Here, we explain the process of extracting features from CIFAR10 dataset images (both training and testing) using pretrained convolutional deep neural networks.
We used the keras python deep learning library. Namely, we used the following networks from keras.applications:
All of them can be initialized to be already pretrained on ImageNET dataset. Here is the example to load the Inception_v3 CNN with keras
from keras.applications.inception_v3 import InceptionV3
model = InceptionV3(include_top=False, weights='imagenet', \
input_tensor=None, input_shape=None, pooling=None, classes=1000)
The most important for transfer learning is to give the paramater
include_top=False
since it builds the CNN model without the last (top) layer which is responsible to classify ImageNET categorized to 1000 classes. As we will see later, such models can be used for extracting the features from CIFAR10 images (only 10 classes) and later to classify them using simple linear classifier like SVM (with linear kernel).
In [1]:
network_names = [ 'incv3', 'resnet50', 'vgg16', 'vgg19' ]
print("Available networks = ", network_names)
cnnid = int( input("Please choose the CNN network [0-{n}]: ".format(n=len(network_names)-1)) )
selected_network = network_names[cnnid]
print("Selected network: ", selected_network)
In [2]:
import time
import myutils
import numpy as np
import tensorflow as tf
from keras.layers import Input, Dense, AveragePooling2D, GlobalAveragePooling2D
from keras import backend as K
Here we use keras.datasets which is pretty similar to our myutils.load_CIFAR10_dataset() procedure.
In [3]:
from keras.datasets import cifar10
(X_train, y_train), (X_test, y_test) = cifar10.load_data()
n_training = X_train.shape[0]
n_testing = X_test.shape[0]
y_train = y_train.flatten()
y_test = y_test.flatten()
print( X_train.shape, y_train.shape, X_test.shape, y_test.shape )
from matplotlib import pyplot as plt
plt.imshow( X_train[0] )
plt.show()
In [4]:
from keras.models import Model
from keras.applications.inception_v3 import InceptionV3
from keras.applications.resnet50 import ResNet50
from keras.applications.vgg16 import VGG16
from keras.applications.vgg19 import VGG19
input_shape = {
'incv3' : (299,299,3),
'resnet50': (224,224,3),
'vgg16' : (224,224,3),
'vgg19' : (224,224,3)
}[selected_network]
def create_model_incv3():
tf_input = Input(shape=input_shape)
model = InceptionV3(input_tensor=tf_input, weights='imagenet', include_top=False)
output_pooled = AveragePooling2D((8, 8), strides=(8, 8))(model.output)
return Model(model.input, output_pooled)
def create_model_resnet50():
tf_input = Input(shape=input_shape)
return ResNet50(input_tensor=tf_input, include_top=False)
def create_model_vgg16():
tf_input = Input(shape=input_shape)
model = VGG16(input_tensor=tf_input, include_top=False)
output_pooled = AveragePooling2D((7, 7))(model.output)
return Model(model.input, output_pooled )
def create_model_vgg19():
tf_input = Input(shape=input_shape)
model = VGG19(input_tensor=tf_input, include_top=False)
output_pooled = AveragePooling2D((7, 7))(model.output)
return Model(model.input, output_pooled )
create_model = {
'incv3' : create_model_incv3,
'resnet50' : create_model_resnet50,
'vgg16' : create_model_vgg16,
'vgg19' : create_model_vgg19
}[selected_network]
In [5]:
# tensorflow placeholder for batch of images from CIFAR10 dataset
batch_of_images_placeholder = tf.placeholder("uint8", (None, 32, 32, 3))
batch_size = {
'incv3' : 16,
'resnet50' : 16,
'vgg16' : 16,
'vgg19' : 16
}[selected_network]
# Inception default size is 299x299
tf_resize_op = tf.image.resize_images(batch_of_images_placeholder, (input_shape[:2]), method=0)
In [6]:
# data generator for tensorflow session
from keras.applications.inception_v3 import preprocess_input as incv3_preprocess_input
from keras.applications.resnet50 import preprocess_input as resnet50_preprocess_input
from keras.applications.vgg16 import preprocess_input as vgg16_preprocess_input
from keras.applications.vgg19 import preprocess_input as vgg19_preprocess_input
preprocess_input = {
'incv3' : incv3_preprocess_input,
'resnet50': resnet50_preprocess_input,
'vgg16' : vgg16_preprocess_input,
'vgg19' : vgg19_preprocess_input
}[selected_network]
def data_generator(sess,data,labels):
def generator():
start = 0
end = start + batch_size
n = data.shape[0]
while True:
batch_of_images_resized = sess.run(tf_resize_op, {batch_of_images_placeholder: data[start:end]})
batch_of_images__preprocessed = preprocess_input(batch_of_images_resized)
batch_of_labels = labels[start:end]
start += batch_size
end += batch_size
if start >= n:
start = 0
end = batch_size
yield (batch_of_images__preprocessed, batch_of_labels)
return generator
In [7]:
with tf.Session() as sess:
# setting tensorflow session to Keras
K.set_session(sess)
# setting phase to training
K.set_learning_phase(0) # 0 - test, 1 - train
model = create_model()
data_train_gen = data_generator(sess, X_train, y_train)
ftrs_training = model.predict_generator(data_train_gen(), n_training/batch_size, verbose=1)
data_test_gen = data_generator(sess, X_test, y_test)
ftrs_testing = model.predict_generator(data_test_gen(), n_testing/batch_size, verbose=1)
In [8]:
features_training = np.array( [ftrs_training[i].flatten() for i in range(n_training)] )
features_testing = np.array( [ftrs_testing[i].flatten() for i in range(n_testing )] )
np.savez_compressed("features/CIFAR10_{}-keras_features.npz".format(selected_network), \
features_training=features_training, \
features_testing=features_testing, \
labels_training=y_train, \
labels_testing=y_test)
In [9]:
features_training.shape, features_testing.shape
Out[9]:
In [10]:
print('Ten first features of X_train[0] (see figure above, with the frog)')
features_training[0][0:10]
Out[10]:
Ten first features of X_train[0] (see figure above, with the frog)
'incv3':
array([ 0.27140674, 0.52113253, 0.24185033, 0.20671989, 0.30322665,
0.34751841, 0.36889949, 0.25563717, 0.27236217, 0.16386187], dtype=float32)
'vgg16'
array([ 5.26894188, 1.65981817, 0.86902153, 6.91321135, 0.55422449,
1.46428168, 2.49574828, 1.16215992, 3.32762074, 3.14453959], dtype=float32)
'vgg19'
In [11]:
import numpy as np
import myutils
data = np.load("features/CIFAR10_{}-keras_features.npz".format(selected_network))
X_training = data['features_training']
y_training = data['labels_training']
X_testing = data['features_testing']
y_testing = data['labels_testing']
# data_training, data_testing = myutils.load_CIFAR_dataset(shuffle=False)
# assert( (np.array( [data_training[i][1] for i in range(len(data_training))] ) == y_training).all() )
# assert( (np.array( [data_testing[i][1] for i in range(len(data_testing))] ) == y_testing).all() )
print( 'X_training size = {}'.format(X_training.shape))
In [12]:
from sklearn import decomposition
pca = decomposition.PCA(n_components=2)
pca.fit(X_training)
Out[12]:
In [13]:
print(pca.explained_variance_ratio_)
In [14]:
X = pca.transform(X_training)
In [15]:
X.shape
Out[15]:
In [16]:
from matplotlib import pyplot as plt
plt.figure( figsize=(15,15) )
plt.scatter( X[:, 0], X[:, 1], c=y_training, cmap='tab10' )
# plt.colorbar()
plt.show()
In [17]:
from sklearn.manifold import TSNE
pca = decomposition.PCA(n_components=10)
X_training_reduced = pca.fit_transform(X_training)
Since t-SNE is quite complex, it is worth to compress the data. Let say that 56% is enough.
In [18]:
np.sum( pca.explained_variance_ratio_ )
Out[18]:
In [19]:
tsne = TSNE(n_components=2)
In [20]:
X_training_reduced_tsne = tsne.fit_transform(X_training_reduced)
In [21]:
X_training_reduced_tsne.shape
Out[21]:
In [22]:
plt.figure( figsize=(15,15) )
plt.scatter( X_training_reduced_tsne[:, 0], X_training_reduced_tsne[:, 1], c=y_training, cmap='tab10' )
plt.show()