In [1]:
# Importing module to load a pre-trained model.
from keras.models import load_model
In [2]:
model = load_model('./saved_checkpoints/Chapter 5.2 - Using convets with small datasets/cats_and_dogs_small_2.h5')
In [3]:
# Summary
model.summary()
In [4]:
# Path
img_path = './/data//Chapter 5.2 - Using convets with small datasets//test//cats//cat.1752.jpg'
In [5]:
from keras.preprocessing import image
import numpy as np
In [6]:
img = image.load_img(img_path,
target_size = (150, 150))
img_tensor = image.img_to_array(img)
In [7]:
img_tensor.shape
Out[7]:
In [8]:
img_tensor = np.expand_dims(img_tensor,
axis = 0)
In [9]:
img_tensor.shape
Out[9]:
In [10]:
# Remember that the model was trained on inputs
# that were preprocessed in the following way:
img_tensor /= 255.
In [11]:
import matplotlib.pyplot as plt
plt.imshow(img_tensor[0])
plt.show()
In [12]:
from keras.models import Model
In [13]:
# Extracts the outputs of the top 8 layers:
layer_outputs = [layer.output for layer in model.layers[:8]]
# Creates a model that will return these outputs, given the model input:
activation_model = Model(inputs = model.input,
outputs = layer_outputs)
In [14]:
activation_model.summary()
In [15]:
activations = activation_model.predict(img_tensor)
In [16]:
# 8 layers, 3 of them are maxpooling layers
len(activations)
Out[16]:
In [17]:
first_layer_activation = activations[0]
print(first_layer_activation.shape)
In [18]:
import matplotlib.pyplot as plt
plt.matshow(first_layer_activation[0, :, :, 0],
cmap = 'viridis')
plt.show()
In [19]:
plt.matshow(first_layer_activation[0, :, :, 20],
cmap = 'viridis')
plt.show()
In [20]:
layer_names = []
for layer in model.layers[:8]:
layer_names.append(layer.name)
In [21]:
images_per_row = 16
In [22]:
# Now let's display our feature maps
for layer_name, layer_activation in zip(layer_names, activations):
# This is the number of features in the feature map
n_features = layer_activation.shape[-1]
# The feature map has shape (1, size, size, n_features)
size = layer_activation.shape[1]
# We will tile the activation channels in this matrix
n_cols = n_features // images_per_row
display_grid = np.zeros((size * n_cols,
images_per_row * size))
# We'll tile each filter into this big horizontal grid
for col in range(n_cols):
for row in range(images_per_row):
channel_image = layer_activation[0,
:, :,
col * images_per_row + row]
# Post-process the feature to make it visually palatable
channel_image -= channel_image.mean()
channel_image /= channel_image.std()
channel_image *= 64
channel_image += 128
channel_image = np.clip(channel_image, 0, 255).astype('uint8')
display_grid[col * size : (col + 1) * size,
row * size : (row + 1) * size] = channel_image
# Display the grid
scale = 1. / size
plt.figure(figsize = (scale * display_grid.shape[1],
scale * display_grid.shape[0]))
plt.title(layer_name)
plt.grid(False)
plt.imshow(display_grid,
aspect = 'auto',
cmap = 'viridis')
plt.show()
In [23]:
from keras.applications import VGG16
from keras import backend as K
model = VGG16(weights='imagenet',
include_top=False)
In [24]:
model.summary()
In [25]:
layer_name = 'block3_conv1'
filter_index = 0
The process is simple: we will build a loss function that maximizes the value of a given filter in a given convolution layer, then we will use stochastic gradient descent to adjust the values of the input image so as to maximize this activation value
In [26]:
layer_output = model.get_layer(layer_name).output
loss = K.mean(layer_output[:, :, :, filter_index])
In [27]:
# The call to `gradients` returns a list of tensors (of size 1 in this case)
# hence we only keep the first element -- which is a tensor.
grads = K.gradients(loss, model.input)[0]
In [28]:
# We add 1e-5 before dividing so as to avoid accidentally dividing by 0.
grads /= (K.sqrt(K.mean(K.square(grads))) + 1e-5)
In [29]:
iterate = K.function([model.input], [loss, grads])
# Let's test it:
import numpy as np
loss_value, grads_value = iterate([np.zeros((1, 150, 150, 3))])
In [30]:
# We start from a gray image with some noise
input_img_data = np.random.random((1, 150, 150, 3)) * 20 + 128.
In [31]:
input_img_data.shape
Out[31]:
In [32]:
plt.matshow(input_img_data[0, :, :, 0],
cmap = 'viridis')
plt.show()
In [33]:
# Run gradient ascent for 40 steps
step = 1. # this is the magnitude of each gradient update
for i in range(40):
# Compute the loss value and gradient value
loss_value, grads_value = iterate([input_img_data])
# Here we adjust the input image in the direction that maximizes the loss
input_img_data += grads_value * step
The resulting image tensor will be a floating point tensor of shape (1, 150, 150, 3), with values that may not be integer within [0, 255]. Hence we would need to post-process this tensor to turn it into a displayable image. We do it with the following straightforward utility function:
In [34]:
def deprocess_image(x):
# normalize tensor: center on 0., ensure std is 0.1
x -= x.mean()
x /= (x.std() + 1e-5)
x *= 0.1
# clip to [0, 1]
x += 0.5
x = np.clip(x, 0, 1)
# convert to RGB array
x *= 255
x = np.clip(x, 0, 255).astype('uint8')
return x
In [35]:
def generate_pattern(layer_name, filter_index, size=150):
# Build a loss function that maximizes the activation
# of the nth filter of the layer considered.
layer_output = model.get_layer(layer_name).output
loss = K.mean(layer_output[:, :, :, filter_index])
# Compute the gradient of the input picture wrt this loss
grads = K.gradients(loss, model.input)[0]
# Normalization trick: we normalize the gradient
grads /= (K.sqrt(K.mean(K.square(grads))) + 1e-5)
# This function returns the loss and grads given the input picture
iterate = K.function([model.input], [loss, grads])
# We start from a gray image with some noise
input_img_data = np.random.random((1, size, size, 3)) * 20 + 128.
# Run gradient ascent for 40 steps
step = 1.
for i in range(40):
loss_value, grads_value = iterate([input_img_data])
input_img_data += grads_value * step
img = input_img_data[0]
return deprocess_image(img)
In [36]:
plt.imshow(generate_pattern('block3_conv1', 0))
plt.show()
In [37]:
# Visualizing every single filter in every layer
for layer_name in ['block1_conv1', 'block2_conv1', 'block3_conv1', 'block4_conv1']:
size = 64
margin = 5
# This a empty (black) image where we will store our results.
results = np.zeros((8 * size + 7 * margin, 8 * size + 7 * margin, 3))
for i in range(8): # iterate over the rows of our results grid
for j in range(8): # iterate over the columns of our results grid
# Generate the pattern for filter `i + (j * 8)` in `layer_name`
filter_img = generate_pattern(layer_name, i + (j * 8),
size = size)
# Put the result in the square `(i, j)` of the results grid
horizontal_start = i * size + i * margin
horizontal_end = horizontal_start + size
vertical_start = j * size + j * margin
vertical_end = vertical_start + size
results[horizontal_start: horizontal_end, vertical_start: vertical_end, :] = filter_img
# Display the results grid
plt.figure(figsize = (20, 20))
plt.imshow(results)
plt.show()
In [38]:
from keras.applications.vgg16 import VGG16
K.clear_session()
# Note that we are including the densely-connected classifier on top;
# all previous times, we were discarding it.
model = VGG16(weights = 'imagenet')
In [39]:
from keras.preprocessing import image
from keras.applications.vgg16 import preprocess_input, decode_predictions
import numpy as np
# The local path to our target image
img_path = r'E:/1_GitHub/arcyfelix/Courses/In Progress-Deep Learning With Python by François Chollet/data/Chapter 5.4 - Visualizing what convnets learn/creative_commons_elephant.jpg'
# `img` is a PIL image of size 224x224
img = image.load_img(img_path,
target_size = (224, 224))
# `x` is a float32 Numpy array of shape (224, 224, 3)
x = image.img_to_array(img)
# We add a dimension to transform our array into a "batch"
# of size (1, 224, 224, 3)
x = np.expand_dims(x, axis = 0)
# Finally we preprocess the batch
# (this does channel-wise color normalization)
x = preprocess_input(x)
In [40]:
# Top-3 predictions
preds = model.predict(x)
print('Predicted:', decode_predictions(preds, top=3)[0])
In [41]:
np.argmax(preds[0])
Out[41]:
In [42]:
# This is the "african elephant" entry in the prediction vector
african_elephant_output = model.output[:, 386]
In [43]:
# The is the output feature map of the `block5_conv3` layer,
# the last convolutional layer in VGG16
last_conv_layer = model.get_layer('block5_conv3')
In [44]:
# This is the gradient of the "african elephant" class with regard to
# the output feature map of `block5_conv3`
grads = K.gradients(african_elephant_output, last_conv_layer.output)[0]
In [45]:
# This is a vector of shape (512,), where each entry
# is the mean intensity of the gradient over a specific feature map channel
pooled_grads = K.mean(grads,
axis = (0, 1, 2))
In [46]:
# This function allows us to access the values of the quantities we just defined:
# `pooled_grads` and the output feature map of `block5_conv3`,
# given a sample image
iterate = K.function([model.input], [pooled_grads, last_conv_layer.output[0]])
In [47]:
# These are the values of these two quantities, as Numpy arrays,
# given our sample image of two elephants
pooled_grads_value, conv_layer_output_value = iterate([x])
In [48]:
# We multiply each channel in the feature map array
# by "how important this channel is" with regard to the elephant class
for i in range(512):
conv_layer_output_value[:, :, i] *= pooled_grads_value[i]
In [49]:
# The channel-wise mean of the resulting feature map
# is our heatmap of class activation
heatmap = np.mean(conv_layer_output_value,
axis = -1)
In [50]:
# Normalizing the heatmap
heatmap = np.maximum(heatmap, 0)
heatmap /= np.max(heatmap)
plt.matshow(heatmap)
plt.show()
In [51]:
import cv2
# We use cv2 to load the original image
img = cv2.imread('.//data//Chapter 5.4 - Visualizing what convnets learn//creative_commons_elephant.jpg')
In [52]:
img.shape
Out[52]:
In [53]:
# We resize the heatmap to have the same size as the original image
heatmap = cv2.resize(heatmap, (img.shape[1], img.shape[0]))
# We convert the heatmap to RGB
heatmap = np.uint8(255 * heatmap)
# We apply the heatmap to the original image
heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET)
# 0.4 here is a heatmap intensity factor
superimposed_img = heatmap * 0.4 + img
In [54]:
superimposed_img.shape
Out[54]:
In [55]:
# Save the image to disk
cv2.imwrite('.//data//Chapter 5.4 - Visualizing what convnets learn///creative_commons_elephant_with_heatmap.jpg',
superimposed_img)
Out[55]:
In [56]:
from IPython.core.display import Image, display
In [57]:
display(Image('.//data//Chapter 5.4 - Visualizing what convnets learn///creative_commons_elephant_with_heatmap.jpg'))