In [9]:
import io
import json
import os.path
import theano as th
import numpy as np
import PIL.Image as img
import matplotlib.pyplot as plt
from pylearn2.utils.serial import load as load_model
from pylearn2.gui.get_weights_report import get_weights_report
import neukrill_net.image_directory_dataset as idd
import neukrill_net.encoding as enc
from IPython.display import display, Image
%matplotlib inline
In [29]:
MODEL_PICKLE_PATH = '${DATA_DIR}/plankton/models/fewer_conv_channels_with_dropout_resume.pkl'
SETTINGS_PATH = '/home/mgraham/projects/neukrill-net-work/settings.json'
RUN_SETTINGS_PATH = '/home/mgraham/projects/neukrill-net-work/run_settings/fewer_conv_channels_with_dropout.json'
WEIGHT_IMAGE_SCALE = 8
LAYER_ACTIV_SCALE = 2
N_CONV_LAYER_ROWS = 4
N_TEST_IMS = 200
SEED = 1234
In [5]:
model = load_model(os.path.expandvars(MODEL_PICKLE_PATH))
In [6]:
input_space = model.get_input_space()
input_axes = input_space.axes
input_height, input_width = input_space.shape
with open(RUN_SETTINGS_PATH, 'r') as f:
run_settings = json.load(f)
if run_settings.has_key('prepreprocessing'):
prepreprocessing = run_settings['prepreprocessing']
else:
prepreprocessing = {'resize' : [input_height, input_width], 'resize_order': 1,
'normalise' : run_settings['preprocessing']['normalise']}
normalise_mu = prepreprocessing['normalise']['mu']
normalise_sigma = prepreprocessing['normalise']['sigma']
prng = np.random.RandomState(SEED)
In [10]:
dataset = idd.ListDataset(transformer=lambda x: None, settings_path=SETTINGS_PATH,
run_settings_path=RUN_SETTINGS_PATH,
training_set_mode='test', force=True,
prepreprocessing=prepreprocessing)
In [11]:
print('## Model structure summary\n')
print(model)
params = model.get_params()
n_params = {p.name : p.get_value().size for p in params}
total_params = sum(n_params.values())
print('\n## Number of parameters\n')
print(' ' + '\n '.join(['{0} : {1} ({2:.1f}%)'.format(k, v, 100.*v/total_params)
for k, v in sorted(n_params.items(), key=lambda x: x[0])]))
print('\nTotal : {0}'.format(total_params))
In [13]:
tr = np.array(model.monitor.channels['valid_y_y_1_nll'].time_record) / 3600.
fig = plt.figure(figsize=(12,8))
ax1 = fig.add_subplot(111)
ax1.plot(model.monitor.channels['valid_y_y_1_nll'].val_record)
ax1.plot(model.monitor.channels['train_y_y_1_nll'].val_record)
ax1.set_xlabel('Epochs')
ax1.legend(['Valid', 'Train'])
ax1.set_ylabel('NLL')
ax1.set_ylim(0., 5.)
ax1.grid(True)
ax2 = ax1.twiny()
ax2.set_xticks(np.arange(0,tr.shape[0],20))
ax2.set_xticklabels(['{0:.2f}'.format(t) for t in tr[::20]])
ax2.set_xlabel('Hours')
print("Minimum validation set NLL {0}".format(min(model.monitor.channels['valid_y_y_1_nll'].val_record)))
In [14]:
pv = get_weights_report(model=model)
w_img = pv.get_img()
w_img = w_img.resize((WEIGHT_IMAGE_SCALE*w_img.size[0], WEIGHT_IMAGE_SCALE*w_img.size[1]))
w_img_data = io.BytesIO()
w_img.save(w_img_data, format='png')
display(Image(data=w_img_data.getvalue(), format='png'))
Plot an example image to check loaded correctly
In [15]:
plt.imshow(dataset.X[0])
Out[15]:
Compile theano function for forward propagating through network and getting all layer activations
In [16]:
X = model.get_input_space().make_theano_batch()
Y = model.fprop( X, True )
model_activ_func = th.function([X], Y)
In [25]:
test_idx = prng.choice(len(dataset.X), N_TEST_IMS, False)
input_arrs = np.array([dataset.X[i].astype(np.float32).reshape(input_height, input_width, 1) for i in test_idx])
input_arrs = (input_arrs - normalise_mu)/normalise_sigma
true_labels = [int(np.where(y)[0]) for y in dataset.y[test_idx,:121]]
if input_axes == ('c', 0, 1, 'b'):
activs = model_activ_func(input_arrs.transpose((3,1,2,0)))
else:
activs = model_activ_func(input_arrs)
In [27]:
def construct_activity_mosaic(layer_activ, pad=1, margin=5, n_rows=None):
n_channels, w, h = layer_activ.shape
if n_rows is None:
n_rows = int(n_channels**0.5)
n_cols = int(((1.*n_channels)/n_rows)+0.5)
assert n_rows * n_cols >= n_channels, "n_rows * n_cols ({0}) < n_channels ({1})".format(n_rows*n_cols, n_channels)
width = n_cols * (w + pad) - pad + 2 * margin
height = n_rows * (h + pad) - pad + 2 * margin
mosaic = np.ones((height, width))
x, y = margin, margin
r, c = 0, 0
for i in range(n_channels):
mosaic[y:y+h, x:x+w] = layer_activ[i].T
x += w + pad
c += 1
if c == n_cols:
c = 0
r += 1
y += h + pad
x = margin
return mosaic
In [53]:
n_classes = np.array([int(run_settings['n_classes_{0}'.format(k)]) for k in range(1, 7)])
superclass_boundaries = np.r_[0,n_classes.cumsum()]
hier = enc.get_hierarchy()
classes = sorted([cls for cls in hier[0]])
encs = [enc.get_encoding(cls, hier) for cls in classes]
In [54]:
norm_conv_activs = [activ[:,:,:,:] for activ in activs[:3]]
norm_conv_activs = [activ - activ.min(axis=(2,3))[:,:,None,None] for activ in norm_conv_activs]
norm_conv_activs = [activ / activ.max(axis=(2,3))[:,:,None,None] for activ in norm_conv_activs]
norm_fc_activs = [activ for activ in activs[3:5]]
norm_fc_activs = [activ - activ.min() for activ in norm_fc_activs]
norm_fc_activs = [activ / activ.max() for activ in norm_fc_activs]
softmax_activs = [activs[-1][:,s:e] for s,e in zip(superclass_boundaries[:-1], superclass_boundaries[1:])]
In [57]:
i = 10
true_y = true_labels[i]
true_enc = encs[true_y]
# input image
input_arr = input_arrs[i].reshape(1, input_height, input_width) * normalise_sigma + normalise_mu
input_arr = construct_activity_mosaic(1.-input_arr, 0, 1)
input_im = img.fromarray(np.uint8((1.-input_arr)*255))
input_im = input_im.resize((LAYER_ACTIV_SCALE*input_im.size[0], LAYER_ACTIV_SCALE*input_im.size[1]))
input_data = io.BytesIO()
input_im.save(input_data, format='png')
display(Image(data=input_data.getvalue(), format='png'))
# convolutional layers
for norm_conv_activ in norm_conv_activs:
mosaic_arr = construct_activity_mosaic(norm_conv_activ[i], 2, 5, N_CONV_LAYER_ROWS)
mosaic_im = img.fromarray(np.uint8((mosaic_arr)*255))
mosaic_im = mosaic_im.resize((LAYER_ACTIV_SCALE*mosaic_im.size[0], LAYER_ACTIV_SCALE*mosaic_im.size[1]))
mosaic_data = io.BytesIO()
mosaic_im.save(mosaic_data, format='png')
display(Image(data=mosaic_data.getvalue(), format='png'))
# fc layers
for norm_fc_activ in norm_fc_activs:
layer_arr = construct_activity_mosaic(norm_fc_activ[i,:].reshape(-1, 8, 1), 0, 2)
layer_im = img.fromarray(np.uint8((1-layer_arr)*255))
layer_im = layer_im.resize((2*LAYER_ACTIV_SCALE*layer_im.size[0], 2*LAYER_ACTIV_SCALE*layer_im.size[1]))
layer_data = io.BytesIO()
layer_im.save(layer_data, format='png')
display(Image(data=layer_data.getvalue(), format='png'))
for j, softmax_activ in enumerate(softmax_activs):
fig = plt.figure(figsize=(1, max(1,n_classes[j]/10.)))
ax1 = fig.add_subplot(111)
ax1.barh(np.arange(n_classes[j]), softmax_activ[i,:], 1)
ax1.set_xticks([0, 1.])
ax1.set_yticks([])
ax1.annotate("True", size=12,
xy=(0.0, int(np.where(np.array(true_enc[j]))[0])+0.5), xycoords='data',
xytext=(-0.8, int(np.where(true_enc[j])[0])+0.5), textcoords='data',
arrowprops=dict(arrowstyle="->",connectionstyle="arc3"),
)
plt.show()