In [1]:
%matplotlib inline
from __future__ import print_function
import matplotlib.pyplot as plt
from keras.models import Sequential
from keras.layers.core import Flatten, Dense, Dropout
from keras.layers.convolutional import Convolution2D, MaxPooling2D, ZeroPadding2D
from keras.optimizers import SGD
import h5py
import numpy as np
import os
DATA = lambda p: os.path.join('../datafiles/zed-subset', p)
def myimread(fname):
import skimage.io, skimage.transform
im = skimage.transform.resize(skimage.io.imread(fname), (224, 224))
return im[...,::-1]*255
def myimshow(im):
plt.figure(figsize=(7,7))
plt.imshow(im[...,::-1]/255)
def myimshow_grey(im):
plt.figure(figsize=(7,7))
plt.imshow(im, cmap=plt.get_cmap('gray'), vmin=0, vmax=255)
test_im = myimread(DATA('556-left.jpeg'))
print(test_im.shape)
myimshow(test_im)
In [2]:
def ConvMap(vgg_weights_path):
REMOVE_EXCESSIVE_POOLING = True
model = Sequential()
model.add(ZeroPadding2D((1,1),input_shape=(3,224,224), name='z11'))
model.add(Convolution2D(64, 3, 3, activation='relu', name='conv11'))
model.add(ZeroPadding2D((1,1), name='z12'))
model.add(Convolution2D(64, 3, 3, activation='relu', name='conv12'))
model.add(MaxPooling2D((2,2), strides=(2,2), name='max1'))
model.add(ZeroPadding2D((1,1), name='z21'))
model.add(Convolution2D(128, 3, 3, activation='relu', name='conv21'))
model.add(ZeroPadding2D((1,1), name='z22'))
model.add(Convolution2D(128, 3, 3, activation='relu', name='conv22'))
model.add(MaxPooling2D((2,2), strides=(2,2), name='max2'))
model.add(ZeroPadding2D((1,1), name='z31'))
model.add(Convolution2D(256, 3, 3, activation='relu', name='conv31'))
model.add(ZeroPadding2D((1,1), name='z32'))
model.add(Convolution2D(256, 3, 3, activation='relu', name='conv32'))
model.add(ZeroPadding2D((1,1), name='z33'))
model.add(Convolution2D(256, 3, 3, activation='relu', name='conv33'))
if not REMOVE_EXCESSIVE_POOLING:
model.add(MaxPooling2D((2,2), strides=(2,2), name='max3'))
model.add(ZeroPadding2D((1,1), name='z41'))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv41'))
model.add(ZeroPadding2D((1,1), name='z42'))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv42'))
model.add(ZeroPadding2D((1,1), name='z43'))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv43'))
if not REMOVE_EXCESSIVE_POOLING:
model.add(MaxPooling2D((2,2), strides=(2,2), name='max4'))
model.add(ZeroPadding2D((1,1), name='z51'))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv51'))
model.add(ZeroPadding2D((1,1), name='z52'))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv52'))
model.add(ZeroPadding2D((1,1), name='z53'))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv53'))
model.add(MaxPooling2D((2,2), strides=(2,2), name='max5'))
# model.add(Flatten())
# model.add(Dense(4096, activation='relu'))
# model.add(Dropout(0.5))
# model.add(Dense(4096, activation='relu'))
# model.add(Dropout(0.5))
# model.add(Dense(1000, activation='softmax'))
layer_key = {
'z11': 'layer_0', 'conv11': 'layer_1',
'z12': 'layer_2', 'conv12': 'layer_3',
'max1': 'layer_4',
'z21': 'layer_5', 'conv21': 'layer_6',
'z22': 'layer_7', 'conv22': 'layer_8',
'max2': 'layer_9',
'z31': 'layer_10', 'conv31': 'layer_11',
'z32': 'layer_12', 'conv32': 'layer_13',
'z33': 'layer_14', 'conv33': 'layer_15',
'max3': 'layer_16',
'z41': 'layer_17', 'conv41': 'layer_18',
'z42': 'layer_19', 'conv42': 'layer_20',
'z43': 'layer_21', 'conv43': 'layer_22',
'max4': 'layer_23',
'z51': 'layer_24', 'conv51': 'layer_25',
'z52': 'layer_26', 'conv52': 'layer_27',
'z53': 'layer_28', 'conv53': 'layer_29',
'max5': 'layer_30',
}
if REMOVE_EXCESSIVE_POOLING:
del layer_key['max3'], layer_key['max4']
with h5py.File(vgg_weights_path) as f:
for name, layer in layer_key.iteritems():
g = f[layer]
weights = [g['param_{}'.format(p)] for p in range(g.attrs['nb_params'])]
model.get_layer(name).set_weights(weights)
print('VGG loaded. (%d layers)'%len(model.layers))
model.add(ZeroPadding2D((1,1), name='z61'))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv61'))
model.add(ZeroPadding2D((1,1), name='z62'))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv62'))
print('ConvMap completed. (%d layers)'%len(model.layers))
for layer in model.layers:
if 'conv' in layer.name or 'max' in layer.name:
print(layer.name, layer.output_shape)
return model
def predict(im):
im[:,:] -= np.array([103.939, 116.779, 123.68])
im = im.transpose((2,0,1))
im = np.expand_dims(im, axis=0)
model = ConvMap('vgg16_weights.h5')
# sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
# model.compile(optimizer=sgd, loss='categorical_crossentropy')
out = model.predict(im)
return out
conv_map = predict(test_im.copy())
print('conv_map', conv_map.shape)
In [3]:
def show_conv_map(im):
N_sumples = 16
sample = np.random.choice(range(conv_map.shape[1]), N_sumples, False)
print('%d convmaps out of %d.'%(N_sumples, conv_map.shape[1]))
print('shape:', conv_map.shape[2:])
plt.figure(figsize=(10,10))
for i in xrange(N_sumples):
plt.subplot(4,4,i+1)
plt.axis('off')
plt.imshow(conv_map[0][sample[i]], cmap=plt.get_cmap('inferno'), vmin=0, vmax=255)
show_conv_map(conv_map)
In [4]:
from skimage.segmentation import slic
from skimage.segmentation import mark_boundaries
N_SUPERPIXELS = 300
def myslic(im):
im_float = im[...,::-1]/255
segments = slic(im/255, n_segments = N_SUPERPIXELS, sigma = 5)
# plt.figure(figsize=(7,7))
# plt.imshow(mark_boundaries(im_float, segments))
return segments
test_segments = myslic(test_im)
In [5]:
def superpixel_weights2(segments, conv_map_shape):
N_segments = np.max(segments)+1
# print('N_segments', N_segments)
subsample = tuple(np.array(segments.shape) / np.array(conv_map_shape))
# print('subsample',subsample)
from skimage.util import view_as_blocks
# print('segments', segments.shape)
W = np.repeat(segments[np.newaxis,...],N_segments,axis=0)
# print('segments dimensionally expended', W.shape)
W = W==np.arange(N_segments)[...,np.newaxis,np.newaxis]
# print('segment masks', W.shape)
W = np.sum(view_as_blocks(W, (1,)+subsample), axis=(3,4,5))
# print('superpixel hits', W.shape)
# W_norm = np.sum(W,axis=(1,2))
# W_norm[W_norm==0] = 1.
# return W.astype(float)/W_norm[...,np.newaxis,np.newaxis]
return W.astype(float)/np.sum(W,axis=(1,2))[...,np.newaxis,np.newaxis]
test_segments_weights = superpixel_weights2(test_segments, conv_map[0][0].shape)
test_segments_weights = np.resize(test_segments_weights, (N_SUPERPIXELS,)+conv_map[0][0].shape)
# Feature map (Unary potential)
print('conv_map', conv_map.shape)
H = np.einsum('tij,kij->tk', test_segments_weights, conv_map[0])
print('superpixel pooling',H.shape)
plt.figure(figsize=(10,7))
plt.imshow(H, cmap='jet')
Out[5]:
In [6]:
def index_dataset():
DATA = lambda p: os.path.join('../datafiles', p)
images = list()
depth = list()
for root, dirs, files in os.walk(DATA('zed-subset')):
for name in files:
if os.path.splitext(name)[1] in ['.jpeg']:
fname = os.path.join(root, name)
if 'depth' in name:
depth.append(fname)
else:
images.append(fname)
return images, depth
(data_images, data_depth) = index_dataset()
In [7]:
def gen_superpixel_pooling_weights(fnames, numSegments, output_path):
print('Output path', output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
for fname in fnames:
try:
im = myimread(fname)
segments = myslic(im)
weights = superpixel_weights2(test_segments, conv_map[0][0].shape)
weights = np.resize(weights, (N_SUPERPIXELS,)+conv_map[0][0].shape)
np.save(os.path.join(output_path, os.path.basename(fname)), weights, allow_pickle=False)
print('.', end='')
except Exception as e:
print(fname, e)
gen_superpixel_pooling_weights(data_images, 300, DATA('SLIC300'))
In [8]:
test_weight = np.load(DATA('SLIC300/556-left.jpeg.npy'))
print(test_weight.shape)
In [ ]: