In [1]:
from __future__ import print_function
from sys import version_info
import matplotlib.pyplot as plt
import numpy as np
import os
import scipy
import theano
import theano.tensor as T
import lasagne
try:
import cPickle as pickle
except ImportError:
import pickle
%matplotlib inline
from scipy.misc import imread, imsave, imresize
from lasagne.utils import floatX
from sklearn.svm import SVC
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import cross_val_score
from sklearn.decomposition import PCA
from sklearn.metrics import accuracy_score
import skimage
from tqdm import tqdm
from scipy.misc import imresize
import xgboost
import skimage.transform
import pandas as pd
from sklearn.neural_network import MLPClassifier
from collections import Counter
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import StratifiedShuffleSplit, StratifiedKFold
In [2]:
# ResNet-50, network from the paper:
# "Deep Residual Learning for Image Recognition"
# http://arxiv.org/pdf/1512.03385v1.pdf
# License: see https://github.com/KaimingHe/deep-residual-networks/blob/master/LICENSE
# Download pretrained weights from:
# https://s3.amazonaws.com/lasagne/recipes/pretrained/imagenet/resnet50.pkl
import lasagne
from lasagne.layers import InputLayer
from lasagne.layers import Conv2DLayer as ConvLayer
from lasagne.layers import BatchNormLayer
from lasagne.layers import Pool2DLayer as PoolLayer
from lasagne.layers import NonlinearityLayer
from lasagne.layers import ElemwiseSumLayer
from lasagne.layers import DenseLayer
from lasagne.nonlinearities import rectify, softmax
def build_simple_block(incoming_layer, names,
num_filters, filter_size, stride, pad,
use_bias=False, nonlin=rectify):
"""Creates stacked Lasagne layers ConvLayer -> BN -> (ReLu)
Parameters:
----------
incoming_layer : instance of Lasagne layer
Parent layer
names : list of string
Names of the layers in block
num_filters : int
Number of filters in convolution layer
filter_size : int
Size of filters in convolution layer
stride : int
Stride of convolution layer
pad : int
Padding of convolution layer
use_bias : bool
Whether to use bias in conlovution layer
nonlin : function
Nonlinearity type of Nonlinearity layer
Returns
-------
tuple: (net, last_layer_name)
net : dict
Dictionary with stacked layers
last_layer_name : string
Last layer name
"""
net = []
net.append((
names[0],
ConvLayer(incoming_layer, num_filters, filter_size, stride, pad,
flip_filters=False, nonlinearity=None) if use_bias
else ConvLayer(incoming_layer, num_filters, filter_size, stride, pad, b=None,
flip_filters=False, nonlinearity=None)
))
net.append((
names[1],
BatchNormLayer(net[-1][1])
))
if nonlin is not None:
net.append((
names[2],
NonlinearityLayer(net[-1][1], nonlinearity=nonlin)
))
return dict(net), net[-1][0]
def build_residual_block(incoming_layer, ratio_n_filter=1.0, ratio_size=1.0, has_left_branch=False,
upscale_factor=4, ix=''):
"""Creates two-branch residual block
Parameters:
----------
incoming_layer : instance of Lasagne layer
Parent layer
ratio_n_filter : float
Scale factor of filter bank at the input of residual block
ratio_size : float
Scale factor of filter size
has_left_branch : bool
if True, then left branch contains simple block
upscale_factor : float
Scale factor of filter bank at the output of residual block
ix : int
Id of residual block
Returns
-------
tuple: (net, last_layer_name)
net : dict
Dictionary with stacked layers
last_layer_name : string
Last layer name
"""
simple_block_name_pattern = ['res%s_branch%i%s', 'bn%s_branch%i%s', 'res%s_branch%i%s_relu']
net = {}
# right branch
net_tmp, last_layer_name = build_simple_block(
incoming_layer, map(lambda s: s % (ix, 2, 'a'), simple_block_name_pattern),
int(lasagne.layers.get_output_shape(incoming_layer)[1]*ratio_n_filter), 1, int(1.0/ratio_size), 0)
net.update(net_tmp)
net_tmp, last_layer_name = build_simple_block(
net[last_layer_name], map(lambda s: s % (ix, 2, 'b'), simple_block_name_pattern),
lasagne.layers.get_output_shape(net[last_layer_name])[1], 3, 1, 1)
net.update(net_tmp)
net_tmp, last_layer_name = build_simple_block(
net[last_layer_name], map(lambda s: s % (ix, 2, 'c'), simple_block_name_pattern),
lasagne.layers.get_output_shape(net[last_layer_name])[1]*upscale_factor, 1, 1, 0,
nonlin=None)
net.update(net_tmp)
right_tail = net[last_layer_name]
left_tail = incoming_layer
# left branch
if has_left_branch:
net_tmp, last_layer_name = build_simple_block(
incoming_layer, map(lambda s: s % (ix, 1, ''), simple_block_name_pattern),
int(lasagne.layers.get_output_shape(incoming_layer)[1]*4*ratio_n_filter), 1, int(1.0/ratio_size), 0,
nonlin=None)
net.update(net_tmp)
left_tail = net[last_layer_name]
net['res%s' % ix] = ElemwiseSumLayer([left_tail, right_tail], coeffs=1)
net['res%s_relu' % ix] = NonlinearityLayer(net['res%s' % ix], nonlinearity=rectify)
return net, 'res%s_relu' % ix
def build_model():
net = {}
net['input'] = InputLayer((None, 3, 224, 224))
sub_net, parent_layer_name = build_simple_block(
net['input'], ['conv1', 'bn_conv1', 'conv1_relu'],
64, 7, 2, 3, use_bias=True)
net.update(sub_net)
net['pool1'] = PoolLayer(net[parent_layer_name], pool_size=3, stride=2, pad=0, mode='max', ignore_border=False)
block_size = list('abc')
parent_layer_name = 'pool1'
for c in block_size:
if c == 'a':
sub_net, parent_layer_name = build_residual_block(net[parent_layer_name], 1, 1, True, 4, ix='2%s' % c)
else:
sub_net, parent_layer_name = build_residual_block(net[parent_layer_name], 1.0/4, 1, False, 4, ix='2%s' % c)
net.update(sub_net)
block_size = list('abcd')
for c in block_size:
if c == 'a':
sub_net, parent_layer_name = build_residual_block(
net[parent_layer_name], 1.0/2, 1.0/2, True, 4, ix='3%s' % c)
else:
sub_net, parent_layer_name = build_residual_block(net[parent_layer_name], 1.0/4, 1, False, 4, ix='3%s' % c)
net.update(sub_net)
block_size = list('abcdef')
for c in block_size:
if c == 'a':
sub_net, parent_layer_name = build_residual_block(
net[parent_layer_name], 1.0/2, 1.0/2, True, 4, ix='4%s' % c)
else:
sub_net, parent_layer_name = build_residual_block(net[parent_layer_name], 1.0/4, 1, False, 4, ix='4%s' % c)
net.update(sub_net)
block_size = list('abc')
for c in block_size:
if c == 'a':
sub_net, parent_layer_name = build_residual_block(
net[parent_layer_name], 1.0/2, 1.0/2, True, 4, ix='5%s' % c)
else:
sub_net, parent_layer_name = build_residual_block(net[parent_layer_name], 1.0/4, 1, False, 4, ix='5%s' % c)
net.update(sub_net)
net['pool5'] = PoolLayer(net[parent_layer_name], pool_size=7, stride=1, pad=0,
mode='average_exc_pad', ignore_border=False)
net['fc1000'] = DenseLayer(net['pool5'], num_units=1000, nonlinearity=None)
net['prob'] = NonlinearityLayer(net['fc1000'], nonlinearity=softmax)
return net
In [3]:
df_labels = pd.read_csv('data/merged_results.csv')
In [4]:
buildings_ids = df_labels['id'].values
files_to_process = list(map(lambda ind: 'data/all_buildings/%05d.bmp' % ind, buildings_ids))
In [5]:
net = build_model()
input_image = T.tensor4('input')
In [6]:
with open('resnet50.pkl', 'rb') as f:
if version_info.major == 2:
resnet_weights = pickle.load(f)
elif version_info.major == 3:
resnet_weights = pickle.load(f, encoding='latin1')
mean_values = resnet_weights['mean_image']
In [7]:
lasagne.layers.set_all_param_values(net['prob'], resnet_weights['values'])
In [8]:
def prep_image(fname):
if fname is None:
ext = url.split('.')[-1]
im = plt.imread(io.BytesIO(urllib.urlopen(url).read()), ext)
else:
ext = fname.split('.')[-1]
im = plt.imread(fname, ext)
h, w, _ = im.shape
if h < w:
im = skimage.transform.resize(im, (256, w*256/h), preserve_range=True)
else:
im = skimage.transform.resize(im, (h*256/w, 256), preserve_range=True)
h, w, _ = im.shape
im = im[h//2-112:h//2+112, w//2-112:w//2+112]
rawim = np.copy(im).astype('uint8')
im = np.swapaxes(np.swapaxes(im, 1, 2), 0, 1)
im = im[::-1, :, :]
im = im - mean_values
return rawim, floatX(im[np.newaxis])
In [9]:
def extract_resnet_features(filenames, layer_name):
X = []
input_image = T.tensor4('input')
output = lasagne.layers.get_output(net[layer_name], input_image, deterministic=True)
prob = theano.function([input_image], output)
for fname in tqdm(filenames):
raw_img, img = prep_image(fname)
features = prob(img).flatten()
X.append(features)
return np.array(X)
In [11]:
X5 = extract_resnet_features(files_to_process, 'pool5')
In [12]:
X = np.concatenate([X5], axis=1)
X = np.sqrt(np.abs(X))
Y = df_labels['is_flat'].values
In [23]:
model = Pipeline([
('scaler', StandardScaler()),
('svm', SVC(C=10))
])
In [24]:
res = cross_val_score(model, X, Y, cv=StratifiedKFold(n_splits=5, shuffle=True, random_state=10), verbose=10)
print('Mean score: ', np.mean(res))
In [25]:
all_files_to_process = map(lambda x: 'data/all_buildings/%s' % x, os.listdir('data/all_buildings/'))
In [26]:
X_all = extract_resnet_features(all_files_to_process, 'pool5')
In [27]:
X_all_transformed = np.sqrt(np.abs(X_all))
In [28]:
model.fit(X, Y)
Out[28]:
In [29]:
all_predictions = model.predict(X_all_transformed)
In [30]:
res = []
for index, fname in enumerate(all_files_to_process):
if fname == 'data/all_buildings/%05d.bmp':
continue
tmp = int(fname.split('/')[-1].split('.')[0])
pred = all_predictions[index]
res.append([tmp, pred])
In [31]:
all_results = pd.DataFrame(res, columns=['id', 'is_flat'])
In [32]:
all_results.to_csv('data/all_results_grodno.csv', index=False)