Note (Mac OS): To ensure all dependencies are available set env virable as follows:
export DYLD_LIBRARY_PATH=/home/user/path_to_CUDNN_folder:$DYLD_LIBRARY_PATH
export CPATH=/home/user/path_to_CUDNN_folder/include:$CPATH
export LIBRARY_PATH=/home/user/path_to_CUDNN_folder:$DYLD_LIBRARY_PATHSee details here.
In [1]:
%pylab inline
In [2]:
import lasagne
import numpy as np
import pickle
import skimage.transform
import scipy
import pickle
import theano
import theano.tensor as T
from lasagne.utils import floatX
In [3]:
theano.config.allow_gc = True
theano.config.scan.allow_output_prealloc = False
theano.config.exception_verbosity = 'high'
theano.config.optimizer = 'fast_compile'
Note: in case of low-memory cards use IMAGEWIDTH <= 450
In [4]:
from lasagne.layers import MaxPool2DLayer
https://s3.amazonaws.com/lasagne/recipes/pretrained/imagenet/vgg19_normalized.pkl (original source: https://bethgelab.org/deepneuralart/)
Note: On Mac install wget via homberew. See details here.
In [5]:
!wget -O ../models/googlenet.pkl https://s3.amazonaws.com/lasagne/recipes/pretrained/imagenet/blvc_googlenet.pkl
In [6]:
IMAGEWIDTH = 600
import googlenet
net = googlenet.build_model(IMAGEWIDTH)
values = pickle.load(open('../models/googlenet.pkl'))['param values']
lasagne.layers.set_all_param_values(net['prob'], [v.astype(np.float32) for v in values])
In [7]:
MEAN_VALUES = np.array([104, 117, 123]).reshape((3,1,1))
def prep_image(im):
if len(im.shape) == 2:
im = im[:, :, np.newaxis]
im = np.repeat(im, 3, axis=2)
h, w, _ = im.shape
if h < w:
im = skimage.transform.resize(im, (IMAGEWIDTH, w*IMAGEWIDTH/h), preserve_range=True)
else:
im = skimage.transform.resize(im, (h*IMAGEWIDTH/w, IMAGEWIDTH), preserve_range=True)
# Central crop
h, w, _ = im.shape
im = im[h//2-IMAGEWIDTH//2:h//2+IMAGEWIDTH//2, w//2-IMAGEWIDTH//2:w//2+IMAGEWIDTH//2]
rawim = np.copy(im).astype('uint8')
# Shuffle axes to c01
im = np.swapaxes(np.swapaxes(im, 1, 2), 0, 1)
# Convert RGB to BGR
im = im[::-1, :, :]
im = im - MEAN_VALUES
return rawim, floatX(im[np.newaxis])
In [8]:
content = imread('../images/content/vasily.jpg')
rawim, content = prep_image(content)
plt.figure(figsize=(8,8))
plt.imshow(rawim)
Out[8]:
In [9]:
style = imread('../images/style/Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg')
rawim, style = prep_image(style)
plt.figure(figsize=(8,8))
plt.imshow(rawim)
Out[9]:
In [10]:
def gram_matrix(x):
x = x.flatten(ndim=3)
g = T.tensordot(x, x, axes=([2], [2]))
return g
def content_loss(P, X, layer):
p = P[layer]
x = X[layer]
loss = 1./2 * ((x - p)**2).sum()
return loss
def style_loss(A, X, layer):
a = A[layer]
x = X[layer]
A = gram_matrix(a)
G = gram_matrix(x)
N = a.shape[1]
M = a.shape[2] * a.shape[3]
loss = 1./(4 * N**2 * M**2) * ((G - A)**2).sum()
return loss
def total_variation_loss(x):
return (((x[:,:,:-1,:-1] - x[:,:,1:,:-1])**2 + (x[:,:,:-1,:-1] - x[:,:,:-1,1:])**2)**1.25).sum()
In [11]:
weights = {"content": {"conv2/3x3": 2e-4, "inception_3a/output": 1-2e-4},
"style": {"conv1/7x7_s2": 0.2, "conv2/3x3": 0.2, "inception_3a/output": 0.2, "inception_4a/output": 0.2,
"inception_5a/output": 0.2}}
In [12]:
layers = weights['content'].keys() + weights['style'].keys()
layers = {k: net[k] for k in layers}
In [13]:
# Precompute layer activations for photo and artwork
input_im_theano = T.tensor4()
outputs = lasagne.layers.get_output(layers.values(), input_im_theano)
content_features = {k: theano.shared(output.eval({input_im_theano: content}))
for k, output in zip(layers.keys(), outputs)}
style_features = {k: theano.shared(output.eval({input_im_theano: style}))
for k, output in zip(layers.keys(), outputs)}
In [14]:
# Get expressions for layer activations for generated image
generated_image = theano.shared(floatX(np.random.uniform(-128, 128, (1, 3, IMAGEWIDTH, IMAGEWIDTH))))
gen_features = lasagne.layers.get_output(layers.values(), generated_image)
gen_features = {k: v for k, v in zip(layers.keys(), gen_features)}
In [15]:
# Define loss function
losses = []
# content loss
for layer, weight in weights['content'].iteritems():
losses.append(weight * content_loss(content_features, gen_features, layer))
# style loss
for layer, weight in weights['style'].iteritems():
losses.append(weight * style_loss(style_features, gen_features, layer))
# total variation penalty
losses.append(0.1e-7 * total_variation_loss(generated_image))
total_loss = sum(losses)
In [16]:
grad = T.grad(total_loss, generated_image)
In [17]:
# Theano functions to evaluate loss and gradient
f_loss = theano.function([], total_loss)
f_grad = theano.function([], grad)
# Helper functions to interface with scipy.optimize
def eval_loss(x0):
x0 = floatX(x0.reshape((1, 3, IMAGEWIDTH, IMAGEWIDTH)))
generated_image.set_value(x0)
return f_loss().astype('float64')
def eval_grad(x0):
x0 = floatX(x0.reshape((1, 3, IMAGEWIDTH, IMAGEWIDTH)))
generated_image.set_value(x0)
return np.array(f_grad()).flatten().astype('float64')
In [23]:
# Initialize with a noise image
generated_image.set_value(floatX(np.random.uniform(-128, 128, (1, 3, IMAGEWIDTH, IMAGEWIDTH))))
x0 = generated_image.get_value().astype('float64')
xs = []
xs.append(x0)
# Optimize, saving the result periodically
for i in range(8):
print(i)
scipy.optimize.fmin_l_bfgs_b(eval_loss, x0.flatten(), fprime=eval_grad, maxfun=40)
x0 = generated_image.get_value().astype('float64')
xs.append(x0)
In [24]:
def deprocess(x):
x = np.copy(x[0])
x += MEAN_VALUES
x = x[::-1]
x = np.swapaxes(np.swapaxes(x, 0, 1), 1, 2)
x = np.clip(x, 0, 255).astype('uint8')
return x
In [25]:
figure(figsize=(12,12))
for i in range(9):
subplot(3, 3, i+1)
gca().xaxis.set_visible(False)
gca().yaxis.set_visible(False)
imshow(deprocess(xs[i]))
tight_layout()
In [26]:
figure(figsize=(8,8))
imshow(deprocess(xs[-1]), interpolation='nearest')
Out[26]:
In [27]:
imsave('../images/result/starry-vasiliy.jpg',deprocess(xs[-1]), )
In [ ]: