In [ ]:
from IPython.display import Image, display
In [ ]:
Image('images/14_deepdream_flowchart.png')
In [ ]:
Image('images/14_deepdream_recursive_flowchart.png')
In [ ]:
%matplotlib inline
In [ ]:
import matplotlib.pyplot as plt
import tensorflow as tf
import numpy as np
import random
import math
# 图片操作
import PIL.Image
from scipy.ndimage.filters import gaussian_filter
In [ ]:
import inception5h
In [ ]:
inception5h.maybe_download()
In [ ]:
model = inception5h.Inception5h()
In [ ]:
len(model.layer_tensors)
In [ ]:
def load_image(filename):
image = PIL.Image.open(filename)
return np.float32(image)
In [ ]:
def save_image(image, filename):
# 确保图片数组的取值在[0,255]之间
image = np.clip(image, 0.0, 255.0)
image = image.astype(np.uint8)
with open(filename, 'wb') as f:
PIL.Image.fromarray(image).save(f, 'jpeg')
绘制图片。使用matplotlib来绘制低像素图片，使用PIL来绘制高清
In [ ]:
def plot_image(image):
if False:
# 使用matplotlib来绘制图片
image = np.clip(image/255.0, 0.0, 1.0)
plt.imshow(image, interpolation='lanczos')
plt.show()
else:
# 使用PIL来展示
image = np.clip(image, 0.0, 255.0)
image = image.astype(np.uint8)
display(PIL.Image.fromarray(image))
In [ ]:
def normalize_image(x):
x_min = x.min()
x_max = x.max()
# 将所有数值规整到[0,1]之间，使用max range方法
x_norm = (x-x_min) / (x_max - x_min)
return x_norm
In [ ]:
def plot_gradient(gradient):
gradient_normalized = normalize_image(gradient)
plt.imshow(gradient_normalized, interpolation='bilinear')
plt.show()
In [ ]:
def resize_image(image, size=None, factor=None):
if factor is not None:
# 按照设定的比例downscale
size = np.array(image.shape[0:2]) * factor
size = size.astype(int)
else:
size = size[0:2]
size = tuple(reversed(size))
img = np.clip(image, 0.0, 255.0)
img = img.astype(np.uint8)
# 使用ndarray创建PIL图片对象
img = PIL.Image.fromarray(img)
img_resized = img.resize(size, PIL.Image.LANCZOS)
# 将8-bit的像素值转化为float
img_resized = np.float32(img_resized)
return img_resized
In [ ]:
def get_tile_size(num_pixels, tile_size=400):
# 切分成多少片，取整
num_tiles = int(round(num_pixels / tile_size))
# 至少有一片
num_tiles = max(1, num_tiles)
# 反过来求取真正的切片大小
actual_tile_size = int(math.ceil(num_pixels / num_tiles))
return actual_tile_size
计算切分图像的gradient。
In [ ]:
def tiled_gradient(gradient, image, tile_size=400):
grad = np.zeros_like(image)
x_max, y_max, _ = image.shape
# x轴防线的tile-size
x_tile_size = get_tile_size(num_pixels=x_max, tile_size=tile_size)
# 1/4
x_tile_size4 = int(x_tile_size // 4)
# y轴方向的tile_size
y_tile_size = get_tile_size(num_pixels=y_max, tile_size=tile_size)
# 1/4
y_tile_size4 = int(y_tile_size // 4)
# 从x轴某个位置开始[-3/4, -1/4]之间的某个位置
x_start = random.randint(-3*x_tile_size4, -x_tile_size4)
while x_start < x_max:
# 如果加上x_tile_size，[1/4, 3/4]作为结束位置
x_end = x_start + x_tile_size
# 边界值
x_start_lim = max(x_start, 0)
x_end_lim = min(x_end, x_max)
y_start = random.randint(-3*y_tile_size4, -y_tile_size4)
while y_start < y_max:
y_end = y_start + y_tile_size
y_start_lim = max(y_start, 0)
y_end_lim = min(y_end, y_max)
img_tile = image[x_start_lim:x_end_lim,
y_start_lim:y_end_lim, :]
feed_dict = model.create_feed_dict(image=img_tile)
g = session.run(gradient, feed_dict=feed_dict)
# 标准化上述所得gradient。这能让gradient更加连贯。
g /= (np.std(g) + 1e-7)
grad[x_start_lim:x_end_lim,
y_start_lim:y_end_lim, :] = g
y_start = y_end
x_start = x_end
return grad
以下是DeepDream算法的主要实现。计算输入图片的在给定网络层的gradient，然后当前gradient的取值会累加到输入图片上。当前操作会执行多次。
In [ ]:
def optimize_image(layer_tensor, image, num_iterations=10, step_size=0.3, tile_size=400, show_gradient=False):
"""
Parameters:
layer_tensor: Reference to a tensor that will be maximized
image: Input image used as the starting point
num_iterations: Number of optimization iterations to perform
step_size: Scale for each step of the gradient ascent
tile_size: Size of the tiles when calculating the gradient
show_gradient: Plot the gradient in each iteration
"""
img = image.copy()
print ("Image before any change:\n")
plot_image(img)
gradient = model.get_gradient(layer_tensor)
for i in range(num_iterations):
grad = tiled_gradient(gradient=gradient, image=img)
"""
Blur the gradient with different amounts and add them together.
The blur amount is also incresed during the optimization. This was
found to give nice, smooth images. You can try change the formulas.
The blur-amount is called sigma(0=no blur, 1=low blur, etc.)
we could call gaussian_filter(grad, sigma=(sigma, sigma, 0.0))
which would not blur the colour-channel. This tends to give psychadelic / pastel
colours in the resulting images. when the color-channel is also blurred the colours
of the input image are mostly retained in the output image.
"""
sigma = (i * 4.0) / num_iterations + 0.5
grad_smooth1 = gaussian_filter(grad, sigma=sigma)
grad_smooth2 = gaussian_filter(grad, sigma=sigma*2)
grad_smooth3 = gaussian_filter(grad, sigma=sigma*0.5)
grad = (grad_smooth1 + grad_smooth2 + grad_smooth3)
step_size_scaled = step_size / (np.std(grad) + 1e-7)
img += grad * step_size_scaled
if show_gradient:
msg = "Gradient min: {0:>9.6f}, max: {1:>9.6f}, stepsize: {2:>9.2f}"
print(msg.format(grad.min(), grad.max(), step_size_scaled))
plot_gradient(grad)
else:
print(". ")
print
print("Image after: ")
plot_image(img)
return img
In [ ]:
def recursive_optimize(layer_tensor, image,
num_repeats=4, rescale_factor=0.7, blend=0.2,
num_iterations=10, step_size=3.0,
tile_size=400):
if num_repeats > 0:
# Blur the input image to prevent artifacts when downscaling.
sigma = 0.5
img_blur = gaussian_filter(image, sigma=(sigma, sigma, 0.0))
img_downscaled = resize_image(image=img_blur, factor=rescale_factor)
img_result = recursive_optimize(layer_tensor=layer_tensor,
image=img_downscaled,
num_repeats=num_repeats-1,
rescale_factor=rescale_factor,
blend=blend,
num_iterations=num_iterations,
step_size=step_size,
tile_size=tile_size)
img_upscaled = resize_image(image=img_result, size=image.shape)
image = blend * image + (1.0 - blend) * img_upscaled
print ("Recursive level:", num_repeats)
img_result = optimize_image(layer_tensor=layer_tensor, image=image, num_iterations=num_iterations, step_size=step_size, tile_size=tile_size)
return img_result
In [ ]:
session = tf.InteractiveSession(graph=model.graph)
In [ ]:
image = load_image(filename='images/hulk.jpg')
plot_image(image)
In [ ]:
layer_tensor = model.layer_tensors[2]
layer_tensor
Now run the DeepDream optimization algorithm for 10 iterations with a step-size of 6.0, which is twice as high as in the recursive optimizations below. We also show the gradient for each iteration and you should note the visible artifacts in the seams between the tiles.
In [ ]:
img_result = optimize_image(layer_tensor, image, num_iterations=10, step_size=6.0, tile_size=400, show_gradient=True)
In [ ]:
save_image(img_result, filename='images/hulk_dream.jpg')
In [ ]:
img_result = recursive_optimize(layer_tensor=layer_tensor,
image=image,
num_iterations=10,
step_size=3.0,
rescale_factor=0.7,
num_repeats=4, blend=0.2
)
In [ ]:
layer_tensor = model.layer_tensors[7][:,:,:,0:3]
img_result = recursive_optimize(layer_tensor=layer_tensor, image=image,
num_iterations=10, step_size=3.0, rescale_factor=0.7,
num_repeats=4, blend=0.2)
In [ ]:
image = load_image(filename='images/giger.jpg')
plot_image(image)
In [ ]:
layer_tensor = model.layer_tensors[5]
img_result = recursive_optimize(layer_tensor=layer_tensor, image=image,
num_iterations=10, step_size=3.0, rescale_factor=0.7,
num_repeats=4, blend=0.2)
In [ ]:
image = load_image(filename='images/escher_planefilling2.jpg')
plot_image(image)
In [ ]:
layer_tensor = model.layer_tensors[6]
img_result = recursive_optimize(layer_tensor=layer_tensor, image=image,
num_iterations=10, step_size=3.0, rescale_factor=0.7,
num_repeats=4, blend=0.2)
In [ ]:
image = load_image(filename='images/simpsons.jpg')
plot_image(image)
In [ ]:
layer_tensor = model.layer_tensors[6]
img_result = recursive_optimize(layer_tensor=layer_tensor, image=image,
num_iterations=5, step_size=3.0, rescale_factor=0.7,
num_repeats=4, blend=0.2)
In [ ]:
image = load_image(filename='images/simpsons2.jpg')
plot_image(image)
In [ ]:
layer_tensor = model.layer_tensors[-1]
img_result = recursive_optimize(layer_tensor=layer_tensor, image=image,
num_iterations=5, step_size=4.0, rescale_factor=0.7,
num_repeats=4, blend=0.2)
In [ ]: