In [ ]:
# Force matplotlib to use inline rendering
%matplotlib inline
import os
import sys
import time
# add path to libraries for ipython
sys.path.append(os.path.expanduser("~/libs"))
import numpy as np
import tensorflow as tf
import tensorlight as light
DATA_ROOT = "data"
TRAIN_DIR = "train-test/datasets"
In [ ]:
def write_animation(dir_path, inputs, targets, fps, index):
concat_tgt = np.concatenate((inputs, targets))
print(concat_tgt.shape)
light.utils.video.write_multi_gif(os.path.join(dir_path, "anim-{:02d}.gif".format(index)),
[concat_tgt],
fps=fps, pad_value=1.0)
light.utils.video.write_multi_image_sequence(os.path.join(dir_path, "timeline-{:02d}.png".format(index)),
[concat_tgt],
pad_value=1.0)
In [ ]:
mnist_train = light.datasets.mnist.MNISTTrainDataset(DATA_ROOT)
mnist_valid = light.datasets.mnist.MNISTValidDataset(DATA_ROOT)
mnist_test = light.datasets.mnist.MNISTTestDataset(DATA_ROOT)
In [ ]:
def display_mnist(dataset):
x, y = dataset.get_batch(1)
light.visualization.display_array(x[0])
print('Label: {}'.format(y))
display_mnist(mnist_train)
display_mnist(mnist_valid)
display_mnist(mnist_test)
In [ ]:
SEQ_LEN = 10
moving_train = light.datasets.moving_mnist.MovingMNISTTrainDataset(
DATA_ROOT, input_shape=[SEQ_LEN,64,64,1], target_shape=[SEQ_LEN,64,64,1])
moving_valid = light.datasets.moving_mnist.MovingMNISTValidDataset(
DATA_ROOT, input_shape=[SEQ_LEN,64,64,1], target_shape=[SEQ_LEN,64,64,1])
moving_test = light.datasets.moving_mnist.MovingMNISTTestDataset(
DATA_ROOT, input_seq_length=SEQ_LEN, target_seq_length=SEQ_LEN)
In [ ]:
def display_moving(dataset, title):
x, y = dataset.get_batch(8)
for i in range(x.shape[0]):
full_seq = np.concatenate((x[i], y[i]))
light.visualization.display_batch(full_seq, nrows=4, ncols=5, title=title)
write_animation(os.path.join(TRAIN_DIR, "mm", "out", title), x[i], y[i], 5, i)
display_moving(moving_train, 'Train')
display_moving(moving_valid, 'Validation')
display_moving(moving_test, 'Test')
In [ ]:
def write_single_images(dataset):
x, y = dataset.get_batch(1)
for i in range(x.shape[1]):
light.utils.image.write(os.path.join("out", "mm-input-{}.png".format(i)), x[0][i])
for i in range(y.shape[1]):
light.utils.image.write(os.path.join("out", "mm-target-{}.png".format(i)), y[0][i])
write_single_images(moving_train)
In [ ]:
ucf11_train = light.datasets.ucf11.UCF11TrainDataset(DATA_ROOT,
input_seq_length=3, target_seq_length=3,
image_scale_factor=0.5, gray_scale=False,
min_examples_in_queue=128, queue_capacitiy=256,
num_threads=16, do_distortion=False,
crop_size=None)
ucf11_valid = light.datasets.ucf11.UCF11ValidDataset(DATA_ROOT,
input_seq_length=3, target_seq_length=3,
image_scale_factor=0.5, gray_scale=False,
crop_size=None)
In [ ]:
batch_x, batch_y = ucf11_train.get_batch(32)
with tf.Session() as sess:
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
try:
print('Starting queue runners...')
x, y = sess.run([batch_x, batch_y])
except tf.errors.OutOfRangeError:
print("Done training -- epoch limit reached")
finally:
# When done, ask the threads to stop
coord.request_stop()
coord.join(threads)
def display_ucf11_queue(x, y, title):
print("x-range: [{}, {}], y-range: [{}, {}]".format(x.min(), x.max(), y.min(), y.max()))
full_seq = np.concatenate((x[0], y[0]))
light.visualization.display_batch(full_seq, nrows=2, ncols=3, title=title)
def display_ucf11_batch(dataset, title):
x, y = dataset.get_batch(2)
print("x-range: [{}, {}], y-range: [{}, {}]".format(x.min(), x.max(), y.min(), y.max()))
light.visualization.display_batch(x[0], nrows=1, ncols=3, title=title + '-Inputs')
light.visualization.display_batch(y[0], nrows=1, ncols=3, title=title + '-Targets')
display_ucf11_queue(x, y, 'Train')
display_ucf11_batch(ucf11_valid, 'Validation')
In [ ]:
ucf101_train = light.datasets.ucf101.UCF101TrainDataset(DATA_ROOT,
input_seq_length=10, target_seq_length=10,
image_scale_factor=0.5, gray_scale=False,
num_threads=16, min_examples_in_queue=32,
queue_capacitiy=64, do_distortion=False,
crop_size=None)
ucf101_valid = light.datasets.ucf101.UCF101ValidDataset(DATA_ROOT,
input_seq_length=10, target_seq_length=10,
image_scale_factor=0.5, gray_scale=False,
double_with_flipped=True,
crop_size=None)
ucf101_test = light.datasets.ucf101.UCF101TestDataset(DATA_ROOT,
input_seq_length=10, target_seq_length=10,
image_scale_factor=0.5, gray_scale=False,
double_with_flipped=False,
crop_size=None)
In [ ]:
print("Train-size: {}".format(ucf101_train.size))
print("Valid-size: {}".format(ucf101_valid.size))
print("Test-size: {}".format(ucf101_test.size))
In [ ]:
with tf.device("/cpu:0"):
batch_x, batch_y = ucf101_train.get_batch(8)
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
try:
print('Starting queue runners...')
x, y = sess.run([batch_x, batch_y])
except tf.errors.OutOfRangeError:
print("Done training -- epoch limit reached")
finally:
# When done, ask the threads to stop
coord.request_stop()
coord.join(threads)
def display_ucf101(x, y, title):
print("x-shape: {}, y-shape: ", x.shape, y.shape)
print("x-range: [{}, {}], y-range: [{}, {}]".format(x.min(), x.max(), y.min(), y.max()))
for i in range(x.shape[0]):
diff = 0.0
for j in range(x.shape[1] - 1):
diff += np.sum(np.square( (x[i,j+1] - x[i,j]) * 2) )
print("Diff @ {}: {}".format(i, diff))
full_seq = np.concatenate((x[i], y[i]))
light.visualization.display_batch(full_seq, nrows=4, ncols=5, title="{}-{:2d}".format(title, i))
write_animation(os.path.join(TRAIN_DIR, "ucf", "out-full", title), x[i], y[i], 5, i)
display_ucf101(x, y, 'Train')
print('-' * 80)
x, y = ucf101_valid.get_batch(8)
display_ucf101(x, y, 'Validation')
print('-' * 80)
x, y = ucf101_test.get_batch(8)
display_ucf101(x, y, 'Test')
In [ ]:
pac_train = light.datasets.ms_pacman.MsPacmanTrainDataset(DATA_ROOT,
input_seq_length=8,
target_seq_length=8,
crop_size=None)
pac_valid = light.datasets.ms_pacman.MsPacmanValidDataset(DATA_ROOT,
input_seq_length=8,
target_seq_length=8,
crop_size=None)
pac_test = light.datasets.ms_pacman.MsPacmanTestDataset(DATA_ROOT,
input_seq_length=8,
target_seq_length=8,
crop_size=None)
print("Dataset-lengths:", pac_train.size, pac_valid.size, pac_test.size)
In [ ]:
def display_moving(dataset, title):
x, y = dataset.get_batch(8)
for i in range(x.shape[0]):
full_seq = np.concatenate((x[i], y[i]))
light.visualization.display_batch(full_seq, nrows=4, ncols=5, title=title)
write_animation(os.path.join(TRAIN_DIR, "pac", "out-full", title), x[i], y[i], 5, i)
display_moving(pac_train, 'Train')
display_moving(pac_valid, 'Validation')
display_moving(pac_test, 'Test')
In [ ]:
class SimplePredictionModel(light.model.AbstractModel):
"""Predicts n future frames from given n frames."""
def __init__(self, weight_decay=0.0):
super(SimplePredictionModel, self).__init__(weight_decay)
@light.utils.attr.override
def inference(self, inputs, targets, feeds,
is_training, device_scope, memory_device):
with tf.variable_scope("Encoder"):
# stack time-dimension on channels (inputs=[bs,t,h,w,c])
unpacked_inputs = tf.unpack(inputs, axis=1) # unpacked_inputs=t*[bs,h,w,c]
stacked_inputs = tf.concat(concat_dim=3, values=unpacked_inputs) # stacked_inputs=[bs,h,w,c*t]
# 1: Conv
conv1 = light.network.conv2d("Conv1", stacked_inputs,
16, (5, 5), (2, 2),
weight_init=tf.contrib.layers.xavier_initializer_conv2d(),
bias_init=0.1,
regularizer=tf.contrib.layers.l2_regularizer(self.weight_decay),
activation=tf.nn.relu)
# 2: Conv
conv2 = light.network.conv2d("Conv2", conv1,
16, (3, 3), (2, 2),
weight_init=tf.contrib.layers.xavier_initializer_conv2d(),
bias_init=0.1,
regularizer=tf.contrib.layers.l2_regularizer(self.weight_decay),
activation=tf.nn.relu)
encoder_out = conv2
with tf.variable_scope("Decoder"):
# 3: Deconv
conv3t = light.network.conv2d_transpose("Deconv1", encoder_out,
16, (3, 3), (2, 2),
weight_init=light.init.bilinear_initializer(),
bias_init=0.1,
regularizer=tf.contrib.layers.l2_regularizer(self.weight_decay),
activation=tf.nn.relu)
# 4: Deconv
target_shape = targets.get_shape().as_list()
channels_out = target_shape[1] * target_shape[4] # t * c
conv4t = light.network.conv2d_transpose("Deconv2", conv3t,
channels_out, (5, 5), (2, 2),
weight_init=light.init.bilinear_initializer(),
bias_init=0.1,
regularizer=tf.contrib.layers.l2_regularizer(self.weight_decay),
activation=tf.nn.sigmoid)
# split channel dimensions
spl_out = tf.split(split_dim=3, num_split=target_shape[1], value=conv4t) # spl_out=t*[b,h,w,c]
decoder_out = tf.pack(spl_out, axis=1) # decoder_out=[b,t,h,w,c]
return decoder_out
@light.utils.attr.override
def loss(self, predictions, targets, device_scope):
return light.loss.bce(predictions, targets)
In [ ]:
runtime = light.core.DefaultRuntime(TRAIN_DIR)
runtime.register_datasets(ucf11_train, ucf11_valid)
runtime.register_model(SimplePredictionModel(weight_decay=0.001))
runtime.build()
In [ ]:
runtime.train(batch_size=16, steps=1000)
In [ ]:
def display_ucf11(seq, title):
light.visualization.display_batch(seq, nrows=1, ncols=3, title=title)
x, y = ucf11_valid.get_batch(1)
pred = runtime.predict(x)
display_ucf11(x[0], "Inputs")
display_ucf11(y[0], "Targets")
display_ucf11(pred[0], "Pred")
Remarks: Using too feq threads can lead to an OutOfRange exception, because we consuming batches in faster speed than the producers can create examples.
The queue might be feel to be slower, because it might perform image-preprocessing, as well as a session is launched in every run.
In [ ]:
LOOPS = 10000
BATCH_SIZE = 32
# define which dataset to test
bench_dataset_queue = ucf11_train
bench_dataset_feeding = ucf11_valid
a = tf.constant(2)
b = tf.constant(3)
c = tf.constant(4)
# do a calcualtion to slow (especially the benchmark with the queue) a little bit down, because it can not
# produce examples in such a high speed. Additionally, this makes both benchmarks more comparable, because
# the feeding_queue needs to run a session as well with this calculation.
calculation = a * b + c
In [ ]:
with tf.device("/cpu:0"):
# Inputs should be done on CPU only (best performance)
batch_x, batch_y = bench_dataset_queue.get_batch(BATCH_SIZE)
with tf.Session() as sess:
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
try:
this_step = 0
overhead = 0.0
while not coord.should_stop():
this_step += 1
if this_step > LOOPS + 1: # extra loop because we skip the first run
break
if this_step == 1:
print("Starting queue runners...")
if this_step == 2:
print("Starting...")
# skipt 1st round, because queue runners have to be filled
start_time = time.time()
if this_step % 1000 == 0:
print(this_step)
x, y, _ = sess.run([batch_x, batch_y, calculation])
except tf.errors.OutOfRangeError:
print("Interrupted: Queue runners are out of range. Epoch limit reached?")
finally:
# When done, ask the threads to stop
duration = time.time() - start_time
coord.request_stop()
coord.join(threads)
print("Duration: {}".format(duration))
In [ ]:
with tf.Session() as sess:
overhead = 0.0
for i in xrange(LOOPS):
if i == 0:
print("Starting...")
start_time = time.time()
start_overhead = time.time()
_ = sess.run(calculation)
overhead += time.time() - start_overhead
batch_x, batch_y = bench_dataset_feeding.get_batch(BATCH_SIZE)
duration = time.time() - start_time
print("Duration: {}".format(duration))
print("Overhead: {}".format(overhead))
In [ ]: