In [ ]:

    

import os
import numpy as np

import tensorflow as tf
from tensorflow.python.framework import ops

import load_data as load
from models.model import Model
from models.customlayers import *
from models.activations import *
from training import *

import moviepy.editor as mpe
from render import render_movie

from models.AELSTM import *
L = tf.layers

import matplotlib.pyplot as plt
% matplotlib inline


    

In [ ]:

    

data_dir = os.path.expanduser('~/Insight/video-representations/frames')


    

In [ ]:

    

# consider wrapping this into a "Trainer" class?
training_epochs = 25
batchsize = 8
sequence_length = 64

model = Model(encoder, lstm_cell, tied_decoder, batchsize, sequence_length)

## LSTM-Encoder Training Graph ##

training_inputs, training_targets = load.inputs('training', batchsize, training_epochs)

encoded, transitioned, decoded = model.build(training_inputs)
frame_loss = tf.reduce_mean(tf.pow(decoded - training_targets, 2))
training_targets_diff = training_targets[:, 1:, :, :, :] - training_targets[:, :-1, :, :, :]

decoded_diff = decoded[:, 1:, :, :, :] - decoded[:, :-1, :, :, :]
transition_loss = tf.reduce_mean(tf.pow(decoded_diff - training_targets_diff, 2))

loss = .9*frame_loss + .1*transition_loss

optimizer = tf.train.AdamOptimizer(learning_rate=.00001)
trainable_vars = tf.trainable_variables()
clipped_gradients, _ = tf.clip_by_global_norm(tf.gradients(loss, trainable_vars), 1)
train_step = optimizer.apply_gradients(zip(clipped_gradients, trainable_vars))

## LSTM-Encoder Validation Graph ##

validation_inputs, validation_targets = load.inputs('validation', batchsize, 1)

encoded_validation, transitioned_validation, decoded_validation = model.build(validation_inputs, reuse=True)
validation_loss = tf.reduce_mean(tf.pow(decoded_validation - validation_targets, 2))


    

In [ ]:

    

np.sum([np.prod(v.get_shape().as_list()) for v in tf.trainable_variables()]) / 1000000


    

In [ ]:

    

# compare this to training.py and merge
saver = tf.train.Saver()
init_global = tf.global_variables_initializer()
init_local = tf.local_variables_initializer()

coord = tf.train.Coordinator()

with tf.Session() as sesh:
    sesh.run([init_global, init_local])
    threads = tf.train.start_queue_runners(sess=sesh, coord=coord)
    
    # initialize lists for tracking
    
    decoder_losses = []
    decoder_validation_losses = []
    
    predictions = []
    encodings = []
    transitions = []
    validation_predictions = []
    validation_transitions = []
    validation_encodings = []
    recovery = []
    validation_recovery = []
    
    # first, encoder training
    try:
        step = 0
        
        while not coord.should_stop():
            _, loss_value, enc, trans, pred, input_recover = sesh.run(
                [train_step, loss, encoded, transitioned, decoded, training_targets]
            )
            
            decoder_losses.append(loss_value)
            
            if step % 250 == 0:
                print(step, loss_value)
                encodings.append(enc)
                transitions.append(trans)
                predictions.append(pred)
                recovery.append(input_recover)
                
            step += 1
            
    except tf.errors.OutOfRangeError:
        print('Encoder trained: {:.2f}'.format(loss_value))
        
    # second, encoder validation
    try:
        step = 0
        
        while not coord.should_stop():
            loss_value, enc, trans, pred, input_recover = sesh.run(
                [validation_loss, encoded_validation, transitioned_validation, 
                 decoded_validation, validation_targets]
            )
            decoder_validation_losses.append(loss_value)
            
            if step % 100 == 0:
                print(step, loss_value)
                validation_encodings.append(enc)
                validation_transitions.append(trans)
                validation_predictions.append(pred)
                validation_recovery.append(input_recover)
                
            step += 1
            
    except tf.errors.OutOfRangeError:
        print('Encoder validated: {:.2f}'.format(loss_value))
        
    finally:
        coord.request_stop()
        
    coord.join(threads)
    saver.save(sesh, 'ptypelstm-tied-relu')


    

In [ ]:

    

fig, axes = plt.subplots(1, 1, figsize=(10, 10))
def rolling_mean(l, w=500):
    c = np.cumsum(l)
    c[w:] = c[w:] - c[:-w]
    return c[w-1:] / w
    

axes.plot(decoder_losses)
axes.plot(rolling_mean(decoder_losses))
plt.setp(axes, ylim=[0, 2000])


    

In [ ]:

    

frame_array = render_movie(recovery[vid][0], 'test_ae_recov2.mp4', 5)