In [1]:

    

import tensorflow as tf
import numpy as np
import pandas as pd
import time, os

from tensorflow.python.ops import variable_scope as vs
from tensorflow.python.ops import math_ops, array_ops
from tensorflow.python.util import nest
from tensorflow.python.ops.nn import rnn_cell

RNNCell = rnn_cell.RNNCell

class config(object):
    # Set network parameters
    # Empirically, depth more important than layer size - output dimension
    mode = 'kst'
    channel = 'rho0'
    n_particles = 5
    rnn_cell = 'gru' # 'gru'
    hidden_size = 512  # Number of neurons per RNN Cell
    keep_prob = 1.0
    input_keep_prob = 0.95
    recurrent_keep_prob = 0.95
    num_epochs = 128
    batch_size = 256
    n_layers = 1 # Note: 3 layers is considered 'deep'
    learning_rate = 4e-3
    lr_epoch_decay = 0.99
    ema_decay = 0.999
    weight_decay = 1e-7
    n_classes = 2
    depth = 5 #  the recurrence depth
    max_grad_norm = 10.
    init_bias = -2.0
    drop_x = 0.00
    drop_i = 0.00
    drop_h = 0.00
    drop_o = 0.00
    
class directories(object):
    data = 'data'
    tensorboard = 'tensorboard'
    checkpoints = 'checkpoints'
    samples = 'samples'
    
architecture = '{} - {} | Base cell: {} | Hidden units: {} | Layers: {} | Batch: {} | Epochs: {}'.format(
    config.channel, config.mode, config.rnn_cell, config.hidden_size, config.n_layers, config.batch_size, config.num_epochs)

class reader():
    def __init__(self, df):
        
        self.df = df
        self.batch_size = config.batch_size
        self.steps_per_epoch = len(df) // config.batch_size
        self.epochs = 0
        self.proceed = True
        self.shuffle()

    def shuffle(self):
        self.df = self.df.sample(frac=1).reset_index(drop=True)
        self.df_X = self.df.drop('Labels', axis = 1)
        self.df_y = self.df['Labels']
        self.pointer = 0

    def next_batch(self, batch_size):
        if self.pointer + 1 >= self.steps_per_epoch:
            inputs = self.df_X.iloc[self.pointer*batch_size:]
            targets = self.df_y.iloc[self.pointer*batch_size:]
            self.epochs += 1
            self.shuffle()
            self.proceed = False
            
        inputs = self.df_X.iloc[self.pointer*batch_size:(self.pointer+1)*batch_size]
        targets = self.df_y.iloc[self.pointer*batch_size:(self.pointer+1)*batch_size]
        self.pointer += 1
                
        return inputs, targets

def save_summary(config, delta_t, train_acc, test_acc):
    import json
    summary = {
        'Timestamp': time.strftime('%c'),
        'Base cell': config.rnn_cell,
        'Hidden units': config.hidden_size,
        'Layers': config.n_layers,
        'Batch_size': config.batch_size,
        'Seq_length': config.seq_length,
        'Dropout': config.keep_prob,
        'Epochs': config.num_epochs,
        'Time': delta_t,
        'Final train acc': train_acc,
        'Final test acc': test_acc
    }
    # Writing JSON data
    if os.path.isfile('rnn_summary.json'):
        with open('rnn_summary_{}.json.format(config.name)', 'r+') as f:
            new = json.load(f)
        new.append(summary)
        with open('rnn_summary.json', 'w') as f:
            json.dump(new, f, indent = 4)
    else:
        with open('rnn_summary.json', 'w') as f:
             json.dump([summary], f, indent = 4)
                
def load_data(file_name, test_size = 0.05):
    from sklearn.model_selection import train_test_split
    df = pd.read_hdf(file_name, 'df')
    df_X_train, df_X_test, df_y_train, df_y_test = train_test_split(df.drop('Labels', axis = 1),
                                                                    df['Labels'], test_size = test_size, random_state=42)
    return df_X_train, df_X_test, df_y_train, df_y_test

def load_reduced_data(file_name, test_size = 0.05):
    from sklearn.model_selection import train_test_split
    df = pd.read_hdf(file_name, 'df')
    df = p_ordering(df)
    df_X_train, df_X_test, df_y_train, df_y_test = train_test_split(df.drop('Labels', axis = 1),
                                                                    df['Labels'], test_size = test_size, random_state=42)
    return df_X_train, df_X_test, df_y_train, df_y_test

def p_ordering(df, ascending = True):
    # Drop errors, order particles by momentum descending
    df = df.drop([column for column in df.columns if column.endswith('Err')], axis = 1)
    labels = df['Labels']
    blocks = np.split(df.drop('Labels', axis = 1), config.n_particles, axis = 1)
    cols_p = [column for column in df.columns if column.endswith('cms_p')]
    p_mean = [df[column].mean() for column in cols_p]
    if ascending:
        p_ordered_frames = [blocks[i] for i in np.argsort(p_mean)]
    else:
        p_ordered_frames = [blocks[i] for i in np.argsort(p_mean)[::-1]]
        
    p_ordered_frames.append(labels)
    df_p_ordered = pd.concat(p_ordered_frames, axis = 1)
    return df_p_ordered

def plot_ROC_curve(network_output, y_true, meta = ''):
#     import matplotlib as mpl
#     mpl.use('pgf')
    import matplotlib.pyplot as plt
    import seaborn as sns
    from sklearn.metrics import roc_curve, auc
    y_score = network_output[:,1]
    
    # Compute ROC curve, integrate
    fpr, tpr, thresholds = roc_curve(y_true, y_score)    
    roc_auc = auc(fpr, tpr)
    
    plt.figure()
    plt.axes([.1,.1,.8,.7])
    plt.figtext(.5,.9, r'$\mathrm{Receiver \;Operating \;Characteristic}$', fontsize=15, ha='center')
    plt.figtext(.5,.85, meta, fontsize=10,ha='center')
    plt.plot(fpr, tpr, color='darkorange',
                     lw=2, label='ROC (area = %0.3f)' % roc_auc)
    plt.plot([0, 1], [0, 1], color='navy', lw=1.0, linestyle='--')
    plt.xlim([-0.05, 1.05])
    plt.ylim([-0.05, 1.05])
    plt.xlabel(r'$\mathrm{False \;Positive \;Rate}$')
    plt.ylabel(r'$\mathrm{True \;Positive \;Rate}$')
    plt.legend(loc="lower right")
    plt.savefig(os.path.join('graphs', '{}_{}_ROC.pdf'.format(config.channel, config.mode)), format='pdf', dpi=1000)
    #plt.savefig(os.path.join('graphs', '{}_{}_ROC.pgf'.format(config.channel, config.mode)), format='pgf', dpi=1000)
    print('AUC: {:.4f}'.format(roc_auc))
    plt.show()
    plt.gcf().clear()
    
def rhn_linear(args, output_size, bias, bias_start=None, scope=None):
    """
    This is a slightly modified version of _linear used by Tensorflow rnn.
    The only change is that we have allowed bias_start=None.
    Linear map: sum_i(args[i] * W[i]), where W[i] is a variable.
    Args:
        args: a 2D Tensor or a list of 2D, batch x n, Tensors.
        output_size: int, second dimension of W[i].
        bias: boolean, whether to add a bias term or not.
        bias_start: starting value to initialize the bias; 0 by default.
        scope: VariableScope for the created subgraph; defaults to "Linear".
    Returns:
        A 2D Tensor with shape [batch x output_size] equal to
        sum_i(args[i] * W[i]), where W[i]s are newly created matrices.
    Raises:
        ValueError: if some of the arguments has unspecified or wrong shape.
    """
    if args is None or (nest.is_sequence(args) and not args):
        raise ValueError("`args` must be specified")
    if not nest.is_sequence(args):
        args = [args]

    # Calculate the total size of arguments on dimension 1.
    total_arg_size = 0
    shapes = [a.get_shape().as_list() for a in args]
    for shape in shapes:
        if len(shape) != 2:
            raise ValueError("Linear is expecting 2D arguments: %s" % str(shapes))
        if not shape[1]:
            raise ValueError("Linear expects shape[1] of arguments: %s" % str(shapes))
        else:
            total_arg_size += shape[1]

    dtype = [a.dtype for a in args][0]

#     # Now the computation.
#     with vs.variable_scope(scope or "Linear"):
#         matrix = vs.get_variable(
#                 "Matrix", [total_arg_size, output_size], dtype=dtype)
#         if len(args) == 1:
#             res = math_ops.matmul(args[0], matrix)
#         else:
#             res = math_ops.matmul(tf.concat(args, axis = 1), matrix)
#         if not bias:
#             return res
#         elif bias_start is None:
#             bias_term = vs.get_variable("Bias", [output_size], dtype=dtype)
#         else:
#             bias_term = vs.get_variable("Bias", [output_size], dtype=dtype,
#                                                                     initializer=tf.constant_initializer(bias_start, dtype=dtype))
            
    # Now the computation.
    with vs.variable_scope(scope or "Linear"):
        matrix = vs.get_variable(
                "weights", [total_arg_size, output_size], initializer=tf.contrib.layers.xavier_initializer())
        if len(args) == 1:
            res = math_ops.matmul(args[0], matrix)
        else:
            res = math_ops.matmul(tf.concat(args, axis = 1), matrix)
        if not bias:
            return res
        elif bias_start is None:
            bias_term = vs.get_variable("Bias", [output_size], dtype=dtype)
        else:
            bias_term = vs.get_variable("Bias", [output_size], dtype=dtype,
                                                                    initializer=tf.constant_initializer(bias_start, dtype=dtype))
            
    return res + bias_term


def get_noise(x, m, drop_x, drop_i, drop_h, drop_o):
    keep_x, keep_i, keep_h, keep_o = 1.0 - drop_x, 1.0 - drop_i, 1.0 - drop_h, 1.0 - drop_o
    if keep_x < 1.0:
        noise_x = (np.random.random_sample((x.shape[0], m.num_steps, config.seq_length)) < keep_x).astype(np.float32) / keep_x
        for b in range(x.shape[0]):
            for n1 in range(m.num_steps):
                for n2 in range(n1 + 1, m.num_steps):
                    if x[b][n2] == x[b][n1]:
                        noise_x[b][n2][0] = noise_x[b][n1][0]
                        break
    else:
        noise_x = np.ones((x.shape[0], m.num_steps, config.seq_length), dtype=np.float32)

    if keep_i < 1.0:
        noise_i = (np.random.random_sample((x.shape[0], m.in_size, m.n_layers)) < keep_i).astype(np.float32) / keep_i
    else:
        noise_i = np.ones((x.shape[0], m.in_size, m.n_layers), dtype=np.float32)
    if keep_h < 1.0:
        noise_h = (np.random.random_sample((x.shape[0], m.size, m.n_layers)) < keep_h).astype(np.float32) / keep_h
    else:
        noise_h = np.ones((x.shape[0], m.size, m.n_layers), dtype=np.float32)
    if keep_o < 1.0:
        noise_o = (np.random.random_sample((x.shape[0], 1, m.size)) < keep_o).astype(np.float32) / keep_o
    else:
        noise_o = np.ones((x.shape[0], 1, m.size), dtype=np.float32)
    return noise_x, noise_i, noise_h, noise_o


    

In [2]:

    

test_file = '/data/projects/punim0011/jtan/data/rnn/rnn_B02rho0gamma_kst.h5'

df_X_train, df_X_test, df_y_train, df_y_test = load_reduced_data(test_file)
df_train = pd.concat([df_X_train, df_y_train], axis = 1)
df_test = pd.concat([df_X_test, df_y_test], axis = 1)

config.n_features = df_train.shape[1] - 1
config.seq_length = config.n_features//config.n_particles
config.steps_per_epoch = len(df_X_train) // config.batch_size
assert config.seq_length == config.n_features/config.n_particles, 'Discrepancy in input feature dimension'

readerTrain = reader(df_train)
readerTest = reader(df_test)


    

In [3]:

    

class Model(object):
    """A Variational RHN model."""
    def __init__(self, is_training, config):
        self.batch_size = batch_size = config.batch_size
        self.num_steps = num_steps = config.n_particles
        vocab_size = config.n_particles
        self.depth = depth = config.depth
        self.size = size = config.hidden_size
        self.in_size = size
        
        self.n_layers = n_layers = config.n_layers

        self.inputs = tf.placeholder(tf.float32, shape = [None, config.n_features])
        self.targets = tf.placeholder(tf.int32, shape = [None])
        
        self._noise_x = tf.placeholder(tf.float32, [None, config.n_particles, config.seq_length])
        self._noise_i = tf.placeholder(tf.float32, [None, size, n_layers])
        self._noise_h = tf.placeholder(tf.float32, [None, size, n_layers])
        self._noise_o = tf.placeholder(tf.float32, [None, 1, size])
        self.global_step = tf.get_variable('global_step', [], initializer=tf.constant_initializer(0), trainable=False)


        self.rnn_inputs = tf.reshape(self.inputs, [-1, config.n_particles, config.seq_length]) * self._noise_x

#         with tf.device("/cpu:0"):
#             embedding = tf.get_variable("embedding", [vocab_size, size])
#             inputs = tf.nn.embedding_lookup(embedding, self._input_data) * self._noise_x

        outputs = []
        self._initial_state = [0] * self.n_layers
        state = [0] * self.n_layers
        self._final_state = [0] * self.n_layers
        for l in range(config.n_layers):
            with tf.variable_scope('RHN_{}'.format((l))):
                cell = RHNCell(size, size, is_training, depth=depth, forget_bias=config.init_bias)
                self._initial_state[l] = cell.zero_state(batch_size, tf.float32)
                state[l] = [self._initial_state[l], self._noise_i[:, :, l], self._noise_h[:, :, l]]
                for time_step in range(num_steps):
                    if time_step > 0:
                        tf.get_variable_scope().reuse_variables()
                    (cell_output, state[l]) = cell(self.rnn_inputs[:, time_step, :], state[l], layer = l)
                    outputs.append(cell_output)
                self.rnn_inputs = tf.stack(outputs, axis=1)
                outputs = []
        
        encoding = self.rnn_inputs * self._noise_o
        #self.output = tf.reshape(self.rnn_inputs * self._noise_o, [-1, size])
        self.output = encoding[:,-1,:]
        softmax_w = tf.get_variable("softmax_w", [size, config.n_classes])
        softmax_b = tf.get_variable("softmax_b", [config.n_classes])
        self.logits_RNN = tf.matmul(self.output, softmax_w) + softmax_b

        self.cross_entropy = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits = self.logits_RNN, labels = self.targets))

        
        # loss = tf.nn.seq2seq.sequence_loss_by_example(
        #       [logits],
        #       [tf.reshape(self._targets, [-1])],
        #       [tf.ones([batch_size * num_steps])])
        # self._final_state = [s[0] for s in state]
        # pred_loss = tf.reduce_sum(loss) / batch_size
        # self._cost = cost = pred_loss

        tvars = tf.trainable_variables()
        l2_loss = tf.add_n([tf.nn.l2_loss(v) for v in tvars])
        self._cost = cost = self.cross_entropy + config.weight_decay * l2_loss

        self._nvars = np.prod(tvars[0].get_shape().as_list())
        print(tvars[0].name, tvars[0].get_shape().as_list())
        for var in tvars[1:]:
            sh = var.get_shape().as_list()
            print(var.name, sh)
            self._nvars += np.prod(sh)
        print(self._nvars, 'total variables')

        self.ema = tf.train.ExponentialMovingAverage(decay = config.ema_decay, num_updates = self.global_step)
        maintain_averages_op = self.ema.apply(tf.trainable_variables())
        
        # Evaluate correctness
        self.predictions = tf.nn.softmax(self.logits_RNN)
        correct_prediction = tf.equal(tf.cast(tf.argmax(self.predictions, 1), tf.int32), self.targets)
        self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
        
        if not is_training:
             return

        opt = tf.train.AdamOptimizer(config.learning_rate)
        grad_var_pairs = opt.compute_gradients(self.cross_entropy)
        grads, tvars = [gv_pair[0] for gv_pair in grad_var_pairs], [gv_pair[1] for gv_pair in grad_var_pairs]
        grads, _ = tf.clip_by_global_norm(grads, config.max_grad_norm)
        
        self.opt_op = opt.apply_gradients(zip(grads, tvars), global_step=self.global_step)

        self.ema = tf.train.ExponentialMovingAverage(decay = config.ema_decay, num_updates = self.global_step)
        maintain_averages_op = self.ema.apply(tf.trainable_variables())
        
        with tf.control_dependencies([self.opt_op]):
                self._train_op = tf.group(maintain_averages_op)


        tf.summary.scalar('cross_entropy', self.cross_entropy)
        tf.summary.scalar('accuracy', self.accuracy)
        self.merge_op = tf.summary.merge_all()

        # grads, _ = tf.clip_by_global_norm(tf.gradients(cost, tvars),
        #                                                                       config.max_grad_norm)
        # optimizer = tf.train.GradientDescentOptimizer(self.lr)
        # self._train_op = optimizer.apply_gradients(zip(grads, tvars))


    @property
    def noise_x(self):
        return self._noise_x

    @property
    def noise_i(self):
        return self._noise_i

    @property
    def noise_h(self):
        return self._noise_h

    @property
    def noise_o(self):
        return self._noise_o

    @property
    def initial_state(self):
        return self._initial_state

    @property
    def cost(self):
        return self._cost

    @property
    def final_state(self):
        return self._final_state

    @property
    def train_op(self):
        return self._train_op

    @property
    def nvars(self):
        return self._nvars
    
    def predict(self, ckpt, metaGraph = None):
        pin_cpu = tf.ConfigProto(allow_soft_placement=True, log_device_placement=True, device_count = {'GPU':0})
    
        # Restore the moving average version of the learned variables for eval.
        #variable_averages = tf.train.ExponentialMovingAverage(config.ema_decay)
        variables_to_restore = self.ema.variables_to_restore()
        #variables_to_restore = variable_averages.variables_to_restore()
        saver = tf.train.Saver(variables_to_restore)
        
        with tf.Session(config=pin_cpu) as sess:
            # Initialize variables
            init_op = tf.global_variables_initializer()
            sess.run(init_op)
            sess.run(tf.local_variables_initializer())
            start_time = time.time()
            assert (ckpt.model_checkpoint_path or metaGraph), 'Missing checkpoint file!'
            
            if metaGraph:
                saver = tf.train.import_meta_graph(metaGraph)
                saver.restore(sess, os.path.splitext(metaGraph)[0])
                print('{} restored.'.format(metaGraph))
            else:    
                saver.restore(sess, ckpt.model_checkpoint_path)
                print('{} restored.'.format(ckpt.model_checkpoint_path))

            # Make predictions using the trained model
            noise_x, noise_i, noise_h, noise_o = get_noise(df_X_test.values, rhn, 0., 0., 0., 0.)
            feed_dict_test = {self.inputs: df_X_test.values, self.targets: df_y_test.values,
                              self.noise_x: noise_x, self.noise_i: noise_i, self.noise_h: noise_h, self.noise_o: noise_o}
            network_output_test, final_v_acc = sess.run(
                [self.predictions, self.accuracy], feed_dict = feed_dict_test)

            print("Validation accuracy: {:g}\nValidation AUC: {:g}".format(final_v_acc, final_v_auc))
            
            plot_ROC_curve(network_output = network_output_test, y_true = df_y_test.values,
                           meta = architecture + ' | Test accuracy: {}'.format(final_v_acc))            
            delta_t = time.time() - start_time
            print("Inference complete. Duration: %g s" %(delta_t))
            
            return network_output_test


class RHNCell(RNNCell):
    """Variational Recurrent Highway Layer
    Reference: https://arxiv.org/abs/1607.03474
    """

    def __init__(self, num_units, size, is_training, depth=3, forget_bias=None):
        self._num_units = num_units
        self._size = size
        self.is_training = is_training
        self.depth = depth
        self.forget_bias = forget_bias

    @property
    def input_size(self):
        return self._size

    @property
    def output_size(self):
        return self._num_units

    @property
    def state_size(self):
        return self._num_units

    def __call__(self, inputs, state, layer, scope=None):
        current_state = state[0]
        noise_i = state[1]
        noise_h = state[2]
        for i in range(self.depth):
            with tf.variable_scope('h_'+str(i)):
                if i == 0:
                    if layer == 0:
                        h = tf.tanh(rhn_linear([inputs, current_state * noise_h], self._num_units, True))
                    else:
                        h = tf.tanh(rhn_linear([inputs * noise_i, current_state * noise_h], self._num_units, True))
                else:
                    h = tf.tanh(rhn_linear([current_state * noise_h], self._num_units, True))
            with tf.variable_scope('t_'+str(i)):
                if i == 0:
                    if layer == 0:
                        t = tf.sigmoid(rhn_linear([inputs, current_state * noise_h], self._num_units, True, self.forget_bias))
                    else:
                        t = tf.sigmoid(rhn_linear([inputs * noise_i, current_state * noise_h], self._num_units, True, self.forget_bias))
                else:
                    t = tf.sigmoid(rhn_linear([current_state * noise_h], self._num_units, True, self.forget_bias))
            current_state = (h - current_state)* t + current_state

        return current_state, [current_state, noise_i, noise_h]

def train(config, restore = False):
    
    rhn = Model(is_training = True, config = config)
    start_time = time.time()
    v_acc_best = 0.
    saver = tf.train.Saver()
    train_writer = tf.summary.FileWriter(
        os.path.join(directories.tensorboard, 'train_{}'.format(time.strftime('%d-%m_%I:%M'))), graph = tf.get_default_graph()) #'%d-%m_%I:%M'
    test_writer = tf.summary.FileWriter(os.path.join(directories.tensorboard, 'test_{}'.format(time.strftime('%d-%m_%I:%M'))))
    ckpt = tf.train.get_checkpoint_state(directories.checkpoints)
    
    with tf.Session() as sess:
        # Initialize variables
        init_op = tf.global_variables_initializer()
        sess.run(init_op)
        
        if restore and ckpt.model_checkpoint_path:
            print('{} restored.'.format(ckpt.model_checkpoint_path))
#             saver = tf.train.import_meta_graph('checkpoints/char-RNN__epoch49.ckpt-49.meta')
#             saver.restore(sess, 'checkpoints/char-RNN__epoch49.ckpt-49')
            saver.restore(sess, ckpt.model_checkpoint_path)
        
        state = [x.eval() for x in rhn.initial_state]
        
        for epoch in range(config.num_epochs):
            
            readerTrain.proceed = True
            begin = time.time()
            step = 0
            print('(*) Entering Epoch {} ({:.3f} s)'.format(epoch, time.time() - start_time))

            # Save every 10 epochs    
            if epoch % 10 == 0:
                save_path = saver.save(sess,
                                       os.path.join(directories.checkpoints,'rhn_{}_{}_epoch{}.ckpt'.format(config.mode, config.channel, epoch)),
                                       global_step = epoch)
                print('Graph saved to file: {}'.format(save_path))

            while(readerTrain.proceed):
                # Iterate through entire corpus
                x_train, y_train = readerTrain.next_batch(config.batch_size)
                noise_x, noise_i, noise_h, noise_o = get_noise(x_train.values, rhn, config.drop_x, config.drop_i, config.drop_h, config.drop_o)
                feed_dict_train = {rhn.inputs: x_train.values, rhn.targets: y_train.values,
                                   rhn.noise_x: noise_x, rhn.noise_i: noise_i, rhn.noise_h: noise_h, rhn.noise_o: noise_o}
                t_op = sess.run(rhn.train_op, feed_dict = feed_dict_train)
                step += 1

                if step % (config.steps_per_epoch // 8) == 0:            
                    # Evaluate model
                    improved = ''
                    x_test, y_test = readerTest.next_batch(config.batch_size)
                    noise_x, noise_i, noise_h, noise_o = get_noise(x_test.values, rhn, 0., 0., 0., 0.)
                    feed_dict_test = {rhn.inputs: x_test.values, rhn.targets: y_test.values,
                                      rhn.noise_x: noise_x, rhn.noise_i: noise_i, rhn.noise_h: noise_h, rhn.noise_o: noise_o}

                    t_acc, t_loss, t_summary = sess.run([rhn.accuracy, rhn.cross_entropy, rhn.merge_op],
                                                        feed_dict = feed_dict_train)
                    v_acc, v_loss, v_summary, = sess.run([rhn.accuracy, rhn.cross_entropy, rhn.merge_op],
                                                        feed_dict = feed_dict_test)

                    train_writer.add_summary(t_summary, step)
                    test_writer.add_summary(v_summary, step)
                    
                    if epoch > 5 and v_acc > v_acc_best:
                        v_acc_best = v_acc
                        improved = '*'
                        save_path = saver.save(sess, os.path.join(directories.checkpoints, 'best.ckpt'), global_step = epoch)
                    
                    print('Epoch {}, Step {} | Training Accuracy: {:.3f} | Test Accuracy: {:.3f} | Training Loss: {:.3f} | Test Loss {:.3f} {}'
                          .format(epoch, step, t_acc, v_acc, t_loss, v_loss, improved))

        save_path = saver.save(sess, os.path.join(directories.checkpoints, 'rhn_end'),
                               global_step = epoch)
        print('Metagraph saved to file: {}'.format(save_path))

        final_train_accuracy = rhn.accuracy.eval(feed_dict = {rhn.inputs: df_X_train.values, rhn.targets: df_y_train.values, 
                                                                rhn.keep_prob: 1.0, rhn.training_phase: False})
        final_test_accuracy = rhn.accuracy.eval(feed_dict = {rhn.inputs: df_X_test.values, rhn.targets: df_y_test.values,
                                                               rhn.keep_prob: 1.0, rhn.training_phase: False})
        delta_t = time.time() - start_time
            
    print("Training Complete. Time elapsed: {:.3f} s".format(delta_t))
    print("Train accuracy: %g\nValidation accuracy: %g" %(final_train_accuracy, final_test_accuracy))

    print('Architecture: {}'.format(architecture))
    save_summary(config, delta_t, final_train_accuracy, final_test_accuracy)


    

In [ ]:

    

model = Model(is_training = True, config = config)


    

In [ ]:

    

train(config)


    

    




RHN_0/h_0/Linear/weights:0 [526, 512]
RHN_0/h_0/Linear/Bias:0 [512]
RHN_0/t_0/Linear/weights:0 [526, 512]
RHN_0/t_0/Linear/Bias:0 [512]
RHN_0/h_1/Linear/weights:0 [512, 512]
RHN_0/h_1/Linear/Bias:0 [512]
RHN_0/t_1/Linear/weights:0 [512, 512]
RHN_0/t_1/Linear/Bias:0 [512]
RHN_0/h_2/Linear/weights:0 [512, 512]
RHN_0/h_2/Linear/Bias:0 [512]
RHN_0/t_2/Linear/weights:0 [512, 512]
RHN_0/t_2/Linear/Bias:0 [512]
RHN_0/h_3/Linear/weights:0 [512, 512]
RHN_0/h_3/Linear/Bias:0 [512]
RHN_0/t_3/Linear/weights:0 [512, 512]
RHN_0/t_3/Linear/Bias:0 [512]
RHN_0/h_4/Linear/weights:0 [512, 512]
RHN_0/h_4/Linear/Bias:0 [512]
RHN_0/t_4/Linear/weights:0 [512, 512]
RHN_0/t_4/Linear/Bias:0 [512]
softmax_w:0 [512, 2]
softmax_b:0 [2]
2641922 total variables
(*) Entering Epoch 0 (11.776 s)
Graph saved to file: checkpoints/rhn_kst_rho0_epoch0.ckpt-0
Epoch 0, Step 237 | Training Accuracy: 0.621 | Test Accuracy: 0.641 | Training Loss: 0.632 | Test Loss 0.610 
Epoch 0, Step 474 | Training Accuracy: 0.730 | Test Accuracy: 0.672 | Training Loss: 0.597 | Test Loss 0.643 
Epoch 0, Step 711 | Training Accuracy: 0.734 | Test Accuracy: 0.734 | Training Loss: 0.522 | Test Loss 0.573 
Epoch 0, Step 948 | Training Accuracy: 0.746 | Test Accuracy: 0.707 | Training Loss: 0.531 | Test Loss 0.517 
Epoch 0, Step 1185 | Training Accuracy: 0.711 | Test Accuracy: 0.762 | Training Loss: 0.543 | Test Loss 0.516 
Epoch 0, Step 1422 | Training Accuracy: 0.770 | Test Accuracy: 0.750 | Training Loss: 0.494 | Test Loss 0.514 
Epoch 0, Step 1659 | Training Accuracy: 0.844 | Test Accuracy: 0.770 | Training Loss: 0.359 | Test Loss 0.443 
Epoch 0, Step 1896 | Training Accuracy: 0.898 | Test Accuracy: 0.781 | Training Loss: 0.327 | Test Loss 0.457 
(*) Entering Epoch 1 (73.272 s)
Epoch 1, Step 237 | Training Accuracy: 0.781 | Test Accuracy: 0.809 | Training Loss: 0.469 | Test Loss 0.435 
Epoch 1, Step 474 | Training Accuracy: 0.828 | Test Accuracy: 0.832 | Training Loss: 0.340 | Test Loss 0.391 
Epoch 1, Step 711 | Training Accuracy: 0.828 | Test Accuracy: 0.789 | Training Loss: 0.398 | Test Loss 0.459 
Epoch 1, Step 948 | Training Accuracy: 0.820 | Test Accuracy: 0.816 | Training Loss: 0.385 | Test Loss 0.388 
Epoch 1, Step 1185 | Training Accuracy: 0.832 | Test Accuracy: 0.801 | Training Loss: 0.388 | Test Loss 0.448 
Epoch 1, Step 1422 | Training Accuracy: 0.832 | Test Accuracy: 0.828 | Training Loss: 0.391 | Test Loss 0.359 
Epoch 1, Step 1659 | Training Accuracy: 0.832 | Test Accuracy: 0.832 | Training Loss: 0.380 | Test Loss 0.393 
Epoch 1, Step 1896 | Training Accuracy: 0.824 | Test Accuracy: 0.840 | Training Loss: 0.372 | Test Loss 0.386 
(*) Entering Epoch 2 (133.823 s)
Epoch 2, Step 237 | Training Accuracy: 0.836 | Test Accuracy: 0.805 | Training Loss: 0.390 | Test Loss 0.422 
Epoch 2, Step 474 | Training Accuracy: 0.828 | Test Accuracy: 0.816 | Training Loss: 0.402 | Test Loss 0.444 
Epoch 2, Step 711 | Training Accuracy: 0.855 | Test Accuracy: 0.793 | Training Loss: 0.332 | Test Loss 0.405 
Epoch 2, Step 948 | Training Accuracy: 0.754 | Test Accuracy: 0.816 | Training Loss: 0.490 | Test Loss 0.414 
Epoch 2, Step 1185 | Training Accuracy: 0.812 | Test Accuracy: 0.793 | Training Loss: 0.392 | Test Loss 0.423 
Epoch 2, Step 1422 | Training Accuracy: 0.816 | Test Accuracy: 0.809 | Training Loss: 0.381 | Test Loss 0.385 
Epoch 2, Step 1659 | Training Accuracy: 0.836 | Test Accuracy: 0.844 | Training Loss: 0.377 | Test Loss 0.347 
Epoch 2, Step 1896 | Training Accuracy: 0.809 | Test Accuracy: 0.809 | Training Loss: 0.392 | Test Loss 0.371 
(*) Entering Epoch 3 (194.346 s)
Epoch 3, Step 237 | Training Accuracy: 0.785 | Test Accuracy: 0.828 | Training Loss: 0.443 | Test Loss 0.394 
Epoch 3, Step 474 | Training Accuracy: 0.824 | Test Accuracy: 0.848 | Training Loss: 0.391 | Test Loss 0.380 
Epoch 3, Step 711 | Training Accuracy: 0.871 | Test Accuracy: 0.844 | Training Loss: 0.324 | Test Loss 0.350 
Epoch 3, Step 948 | Training Accuracy: 0.855 | Test Accuracy: 0.781 | Training Loss: 0.350 | Test Loss 0.441 
Epoch 3, Step 1185 | Training Accuracy: 0.855 | Test Accuracy: 0.812 | Training Loss: 0.334 | Test Loss 0.400 
Epoch 3, Step 1422 | Training Accuracy: 0.809 | Test Accuracy: 0.812 | Training Loss: 0.425 | Test Loss 0.411 
Epoch 3, Step 1659 | Training Accuracy: 0.812 | Test Accuracy: 0.863 | Training Loss: 0.384 | Test Loss 0.338 
Epoch 3, Step 1896 | Training Accuracy: 0.797 | Test Accuracy: 0.777 | Training Loss: 0.410 | Test Loss 0.469 
(*) Entering Epoch 4 (254.857 s)
Epoch 4, Step 237 | Training Accuracy: 0.805 | Test Accuracy: 0.805 | Training Loss: 0.397 | Test Loss 0.389 
Epoch 4, Step 474 | Training Accuracy: 0.852 | Test Accuracy: 0.820 | Training Loss: 0.348 | Test Loss 0.368 
Epoch 4, Step 711 | Training Accuracy: 0.859 | Test Accuracy: 0.805 | Training Loss: 0.331 | Test Loss 0.432 
Epoch 4, Step 948 | Training Accuracy: 0.852 | Test Accuracy: 0.785 | Training Loss: 0.329 | Test Loss 0.422 
Epoch 4, Step 1185 | Training Accuracy: 0.820 | Test Accuracy: 0.816 | Training Loss: 0.406 | Test Loss 0.405 
Epoch 4, Step 1422 | Training Accuracy: 0.836 | Test Accuracy: 0.801 | Training Loss: 0.367 | Test Loss 0.389 
Epoch 4, Step 1659 | Training Accuracy: 0.852 | Test Accuracy: 0.828 | Training Loss: 0.359 | Test Loss 0.380 
Epoch 4, Step 1896 | Training Accuracy: 0.840 | Test Accuracy: 0.824 | Training Loss: 0.379 | Test Loss 0.378 
(*) Entering Epoch 5 (315.372 s)
Epoch 5, Step 237 | Training Accuracy: 0.824 | Test Accuracy: 0.805 | Training Loss: 0.362 | Test Loss 0.402 
Epoch 5, Step 474 | Training Accuracy: 0.836 | Test Accuracy: 0.812 | Training Loss: 0.365 | Test Loss 0.417 
Epoch 5, Step 711 | Training Accuracy: 0.844 | Test Accuracy: 0.844 | Training Loss: 0.388 | Test Loss 0.360 
Epoch 5, Step 948 | Training Accuracy: 0.824 | Test Accuracy: 0.863 | Training Loss: 0.385 | Test Loss 0.358 
Epoch 5, Step 1185 | Training Accuracy: 0.828 | Test Accuracy: 0.820 | Training Loss: 0.368 | Test Loss 0.415 
Epoch 5, Step 1422 | Training Accuracy: 0.832 | Test Accuracy: 0.805 | Training Loss: 0.419 | Test Loss 0.422 
Epoch 5, Step 1659 | Training Accuracy: 0.801 | Test Accuracy: 0.816 | Training Loss: 0.414 | Test Loss 0.427 
Epoch 5, Step 1896 | Training Accuracy: 0.859 | Test Accuracy: 0.824 | Training Loss: 0.336 | Test Loss 0.351 
(*) Entering Epoch 6 (375.897 s)
Epoch 6, Step 237 | Training Accuracy: 0.840 | Test Accuracy: 0.832 | Training Loss: 0.366 | Test Loss 0.324 *
Epoch 6, Step 474 | Training Accuracy: 0.805 | Test Accuracy: 0.836 | Training Loss: 0.406 | Test Loss 0.396 *
Epoch 6, Step 711 | Training Accuracy: 0.863 | Test Accuracy: 0.875 | Training Loss: 0.360 | Test Loss 0.311 *
Epoch 6, Step 948 | Training Accuracy: 0.816 | Test Accuracy: 0.836 | Training Loss: 0.370 | Test Loss 0.361 
Epoch 6, Step 1185 | Training Accuracy: 0.871 | Test Accuracy: 0.773 | Training Loss: 0.325 | Test Loss 0.452 
Epoch 6, Step 1422 | Training Accuracy: 0.820 | Test Accuracy: 0.812 | Training Loss: 0.371 | Test Loss 0.370 
Epoch 6, Step 1659 | Training Accuracy: 0.812 | Test Accuracy: 0.812 | Training Loss: 0.369 | Test Loss 0.391 
Epoch 6, Step 1896 | Training Accuracy: 0.848 | Test Accuracy: 0.844 | Training Loss: 0.372 | Test Loss 0.371 
(*) Entering Epoch 7 (437.643 s)
Epoch 7, Step 237 | Training Accuracy: 0.836 | Test Accuracy: 0.812 | Training Loss: 0.362 | Test Loss 0.395 
Epoch 7, Step 474 | Training Accuracy: 0.824 | Test Accuracy: 0.852 | Training Loss: 0.370 | Test Loss 0.366 
Epoch 7, Step 711 | Training Accuracy: 0.852 | Test Accuracy: 0.832 | Training Loss: 0.333 | Test Loss 0.362 
Epoch 7, Step 948 | Training Accuracy: 0.805 | Test Accuracy: 0.828 | Training Loss: 0.402 | Test Loss 0.396 
Epoch 7, Step 1185 | Training Accuracy: 0.816 | Test Accuracy: 0.863 | Training Loss: 0.392 | Test Loss 0.314 
Epoch 7, Step 1422 | Training Accuracy: 0.832 | Test Accuracy: 0.875 | Training Loss: 0.381 | Test Loss 0.313 
Epoch 7, Step 1659 | Training Accuracy: 0.816 | Test Accuracy: 0.816 | Training Loss: 0.405 | Test Loss 0.389 
Epoch 7, Step 1896 | Training Accuracy: 0.828 | Test Accuracy: 0.840 | Training Loss: 0.352 | Test Loss 0.388 
(*) Entering Epoch 8 (498.191 s)
Epoch 8, Step 237 | Training Accuracy: 0.832 | Test Accuracy: 0.848 | Training Loss: 0.400 | Test Loss 0.380 
Epoch 8, Step 474 | Training Accuracy: 0.840 | Test Accuracy: 0.840 | Training Loss: 0.353 | Test Loss 0.389 
Epoch 8, Step 711 | Training Accuracy: 0.840 | Test Accuracy: 0.836 | Training Loss: 0.350 | Test Loss 0.412 
Epoch 8, Step 948 | Training Accuracy: 0.844 | Test Accuracy: 0.824 | Training Loss: 0.364 | Test Loss 0.375 
Epoch 8, Step 1185 | Training Accuracy: 0.809 | Test Accuracy: 0.871 | Training Loss: 0.376 | Test Loss 0.342 
Epoch 8, Step 1422 | Training Accuracy: 0.836 | Test Accuracy: 0.832 | Training Loss: 0.354 | Test Loss 0.382 
Epoch 8, Step 1659 | Training Accuracy: 0.812 | Test Accuracy: 0.812 | Training Loss: 0.369 | Test Loss 0.414 
Epoch 8, Step 1896 | Training Accuracy: 0.832 | Test Accuracy: 0.844 | Training Loss: 0.391 | Test Loss 0.375 
(*) Entering Epoch 9 (558.694 s)
Epoch 9, Step 237 | Training Accuracy: 0.832 | Test Accuracy: 0.758 | Training Loss: 0.346 | Test Loss 0.491 
Epoch 9, Step 474 | Training Accuracy: 0.832 | Test Accuracy: 0.793 | Training Loss: 0.368 | Test Loss 0.407 
Epoch 9, Step 711 | Training Accuracy: 0.801 | Test Accuracy: 0.789 | Training Loss: 0.403 | Test Loss 0.417 
Epoch 9, Step 948 | Training Accuracy: 0.859 | Test Accuracy: 0.816 | Training Loss: 0.331 | Test Loss 0.395 
Epoch 9, Step 1185 | Training Accuracy: 0.863 | Test Accuracy: 0.820 | Training Loss: 0.339 | Test Loss 0.398 
Epoch 9, Step 1422 | Training Accuracy: 0.859 | Test Accuracy: 0.781 | Training Loss: 0.349 | Test Loss 0.430 
Epoch 9, Step 1659 | Training Accuracy: 0.859 | Test Accuracy: 0.848 | Training Loss: 0.332 | Test Loss 0.335 
Epoch 9, Step 1896 | Training Accuracy: 0.836 | Test Accuracy: 0.848 | Training Loss: 0.357 | Test Loss 0.360 
(*) Entering Epoch 10 (619.189 s)
Graph saved to file: checkpoints/rhn_kst_rho0_epoch10.ckpt-10
Epoch 10, Step 237 | Training Accuracy: 0.816 | Test Accuracy: 0.875 | Training Loss: 0.364 | Test Loss 0.302 
Epoch 10, Step 474 | Training Accuracy: 0.836 | Test Accuracy: 0.832 | Training Loss: 0.365 | Test Loss 0.420 
Epoch 10, Step 711 | Training Accuracy: 0.840 | Test Accuracy: 0.828 | Training Loss: 0.375 | Test Loss 0.407 
Epoch 10, Step 948 | Training Accuracy: 0.812 | Test Accuracy: 0.812 | Training Loss: 0.380 | Test Loss 0.420 
Epoch 10, Step 1185 | Training Accuracy: 0.824 | Test Accuracy: 0.816 | Training Loss: 0.372 | Test Loss 0.424 
Epoch 10, Step 1422 | Training Accuracy: 0.836 | Test Accuracy: 0.812 | Training Loss: 0.331 | Test Loss 0.399 
Epoch 10, Step 1659 | Training Accuracy: 0.805 | Test Accuracy: 0.809 | Training Loss: 0.382 | Test Loss 0.397 
Epoch 10, Step 1896 | Training Accuracy: 0.863 | Test Accuracy: 0.863 | Training Loss: 0.361 | Test Loss 0.326 
(*) Entering Epoch 11 (680.079 s)
Epoch 11, Step 237 | Training Accuracy: 0.777 | Test Accuracy: 0.852 | Training Loss: 0.410 | Test Loss 0.348 
Epoch 11, Step 474 | Training Accuracy: 0.820 | Test Accuracy: 0.820 | Training Loss: 0.381 | Test Loss 0.390 
Epoch 11, Step 711 | Training Accuracy: 0.836 | Test Accuracy: 0.828 | Training Loss: 0.355 | Test Loss 0.382 
Epoch 11, Step 948 | Training Accuracy: 0.824 | Test Accuracy: 0.816 | Training Loss: 0.352 | Test Loss 0.395 
Epoch 11, Step 1185 | Training Accuracy: 0.840 | Test Accuracy: 0.840 | Training Loss: 0.341 | Test Loss 0.363 
Epoch 11, Step 1422 | Training Accuracy: 0.844 | Test Accuracy: 0.828 | Training Loss: 0.347 | Test Loss 0.395 
Epoch 11, Step 1659 | Training Accuracy: 0.859 | Test Accuracy: 0.797 | Training Loss: 0.336 | Test Loss 0.398 
Epoch 11, Step 1896 | Training Accuracy: 0.855 | Test Accuracy: 0.852 | Training Loss: 0.360 | Test Loss 0.388 
(*) Entering Epoch 12 (740.595 s)
Epoch 12, Step 237 | Training Accuracy: 0.844 | Test Accuracy: 0.844 | Training Loss: 0.364 | Test Loss 0.384 
Epoch 12, Step 474 | Training Accuracy: 0.863 | Test Accuracy: 0.832 | Training Loss: 0.304 | Test Loss 0.356 
Epoch 12, Step 711 | Training Accuracy: 0.812 | Test Accuracy: 0.816 | Training Loss: 0.361 | Test Loss 0.394 
Epoch 12, Step 948 | Training Accuracy: 0.852 | Test Accuracy: 0.871 | Training Loss: 0.335 | Test Loss 0.334 
Epoch 12, Step 1185 | Training Accuracy: 0.809 | Test Accuracy: 0.848 | Training Loss: 0.375 | Test Loss 0.336 
Epoch 12, Step 1422 | Training Accuracy: 0.812 | Test Accuracy: 0.816 | Training Loss: 0.369 | Test Loss 0.384 
Epoch 12, Step 1659 | Training Accuracy: 0.812 | Test Accuracy: 0.824 | Training Loss: 0.378 | Test Loss 0.405 
Epoch 12, Step 1896 | Training Accuracy: 0.809 | Test Accuracy: 0.812 | Training Loss: 0.411 | Test Loss 0.405 
(*) Entering Epoch 13 (801.146 s)
Epoch 13, Step 237 | Training Accuracy: 0.844 | Test Accuracy: 0.852 | Training Loss: 0.341 | Test Loss 0.379 
Epoch 13, Step 474 | Training Accuracy: 0.820 | Test Accuracy: 0.797 | Training Loss: 0.355 | Test Loss 0.421 
Epoch 13, Step 711 | Training Accuracy: 0.867 | Test Accuracy: 0.871 | Training Loss: 0.275 | Test Loss 0.333 
Epoch 13, Step 948 | Training Accuracy: 0.824 | Test Accuracy: 0.836 | Training Loss: 0.390 | Test Loss 0.387 
Epoch 13, Step 1185 | Training Accuracy: 0.859 | Test Accuracy: 0.848 | Training Loss: 0.323 | Test Loss 0.356 
Epoch 13, Step 1422 | Training Accuracy: 0.840 | Test Accuracy: 0.840 | Training Loss: 0.389 | Test Loss 0.401 
Epoch 13, Step 1659 | Training Accuracy: 0.824 | Test Accuracy: 0.859 | Training Loss: 0.404 | Test Loss 0.352 
Epoch 13, Step 1896 | Training Accuracy: 0.809 | Test Accuracy: 0.855 | Training Loss: 0.395 | Test Loss 0.366 
(*) Entering Epoch 14 (861.699 s)
Epoch 14, Step 237 | Training Accuracy: 0.836 | Test Accuracy: 0.828 | Training Loss: 0.375 | Test Loss 0.371 
Epoch 14, Step 474 | Training Accuracy: 0.852 | Test Accuracy: 0.812 | Training Loss: 0.364 | Test Loss 0.393 
Epoch 14, Step 711 | Training Accuracy: 0.766 | Test Accuracy: 0.859 | Training Loss: 0.439 | Test Loss 0.307 
Epoch 14, Step 948 | Training Accuracy: 0.812 | Test Accuracy: 0.836 | Training Loss: 0.422 | Test Loss 0.409 
Epoch 14, Step 1185 | Training Accuracy: 0.820 | Test Accuracy: 0.844 | Training Loss: 0.378 | Test Loss 0.377 
Epoch 14, Step 1422 | Training Accuracy: 0.848 | Test Accuracy: 0.852 | Training Loss: 0.356 | Test Loss 0.333 
Epoch 14, Step 1659 | Training Accuracy: 0.812 | Test Accuracy: 0.832 | Training Loss: 0.386 | Test Loss 0.376 
Epoch 14, Step 1896 | Training Accuracy: 0.840 | Test Accuracy: 0.848 | Training Loss: 0.375 | Test Loss 0.388 
(*) Entering Epoch 15 (922.248 s)
Epoch 15, Step 237 | Training Accuracy: 0.820 | Test Accuracy: 0.848 | Training Loss: 0.381 | Test Loss 0.384 
Epoch 15, Step 474 | Training Accuracy: 0.867 | Test Accuracy: 0.832 | Training Loss: 0.334 | Test Loss 0.370 
Epoch 15, Step 711 | Training Accuracy: 0.824 | Test Accuracy: 0.797 | Training Loss: 0.366 | Test Loss 0.391 
Epoch 15, Step 948 | Training Accuracy: 0.891 | Test Accuracy: 0.824 | Training Loss: 0.305 | Test Loss 0.403 
Epoch 15, Step 1185 | Training Accuracy: 0.797 | Test Accuracy: 0.816 | Training Loss: 0.437 | Test Loss 0.376 
Epoch 15, Step 1422 | Training Accuracy: 0.836 | Test Accuracy: 0.824 | Training Loss: 0.399 | Test Loss 0.351 
Epoch 15, Step 1659 | Training Accuracy: 0.832 | Test Accuracy: 0.840 | Training Loss: 0.366 | Test Loss 0.363 
Epoch 15, Step 1896 | Training Accuracy: 0.828 | Test Accuracy: 0.809 | Training Loss: 0.392 | Test Loss 0.385 
(*) Entering Epoch 16 (982.810 s)
Epoch 16, Step 237 | Training Accuracy: 0.832 | Test Accuracy: 0.844 | Training Loss: 0.374 | Test Loss 0.325 
Epoch 16, Step 474 | Training Accuracy: 0.852 | Test Accuracy: 0.836 | Training Loss: 0.397 | Test Loss 0.356 
Epoch 16, Step 711 | Training Accuracy: 0.797 | Test Accuracy: 0.844 | Training Loss: 0.402 | Test Loss 0.323 
Epoch 16, Step 948 | Training Accuracy: 0.844 | Test Accuracy: 0.836 | Training Loss: 0.369 | Test Loss 0.374 
Epoch 16, Step 1185 | Training Accuracy: 0.883 | Test Accuracy: 0.812 | Training Loss: 0.287 | Test Loss 0.385 
Epoch 16, Step 1422 | Training Accuracy: 0.871 | Test Accuracy: 0.809 | Training Loss: 0.303 | Test Loss 0.424 
Epoch 16, Step 1659 | Training Accuracy: 0.828 | Test Accuracy: 0.863 | Training Loss: 0.365 | Test Loss 0.338 
Epoch 16, Step 1896 | Training Accuracy: 0.855 | Test Accuracy: 0.848 | Training Loss: 0.326 | Test Loss 0.353 
(*) Entering Epoch 17 (1043.375 s)
Epoch 17, Step 237 | Training Accuracy: 0.848 | Test Accuracy: 0.816 | Training Loss: 0.375 | Test Loss 0.389 
Epoch 17, Step 474 | Training Accuracy: 0.809 | Test Accuracy: 0.801 | Training Loss: 0.365 | Test Loss 0.383 
Epoch 17, Step 711 | Training Accuracy: 0.801 | Test Accuracy: 0.859 | Training Loss: 0.425 | Test Loss 0.345 
Epoch 17, Step 948 | Training Accuracy: 0.844 | Test Accuracy: 0.844 | Training Loss: 0.350 | Test Loss 0.358 
Epoch 17, Step 1185 | Training Accuracy: 0.855 | Test Accuracy: 0.801 | Training Loss: 0.322 | Test Loss 0.384 
Epoch 17, Step 1422 | Training Accuracy: 0.852 | Test Accuracy: 0.844 | Training Loss: 0.350 | Test Loss 0.393 
Epoch 17, Step 1659 | Training Accuracy: 0.852 | Test Accuracy: 0.852 | Training Loss: 0.346 | Test Loss 0.341 
Epoch 17, Step 1896 | Training Accuracy: 0.852 | Test Accuracy: 0.816 | Training Loss: 0.346 | Test Loss 0.381 
(*) Entering Epoch 18 (1103.946 s)
Epoch 18, Step 237 | Training Accuracy: 0.875 | Test Accuracy: 0.793 | Training Loss: 0.322 | Test Loss 0.431 
Epoch 18, Step 474 | Training Accuracy: 0.840 | Test Accuracy: 0.867 | Training Loss: 0.378 | Test Loss 0.335 
Epoch 18, Step 711 | Training Accuracy: 0.789 | Test Accuracy: 0.812 | Training Loss: 0.392 | Test Loss 0.390 
Epoch 18, Step 948 | Training Accuracy: 0.789 | Test Accuracy: 0.855 | Training Loss: 0.416 | Test Loss 0.362 
Epoch 18, Step 1185 | Training Accuracy: 0.852 | Test Accuracy: 0.824 | Training Loss: 0.342 | Test Loss 0.397 
Epoch 18, Step 1422 | Training Accuracy: 0.840 | Test Accuracy: 0.793 | Training Loss: 0.361 | Test Loss 0.433 
Epoch 18, Step 1659 | Training Accuracy: 0.824 | Test Accuracy: 0.828 | Training Loss: 0.373 | Test Loss 0.381 
Epoch 18, Step 1896 | Training Accuracy: 0.852 | Test Accuracy: 0.828 | Training Loss: 0.328 | Test Loss 0.375 
(*) Entering Epoch 19 (1164.518 s)
Epoch 19, Step 237 | Training Accuracy: 0.832 | Test Accuracy: 0.836 | Training Loss: 0.364 | Test Loss 0.351 
Epoch 19, Step 474 | Training Accuracy: 0.676 | Test Accuracy: 0.699 | Training Loss: 0.596 | Test Loss 0.573 
Epoch 19, Step 711 | Training Accuracy: 0.738 | Test Accuracy: 0.742 | Training Loss: 0.572 | Test Loss 0.520 
Epoch 19, Step 948 | Training Accuracy: 0.758 | Test Accuracy: 0.754 | Training Loss: 0.483 | Test Loss 0.502 
Epoch 19, Step 1185 | Training Accuracy: 0.801 | Test Accuracy: 0.789 | Training Loss: 0.448 | Test Loss 0.442 
Epoch 19, Step 1422 | Training Accuracy: 0.832 | Test Accuracy: 0.801 | Training Loss: 0.414 | Test Loss 0.453 
Epoch 19, Step 1659 | Training Accuracy: 0.785 | Test Accuracy: 0.793 | Training Loss: 0.487 | Test Loss 0.455 
Epoch 19, Step 1896 | Training Accuracy: 0.770 | Test Accuracy: 0.805 | Training Loss: 0.463 | Test Loss 0.450 
(*) Entering Epoch 20 (1225.049 s)
Graph saved to file: checkpoints/rhn_kst_rho0_epoch20.ckpt-20
Epoch 20, Step 237 | Training Accuracy: 0.812 | Test Accuracy: 0.762 | Training Loss: 0.405 | Test Loss 0.539 
Epoch 20, Step 474 | Training Accuracy: 0.801 | Test Accuracy: 0.801 | Training Loss: 0.419 | Test Loss 0.436 
Epoch 20, Step 711 | Training Accuracy: 0.777 | Test Accuracy: 0.816 | Training Loss: 0.444 | Test Loss 0.423 
Epoch 20, Step 948 | Training Accuracy: 0.816 | Test Accuracy: 0.852 | Training Loss: 0.412 | Test Loss 0.356 
Epoch 20, Step 1185 | Training Accuracy: 0.836 | Test Accuracy: 0.832 | Training Loss: 0.358 | Test Loss 0.415 
Epoch 20, Step 1422 | Training Accuracy: 0.840 | Test Accuracy: 0.781 | Training Loss: 0.367 | Test Loss 0.454 
Epoch 20, Step 1659 | Training Accuracy: 0.852 | Test Accuracy: 0.836 | Training Loss: 0.338 | Test Loss 0.388 
Epoch 20, Step 1896 | Training Accuracy: 0.824 | Test Accuracy: 0.789 | Training Loss: 0.370 | Test Loss 0.440 
(*) Entering Epoch 21 (1286.102 s)
Epoch 21, Step 237 | Training Accuracy: 0.816 | Test Accuracy: 0.754 | Training Loss: 0.417 | Test Loss 0.488 
Epoch 21, Step 474 | Training Accuracy: 0.848 | Test Accuracy: 0.816 | Training Loss: 0.338 | Test Loss 0.404 
Epoch 21, Step 711 | Training Accuracy: 0.840 | Test Accuracy: 0.828 | Training Loss: 0.376 | Test Loss 0.409 
Epoch 21, Step 948 | Training Accuracy: 0.812 | Test Accuracy: 0.805 | Training Loss: 0.412 | Test Loss 0.434 
Epoch 21, Step 1185 | Training Accuracy: 0.836 | Test Accuracy: 0.812 | Training Loss: 0.353 | Test Loss 0.415 
Epoch 21, Step 1422 | Training Accuracy: 0.859 | Test Accuracy: 0.801 | Training Loss: 0.347 | Test Loss 0.441 
Epoch 21, Step 1659 | Training Accuracy: 0.832 | Test Accuracy: 0.859 | Training Loss: 0.357 | Test Loss 0.329 
Epoch 21, Step 1896 | Training Accuracy: 0.844 | Test Accuracy: 0.844 | Training Loss: 0.336 | Test Loss 0.354 
(*) Entering Epoch 22 (1346.653 s)
Epoch 22, Step 237 | Training Accuracy: 0.805 | Test Accuracy: 0.852 | Training Loss: 0.370 | Test Loss 0.352 
Epoch 22, Step 474 | Training Accuracy: 0.820 | Test Accuracy: 0.801 | Training Loss: 0.381 | Test Loss 0.387 
Epoch 22, Step 711 | Training Accuracy: 0.840 | Test Accuracy: 0.785 | Training Loss: 0.342 | Test Loss 0.425 
Epoch 22, Step 948 | Training Accuracy: 0.805 | Test Accuracy: 0.785 | Training Loss: 0.412 | Test Loss 0.476 
Epoch 22, Step 1185 | Training Accuracy: 0.812 | Test Accuracy: 0.836 | Training Loss: 0.397 | Test Loss 0.384 
Epoch 22, Step 1422 | Training Accuracy: 0.801 | Test Accuracy: 0.848 | Training Loss: 0.389 | Test Loss 0.346 
Epoch 22, Step 1659 | Training Accuracy: 0.828 | Test Accuracy: 0.824 | Training Loss: 0.404 | Test Loss 0.396 
Epoch 22, Step 1896 | Training Accuracy: 0.809 | Test Accuracy: 0.766 | Training Loss: 0.422 | Test Loss 0.464 
(*) Entering Epoch 23 (1407.214 s)
Epoch 23, Step 237 | Training Accuracy: 0.812 | Test Accuracy: 0.840 | Training Loss: 0.399 | Test Loss 0.336 
Epoch 23, Step 474 | Training Accuracy: 0.883 | Test Accuracy: 0.828 | Training Loss: 0.337 | Test Loss 0.414 
Epoch 23, Step 711 | Training Accuracy: 0.836 | Test Accuracy: 0.848 | Training Loss: 0.396 | Test Loss 0.363 
Epoch 23, Step 948 | Training Accuracy: 0.824 | Test Accuracy: 0.801 | Training Loss: 0.430 | Test Loss 0.393 
Epoch 23, Step 1185 | Training Accuracy: 0.871 | Test Accuracy: 0.781 | Training Loss: 0.300 | Test Loss 0.424 
Epoch 23, Step 1422 | Training Accuracy: 0.809 | Test Accuracy: 0.797 | Training Loss: 0.411 | Test Loss 0.438 
Epoch 23, Step 1659 | Training Accuracy: 0.832 | Test Accuracy: 0.801 | Training Loss: 0.389 | Test Loss 0.395 
Epoch 23, Step 1896 | Training Accuracy: 0.828 | Test Accuracy: 0.781 | Training Loss: 0.365 | Test Loss 0.457 
(*) Entering Epoch 24 (1467.792 s)
Epoch 24, Step 237 | Training Accuracy: 0.820 | Test Accuracy: 0.852 | Training Loss: 0.368 | Test Loss 0.368 
Epoch 24, Step 474 | Training Accuracy: 0.820 | Test Accuracy: 0.852 | Training Loss: 0.409 | Test Loss 0.329 
Epoch 24, Step 711 | Training Accuracy: 0.828 | Test Accuracy: 0.805 | Training Loss: 0.428 | Test Loss 0.423 
Epoch 24, Step 948 | Training Accuracy: 0.824 | Test Accuracy: 0.816 | Training Loss: 0.389 | Test Loss 0.406 
Epoch 24, Step 1185 | Training Accuracy: 0.824 | Test Accuracy: 0.770 | Training Loss: 0.364 | Test Loss 0.449 
Epoch 24, Step 1422 | Training Accuracy: 0.828 | Test Accuracy: 0.809 | Training Loss: 0.373 | Test Loss 0.428 
Epoch 24, Step 1659 | Training Accuracy: 0.750 | Test Accuracy: 0.848 | Training Loss: 0.462 | Test Loss 0.331 
Epoch 24, Step 1896 | Training Accuracy: 0.812 | Test Accuracy: 0.816 | Training Loss: 0.404 | Test Loss 0.398 
(*) Entering Epoch 25 (1528.363 s)
Epoch 25, Step 237 | Training Accuracy: 0.820 | Test Accuracy: 0.793 | Training Loss: 0.397 | Test Loss 0.415 
Epoch 25, Step 474 | Training Accuracy: 0.867 | Test Accuracy: 0.832 | Training Loss: 0.340 | Test Loss 0.375 
Epoch 25, Step 711 | Training Accuracy: 0.754 | Test Accuracy: 0.824 | Training Loss: 0.523 | Test Loss 0.381 
Epoch 25, Step 948 | Training Accuracy: 0.828 | Test Accuracy: 0.816 | Training Loss: 0.410 | Test Loss 0.396 
Epoch 25, Step 1185 | Training Accuracy: 0.809 | Test Accuracy: 0.855 | Training Loss: 0.397 | Test Loss 0.350 
Epoch 25, Step 1422 | Training Accuracy: 0.859 | Test Accuracy: 0.840 | Training Loss: 0.333 | Test Loss 0.374 
Epoch 25, Step 1659 | Training Accuracy: 0.809 | Test Accuracy: 0.812 | Training Loss: 0.387 | Test Loss 0.338 
Epoch 25, Step 1896 | Training Accuracy: 0.848 | Test Accuracy: 0.797 | Training Loss: 0.343 | Test Loss 0.414 
(*) Entering Epoch 26 (1588.896 s)
Epoch 26, Step 237 | Training Accuracy: 0.809 | Test Accuracy: 0.820 | Training Loss: 0.371 | Test Loss 0.381 
Epoch 26, Step 474 | Training Accuracy: 0.859 | Test Accuracy: 0.816 | Training Loss: 0.382 | Test Loss 0.380 
Epoch 26, Step 711 | Training Accuracy: 0.824 | Test Accuracy: 0.777 | Training Loss: 0.357 | Test Loss 0.421 
Epoch 26, Step 948 | Training Accuracy: 0.797 | Test Accuracy: 0.832 | Training Loss: 0.402 | Test Loss 0.385 
Epoch 26, Step 1185 | Training Accuracy: 0.824 | Test Accuracy: 0.812 | Training Loss: 0.405 | Test Loss 0.384 
Epoch 26, Step 1422 | Training Accuracy: 0.824 | Test Accuracy: 0.805 | Training Loss: 0.369 | Test Loss 0.425 
Epoch 26, Step 1659 | Training Accuracy: 0.836 | Test Accuracy: 0.836 | Training Loss: 0.354 | Test Loss 0.373 
Epoch 26, Step 1896 | Training Accuracy: 0.820 | Test Accuracy: 0.809 | Training Loss: 0.371 | Test Loss 0.452 
(*) Entering Epoch 27 (1649.437 s)
Epoch 27, Step 237 | Training Accuracy: 0.836 | Test Accuracy: 0.809 | Training Loss: 0.373 | Test Loss 0.404 
Epoch 27, Step 474 | Training Accuracy: 0.848 | Test Accuracy: 0.855 | Training Loss: 0.368 | Test Loss 0.355 
Epoch 27, Step 711 | Training Accuracy: 0.824 | Test Accuracy: 0.816 | Training Loss: 0.403 | Test Loss 0.418 
Epoch 27, Step 948 | Training Accuracy: 0.836 | Test Accuracy: 0.793 | Training Loss: 0.374 | Test Loss 0.430 
Epoch 27, Step 1185 | Training Accuracy: 0.871 | Test Accuracy: 0.848 | Training Loss: 0.324 | Test Loss 0.344 
Epoch 27, Step 1422 | Training Accuracy: 0.871 | Test Accuracy: 0.828 | Training Loss: 0.329 | Test Loss 0.436 
Epoch 27, Step 1659 | Training Accuracy: 0.812 | Test Accuracy: 0.816 | Training Loss: 0.419 | Test Loss 0.398 
Epoch 27, Step 1896 | Training Accuracy: 0.852 | Test Accuracy: 0.848 | Training Loss: 0.345 | Test Loss 0.393 
(*) Entering Epoch 28 (1709.968 s)
Epoch 28, Step 237 | Training Accuracy: 0.832 | Test Accuracy: 0.812 | Training Loss: 0.375 | Test Loss 0.378 
Epoch 28, Step 474 | Training Accuracy: 0.785 | Test Accuracy: 0.844 | Training Loss: 0.438 | Test Loss 0.380 
Epoch 28, Step 711 | Training Accuracy: 0.820 | Test Accuracy: 0.801 | Training Loss: 0.376 | Test Loss 0.376 
Epoch 28, Step 948 | Training Accuracy: 0.785 | Test Accuracy: 0.828 | Training Loss: 0.415 | Test Loss 0.388 
Epoch 28, Step 1185 | Training Accuracy: 0.859 | Test Accuracy: 0.844 | Training Loss: 0.330 | Test Loss 0.352 
Epoch 28, Step 1422 | Training Accuracy: 0.824 | Test Accuracy: 0.797 | Training Loss: 0.355 | Test Loss 0.459 
Epoch 28, Step 1659 | Training Accuracy: 0.824 | Test Accuracy: 0.855 | Training Loss: 0.379 | Test Loss 0.333 
Epoch 28, Step 1896 | Training Accuracy: 0.824 | Test Accuracy: 0.840 | Training Loss: 0.400 | Test Loss 0.370 
(*) Entering Epoch 29 (1770.573 s)
Epoch 29, Step 237 | Training Accuracy: 0.852 | Test Accuracy: 0.848 | Training Loss: 0.335 | Test Loss 0.317 
Epoch 29, Step 474 | Training Accuracy: 0.828 | Test Accuracy: 0.805 | Training Loss: 0.406 | Test Loss 0.421 
Epoch 29, Step 711 | Training Accuracy: 0.812 | Test Accuracy: 0.852 | Training Loss: 0.390 | Test Loss 0.370 
Epoch 29, Step 948 | Training Accuracy: 0.820 | Test Accuracy: 0.852 | Training Loss: 0.392 | Test Loss 0.392 
Epoch 29, Step 1185 | Training Accuracy: 0.809 | Test Accuracy: 0.832 | Training Loss: 0.388 | Test Loss 0.382 
Epoch 29, Step 1422 | Training Accuracy: 0.852 | Test Accuracy: 0.797 | Training Loss: 0.348 | Test Loss 0.448 
Epoch 29, Step 1659 | Training Accuracy: 0.848 | Test Accuracy: 0.832 | Training Loss: 0.382 | Test Loss 0.361 
Epoch 29, Step 1896 | Training Accuracy: 0.836 | Test Accuracy: 0.848 | Training Loss: 0.356 | Test Loss 0.345 
(*) Entering Epoch 30 (1831.175 s)
Graph saved to file: checkpoints/rhn_kst_rho0_epoch30.ckpt-30
Epoch 30, Step 237 | Training Accuracy: 0.848 | Test Accuracy: 0.832 | Training Loss: 0.365 | Test Loss 0.375 
Epoch 30, Step 474 | Training Accuracy: 0.812 | Test Accuracy: 0.840 | Training Loss: 0.394 | Test Loss 0.361 
Epoch 30, Step 711 | Training Accuracy: 0.891 | Test Accuracy: 0.832 | Training Loss: 0.324 | Test Loss 0.385 
Epoch 30, Step 948 | Training Accuracy: 0.797 | Test Accuracy: 0.859 | Training Loss: 0.399 | Test Loss 0.349 
Epoch 30, Step 1185 | Training Accuracy: 0.824 | Test Accuracy: 0.816 | Training Loss: 0.382 | Test Loss 0.397 
Epoch 30, Step 1422 | Training Accuracy: 0.844 | Test Accuracy: 0.848 | Training Loss: 0.367 | Test Loss 0.337 
Epoch 30, Step 1659 | Training Accuracy: 0.797 | Test Accuracy: 0.812 | Training Loss: 0.385 | Test Loss 0.409 
Epoch 30, Step 1896 | Training Accuracy: 0.828 | Test Accuracy: 0.801 | Training Loss: 0.375 | Test Loss 0.390 
(*) Entering Epoch 31 (1892.220 s)
Epoch 31, Step 237 | Training Accuracy: 0.844 | Test Accuracy: 0.805 | Training Loss: 0.403 | Test Loss 0.425 
Epoch 31, Step 474 | Training Accuracy: 0.812 | Test Accuracy: 0.836 | Training Loss: 0.385 | Test Loss 0.367 
Epoch 31, Step 711 | Training Accuracy: 0.836 | Test Accuracy: 0.773 | Training Loss: 0.340 | Test Loss 0.439 
Epoch 31, Step 948 | Training Accuracy: 0.812 | Test Accuracy: 0.828 | Training Loss: 0.363 | Test Loss 0.415 
Epoch 31, Step 1185 | Training Accuracy: 0.863 | Test Accuracy: 0.832 | Training Loss: 0.315 | Test Loss 0.390 
Epoch 31, Step 1422 | Training Accuracy: 0.840 | Test Accuracy: 0.809 | Training Loss: 0.380 | Test Loss 0.393 
Epoch 31, Step 1659 | Training Accuracy: 0.809 | Test Accuracy: 0.828 | Training Loss: 0.406 | Test Loss 0.394 
Epoch 31, Step 1896 | Training Accuracy: 0.816 | Test Accuracy: 0.812 | Training Loss: 0.408 | Test Loss 0.385 
(*) Entering Epoch 32 (1952.735 s)
Epoch 32, Step 237 | Training Accuracy: 0.812 | Test Accuracy: 0.801 | Training Loss: 0.386 | Test Loss 0.406 
Epoch 32, Step 474 | Training Accuracy: 0.844 | Test Accuracy: 0.844 | Training Loss: 0.326 | Test Loss 0.344 
Epoch 32, Step 711 | Training Accuracy: 0.840 | Test Accuracy: 0.820 | Training Loss: 0.322 | Test Loss 0.388 
Epoch 32, Step 948 | Training Accuracy: 0.863 | Test Accuracy: 0.809 | Training Loss: 0.334 | Test Loss 0.428 
Epoch 32, Step 1185 | Training Accuracy: 0.820 | Test Accuracy: 0.863 | Training Loss: 0.366 | Test Loss 0.337 
Epoch 32, Step 1422 | Training Accuracy: 0.836 | Test Accuracy: 0.855 | Training Loss: 0.347 | Test Loss 0.371 
Epoch 32, Step 1659 | Training Accuracy: 0.840 | Test Accuracy: 0.879 | Training Loss: 0.431 | Test Loss 0.312 *
Epoch 32, Step 1896 | Training Accuracy: 0.793 | Test Accuracy: 0.859 | Training Loss: 0.437 | Test Loss 0.340 
(*) Entering Epoch 33 (2013.958 s)
Epoch 33, Step 237 | Training Accuracy: 0.832 | Test Accuracy: 0.832 | Training Loss: 0.380 | Test Loss 0.362 
Epoch 33, Step 474 | Training Accuracy: 0.848 | Test Accuracy: 0.797 | Training Loss: 0.346 | Test Loss 0.375 
Epoch 33, Step 711 | Training Accuracy: 0.828 | Test Accuracy: 0.832 | Training Loss: 0.377 | Test Loss 0.361 
Epoch 33, Step 948 | Training Accuracy: 0.812 | Test Accuracy: 0.840 | Training Loss: 0.403 | Test Loss 0.338 
Epoch 33, Step 1185 | Training Accuracy: 0.844 | Test Accuracy: 0.828 | Training Loss: 0.357 | Test Loss 0.395 
Epoch 33, Step 1422 | Training Accuracy: 0.836 | Test Accuracy: 0.824 | Training Loss: 0.385 | Test Loss 0.357 
Epoch 33, Step 1659 | Training Accuracy: 0.859 | Test Accuracy: 0.816 | Training Loss: 0.341 | Test Loss 0.377 
Epoch 33, Step 1896 | Training Accuracy: 0.859 | Test Accuracy: 0.816 | Training Loss: 0.342 | Test Loss 0.381 
(*) Entering Epoch 34 (2074.534 s)
Epoch 34, Step 237 | Training Accuracy: 0.809 | Test Accuracy: 0.812 | Training Loss: 0.400 | Test Loss 0.377 
Epoch 34, Step 474 | Training Accuracy: 0.816 | Test Accuracy: 0.852 | Training Loss: 0.372 | Test Loss 0.347 
Epoch 34, Step 711 | Training Accuracy: 0.805 | Test Accuracy: 0.824 | Training Loss: 0.403 | Test Loss 0.384 
Epoch 34, Step 948 | Training Accuracy: 0.820 | Test Accuracy: 0.848 | Training Loss: 0.400 | Test Loss 0.368 
Epoch 34, Step 1185 | Training Accuracy: 0.816 | Test Accuracy: 0.828 | Training Loss: 0.388 | Test Loss 0.385 
Epoch 34, Step 1422 | Training Accuracy: 0.855 | Test Accuracy: 0.801 | Training Loss: 0.347 | Test Loss 0.395 
Epoch 34, Step 1659 | Training Accuracy: 0.797 | Test Accuracy: 0.832 | Training Loss: 0.411 | Test Loss 0.352 
Epoch 34, Step 1896 | Training Accuracy: 0.805 | Test Accuracy: 0.855 | Training Loss: 0.406 | Test Loss 0.355 
(*) Entering Epoch 35 (2135.087 s)
Epoch 35, Step 237 | Training Accuracy: 0.812 | Test Accuracy: 0.801 | Training Loss: 0.381 | Test Loss 0.420 
Epoch 35, Step 474 | Training Accuracy: 0.832 | Test Accuracy: 0.797 | Training Loss: 0.358 | Test Loss 0.417 
Epoch 35, Step 711 | Training Accuracy: 0.844 | Test Accuracy: 0.789 | Training Loss: 0.398 | Test Loss 0.445 
Epoch 35, Step 948 | Training Accuracy: 0.812 | Test Accuracy: 0.824 | Training Loss: 0.363 | Test Loss 0.407 
Epoch 35, Step 1185 | Training Accuracy: 0.797 | Test Accuracy: 0.863 | Training Loss: 0.412 | Test Loss 0.321 
Epoch 35, Step 1422 | Training Accuracy: 0.871 | Test Accuracy: 0.797 | Training Loss: 0.330 | Test Loss 0.442 
Epoch 35, Step 1659 | Training Accuracy: 0.812 | Test Accuracy: 0.809 | Training Loss: 0.392 | Test Loss 0.422 
Epoch 35, Step 1896 | Training Accuracy: 0.824 | Test Accuracy: 0.828 | Training Loss: 0.387 | Test Loss 0.450 
(*) Entering Epoch 36 (2195.635 s)
Epoch 36, Step 237 | Training Accuracy: 0.805 | Test Accuracy: 0.824 | Training Loss: 0.396 | Test Loss 0.407 
Epoch 36, Step 474 | Training Accuracy: 0.793 | Test Accuracy: 0.777 | Training Loss: 0.433 | Test Loss 0.438 
Epoch 36, Step 711 | Training Accuracy: 0.844 | Test Accuracy: 0.844 | Training Loss: 0.361 | Test Loss 0.379 
Epoch 36, Step 948 | Training Accuracy: 0.852 | Test Accuracy: 0.805 | Training Loss: 0.347 | Test Loss 0.416 
Epoch 36, Step 1185 | Training Accuracy: 0.848 | Test Accuracy: 0.809 | Training Loss: 0.341 | Test Loss 0.393 
Epoch 36, Step 1422 | Training Accuracy: 0.820 | Test Accuracy: 0.801 | Training Loss: 0.384 | Test Loss 0.402 
Epoch 36, Step 1659 | Training Accuracy: 0.852 | Test Accuracy: 0.820 | Training Loss: 0.314 | Test Loss 0.371 
Epoch 36, Step 1896 | Training Accuracy: 0.828 | Test Accuracy: 0.809 | Training Loss: 0.367 | Test Loss 0.384 
(*) Entering Epoch 37 (2256.171 s)
Epoch 37, Step 237 | Training Accuracy: 0.848 | Test Accuracy: 0.809 | Training Loss: 0.362 | Test Loss 0.418 
Epoch 37, Step 474 | Training Accuracy: 0.852 | Test Accuracy: 0.832 | Training Loss: 0.310 | Test Loss 0.381 
Epoch 37, Step 711 | Training Accuracy: 0.770 | Test Accuracy: 0.848 | Training Loss: 0.437 | Test Loss 0.368 
Epoch 37, Step 948 | Training Accuracy: 0.824 | Test Accuracy: 0.781 | Training Loss: 0.400 | Test Loss 0.422 
Epoch 37, Step 1185 | Training Accuracy: 0.832 | Test Accuracy: 0.824 | Training Loss: 0.393 | Test Loss 0.364 
Epoch 37, Step 1422 | Training Accuracy: 0.848 | Test Accuracy: 0.781 | Training Loss: 0.366 | Test Loss 0.461 
Epoch 37, Step 1659 | Training Accuracy: 0.781 | Test Accuracy: 0.852 | Training Loss: 0.417 | Test Loss 0.372 
Epoch 37, Step 1896 | Training Accuracy: 0.859 | Test Accuracy: 0.840 | Training Loss: 0.357 | Test Loss 0.393 
(*) Entering Epoch 38 (2316.718 s)
Epoch 38, Step 237 | Training Accuracy: 0.828 | Test Accuracy: 0.805 | Training Loss: 0.397 | Test Loss 0.380 
Epoch 38, Step 474 | Training Accuracy: 0.824 | Test Accuracy: 0.844 | Training Loss: 0.374 | Test Loss 0.369 
Epoch 38, Step 711 | Training Accuracy: 0.848 | Test Accuracy: 0.852 | Training Loss: 0.385 | Test Loss 0.365 
Epoch 38, Step 948 | Training Accuracy: 0.863 | Test Accuracy: 0.840 | Training Loss: 0.351 | Test Loss 0.353 
Epoch 38, Step 1185 | Training Accuracy: 0.840 | Test Accuracy: 0.867 | Training Loss: 0.362 | Test Loss 0.306 
Epoch 38, Step 1422 | Training Accuracy: 0.852 | Test Accuracy: 0.809 | Training Loss: 0.370 | Test Loss 0.442 
Epoch 38, Step 1659 | Training Accuracy: 0.836 | Test Accuracy: 0.812 | Training Loss: 0.390 | Test Loss 0.391 
Epoch 38, Step 1896 | Training Accuracy: 0.820 | Test Accuracy: 0.840 | Training Loss: 0.383 | Test Loss 0.396 
(*) Entering Epoch 39 (2377.236 s)
Epoch 39, Step 237 | Training Accuracy: 0.863 | Test Accuracy: 0.832 | Training Loss: 0.319 | Test Loss 0.399 
Epoch 39, Step 474 | Training Accuracy: 0.801 | Test Accuracy: 0.781 | Training Loss: 0.371 | Test Loss 0.442 
Epoch 39, Step 711 | Training Accuracy: 0.859 | Test Accuracy: 0.840 | Training Loss: 0.352 | Test Loss 0.334 
Epoch 39, Step 948 | Training Accuracy: 0.844 | Test Accuracy: 0.805 | Training Loss: 0.381 | Test Loss 0.391 
Epoch 39, Step 1185 | Training Accuracy: 0.816 | Test Accuracy: 0.820 | Training Loss: 0.370 | Test Loss 0.402 
Epoch 39, Step 1422 | Training Accuracy: 0.805 | Test Accuracy: 0.828 | Training Loss: 0.412 | Test Loss 0.416 
Epoch 39, Step 1659 | Training Accuracy: 0.816 | Test Accuracy: 0.812 | Training Loss: 0.388 | Test Loss 0.385 
Epoch 39, Step 1896 | Training Accuracy: 0.828 | Test Accuracy: 0.820 | Training Loss: 0.391 | Test Loss 0.379 
(*) Entering Epoch 40 (2437.769 s)
Graph saved to file: checkpoints/rhn_kst_rho0_epoch40.ckpt-40
Epoch 40, Step 237 | Training Accuracy: 0.812 | Test Accuracy: 0.836 | Training Loss: 0.369 | Test Loss 0.390 
Epoch 40, Step 474 | Training Accuracy: 0.844 | Test Accuracy: 0.777 | Training Loss: 0.368 | Test Loss 0.465 
Epoch 40, Step 711 | Training Accuracy: 0.855 | Test Accuracy: 0.770 | Training Loss: 0.341 | Test Loss 0.435 
Epoch 40, Step 948 | Training Accuracy: 0.855 | Test Accuracy: 0.867 | Training Loss: 0.367 | Test Loss 0.355 
Epoch 40, Step 1185 | Training Accuracy: 0.785 | Test Accuracy: 0.809 | Training Loss: 0.420 | Test Loss 0.434 
Epoch 40, Step 1422 | Training Accuracy: 0.816 | Test Accuracy: 0.805 | Training Loss: 0.434 | Test Loss 0.375 
Epoch 40, Step 1659 | Training Accuracy: 0.836 | Test Accuracy: 0.785 | Training Loss: 0.348 | Test Loss 0.375 
Epoch 40, Step 1896 | Training Accuracy: 0.848 | Test Accuracy: 0.879 | Training Loss: 0.377 | Test Loss 0.313 
(*) Entering Epoch 41 (2498.833 s)
Epoch 41, Step 237 | Training Accuracy: 0.852 | Test Accuracy: 0.828 | Training Loss: 0.349 | Test Loss 0.362 
Epoch 41, Step 474 | Training Accuracy: 0.883 | Test Accuracy: 0.828 | Training Loss: 0.299 | Test Loss 0.386 
Epoch 41, Step 711 | Training Accuracy: 0.844 | Test Accuracy: 0.879 | Training Loss: 0.356 | Test Loss 0.332 
Epoch 41, Step 948 | Training Accuracy: 0.840 | Test Accuracy: 0.812 | Training Loss: 0.370 | Test Loss 0.395 
Epoch 41, Step 1185 | Training Accuracy: 0.848 | Test Accuracy: 0.855 | Training Loss: 0.378 | Test Loss 0.384 
Epoch 41, Step 1422 | Training Accuracy: 0.805 | Test Accuracy: 0.848 | Training Loss: 0.419 | Test Loss 0.327 
Epoch 41, Step 1659 | Training Accuracy: 0.855 | Test Accuracy: 0.875 | Training Loss: 0.341 | Test Loss 0.334 
Epoch 41, Step 1896 | Training Accuracy: 0.820 | Test Accuracy: 0.844 | Training Loss: 0.390 | Test Loss 0.366 
(*) Entering Epoch 42 (2559.348 s)
Epoch 42, Step 237 | Training Accuracy: 0.871 | Test Accuracy: 0.801 | Training Loss: 0.327 | Test Loss 0.420 
Epoch 42, Step 474 | Training Accuracy: 0.871 | Test Accuracy: 0.797 | Training Loss: 0.301 | Test Loss 0.409 
Epoch 42, Step 711 | Training Accuracy: 0.828 | Test Accuracy: 0.852 | Training Loss: 0.411 | Test Loss 0.385 
Epoch 42, Step 948 | Training Accuracy: 0.891 | Test Accuracy: 0.832 | Training Loss: 0.311 | Test Loss 0.357 
Epoch 42, Step 1185 | Training Accuracy: 0.844 | Test Accuracy: 0.812 | Training Loss: 0.339 | Test Loss 0.400 
Epoch 42, Step 1422 | Training Accuracy: 0.840 | Test Accuracy: 0.859 | Training Loss: 0.359 | Test Loss 0.336 
Epoch 42, Step 1659 | Training Accuracy: 0.840 | Test Accuracy: 0.805 | Training Loss: 0.347 | Test Loss 0.434 
Epoch 42, Step 1896 | Training Accuracy: 0.809 | Test Accuracy: 0.773 | Training Loss: 0.409 | Test Loss 0.474 
(*) Entering Epoch 43 (2619.878 s)
Epoch 43, Step 237 | Training Accuracy: 0.828 | Test Accuracy: 0.855 | Training Loss: 0.392 | Test Loss 0.362 
Epoch 43, Step 474 | Training Accuracy: 0.852 | Test Accuracy: 0.801 | Training Loss: 0.328 | Test Loss 0.411 
Epoch 43, Step 711 | Training Accuracy: 0.832 | Test Accuracy: 0.844 | Training Loss: 0.385 | Test Loss 0.348 
Epoch 43, Step 948 | Training Accuracy: 0.855 | Test Accuracy: 0.836 | Training Loss: 0.347 | Test Loss 0.356 
Epoch 43, Step 1185 | Training Accuracy: 0.828 | Test Accuracy: 0.863 | Training Loss: 0.358 | Test Loss 0.336 
Epoch 43, Step 1422 | Training Accuracy: 0.844 | Test Accuracy: 0.816 | Training Loss: 0.359 | Test Loss 0.399 
Epoch 43, Step 1659 | Training Accuracy: 0.848 | Test Accuracy: 0.793 | Training Loss: 0.334 | Test Loss 0.428 
Epoch 43, Step 1896 | Training Accuracy: 0.801 | Test Accuracy: 0.836 | Training Loss: 0.412 | Test Loss 0.377 
(*) Entering Epoch 44 (2680.385 s)
Epoch 44, Step 237 | Training Accuracy: 0.840 | Test Accuracy: 0.844 | Training Loss: 0.373 | Test Loss 0.380 
Epoch 44, Step 474 | Training Accuracy: 0.816 | Test Accuracy: 0.836 | Training Loss: 0.369 | Test Loss 0.364 
Epoch 44, Step 711 | Training Accuracy: 0.840 | Test Accuracy: 0.824 | Training Loss: 0.398 | Test Loss 0.400 
Epoch 44, Step 948 | Training Accuracy: 0.820 | Test Accuracy: 0.855 | Training Loss: 0.387 | Test Loss 0.318 
Epoch 44, Step 1185 | Training Accuracy: 0.828 | Test Accuracy: 0.777 | Training Loss: 0.381 | Test Loss 0.440 
Epoch 44, Step 1422 | Training Accuracy: 0.859 | Test Accuracy: 0.871 | Training Loss: 0.294 | Test Loss 0.316 
Epoch 44, Step 1659 | Training Accuracy: 0.824 | Test Accuracy: 0.820 | Training Loss: 0.392 | Test Loss 0.381 
Epoch 44, Step 1896 | Training Accuracy: 0.797 | Test Accuracy: 0.816 | Training Loss: 0.405 | Test Loss 0.415 
(*) Entering Epoch 45 (2740.949 s)
Epoch 45, Step 237 | Training Accuracy: 0.863 | Test Accuracy: 0.855 | Training Loss: 0.312 | Test Loss 0.352 
Epoch 45, Step 474 | Training Accuracy: 0.840 | Test Accuracy: 0.852 | Training Loss: 0.370 | Test Loss 0.376 
Epoch 45, Step 711 | Training Accuracy: 0.793 | Test Accuracy: 0.840 | Training Loss: 0.370 | Test Loss 0.329 
Epoch 45, Step 948 | Training Accuracy: 0.773 | Test Accuracy: 0.852 | Training Loss: 0.416 | Test Loss 0.345 
Epoch 45, Step 1185 | Training Accuracy: 0.820 | Test Accuracy: 0.820 | Training Loss: 0.383 | Test Loss 0.399 
Epoch 45, Step 1422 | Training Accuracy: 0.836 | Test Accuracy: 0.781 | Training Loss: 0.392 | Test Loss 0.439 
Epoch 45, Step 1659 | Training Accuracy: 0.840 | Test Accuracy: 0.809 | Training Loss: 0.358 | Test Loss 0.393 
Epoch 45, Step 1896 | Training Accuracy: 0.848 | Test Accuracy: 0.875 | Training Loss: 0.328 | Test Loss 0.309 
(*) Entering Epoch 46 (2801.460 s)
Epoch 46, Step 237 | Training Accuracy: 0.824 | Test Accuracy: 0.855 | Training Loss: 0.392 | Test Loss 0.384 
Epoch 46, Step 474 | Training Accuracy: 0.828 | Test Accuracy: 0.844 | Training Loss: 0.376 | Test Loss 0.397 
Epoch 46, Step 711 | Training Accuracy: 0.895 | Test Accuracy: 0.820 | Training Loss: 0.307 | Test Loss 0.384 
Epoch 46, Step 948 | Training Accuracy: 0.863 | Test Accuracy: 0.816 | Training Loss: 0.336 | Test Loss 0.344 
Epoch 46, Step 1185 | Training Accuracy: 0.883 | Test Accuracy: 0.809 | Training Loss: 0.304 | Test Loss 0.399 
Epoch 46, Step 1422 | Training Accuracy: 0.816 | Test Accuracy: 0.785 | Training Loss: 0.360 | Test Loss 0.399 
Epoch 46, Step 1659 | Training Accuracy: 0.840 | Test Accuracy: 0.824 | Training Loss: 0.343 | Test Loss 0.386 
Epoch 46, Step 1896 | Training Accuracy: 0.863 | Test Accuracy: 0.832 | Training Loss: 0.347 | Test Loss 0.366 
(*) Entering Epoch 47 (2862.012 s)
Epoch 47, Step 237 | Training Accuracy: 0.832 | Test Accuracy: 0.832 | Training Loss: 0.368 | Test Loss 0.376 
Epoch 47, Step 474 | Training Accuracy: 0.824 | Test Accuracy: 0.863 | Training Loss: 0.379 | Test Loss 0.317 
Epoch 47, Step 711 | Training Accuracy: 0.832 | Test Accuracy: 0.766 | Training Loss: 0.388 | Test Loss 0.510 
Epoch 47, Step 948 | Training Accuracy: 0.852 | Test Accuracy: 0.824 | Training Loss: 0.317 | Test Loss 0.385 
Epoch 47, Step 1185 | Training Accuracy: 0.871 | Test Accuracy: 0.832 | Training Loss: 0.326 | Test Loss 0.416 
Epoch 47, Step 1422 | Training Accuracy: 0.852 | Test Accuracy: 0.867 | Training Loss: 0.331 | Test Loss 0.370 
Epoch 47, Step 1659 | Training Accuracy: 0.844 | Test Accuracy: 0.836 | Training Loss: 0.323 | Test Loss 0.358 
Epoch 47, Step 1896 | Training Accuracy: 0.828 | Test Accuracy: 0.852 | Training Loss: 0.388 | Test Loss 0.367 
(*) Entering Epoch 48 (2922.557 s)
Epoch 48, Step 237 | Training Accuracy: 0.852 | Test Accuracy: 0.863 | Training Loss: 0.361 | Test Loss 0.364 
Epoch 48, Step 474 | Training Accuracy: 0.879 | Test Accuracy: 0.820 | Training Loss: 0.307 | Test Loss 0.357 
Epoch 48, Step 711 | Training Accuracy: 0.863 | Test Accuracy: 0.824 | Training Loss: 0.320 | Test Loss 0.402 
Epoch 48, Step 948 | Training Accuracy: 0.840 | Test Accuracy: 0.875 | Training Loss: 0.386 | Test Loss 0.313 
Epoch 48, Step 1185 | Training Accuracy: 0.812 | Test Accuracy: 0.836 | Training Loss: 0.379 | Test Loss 0.349 
Epoch 48, Step 1422 | Training Accuracy: 0.824 | Test Accuracy: 0.797 | Training Loss: 0.373 | Test Loss 0.424 
Epoch 48, Step 1659 | Training Accuracy: 0.812 | Test Accuracy: 0.789 | Training Loss: 0.392 | Test Loss 0.467 
Epoch 48, Step 1896 | Training Accuracy: 0.832 | Test Accuracy: 0.867 | Training Loss: 0.384 | Test Loss 0.354 
(*) Entering Epoch 49 (2983.112 s)
Epoch 49, Step 237 | Training Accuracy: 0.852 | Test Accuracy: 0.801 | Training Loss: 0.315 | Test Loss 0.393 
Epoch 49, Step 474 | Training Accuracy: 0.863 | Test Accuracy: 0.766 | Training Loss: 0.335 | Test Loss 0.449 
Epoch 49, Step 711 | Training Accuracy: 0.836 | Test Accuracy: 0.852 | Training Loss: 0.361 | Test Loss 0.383 
Epoch 49, Step 948 | Training Accuracy: 0.816 | Test Accuracy: 0.840 | Training Loss: 0.391 | Test Loss 0.381 
Epoch 49, Step 1185 | Training Accuracy: 0.828 | Test Accuracy: 0.852 | Training Loss: 0.374 | Test Loss 0.351 
Epoch 49, Step 1422 | Training Accuracy: 0.816 | Test Accuracy: 0.820 | Training Loss: 0.392 | Test Loss 0.399 
Epoch 49, Step 1659 | Training Accuracy: 0.832 | Test Accuracy: 0.844 | Training Loss: 0.387 | Test Loss 0.310 
Epoch 49, Step 1896 | Training Accuracy: 0.863 | Test Accuracy: 0.848 | Training Loss: 0.327 | Test Loss 0.342 
(*) Entering Epoch 50 (3043.675 s)
Graph saved to file: checkpoints/rhn_kst_rho0_epoch50.ckpt-50
Epoch 50, Step 237 | Training Accuracy: 0.820 | Test Accuracy: 0.840 | Training Loss: 0.393 | Test Loss 0.310 
Epoch 50, Step 474 | Training Accuracy: 0.812 | Test Accuracy: 0.785 | Training Loss: 0.406 | Test Loss 0.430 
Epoch 50, Step 711 | Training Accuracy: 0.836 | Test Accuracy: 0.848 | Training Loss: 0.366 | Test Loss 0.371 
Epoch 50, Step 948 | Training Accuracy: 0.820 | Test Accuracy: 0.801 | Training Loss: 0.353 | Test Loss 0.401 
Epoch 50, Step 1185 | Training Accuracy: 0.816 | Test Accuracy: 0.836 | Training Loss: 0.394 | Test Loss 0.363 
Epoch 50, Step 1422 | Training Accuracy: 0.836 | Test Accuracy: 0.824 | Training Loss: 0.340 | Test Loss 0.431 
Epoch 50, Step 1659 | Training Accuracy: 0.816 | Test Accuracy: 0.824 | Training Loss: 0.417 | Test Loss 0.437 
Epoch 50, Step 1896 | Training Accuracy: 0.832 | Test Accuracy: 0.801 | Training Loss: 0.396 | Test Loss 0.412 
(*) Entering Epoch 51 (3104.729 s)
Epoch 51, Step 237 | Training Accuracy: 0.883 | Test Accuracy: 0.812 | Training Loss: 0.315 | Test Loss 0.399 
Epoch 51, Step 474 | Training Accuracy: 0.863 | Test Accuracy: 0.820 | Training Loss: 0.322 | Test Loss 0.448 
Epoch 51, Step 711 | Training Accuracy: 0.832 | Test Accuracy: 0.824 | Training Loss: 0.365 | Test Loss 0.380 
Epoch 51, Step 948 | Training Accuracy: 0.836 | Test Accuracy: 0.824 | Training Loss: 0.373 | Test Loss 0.427 
Epoch 51, Step 1185 | Training Accuracy: 0.824 | Test Accuracy: 0.809 | Training Loss: 0.368 | Test Loss 0.420 
Epoch 51, Step 1422 | Training Accuracy: 0.824 | Test Accuracy: 0.816 | Training Loss: 0.387 | Test Loss 0.361 
Epoch 51, Step 1659 | Training Accuracy: 0.840 | Test Accuracy: 0.844 | Training Loss: 0.349 | Test Loss 0.357 
Epoch 51, Step 1896 | Training Accuracy: 0.820 | Test Accuracy: 0.816 | Training Loss: 0.411 | Test Loss 0.417 
(*) Entering Epoch 52 (3165.263 s)
Epoch 52, Step 237 | Training Accuracy: 0.824 | Test Accuracy: 0.848 | Training Loss: 0.372 | Test Loss 0.366 
Epoch 52, Step 474 | Training Accuracy: 0.852 | Test Accuracy: 0.812 | Training Loss: 0.353 | Test Loss 0.402 
Epoch 52, Step 711 | Training Accuracy: 0.824 | Test Accuracy: 0.828 | Training Loss: 0.373 | Test Loss 0.350 
Epoch 52, Step 948 | Training Accuracy: 0.809 | Test Accuracy: 0.832 | Training Loss: 0.426 | Test Loss 0.380 
Epoch 52, Step 1185 | Training Accuracy: 0.820 | Test Accuracy: 0.855 | Training Loss: 0.421 | Test Loss 0.334 
Epoch 52, Step 1422 | Training Accuracy: 0.844 | Test Accuracy: 0.844 | Training Loss: 0.357 | Test Loss 0.374 
Epoch 52, Step 1659 | Training Accuracy: 0.832 | Test Accuracy: 0.789 | Training Loss: 0.387 | Test Loss 0.443 
Epoch 52, Step 1896 | Training Accuracy: 0.840 | Test Accuracy: 0.875 | Training Loss: 0.385 | Test Loss 0.305 
(*) Entering Epoch 53 (3225.821 s)
Epoch 53, Step 237 | Training Accuracy: 0.793 | Test Accuracy: 0.816 | Training Loss: 0.411 | Test Loss 0.421 
Epoch 53, Step 474 | Training Accuracy: 0.848 | Test Accuracy: 0.844 | Training Loss: 0.353 | Test Loss 0.340 
Epoch 53, Step 711 | Training Accuracy: 0.836 | Test Accuracy: 0.848 | Training Loss: 0.363 | Test Loss 0.313 
Epoch 53, Step 948 | Training Accuracy: 0.852 | Test Accuracy: 0.805 | Training Loss: 0.352 | Test Loss 0.390 
Epoch 53, Step 1185 | Training Accuracy: 0.828 | Test Accuracy: 0.840 | Training Loss: 0.337 | Test Loss 0.339 
Epoch 53, Step 1422 | Training Accuracy: 0.828 | Test Accuracy: 0.781 | Training Loss: 0.369 | Test Loss 0.476 
Epoch 53, Step 1659 | Training Accuracy: 0.840 | Test Accuracy: 0.809 | Training Loss: 0.359 | Test Loss 0.425 
Epoch 53, Step 1896 | Training Accuracy: 0.895 | Test Accuracy: 0.816 | Training Loss: 0.282 | Test Loss 0.389 
(*) Entering Epoch 54 (3286.412 s)
Epoch 54, Step 237 | Training Accuracy: 0.836 | Test Accuracy: 0.812 | Training Loss: 0.418 | Test Loss 0.410 
Epoch 54, Step 474 | Training Accuracy: 0.820 | Test Accuracy: 0.836 | Training Loss: 0.378 | Test Loss 0.375 
Epoch 54, Step 711 | Training Accuracy: 0.855 | Test Accuracy: 0.848 | Training Loss: 0.345 | Test Loss 0.345 
Epoch 54, Step 948 | Training Accuracy: 0.855 | Test Accuracy: 0.891 | Training Loss: 0.359 | Test Loss 0.319 *
Epoch 54, Step 1185 | Training Accuracy: 0.867 | Test Accuracy: 0.820 | Training Loss: 0.316 | Test Loss 0.364 
Epoch 54, Step 1422 | Training Accuracy: 0.812 | Test Accuracy: 0.809 | Training Loss: 0.394 | Test Loss 0.401 
Epoch 54, Step 1659 | Training Accuracy: 0.836 | Test Accuracy: 0.836 | Training Loss: 0.385 | Test Loss 0.380 
Epoch 54, Step 1896 | Training Accuracy: 0.855 | Test Accuracy: 0.785 | Training Loss: 0.351 | Test Loss 0.434 
(*) Entering Epoch 55 (3347.396 s)
Epoch 55, Step 237 | Training Accuracy: 0.887 | Test Accuracy: 0.809 | Training Loss: 0.303 | Test Loss 0.380 
Epoch 55, Step 474 | Training Accuracy: 0.848 | Test Accuracy: 0.797 | Training Loss: 0.380 | Test Loss 0.427 
Epoch 55, Step 711 | Training Accuracy: 0.820 | Test Accuracy: 0.848 | Training Loss: 0.368 | Test Loss 0.353 
Epoch 55, Step 948 | Training Accuracy: 0.871 | Test Accuracy: 0.812 | Training Loss: 0.326 | Test Loss 0.337 
Epoch 55, Step 1185 | Training Accuracy: 0.840 | Test Accuracy: 0.785 | Training Loss: 0.356 | Test Loss 0.426 
Epoch 55, Step 1422 | Training Accuracy: 0.855 | Test Accuracy: 0.844 | Training Loss: 0.350 | Test Loss 0.379 
Epoch 55, Step 1659 | Training Accuracy: 0.801 | Test Accuracy: 0.848 | Training Loss: 0.403 | Test Loss 0.357 
Epoch 55, Step 1896 | Training Accuracy: 0.840 | Test Accuracy: 0.848 | Training Loss: 0.315 | Test Loss 0.381 
(*) Entering Epoch 56 (3407.959 s)
Epoch 56, Step 237 | Training Accuracy: 0.820 | Test Accuracy: 0.836 | Training Loss: 0.428 | Test Loss 0.357 
Epoch 56, Step 474 | Training Accuracy: 0.840 | Test Accuracy: 0.840 | Training Loss: 0.348 | Test Loss 0.363 
Epoch 56, Step 711 | Training Accuracy: 0.828 | Test Accuracy: 0.828 | Training Loss: 0.374 | Test Loss 0.350 
Epoch 56, Step 948 | Training Accuracy: 0.855 | Test Accuracy: 0.848 | Training Loss: 0.304 | Test Loss 0.382 
Epoch 56, Step 1185 | Training Accuracy: 0.840 | Test Accuracy: 0.809 | Training Loss: 0.396 | Test Loss 0.405 
Epoch 56, Step 1422 | Training Accuracy: 0.832 | Test Accuracy: 0.809 | Training Loss: 0.366 | Test Loss 0.378 
Epoch 56, Step 1659 | Training Accuracy: 0.840 | Test Accuracy: 0.844 | Training Loss: 0.375 | Test Loss 0.348 
Epoch 56, Step 1896 | Training Accuracy: 0.809 | Test Accuracy: 0.828 | Training Loss: 0.413 | Test Loss 0.362 
(*) Entering Epoch 57 (3468.531 s)
Epoch 57, Step 237 | Training Accuracy: 0.859 | Test Accuracy: 0.816 | Training Loss: 0.332 | Test Loss 0.396 
Epoch 57, Step 474 | Training Accuracy: 0.809 | Test Accuracy: 0.855 | Training Loss: 0.408 | Test Loss 0.367 
Epoch 57, Step 711 | Training Accuracy: 0.852 | Test Accuracy: 0.793 | Training Loss: 0.324 | Test Loss 0.425 
Epoch 57, Step 948 | Training Accuracy: 0.832 | Test Accuracy: 0.855 | Training Loss: 0.347 | Test Loss 0.346 
Epoch 57, Step 1185 | Training Accuracy: 0.836 | Test Accuracy: 0.812 | Training Loss: 0.358 | Test Loss 0.410 
Epoch 57, Step 1422 | Training Accuracy: 0.828 | Test Accuracy: 0.836 | Training Loss: 0.399 | Test Loss 0.350 
Epoch 57, Step 1659 | Training Accuracy: 0.848 | Test Accuracy: 0.809 | Training Loss: 0.361 | Test Loss 0.418 
Epoch 57, Step 1896 | Training Accuracy: 0.848 | Test Accuracy: 0.832 | Training Loss: 0.363 | Test Loss 0.362 
(*) Entering Epoch 58 (3529.101 s)
Epoch 58, Step 237 | Training Accuracy: 0.832 | Test Accuracy: 0.844 | Training Loss: 0.358 | Test Loss 0.374 
Epoch 58, Step 474 | Training Accuracy: 0.840 | Test Accuracy: 0.812 | Training Loss: 0.384 | Test Loss 0.412 
Epoch 58, Step 711 | Training Accuracy: 0.855 | Test Accuracy: 0.809 | Training Loss: 0.333 | Test Loss 0.421 
Epoch 58, Step 948 | Training Accuracy: 0.863 | Test Accuracy: 0.801 | Training Loss: 0.319 | Test Loss 0.446 
Epoch 58, Step 1185 | Training Accuracy: 0.820 | Test Accuracy: 0.824 | Training Loss: 0.419 | Test Loss 0.390 
Epoch 58, Step 1422 | Training Accuracy: 0.871 | Test Accuracy: 0.852 | Training Loss: 0.317 | Test Loss 0.331 
Epoch 58, Step 1659 | Training Accuracy: 0.840 | Test Accuracy: 0.859 | Training Loss: 0.355 | Test Loss 0.329 
Epoch 58, Step 1896 | Training Accuracy: 0.828 | Test Accuracy: 0.809 | Training Loss: 0.348 | Test Loss 0.377 
(*) Entering Epoch 59 (3589.719 s)
Epoch 59, Step 237 | Training Accuracy: 0.832 | Test Accuracy: 0.812 | Training Loss: 0.385 | Test Loss 0.427 
Epoch 59, Step 474 | Training Accuracy: 0.785 | Test Accuracy: 0.805 | Training Loss: 0.422 | Test Loss 0.465 
Epoch 59, Step 711 | Training Accuracy: 0.820 | Test Accuracy: 0.836 | Training Loss: 0.398 | Test Loss 0.341 
Epoch 59, Step 948 | Training Accuracy: 0.809 | Test Accuracy: 0.832 | Training Loss: 0.418 | Test Loss 0.387 
Epoch 59, Step 1185 | Training Accuracy: 0.832 | Test Accuracy: 0.805 | Training Loss: 0.348 | Test Loss 0.405 
Epoch 59, Step 1422 | Training Accuracy: 0.832 | Test Accuracy: 0.848 | Training Loss: 0.377 | Test Loss 0.377 
Epoch 59, Step 1659 | Training Accuracy: 0.805 | Test Accuracy: 0.781 | Training Loss: 0.392 | Test Loss 0.428 
Epoch 59, Step 1896 | Training Accuracy: 0.840 | Test Accuracy: 0.820 | Training Loss: 0.357 | Test Loss 0.387 
(*) Entering Epoch 60 (3650.327 s)
Graph saved to file: checkpoints/rhn_kst_rho0_epoch60.ckpt-60
Epoch 60, Step 237 | Training Accuracy: 0.820 | Test Accuracy: 0.832 | Training Loss: 0.398 | Test Loss 0.387 
Epoch 60, Step 474 | Training Accuracy: 0.836 | Test Accuracy: 0.836 | Training Loss: 0.358 | Test Loss 0.373 
Epoch 60, Step 711 | Training Accuracy: 0.852 | Test Accuracy: 0.828 | Training Loss: 0.366 | Test Loss 0.351 
Epoch 60, Step 948 | Training Accuracy: 0.836 | Test Accuracy: 0.871 | Training Loss: 0.366 | Test Loss 0.304 
Epoch 60, Step 1185 | Training Accuracy: 0.875 | Test Accuracy: 0.809 | Training Loss: 0.347 | Test Loss 0.395 
Epoch 60, Step 1422 | Training Accuracy: 0.824 | Test Accuracy: 0.828 | Training Loss: 0.372 | Test Loss 0.386 
Epoch 60, Step 1659 | Training Accuracy: 0.816 | Test Accuracy: 0.844 | Training Loss: 0.348 | Test Loss 0.353 
Epoch 60, Step 1896 | Training Accuracy: 0.824 | Test Accuracy: 0.875 | Training Loss: 0.409 | Test Loss 0.336 
(*) Entering Epoch 61 (3711.338 s)
Epoch 61, Step 237 | Training Accuracy: 0.812 | Test Accuracy: 0.852 | Training Loss: 0.438 | Test Loss 0.326 
Epoch 61, Step 474 | Training Accuracy: 0.863 | Test Accuracy: 0.871 | Training Loss: 0.312 | Test Loss 0.324 
Epoch 61, Step 711 | Training Accuracy: 0.801 | Test Accuracy: 0.859 | Training Loss: 0.420 | Test Loss 0.343 
Epoch 61, Step 948 | Training Accuracy: 0.883 | Test Accuracy: 0.805 | Training Loss: 0.288 | Test Loss 0.409 
Epoch 61, Step 1185 | Training Accuracy: 0.852 | Test Accuracy: 0.832 | Training Loss: 0.355 | Test Loss 0.394 
Epoch 61, Step 1422 | Training Accuracy: 0.812 | Test Accuracy: 0.832 | Training Loss: 0.390 | Test Loss 0.382 
Epoch 61, Step 1659 | Training Accuracy: 0.828 | Test Accuracy: 0.801 | Training Loss: 0.363 | Test Loss 0.403 
Epoch 61, Step 1896 | Training Accuracy: 0.859 | Test Accuracy: 0.824 | Training Loss: 0.357 | Test Loss 0.394 
(*) Entering Epoch 62 (3771.930 s)
Epoch 62, Step 237 | Training Accuracy: 0.809 | Test Accuracy: 0.879 | Training Loss: 0.367 | Test Loss 0.332 
Epoch 62, Step 474 | Training Accuracy: 0.828 | Test Accuracy: 0.820 | Training Loss: 0.381 | Test Loss 0.392 
Epoch 62, Step 711 | Training Accuracy: 0.867 | Test Accuracy: 0.855 | Training Loss: 0.355 | Test Loss 0.340 
Epoch 62, Step 948 | Training Accuracy: 0.844 | Test Accuracy: 0.781 | Training Loss: 0.368 | Test Loss 0.421 
Epoch 62, Step 1185 | Training Accuracy: 0.840 | Test Accuracy: 0.848 | Training Loss: 0.351 | Test Loss 0.372 
Epoch 62, Step 1422 | Training Accuracy: 0.820 | Test Accuracy: 0.812 | Training Loss: 0.386 | Test Loss 0.392 
Epoch 62, Step 1659 | Training Accuracy: 0.820 | Test Accuracy: 0.836 | Training Loss: 0.413 | Test Loss 0.367 
Epoch 62, Step 1896 | Training Accuracy: 0.801 | Test Accuracy: 0.836 | Training Loss: 0.435 | Test Loss 0.357 
(*) Entering Epoch 63 (3832.468 s)
Epoch 63, Step 237 | Training Accuracy: 0.824 | Test Accuracy: 0.875 | Training Loss: 0.360 | Test Loss 0.314 
Epoch 63, Step 474 | Training Accuracy: 0.867 | Test Accuracy: 0.832 | Training Loss: 0.329 | Test Loss 0.387 
Epoch 63, Step 711 | Training Accuracy: 0.855 | Test Accuracy: 0.773 | Training Loss: 0.361 | Test Loss 0.419 
Epoch 63, Step 948 | Training Accuracy: 0.809 | Test Accuracy: 0.848 | Training Loss: 0.401 | Test Loss 0.368 
Epoch 63, Step 1185 | Training Accuracy: 0.836 | Test Accuracy: 0.801 | Training Loss: 0.409 | Test Loss 0.424 
Epoch 63, Step 1422 | Training Accuracy: 0.848 | Test Accuracy: 0.844 | Training Loss: 0.395 | Test Loss 0.395 
Epoch 63, Step 1659 | Training Accuracy: 0.809 | Test Accuracy: 0.793 | Training Loss: 0.418 | Test Loss 0.452 
Epoch 63, Step 1896 | Training Accuracy: 0.848 | Test Accuracy: 0.844 | Training Loss: 0.343 | Test Loss 0.389 
(*) Entering Epoch 64 (3893.003 s)
Epoch 64, Step 237 | Training Accuracy: 0.816 | Test Accuracy: 0.875 | Training Loss: 0.385 | Test Loss 0.323 
Epoch 64, Step 474 | Training Accuracy: 0.844 | Test Accuracy: 0.844 | Training Loss: 0.363 | Test Loss 0.362 
Epoch 64, Step 711 | Training Accuracy: 0.789 | Test Accuracy: 0.828 | Training Loss: 0.420 | Test Loss 0.376 
Epoch 64, Step 948 | Training Accuracy: 0.832 | Test Accuracy: 0.824 | Training Loss: 0.360 | Test Loss 0.385 
Epoch 64, Step 1185 | Training Accuracy: 0.824 | Test Accuracy: 0.809 | Training Loss: 0.378 | Test Loss 0.413 
Epoch 67, Step 1896 | Training Accuracy: 0.848 | Test Accuracy: 0.824 | Training Loss: 0.333 | Test Loss 0.379 

In [4]:

    

ckpt = tf.train.get_checkpoint_state(directories.checkpoints)
rhn = Model(is_training = False, config = config)
network_output = rhn.predict(ckpt)

np.save(os.path.join(directories.checkpoints, '{}_{}_y_pred.npy'.format(config.channel, config.mode)), network_output)
np.save(os.path.join(directories.checkpoints, '{}_{}_y_test.npy'.format(config.channel, config.mode)), df_y_test.values)


    

    




RHN_0/h_0/Linear/weights:0 [526, 512]
RHN_0/h_0/Linear/Bias:0 [512]
RHN_0/t_0/Linear/weights:0 [526, 512]
RHN_0/t_0/Linear/Bias:0 [512]
RHN_0/h_1/Linear/weights:0 [512, 512]
RHN_0/h_1/Linear/Bias:0 [512]
RHN_0/t_1/Linear/weights:0 [512, 512]
RHN_0/t_1/Linear/Bias:0 [512]
RHN_0/h_2/Linear/weights:0 [512, 512]
RHN_0/h_2/Linear/Bias:0 [512]
RHN_0/t_2/Linear/weights:0 [512, 512]
RHN_0/t_2/Linear/Bias:0 [512]
RHN_0/h_3/Linear/weights:0 [512, 512]
RHN_0/h_3/Linear/Bias:0 [512]
RHN_0/t_3/Linear/weights:0 [512, 512]
RHN_0/t_3/Linear/Bias:0 [512]
softmax_w:0 [512, 2]
softmax_b:0 [2]
2116610 total variables
INFO:tensorflow:Restoring parameters from checkpoints/rhn_end-127
checkpoints/rhn_end-127 restored.






    




---------------------------------------------------------------------------
InvalidArgumentError                      Traceback (most recent call last)
/home/jtan/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
   1284     try:
-> 1285       return fn(*args)
   1286     except errors.OpError as e:

/home/jtan/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py in _run_fn(session, feed_dict, fetch_list, target_list, options, run_metadata)
   1263                                    feed_dict, fetch_list, target_list,
-> 1264                                    status, run_metadata)
   1265 

/home/jtan/anaconda3/lib/python3.6/contextlib.py in __exit__(self, type, value, traceback)
     88             try:
---> 89                 next(self.gen)
     90             except StopIteration:

/home/jtan/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/errors_impl.py in raise_exception_on_not_ok_status()
    465           compat.as_text(pywrap_tensorflow.TF_Message(status)),
--> 466           pywrap_tensorflow.TF_GetCode(status))
    467   finally:

InvalidArgumentError: Incompatible shapes: [256,512] vs. [25608,512]
	 [[Node: RHN_0/h_0/mul = Mul[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"](RHN_0/RHNCellZeroState/zeros, RHN_0/strided_slice_1)]]

During handling of the above exception, another exception occurred:

InvalidArgumentError                      Traceback (most recent call last)
<ipython-input-4-ba615a2a1c7c> in <module>()
      1 ckpt = tf.train.get_checkpoint_state(directories.checkpoints)
      2 rhn = Model(is_training = False, config = config)
----> 3 network_output = rhn.predict(ckpt)
      4 
      5 np.save(os.path.join(directories.checkpoints, '{}_{}_y_pred.npy'.format(config.channel, config.mode)), network_output)

<ipython-input-2-bc21a95f05bd> in predict(self, ckpt, metaGraph)
    175                               self.noise_x: noise_x, self.noise_i: noise_i, self.noise_h: noise_h, self.noise_o: noise_o}
    176             network_output_test, final_v_acc = sess.run(
--> 177                 [self.predictions, self.accuracy], feed_dict = feed_dict_test)
    178 
    179             print("Validation accuracy: {:g}\nValidation AUC: {:g}".format(final_v_acc, final_v_auc))

/home/jtan/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py in run(self, fetches, feed_dict, options, run_metadata)
    894     try:
    895       result = self._run(None, fetches, feed_dict, options_ptr,
--> 896                          run_metadata_ptr)
    897       if run_metadata:
    898         proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)

/home/jtan/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
   1122     if final_fetches or final_targets or (handle and feed_dict_tensor):
   1123       results = self._do_run(handle, final_targets, final_fetches,
-> 1124                              feed_dict_tensor, options, run_metadata)
   1125     else:
   1126       results = []

/home/jtan/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
   1277     if handle is None:
   1278       return self._do_call(_run_fn, self._session, feeds, fetches, targets,
-> 1279                            options, run_metadata)
   1280     else:
   1281       return self._do_call(_prun_fn, self._session, handle, feeds, fetches)

/home/jtan/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
   1296         except KeyError:
   1297           pass
-> 1298       raise type(e)(node_def, op, message)
   1299 
   1300   def _extend_graph(self):

InvalidArgumentError: Incompatible shapes: [256,512] vs. [25608,512]
	 [[Node: RHN_0/h_0/mul = Mul[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"](RHN_0/RHNCellZeroState/zeros, RHN_0/strided_slice_1)]]

Caused by op 'RHN_0/h_0/mul', defined at:
  File "/home/jtan/anaconda3/lib/python3.6/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/home/jtan/anaconda3/lib/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/home/jtan/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py", line 16, in <module>
    app.launch_new_instance()
  File "/home/jtan/anaconda3/lib/python3.6/site-packages/traitlets/config/application.py", line 658, in launch_instance
    app.start()
  File "/home/jtan/anaconda3/lib/python3.6/site-packages/ipykernel/kernelapp.py", line 477, in start
    ioloop.IOLoop.instance().start()
  File "/home/jtan/anaconda3/lib/python3.6/site-packages/zmq/eventloop/ioloop.py", line 177, in start
    super(ZMQIOLoop, self).start()
  File "/home/jtan/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 888, in start
    handler_func(fd_obj, events)
  File "/home/jtan/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/home/jtan/anaconda3/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
    self._handle_recv()
  File "/home/jtan/anaconda3/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
    self._run_callback(callback, msg)
  File "/home/jtan/anaconda3/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
    callback(*args, **kwargs)
  File "/home/jtan/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/home/jtan/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 283, in dispatcher
    return self.dispatch_shell(stream, msg)
  File "/home/jtan/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 235, in dispatch_shell
    handler(stream, idents, msg)
  File "/home/jtan/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 399, in execute_request
    user_expressions, allow_stdin)
  File "/home/jtan/anaconda3/lib/python3.6/site-packages/ipykernel/ipkernel.py", line 196, in do_execute
    res = shell.run_cell(code, store_history=store_history, silent=silent)
  File "/home/jtan/anaconda3/lib/python3.6/site-packages/ipykernel/zmqshell.py", line 533, in run_cell
    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
  File "/home/jtan/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2717, in run_cell
    interactivity=interactivity, compiler=compiler, result=result)
  File "/home/jtan/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2821, in run_ast_nodes
    if self.run_code(code, result):
  File "/home/jtan/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2881, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-4-ba615a2a1c7c>", line 2, in <module>
    rhn = Model(is_training = False, config = config)
  File "<ipython-input-2-bc21a95f05bd>", line 41, in __init__
    (cell_output, state[l]) = cell(self.rnn_inputs[:, time_step, :], state[l], layer = l)
  File "<ipython-input-2-bc21a95f05bd>", line 221, in __call__
    h = tf.tanh(linear([inputs, current_state * noise_h], self._num_units, True))
  File "/home/jtan/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py", line 846, in binary_op_wrapper
    return func(x, y, name=name)
  File "/home/jtan/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py", line 1069, in _mul_dispatch
    return gen_math_ops._mul(x, y, name=name)
  File "/home/jtan/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gen_math_ops.py", line 1402, in _mul
    result = _op_def_lib.apply_op("Mul", x=x, y=y, name=name)
  File "/home/jtan/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 767, in apply_op
    op_def=op_def)
  File "/home/jtan/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 2534, in create_op
    original_op=self._default_original_op, op_def=op_def)
  File "/home/jtan/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1203, in __init__
    self._traceback = self._graph._extract_stack()  # pylint: disable=protected-access

InvalidArgumentError (see above for traceback): Incompatible shapes: [256,512] vs. [25608,512]
	 [[Node: RHN_0/h_0/mul = Mul[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"](RHN_0/RHNCellZeroState/zeros, RHN_0/strided_slice_1)]]