Simple Character-level Language Model using ANN

In [1]:

    

import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt

# Input/Ouput data
char_raw = 'hello_world_good_morning_see_you_hello_great'
char_list = list(set(char_raw))
char_to_idx = {c: i for i, c in enumerate(char_list)}
idx_to_char = {i: c for i, c in enumerate(char_list)}
char_data = [char_to_idx[c] for c in char_raw]
char_data_one_hot = tf.one_hot(char_data, depth=len(
    char_list), on_value=1., off_value=0., axis=1, dtype=tf.float32)
char_input = char_data_one_hot[:-1, :]  # 'hello_world_good_morning_see_you_hello_grea'
char_output = char_data_one_hot[1:, :]  # 'ello_world_good_morning_see_you_hello_great'
with tf.Session() as sess:
    char_input = char_input.eval()
    char_output = char_output.eval()


    

In [2]:

    

# Parameters
learning_rate = 0.001
max_iter = 200

# Network Parameters
n_hidden_1 = 256 # 1st hidden layer number of features (neurons)
n_hidden_2 = 256 # 2nd hidden layer number of features (neurons)
n_input = char_input.shape[1] # input features
n_output = char_input.shape[1] # output features

# tf Graph input
x = tf.placeholder("float", [None, n_input])
y = tf.placeholder("float", [None, n_output])

# Create model
def multilayer_perceptron(x, weights, biases):
    # Hidden layer with RELU activation
    layer_1 = tf.add(tf.matmul(x, weights['h1']), biases['b1'])
    layer_1 = tf.nn.sigmoid(layer_1)
    # Output layer with linear activation
    out_layer = tf.matmul(layer_1, weights['out']) + biases['out']
    return tf.tanh(out_layer)


    

In [3]:

    

# Store layers weight & bias
weights = {
    'h1': tf.Variable(tf.random_normal([n_input, n_hidden_1])),
    'out': tf.Variable(tf.random_normal([n_hidden_1, n_output]))
}
biases = {
    'b1': tf.Variable(tf.random_normal([n_hidden_1])),
    'out': tf.Variable(tf.random_normal([n_output]))
}

# Construct model
pred = multilayer_perceptron(x, weights, biases)

# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)

# Initializing the variables
init = tf.global_variables_initializer()

def softmax(x):
    rowmax = np.max(x, axis=1)
    x -= rowmax.reshape((x.shape[0] ,1)) # for numerical stability
    x = np.exp(x)
    sum_x = np.sum(x, axis=1).reshape((x.shape[0],1))
    return x / sum_x


    

In [4]:

    

# Learning (online training)
with tf.Session() as sess:
    sess.run(init)

    # Training cycle
    pred_act = np.empty((0,17))
    for i in range(max_iter):
        avg_cost = 0.
        total_chars = len(char_input)
        pred_seq = []
        # Loop over all characters
        for c in range(total_chars):
            train_x = char_input[c,:].reshape((-1,n_input))
            train_y = char_output[c,:].reshape((-1,n_output))
            _, l, p = sess.run([optimizer, cost, pred], 
                               feed_dict={x: train_x, y: train_y})
            if i is max_iter-1:
                pred_act = np.vstack((pred_act, p))
            pred_out = int(np.argmax(p, axis=1))
            pred_seq.append(idx_to_char[pred_out])
            # Compute average loss
            avg_cost += l / total_chars
        # Display logs per epoch step
        print('Epoch: {:>4}'.format(i+1), 
              'Cost: {:4f}'.format(avg_cost), 
              'Predict:', ''.join(pred_seq))
    pred_act = softmax(pred_act)


    

    




Epoch:    1 Cost: 3.438902 Predict: aaaaaaaaaaaaaaaaaaaaawaaaaaaaaaaaaaaaaaaaaa
Epoch:    2 Cost: 3.249028 Predict: aaaaaaaaaaaaaaaaaaaaawaaaaaaaaaaaaaaaaaaaaa
Epoch:    3 Cost: 3.098925 Predict: aaaaaaaaaaaaaaaaaaaaahaaaaaaaaaaaaaaaaaaaaa
Epoch:    4 Cost: 2.967775 Predict: aaaaaaaaaaaaaaaaaaaaahaaaaaaaaaaaaaaaaaaaaa
Epoch:    5 Cost: 2.911683 Predict: aaaaaaaaaaaaaaaaaaaaaohaaaaaaaaaaawaaaaaaha
Epoch:    6 Cost: 2.874949 Predict: aaaaaaaaaahaaaahaaaahohaaaooaaaaahwaaaaaawa
Epoch:    7 Cost: 2.855519 Predict: hhaaaaaaaahaaaahaaaahohaaawhaaaaawwaaaaaawa
Epoch:    8 Cost: 2.813224 Predict: wwaaaaaaaahaaaawaaaawowaawowaoawawonnaaaaow
Epoch:    9 Cost: 2.771241 Predict: wwnnaawaanwaaaawaaaawowaawwwaowwawwnnwtwwww
Epoch:   10 Cost: 2.681875 Predict: wonnwtwwanwtwwwwtlwawowwawwwtowwtwwnnwtwaww
Epoch:   11 Cost: 2.651154 Predict: wwnnwtwwanwtwwwwtlwawowwtwootowwtwwnnwtwwww
Epoch:   12 Cost: 2.632099 Predict: wwnnwtwwanwtwwwwtlwwwowwtwwotowwtwwnnwtwwww
Epoch:   13 Cost: 2.603506 Predict: wwnnwtwwwnwtwwwwtlwwwowwtwwwtowwtwwnnwtwwww
Epoch:   14 Cost: 2.576422 Predict: wwnnwtwwwnwtwwwwtlwwwowwtwwwgowwgwonnwgwwww
Epoch:   15 Cost: 2.541513 Predict: wwnnwgwwwnwgwwwwglwwwowwgwwwgowwgwwnnwgwwww
Epoch:   16 Cost: 2.513230 Predict: wwnnwgwwwnwgwwwwglwwwowwgwwwgowwgwwnnwgwwww
Epoch:   17 Cost: 2.481514 Predict: wwnnwgwwwnwgwwwwglwwwowwgwwwgowwgwwnnwgwwww
Epoch:   18 Cost: 2.430895 Predict: wwnnwgwwwlwgwwwwglwwwowwgwwwgowwgwwnnwgwwww
Epoch:   19 Cost: 2.395654 Predict: wwnnwgwwwnwgwwwwglwwwowwgwwwgowwgwwnnwgwwww
Epoch:   20 Cost: 2.320129 Predict: wwnnwgwwwnwgwwwwglwwwowwgwwwgowwgwwnnwgwwww
Epoch:   21 Cost: 2.310483 Predict: wwnnwgwwwnwgwwwwglwwwowwgwwwgowwgwwnnwgwwww
Epoch:   22 Cost: 2.304042 Predict: wwnnwgwwwnwgwwwwglwwwowwgwwwgowwgwwnnwgwwww
Epoch:   23 Cost: 2.296958 Predict: wwnnwgwwwnwgwwwwglwwwowwgwwwgowwgwwnnwgwwww
Epoch:   24 Cost: 2.282331 Predict: wwnnwgwwwnwgwwwwglwwwowwgwwwgowwgwwnnwgwwww
Epoch:   25 Cost: 2.262126 Predict: wwnnwgwwwnwgwwwwglwwwowwgwwwgowwgwwnnwgwwww
Epoch:   26 Cost: 2.252340 Predict: wwnnwgwwwnwgwwwwglwwwowwgwwwgowwgwwnnwgwwww
Epoch:   27 Cost: 2.248322 Predict: wwnnwgwwwlwgwwwwglwwwowwgwwwgowwgwwnnwgwwww
Epoch:   28 Cost: 2.244619 Predict: wwnnwgwwwlwgwwwwglwwwowwgwwwgowwgwwnnwgwwww
Epoch:   29 Cost: 2.238382 Predict: wwllwgwwwlwgwwwwglwwwowwgwwwgowwgwwoowwwwww
Epoch:   30 Cost: 2.232178 Predict: wwllwgwwwlwgwwwwglwwwowwgwwwgowwgwwoowgwwww
Epoch:   31 Cost: 2.230264 Predict: wwllwgwwwlwgwwwwglwwwowwgwwwgowwgwwoowgwwww
Epoch:   32 Cost: 2.228911 Predict: wwllwgwwwlwgwwwwglwwwowwgwwwgowwgwwoowgwwww
Epoch:   33 Cost: 2.227805 Predict: wwllwgwwwlwgwwwwglwwwowwgwwwgowwgwwoowgwwww
Epoch:   34 Cost: 2.226705 Predict: wwllwgwwwlwgwwwwglwwwowwgwwwgowwgwwoowgwwww
Epoch:   35 Cost: 2.225200 Predict: wwllwgwwwlwgwwwwglwwwowwgwwwgowwgwwoowgwwww
Epoch:   36 Cost: 2.222227 Predict: wwllwgwwwlwgwwwwglwwwowwgwwwgowwgwwoowgwwww
Epoch:   37 Cost: 2.219227 Predict: wwolwgwwwlwgwwwwglwwgogwgwwwgowwgwwoowgwnww
Epoch:   38 Cost: 2.200884 Predict: wwllwgwwwlwgwwwwglwwgogwgwwwgowwgwwoowgwwww
Epoch:   39 Cost: 2.198013 Predict: wwllwgwwwlwgwwwwglwwgogwgwwwgowwgwwllwgwwww
Epoch:   40 Cost: 2.196212 Predict: wwllwgwwwlwgwwwwglwwgogwgwwwgowwgwwllwgwnww
Epoch:   41 Cost: 2.193012 Predict: wwllwgwwwlwgwwwwglwwgogwgwwwgowwgwwllwgwnww
Epoch:   42 Cost: 2.190624 Predict: wwllwgwwnlwgwwwwglwwgogwgwwwgowwgwwllwgwnww
Epoch:   43 Cost: 2.188183 Predict: wwllwgwwnlwgwwwwglwlgogwgwwwgowwgwwllwgwnww
Epoch:   44 Cost: 2.187684 Predict: wwllwgwwnlwgwwwwglwngogwgwwwgowwgwwolwgwnww
Epoch:   45 Cost: 2.187038 Predict: wollwgwwnlwgwwwwglwngogwgwwwgowwgwooowgwnow
Epoch:   46 Cost: 2.185760 Predict: wollwgwwnlwgwwwwglwngogwgwoogow_gwooowgwno_
Epoch:   47 Cost: 2.180619 Predict: wooowgwwnlwgwwwwglwngogwgwoogow_gwooowgwno_
Epoch:   48 Cost: 2.173514 Predict: wooowg_wno_wwwwwwlwngogwwwoowow_wwooowywno_
Epoch:   49 Cost: 2.150439 Predict: wooowg_wno_gwww_gowngogwgwoogow_gwooowgono_
Epoch:   50 Cost: 2.156310 Predict: wooowg_wno_gwww_gowngogwgwoogow_gwooowgwno_
Epoch:   51 Cost: 2.146388 Predict: wooowg_wno_gwww_gowngogwgwoogow_gwooowgono_
Epoch:   52 Cost: 2.139970 Predict: wooowg_wno_gwww_gowngogogwoogow_gwooorgono_
Epoch:   53 Cost: 2.133894 Predict: wooorg_rno_gorr_gorngogogwoogor_gwooorgono_
Epoch:   54 Cost: 2.073794 Predict: wooorg_rno_gorr_gorngogogooogor_gwooorgono_
Epoch:   55 Cost: 2.056855 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   56 Cost: 2.054401 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   57 Cost: 2.053523 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   58 Cost: 2.053105 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   59 Cost: 2.052948 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   60 Cost: 2.052738 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   61 Cost: 2.052618 Predict: oooorg_rnl_gorr_gorngogogooogor_goooorgono_
Epoch:   62 Cost: 2.052476 Predict: oooorg_rnl_gorr_gorngogogooogor_goooorgono_
Epoch:   63 Cost: 2.052372 Predict: oooorg_rnl_gorr_gorngogogooogor_goooorgono_
Epoch:   64 Cost: 2.052261 Predict: oooorg_rnl_gorr_gorngogogooogor_goooorgono_
Epoch:   65 Cost: 2.052169 Predict: oooorg_rnl_gorr_gorngogogooogor_goooorgono_
Epoch:   66 Cost: 2.052077 Predict: oooorg_rnl_gorr_gorngogogooogor_goooorgono_
Epoch:   67 Cost: 2.051995 Predict: oooorg_rnl_gorr_gorngogogooogor_goooorgono_
Epoch:   68 Cost: 2.051915 Predict: oooorg_rnl_gorr_gorngogogooogor_goooorgono_
Epoch:   69 Cost: 2.051840 Predict: oooorg_rnl_gorr_gorngogogooogor_goooorgono_
Epoch:   70 Cost: 2.051768 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   71 Cost: 2.051699 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   72 Cost: 2.051632 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   73 Cost: 2.051567 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   74 Cost: 2.051502 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   75 Cost: 2.051436 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   76 Cost: 2.051367 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   77 Cost: 2.051292 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   78 Cost: 2.051204 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   79 Cost: 2.051083 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   80 Cost: 2.050865 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   81 Cost: 2.050206 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   82 Cost: 2.047032 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   83 Cost: 2.045114 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   84 Cost: 2.041882 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   85 Cost: 2.041845 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   86 Cost: 2.041538 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   87 Cost: 2.041383 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   88 Cost: 2.041267 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   89 Cost: 2.041171 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   90 Cost: 2.041092 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   91 Cost: 2.041024 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   92 Cost: 2.040965 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   93 Cost: 2.040912 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   94 Cost: 2.040864 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   95 Cost: 2.040821 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   96 Cost: 2.040781 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   97 Cost: 2.040745 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   98 Cost: 2.040710 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:   99 Cost: 2.040678 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:  100 Cost: 2.040648 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:  101 Cost: 2.040620 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:  102 Cost: 2.040593 Predict: oooorg_rno_gorr_gorngogogooogor_goooorgono_
Epoch:  103 Cost: 2.040567 Predict: oooorg_rno_gorr_gorngngogooogor_goooorgono_
Epoch:  104 Cost: 2.040543 Predict: oooorg_rno_gorr_gorngngogooogor_goooorgono_
Epoch:  105 Cost: 2.040520 Predict: oooorg_rno_gorr_gorngngogooogor_goooorgono_
Epoch:  106 Cost: 2.040498 Predict: oooorg_rno_gorr_gorngngogooogor_goooorgono_
Epoch:  107 Cost: 2.040476 Predict: oooorg_rno_gorr_gorngngogooogor_goooorgono_
Epoch:  108 Cost: 2.040456 Predict: oooorg_rno_gorr_gorngngogooogor_goooorgono_
Epoch:  109 Cost: 2.040436 Predict: oooorg_rno_gorr_gorngngogooogor_goooorgono_
Epoch:  110 Cost: 2.040417 Predict: oooorg_rno_gorr_gorngngogooogor_goooorgono_
Epoch:  111 Cost: 2.040399 Predict: oooorg_rno_gorr_gorngngogooogor_goooorgono_
Epoch:  112 Cost: 2.040382 Predict: oooorg_rno_gorr_gorngngogooogor_goooorgono_
Epoch:  113 Cost: 2.040365 Predict: oooorg_rno_gorr_gorngngogooogor_goooorgono_
Epoch:  114 Cost: 2.040348 Predict: oooorg_rno_gorr_gorngngogooogor_goooorgono_
Epoch:  115 Cost: 2.040332 Predict: oooorg_rno_gorr_gorngngogooogor_goooorgono_
Epoch:  116 Cost: 2.040317 Predict: oooorg_rnl_gorr_gorngngogooogor_goooorgono_
Epoch:  117 Cost: 2.040302 Predict: oooorg_rnl_gorr_gorngngogooogor_goooorgono_
Epoch:  118 Cost: 2.040287 Predict: oooorg_rnl_gorr_gorngngogooogor_goooorgono_
Epoch:  119 Cost: 2.040273 Predict: oooorg_rnl_gorr_gorngngogooogor_goooorgono_
Epoch:  120 Cost: 2.040260 Predict: oooorg_rnl_gorr_gorngngogooogor_goooorgono_
Epoch:  121 Cost: 2.040246 Predict: oooorg_rnl_gorr_gorngngogooogor_goooorgono_
Epoch:  122 Cost: 2.040233 Predict: oooorg_rnl_gorr_gorngngogooogor_goooorgono_
Epoch:  123 Cost: 2.040221 Predict: oooorg_rnl_gorr_gorngngogooogor_goooorgono_
Epoch:  124 Cost: 2.040208 Predict: oooorg_rnl_gorr_gorngngogooogor_goooorgono_
Epoch:  125 Cost: 2.040196 Predict: oooorg_rnl_gorr_gorngngogooogor_goooorgono_
Epoch:  126 Cost: 2.040185 Predict: oooorg_rnl_gorr_gorngngogooogor_goooorgono_
Epoch:  127 Cost: 2.040173 Predict: oooorg_rnl_gorr_gorngngogooogor_goooorgono_
Epoch:  128 Cost: 2.040162 Predict: oooorg_rnl_gorr_gorngngogooogor_goooorgono_
Epoch:  129 Cost: 2.040151 Predict: oooorg_rnl_gorr_gorngngogooogor_goooorgono_
Epoch:  130 Cost: 2.040140 Predict: oooorg_rnl_gorr_gorngngogooogor_goooorgono_
Epoch:  131 Cost: 2.040129 Predict: oooorg_rnl_gorr_gorngngogooogor_goooorgono_
Epoch:  132 Cost: 2.040119 Predict: oooorg_rnl_gorr_gorngngogooogor_goooorgono_
Epoch:  133 Cost: 2.040109 Predict: oooorg_rnl_gorr_gorngngogooogor_goooorgono_
Epoch:  134 Cost: 2.040098 Predict: oooorg_rnl_gorr_gorngngogooogor_goooorgono_
Epoch:  135 Cost: 2.040088 Predict: oooorg_rnl_gorr_gorngngogooogor_goooorgono_
Epoch:  136 Cost: 2.040077 Predict: oooorg_rnl_gorr_gorngngogooogor_goooorgono_
Epoch:  137 Cost: 2.040066 Predict: oooorg_rnl_gorr_gorngngogooogor_goooorgono_
Epoch:  138 Cost: 2.040055 Predict: ooolrg_rnl_gorr_gorngngogooogor_goooorgono_
Epoch:  139 Cost: 2.040043 Predict: ooolrg_rnl_gorr_gorngngog_oogor_goooorgono_
Epoch:  140 Cost: 2.040028 Predict: oollrg_rnl_gorr_gorngngog_oogor_goooorgono_
Epoch:  141 Cost: 2.040009 Predict: oollrg_rnl_gorr_gorngngog_oogor_goooorgono_
Epoch:  142 Cost: 2.039981 Predict: oollrg_rnl_gorr_gorngngog_oogor_goooorgono_
Epoch:  143 Cost: 2.039927 Predict: oollrg_rnl_gorr_gorngngog_oogor_goollrgono_
Epoch:  144 Cost: 2.039780 Predict: oollrg_rnl_g_rr_gorngngog_oogor_goollrgono_
Epoch:  145 Cost: 2.039202 Predict: nollrg_rnl_g_rr_gorngngog_oogor_gnollrg_no_
Epoch:  146 Cost: 2.039112 Predict: nollrg_rnl_g_rr_gorngngog_oogor_gnollrg_no_
Epoch:  147 Cost: 2.034441 Predict: nollrg_rnl_g_rr_gorngng_g_oogor_gnollrg_no_
Epoch:  148 Cost: 2.034576 Predict: nollrg_rnl_g_rr_gorngng_g_oogor_gnollrg_no_
Epoch:  149 Cost: 2.034561 Predict: nollrg_rnl_g_rr_gorngng_g_oogor_gnollrg_no_
Epoch:  150 Cost: 2.034501 Predict: nollrg_rnl_g_rr_gorngng_g_oogor_gnollrg_no_
Epoch:  151 Cost: 2.034480 Predict: nollrg_rnl_g_rr_gorngng_g_oogor_gnollrg_no_
Epoch:  152 Cost: 2.034452 Predict: nollrg_rnl_g_rr_gorngng_g_oogor_gnollrg_no_
Epoch:  153 Cost: 2.034435 Predict: nollrg_rnl_g_rr_gorngng_g_oogor_gnollrg_no_
Epoch:  154 Cost: 2.034416 Predict: nollrg_rnl_g_rr_gorngngog_oogor_gnollrg_no_
Epoch:  155 Cost: 2.034402 Predict: nollrg_rnl_g_rr_gorngngog_oogor_gnollrg_no_
Epoch:  156 Cost: 2.034388 Predict: nollrg_rnl_g_rr_gorngngog_oogor_gnollrg_no_
Epoch:  157 Cost: 2.034377 Predict: nollrg_rnl_g_rr_gorngngog_oogor_gnollrg_no_
Epoch:  158 Cost: 2.034365 Predict: nollrg_rnl_g_rr_gorngngog_oogor_gnollrg_no_
Epoch:  159 Cost: 2.034356 Predict: nollrg_rnl_g_rr_gorngngog_oogor_gnollrgono_
Epoch:  160 Cost: 2.034346 Predict: nollrg_rnl_g_rr_gorngngog_oogor_gnollrgono_
Epoch:  161 Cost: 2.034338 Predict: nollrg_rnl_g_rr_gorngngog_oogor_gnollrgono_
Epoch:  162 Cost: 2.034330 Predict: nollrg_rnl_g_rr_gorngngog_oogor_gnollrgono_
Epoch:  163 Cost: 2.034322 Predict: nollrg_rnl_g_rr_gorngngog_oogor_gnollrgono_
Epoch:  164 Cost: 2.034315 Predict: nollrg_rnl_g_rr_gorngngog_oogor_gnollrgono_
Epoch:  165 Cost: 2.034308 Predict: nollrg_rnl_g_rr_gorngngog_oogor_gnollrgono_
Epoch:  166 Cost: 2.034302 Predict: nollrg_rnl_g_rr_gorngngog_oogor_gnollrgono_
Epoch:  167 Cost: 2.034295 Predict: nollrg_rnl_g_rr_gorngngog_oogor_gnollrgono_
Epoch:  168 Cost: 2.034289 Predict: nollrg_rnl_g_rr_gorngngog_oogor_gnollrgono_
Epoch:  169 Cost: 2.034284 Predict: nollrg_rnl_g_rr_gorngngog_oogor_gnollrgono_
Epoch:  170 Cost: 2.034278 Predict: nollrg_rnl_g_rr_gorngngog_oogor_gnollrgono_
Epoch:  171 Cost: 2.034273 Predict: nollrg_rnl_g_rr_gorngngog_oogor_gnollrgono_
Epoch:  172 Cost: 2.034268 Predict: nollrg_rnl_g_rr_gorngngog_oogor_gnollrgono_
Epoch:  173 Cost: 2.034263 Predict: nollrg_rnl_g_rr_gorngngog_oogor_gnollrgono_
Epoch:  174 Cost: 2.034258 Predict: nollrg_rnl_g_rr_gorngngog_oogor_gnollrgono_
Epoch:  175 Cost: 2.034253 Predict: nollrg_rnl_g_rr_gorngngog_oogor_gnollrgono_
Epoch:  176 Cost: 2.034249 Predict: nollrg_rnl_g_rr_gorngngog_oogor_gnollrgono_
Epoch:  177 Cost: 2.034244 Predict: nollrg_rnl_g_rr_gorngngog_oogor_gnollrgono_
Epoch:  178 Cost: 2.034240 Predict: nollrg_rnl_g_rr_gorngngog_oogor_gnollrgono_
Epoch:  179 Cost: 2.034236 Predict: nollrg_rnl_g_rr_gorngngog_oogor_gnollrgono_
Epoch:  180 Cost: 2.034232 Predict: nollrg_rnl_g_rr_gorngngog_oogor_gnollrgono_
Epoch:  181 Cost: 2.034228 Predict: nollrg_rnl_g_rr_gorngngog_oogor_gnollrgono_
Epoch:  182 Cost: 2.034224 Predict: nollrg_rnl_gorr_gorngngog_oogor_gnollrgono_
Epoch:  183 Cost: 2.034220 Predict: nollrg_rnl_gorr_gorngngog_oogor_gnoolrgono_
Epoch:  184 Cost: 2.034216 Predict: nollrg_rnl_gorr_gorngngog_oogor_gnoolrgono_
Epoch:  185 Cost: 2.034212 Predict: nollrg_rnl_gorr_gorngngog_oogor_gnooorgono_
Epoch:  186 Cost: 2.034208 Predict: nollrg_rnl_gorr_gorngngog_oogor_gnooorgono_
Epoch:  187 Cost: 2.034204 Predict: nollrg_rnl_gorr_gorngngog_oogor_gnooorgono_
Epoch:  188 Cost: 2.034201 Predict: nollrg_rnl_gorr_gorngngog_oogor_gnooorgono_
Epoch:  189 Cost: 2.034197 Predict: nollrg_rnl_gorr_gorngngog_oogor_gnooorgono_
Epoch:  190 Cost: 2.034193 Predict: nollrg_rnl_gorr_gorngngog_oogor_gnooorgono_
Epoch:  191 Cost: 2.034189 Predict: nollrg_rnl_gorr_gorngngog_oogor_gnooorgono_
Epoch:  192 Cost: 2.034185 Predict: nollrg_rnl_gorr_gorngngog_oogor_goooorgono_
Epoch:  193 Cost: 2.034180 Predict: nollrg_rnl_gorr_gorngngog_oogor_goooorgono_
Epoch:  194 Cost: 2.034175 Predict: nollrg_rnl_gorr_gorngngog_oogor_goooorgono_
Epoch:  195 Cost: 2.034169 Predict: nollrg_rnl_gorr_gorngngog_oogor_goooorgono_
Epoch:  196 Cost: 2.034163 Predict: nollrg_rnl_gorr_gorngngog_oogor_goooorgono_
Epoch:  197 Cost: 2.034155 Predict: nollrg_rnl_gorr_gorngngog_oogor_goooorgono_
Epoch:  198 Cost: 2.034146 Predict: nollrg_rnl_gorr_gorngngog_oogor_goooorgono_
Epoch:  199 Cost: 2.034138 Predict: nollrg_rnl_gorr_gorngngog_oogor_gwooorgono_
Epoch:  200 Cost: 2.034139 Predict: woolrg_rnl_gorr_gorngngog_oogor_gwooorgono_

In [5]:

    

# Probability plot
fig, ax = plt.subplots()
fig.set_size_inches(15,20)
plt.title('Input Sequence', y=1.08, fontsize=20)
plt.xlabel('Probability of Next Character(y) Given Current One(x)', fontsize=25, y=1.5)
plt.ylabel('Character List', fontsize=20)
plot = plt.imshow(pred_act.T, cmap=plt.get_cmap('plasma'))
cbar = fig.colorbar(plot, fraction=0.015, pad=0.08)
plt.xticks(np.arange(len(char_data)-1), list(char_raw)[:-1], fontsize=15)
plt.yticks(np.arange(len(char_list)), [idx_to_char[i] for i in range(len(char_list))], fontsize=15)
ax.xaxis.tick_top()

# Annotate
for i, c in zip(range(len(pred_seq)), pred_seq):
    idx = char_to_idx[c]
    ax.annotate(c, xy=(i-0.2, idx+0.2), fontsize=12)
    
plt.show()