In [1]:

    

# Simple character-level prediction using RNN
# 2017-03-30 jkang
# 
# 'hello_world_good_morning_see_you_hello_great'
#
# input:  'ello_world_good_morning_see_you_hello_great'
# output: 'hello_world_good_morning_see_you_hello_grea'
# 
# Python3.5
# Tensorflow1.0.1
# ref: https://hunkim.github.io/ml/

import tensorflow as tf
import numpy as np


    

In [2]:

    

# Make input data
char_raw = 'hello_world_good_morning_see_you_hello_great'
char_list = list(set(char_raw))
char_idx = {c: i for i, c in enumerate(char_list)} # character with index
char_data = [char_idx[c] for c in char_raw]
char_data_onehot = tf.one_hot(char_data, 
                              depth=len(char_list), 
                              on_value=1., 
                              off_value=0.,
                              axis=1, 
                              dtype=tf.float32)
char_input = char_data_onehot[:-1] # 'ello_world_good_morning_see_you_hello_great'
char_output = char_data_onehot[1:] # 'hello_world_good_morning_see_you_hello_grea'
print('char_data:', char_data)
print('char_data_onehot:', char_data_onehot.shape)
print('char_input:', char_input.shape)
print('char_output:', char_output.shape)


    

    




char_data: [8, 12, 3, 3, 10, 4, 0, 10, 13, 3, 15, 4, 9, 10, 10, 15, 4, 5, 10, 13, 2, 11, 2, 9, 4, 6, 12, 12, 4, 1, 10, 16, 4, 8, 12, 3, 3, 10, 4, 9, 13, 12, 7, 14]
char_data_onehot: (44, 17)
char_input: (43, 17)
char_output: (43, 17)

In [3]:

    

# Set configurations
n_char = len(char_list)
rnn_size = n_char # number of one-hot coding vectors == output size for each cell
n_timestep = char_input.shape.as_list()[0] # length of the input
batch_size = 1  # one example
max_iter = 300

# Set RNN
rnn_cell = tf.contrib.rnn.BasicRNNCell(rnn_size)
init_state = tf.zeros([batch_size, rnn_cell.state_size])
input_split = tf.split(value=char_input, num_or_size_splits=n_timestep, axis=0)
outputs, state = tf.contrib.rnn.static_rnn(rnn_cell, input_split, init_state)


    

In [4]:

    

# logits: A 3D Tensor of shape [batch_size x sequence_length x num_decoder_symbols] and dtype float. 
# targets: A 2D Tensor of shape [batch_size x sequence_length] and dtype int.
# weights: A 2D Tensor of shape [batch_size x sequence_length] and dtype float.
# logits = tf.reshape(tf.concat(values=char_output, axis=1), [batch_size, n_timestep, rnn_size])
logits = tf.reshape(outputs, [batch_size, n_timestep, rnn_size])
targets = tf.reshape(char_data[1:], [batch_size, n_timestep]) # target as index
weights = tf.ones((batch_size, n_timestep))

loss = tf.contrib.seq2seq.sequence_loss(logits, targets, weights)
cost = tf.reduce_sum(loss) / batch_size
train_op = tf.train.RMSPropOptimizer(learning_rate = 0.01, decay = 0.9).minimize(cost)


    

In [5]:

    

# Launch the graph in a session
with tf.Session() as sess:
    tf.global_variables_initializer().run()
    for i in range(max_iter):
        _, c = sess.run([train_op, cost])
        result = sess.run(tf.arg_max(logits, 2))
        print('Epoch: {:>4}'.format(i + 1), '/', str(max_iter),
              'Cost: {:4f}'.format(c), 'Predict:', ''.join([char_list[t] for t in result[0]]))


    

    




Epoch:    1 / 300 Cost: 2.939752 Predict: iysnellehlywleellhehwrwrweygdeewlmynnellh_h
Epoch:    2 / 300 Cost: 2.937397 Predict: iysnellehlywleellhehwrwrweygdeewlmynnellh_h
Epoch:    3 / 300 Cost: 2.934924 Predict: iysnellehlywleelleehwrwrweygdeewlmynnellh_h
Epoch:    4 / 300 Cost: 2.932330 Predict: iysnellehlywleelleehwrwrweygdeewleynnellh_h
Epoch:    5 / 300 Cost: 2.929610 Predict: iysnellehlylleelleehwrwrweygdeewleynnellh_h
Epoch:    6 / 300 Cost: 2.926758 Predict: iysnellehlylleelleehwrwrweygdeewleynnellh_h
Epoch:    7 / 300 Cost: 2.923770 Predict: iysnellehlylleelleehwrwrweygdeewleynnellh_h
Epoch:    8 / 300 Cost: 2.920639 Predict: iysnellehlylleelleehwrwrweygdeewleynnellh_h
Epoch:    9 / 300 Cost: 2.917361 Predict: iysnellehlylleelleehwrerleygdeewleynnellh_h
Epoch:   10 / 300 Cost: 2.913930 Predict: iysnellehlylleelleehwrerleygdeewleynnellh_h
Epoch:   11 / 300 Cost: 2.910341 Predict: iysnellehlylleelleehwrerleygdeewleydnellh_h
Epoch:   12 / 300 Cost: 2.906589 Predict: iysnellehlylleelleehweerleygdeewleydnellh_h
Epoch:   13 / 300 Cost: 2.902666 Predict: iysnellehlylleelleehweerleygdeewleydnellh_h
Epoch:   14 / 300 Cost: 2.898569 Predict: iysnellehlylleelleehweerleygdeewleydlellh_h
Epoch:   15 / 300 Cost: 2.894293 Predict: iysneleehlelleelleeheeerleygdeewleydlellhlh
Epoch:   16 / 300 Cost: 2.889831 Predict: iydneleehlelleelleeheeerleygdeewleydlellheh
Epoch:   17 / 300 Cost: 2.885179 Predict: iydneleehlelleelleeheeerleygdeewleydlellheh
Epoch:   18 / 300 Cost: 2.880331 Predict: iydneleehlelleelleeheeerleygdeewlegdlellheh
Epoch:   19 / 300 Cost: 2.875284 Predict: iydneleehlelleelleeheeerleygdeewlegdlellheh
Epoch:   20 / 300 Cost: 2.870033 Predict: iydneleehlelleelleeheeerleygdeewlegdeellheh
Epoch:   21 / 300 Cost: 2.864575 Predict: iydneleehlelleelleeheeerleygdeewlegdeellheh
Epoch:   22 / 300 Cost: 2.858906 Predict: iydneleehrelleelleeheeerleygdeewlegdeellheh
Epoch:   23 / 300 Cost: 2.853024 Predict: iydneleehrelleelleeheeerleygdeewlegdeellheh
Epoch:   24 / 300 Cost: 2.846927 Predict: igdneleehrelleelleeheeerleygdeewlegdeellheh
Epoch:   25 / 300 Cost: 2.840615 Predict: igdneleehrelleelleeheeerleygdeewlegdeellheh
Epoch:   26 / 300 Cost: 2.834087 Predict: igdneleehrelleelleeheeerleygdeewlegdeellheh
Epoch:   27 / 300 Cost: 2.827343 Predict: igdneweehredleeeleeheeerleygdeewlegdeelrheh
Epoch:   28 / 300 Cost: 2.820387 Predict: igdneweehredleeeleeheeerleygdeewlegdeelrheh
Epoch:   29 / 300 Cost: 2.813221 Predict: igdneweehredeeeeleeheeerleggdeewlegdeelrheh
Epoch:   30 / 300 Cost: 2.805848 Predict: igdeeweehredeeeeleeheeerleggdeewlegdeelrheh
Epoch:   31 / 300 Cost: 2.798274 Predict: igdeeweehredeeeeleeheeerleggdeewdegdeelrheh
Epoch:   32 / 300 Cost: 2.790502 Predict: igdeeweehredeeeeleeheeerleggdeewregdeelreeh
Epoch:   33 / 300 Cost: 2.782541 Predict: igdeeweehredeeeedeeheeeroeggdeewregdeelreeh
Epoch:   34 / 300 Cost: 2.774395 Predict: igdeeweehredeeeedeeheeeroeggdeewregdeelreeh
Epoch:   35 / 300 Cost: 2.766073 Predict: igdeeweehredeeeedeeheeeroeggdeewregdeewreeh
Epoch:   36 / 300 Cost: 2.757581 Predict: igdeeweehredeeeedeeheeeroeggdeewregdeewreer
Epoch:   37 / 300 Cost: 2.748927 Predict: egdeeweeeredeeeedeeheeeroeggdeewregdeewreer
Epoch:   38 / 300 Cost: 2.740121 Predict: egdeeweeereoeeeedeeheeeroeggdeewregdeeoreer
Epoch:   39 / 300 Cost: 2.731172 Predict: egdeeweeereoeeeedeeheeeroeggdeewregdeeoreer
Epoch:   40 / 300 Cost: 2.722088 Predict: egdeeweeereoeeeedeeheeeroeggdeewregdeeoreer
Epoch:   41 / 300 Cost: 2.712881 Predict: egdeeweeereoeeeedeeheeeroeggdeewregdeeoreer
Epoch:   42 / 300 Cost: 2.703560 Predict: egdeeoeeereoeeeedeeheeeroeggdeewregdeeoreer
Epoch:   43 / 300 Cost: 2.694138 Predict: egdeeoeeereoeeeeoeeheeeroeggdeewregdeeoreer
Epoch:   44 / 300 Cost: 2.684624 Predict: egdeeoeeereoeeeeoeeheeeroeggdeewregdeeoreer
Epoch:   45 / 300 Cost: 2.675027 Predict: egdoeoeeerooeeeeoeeheeeroeggdeewregdeeoreer
Epoch:   46 / 300 Cost: 2.665354 Predict: egdoeoeeerogeeeeoeeheeeroeggdeewregdeeoreer
Epoch:   47 / 300 Cost: 2.655606 Predict: egdoeoeeerogeeegoeeheeeroeggoeewregdeeore_r
Epoch:   48 / 300 Cost: 2.645780 Predict: egdoeoeeerogeeegoeeheeeroeggoeewregdoeoro_r
Epoch:   49 / 300 Cost: 2.635862 Predict: egdoeoeeerogeeegoeehreeroeggoeewregdoeoro_r
Epoch:   50 / 300 Cost: 2.625831 Predict: egdoeoeeerogeeegoeehreeroeggoeewregdoeoro_r
Epoch:   51 / 300 Cost: 2.615652 Predict: eldoeoeehrogreegoeehreeroeggoeewregloeoro_r
Epoch:   52 / 300 Cost: 2.605280 Predict: eldoeoee_rogreegoee_rreroeggoeewregloe_ro_r
Epoch:   53 / 300 Cost: 2.594665 Predict: elloe_eenrogreegoee_rreroeggoeewregloe_ro_r
Epoch:   54 / 300 Cost: 2.583755 Predict: elloe_eenronreegoee_rreroeggoeewregloe_ro_r
Epoch:   55 / 300 Cost: 2.572506 Predict: ell_e_eenronreegoee_rrgroeggoeewregloe_ro_r
Epoch:   56 / 300 Cost: 2.560903 Predict: ell_e_eenronreegoee_rrgroeggoeewregloe_ro_r
Epoch:   57 / 300 Cost: 2.548961 Predict: ell_e_oenronreegoee_rrgroe_goeewregloe_ro_r
Epoch:   58 / 300 Cost: 2.536736 Predict: ell_e_oenronreegooenrrgroe_goeewregloewro_r
Epoch:   59 / 300 Cost: 2.524320 Predict: ell___oenronreegooenrrgroe_goeewregloewro_r
Epoch:   60 / 300 Cost: 2.511822 Predict: ell___oenronieegoo_nregroe_goeewregloewro_r
Epoch:   61 / 300 Cost: 2.499345 Predict: ell___oenronieegoo_nregroo_goegwregloewio_r
Epoch:   62 / 300 Cost: 2.486955 Predict: ell___oenronieegoo_nregroo_goegwreglo_gio_r
Epoch:   63 / 300 Cost: 2.474675 Predict: ell___oenronieegoo_nregroo_goegwreglo_gio_r
Epoch:   64 / 300 Cost: 2.462488 Predict: ell___oenrogieegro_nregroo_goegwreglo_gio_r
Epoch:   65 / 300 Cost: 2.450351 Predict: ell___oenrogieegro_nregroo_goegwreglo_gio_r
Epoch:   66 / 300 Cost: 2.438211 Predict: ell___oenrogieegro_nregroo_goegwreglo_gio_r
Epoch:   67 / 300 Cost: 2.426006 Predict: ell___oenrogieegro_nregroo_goegwreglo_gio_r
Epoch:   68 / 300 Cost: 2.413675 Predict: ell___oenrogieegro_nregroo_goegwreglo_gio_r
Epoch:   69 / 300 Cost: 2.401151 Predict: ell__goenrogieegro_nregro__goegwoeglo_gio_r
Epoch:   70 / 300 Cost: 2.388368 Predict: ell__goenrogieegro_nregro__goegwoeglo_gio_n
Epoch:   71 / 300 Cost: 2.375256 Predict: ell__goenrogiee_ro_nregro__goegwoeglo_gio_n
Epoch:   72 / 300 Cost: 2.361747 Predict: ell__goenr_giee_ro_nregro__goegwoeglo_gio_n
Epoch:   73 / 300 Cost: 2.347782 Predict: ell__goenr_giee_ro_nregro__goegwoeglo_gio_n
Epoch:   74 / 300 Cost: 2.333322 Predict: ell__goenr_gioe__o_nregro__goegwoeglo_gio_n
Epoch:   75 / 300 Cost: 2.318362 Predict: ell__goenr_gioe__o_nregro__goegwoeglo_gio_n
Epoch:   76 / 300 Cost: 2.302939 Predict: ell__gienr_giol_nornregro__gdeuwoeglo_gio_n
Epoch:   77 / 300 Cost: 2.287112 Predict: ell__gienr_giol_nornregro__gdeuwoello_gio_n
Epoch:   78 / 300 Cost: 2.270928 Predict: ell__gienr_giol_nornregro__gdeuwoello_gio_n
Epoch:   79 / 300 Cost: 2.254416 Predict: ell__gielr_giol_nornregro__gdeuwoello_gio_n
Epoch:   80 / 300 Cost: 2.237600 Predict: ell__gielr_giol_nornregro__gdeuwoello_gio_n
Epoch:   81 / 300 Cost: 2.220518 Predict: ell__giel__giol_nornregro__gdeuwoello_gio_n
Epoch:   82 / 300 Cost: 2.203216 Predict: ell__giel__giol_nornrngro__gdeuwoello_gio_n
Epoch:   83 / 300 Cost: 2.185748 Predict: ell__giel__giol_nornrngro__gdeuwoello_gio_n
Epoch:   84 / 300 Cost: 2.168167 Predict: ell__giel__giol_nornrngro__gdeuwoello_gio_n
Epoch:   85 / 300 Cost: 2.150524 Predict: ell__giel__giol_nornrngro__gdeu_oello_gio_n
Epoch:   86 / 300 Cost: 2.132867 Predict: ell__giel__giol_nornrngro__gdeu_oello_gio_n
Epoch:   87 / 300 Cost: 2.115398 Predict: ell__goel__giol__ornregro___deu_oello_gie_n
Epoch:   88 / 300 Cost: 2.102943 Predict: ell__iirl__girl_nirnmngro__giol_mello_niiln
Epoch:   89 / 300 Cost: 2.166343 Predict: ell__goel__gioe_ro_noegro___deuwoello_goe_n
Epoch:   90 / 300 Cost: 2.126815 Predict: ell__iiel__giel__ornoegr___ghoutoello_gie_n
Epoch:   91 / 300 Cost: 2.088155 Predict: ell__iiel__giol_gornregroe__houthello_gio_n
Epoch:   92 / 300 Cost: 2.064417 Predict: ell__iorl__giel__ornongro___heutoello_iieln
Epoch:   93 / 300 Cost: 2.061842 Predict: ell__girl__giol_gornrngro___houthello_gioan
Epoch:   94 / 300 Cost: 2.042830 Predict: ell__iorl__giel_nornongro___heutoello_iieln
Epoch:   95 / 300 Cost: 2.040092 Predict: ell__iorl__giol_gornrngro___houthello_gio_n
Epoch:   96 / 300 Cost: 2.008363 Predict: ell__iorl__giol__ornongro___heuthello_gie_n
Epoch:   97 / 300 Cost: 1.998788 Predict: ell__iorl__giol_gornmnord__gioutrello_gie_n
Epoch:   98 / 300 Cost: 1.995706 Predict: ell__goel__giol__ornongro___heuthello_gie_n
Epoch:   99 / 300 Cost: 1.990492 Predict: ell__iorl__giol_gornmnors___houthello_gie_n
Epoch:  100 / 300 Cost: 1.974005 Predict: ell__goel__giol__ornongro___heuthello_gie_n
Epoch:  101 / 300 Cost: 1.967864 Predict: ell__iorl__giol_mornmnors___houthello_gie_n
Epoch:  102 / 300 Cost: 1.958650 Predict: ell__goel__giol__ornongro___heutoello_goe_t
Epoch:  103 / 300 Cost: 1.949437 Predict: ell__iorl__giol_morninors___houthello_gie_n
Epoch:  104 / 300 Cost: 1.931423 Predict: ell__gorl__giol__ornongro___houthello_gie_t
Epoch:  105 / 300 Cost: 1.922591 Predict: ell__iorl__giol_morninors___houthello_gie_n
Epoch:  106 / 300 Cost: 1.911846 Predict: ell__gorl__giol__ornongro___houthello_gie_t
Epoch:  107 / 300 Cost: 1.908348 Predict: ell__gorl__giol_gorninors___houthello_gie_t
Epoch:  108 / 300 Cost: 1.891867 Predict: ello_worl__giel_morningro___houthello_gie_t
Epoch:  109 / 300 Cost: 1.885497 Predict: ello_gorl__giol_rorninorso__houthello_gie_t
Epoch:  110 / 300 Cost: 1.874098 Predict: ello_woel__giel_morninors___youthello_gie_t
Epoch:  111 / 300 Cost: 1.871541 Predict: ello_gorl__giol_rorningrso__houthello_gieat
Epoch:  112 / 300 Cost: 1.863660 Predict: ello_goel__goel_gorninord___you_hello_gie_t
Epoch:  113 / 300 Cost: 1.866437 Predict: ello_worl__giol_morningrso__houthello_gieat
Epoch:  114 / 300 Cost: 1.843270 Predict: ello_gorl__goel_gorninors___you_hello_gie_t
Epoch:  115 / 300 Cost: 1.834836 Predict: ello_worl__giol_morninorso__houthello_gieat
Epoch:  116 / 300 Cost: 1.827563 Predict: ello_goel__goel_gorningrs___you_hello_goe_t
Epoch:  117 / 300 Cost: 1.835135 Predict: ello_worl__giol_morninorso__houthello_gieat
Epoch:  118 / 300 Cost: 1.811498 Predict: ello_worl__goel_morningrse__hou_hello_goeat
Epoch:  119 / 300 Cost: 1.805474 Predict: ello_worl__gool_morninorso__houthello_gieat
Epoch:  120 / 300 Cost: 1.796636 Predict: ello_woel__goel_morning_s___you_hello_goeat
Epoch:  121 / 300 Cost: 1.797644 Predict: ello_worl__gool_morninorso__houthello_gieat
Epoch:  122 / 300 Cost: 1.784358 Predict: ello_woel__goel_morninors___yol_hello_gie_t
Epoch:  123 / 300 Cost: 1.797981 Predict: ello_worl__giol_morningrso__hou_hello_gieat
Epoch:  124 / 300 Cost: 1.768598 Predict: ello_gorl__goed_morninorso__you_hello_gieat
Epoch:  125 / 300 Cost: 1.761646 Predict: ello_worl__gool_morningrsoe_houthello_gieat
Epoch:  126 / 300 Cost: 1.758162 Predict: ello_goel__goel_morning_s_e_you_hello_goeat
Epoch:  127 / 300 Cost: 1.777731 Predict: ello_worl__goel_morninorso__houthello_gieat
Epoch:  128 / 300 Cost: 1.741283 Predict: ello_worl__goed_morningrso__hou_hello_gieat
Epoch:  129 / 300 Cost: 1.732208 Predict: ello_worl__good_morninorsoe_houthello_gieat
Epoch:  130 / 300 Cost: 1.731410 Predict: ello_woel__goel_morning_s___yol_hello_goeat
Epoch:  131 / 300 Cost: 1.754478 Predict: ello_worl__gool_morningrso__hou_hello_gieat
Epoch:  132 / 300 Cost: 1.716942 Predict: ello_gorl__goed_morninorsoe_you_hello_gieat
Epoch:  133 / 300 Cost: 1.711673 Predict: ello_world_gool_morningrsoe_houthello_gieat
Epoch:  134 / 300 Cost: 1.711298 Predict: ello_gorl__goed_morning_s_e_you_hello_goeat
Epoch:  135 / 300 Cost: 1.722293 Predict: ello_worl__good_morningrsoe_houthello_gieat
Epoch:  136 / 300 Cost: 1.699834 Predict: ello_worl__goed_morning_s___yeu_hello_goeat
Epoch:  137 / 300 Cost: 1.714714 Predict: ello_gorl__good_morninorsoe_hol_hello_gieat
Epoch:  138 / 300 Cost: 1.685810 Predict: ello_world_goed_morningrso__hou_hello_gieat
Epoch:  139 / 300 Cost: 1.678744 Predict: ello_gorl__good_morninorsoe_hou_hello_gieat
Epoch:  140 / 300 Cost: 1.675250 Predict: ello_world_goed_morningrsoe_you_hello_gieat
Epoch:  141 / 300 Cost: 1.673756 Predict: ello_gorld_good_morninorsoe_hou_hello_gieat
Epoch:  142 / 300 Cost: 1.668467 Predict: ello_gorld_goed_morning_soe_yol_hello_gieat
Epoch:  143 / 300 Cost: 1.688149 Predict: ello_world_giod_morningrso__hou_hello_gieat
Epoch:  144 / 300 Cost: 1.668076 Predict: ello_gorl__good_morning_soe_yol_hello_gieat
Epoch:  145 / 300 Cost: 1.668572 Predict: ello_world_good_morningrsoe_hou_hello_wieat
Epoch:  146 / 300 Cost: 1.652323 Predict: ello_gorld_goed_morning_soe_you_hello_gieat
Epoch:  147 / 300 Cost: 1.650169 Predict: ello_gorld_good_morningrsoe_hou_hello_gieat
Epoch:  148 / 300 Cost: 1.640975 Predict: ello_world_goed_morning_soe_you_hello_gieat
Epoch:  149 / 300 Cost: 1.645409 Predict: ello_gorld_good_morningrsoe_you_hello_gieat
Epoch:  150 / 300 Cost: 1.628424 Predict: ello_gorld_good_morningrsoe_you_hello_gieat
Epoch:  151 / 300 Cost: 1.625416 Predict: ello_world_good_morningrsoe_you_hello_gieat
Epoch:  152 / 300 Cost: 1.621414 Predict: ello_gorld_goed_morning_see_yol_hello_gieat
Epoch:  153 / 300 Cost: 1.626111 Predict: ello_gorld_good_morningrsoe_you_hello_gieat
Epoch:  154 / 300 Cost: 1.613229 Predict: ello_world_goed_morning_soe_yol_hello_gieat
Epoch:  155 / 300 Cost: 1.616034 Predict: ello_gorld_good_morningrsoe_you_hello_gieat
Epoch:  156 / 300 Cost: 1.604866 Predict: ello_world_goed_morning_see_yol_hello_gieat
Epoch:  157 / 300 Cost: 1.609048 Predict: ello_gorld_good_morningrsoe_you_hello_gieat
Epoch:  158 / 300 Cost: 1.596039 Predict: ello_world_goed_morning_soe_yol_hello_gieat
Epoch:  159 / 300 Cost: 1.597261 Predict: ello_gorld_good_morningrsoe_you_hello_gieat
Epoch:  160 / 300 Cost: 1.589508 Predict: ello_world_goed_morning_se__yel_hello_gieat
Epoch:  161 / 300 Cost: 1.599150 Predict: ello_gorld_good_morningrsoe_you_hello_gieat
Epoch:  162 / 300 Cost: 1.585626 Predict: ello_world_goed_morningrsoelyou_hello_wieat
Epoch:  163 / 300 Cost: 1.582433 Predict: ello_gorld_good_morningrsoe_you_hello_gieat
Epoch:  164 / 300 Cost: 1.588420 Predict: ello_world_giod_morningrsoe_you_hello_wieat
Epoch:  165 / 300 Cost: 1.578786 Predict: ello_gorld_goed_morning_soe_you_hello_gieat
Epoch:  166 / 300 Cost: 1.579695 Predict: ello_gorld_goed_morningrsoe_you_hello_gieat
Epoch:  167 / 300 Cost: 1.565691 Predict: ello_gorld_goed_morning_soe_you_hello_gieat
Epoch:  168 / 300 Cost: 1.562391 Predict: ello_gorld_good_morningrsoe_you_hello_gieat
Epoch:  169 / 300 Cost: 1.561003 Predict: ello_world_gied_morningisee_yol_hello_gieat
Epoch:  170 / 300 Cost: 1.585567 Predict: ello_gorld_good_morning_soe_you_hello_gieat
Epoch:  171 / 300 Cost: 1.553437 Predict: ello_gorld_goed_morning_soe_you_hello_gieat
Epoch:  172 / 300 Cost: 1.548551 Predict: ello_gorld_giod_morningrsoe_you_hello_gieat
Epoch:  173 / 300 Cost: 1.546399 Predict: ello_world_goed_morning_se__yol_hello_gieat
Epoch:  174 / 300 Cost: 1.552279 Predict: ello_gorld_good_morningrsoe_you_hello_gieat
Epoch:  175 / 300 Cost: 1.545982 Predict: ello_world_goed_morningrsoe_you_hello_gieat
Epoch:  176 / 300 Cost: 1.538199 Predict: ello_gorld_good_morning_see_you_hello_gieat
Epoch:  177 / 300 Cost: 1.537025 Predict: ello_gorld_good_morningrsoelyou_hello_gieat
Epoch:  178 / 300 Cost: 1.539675 Predict: ello_gorld_gied_morning_see_yol_hello_gieat
Epoch:  179 / 300 Cost: 1.576663 Predict: ello_gorld_giod_morning_see_you_hello_gieat
Epoch:  180 / 300 Cost: 1.532155 Predict: ello_gorld_giod_morning_see_you_hello_gieat
Epoch:  181 / 300 Cost: 1.524811 Predict: ello_gorld_good_morning_see_you_hello_gieat
Epoch:  182 / 300 Cost: 1.521078 Predict: ello_gorld_good_morning_see_you_hello_gieat
Epoch:  183 / 300 Cost: 1.519551 Predict: ello_gorld_giod_morningrsee_you_hello_gieat
Epoch:  184 / 300 Cost: 1.520990 Predict: ello_gorld_good_morning_see_you_hello_gieat
Epoch:  185 / 300 Cost: 1.518450 Predict: ello_world_goed_morningise__yel_hello_wieat
Epoch:  186 / 300 Cost: 1.552299 Predict: ello_gorld_good_morning_soe_you_hello_gieat
Epoch:  187 / 300 Cost: 1.528664 Predict: ello_world_goed_morning_soelyou_hello_gieat
Epoch:  188 / 300 Cost: 1.511559 Predict: ello_gorld_good_morning_see_you_hello_gieat
Epoch:  189 / 300 Cost: 1.507064 Predict: ello_gorld_good_morningrsoe_you_hello_gieat
Epoch:  190 / 300 Cost: 1.505124 Predict: ello_gorld_good_morning_see_you_hello_gieat
Epoch:  191 / 300 Cost: 1.512019 Predict: ello_gorld_good_morningrseelyou_hello_gieat
Epoch:  192 / 300 Cost: 1.508105 Predict: ello_gorld_goed_morning_see_you_hello_gieat
Epoch:  193 / 300 Cost: 1.535719 Predict: ello_gorld_good_morning_see_you_hello_gieat
Epoch:  194 / 300 Cost: 1.502756 Predict: ello_gorld_good_morning_see_you_hello_gieat
Epoch:  195 / 300 Cost: 1.495261 Predict: ello_gorld_grod_morningrsee_you_hello_gieat
Epoch:  196 / 300 Cost: 1.494510 Predict: ello_gorld_good_morning_see_you_hello_gieat
Epoch:  197 / 300 Cost: 1.496581 Predict: ello_gorld_grod_morningrsee_you_hello_gieat
Epoch:  198 / 300 Cost: 1.494745 Predict: ello_world_goed_morning_see_you_hello_gieat
Epoch:  199 / 300 Cost: 1.509108 Predict: ello_gorld_good_morning_see_you_hello_gieat
Epoch:  200 / 300 Cost: 1.486547 Predict: ello_gorld_good_morning_see_you_hello_gieat
Epoch:  201 / 300 Cost: 1.487938 Predict: ello_gorld_good_morningrseelyou_hello_gieat
Epoch:  202 / 300 Cost: 1.489174 Predict: ello_gorld_goed_morning_see_yol_hello_gieat
Epoch:  203 / 300 Cost: 1.519983 Predict: ello_gorld_good_morning_see_you_hello_gieat
Epoch:  204 / 300 Cost: 1.488488 Predict: ello_gorld_grod_morningrsee_you_hello_gieat
Epoch:  205 / 300 Cost: 1.488149 Predict: ello_gorld_grod_morning_see_you_hello_gieat
Epoch:  206 / 300 Cost: 1.478021 Predict: ello_gorld_grod_morningrsee_you_hello_gieat
Epoch:  207 / 300 Cost: 1.476519 Predict: ello_gorld_good_morning_see_you_hello_gieat
Epoch:  208 / 300 Cost: 1.484544 Predict: ello_gorld_grod_morning_see_you_hello_gieat
Epoch:  209 / 300 Cost: 1.472679 Predict: ello_gorld_good_morning_see_you_hello_gieat
Epoch:  210 / 300 Cost: 1.477017 Predict: ello_gorld_grod_morning_see_you_hello_gieat
Epoch:  211 / 300 Cost: 1.469321 Predict: ello_gorld_good_morning_see_you_hello_gieat
Epoch:  212 / 300 Cost: 1.475511 Predict: ello_gorld_grod_morning_see_you_hello_gieat
Epoch:  213 / 300 Cost: 1.464507 Predict: ello_gorld_good_morning_see_you_hello_gieat
Epoch:  214 / 300 Cost: 1.468169 Predict: ello_gorld_grod_morning_seelyou_hello_gieat
Epoch:  215 / 300 Cost: 1.475151 Predict: ello_gorld_good_morning_se__you_hello_gieat
Epoch:  216 / 300 Cost: 1.505914 Predict: ello_gorld_good_morning_see_you_hello_gieat
Epoch:  217 / 300 Cost: 1.477022 Predict: ello_world_good_morning_see_you_hello_wieat
Epoch:  218 / 300 Cost: 1.468951 Predict: ello_gorld_grod_morningrsee_you_mello_gieat
Epoch:  219 / 300 Cost: 1.482975 Predict: ello_world_grod_morning_see_you_hello_wieat
Epoch:  220 / 300 Cost: 1.476871 Predict: ello_gorld_good_morning_see_you_hello_gieat
Epoch:  221 / 300 Cost: 1.456326 Predict: ello_gorld_good_morning_see_you_hello_gieat
Epoch:  222 / 300 Cost: 1.454953 Predict: ello_gorld_grod_morning_see_you_hello_gieat
Epoch:  223 / 300 Cost: 1.452251 Predict: ello_gorld_good_morning_see_you_hello_gieat
Epoch:  224 / 300 Cost: 1.453502 Predict: ello_gorld_grod_morning_see_you_hello_gieat
Epoch:  225 / 300 Cost: 1.450310 Predict: ello_gorld_good_morning_see_you_hello_gieat
Epoch:  226 / 300 Cost: 1.453303 Predict: ello_gorld_grod_morningrsee_you_hello_great
Epoch:  227 / 300 Cost: 1.450397 Predict: ello_gorld_good_morning_see_you_hello_gieat
Epoch:  228 / 300 Cost: 1.452921 Predict: ello_gorld_grod_morningrsee_you_hello_great
Epoch:  229 / 300 Cost: 1.451674 Predict: ello_gorld_good_morning_see_you_hello_gieat
Epoch:  230 / 300 Cost: 1.457561 Predict: ello_gorld_good_morningrseelyou_hello_great
Epoch:  231 / 300 Cost: 1.454077 Predict: ello_gorld_good_morning_see_you_hello_gieat
Epoch:  232 / 300 Cost: 1.454745 Predict: ello_gorld_good_morning_see_you_hello_gieat
Epoch:  233 / 300 Cost: 1.443933 Predict: ello_gorld_grod_morningrsee_you_hello_gieat
Epoch:  234 / 300 Cost: 1.452868 Predict: ello_gorld_grod_morningrsee_you_hello_great
Epoch:  235 / 300 Cost: 1.442820 Predict: ello_gorld_grod_morning_see_you_hello_great
Epoch:  236 / 300 Cost: 1.442411 Predict: ello_world_grod_morningrsel_you_hello_wreat
Epoch:  237 / 300 Cost: 1.455821 Predict: ello_gorld_good_morning_see_you_hello_great
Epoch:  238 / 300 Cost: 1.446138 Predict: ello_gorld_good_morning_see_you_hello_gieat
Epoch:  239 / 300 Cost: 1.440606 Predict: ello_gorld_grod_morningrsee_you_hello_great
Epoch:  240 / 300 Cost: 1.433103 Predict: ello_gorld_good_morning_see_you_hello_great
Epoch:  241 / 300 Cost: 1.431352 Predict: ello_gorld_grod_morningrsee_you_hello_great
Epoch:  242 / 300 Cost: 1.429746 Predict: ello_gorld_good_morning_see_you_hello_great
Epoch:  243 / 300 Cost: 1.429327 Predict: ello_gorld_grod_morningrsee_you_hello_great
Epoch:  244 / 300 Cost: 1.428384 Predict: ello_gorld_good_morning_see_you_hello_great
Epoch:  245 / 300 Cost: 1.429590 Predict: ello_gorld_grod_morningrsee_you_hello_great
Epoch:  246 / 300 Cost: 1.428630 Predict: ello_gorld_good_morning_see_you_hello_great
Epoch:  247 / 300 Cost: 1.429975 Predict: ello_gorld_grod_morningrsee_you_hello_great
Epoch:  248 / 300 Cost: 1.432593 Predict: ello_world_good_morning_see_you_hello_great
Epoch:  249 / 300 Cost: 1.431821 Predict: ello_gorld_good_morning_soelyou_hello_great
Epoch:  250 / 300 Cost: 1.435976 Predict: ello_gorld_good_morningrsee_you_hello_great
Epoch:  251 / 300 Cost: 1.425890 Predict: ello_gorld_good_morningrsee_you_hello_great
Epoch:  252 / 300 Cost: 1.420913 Predict: ello_gorld_good_morning_see_you_hello_great
Epoch:  253 / 300 Cost: 1.422682 Predict: ello_world_good_morningrseelyou_hello_wreat
Epoch:  254 / 300 Cost: 1.429956 Predict: ello_gorld_good_morning_see_you_hello_great
Epoch:  255 / 300 Cost: 1.427094 Predict: ello_world_good_morningrsee_you_hello_great
Epoch:  256 / 300 Cost: 1.417549 Predict: ello_gorld_good_morning_see_you_hello_great
Epoch:  257 / 300 Cost: 1.416286 Predict: ello_gorld_good_morningrsee_you_hello_great
Epoch:  258 / 300 Cost: 1.415319 Predict: ello_gorld_good_morning_see_you_hello_great
Epoch:  259 / 300 Cost: 1.414380 Predict: ello_gorld_good_morning_see_you_hello_great
Epoch:  260 / 300 Cost: 1.414619 Predict: ello_gorld_good_morning_see_you_hello_great
Epoch:  261 / 300 Cost: 1.419433 Predict: ello_world_good_morning_see_you_hello_wreat
Epoch:  262 / 300 Cost: 1.427747 Predict: ello_gorld_good_morning_see_you_hello_great
Epoch:  263 / 300 Cost: 1.420115 Predict: ello_gorld_good_morning_see_you_hello_great
Epoch:  264 / 300 Cost: 1.412811 Predict: ello_gorld_good_morningrsee_you_hello_great
Epoch:  265 / 300 Cost: 1.408913 Predict: ello_gorld_good_morning_see_you_hello_great
Epoch:  266 / 300 Cost: 1.410859 Predict: ello_world_good_morningrseelyou_hello_wreat
Epoch:  267 / 300 Cost: 1.412407 Predict: ello_gorld_grod_morning_see_you_hello_great
Epoch:  268 / 300 Cost: 1.410043 Predict: ello_gorld_good_morningrsee_you_hello_great
Epoch:  269 / 300 Cost: 1.403924 Predict: ello_gorld_good_morning_see_you_hello_great
Epoch:  270 / 300 Cost: 1.403058 Predict: ello_gorld_good_morningrsee_you_hello_great
Epoch:  271 / 300 Cost: 1.402425 Predict: ello_gorld_good_morning_see_you_hello_great
Epoch:  272 / 300 Cost: 1.401601 Predict: ello_gorld_good_morningrsee_you_hello_great
Epoch:  273 / 300 Cost: 1.401495 Predict: ello_gorld_good_morning_see_you_hello_great
Epoch:  274 / 300 Cost: 1.401267 Predict: ello_gorld_grod_morningrsee_you_hello_great
Epoch:  275 / 300 Cost: 1.404285 Predict: ello_gorld_good_morning_see_you_hello_great
Epoch:  276 / 300 Cost: 1.406441 Predict: ello_gorld_grod_gorningrsee_you_mello_great
Epoch:  277 / 300 Cost: 1.432289 Predict: ello_world_grod_morningioel_you_hello_wreat
Epoch:  278 / 300 Cost: 1.455354 Predict: ello_gorld_good_morning_see_you_hello_great
Epoch:  279 / 300 Cost: 1.417502 Predict: ello_world_good_morning_seelyou_hello_wreat
Epoch:  280 / 300 Cost: 1.406462 Predict: ello_gorld_good_morning_see_you_hello_great
Epoch:  281 / 300 Cost: 1.401947 Predict: ello_gorld_good_morning_see_you_hello_great
Epoch:  282 / 300 Cost: 1.398957 Predict: ello_gorld_good_morning_see_you_hello_great
Epoch:  283 / 300 Cost: 1.398376 Predict: ello_gorld_good_morningrsee_you_hello_great
Epoch:  284 / 300 Cost: 1.395116 Predict: ello_gorld_good_morning_see_you_hello_great
Epoch:  285 / 300 Cost: 1.395139 Predict: ello_gorld_good_morningrsee_you_hello_great
Epoch:  286 / 300 Cost: 1.392765 Predict: ello_gorld_good_morning_see_you_hello_great
Epoch:  287 / 300 Cost: 1.393023 Predict: ello_gorld_good_morningrsee_you_hello_great
Epoch:  288 / 300 Cost: 1.391586 Predict: ello_gorld_good_morning_see_you_hello_great
Epoch:  289 / 300 Cost: 1.391969 Predict: ello_gorld_good_morningrsee_you_hello_great
Epoch:  290 / 300 Cost: 1.390326 Predict: ello_gorld_good_morning_see_you_hello_great
Epoch:  291 / 300 Cost: 1.390108 Predict: ello_gorld_good_morningrsee_you_hello_great
Epoch:  292 / 300 Cost: 1.388595 Predict: ello_gorld_good_morning_see_you_hello_great
Epoch:  293 / 300 Cost: 1.388129 Predict: ello_gorld_good_morningrsee_you_hello_great
Epoch:  294 / 300 Cost: 1.387080 Predict: ello_gorld_good_morning_see_you_hello_great
Epoch:  295 / 300 Cost: 1.386714 Predict: ello_gorld_good_morningrsee_you_hello_great
Epoch:  296 / 300 Cost: 1.385922 Predict: ello_gorld_grod_morning_see_you_hello_great
Epoch:  297 / 300 Cost: 1.385547 Predict: ello_gorld_good_morningrsee_you_hello_great
Epoch:  298 / 300 Cost: 1.384636 Predict: ello_gorld_grod_morning_see_you_hello_great
Epoch:  299 / 300 Cost: 1.384199 Predict: ello_gorld_good_morningrsee_you_hello_great
Epoch:  300 / 300 Cost: 1.382960 Predict: ello_gorld_grod_morning_see_you_hello_great