In [1]:

    

import os
import re
import string
import requests
import numpy as np
import collections
import random
import pickle
import matplotlib.pyplot as plt
import tensorflow as tf


    

In [2]:

    

sess = tf.Session()

min_word_freq = 5
rnn_size = 128
epochs = 10
batch_size = 100
learning_rate = 0.001
training_seq_len = 50
embedding_size = rnn_size
save_every = 500
eval_every = 50

prime_texts = ['thou art more', 'to be or not to', 'wherefore art thou']


    

In [3]:

    

#テキスト　クレンジング
data_dir = 'temp'
data_file = 'shakespeare.txt'
model_path = 'shakespeare_model'
full_model_dir = os.path.join(data_dir, model_path)

# ハイフンとアポストロフィ以外の句読点を削除
punctuation = string.punctuation
punctuation = ''.join([x for x in punctuation if x not in ['-', "'"]])


    

In [4]:

    

# モデルフォルダを作成
if not os.path.exists(full_model_dir):
    os.makedirs(full_model_dir)
    
# データフォルダを作成
if not os.path.exists(data_dir):
    os.makedirs(data_dir)
    
print("Loding Shakespeare data.")
if not os.path.isfile(os.path.join(data_dir, data_file)):
    print('Not found, downloading Shakespeare texts from www.gutenberg.org')
    
    shakespeare_url = 'http://www.gutenberg.org/cache/epub/100/pg100.txt'
    response = requests.get(shakespeare_url)
    shakespeare_file = response.content
    s_text = shakespeare_file.decode('utf-8')
    s_text = s_text[7675:]
    s_text = s_text.replace('\r\n', '')
    s_text = s_text.replace('\n', '')
    
    with open(os.path.join(data_dir, data_file), 'w') as out_conn:
        out_conn.write(s_text)
        
else:
    with open(os.path.join(data_dir, data_file), 'r') as file_conn:
        s_text = file_conn.read().replace('\n', '')


    

    




Loding Shakespeare data.

In [5]:

    

s_text = re.sub(r'[{}]'.format(punctuation), ' ', s_text)
s_text = re.sub('\s+', ' ', s_text).strip().lower()


    

In [6]:

    

# 語彙の辞書を作成する
def build_vocab(text, min_word_freq):
    word_counts = collections.Counter(text.split(' '))
    word_counts = {key:val for key, val in word_counts.items() if val > min_word_freq}
    
    words = word_counts.keys()
    vocab_to_ix_dict = {key:(ix+1) for ix, key in enumerate(words)}
    vocab_to_ix_dict['unknown'] = 0
    
    ix_to_vocav_dict = {val:key for key, val in vocab_to_ix_dict.items()}
    
    return (ix_to_vocav_dict, vocab_to_ix_dict)

ix2vocab, vocab2ix = build_vocab(s_text, min_word_freq)
vocab_size = len(ix2vocab) + 1


    

In [7]:

    

# テキストデータを単語ベクトルに変換する
s_text_words = s_text.split(' ')
s_text_ix = []

for ix, x in enumerate(s_text_words):
    try:
        s_text_ix.append(vocab2ix[x])
    except:
        s_text_ix.append(0)
        
s_text_ix = np.array(s_text_ix)


    

In [8]:

    

class LSTM_Model():
    
    def __init__(self, rnn_size, batch_size, learning_rate, seq_len, vocab_size, infer_sample=False):
        self.rnn_size = rnn_size
        self.vocab_size = vocab_size
        self.infer_sample = infer_sample
        self.learning_rate = learning_rate
        
        if infer_sample:
            self.batch_size = 1
            self.seq_len = 1
        else:
            self.batch_size = batch_size
            self.seq_len = seq_len
            
        self.lstm_cell = tf.contrib.rnn.BasicLSTMCell(rnn_size)
        self.initial_state = self.lstm_cell.zero_state(self.batch_size, tf.float32)
        
        self.x_data = tf.placeholder(tf.int32, [self.batch_size, self.seq_len])
        self.y_output = tf.placeholder(tf.int32, [self.batch_size, self.seq_len])
        
        with tf.variable_scope('lstm_vars'):
            W = tf.get_variable('W', [self.rnn_size, self.vocab_size],
                                tf.float32, tf.random_normal_initializer())
            b = tf.get_variable('b', [self.vocab_size],
                                tf.float32, tf.constant_initializer(0.0))
            
            embedding_mat = tf.get_variable('embedding_mat', [self.vocab_size, self.rnn_size],
                                            tf.float32,
                                            tf.random_normal_initializer())
            
            embedding_output = tf.nn.embedding_lookup(embedding_mat, self.x_data)
            
            rnn_inputs = tf.split(axis=1, num_or_size_splits=self.seq_len,
                                  value=embedding_output)
            rnn_inputs_trimmed = [tf.squeeze(x, [1]) for x in rnn_inputs]
            
        # テキスト生成中はループ関数を追加
        # i番目の出力から i+1 番目の入力を取得する方法を定義
        def inferred_loop(prev, count):
            # 隠れそう
            prev_transformed = tf.matmul(prev, W) + b
            prev_symbol = tf.stop_gradient(tf.argmax(prev_transformed, 1))
            output = tf.nn.embedding_lookup(embedding_mat, prev_symbol)
            return (output)
        
        decoder = tf.contrib.legacy_seq2seq.rnn_decoder
        outputs, last_state = decoder(
            rnn_inputs_trimmed,
            self.initial_state,
            self.lstm_cell,
            loop_function=inferred_loop if infer_sample else None)
        
        # 推測されていない出力
        output = tf.reshape(tf.concat(axis=1, values=outputs), [-1, self.rnn_size])
        
        # ロジットと出力
        self.logit_output = tf.matmul(output, W) + b
        self.model_output = tf.nn.softmax(self.logit_output)
        
        # 損失関数
        loss_func = tf.contrib.legacy_seq2seq.sequence_loss_by_example
        loss = loss_func(
            [self.logit_output],
            [tf.reshape(self.y_output, [-1])],
            [tf.ones([self.batch_size * self.seq_len])],
            self.vocab_size)
        
        self.cost = tf.reduce_sum(loss) / (self.batch_size * self.seq_len)
        self.final_state = last_state
        gradients, _ = tf.clip_by_global_norm(tf.gradients(self.cost, tf.trainable_variables()), 4.5)
        optimizer = tf.train.AdamOptimizer(self.learning_rate)
        self.train_op = optimizer.apply_gradients(zip(gradients, tf.trainable_variables()))
        
    def sample(self, sess, words=ix2vocab, vocab=vocab2ix, num=10, prime_text='thou art'):
        state = sess.run(self.lstm_cell.zero_state(1, tf.float32))
        word_list = prime_text.split()
        
        for word in word_list[:-1]:
            x = np.zeros((1, 1))
            x[0, 0] = vocab[word]
            feed_dict = {self.x_data: x, self.initial_state: state}
            [state] = sess.run([self.final_state], feed_dict=feed_dict)
            
        out_sentence = prime_text
        word = word_list[-1]
        for n in range(num):
            x = np.zeros((1, 1))
            x[0, 0] = vocab[word]
            feed_dict = {self.x_data: x, self.initial_state: state}
            [model_output, state] = sess.run(
                [self.model_output, self.final_state], feed_dict=feed_dict)
            sample = np.argmax(model_output[0])
            
            if sample == 0:
                break
                
            word = words[sample]
            out_sentence = out_sentence + ' ' + word
            
        return (out_sentence)


    

In [9]:

    

lstm_model = LSTM_Model(rnn_size=rnn_size, batch_size=batch_size, learning_rate=learning_rate, seq_len=training_seq_len, vocab_size=vocab_size, infer_sample=False)

with tf.variable_scope(tf.get_variable_scope(), reuse=True):
    test_lstm_model = LSTM_Model(rnn_size=rnn_size, batch_size=batch_size, learning_rate=learning_rate,
                        seq_len=training_seq_len, vocab_size=vocab_size, infer_sample=True)


    

In [10]:

    

saver = tf.train.Saver(tf.global_variables())

num_batches = int(len(s_text_ix) / (batch_size * training_seq_len)) + 1

batches = np.array_split(s_text_ix, num_batches)
batches = [np.resize(x, [batch_size, training_seq_len]) for x in batches]

# 全ての変数を初期化
init = tf.global_variables_initializer()
sess.run(init)


    

In [11]:

    

train_loss = []
iteration_count = 1

for epoch in range(epochs):
    random.shuffle(batches)
    targets = [np.roll(x, -1, axis=1) for x in batches]
    print('Starting Epoch {}/{}...'.format(epoch+1, epochs))
    
    state = sess.run(lstm_model.initial_state)
    for ix, batch in enumerate(batches):
        training_dict = {lstm_model.x_data: batch,
                         lstm_model.y_output: targets[ix]}
        c, h = lstm_model.initial_state
        training_dict[c] = state.c
        training_dict[h] = state.h
        
        temp_loss, state, _ = sess.run(
            [lstm_model.cost, lstm_model.final_state, lstm_model.train_op],
            feed_dict=training_dict)
        train_loss.append(temp_loss)
        
        if iteration_count % 10 == 0:
            summary_nums = (iteration_count, epoch+1, ix+1, num_batches+1, temp_loss)
            print('Iteration: {}, Epoch: {}, Batch: {} out of {}, \
                  Loss: {:.2f}'.format(*summary_nums))
            
        if iteration_count % save_every == 0:
            model_file_name = os.path.join(full_model_dir, 'model')
            saver.save(sess, model_file_name, global_step=iteration_count)
            print('Model saved to: {}'.format(model_file_name))
            
            dictionary_file = os.path.join(full_model_dir, 'vocab.pkl')
            with open(dictionary_file, 'wb') as dict_file_conn:
                pickle.dump([vocab2ix, ix2vocab], dict_file_conn)
                
        if iteration_count % eval_every == 0:
            for sample in prime_texts:
                print(test_lstm_model.sample(sess, ix2vocab, vocab2ix, num=10, prime_text=sample))
                
        iteration_count += 1


    

    




Starting Epoch 1/10...
Iteration: 10, Epoch: 1, Batch: 10 out of 182,                   Loss: 9.90
Iteration: 20, Epoch: 1, Batch: 20 out of 182,                   Loss: 9.26
Iteration: 30, Epoch: 1, Batch: 30 out of 182,                   Loss: 8.73
Iteration: 40, Epoch: 1, Batch: 40 out of 182,                   Loss: 8.39
Iteration: 50, Epoch: 1, Batch: 50 out of 182,                   Loss: 8.07
thou art more think saturn make i have to to the
to be or not to to the
wherefore art thou
Iteration: 60, Epoch: 1, Batch: 60 out of 182,                   Loss: 7.82
Iteration: 70, Epoch: 1, Batch: 70 out of 182,                   Loss: 7.58
Iteration: 80, Epoch: 1, Batch: 80 out of 182,                   Loss: 7.50
Iteration: 90, Epoch: 1, Batch: 90 out of 182,                   Loss: 7.39
Iteration: 100, Epoch: 1, Batch: 100 out of 182,                   Loss: 7.11
thou art more think the
to be or not to the
wherefore art thou hast thou hast to the
Iteration: 110, Epoch: 1, Batch: 110 out of 182,                   Loss: 6.98
Iteration: 120, Epoch: 1, Batch: 120 out of 182,                   Loss: 6.83
Iteration: 130, Epoch: 1, Batch: 130 out of 182,                   Loss: 6.81
Iteration: 140, Epoch: 1, Batch: 140 out of 182,                   Loss: 6.71
Iteration: 150, Epoch: 1, Batch: 150 out of 182,                   Loss: 6.60
thou art more than
to be or not to the
wherefore art thou hast thou hast thou art thou art thou art thou
Iteration: 160, Epoch: 1, Batch: 160 out of 182,                   Loss: 6.59
Iteration: 170, Epoch: 1, Batch: 170 out of 182,                   Loss: 6.30
Iteration: 180, Epoch: 1, Batch: 180 out of 182,                   Loss: 6.51
Starting Epoch 2/10...
Iteration: 190, Epoch: 2, Batch: 9 out of 182,                   Loss: 6.98
Iteration: 200, Epoch: 2, Batch: 19 out of 182,                   Loss: 6.99
thou art more than
to be or not to the
wherefore art thou hast thou hast thou art thou art thou art thou
Iteration: 210, Epoch: 2, Batch: 29 out of 182,                   Loss: 6.57
Iteration: 220, Epoch: 2, Batch: 39 out of 182,                   Loss: 6.48
Iteration: 230, Epoch: 2, Batch: 49 out of 182,                   Loss: 6.31
Iteration: 240, Epoch: 2, Batch: 59 out of 182,                   Loss: 6.45
Iteration: 250, Epoch: 2, Batch: 69 out of 182,                   Loss: 6.47
thou art more than
to be or not to the
wherefore art thou hast thou hast thou hast thou hast thou art thou
Iteration: 260, Epoch: 2, Batch: 79 out of 182,                   Loss: 6.46
Iteration: 270, Epoch: 2, Batch: 89 out of 182,                   Loss: 6.57
Iteration: 280, Epoch: 2, Batch: 99 out of 182,                   Loss: 6.39
Iteration: 290, Epoch: 2, Batch: 109 out of 182,                   Loss: 6.47
Iteration: 300, Epoch: 2, Batch: 119 out of 182,                   Loss: 6.40
thou art more than
to be or not to the
wherefore art thou hast thou hast thou hast thou art thou art thou
Iteration: 310, Epoch: 2, Batch: 129 out of 182,                   Loss: 6.40
Iteration: 320, Epoch: 2, Batch: 139 out of 182,                   Loss: 5.96
Iteration: 330, Epoch: 2, Batch: 149 out of 182,                   Loss: 6.31
Iteration: 340, Epoch: 2, Batch: 159 out of 182,                   Loss: 6.29
Iteration: 350, Epoch: 2, Batch: 169 out of 182,                   Loss: 6.45
thou art more than
to be or not to the
wherefore art thou hast thou hast thou hast thou art thou hast thou
Iteration: 360, Epoch: 2, Batch: 179 out of 182,                   Loss: 6.25
Starting Epoch 3/10...
Iteration: 370, Epoch: 3, Batch: 8 out of 182,                   Loss: 6.13
Iteration: 380, Epoch: 3, Batch: 18 out of 182,                   Loss: 6.16
Iteration: 390, Epoch: 3, Batch: 28 out of 182,                   Loss: 5.98
Iteration: 400, Epoch: 3, Batch: 38 out of 182,                   Loss: 6.27
thou art more than
to be or not to the
wherefore art thou hast thou hast thou hast thou hast thou art thou
Iteration: 410, Epoch: 3, Batch: 48 out of 182,                   Loss: 6.24
Iteration: 420, Epoch: 3, Batch: 58 out of 182,                   Loss: 5.93
Iteration: 430, Epoch: 3, Batch: 68 out of 182,                   Loss: 6.15
Iteration: 440, Epoch: 3, Batch: 78 out of 182,                   Loss: 6.28
Iteration: 450, Epoch: 3, Batch: 88 out of 182,                   Loss: 6.34
thou art more than
to be or not to the
wherefore art thou hast thou hast thou art thou art thou art thou
Iteration: 460, Epoch: 3, Batch: 98 out of 182,                   Loss: 6.03
Iteration: 470, Epoch: 3, Batch: 108 out of 182,                   Loss: 6.25
Iteration: 480, Epoch: 3, Batch: 118 out of 182,                   Loss: 6.22
Iteration: 490, Epoch: 3, Batch: 128 out of 182,                   Loss: 6.26
Iteration: 500, Epoch: 3, Batch: 138 out of 182,                   Loss: 6.31
Model saved to: temp/shakespeare_model/model
thou art more than they are not distributed or usedcommercially prohibited commercial distribution
to be or not to the
wherefore art thou hast thou hast thou hast thou hast thou hast thou
Iteration: 510, Epoch: 3, Batch: 148 out of 182,                   Loss: 6.08
Iteration: 520, Epoch: 3, Batch: 158 out of 182,                   Loss: 6.11
Iteration: 530, Epoch: 3, Batch: 168 out of 182,                   Loss: 6.31
Iteration: 540, Epoch: 3, Batch: 178 out of 182,                   Loss: 6.06
Starting Epoch 4/10...
Iteration: 550, Epoch: 4, Batch: 7 out of 182,                   Loss: 6.32
thou art more than they are not distributed or usedcommercially prohibited commercial distribution
to be or not to the
wherefore art thou hast thou hast thou hast thou hast thou art thou
Iteration: 560, Epoch: 4, Batch: 17 out of 182,                   Loss: 6.26
Iteration: 570, Epoch: 4, Batch: 27 out of 182,                   Loss: 6.20
Iteration: 580, Epoch: 4, Batch: 37 out of 182,                   Loss: 6.24
Iteration: 590, Epoch: 4, Batch: 47 out of 182,                   Loss: 5.83
Iteration: 600, Epoch: 4, Batch: 57 out of 182,                   Loss: 6.16
thou art more than they were
to be or not to the
wherefore art thou hast thou hast thou hast thy thy
Iteration: 610, Epoch: 4, Batch: 67 out of 182,                   Loss: 6.09
Iteration: 620, Epoch: 4, Batch: 77 out of 182,                   Loss: 6.13
Iteration: 630, Epoch: 4, Batch: 87 out of 182,                   Loss: 6.33
Iteration: 640, Epoch: 4, Batch: 97 out of 182,                   Loss: 5.85
Iteration: 650, Epoch: 4, Batch: 107 out of 182,                   Loss: 6.05
thou art more than they are not so much as i am not
to be or not to the
wherefore art thou hast thou hast thy thy
Iteration: 660, Epoch: 4, Batch: 117 out of 182,                   Loss: 6.08
Iteration: 670, Epoch: 4, Batch: 127 out of 182,                   Loss: 6.00
Iteration: 680, Epoch: 4, Batch: 137 out of 182,                   Loss: 6.27
Iteration: 690, Epoch: 4, Batch: 147 out of 182,                   Loss: 5.99
Iteration: 700, Epoch: 4, Batch: 157 out of 182,                   Loss: 6.16
thou art more than
to be or not to the
wherefore art thou hast thou hast thou hast thou hast thou art thou
Iteration: 710, Epoch: 4, Batch: 167 out of 182,                   Loss: 6.27
Iteration: 720, Epoch: 4, Batch: 177 out of 182,                   Loss: 6.07
Starting Epoch 5/10...
Iteration: 730, Epoch: 5, Batch: 6 out of 182,                   Loss: 5.98
Iteration: 740, Epoch: 5, Batch: 16 out of 182,                   Loss: 5.93
Iteration: 750, Epoch: 5, Batch: 26 out of 182,                   Loss: 6.05
thou art more than they are not distributed or usedcommercially prohibited commercial distribution
to be or not to the
wherefore art thou hast thou shalt thy
Iteration: 760, Epoch: 5, Batch: 36 out of 182,                   Loss: 6.03
Iteration: 770, Epoch: 5, Batch: 46 out of 182,                   Loss: 6.01
Iteration: 780, Epoch: 5, Batch: 56 out of 182,                   Loss: 5.79
Iteration: 790, Epoch: 5, Batch: 66 out of 182,                   Loss: 6.10
Iteration: 800, Epoch: 5, Batch: 76 out of 182,                   Loss: 6.00
thou art more than
to be or not to the
wherefore art thou hast thou hast thy
Iteration: 810, Epoch: 5, Batch: 86 out of 182,                   Loss: 5.94
Iteration: 820, Epoch: 5, Batch: 96 out of 182,                   Loss: 6.01
Iteration: 830, Epoch: 5, Batch: 106 out of 182,                   Loss: 6.01
Iteration: 840, Epoch: 5, Batch: 116 out of 182,                   Loss: 5.73
Iteration: 850, Epoch: 5, Batch: 126 out of 182,                   Loss: 5.74
thou art more than they are not so much as i am a
to be or not to the
wherefore art thou hast thou shalt not so
Iteration: 860, Epoch: 5, Batch: 136 out of 182,                   Loss: 6.01
Iteration: 870, Epoch: 5, Batch: 146 out of 182,                   Loss: 6.07
Iteration: 880, Epoch: 5, Batch: 156 out of 182,                   Loss: 5.87
Iteration: 890, Epoch: 5, Batch: 166 out of 182,                   Loss: 6.08
Iteration: 900, Epoch: 5, Batch: 176 out of 182,                   Loss: 5.95
thou art more than
to be or not to the
wherefore art thou shalt not be
Starting Epoch 6/10...
Iteration: 910, Epoch: 6, Batch: 5 out of 182,                   Loss: 5.86
Iteration: 920, Epoch: 6, Batch: 15 out of 182,                   Loss: 6.02
Iteration: 930, Epoch: 6, Batch: 25 out of 182,                   Loss: 6.10
Iteration: 940, Epoch: 6, Batch: 35 out of 182,                   Loss: 6.05
Iteration: 950, Epoch: 6, Batch: 45 out of 182,                   Loss: 6.05
thou art more than
to be or not to the
wherefore art thou hast thou shalt not so
Iteration: 960, Epoch: 6, Batch: 55 out of 182,                   Loss: 5.95
Iteration: 970, Epoch: 6, Batch: 65 out of 182,                   Loss: 5.99
Iteration: 980, Epoch: 6, Batch: 75 out of 182,                   Loss: 5.90
Iteration: 990, Epoch: 6, Batch: 85 out of 182,                   Loss: 5.97
Iteration: 1000, Epoch: 6, Batch: 95 out of 182,                   Loss: 5.83
Model saved to: temp/shakespeare_model/model
thou art more than
to be or not to the
wherefore art thou hast thou hast thou hast thy
Iteration: 1010, Epoch: 6, Batch: 105 out of 182,                   Loss: 5.97
Iteration: 1020, Epoch: 6, Batch: 115 out of 182,                   Loss: 5.72
Iteration: 1030, Epoch: 6, Batch: 125 out of 182,                   Loss: 5.95
Iteration: 1040, Epoch: 6, Batch: 135 out of 182,                   Loss: 6.13
Iteration: 1050, Epoch: 6, Batch: 145 out of 182,                   Loss: 6.02
thou art more than they were
to be or not to the
wherefore art thou shalt not so
Iteration: 1060, Epoch: 6, Batch: 155 out of 182,                   Loss: 5.97
Iteration: 1070, Epoch: 6, Batch: 165 out of 182,                   Loss: 5.89
Iteration: 1080, Epoch: 6, Batch: 175 out of 182,                   Loss: 5.89
Starting Epoch 7/10...
Iteration: 1090, Epoch: 7, Batch: 4 out of 182,                   Loss: 6.08
Iteration: 1100, Epoch: 7, Batch: 14 out of 182,                   Loss: 5.90
thou art more than than they were not in the
to be or not to the
wherefore art thou shalt not so
Iteration: 1110, Epoch: 7, Batch: 24 out of 182,                   Loss: 6.06
Iteration: 1120, Epoch: 7, Batch: 34 out of 182,                   Loss: 5.83
Iteration: 1130, Epoch: 7, Batch: 44 out of 182,                   Loss: 5.90
Iteration: 1140, Epoch: 7, Batch: 54 out of 182,                   Loss: 5.92
Iteration: 1150, Epoch: 7, Batch: 64 out of 182,                   Loss: 5.95
thou art more than
to be or not to the
wherefore art thou hast thou shalt not so
Iteration: 1160, Epoch: 7, Batch: 74 out of 182,                   Loss: 5.88
Iteration: 1170, Epoch: 7, Batch: 84 out of 182,                   Loss: 6.02
Iteration: 1180, Epoch: 7, Batch: 94 out of 182,                   Loss: 5.85
Iteration: 1190, Epoch: 7, Batch: 104 out of 182,                   Loss: 5.97
Iteration: 1200, Epoch: 7, Batch: 114 out of 182,                   Loss: 5.74
thou art more than a
to be or not to the
wherefore art thou shalt not
Iteration: 1210, Epoch: 7, Batch: 124 out of 182,                   Loss: 6.19
Iteration: 1220, Epoch: 7, Batch: 134 out of 182,                   Loss: 5.58
Iteration: 1230, Epoch: 7, Batch: 144 out of 182,                   Loss: 6.05
Iteration: 1240, Epoch: 7, Batch: 154 out of 182,                   Loss: 5.72
Iteration: 1250, Epoch: 7, Batch: 164 out of 182,                   Loss: 5.91
thou art more than a
to be or not to the
wherefore art thou shalt not so
Iteration: 1260, Epoch: 7, Batch: 174 out of 182,                   Loss: 5.88
Starting Epoch 8/10...
Iteration: 1270, Epoch: 8, Batch: 3 out of 182,                   Loss: 5.87
Iteration: 1280, Epoch: 8, Batch: 13 out of 182,                   Loss: 6.02
Iteration: 1290, Epoch: 8, Batch: 23 out of 182,                   Loss: 5.86
Iteration: 1300, Epoch: 8, Batch: 33 out of 182,                   Loss: 5.79
thou art more than
to be or not to the
wherefore art thou shalt not so
Iteration: 1310, Epoch: 8, Batch: 43 out of 182,                   Loss: 5.77
Iteration: 1320, Epoch: 8, Batch: 53 out of 182,                   Loss: 5.81
Iteration: 1330, Epoch: 8, Batch: 63 out of 182,                   Loss: 5.84
Iteration: 1340, Epoch: 8, Batch: 73 out of 182,                   Loss: 6.13
Iteration: 1350, Epoch: 8, Batch: 83 out of 182,                   Loss: 5.91
thou art more than
to be or not to the
wherefore art thou hast thou shalt not so
Iteration: 1360, Epoch: 8, Batch: 93 out of 182,                   Loss: 5.73
Iteration: 1370, Epoch: 8, Batch: 103 out of 182,                   Loss: 5.97
Iteration: 1380, Epoch: 8, Batch: 113 out of 182,                   Loss: 5.66
Iteration: 1390, Epoch: 8, Batch: 123 out of 182,                   Loss: 5.68
Iteration: 1400, Epoch: 8, Batch: 133 out of 182,                   Loss: 5.73
thou art more than
to be or not to the
wherefore art thou shalt not so
Iteration: 1410, Epoch: 8, Batch: 143 out of 182,                   Loss: 5.63
Iteration: 1420, Epoch: 8, Batch: 153 out of 182,                   Loss: 5.82
Iteration: 1430, Epoch: 8, Batch: 163 out of 182,                   Loss: 5.86
Iteration: 1440, Epoch: 8, Batch: 173 out of 182,                   Loss: 5.97
Starting Epoch 9/10...
Iteration: 1450, Epoch: 9, Batch: 2 out of 182,                   Loss: 5.83
thou art more than
to be or not to the
wherefore art thou shalt not be so
Iteration: 1460, Epoch: 9, Batch: 12 out of 182,                   Loss: 5.83
Iteration: 1470, Epoch: 9, Batch: 22 out of 182,                   Loss: 5.54
Iteration: 1480, Epoch: 9, Batch: 32 out of 182,                   Loss: 5.82
Iteration: 1490, Epoch: 9, Batch: 42 out of 182,                   Loss: 5.93
Iteration: 1500, Epoch: 9, Batch: 52 out of 182,                   Loss: 5.80
Model saved to: temp/shakespeare_model/model
thou art more than
to be or not to the
wherefore art thou shalt not be
Iteration: 1510, Epoch: 9, Batch: 62 out of 182,                   Loss: 5.94
Iteration: 1520, Epoch: 9, Batch: 72 out of 182,                   Loss: 5.83
Iteration: 1530, Epoch: 9, Batch: 82 out of 182,                   Loss: 5.78
Iteration: 1540, Epoch: 9, Batch: 92 out of 182,                   Loss: 5.47
Iteration: 1550, Epoch: 9, Batch: 102 out of 182,                   Loss: 5.77
thou art more than a
to be or not to the
wherefore art thou shalt be
Iteration: 1560, Epoch: 9, Batch: 112 out of 182,                   Loss: 5.99
Iteration: 1570, Epoch: 9, Batch: 122 out of 182,                   Loss: 5.87
Iteration: 1580, Epoch: 9, Batch: 132 out of 182,                   Loss: 5.84
Iteration: 1590, Epoch: 9, Batch: 142 out of 182,                   Loss: 5.74
Iteration: 1600, Epoch: 9, Batch: 152 out of 182,                   Loss: 5.88
thou art more than a
to be or not to the
wherefore art thou shalt not be a
Iteration: 1610, Epoch: 9, Batch: 162 out of 182,                   Loss: 5.82
Iteration: 1620, Epoch: 9, Batch: 172 out of 182,                   Loss: 5.43
Starting Epoch 10/10...
Iteration: 1630, Epoch: 10, Batch: 1 out of 182,                   Loss: 5.49
Iteration: 1640, Epoch: 10, Batch: 11 out of 182,                   Loss: 5.77
Iteration: 1650, Epoch: 10, Batch: 21 out of 182,                   Loss: 5.76
thou art more than a
to be or not to the
wherefore art thou shalt not be
Iteration: 1660, Epoch: 10, Batch: 31 out of 182,                   Loss: 5.79
Iteration: 1670, Epoch: 10, Batch: 41 out of 182,                   Loss: 5.69
Iteration: 1680, Epoch: 10, Batch: 51 out of 182,                   Loss: 5.81
Iteration: 1690, Epoch: 10, Batch: 61 out of 182,                   Loss: 5.81
Iteration: 1700, Epoch: 10, Batch: 71 out of 182,                   Loss: 5.90
thou art more than a
to be or not to the
wherefore art thou shalt not be
Iteration: 1710, Epoch: 10, Batch: 81 out of 182,                   Loss: 5.94
Iteration: 1720, Epoch: 10, Batch: 91 out of 182,                   Loss: 5.72
Iteration: 1730, Epoch: 10, Batch: 101 out of 182,                   Loss: 5.79
Iteration: 1740, Epoch: 10, Batch: 111 out of 182,                   Loss: 5.69
Iteration: 1750, Epoch: 10, Batch: 121 out of 182,                   Loss: 5.72
thou art more than
to be or not to the
wherefore art thou shalt be
Iteration: 1760, Epoch: 10, Batch: 131 out of 182,                   Loss: 5.86
Iteration: 1770, Epoch: 10, Batch: 141 out of 182,                   Loss: 5.90
Iteration: 1780, Epoch: 10, Batch: 151 out of 182,                   Loss: 5.76
Iteration: 1790, Epoch: 10, Batch: 161 out of 182,                   Loss: 5.72
Iteration: 1800, Epoch: 10, Batch: 171 out of 182,                   Loss: 5.88
thou art more than
to be or not to the
wherefore art thou shalt not be
Iteration: 1810, Epoch: 10, Batch: 181 out of 182,                   Loss: 5.80

In [12]:

    

plt.plot(train_loss, 'k-')
plt.title('Seq2Seq loss')
plt.xlabel('Iterations')
plt.ylabel('Loss')
plt.show()


    

In [ ]: