In [1]:

    

# https://raw.githubusercontent.com/fchollet/keras/master/examples/lstm_text_generation.py
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

# tensorflow
import tensorflow as tf
import tensorflow.contrib.rnn as rnn
import tensorflow.contrib.learn as tflearn
import tensorflow.contrib.layers as tflayers

# keras
from tensorflow.contrib.keras.python.keras.layers import Dense, LSTM, GRU, Activation
from tensorflow.contrib.keras.python.keras.utils.data_utils import get_file

# input data
from tensorflow.examples.tutorials.mnist import input_data

# estimators
from tensorflow.contrib import learn

# estimator "builder"
from tensorflow.contrib.learn.python.learn.estimators import model_fn as model_fn_lib

# helpers
import numpy as np
import random
import sys

# enable logs
tf.logging.set_verbosity(tf.logging.INFO)

def sample(preds, temperature=1.0):
    # helper function to sample an index from a probability array
    preds = np.asarray(preds).astype('float64')
    preds = np.log(preds) / temperature
    exp_preds = np.exp(preds)
    preds = exp_preds / np.sum(exp_preds)
    probas = np.random.multinomial(1, preds, 1)
    return np.argmax(probas)

# THE MODEL
def model_fn(features, targets, mode, params):
    """Model function for Estimator."""
    
    # 1. Configure the model via TensorFlow operations
    # First, build all the model, a good idea is using Keras or tf.layers
    # since these are high-level API's
    #lstm = GRU(128, input_shape=(params["maxlen"], params["vocab_size"]))(features)
    #preds = Dense(params["vocab_size"], activation='sigmoid')(lstm)
    
    # 0. Reformat input shape to become a sequence
    lstm1 = GRU(128, input_shape=(params["maxlen"], params["vocab_size"]),
                return_sequences=False)(features)
    #lstm2 = GRU(128)(lstm1)
    preds = Dense(params["vocab_size"])(lstm1)
    preds_softmax = Activation("softmax")(preds)

    # 2. Define the loss function for training/evaluation
    loss = None
    train_op = None
    
    # Calculate Loss (for both TRAIN and EVAL modes)
    if mode != learn.ModeKeys.PREDICT:
        loss = tf.losses.softmax_cross_entropy(
            onehot_labels=targets, logits=preds)

    # 3. Define the training operation/optimizer
    
    # Configure the Training Op (for TRAIN mode)
    if mode == learn.ModeKeys.TRAIN:
        train_op = tf.contrib.layers.optimize_loss(
            loss=loss,
            global_step=tf.contrib.framework.get_global_step(),
            learning_rate=params["learning_rate"],
            optimizer="RMSProp",
        )

    # 4. Generate predictions
    predictions_dict = {
      "preds": preds_softmax
    }
    
    # 5. Define how you want to evaluate the model
    metrics = {
        "accuracy": tf.metrics.accuracy(tf.argmax(input=preds_softmax, axis=1), tf.argmax(input=targets, axis=1))
    }
    
    # 6. Return predictions/loss/train_op/eval_metric_ops in ModelFnOps object
    return model_fn_lib.ModelFnOps(
      mode=mode,
      predictions=predictions_dict,
      loss=loss,
      train_op=train_op,
      eval_metric_ops=metrics)


    

In [2]:

    

print('Getting data')

#path = get_file('nietzsche.txt', origin='https://s3.amazonaws.com/text-datasets/nietzsche.txt')
path = 'shakespeare.txt'
text = open(path).read().lower()
print('corpus length:', len(text))

chars = sorted(list(set(text)))
print('total chars:', len(chars))
char_indices = dict((c, i) for i, c in enumerate(chars))
indices_char = dict((i, c) for i, c in enumerate(chars))

# cut the text in semi-redundant sequences of maxlen characters
maxlen = 40
step = 1
sentences = []
next_chars = []
for i in range(0, len(text) - maxlen, step):
    sentences.append(text[i: i + maxlen])
    next_chars.append(text[i + maxlen])
print('nb sequences:', len(sentences))

print('Vectorization...')
X = np.zeros((len(sentences), maxlen, len(chars)), dtype=np.float32)
y = np.zeros((len(sentences), len(chars)), dtype=np.float32)
for i, sentence in enumerate(sentences):
    for t, char in enumerate(sentence):
        X[i, t, char_indices[char]] = 1
    y[i, char_indices[next_chars[i]]] = 1

print(X[0])


    

    




Getting data
corpus length: 1115394
total chars: 39
nb sequences: 1115354
Vectorization...






    




---------------------------------------------------------------------------
MemoryError                               Traceback (most recent call last)
<ipython-input-2-38e67f5d7129> in <module>()
     22 
     23 print('Vectorization...')
---> 24 X = np.zeros((len(sentences), maxlen, len(chars)), dtype=np.float32)
     25 y = np.zeros((len(sentences), len(chars)), dtype=np.float32)
     26 for i, sentence in enumerate(sentences):

MemoryError: 

In [ ]: