In [3]:

    

# https://www.tensorflow.org/extend/estimators
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

# tensorflow
import tensorflow as tf
import tensorflow.contrib.rnn as rnn
import tensorflow.contrib.learn as tflearn
import tensorflow.contrib.layers as tflayers

# keras
from tensorflow.contrib.keras.python.keras.layers import Dense, LSTM, GRU, Activation
from tensorflow.contrib.keras.python.keras.utils.data_utils import get_file

# input data
from tensorflow.examples.tutorials.mnist import input_data

# estimators
from tensorflow.contrib import learn

# estimator "builder"
from tensorflow.contrib.learn.python.learn.estimators import model_fn as model_fn_lib

# helpers
import numpy as np
import random
import sys

# enable logs
tf.logging.set_verbosity(tf.logging.INFO)

def sample(preds, temperature=1.0):
    #print(preds)
    return np.argmax(preds)

# THE MODEL
def model_fn(features, targets, mode, params):
    """Model function for Estimator."""
    
    # 1. Configure the model via TensorFlow operations
    # First, build all the model, a good idea is using Keras or tf.layers
    # since these are high-level API's
    #lstm = GRU(128, input_shape=(params["maxlen"], params["vocab_size"]))(features)
    #preds = Dense(params["vocab_size"], activation='sigmoid')(lstm)
    
    # 0. Reformat input shape to become a sequence
    lstm1 = GRU(128, input_shape=(params["maxlen"], params["vocab_size"]),
                return_sequences=False)(features)
    #lstm2 = GRU(128)(lstm1)
    preds = Dense(params["vocab_size"])(lstm1)
    preds_softmax = Activation("softmax")(preds)

    # 2. Define the loss function for training/evaluation
    loss = None
    train_op = None
    
    # Calculate Loss (for both TRAIN and EVAL modes)
    if mode != learn.ModeKeys.PREDICT:
        loss = tf.losses.softmax_cross_entropy(
            onehot_labels=targets, logits=preds)

    # 3. Define the training operation/optimizer
    
    # Configure the Training Op (for TRAIN mode)
    if mode == learn.ModeKeys.TRAIN:
        train_op = tf.contrib.layers.optimize_loss(
            loss=loss,
            global_step=tf.contrib.framework.get_global_step(),
            learning_rate=params["learning_rate"],
            optimizer="RMSProp",
        )

    # 4. Generate predictions
    predictions_dict = {
      "preds": preds_softmax
    }
    
    # 5. Define how you want to evaluate the model
    metrics = {
        "accuracy": tf.metrics.accuracy(tf.argmax(input=preds_softmax, axis=1), tf.argmax(input=targets, axis=1))
    }
    
    # 6. Return predictions/loss/train_op/eval_metric_ops in ModelFnOps object
    return model_fn_lib.ModelFnOps(
      mode=mode,
      predictions=predictions_dict,
      loss=loss,
      train_op=train_op,
      eval_metric_ops=metrics)


    

In [4]:

    

print('Getting data')

#path = get_file('nietzsche.txt', origin='https://s3.amazonaws.com/text-datasets/nietzsche.txt')
path = 'shakespeare.txt'
text = open(path).read().lower()
print('corpus length:', len(text))

chars = sorted(list(set(text)))
print('total chars:', len(chars))
char_indices = dict((c, i) for i, c in enumerate(chars))
indices_char = dict((i, c) for i, c in enumerate(chars))

# cut the text in semi-redundant sequences of maxlen characters
maxlen = 40
step = 1
sentences = []
next_chars = []
for i in range(0, len(text) - maxlen, step):
    sentences.append(text[i: i + maxlen])
    next_chars.append(text[i + maxlen])
print('nb sequences:', len(sentences))

print('Vectorization...')
X = np.zeros((len(sentences), maxlen, len(chars)), dtype=np.float32)
y = np.zeros((len(sentences), len(chars)), dtype=np.float32)
for i, sentence in enumerate(sentences):
    for t, char in enumerate(sentence):
        X[i, t, char_indices[char]] = 1
    y[i, char_indices[next_chars[i]]] = 1

print(X[0])


    

    




Getting data
corpus length: 1115394
total chars: 39
nb sequences: 1115354
Vectorization...
[[ 0.  0.  0. ...,  0.  0.  0.]
 [ 0.  0.  0. ...,  0.  0.  0.]
 [ 0.  0.  0. ...,  0.  0.  0.]
 ..., 
 [ 0.  0.  0. ...,  0.  0.  0.]
 [ 0.  0.  0. ...,  0.  0.  0.]
 [ 0.  0.  0. ...,  0.  0.  0.]]

In [ ]:

    

# PARAMETERS
LEARNING_RATE = 0.01
BATCH_SIZE = 1
STEPS = len(sentences)

NUM_OUTPUTS_PRED = 500 # Number of test characters of text to generate after training the network

# Set model params
model_params = {"learning_rate": LEARNING_RATE, "vocab_size": len(chars), "maxlen": maxlen}

# Instantiate Estimator
nn = tf.contrib.learn.Estimator(model_fn=model_fn, params=model_params)

# Score accuracy
# Fit
print('-' * 40)
print("Training")
print('-' * 40)
nn.fit(x=X, y=y, steps=STEPS, batch_size=BATCH_SIZE)

# choose a random sentence
start_index = random.randint(0, len(text) - maxlen - 1)
sentence = text[start_index: start_index + maxlen]

# generate output using the RNN model
original_sentence = sentence
generated = sentence
for i in range(NUM_OUTPUTS_PRED):
    x = np.zeros((1, maxlen, len(chars)), dtype=np.float32)
    for t, char in enumerate(sentence):
        x[0, t, char_indices[char]] = 1.

    p = None
    for e in nn.predict(x):
        if p is None: p = e["preds"]
    next_index = sample(p)
    next_char = indices_char[next_index]

    generated += next_char
    sentence = sentence[1:] + next_char

print('\n' * 10, '-' * 100)
print('HERE')
print(generated)
print(original_sentence)
print('-' * 100, '\n' * 10)


    

    




INFO:tensorflow:Using default config.
INFO:tensorflow:Using config: {'_task_id': 0, '_save_checkpoints_secs': 600, '_keep_checkpoint_every_n_hours': 10000, '_master': '', '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f48cb050c88>, '_keep_checkpoint_max': 5, '_evaluation_master': '', '_tf_random_seed': None, '_is_chief': True, '_num_ps_replicas': 0, '_environment': 'local', '_model_dir': None, '_tf_config': gpu_options {
  per_process_gpu_memory_fraction: 1
}
, '_num_worker_replicas': 0, '_save_summary_steps': 100, '_task_type': None, '_save_checkpoints_steps': None}
WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmpsnevr3y3
----------------------------------------
Training
----------------------------------------
WARNING:tensorflow:From <ipython-input-7-0de7a2223bd3>:19: calling BaseEstimator.fit (from tensorflow.contrib.learn.python.learn.estimators.estimator) with y is deprecated and will be removed after 2016-12-01.
Instructions for updating:
Estimator is decoupled from Scikit Learn interface by moving into
separate class SKCompat. Arguments x, y and batch_size are only
available in the SKCompat class, Estimator will only accept input_fn.
Example conversion:
  est = Estimator(...) -> est = SKCompat(Estimator(...))
WARNING:tensorflow:From <ipython-input-7-0de7a2223bd3>:19: calling BaseEstimator.fit (from tensorflow.contrib.learn.python.learn.estimators.estimator) with batch_size is deprecated and will be removed after 2016-12-01.
Instructions for updating:
Estimator is decoupled from Scikit Learn interface by moving into
separate class SKCompat. Arguments x, y and batch_size are only
available in the SKCompat class, Estimator will only accept input_fn.
Example conversion:
  est = Estimator(...) -> est = SKCompat(Estimator(...))
WARNING:tensorflow:From <ipython-input-7-0de7a2223bd3>:19: calling BaseEstimator.fit (from tensorflow.contrib.learn.python.learn.estimators.estimator) with x is deprecated and will be removed after 2016-12-01.
Instructions for updating:
Estimator is decoupled from Scikit Learn interface by moving into
separate class SKCompat. Arguments x, y and batch_size are only
available in the SKCompat class, Estimator will only accept input_fn.
Example conversion:
  est = Estimator(...) -> est = SKCompat(Estimator(...))






    




/usr/local/lib/python3.4/dist-packages/tensorflow/python/util/deprecation.py:248: FutureWarning: comparison to `None` will result in an elementwise object comparison in the future.
  equality = a == b






    




INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Saving checkpoints for 1 into /tmp/tmpsnevr3y3/model.ckpt.
INFO:tensorflow:loss = 3.64558, step = 1
INFO:tensorflow:global_step/sec: 26.5339
INFO:tensorflow:loss = 4.03005, step = 101 (3.770 sec)
INFO:tensorflow:global_step/sec: 27.2854
INFO:tensorflow:loss = 2.77961, step = 201 (3.665 sec)
INFO:tensorflow:global_step/sec: 26.7679
INFO:tensorflow:loss = 3.10089, step = 301 (3.739 sec)
INFO:tensorflow:global_step/sec: 26.8403
INFO:tensorflow:loss = 5.49769, step = 401 (3.729 sec)
INFO:tensorflow:global_step/sec: 26.7837
INFO:tensorflow:loss = 3.52534, step = 501 (3.728 sec)
INFO:tensorflow:global_step/sec: 25.8704
INFO:tensorflow:loss = 1.59166, step = 601 (3.864 sec)
INFO:tensorflow:global_step/sec: 26.789
INFO:tensorflow:loss = 6.09143, step = 701 (3.734 sec)
INFO:tensorflow:global_step/sec: 26.8786
INFO:tensorflow:loss = 2.4158, step = 801 (3.720 sec)
INFO:tensorflow:global_step/sec: 27.3595
INFO:tensorflow:loss = 2.93907, step = 901 (3.655 sec)
INFO:tensorflow:global_step/sec: 28.0323
INFO:tensorflow:loss = 3.851, step = 1001 (3.567 sec)
INFO:tensorflow:global_step/sec: 28.8611
INFO:tensorflow:loss = 0.611839, step = 1101 (3.465 sec)
INFO:tensorflow:global_step/sec: 29.357
INFO:tensorflow:loss = 2.95856, step = 1201 (3.408 sec)
INFO:tensorflow:global_step/sec: 29.8971
INFO:tensorflow:loss = 4.17606, step = 1301 (3.343 sec)
INFO:tensorflow:global_step/sec: 29.2152
INFO:tensorflow:loss = 4.62098, step = 1401 (3.424 sec)
INFO:tensorflow:global_step/sec: 28.8972
INFO:tensorflow:loss = 12.109, step = 1501 (3.462 sec)
INFO:tensorflow:global_step/sec: 28.8362
INFO:tensorflow:loss = 0.856251, step = 1601 (3.466 sec)
INFO:tensorflow:global_step/sec: 28.6669
INFO:tensorflow:loss = 5.55303, step = 1701 (3.489 sec)
INFO:tensorflow:global_step/sec: 28.8359
INFO:tensorflow:loss = 4.22769, step = 1801 (3.468 sec)
INFO:tensorflow:global_step/sec: 28.104
INFO:tensorflow:loss = 1.79838, step = 1901 (3.559 sec)
INFO:tensorflow:global_step/sec: 27.9153
INFO:tensorflow:loss = 5.30141, step = 2001 (3.581 sec)
INFO:tensorflow:global_step/sec: 27.4383
INFO:tensorflow:loss = 1.5258, step = 2101 (3.644 sec)
INFO:tensorflow:global_step/sec: 28.4042
INFO:tensorflow:loss = 4.01015, step = 2201 (3.525 sec)
INFO:tensorflow:global_step/sec: 27.4098
INFO:tensorflow:loss = 5.74606, step = 2301 (3.644 sec)
INFO:tensorflow:global_step/sec: 27.6994
INFO:tensorflow:loss = 1.68145, step = 2401 (3.610 sec)
INFO:tensorflow:global_step/sec: 27.3696
INFO:tensorflow:loss = 4.82453, step = 2501 (3.655 sec)
INFO:tensorflow:global_step/sec: 27.8929
INFO:tensorflow:loss = 3.43846, step = 2601 (3.585 sec)
INFO:tensorflow:global_step/sec: 27.4665
INFO:tensorflow:loss = 2.55553, step = 2701 (3.640 sec)
INFO:tensorflow:global_step/sec: 27.4363
INFO:tensorflow:loss = 3.86818, step = 2801 (3.645 sec)
INFO:tensorflow:global_step/sec: 27.1478
INFO:tensorflow:loss = 5.41699, step = 2901 (3.690 sec)
INFO:tensorflow:global_step/sec: 27.7145
INFO:tensorflow:loss = 4.83035, step = 3001 (3.605 sec)
INFO:tensorflow:global_step/sec: 27.5017
INFO:tensorflow:loss = 8.22265, step = 3101 (3.634 sec)
INFO:tensorflow:global_step/sec: 27.3713
INFO:tensorflow:loss = 2.39176, step = 3201 (3.658 sec)
INFO:tensorflow:global_step/sec: 27.5985
INFO:tensorflow:loss = 2.68953, step = 3301 (3.618 sec)
INFO:tensorflow:global_step/sec: 26.2421
INFO:tensorflow:loss = 2.22034, step = 3401 (3.811 sec)
INFO:tensorflow:global_step/sec: 26.5815
INFO:tensorflow:loss = 3.14892, step = 3501 (3.764 sec)
INFO:tensorflow:global_step/sec: 27.143
INFO:tensorflow:loss = 13.8441, step = 3601 (3.682 sec)
INFO:tensorflow:global_step/sec: 27.5162
INFO:tensorflow:loss = 3.48399, step = 3701 (3.635 sec)
INFO:tensorflow:global_step/sec: 27.8807
INFO:tensorflow:loss = 4.41308, step = 3801 (3.586 sec)
INFO:tensorflow:global_step/sec: 27.7331
INFO:tensorflow:loss = 2.75933, step = 3901 (3.606 sec)
INFO:tensorflow:global_step/sec: 27.0074
INFO:tensorflow:loss = 4.71359, step = 4001 (3.703 sec)
INFO:tensorflow:global_step/sec: 27.1761
INFO:tensorflow:loss = 6.20922, step = 4101 (3.683 sec)
INFO:tensorflow:global_step/sec: 27.7613
INFO:tensorflow:loss = 7.42886, step = 4201 (3.599 sec)
INFO:tensorflow:global_step/sec: 27.8759
INFO:tensorflow:loss = 5.37514, step = 4301 (3.592 sec)
INFO:tensorflow:global_step/sec: 27.5513
INFO:tensorflow:loss = 13.9273, step = 4401 (3.625 sec)
INFO:tensorflow:global_step/sec: 28.1532
INFO:tensorflow:loss = 6.04067, step = 4501 (3.552 sec)
INFO:tensorflow:global_step/sec: 28.0404
INFO:tensorflow:loss = 4.4146, step = 4601 (3.566 sec)
INFO:tensorflow:global_step/sec: 27.3456
INFO:tensorflow:loss = 5.5622, step = 4701 (3.659 sec)
INFO:tensorflow:global_step/sec: 27.5725
INFO:tensorflow:loss = 2.1977, step = 4801 (3.625 sec)
INFO:tensorflow:global_step/sec: 26.4702
INFO:tensorflow:loss = 3.58606, step = 4901 (3.779 sec)
INFO:tensorflow:global_step/sec: 26.6982
INFO:tensorflow:loss = 15.4288, step = 5001 (3.744 sec)
INFO:tensorflow:global_step/sec: 25.7548
INFO:tensorflow:loss = 4.57986, step = 5101 (3.885 sec)
INFO:tensorflow:global_step/sec: 26.7088
INFO:tensorflow:loss = 6.03905, step = 5201 (3.741 sec)
INFO:tensorflow:global_step/sec: 27.1871
INFO:tensorflow:loss = 0.596038, step = 5301 (3.678 sec)
INFO:tensorflow:global_step/sec: 26.6075
INFO:tensorflow:loss = 3.52885, step = 5401 (3.763 sec)
INFO:tensorflow:global_step/sec: 27.6366
INFO:tensorflow:loss = 0.497641, step = 5501 (3.614 sec)
INFO:tensorflow:global_step/sec: 29.6364
INFO:tensorflow:loss = 2.15703, step = 5601 (3.374 sec)
INFO:tensorflow:global_step/sec: 29.2636
INFO:tensorflow:loss = 1.25765, step = 5701 (3.417 sec)
INFO:tensorflow:global_step/sec: 28.6137
INFO:tensorflow:loss = 10.8849, step = 5801 (3.495 sec)
INFO:tensorflow:global_step/sec: 28.5661
INFO:tensorflow:loss = 0.991155, step = 5901 (3.501 sec)
INFO:tensorflow:global_step/sec: 29.329
INFO:tensorflow:loss = 2.25538, step = 6001 (3.410 sec)
INFO:tensorflow:global_step/sec: 29.654
INFO:tensorflow:loss = 2.80958, step = 6101 (3.372 sec)
INFO:tensorflow:global_step/sec: 29.0092
INFO:tensorflow:loss = 4.61165, step = 6201 (3.447 sec)
INFO:tensorflow:global_step/sec: 28.2773
INFO:tensorflow:loss = 1.52782, step = 6301 (3.537 sec)
INFO:tensorflow:global_step/sec: 26.5083
INFO:tensorflow:loss = 4.04971, step = 6401 (3.775 sec)
INFO:tensorflow:global_step/sec: 26.2241
INFO:tensorflow:loss = 4.33619, step = 6501 (3.818 sec)
INFO:tensorflow:global_step/sec: 27.4841
INFO:tensorflow:loss = 1.95557, step = 6601 (3.632 sec)
INFO:tensorflow:global_step/sec: 29.4218
INFO:tensorflow:loss = 2.56419, step = 6701 (3.398 sec)
INFO:tensorflow:global_step/sec: 29.261
INFO:tensorflow:loss = 1.37078, step = 6801 (3.420 sec)
INFO:tensorflow:global_step/sec: 29.3693
INFO:tensorflow:loss = 4.31274, step = 6901 (3.403 sec)
INFO:tensorflow:global_step/sec: 29.7096
INFO:tensorflow:loss = 2.8041, step = 7001 (3.367 sec)
INFO:tensorflow:global_step/sec: 28.5311
INFO:tensorflow:loss = 4.65558, step = 7101 (3.504 sec)
INFO:tensorflow:global_step/sec: 30.1243
INFO:tensorflow:loss = 3.10305, step = 7201 (3.320 sec)
INFO:tensorflow:global_step/sec: 28.8737
INFO:tensorflow:loss = 2.63041, step = 7301 (3.462 sec)
INFO:tensorflow:global_step/sec: 29.6751
INFO:tensorflow:loss = 1.30979, step = 7401 (3.370 sec)
INFO:tensorflow:global_step/sec: 28.8262
INFO:tensorflow:loss = 4.16254, step = 7501 (3.469 sec)
INFO:tensorflow:global_step/sec: 30.2222
INFO:tensorflow:loss = 2.52677, step = 7601 (3.309 sec)
INFO:tensorflow:global_step/sec: 29.4587
INFO:tensorflow:loss = 5.07622, step = 7701 (3.396 sec)
INFO:tensorflow:global_step/sec: 28.6201
INFO:tensorflow:loss = 7.21718, step = 7801 (3.493 sec)
INFO:tensorflow:global_step/sec: 27.1965
INFO:tensorflow:loss = 3.01447, step = 7901 (3.680 sec)
INFO:tensorflow:global_step/sec: 26.8171
INFO:tensorflow:loss = 2.93606, step = 8001 (3.729 sec)
INFO:tensorflow:global_step/sec: 26.305
INFO:tensorflow:loss = 3.60068, step = 8101 (3.799 sec)
INFO:tensorflow:global_step/sec: 26.6941
INFO:tensorflow:loss = 1.13325, step = 8201 (3.746 sec)
INFO:tensorflow:global_step/sec: 27.7305
INFO:tensorflow:loss = 3.15776, step = 8301 (3.608 sec)
INFO:tensorflow:global_step/sec: 28.0493
INFO:tensorflow:loss = 3.88607, step = 8401 (3.563 sec)
INFO:tensorflow:global_step/sec: 27.2645
INFO:tensorflow:loss = 3.29669, step = 8501 (3.670 sec)
INFO:tensorflow:global_step/sec: 27.0021
INFO:tensorflow:loss = 2.18847, step = 8601 (3.702 sec)
INFO:tensorflow:global_step/sec: 26.7854
INFO:tensorflow:loss = 3.34612, step = 8701 (3.733 sec)
INFO:tensorflow:global_step/sec: 27.8074
INFO:tensorflow:loss = 1.53164, step = 8801 (3.599 sec)
INFO:tensorflow:global_step/sec: 26.6641
INFO:tensorflow:loss = 6.73189, step = 8901 (3.748 sec)
INFO:tensorflow:global_step/sec: 26.6922
INFO:tensorflow:loss = 1.76937, step = 9001 (3.751 sec)

In [ ]: