Build Experiment from keras model

Embeds a 3 layer FCN model to predict MNIST handwritten digits in a Tensorflow Experiment. The Estimator here is a Keras model.

DOES NOT WORK CURRENTLY with Tensorflow 1.2


In [1]:
from __future__ import division, print_function
from tensorflow.contrib.learn.python.learn.estimators import model_fn as model_fn_lib
from tensorflow.contrib import keras
import matplotlib.pyplot as plt
import numpy as np
import os
import shutil
import tensorflow as tf

In [7]:
DATA_DIR = "../../data"
TRAIN_FILE = os.path.join(DATA_DIR, "mnist_train.csv")
TEST_FILE = os.path.join(DATA_DIR, "mnist_test.csv")

MODEL_DIR = os.path.join(DATA_DIR, "expt-keras-model")

NUM_FEATURES = 784
NUM_CLASSES = 10
NUM_STEPS = 10
LEARNING_RATE = 1e-3
BATCH_SIZE = 128

tf.logging.set_verbosity(tf.logging.INFO)

Prepare Data


In [13]:
def parse_file(filename):
    xdata, ydata = [], []
    fin = open(filename, "rb")
    i = 0
    for line in fin:
        if i % 10000 == 0:
            print("{:s}: {:d} lines read".format(
                os.path.basename(filename), i))
        cols = line.strip().split(",")
        onehot_label = np.zeros((NUM_CLASSES))
        onehot_label[int(cols[0])] = 1
        ydata.append(onehot_label)
        xdata.append([float(x) / 255. for x in cols[1:]])
        i += 1
    fin.close()
    print("{:s}: {:d} lines read".format(os.path.basename(filename), i))
    y = np.array(ydata, dtype=np.float32)
    X = np.array(xdata, dtype=np.float32)
    return X, y

Xtrain, ytrain = parse_file(TRAIN_FILE)
Xtest, ytest = parse_file(TEST_FILE)
print(Xtrain.shape, ytrain.shape, Xtest.shape, ytest.shape)


mnist_train.csv: 0 lines read
mnist_train.csv: 10000 lines read
mnist_train.csv: 20000 lines read
mnist_train.csv: 30000 lines read
mnist_train.csv: 40000 lines read
mnist_train.csv: 50000 lines read
mnist_train.csv: 60000 lines read
mnist_test.csv: 0 lines read
mnist_test.csv: 10000 lines read
(60000, 784) (60000, 10) (10000, 784) (10000, 10)

The train_input_fn and test_input_fn below are equivalent to using the full batches. There is some information on building batch oriented input functions, but I was unable to make it work. Commented out block is adapted from a Keras data generator, but that does not work either.


In [8]:
def train_input_fn():
    return tf.constant(Xtrain), tf.constant(ytrain)

def test_input_fn():
    return tf.constant(Xtest), tf.constant(ytest)

# def batch_input_fn(X, y, batch_size=BATCH_SIZE, 
#                    num_epochs=NUM_STEPS):
#     for e in range(num_epochs):
#         num_recs = X.shape[0]
#         sids = np.random.permutation(np.arange(num_recs))
#         num_batches = num_recs // batch_size
#         for bid in range(num_batches):
#             sids_b = sids[bid * batch_size : (bid + 1) * batch_size]
#             X_b = np.zeros((batch_size, NUM_FEATURES))
#             y_b = np.zeros((batch_size,))
#             for i in range(batch_size):
#                 X_b[i] = X[sids_b[i]]
#                 y_b[i] = y[sids_b[i]]
#             yield tf.constant(X_b, dtype=tf.float32), \
#                   tf.constant(y_b, dtype=tf.float32)

# def train_input_fn():
#     return batch_input_fn(Xtrain, ytrain, BATCH_SIZE).next()

# def test_input_fn():
#     return batch_input_fn(Xtest, ytest, BATCH_SIZE).next()

Define Estimator

Apparently Keras model is an Estimator (or there is some functional equivalence).


In [14]:
model = keras.models.Sequential()
model.add(keras.layers.Dense(512, activation="relu", 
                             input_shape=(NUM_FEATURES,)))
model.add(keras.layers.Dropout(0.2))
model.add(keras.layers.Dense(256, activation="relu"))
model.add(keras.layers.Dropout(0.2))
model.add(keras.layers.Dense(NUM_CLASSES, activation="softmax"))

In [17]:
model.compile(loss=keras.losses.categorical_crossentropy,
             optimizer=keras.optimizers.Adam(),
             metrics=["accuracy"])

Train Estimator

Using the parameters x, y and batch are deprecated and the warnings say to use the input_fn instead. However, using that results in very slow fit and evaluate. The solution is to use batch oriented input_fns. The commented portions will be opened up once I figure out how to make the batch oriented input_fns work.


In [18]:
model.fit(x=Xtrain, y=ytrain, 
            batch_size=BATCH_SIZE, 
            epochs=NUM_STEPS)
# estimator.fit(input_fn=train_input_fn, steps=NUM_STEPS)


Epoch 1/10
60000/60000 [==============================] - 4s - loss: 0.0874 - acc: 0.9735       
Epoch 2/10
60000/60000 [==============================] - 4s - loss: 0.0797 - acc: 0.9747     
Epoch 3/10
60000/60000 [==============================] - 4s - loss: 0.0739 - acc: 0.9771     
Epoch 4/10
60000/60000 [==============================] - 4s - loss: 0.0698 - acc: 0.9777     
Epoch 5/10
60000/60000 [==============================] - 4s - loss: 0.0608 - acc: 0.9809     
Epoch 6/10
60000/60000 [==============================] - 4s - loss: 0.0601 - acc: 0.9815     
Epoch 7/10
60000/60000 [==============================] - 4s - loss: 0.0599 - acc: 0.9815     
Epoch 8/10
60000/60000 [==============================] - 4s - loss: 0.0595 - acc: 0.9816     
Epoch 9/10
60000/60000 [==============================] - 4s - loss: 0.0553 - acc: 0.9834     
Epoch 10/10
60000/60000 [==============================] - 4s - loss: 0.0501 - acc: 0.9844     
Out[18]:
<tensorflow.contrib.keras.python.keras.callbacks.History at 0x122ad2d10>

Evaluate Estimator


In [20]:
results = model.evaluate(x=Xtest, y=ytest)
# results = estimator.evaluate(input_fn=test_input_fn)
print(results)


 9152/10000 [==========================>...] - ETA: 0s [0.062144852349229771, 0.98340000000000005]

alternatively...

Define Experiment

A model is wrapped in an Estimator, which is then wrapped in an Experiment. Once you have an Experiment, you can run this in a distributed manner on CPU or GPU.


In [24]:
NUM_STEPS = 20

def experiment_fn(run_config, params):
    # define and compile model
    model = keras.models.Sequential()
    model.add(keras.layers.Dense(512, activation="relu", 
                                 input_shape=(NUM_FEATURES,)))
    model.add(keras.layers.Dropout(0.2))
    model.add(keras.layers.Dense(256, activation="relu"))
    model.add(keras.layers.Dropout(0.2))
    model.add(keras.layers.Dense(NUM_CLASSES, activation="softmax"))
    
    model.compile(loss=keras.losses.categorical_crossentropy,
                  optimizer=keras.optimizers.Adam(),
                  metrics=["accuracy"])
    estimator = model.get_estimator(config={})
    return tf.contrib.learn.Experiment(
        estimator=estimator,
        train_input_fn=train_input_fn,
        train_steps=NUM_STEPS,
        eval_input_fn=test_input_fn)

Run Experiment


In [25]:
shutil.rmtree(MODEL_DIR, ignore_errors=True)
tf.contrib.learn.learn_runner.run(experiment_fn, 
    run_config=tf.contrib.learn.RunConfig(
        model_dir=MODEL_DIR))


WARNING:tensorflow:uid (from tensorflow.contrib.learn.python.learn.estimators.run_config) is experimental and may change or be removed at any time, and without warning.
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-25-1db2108eda94> in <module>()
      2 tf.contrib.learn.learn_runner.run(experiment_fn, 
      3     run_config=tf.contrib.learn.RunConfig(
----> 4         model_dir=MODEL_DIR))

/Users/palsujit/anaconda2/lib/python2.7/site-packages/tensorflow/contrib/learn/python/learn/learn_runner.pyc in run(experiment_fn, output_dir, schedule, run_config, hparams)
    191   if run_config is not None:
    192     wrapped_experiment_fn = _wrapped_experiment_fn_with_uid_check(experiment_fn)
--> 193     experiment = wrapped_experiment_fn(run_config=run_config, hparams=hparams)
    194   else:
    195     if not output_dir:

/Users/palsujit/anaconda2/lib/python2.7/site-packages/tensorflow/contrib/learn/python/learn/learn_runner.pyc in wrapped_experiment_fn(run_config, hparams)
     77 
     78     expected_uid = run_config.uid()
---> 79     experiment = experiment_fn(run_config, hparams)
     80 
     81     if not isinstance(experiment, Experiment):

<ipython-input-24-cf7317f310f5> in experiment_fn(run_config, params)
     14                   optimizer=keras.optimizers.Adam(),
     15                   metrics=["accuracy"])
---> 16     estimator = model.get_estimator(config={})
     17     return tf.contrib.learn.Experiment(
     18         estimator=estimator,

AttributeError: 'Sequential' object has no attribute 'get_estimator'

In [ ]: