In [2]:
from keras.datasets import imdb
from keras.preprocessing import sequence
from keras.models import Sequential
import keras.layers as kl
from keras.optimizers import Adam

from hyperopt import fmin, tpe, hp, STATUS_OK, Trials
from kopt import CompileFN, KMongoTrials, test_fn

# 1. define the data function returning training, (validation, test) data
def data(max_features=5000, maxlen=80):
    (x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features)
    x_train = sequence.pad_sequences(x_train, maxlen=maxlen)
    x_test = sequence.pad_sequences(x_test, maxlen=maxlen)
    return (x_train[:100], y_train[:100], max_features), (x_test, y_test)

In [3]:
# 2. Define the model function returning a compiled Keras model
def model(train_data, lr=0.001,
          embedding_dims=128, rnn_units=64,
          dropout=0.2):
    # extract data dimensions
    max_features = train_data[2]

    model = Sequential()
    model.add(kl.Embedding(max_features, embedding_dims))
    model.add(kl.LSTM(rnn_units, dropout=dropout, recurrent_dropout=dropout))
    model.add(kl.Dense(1, activation='sigmoid'))

    model.compile(loss='binary_crossentropy',
                  optimizer=Adam(lr=lr),
                  metrics=['accuracy'])
    return model

In [4]:
# Specify the objective function
db_name = "imdb"
exp_name = "myexp1"

objective = CompileFN(db_name, exp_name,
                      data_fn=data,
                      model_fn=model,
                      loss_metric="acc", # which metric to optimize for
                      loss_metric_mode="max",  # try to maximize the metric
                      valid_split=.2, # use 20% of the training data for the validation set
                      save_model='best', # checkpoint the best model
                      save_results=True, # save the results as .json (in addition to mongoDB)
                      save_dir="./saved_models/")  # place to store the models

In [5]:
# define the hyper-parameter ranges
# see https://github.com/hyperopt/hyperopt/wiki/FMin for more info
hyper_params = {
    "data": {
        "max_features": 100,
        "maxlen": 80,
    },
    "model": {
        "lr": hp.loguniform("m_lr", np.log(1e-4), np.log(1e-2)), # 0.0001 - 0.01
        "embedding_dims": hp.choice("m_emb", (64, 128)),
        "rnn_units": 64,
        "dropout": hp.uniform("m_do", 0, 0.5),
},
    "fit": {
        "epochs": 20
    }
}

In [6]:
# test model training, on a small subset for one epoch
test_fn(objective, hyper_params)


2017-10-07 17:53:55,100 [INFO] Load data...
2017-10-07 17:54:00,418 [INFO] Fit...
['loss', 'acc']
Train on 80 samples, validate on 20 samples
Epoch 1/1
0s - loss: 0.6931 - acc: 0.5125 - val_loss: 0.6699 - val_acc: 0.7000
2017-10-07 17:54:03,808 [INFO] Evaluate...
2017-10-07 17:54:04,060 [INFO] Done!
Returned value:
{'eval': {'acc': 0.69999998807907104, 'loss': 0.66988599300384521},
 'history': {'loss': {'acc': [0.51249999999999996],
                      'epoch': [0],
                      'loss': [0.6931055903434753],
                      'val_acc': [0.69999998807907104],
                      'val_loss': [0.66988599300384521]},
             'params': {'batch_size': 32,
                        'do_validation': True,
                        'epochs': 1,
                        'metrics': ['loss', 'acc', 'val_loss', 'val_acc'],
                        'samples': 80,
                        'steps': None,
                        'verbose': 2}},
 'loss': -0.69999998807907104,
 'name': {'data': 'data',
          'model': 'model',
          'optim_metric': 'acc',
          'optim_metric_mode': 'acc'},
 'param': {'data': {'max_features': 100, 'maxlen': 80},
           'fit': {'batch_size': 32,
                   'early_stop_monitor': 'val_loss',
                   'epochs': 1,
                   'patience': 10},
           'model': {'dropout': 0.31760152349994536,
                     'embedding_dims': 64,
                     'lr': 0.0035943038525379817,
                     'rnn_units': 64}},
 'path': {'model': '/tmp/kopt_test//imdb/myexp1//train_models/5dbfcb40-28c9-4aa3-9176-178ac5c3e85f.h5',
          'results': '/tmp/kopt_test//imdb/myexp1//train_models/5dbfcb40-28c9-4aa3-9176-178ac5c3e85f.json'},
 'status': 'ok',
 'time': {'duration': {'dataload': 4.920822,
                       'total': 8.960585,
                       'training': 4.039763},
          'end': '2017-10-07 17:54:04.059751',
          'start': '2017-10-07 17:53:55.099166'}}

In [7]:
# run hyper-parameter optimization sequentially (without any database)
trials = Trials()
best = fmin(objective, hyper_params, trials=trials, algo=tpe.suggest, max_evals=2)


2017-10-07 17:54:08,950 [INFO] tpe_transform took 0.002530 seconds
2017-10-07 17:54:08,951 [INFO] TPE using 0 trials
2017-10-07 17:54:08,954 [INFO] Load data...
2017-10-07 17:54:13,933 [INFO] Fit...
['loss', 'acc']
Train on 80 samples, validate on 20 samples
Epoch 1/20
0s - loss: 0.6929 - acc: 0.5125 - val_loss: 0.6252 - val_acc: 0.7000
Epoch 2/20
0s - loss: 0.6853 - acc: 0.5500 - val_loss: 0.6495 - val_acc: 0.6500
Epoch 3/20
0s - loss: 0.6691 - acc: 0.6500 - val_loss: 0.6532 - val_acc: 0.7000
Epoch 4/20
0s - loss: 0.6291 - acc: 0.7500 - val_loss: 0.5904 - val_acc: 0.6000
Epoch 5/20
0s - loss: 0.5450 - acc: 0.7375 - val_loss: 0.6378 - val_acc: 0.7000
Epoch 6/20
0s - loss: 0.4470 - acc: 0.7625 - val_loss: 0.6246 - val_acc: 0.7000
Epoch 7/20
0s - loss: 0.5676 - acc: 0.7500 - val_loss: 0.6697 - val_acc: 0.7500
Epoch 8/20
0s - loss: 0.3634 - acc: 0.8625 - val_loss: 0.7863 - val_acc: 0.6000
Epoch 9/20
0s - loss: 0.3516 - acc: 0.8625 - val_loss: 0.8624 - val_acc: 0.6500
Epoch 10/20
0s - loss: 0.2364 - acc: 0.9125 - val_loss: 0.7997 - val_acc: 0.5000
Epoch 11/20
0s - loss: 0.2183 - acc: 0.9500 - val_loss: 1.0836 - val_acc: 0.4500
Epoch 12/20
0s - loss: 0.1658 - acc: 0.9500 - val_loss: 1.1220 - val_acc: 0.4500
Epoch 13/20
0s - loss: 0.1018 - acc: 0.9750 - val_loss: 1.4189 - val_acc: 0.5000
Epoch 14/20
0s - loss: 0.1085 - acc: 0.9625 - val_loss: 1.3545 - val_acc: 0.5500
Epoch 15/20
0s - loss: 0.2012 - acc: 0.9375 - val_loss: 1.2497 - val_acc: 0.5500
2017-10-07 17:54:21,912 [INFO] Evaluate...
2017-10-07 17:54:22,468 [INFO] Done!
2017-10-07 17:54:22,474 [INFO] tpe_transform took 0.002084 seconds
2017-10-07 17:54:22,475 [INFO] TPE using 1/1 trials with best loss -0.600000
2017-10-07 17:54:22,479 [INFO] Load data...
2017-10-07 17:54:27,459 [INFO] Fit...
['loss', 'acc']
Train on 80 samples, validate on 20 samples
Epoch 1/20
1s - loss: 0.6927 - acc: 0.5500 - val_loss: 0.6739 - val_acc: 0.7000
Epoch 2/20
0s - loss: 0.6863 - acc: 0.5500 - val_loss: 0.6485 - val_acc: 0.7000
Epoch 3/20
0s - loss: 0.6751 - acc: 0.5500 - val_loss: 0.6320 - val_acc: 0.7000
Epoch 4/20
0s - loss: 0.6750 - acc: 0.5500 - val_loss: 0.6221 - val_acc: 0.7000
Epoch 5/20
0s - loss: 0.6869 - acc: 0.5500 - val_loss: 0.6367 - val_acc: 0.7000
Epoch 6/20
0s - loss: 0.6581 - acc: 0.5750 - val_loss: 0.6323 - val_acc: 0.6500
Epoch 7/20
0s - loss: 0.6562 - acc: 0.6000 - val_loss: 0.6239 - val_acc: 0.6500
Epoch 8/20
0s - loss: 0.6367 - acc: 0.6125 - val_loss: 0.5990 - val_acc: 0.6500
Epoch 9/20
0s - loss: 0.6026 - acc: 0.7000 - val_loss: 0.6150 - val_acc: 0.7500
Epoch 10/20
0s - loss: 0.5760 - acc: 0.7875 - val_loss: 0.5984 - val_acc: 0.8000
Epoch 11/20
0s - loss: 0.5383 - acc: 0.8125 - val_loss: 0.6042 - val_acc: 0.7500
Epoch 12/20
0s - loss: 0.5136 - acc: 0.8000 - val_loss: 0.5976 - val_acc: 0.6500
Epoch 13/20
0s - loss: 0.4534 - acc: 0.7875 - val_loss: 0.6000 - val_acc: 0.7000
Epoch 14/20
0s - loss: 0.4166 - acc: 0.8625 - val_loss: 0.6372 - val_acc: 0.6500
Epoch 15/20
0s - loss: 0.3760 - acc: 0.8625 - val_loss: 0.6083 - val_acc: 0.7000
Epoch 16/20
0s - loss: 0.3851 - acc: 0.8375 - val_loss: 0.6039 - val_acc: 0.7000
Epoch 17/20
0s - loss: 0.3941 - acc: 0.8125 - val_loss: 0.6135 - val_acc: 0.7500
Epoch 18/20
0s - loss: 0.4082 - acc: 0.7875 - val_loss: 0.6570 - val_acc: 0.7000
Epoch 19/20
0s - loss: 0.3180 - acc: 0.9125 - val_loss: 0.6680 - val_acc: 0.6500
Epoch 20/20
0s - loss: 0.3227 - acc: 0.8875 - val_loss: 0.6794 - val_acc: 0.6500
2017-10-07 17:54:37,040 [INFO] Evaluate...
2017-10-07 17:54:37,782 [INFO] Done!

Distributed workers - (K)MongoTrials


In [73]:
import tempfile
import os
import subprocess
# Start the mongodb database and a worker
mongodb_path = tempfile.mkdtemp()
results_path = tempfile.mkdtemp()

proc_args = ["mongod",
             "--dbpath=%s" % mongodb_path,
             "--noprealloc",
             "--port=22334"]
print("starting mongod", proc_args)
mongodb_proc = subprocess.Popen(
        proc_args,
        cwd=mongodb_path,
)


starting mongod ['mongod', '--dbpath=/tmp/tmp7e0lvesw', '--noprealloc', '--port=22334']

In [74]:
# Start the worker
from kopt.utils import merge_dicts
proc_args_worker = ["hyperopt-mongo-worker",
                    "--mongo=localhost:22334/imdb",
                    "--poll-interval=0.1"]

mongo_worker_proc = subprocess.Popen(
    proc_args_worker,
    env=merge_dicts(os.environ, {"PYTHONPATH": os.getcwd()}),
)

In [75]:
## In Order for pickling of the functions to work,
## we need to import the functions from a module different
## than __main___
## I've implemented them in model.py and data.py
import model
import data
objective.data_fn = data.data
objective.model_fn = model.model
objective.save_dir = results_path

In [76]:
# run hyper-parameter optimization in parallel (saving the results to MonogoDB)
# Follow the hyperopt guide:
# https://github.com/hyperopt/hyperopt/wiki/Parallelizing-Evaluations-During-Search-via-MongoDB
# KMongoTrials extends hyperopt.MongoTrials with convenience methods
trials = KMongoTrials(db_name, exp_name,
                      ip="localhost",
                      port=22334)
best = fmin(objective, hyper_params, trials=trials, algo=tpe.suggest, max_evals=2)


2017-10-07 18:26:16,861 [INFO] PROTOCOL mongo
2017-10-07 18:26:16,863 [INFO] USERNAME None
2017-10-07 18:26:16,865 [INFO] HOSTNAME localhost
2017-10-07 18:26:16,866 [INFO] PORT 22334
2017-10-07 18:26:16,867 [INFO] PATH /imdb/jobs
2017-10-07 18:26:16,868 [INFO] DB imdb
2017-10-07 18:26:16,870 [INFO] COLLECTION jobs
2017-10-07 18:26:16,871 [INFO] PASS None
2017-10-07 18:26:17,305 [WARNING] no last_id found, re-trying
2017-10-07 18:26:18,315 [INFO] tpe_transform took 0.002277 seconds
2017-10-07 18:26:18,316 [INFO] TPE using 0 trials
2017-10-07 18:26:19,336 [INFO] tpe_transform took 0.003541 seconds
2017-10-07 18:26:19,338 [INFO] TPE using 1/1 trials with best loss inf
2017-10-07 18:26:20,348 [INFO] Waiting for 2 jobs to finish ...
2017-10-07 18:26:58,425 [INFO] Queue empty, exiting run.

Analyze the trials


In [91]:
# Number of submitted trials
len(trials)


Out[91]:
2

In [78]:
# ALl the traial information in one tidy pd.DataFrame
trials.as_df()


Out[78]:
tid loss status eval.acc eval.loss name.data name.model name.optim_metric name.optim_metric_mode param.data.max_features param.data.maxlen param.fit.batch_size param.fit.early_stop_monitor param.fit.epochs param.fit.patience param.model.dropout param.model.embedding_dims param.model.lr param.model.rnn_units path.model path.results time.duration.dataload time.duration.total time.duration.training time.end time.start n_epoch
0 1 -0.75 ok 0.75 0.5620 data model acc acc 100 80 32 val_loss 20 10 0.0161 64 0.0083 64 /tmp/tmp2cmnwyi8/imdb... /tmp/tmp2cmnwyi8/imdb... 5.7646 18.6963 12.9317 2017-10-07 18:26:57.1... 2017-10-07 18:26:38.4... 18
1 2 -0.80 ok 0.80 0.6083 data model acc acc 100 80 32 val_loss 20 10 0.3088 64 0.0009 64 /tmp/tmp2cmnwyi8/imdb... /tmp/tmp2cmnwyi8/imdb... 5.5815 18.7385 13.1570 2017-10-07 18:26:58.3... 2017-10-07 18:26:39.6... 19

In [79]:
# load the best model
model = trials.load_model(trials.best_trial_tid())
model


Out[79]:
<keras.models.Sequential at 0x7fae6023ef28>

In [83]:
# see the training history of the best model
train_hist = trials.train_history(trials.best_trial_tid())
train_hist


Out[83]:
tid epoch acc loss val_acc val_loss
0 2 0 0.5375 0.6933 0.7 0.6887
1 2 1 0.5625 0.6898 0.7 0.6781
2 2 2 0.5500 0.6864 0.7 0.6696
... ... ... ... ... ... ...
17 2 17 0.7000 0.6345 0.7 0.6209
18 2 18 0.6750 0.6151 0.8 0.6083
19 2 19 0.7375 0.5972 0.8 0.6119

20 rows × 6 columns


In [89]:
# close the processes (mongodb & hyperopt worker)
mongo_worker_proc.kill()
mongodb_proc.kill()