In [ ]:
from neon.backends import gen_backend
from neon.initializers import Gaussian
from neon.layers import Affine
from neon.data import MNIST
from neon.transforms import Rectlin, Softmax
from neon.models import Model
from neon.layers import GeneralizedCost
from neon.transforms import CrossEntropyMulti
from neon.optimizers import GradientDescentMomentum
be = gen_backend(batch_size=128)
mnist = MNIST(path='data/')
train_set = mnist.train_iter
test_set = mnist.valid_iter
init_norm = Gaussian(loc=0.0, scale=0.01)
layers = []
layers.append(Affine(nout=100, init=init_norm, activation=Rectlin()))
layers.append(Affine(nout=10, init=init_norm,
activation=Softmax()))
mlp = Model(layers=layers)
cost = GeneralizedCost(costfunc=CrossEntropyMulti())
optimizer = GradientDescentMomentum(0.1, momentum_coef=0.9)
This callback makes use of new features in bokeh 0.11, which needs to be installed before running the callback.
We can install the pip package using the notebook terminal or from inside the notebook itself.
After installation, execute 'Kernel-> restart and run all' to reload the kernel with the newly installed package version.
In [ ]:
import subprocess
subprocess.check_output(['pip', 'install', 'bokeh==0.11'])
Neon provides an API for calling operations during the model fit. The progress bars displayed during training are an example of a callback, and we'll go through the process of adding a new callback that visualizes cost graphically instead of printing to screen.
To make a new callback, subclass from Callback
, and implement the desired callback methods.
Each of the callback functions have access to callback_data
and model
objects. callback_data
is an H5 file that is saved when supplying the -o
flag to neon, and callbacks should store any computed data into callback_data
. Visualization callbacks can read already computed data such as training or validation cost from callback_data
.
This callback implements the subset of the available callback functions that it needs: http://neon.nervanasys.com/docs/latest/callbacks.html#creating-callbacks
In [ ]:
from neon.callbacks.callbacks import Callbacks, Callback
from bokeh.plotting import output_notebook, figure, ColumnDataSource, show
from bokeh.io import push_notebook
from timeit import default_timer
class CostVisCallback(Callback):
"""
Callback providing a live updating console based progress bar.
"""
def __init__(self, epoch_freq=1,
minibatch_freq=1, update_thresh_s=0.65):
super(CostVisCallback, self).__init__(epoch_freq=epoch_freq,
minibatch_freq=minibatch_freq)
self.update_thresh_s = update_thresh_s
output_notebook()
self.fig = figure(name="cost", title="Cost", x_axis_label="Epoch", plot_width=900)
self.train_source = ColumnDataSource(data=dict(x=[], y0=[]))
self.train_cost = self.fig.line(x=[], y=[], source=self.train_source)
self.val_source = ColumnDataSource(data=dict(x=[], y0=[]))
self.val_cost = self.fig.line(x=[], y=[], source=self.val_source, color='red')
def on_train_begin(self, callback_data, model, epochs):
"""
A good place for one-time startup operations, such as displaying the figure.
"""
show(self.fig)
def on_epoch_begin(self, callback_data, model, epoch):
"""
Since the number of minibatches per epoch is not constant, calculate it here.
"""
self.start_epoch = self.last_update = default_timer()
self.nbatches = model.nbatches
def on_minibatch_end(self, callback_data, model, epoch, minibatch):
"""
Read the training cost already computed by the TrainCostCallback out of 'callback_data', and display it.
"""
now = default_timer()
mb_complete = minibatch + 1
mbstart = callback_data['time_markers/minibatch'][epoch-1] if epoch > 0 else 0
train_cost = callback_data['cost/train'][mbstart + minibatch]
mb_epoch_scale = epoch + minibatch / float(self.nbatches)
self.train_source.data['x'].append(mb_epoch_scale)
self.train_source.data['y'].append(train_cost)
if (now - self.last_update > self.update_thresh_s or mb_complete == self.nbatches):
self.last_update = now
push_notebook()
def on_epoch_end(self, callback_data, model, epoch):
"""
If per-epoch validation cost is being computed by the LossCallback, plot that too.
"""
_eil = self._get_cached_epoch_loss(callback_data, model, epoch, 'loss')
if _eil:
self.val_source.data['x'].append(1 + epoch)
self.val_source.data['y'].append(_eil['cost'])
push_notebook()
In [ ]:
callbacks = Callbacks(mlp, eval_set=test_set)
cv = CostVisCallback()
callbacks.add_callback(cv)
mlp.fit(train_set, optimizer=optimizer, num_epochs=10, cost=cost, callbacks=callbacks)
In [ ]: