In [1]:
from scipy.io import loadmat
import plotly.offline as py
import plotly.graph_objs as go
from plotly import tools
import numpy as np
import tensorbuilder
from tensorbuilder import tensorbuilder as tb
import tensorflow as tf
from sklearn.preprocessing import OneHotEncoder
from datetime import datetime
reload(tensorbuilder)
py.init_notebook_mode()
In [2]:
mat = loadmat('mnist.mat')
x_data = mat['X']
y_data = mat['y']
x_data = x_data.reshape([5000, 20, 20, 1])
x_data = np.transpose(x_data, [0, 2, 1, 3]) #transpose
x_data = x_data[:,::-1, :, :] #flip
y_data = y_data % 10
y_data = OneHotEncoder().fit_transform(y_data).todense()
data = tb.data(
x=x_data,
y=y_data
)
[training_data, test_data] = data.split(0.8, 0.2)
In [3]:
axis_ops = dict(
autorange=True,
showgrid=False,
zeroline=False,
showline=False,
autotick=True,
ticks='',
showticklabels=False
)
layout = go.Layout(xaxis=axis_ops, yaxis=axis_ops)
fig = tools.make_subplots(rows=2, cols=5, print_grid=False)
def heatmap(z, smooth='best', reversescale=True):
return go.Heatmap(
z=z,
colorscale='Greys',
reversescale=reversescale,
showscale=False,
showlegend=False,
zsmooth=smooth
)
def printmat(x):
print(np.array2string(np.abs(x[::-1, :]), formatter={'float_kind':'{0:.1f}'.format}))
for row in range(1, 3):
for col in range(1, 6):
i = 500 * ((row - 1) * 5 + (col - 1)) + 250
trace = heatmap(x_data[i][:, :, 0])
fig.append_trace(trace, row, col)
for i in range(10):
fig['layout']['xaxis{0}'.format(i+1)].update(**axis_ops)
fig['layout']['yaxis{0}'.format(i+1)].update(**axis_ops)
py.iplot(fig)
In [4]:
sample_indexes = [ 500 * i + 250 for i in range(10) ]
sample_features = x_data[sample_indexes]
sample_labels = y_data[sample_indexes]
sample = sample_features[0][:, 2:-2, 0]
hmap = heatmap(sample)
hlayout = layout.copy()
hlayout.update(width=700,height=700)
py.iplot(go.Figure(data=[hmap], layout=hlayout))
printmat(sample)
In [14]:
graph = tf.Graph()
with graph.as_default():
with tf.name_scope('inputs'):
x = tf.placeholder('float', shape=[None, 20, 20, 1], name='x')
y = tf.placeholder('float', shape=[None, 10], name='y')
learning_rate = tf.placeholder('float', name='learning_rate')
keep_prob = tf.placeholder('float', name='keep_prob')
#output builders
# with tf.device('/gpu:0'):
[h, trainer, conv1, conv2, loss] = tb(
x,
tb
.convolution2d(16, 3).on('conv1') #, scope='conv2d_16_relu_3x3')
.max_pool2d(2)
.convolution2d(32, 3).on('conv2') #, scope='conv2d_32_relu_3x3')
.max_pool2d(2)
.flatten()
.relu_layer(1024)
.dropout(keep_prob)
.linear_layer(10), #, scope='logits'),
[
tb.softmax(name='h').on('h')
,
(
tb.With( tf.name_scope('loss'),
tb.softmax_cross_entropy_with_logits(y).reduce_mean().make_scalar_summary('loss').on('loss')
),
tb.minimize(tf.train.AdadeltaOptimizer(learning_rate)).on('trainer')
)
],
['h', 'trainer', 'conv1', 'conv2', 'loss']
)
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(h, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.scalar_summary('accuracy', accuracy)
summaries = tf.merge_all_summaries()
saver = tf.train.Saver()
logs_dir = "/logs/" + datetime.now().strftime('%m_%d_%Y_%H_%M_%S')
writter = tf.train.SummaryWriter(logs_dir, graph=graph)
def feed_dict(data, rate, prob):
return {
x: data.x,
y: data.y,
learning_rate: rate,
keep_prob: prob
}
In [15]:
model_dir = 'mnist.model'
with tf.Session(graph=graph, config=tf.ConfigProto(log_device_placement=True)) as sess:
#sess.run(tf.initialize_all_variables());
best = 0.0
rate = 0.01
saver.restore(sess, model_dir)
for step, datum in training_data.epochs(20000).batches_of(500).enumerated():
[_, _summaries] = sess.run(
[trainer, summaries],
feed_dict=feed_dict(datum, rate, 0.4)
)
if step % 5 == 0:
writter.add_summary(_summaries, step)
if step % 200 == 0:
[test_loss, test_accuracy] = sess.run(
[loss, accuracy],
feed_dict=feed_dict(test_data, rate, 1.0)
)
[train_loss, train_accuracy] = sess.run(
[loss, accuracy],
feed_dict=feed_dict(training_data, rate, 1.0)
)
print "Step ", step
print "[Test] loss: {0}, accuracy: {1}".format(test_loss, test_accuracy)
print "[Train] loss: {0}, accuracy: {1}".format(train_loss, train_accuracy)
if test_accuracy > best:
print "Saving"
saver.save(sess, model_dir)
best = test_accuracy
print ""
In [8]:
image_index = 7
with tf.Session(graph=graph) as sess:
saver.restore(sess, model_dir)
with tf.variable_scope("Conv", reuse=True):
conv_weights = tf.get_variable("weights").eval().reshape([16, 3, 3])
[_conv1, _conv2] = sess.run([conv1, conv2], feed_dict={x: sample_features[image_index:image_index+1, :, :, :]})
# weight plots
fig = tools.make_subplots(rows=4, cols=4, print_grid=False)
for row in range(4):
for col in range(4):
i = 4 * row + col
trace = heatmap(conv_weights[i], smooth=None, reversescale=False)
fig.append_trace(trace, row+1, col+1)
for i in range(16):
fig['layout']['xaxis{0}'.format(i+1)].update(**axis_ops)
fig['layout']['yaxis{0}'.format(i+1)].update(**axis_ops)
py.iplot(fig)
# conv1 plots
hmap = heatmap(sample_features[image_index, :, :, 0])
py.iplot(go.Figure(data=[hmap], layout=hlayout))
fig = tools.make_subplots(rows=4, cols=4, print_grid=False)
for row in range(4):
for col in range(4):
i = 4 * row + col
trace = heatmap(_conv1[0, :, :, i], smooth=None)
fig.append_trace(trace, row+1, col+1)
for i in range(16):
fig['layout']['xaxis{0}'.format(i+1)].update(**axis_ops)
fig['layout']['yaxis{0}'.format(i+1)].update(**axis_ops)
fig['layout'].update(width=1000,height=1000)
py.iplot(fig)
# conv2 plots
fig = tools.make_subplots(rows=8, cols=4, print_grid=False)
for row in range(8):
for col in range(4):
i = 4 * row + col
trace = heatmap(_conv2[0, :, :, i], smooth=None)
fig.append_trace(trace, row+1, col+1)
for i in range(32):
fig['layout']['xaxis{0}'.format(i+1)].update(**axis_ops)
fig['layout']['yaxis{0}'.format(i+1)].update(**axis_ops)
fig['layout'].update(width=1000,height=1000)
py.iplot(fig)
In [9]:
_conv1.shape
Out[9]:
In [11]:
extra_answers = [4, 6 , 8]
extra_samples = np.array([loadmat('num_{0}.mat'.format(n))['im'][::-1, :] for n in extra_answers])
extra_samples = extra_samples.reshape([3, 20, 20, 1])
answers = range(10) + extra_answers
samples = np.vstack((sample_features, extra_samples))
with tf.Session(graph=graph) as sess:
saver.restore(sess, model_dir)
predictions = sess.run(h, feed_dict={x: samples, keep_prob: 1.0})
prediction_vals = np.argmax(predictions, 1)
for answer, sample, prediction, prediction_val in zip(answers, samples, predictions, prediction_vals):
fig = tools.make_subplots(
rows=1, cols=2, print_grid=False,
subplot_titles=(
'Answer {0}'.format(answer),
'Prediction {0} - {1}'.format(prediction_val, prediction_val == answer)
)
)
heatmap_trace = heatmap(sample[:, :, 0])
bar_trace = go.Bar(
x=range(10),
y=prediction
)
fig.append_trace(heatmap_trace, 1, 1)
fig.append_trace(bar_trace, 1, 2)
for i in range(2):
fig['layout']['xaxis{0}'.format(i+1)].update(**axis_ops)
fig['layout']['yaxis{0}'.format(i+1)].update(**axis_ops)
py.iplot(fig)
In [ ]: