In [1]:

    

from scipy.io import loadmat
import plotly.offline as py
import plotly.graph_objs as go
from plotly import tools
import numpy as np
from tensorbuilder import tensorbuilder as tb
import tensorflow as tf
from sklearn.preprocessing import OneHotEncoder
from datetime import datetime

py.init_notebook_mode()


    

    




/usr/local/lib/python2.7/dist-packages/requests/packages/urllib3/util/ssl_.py:318: SNIMissingWarning: An HTTPS request has been made, but the SNI (Subject Name Indication) extension to TLS is not available on this platform. This may cause the server to present an incorrect TLS certificate, which can cause validation failures. You can upgrade to a newer version of Python to solve this. For more information, see https://urllib3.readthedocs.io/en/latest/security.html#snimissingwarning.
  SNIMissingWarning
/usr/local/lib/python2.7/dist-packages/requests/packages/urllib3/util/ssl_.py:122: InsecurePlatformWarning: A true SSLContext object is not available. This prevents urllib3 from configuring SSL appropriately and may cause certain SSL connections to fail. You can upgrade to a newer version of Python to solve this. For more information, see https://urllib3.readthedocs.io/en/latest/security.html#insecureplatformwarning.
  InsecurePlatformWarning






    

In [2]:

    

mat = loadmat('mnist.mat')
x_data = mat['X']
y_data = mat['y']

x_data = x_data.reshape([5000, 20, 20, 1])
x_data = np.transpose(x_data, [0, 2, 1, 3]) #transpose
x_data = x_data[:,::-1, :, :] #flip

y_data = y_data % 10
y_data = OneHotEncoder().fit_transform(y_data).todense()

data = tb.data(
    x=x_data, 
    y=y_data
)

[training_data, test_data] = data.split(0.8, 0.2)


    

In [1]:

    

axis_ops = dict(
    autorange=True,
    showgrid=False,
    zeroline=False,
    showline=False,
    autotick=True,
    ticks='',
    showticklabels=False
)
layout = go.Layout(xaxis=axis_ops, yaxis=axis_ops)
fig = tools.make_subplots(rows=2, cols=5, print_grid=False)
def heatmap(z, smooth='best', reversescale=True):
    return go.Heatmap(
        z=z,
        colorscale='Greys',
        reversescale=reversescale,
        showscale=False,
        showlegend=False,
        zsmooth=smooth
    )
def printmat(x):
    print(np.array2string(np.abs(x[::-1, :]), formatter={'float_kind':'{0:.1f}'.format}))

for row in range(1, 3):
    for col in range(1, 6):
        i = 500 * ((row - 1) * 5 + (col - 1)) + 250
        trace = heatmap(x_data[i][:, :, 0])
        fig.append_trace(trace, row, col)

for i in range(10):
    fig['layout']['xaxis{0}'.format(i+1)].update(**axis_ops)
    fig['layout']['yaxis{0}'.format(i+1)].update(**axis_ops)


py.iplot(fig)


    

    





NameErrorTraceback (most recent call last)
<ipython-input-1-f14c237d1257> in <module>()
      8     showticklabels=False
      9 )
---> 10 layout = go.Layout(xaxis=axis_ops, yaxis=axis_ops)
     11 fig = tools.make_subplots(rows=2, cols=5, print_grid=False)
     12 def heatmap(z, smooth='best', reversescale=True):

NameError: name 'go' is not defined

In [4]:

    

sample_indexes = [ 500 * i + 250 for i in range(10) ]

sample_features = x_data[sample_indexes]
sample_labels = y_data[sample_indexes]

sample = sample_features[0][:, 2:-2, 0]

hmap = heatmap(sample)
hlayout = layout.copy()
hlayout.update(width=700,height=700)

py.iplot(go.Figure(data=[hmap], layout=hlayout))
printmat(sample)


    

    












    




[[0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]
 [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]
 [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]
 [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.1 0.3 0.4 0.2 0.0 0.0 0.0 0.0 0.0]
 [0.0 0.0 0.0 0.0 0.0 0.0 0.1 0.7 0.9 1.0 1.0 0.6 0.1 0.0 0.0 0.0]
 [0.0 0.0 0.0 0.0 0.0 0.0 0.3 1.0 1.0 1.0 0.9 1.0 0.7 0.1 0.0 0.0]
 [0.0 0.0 0.0 0.0 0.0 0.1 0.7 1.0 1.0 0.9 0.2 0.6 1.1 0.5 0.0 0.0]
 [0.0 0.0 0.0 0.0 0.0 0.6 1.0 0.8 0.5 0.2 0.1 0.4 1.1 0.6 0.0 0.0]
 [0.0 0.0 0.0 0.0 0.3 1.0 0.8 0.1 0.0 0.0 0.0 0.2 0.9 0.6 0.0 0.0]
 [0.0 0.0 0.0 0.1 0.8 1.0 0.4 0.0 0.0 0.0 0.0 0.1 0.8 0.6 0.0 0.0]
 [0.0 0.0 0.0 0.5 1.0 0.7 0.0 0.0 0.0 0.0 0.0 0.2 1.0 0.6 0.0 0.0]
 [0.0 0.0 0.2 0.9 1.0 0.2 0.0 0.0 0.0 0.0 0.0 0.6 1.0 0.3 0.0 0.0]
 [0.0 0.0 0.6 1.0 0.6 0.0 0.0 0.0 0.0 0.0 0.4 1.0 0.7 0.0 0.0 0.0]
 [0.0 0.0 0.6 0.9 0.2 0.0 0.0 0.0 0.0 0.3 1.0 0.9 0.2 0.0 0.0 0.0]
 [0.0 0.0 0.7 0.8 0.1 0.0 0.0 0.1 0.4 1.0 0.9 0.2 0.0 0.0 0.0 0.0]
 [0.0 0.0 0.8 0.9 0.3 0.2 0.4 0.8 1.0 0.8 0.2 0.0 0.0 0.0 0.0 0.0]
 [0.0 0.0 0.5 1.0 1.0 1.0 1.0 0.9 0.5 0.1 0.0 0.0 0.0 0.0 0.0 0.0]
 [0.0 0.0 0.0 0.4 0.7 0.7 0.5 0.2 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]
 [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]
 [0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0]]

In [13]:

    

graph = tf.Graph()
with graph.as_default():
    
    with tf.name_scope('inputs'):
        x = tf.placeholder('float', shape=[None, 20, 20, 1], name='x')
        y = tf.placeholder('float', shape=[None, 10], name='y')
        learning_rate = tf.placeholder('float', name='learning_rate')
        keep_prob = tf.placeholder('float', name='keep_prob')
        
    
    #output builders
    
#     with tf.device('/gpu:0'):
    [h, trainer, loss] = tb(
        x,
        tb
        .inception_layer(4, activation_fn=tf.nn.elu), {'inception1'}

        .inception_layer(8, activation_fn=tf.nn.elu), {'inception2'}
        
        .elu_conv2d_layer(64, [3, 3], stride=2)
        
        .elu_conv2d_layer(128, [3, 3], stride=2)
        
        .elu_conv2d_layer(256, [3, 3], padding='VALID')
        
        .elu_conv2d_layer(512, [3, 3], padding='VALID')

        .flatten()

        .dropout(keep_prob)
        
        .linear_layer(10), #, scope='logits'),
        [
            tb.softmax(name='h').on('h')
        ,
            (
                tb.With( tf.name_scope('loss'),
                    tb.softmax_cross_entropy_with_logits(y).reduce_mean().make_scalar_summary('loss').on('loss')
                ),
                tb.minimize(tf.train.AdadeltaOptimizer(learning_rate)).on('trainer')
            )
        ]
        ,
        ['h', 'trainer', 'loss']
    )
    
    with tf.name_scope('accuracy'):
        correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(h, 1))
        accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
        tf.scalar_summary('accuracy', accuracy)
    
    summaries = tf.merge_all_summaries()
    saver = tf.train.Saver()
    
    model_name = "mnist-inception-big"
    logs_dir = "/logs/{0}".format(model_name)
    writter = tf.train.SummaryWriter(logs_dir, graph=graph)

    
def feed_dict(data, rate, prob):
    return {
        x: data.x,
        y: data.y,
        learning_rate: rate,
        keep_prob: prob
    }

def count_parameters(graph):
    import operator
    with graph.as_default():
        prod = lambda iterable: reduce(operator.mul, iterable, 1)
        dims = [ list([ int(d) for d in  v.get_shape()]) for v in tf.all_variables() ]
        vars = [ prod(shape) for shape in dims ]
        return sum(vars)


print "Number of paramters:", count_parameters(graph)


    

    




Number of paramters: 4728390

In [14]:

    

model_dir = model_name + '.model'
    
with tf.Session(graph=graph, config=tf.ConfigProto(log_device_placement=True)) as sess:
    #sess.run(tf.initialize_all_variables()); 
    best = 0.0
    rate = 0.01
    saver.restore(sess, model_dir)
    
    for step, datum in training_data.epochs(20000).batches_of(500).enumerated():
        
        
        [_, _summaries] = sess.run(
            [trainer, summaries], 
            feed_dict=feed_dict(datum, rate, 0.4)
        )
        
        if step % 5 == 0:
            writter.add_summary(_summaries, step)
        
        
        if step % 200 == 0:
            [test_loss, test_accuracy] = sess.run(
                [loss, accuracy],
                feed_dict=feed_dict(test_data, rate, 1.0)
            )
            [train_loss, train_accuracy] = sess.run(
                [loss, accuracy],
                feed_dict=feed_dict(training_data, rate, 1.0)
            )
            
            print "Step ", step
            print "[Test]  loss: {0}, accuracy: {1}".format(test_loss, test_accuracy)
            print "[Train] loss: {0}, accuracy: {1}".format(train_loss, train_accuracy)
            
            if test_accuracy > best:
                print "Saving"
                saver.save(sess, model_dir)
                best = test_accuracy
                
            print ""


    

    




Step  0
[Test]  loss: 2.31219911575, accuracy: 0.0520000010729
[Train] loss: 2.30888175964, accuracy: 0.0572499968112
Saving

Step  200
[Test]  loss: 0.517391860485, accuracy: 0.856999993324
[Train] loss: 0.482131004333, accuracy: 0.869750022888
Saving

Step  400
[Test]  loss: 0.358663737774, accuracy: 0.889000058174
[Train] loss: 0.327243000269, accuracy: 0.903250098228
Saving

Step  600
[Test]  loss: 0.300055027008, accuracy: 0.90600001812
[Train] loss: 0.266403317451, accuracy: 0.920000076294
Saving

Step  800
[Test]  loss: 0.263458251953, accuracy: 0.920000016689
[Train] loss: 0.221202522516, accuracy: 0.937500059605
Saving

Step  1000
[Test]  loss: 0.239001855254, accuracy: 0.924999952316
[Train] loss: 0.185490608215, accuracy: 0.945750117302
Saving

Step  1200
[Test]  loss: 0.213777437806, accuracy: 0.935000061989
[Train] loss: 0.155624687672, accuracy: 0.95500010252
Saving

Step  1400
[Test]  loss: 0.196519300342, accuracy: 0.941000044346
[Train] loss: 0.130503833294, accuracy: 0.962500095367
Saving

Step  1600
[Test]  loss: 0.177425757051, accuracy: 0.945000052452
[Train] loss: 0.108266323805, accuracy: 0.971250116825
Saving

Step  1800
[Test]  loss: 0.1664237082, accuracy: 0.944000124931
[Train] loss: 0.0904407203197, accuracy: 0.976750075817

Step  2000
[Test]  loss: 0.154519081116, accuracy: 0.950000047684
[Train] loss: 0.075721912086, accuracy: 0.980250179768
Saving

Step  2200
[Test]  loss: 0.147462129593, accuracy: 0.953000068665
[Train] loss: 0.0617688894272, accuracy: 0.983500123024
Saving

Step  2400
[Test]  loss: 0.138149544597, accuracy: 0.955000042915
[Train] loss: 0.0505511723459, accuracy: 0.986500024796
Saving

Step  2600
[Test]  loss: 0.134580880404, accuracy: 0.959999978542
[Train] loss: 0.0419578701258, accuracy: 0.989500045776
Saving

Step  2800
[Test]  loss: 0.125545084476, accuracy: 0.960000038147
[Train] loss: 0.0334069170058, accuracy: 0.992000102997
Saving

Step  3000
[Test]  loss: 0.120589375496, accuracy: 0.963999986649
[Train] loss: 0.0272133219987, accuracy: 0.993750035763
Saving

Step  3200
[Test]  loss: 0.118678785861, accuracy: 0.963000059128
[Train] loss: 0.0229837074876, accuracy: 0.994000136852

Step  3400
[Test]  loss: 0.113171651959, accuracy: 0.966000020504
[Train] loss: 0.0182008910924, accuracy: 0.996000051498
Saving

Step  3600
[Test]  loss: 0.114047124982, accuracy: 0.966000020504
[Train] loss: 0.0140790846199, accuracy: 0.996250033379

Step  3800
[Test]  loss: 0.112027212977, accuracy: 0.962000012398
[Train] loss: 0.0110533609986, accuracy: 0.997500121593

Step  4000
[Test]  loss: 0.10522569716, accuracy: 0.969000041485
[Train] loss: 0.00902753416449, accuracy: 0.998000144958
Saving

Step  4200
[Test]  loss: 0.110414326191, accuracy: 0.967000067234
[Train] loss: 0.00714071886614, accuracy: 0.998000025749

Step  4400
[Test]  loss: 0.106964975595, accuracy: 0.967999994755
[Train] loss: 0.00561980204657, accuracy: 0.999250054359

Step  4600
[Test]  loss: 0.112954348326, accuracy: 0.965000033379
[Train] loss: 0.00504284724593, accuracy: 0.999250054359

Step  4800
[Test]  loss: 0.10879112035, accuracy: 0.967000007629
[Train] loss: 0.00370294018649, accuracy: 0.99950003624

Step  5000
[Test]  loss: 0.107033208013, accuracy: 0.97000002861
[Train] loss: 0.00306889181957, accuracy: 0.99975001812
Saving

Step  5200
[Test]  loss: 0.110818549991, accuracy: 0.969000101089
[Train] loss: 0.00255636591464, accuracy: 0.99975001812

Step  5400
[Test]  loss: 0.108322635293, accuracy: 0.965000033379
[Train] loss: 0.00205508712679, accuracy: 1.00000011921

Step  5600
[Test]  loss: 0.109655037522, accuracy: 0.97000002861
[Train] loss: 0.00168000813574, accuracy: 1.00000011921

Step  5800
[Test]  loss: 0.110635280609, accuracy: 0.969000041485
[Train] loss: 0.00152780453209, accuracy: 0.99975001812

Step  6000
[Test]  loss: 0.110643282533, accuracy: 0.971000015736
[Train] loss: 0.0012387256138, accuracy: 1.00000011921
Saving

Step  6200
[Test]  loss: 0.108480110765, accuracy: 0.973000049591
[Train] loss: 0.000999300740659, accuracy: 1.00000011921
Saving

Step  6400
[Test]  loss: 0.109135389328, accuracy: 0.97100007534
[Train] loss: 0.000879580446053, accuracy: 1.00000011921







    





KeyboardInterruptTraceback (most recent call last)
<ipython-input-14-6cd1338bc26a> in <module>()
     12         [_, _summaries] = sess.run(
     13             [trainer, summaries],
---> 14             feed_dict=feed_dict(datum, rate, 0.4)
     15         )
     16 

/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
    715     try:
    716       result = self._run(None, fetches, feed_dict, options_ptr,
--> 717                          run_metadata_ptr)
    718       if run_metadata:
    719         proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)

/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
    913     if final_fetches or final_targets:
    914       results = self._do_run(handle, final_targets, final_fetches,
--> 915                              feed_dict_string, options, run_metadata)
    916     else:
    917       results = []

/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
    963     if handle is None:
    964       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 965                            target_list, options, run_metadata)
    966     else:
    967       return self._do_call(_prun_fn, self._session, handle, feed_dict,

/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
    970   def _do_call(self, fn, *args):
    971     try:
--> 972       return fn(*args)
    973     except errors.OpError as e:
    974       message = compat.as_text(e.message)

/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc in _run_fn(session, feed_dict, fetch_list, target_list, options, run_metadata)
    952         return tf_session.TF_Run(session, options,
    953                                  feed_dict, fetch_list, target_list,
--> 954                                  status, run_metadata)
    955 
    956     def _prun_fn(session, handle, feed_dict, fetch_list):

KeyboardInterrupt: 

In [9]:

    

image_index = 4

with tf.Session(graph=graph) as sess:
    saver.restore(sess, model_dir)
    
    with tf.variable_scope("Conv", reuse=True):
        conv_weights = tf.get_variable("weights").eval().reshape([16, 3, 3])
        
    [_conv1, _conv2] = sess.run([conv1, conv2], feed_dict={x: sample_features[image_index:image_index+1, :, :, :]})
    
        

# weight plots
fig = tools.make_subplots(rows=4, cols=4, print_grid=False)

for row in range(4):
    for col in range(4):
        i = 4 * row + col
        trace = heatmap(conv_weights[i], smooth=None, reversescale=False)
        fig.append_trace(trace, row+1, col+1)

for i in range(16):
    fig['layout']['xaxis{0}'.format(i+1)].update(**axis_ops)
    fig['layout']['yaxis{0}'.format(i+1)].update(**axis_ops)


py.iplot(fig)

# conv1 plots
hmap = heatmap(sample_features[image_index, :, :, 0])
py.iplot(go.Figure(data=[hmap], layout=hlayout))

fig = tools.make_subplots(rows=4, cols=4, print_grid=False)

for row in range(4):
    for col in range(4):
        i = 4 * row + col
        trace = heatmap(_conv1[0, :, :, i], smooth=None)
        fig.append_trace(trace, row+1, col+1)

for i in range(16):
    fig['layout']['xaxis{0}'.format(i+1)].update(**axis_ops)
    fig['layout']['yaxis{0}'.format(i+1)].update(**axis_ops)
    
fig['layout'].update(width=1000,height=1000)


py.iplot(fig)

# conv2 plots
fig = tools.make_subplots(rows=8, cols=4, print_grid=False)

for row in range(8):
    for col in range(4):
        i = 4 * row + col
        trace = heatmap(_conv2[0, :, :, i], smooth=None)
        fig.append_trace(trace, row+1, col+1)

for i in range(32):
    fig['layout']['xaxis{0}'.format(i+1)].update(**axis_ops)
    fig['layout']['yaxis{0}'.format(i+1)].update(**axis_ops)
    
fig['layout'].update(width=1000,height=1000)


py.iplot(fig)


    

In [15]:

    

extra_answers = [4, 6 , 8]
extra_samples = np.array([loadmat('num_{0}.mat'.format(n))['im'][::-1, :] for n in extra_answers])
extra_samples = extra_samples.reshape([3, 20, 20, 1])

answers = range(10) + extra_answers
samples = np.vstack((sample_features, extra_samples))

with tf.Session(graph=graph) as sess:
    saver.restore(sess, model_dir)
    predictions = sess.run(h, feed_dict={x: samples, keep_prob: 1.0})
    prediction_vals = np.argmax(predictions, 1)
    
    
for answer, sample, prediction, prediction_val in zip(answers, samples, predictions, prediction_vals):
    fig = tools.make_subplots(
        rows=1, cols=2, print_grid=False,
        subplot_titles=(
            'Answer {0}'.format(answer),
            'Prediction {0} - {1}'.format(prediction_val, prediction_val == answer)
        )
    )

    heatmap_trace = heatmap(sample[:, :, 0])
    bar_trace = go.Bar(
        x=range(10),
        y=prediction
    )

    fig.append_trace(heatmap_trace, 1, 1)
    fig.append_trace(bar_trace, 1, 2)

    for i in range(2):
        fig['layout']['xaxis{0}'.format(i+1)].update(**axis_ops)
        fig['layout']['yaxis{0}'.format(i+1)].update(**axis_ops)


    py.iplot(fig)


    

In [ ]: