In [2]:

    

import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf
import os
import sys
from six.moves import cPickle as pickle
%matplotlib inline


    

    




/home/josh/anaconda2/envs/tensorflow/lib/python2.7/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.
  warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')

In [3]:

    

#pickle_file = 'train.pickle'
'''
with open(pickle_file, 'rb') as f:
    save = pickle.load(f)
    train_X_1 = save['data']
    train_outcome_1 = save['outcome']
    del save  # hint to help gc free up memory
'''
    
pickle_file = 'train2.pickle'

with open(pickle_file, 'rb') as f:
    save = pickle.load(f)
    train_X_0 = save['data']
    train_outcome_0 = save['outcome']
    del save  # hint to help gc free up memory
    
'''
pickle_file = 'test.pickle'

with open(pickle_file, 'rb') as f:
    save = pickle.load(f)
    test_X_1 = save['data']
    test_outcome_1 = save['outcome']
    del save  # hint to help gc free up memory
    
'''

pickle_file = 'test2.pickle'

with open(pickle_file, 'rb') as f:
    save = pickle.load(f)
    test_X_0 = save['data']
    test_outcome_0 = save['outcome']
    del save  # hint to help gc free up memory


    

In [4]:

    

#reformat the label
#for each digit, add a 'end_digit' as '10'
#for each label, add a digit size
#each of them is a one-hot coding

def label_reformat(label, max_size = 5):
    digit_size = np.asarray([len(x) for x in label])
    digit_size[digit_size > max_size]= max_size
    digit_size = ((np.arange(max_size)+1) == digit_size[:,None]).astype(np.float32)
    
    digits = {}
    end_digit = 10.0
    for i in range(max_size):
        digit_coding = np.asarray( [x[i] if len(x)>i else end_digit for x in label])
        digit_coding = (np.arange(end_digit+1) == digit_coding[:,None]).astype(np.float32)
        digits['digit_'+ str(i)] = digit_coding
        
    return digit_size, digits


    

In [5]:

    

#train_X_0 = np.vstack((train_X_1 ,train_X_2 ))


    

In [6]:

    

train_X_0.shape


    

    
Out[6]:





(33402, 64, 64, 3)

In [7]:

    

#train_X_0 = np.vstack((train_X_1 ,train_X_2 ))

image_size = train_X_0.shape[1]
num_channels = train_X_0.shape[3]
batch_size = 100
val_size = 50
test_size = 50


#train_label = train_outcome_1['label'] +  train_outcome_2['label']
train_label = train_outcome_0['label']
train_digit_size, train_digits = label_reformat(train_label)
train_X = train_X_0


val_label = test_outcome_0['label']
val_digit_size, val_digits = label_reformat(val_label)
val_X = test_X_0

val_size = val_X.shape[0]


    

In [8]:

    

print train_digit_size.shape
print train_digits['digit_0'].shape
print train_X.shape


    

    




(33402, 5)
(33402, 11)
(33402, 64, 64, 3)

In [9]:

    

plt.imshow(train_X[0,:,:,:])
plt.show()
print train_digits['digit_0'][0]
print train_digits['digit_1'][0]


    

    













    




[ 0.  1.  0.  0.  0.  0.  0.  0.  0.  0.  0.]
[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  1.  0.]

In [10]:

    

plt.imshow(val_X[1,:,:,:])
plt.show()
print val_digits['digit_0'][1]
print val_digits['digit_1'][1]


    

    













    




[ 0.  0.  1.  0.  0.  0.  0.  0.  0.  0.  0.]
[ 0.  1.  0.  0.  0.  0.  0.  0.  0.  0.  0.]

In [11]:

    

def next_batch(X, y_dsize, y_ds, batch_size=50, replace = True):
    idx = np.random.choice(X.shape[0],batch_size, replace = replace)
    batch_x = X[idx,:,:,:]
    batch_y_dsize = y_dsize[idx,:]
    batch_y_d1 = y_ds['digit_0'][idx,:]
    batch_y_d2 = y_ds['digit_1'][idx,:]
    batch_y_d3 = y_ds['digit_2'][idx,:]
    batch_y_d4 = y_ds['digit_3'][idx,:]
    batch_y_d5 = y_ds['digit_4'][idx,:]
    
    return batch_x, batch_y_dsize, batch_y_d1, batch_y_d2, batch_y_d3, batch_y_d4, batch_y_d5


    

In [12]:

    

reg = 1e-4

graph = tf.Graph()
with graph.as_default():
    
    def weight_variable(shape):
        initial = tf.truncated_normal(shape, stddev=0.01)
        return tf.Variable(initial)

    def bias_variable(shape):
        initial = tf.constant(1.0, shape=shape)
        return tf.Variable(initial)

    def conv2d(x, W):
        conv = tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')
        return conv

    def max_pool_2x2(x):
        return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
    
    def max_pool_2x2_same(x):
        return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 1, 1, 1], padding='SAME')
    
    x_image = tf.placeholder(tf.float32, shape=(batch_size, image_size, image_size, num_channels))

    y_d1 = tf.placeholder(tf.float32, shape=(batch_size, 11))
    y_d2 = tf.placeholder(tf.float32, shape=(batch_size, 11))
    y_d3 = tf.placeholder(tf.float32, shape=(batch_size, 11))
    y_d4 = tf.placeholder(tf.float32, shape=(batch_size, 11))
    y_d5 = tf.placeholder(tf.float32, shape=(batch_size, 11))

    y_dsize = tf.placeholder(tf.float32, shape=(batch_size, 5))
    
    val_x_image = tf.placeholder(tf.float32, shape=(val_size, image_size, image_size, num_channels))

    val_y_d1 = tf.placeholder(tf.float32, shape=(val_size, 11))
    val_y_d2 = tf.placeholder(tf.float32, shape=(val_size, 11))
    val_y_d3 = tf.placeholder(tf.float32, shape=(val_size, 11))
    val_y_d4 = tf.placeholder(tf.float32, shape=(val_size, 11))
    val_y_d5 = tf.placeholder(tf.float32, shape=(val_size, 11))

    val_y_dsize = tf.placeholder(tf.float32, shape=(val_size, 5))

    test_x_image = tf.placeholder(tf.float32, shape=(test_size, image_size, image_size, num_channels))

    test_y_d1 = tf.placeholder(tf.float32, shape=(test_size, 11))
    test_y_d2 = tf.placeholder(tf.float32, shape=(test_size, 11))
    test_y_d3 = tf.placeholder(tf.float32, shape=(test_size, 11))
    test_y_d4 = tf.placeholder(tf.float32, shape=(test_size, 11))
    test_y_d5 = tf.placeholder(tf.float32, shape=(test_size, 11))

    test_y_dsize = tf.placeholder(tf.float32, shape=(test_size, 5))
    
    
    W_conv1 = weight_variable([5, 5, num_channels, 32])
    b_conv1 = bias_variable([32])

    h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)
    h_pool1 = max_pool_2x2(h_conv1)
    
    
    W_conv2 = weight_variable([5, 5, 32, 64])
    b_conv2 = bias_variable([64])

    h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
    h_pool2 = max_pool_2x2_same(h_conv2)
    
    #W_fc1 = weight_variable([16 * 16 * 64, 1024])
    #b_fc1 = bias_variable([1024])

    #h_pool2_flat = tf.reshape(h_pool2, [-1, 16*16*64])
    #h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)
    
    W_conv3 = weight_variable([5, 5, 64, 128])
    b_conv3 = bias_variable([128])

    h_conv3 = tf.nn.relu(conv2d(h_pool2, W_conv3) + b_conv3)
    h_pool3 = max_pool_2x2(h_conv3)
    
        
    W_conv4 = weight_variable([5, 5, 128, 160])
    b_conv4 = bias_variable([160])

    h_conv4 = tf.nn.relu(conv2d(h_pool3, W_conv4) + b_conv4)
    h_pool4 = max_pool_2x2_same(h_conv4)
    
    '''
    W_conv5 = weight_variable([5, 5, 160, 160])
    b_conv5 = bias_variable([160])

    h_conv5 = tf.nn.relu(conv2d(h_pool4, W_conv5) + b_conv5)
    h_pool5 = max_pool_2x2_same(h_conv5)
    
    
    W_conv6 = weight_variable([5, 5, 180, 180])
    b_conv6 = bias_variable([180])

    h_conv6 = tf.nn.relu(conv2d(h_pool5, W_conv6) + b_conv6)
    h_pool6 = max_pool_2x2_same(h_conv6)
    
    '''

    W_fc1 = weight_variable([16 * 16 * 160, 1024])
    b_fc1 = bias_variable([1024])

    h_pool6_flat = tf.reshape(h_pool4, [-1, 16*16*160])
    h_fc1 = tf.nn.relu(tf.matmul(h_pool6_flat, W_fc1) + b_fc1)
    
    keep_prob = tf.placeholder(tf.float32)
    h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
    
    
    #first digit
    W_fc2_d1 = weight_variable([1024, 11])
    b_fc2_d1 = bias_variable([11])

    y_conv_d1 = tf.matmul(h_fc1_drop, W_fc2_d1) + b_fc2_d1

    #second digit
    W_fc2_d2 = weight_variable([1024, 11])
    b_fc2_d2 = bias_variable([11])

    y_conv_d2 = tf.matmul(h_fc1_drop, W_fc2_d2) + b_fc2_d2

    #third digit
    W_fc2_d3 = weight_variable([1024, 11])
    b_fc2_d3 = bias_variable([11])

    y_conv_d3 = tf.matmul(h_fc1_drop, W_fc2_d3) + b_fc2_d3

    #fourth digit
    W_fc2_d4 = weight_variable([1024, 11])
    b_fc2_d4 = bias_variable([11])

    y_conv_d4 = tf.matmul(h_fc1_drop, W_fc2_d4) + b_fc2_d4

    #fifth digit
    W_fc2_d5 = weight_variable([1024, 11])
    b_fc2_d5 = bias_variable([11])

    y_conv_d5 = tf.matmul(h_fc1_drop, W_fc2_d5) + b_fc2_d5

    #digit size
    W_fc2_dsize = weight_variable([1024, 5])
    b_fc2_dsize = bias_variable([5])

    y_conv_dsize = tf.matmul(h_fc1_drop, W_fc2_dsize) + b_fc2_dsize

    
    cross_entropy = ( tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(y_conv_d1, y_d1)) 
                     + tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(y_conv_d2, y_d2))
                     + tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(y_conv_d3, y_d3))
                     + tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(y_conv_d4, y_d4))
                     + tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(y_conv_d5, y_d5))
                     + tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(y_conv_dsize, y_dsize))
                     ) + reg *(tf.nn.l2_loss(W_conv1) + tf.nn.l2_loss(W_conv2) 
                               + tf.nn.l2_loss(W_conv3) + tf.nn.l2_loss(W_conv4) 
                               + tf.nn.l2_loss(W_fc1)
                               + tf.nn.l2_loss(W_fc2_d1) + tf.nn.l2_loss(W_fc2_d2) 
                               + tf.nn.l2_loss(W_fc2_d3) + tf.nn.l2_loss(W_fc2_d4) 
                               + tf.nn.l2_loss(W_fc2_d5) + tf.nn.l2_loss(W_fc2_dsize) 
                              ) 

    train_step = tf.train.AdamOptimizer(1e-2,epsilon=0.1).minimize(cross_entropy)
    #train_step = tf.train.tf.train.RMSPropOptimizer(1e-4).minimize(cross_entropy)
    
    #let's just check the first digit
    correct_prediction = tf.equal(tf.argmax(y_conv_d1,1), tf.argmax(y_d1,1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))


    

In [13]:

    

num_steps = 50000
summary_frequency = 100

with tf.Session(graph=graph) as session:

    tf.initialize_all_variables().run()
    print('Initialized')

    for i in range(num_steps):
        (batch_x, batch_y_dsize, 
         batch_y_d1, batch_y_d2, 
         batch_y_d3, batch_y_d4, batch_y_d5) = next_batch(train_X, 
                                                          train_digit_size, 
                                                          train_digits, batch_size)
        feed_dict={
                x_image: batch_x, y_dsize: batch_y_dsize,
                y_d1: batch_y_d1, y_d2: batch_y_d2, y_d3: batch_y_d3,
                y_d4: batch_y_d4, y_d5: batch_y_d5,
                keep_prob: 0.5}
        
        session.run(train_step,feed_dict=feed_dict)
    
        if i%summary_frequency == 0:
            train_accuracy = accuracy.eval(feed_dict=feed_dict)
            print("step %d, training accuracy %g"%(i, train_accuracy))
        
        if i%(summary_frequency*10) ==0:
            (batch_x, batch_y_dsize, 
             batch_y_d1, batch_y_d2, 
             batch_y_d3, batch_y_d4, batch_y_d5) = next_batch(val_X, 
                                                              val_digit_size, 
                                                              val_digits, batch_size)
            val_accuracy = accuracy.eval(feed_dict={
                        x_image: batch_x, y_dsize: batch_y_dsize,
                        y_d1: batch_y_d1, y_d2: batch_y_d2, y_d3: batch_y_d3,
                        y_d4: batch_y_d4, y_d5: batch_y_d5,
                        keep_prob: 1.0})
            print("step %d, val accuracy %g"%(i, val_accuracy))


    

    




Initialized
step 0, training accuracy 0.29
step 0, val accuracy 0.33
step 100, training accuracy 0.23
step 200, training accuracy 0.23
step 300, training accuracy 0.18
step 400, training accuracy 0.25
step 500, training accuracy 0.32
step 600, training accuracy 0.28
step 700, training accuracy 0.23
step 800, training accuracy 0.27
step 900, training accuracy 0.33
step 1000, training accuracy 0.32
step 1000, val accuracy 0.27
step 1100, training accuracy 0.28
step 1200, training accuracy 0.28
step 1300, training accuracy 0.26
step 1400, training accuracy 0.29
step 1500, training accuracy 0.3
step 1600, training accuracy 0.25
step 1700, training accuracy 0.23
step 1800, training accuracy 0.24
step 1900, training accuracy 0.24
step 2000, training accuracy 0.27
step 2000, val accuracy 0.21
step 2100, training accuracy 0.25
step 2200, training accuracy 0.26
step 2300, training accuracy 0.2
step 2400, training accuracy 0.31
step 2500, training accuracy 0.34
step 2600, training accuracy 0.24
step 2700, training accuracy 0.29
step 2800, training accuracy 0.24
step 2900, training accuracy 0.26
step 3000, training accuracy 0.23
step 3000, val accuracy 0.22
step 3100, training accuracy 0.27
step 3200, training accuracy 0.39
step 3300, training accuracy 0.5
step 3400, training accuracy 0.6
step 3500, training accuracy 0.64
step 3600, training accuracy 0.72
step 3700, training accuracy 0.67
step 3800, training accuracy 0.69
step 3900, training accuracy 0.79
step 4000, training accuracy 0.8
step 4000, val accuracy 0.71
step 4100, training accuracy 0.79
step 4200, training accuracy 0.78
step 4300, training accuracy 0.88
step 4400, training accuracy 0.77
step 4500, training accuracy 0.85
step 4600, training accuracy 0.8
step 4700, training accuracy 0.89
step 4800, training accuracy 0.86
step 4900, training accuracy 0.88
step 5000, training accuracy 0.92
step 5000, val accuracy 0.8
step 5100, training accuracy 0.88
step 5200, training accuracy 0.9
step 5300, training accuracy 0.95
step 5400, training accuracy 0.87
step 5500, training accuracy 0.91
step 5600, training accuracy 0.91
step 5700, training accuracy 0.96
step 5800, training accuracy 0.94
step 5900, training accuracy 0.92
step 6000, training accuracy 0.94
step 6000, val accuracy 0.85
step 6100, training accuracy 0.89
step 6200, training accuracy 0.91
step 6300, training accuracy 0.92
step 6400, training accuracy 0.89
step 6500, training accuracy 0.88
step 6600, training accuracy 0.94
step 6700, training accuracy 0.91
step 6800, training accuracy 0.98
step 6900, training accuracy 0.96
step 7000, training accuracy 0.92
step 7000, val accuracy 0.84
step 7100, training accuracy 0.9
step 7200, training accuracy 0.95
step 7300, training accuracy 0.91
step 7400, training accuracy 0.96
step 7500, training accuracy 0.93
step 7600, training accuracy 0.99
step 7700, training accuracy 0.96
step 7800, training accuracy 0.95
step 7900, training accuracy 0.96
step 8000, training accuracy 0.98
step 8000, val accuracy 0.92
step 8100, training accuracy 0.94
step 8200, training accuracy 0.92
step 8300, training accuracy 0.96
step 8400, training accuracy 0.95
step 8500, training accuracy 0.92
step 8600, training accuracy 0.97
step 8700, training accuracy 0.95
step 8800, training accuracy 0.89
step 8900, training accuracy 0.93
step 9000, training accuracy 0.95
step 9000, val accuracy 0.86
step 9100, training accuracy 0.97
step 9200, training accuracy 0.95
step 9300, training accuracy 0.95
step 9400, training accuracy 0.92
step 9500, training accuracy 0.95
step 9600, training accuracy 0.96
step 9700, training accuracy 0.98
step 9800, training accuracy 0.97
step 9900, training accuracy 0.98
step 10000, training accuracy 0.94
step 10000, val accuracy 0.86
step 10100, training accuracy 0.96
step 10200, training accuracy 0.97
step 10300, training accuracy 0.94
step 10400, training accuracy 1
step 10500, training accuracy 0.98
step 10600, training accuracy 0.96
step 10700, training accuracy 0.97
step 10800, training accuracy 0.99
step 10900, training accuracy 0.98
step 11000, training accuracy 0.95
step 11000, val accuracy 0.85
step 11100, training accuracy 0.97
step 11200, training accuracy 0.99
step 11300, training accuracy 0.93
step 11400, training accuracy 0.99
step 11500, training accuracy 0.97
step 11600, training accuracy 1
step 11700, training accuracy 0.99
step 11800, training accuracy 0.96
step 11900, training accuracy 0.97
step 12000, training accuracy 0.96
step 12000, val accuracy 0.89
step 12100, training accuracy 0.98
step 12200, training accuracy 0.96






    




---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-13-731d8adccdf4> in <module>()
     19                 keep_prob: 0.5}
     20 
---> 21         session.run(train_step,feed_dict=feed_dict)
     22 
     23         if i%summary_frequency == 0:

/home/josh/anaconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
    715     try:
    716       result = self._run(None, fetches, feed_dict, options_ptr,
--> 717                          run_metadata_ptr)
    718       if run_metadata:
    719         proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)

/home/josh/anaconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
    913     if final_fetches or final_targets:
    914       results = self._do_run(handle, final_targets, final_fetches,
--> 915                              feed_dict_string, options, run_metadata)
    916     else:
    917       results = []

/home/josh/anaconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
    963     if handle is None:
    964       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 965                            target_list, options, run_metadata)
    966     else:
    967       return self._do_call(_prun_fn, self._session, handle, feed_dict,

/home/josh/anaconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
    970   def _do_call(self, fn, *args):
    971     try:
--> 972       return fn(*args)
    973     except errors.OpError as e:
    974       message = compat.as_text(e.message)

/home/josh/anaconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run_fn(session, feed_dict, fetch_list, target_list, options, run_metadata)
    952         return tf_session.TF_Run(session, options,
    953                                  feed_dict, fetch_list, target_list,
--> 954                                  status, run_metadata)
    955 
    956     def _prun_fn(session, handle, feed_dict, fetch_list):

KeyboardInterrupt: 

In [ ]:

    




    

In [ ]:

    




    

In [ ]: