In [1]:

    

import os
import numpy as np
import random
import theano
from pandas.io.parsers import read_csv
from sklearn.utils import shuffle
from scipy import misc
import pandas as pd
from matplotlib import pyplot
from sklearn.metrics import accuracy_score
from sklearn.metrics import log_loss


    

    




Using gpu device 0: GeForce GTX 760

In [2]:

    

import matplotlib.pyplot as plt
 
pd.set_option('display.max_columns',1000)
pd.set_option('display.max_columns', 50)
pd.set_option('display.max_colwidth', 100)
 
plt.style.use('ggplot')


    

In [ ]:

    




    

In [3]:

    

from preprocess_data_lib import *
(X_train, y_train, X_valid, y_valid, X_test, y_test) = getDataRot(cast=True)


    

In [4]:

    

(X_train.shape, y_train.shape, X_valid.shape, y_valid.shape, X_test.shape, y_test.shape)


    

    
Out[4]:





((126000, 784), (126000,), (0, 784), (0,), (28000, 784), (28000,))

In [5]:

    

image_width = image_height = int(X_train.shape[1] ** .5)


    

In [6]:

    

X_train_reshaped = X_train.reshape(X_train.shape[0], 1, image_height, image_width)
X_test_reshaped = X_test.reshape(X_test.shape[0], 1, image_height, image_width)


    

In [ ]:

    




    

In [ ]:

    




    

In [7]:

    

from lasagne import layers
from lasagne.nonlinearities import  softmax, rectify
from lasagne.updates import nesterov_momentum
from nolearn.lasagne import NeuralNet


    

In [ ]:

    




    

In [ ]:

    




    

In [8]:

    

def float32(k):
    return np.cast['float32'](k)


    

In [9]:

    

class AdjustVariable(object):
    def __init__(self, name, start=0.03, stop=0.001):
        self.name = name
        self.start, self.stop = start, stop
        self.ls = None

    def __call__(self, nn, train_history):
        if self.ls is None:
            self.ls = np.linspace(self.start, self.stop, nn.max_epochs)

        epoch = train_history[-1]['epoch']
        new_value = float32(self.ls[epoch - 1])
        getattr(nn, self.name).set_value(new_value)


    

In [ ]:

    




    

In [ ]:

    




    

In [ ]:

    




    

In [10]:

    

ffNet1 = NeuralNet(
    layers=[  # three layers: one hidden layer
        ('input', layers.InputLayer),
        ('hidden', layers.DenseLayer),
        ('output', layers.DenseLayer),
        ],
    # layer parameters:
    input_shape=(None, X_train.shape[1]),  # 96x96 input pixels per batch
    hidden_num_units=20,  # number of units in hidden layer
    output_nonlinearity=softmax,  # output layer uses identity function
    output_num_units=10,  # 30 target values

    # optimization method:
    update=nesterov_momentum,
    update_learning_rate=theano.shared(float32(0.03)),
    update_momentum=theano.shared(float32(0.9)),
    
    regression=False,
    max_epochs=5,  # we want to train this many epochs
    verbose=1,
    
    on_epoch_finished=[
        AdjustVariable('update_learning_rate', start=0.03, stop=0.0001),
        AdjustVariable('update_momentum', start=0.9, stop=0.999),
        ],
    
    )
ffNet1.fit(X_train, y_train)


    

    




  input             	(None, 784)         	produces     784 outputs
  hidden            	(None, 20)          	produces      20 outputs
  output            	(None, 10)          	produces      10 outputs
|   epoch |   train loss |   valid loss |   valid best |   train/val |   val acc |    dur |
|--------:|-------------:|-------------:|-------------:|------------:|----------:|-------:|
|       1 |       0.4410 |       0.3117 |       0.3117 |      1.4148 |    0.9073 | 0.8474 |
|       2 |       0.2698 |       0.2656 |       0.2656 |      1.0158 |    0.9211 | 0.8548 |
|       3 |       0.2393 |       0.2441 |       0.2441 |      0.9805 |    0.9276 | 0.8459 |
|       4 |       0.2225 |       0.2362 |       0.2362 |      0.9421 |    0.9288 | 0.8510 |
|       5 |       0.2115 |       0.2339 |       0.2339 |      0.9040 |    0.9286 | 0.8487 |






    




/home/issam/anaconda/lib/python2.7/site-packages/Lasagne-0.1.dev0-py2.7.egg/lasagne/init.py:86: UserWarning: The uniform initializer no longer uses Glorot et al.'s approach to determine the bounds, but defaults to the range (-0.01, 0.01) instead. Please use the new GlorotUniform initializer to get the old behavior. GlorotUniform is now the default for all layers.
  warnings.warn("The uniform initializer no longer uses Glorot et al.'s "






    
Out[10]:





NeuralNet(X_tensor_type=<function matrix at 0x7f31503d0320>,
     batch_iterator_test=<nolearn.lasagne.base.BatchIterator object at 0x7f313618b390>,
     batch_iterator_train=<nolearn.lasagne.base.BatchIterator object at 0x7f313618b850>,
     custom_score=None, eval_size=0.2, hidden_num_units=20,
     input_shape=(None, 784),
     layers=[('input', <class 'lasagne.layers.input.InputLayer'>), ('hidden', <class 'lasagne.layers.dense.DenseLayer'>), ('output', <class 'lasagne.layers.dense.DenseLayer'>)],
     loss=None, max_epochs=5, more_params={},
     objective=<class 'lasagne.objectives.Objective'>,
     objective_loss_function=<function categorical_crossentropy at 0x7f314ee94758>,
     on_epoch_finished=[<__main__.AdjustVariable object at 0x7f313d319990>, <__main__.AdjustVariable object at 0x7f3135f99350>],
     on_training_finished=(),
     output_nonlinearity=<function softmax at 0x7f3137aff668>,
     output_num_units=10, regression=False,
     update=<function nesterov_momentum at 0x7f31372412a8>,
     update_learning_rate=<CudaNdarrayType(float32, scalar)>,
     update_momentum=<CudaNdarrayType(float32, scalar)>,
     use_label_encoder=False, verbose=1,
     y_tensor_type=TensorType(int32, vector))

In [12]:

    

ffNet2 = NeuralNet(
        layers=[  #list the layers here
            ('input', layers.InputLayer),
            ('hidden1', layers.DenseLayer),
            ('hidden2', layers.DenseLayer),
            ('output', layers.DenseLayer),
            ],

        # layer parameters:
        input_shape=(None, X_train.shape[1]),
        hidden1_num_units=200, hidden1_nonlinearity=rectify,  #params of first layer
        hidden2_num_units=200, hidden2_nonlinearity=rectify,
        output_nonlinearity=softmax,  # softmax for classification problems
        output_num_units=10,  # 10 target values

        # optimization method:
        update=nesterov_momentum,
        update_learning_rate=theano.shared(float32(0.05)),
        update_momentum=theano.shared(float32(0.7)),

        on_epoch_finished=[
            AdjustVariable('update_learning_rate', start=0.05, stop=0.0001),
            AdjustVariable('update_momentum', start=0.7, stop=0.999),
            ],    
    
        regression=False,
        max_epochs=50,  # Intentionally limited for execution speed
        verbose=1,
        )
ffNet2.fit(X_train, y_train)


    

    




  input             	(None, 784)         	produces     784 outputs
  hidden1           	(None, 200)         	produces     200 outputs
  hidden2           	(None, 200)         	produces     200 outputs
  output            	(None, 10)          	produces      10 outputs
|   epoch |   train loss |   valid loss |   valid best |   train/val |   val acc |    dur |
|--------:|-------------:|-------------:|-------------:|------------:|----------:|-------:|
|       1 |       0.3437 |       0.1936 |       0.1936 |      1.7751 |    0.9414 | 1.3447 |
|       2 |       0.1432 |       0.1343 |       0.1343 |      1.0662 |    0.9586 | 1.3033 |
|       3 |       0.1009 |       0.1124 |       0.1124 |      0.8972 |    0.9659 | 1.3002 |
|       4 |       0.0789 |       0.1007 |       0.1007 |      0.7834 |    0.9704 | 1.3019 |
|       5 |       0.0641 |       0.0939 |       0.0939 |      0.6820 |    0.9716 | 1.3038 |
|       6 |       0.0528 |       0.0895 |       0.0895 |      0.5903 |    0.9726 | 1.3042 |
|       7 |       0.0440 |       0.0861 |       0.0861 |      0.5110 |    0.9736 | 1.3024 |
|       8 |       0.0368 |       0.0830 |       0.0830 |      0.4431 |    0.9747 | 1.3020 |
|       9 |       0.0307 |       0.0816 |       0.0816 |      0.3758 |    0.9754 | 1.3002 |
|      10 |       0.0258 |       0.0804 |       0.0804 |      0.3203 |    0.9757 | 1.3056 |
|      11 |       0.0216 |       0.0814 |              |      0.2650 |    0.9753 | 1.2994 |
|      12 |       0.0182 |       0.0800 |       0.0800 |      0.2272 |    0.9766 | 1.3014 |
|      13 |       0.0152 |       0.0811 |              |      0.1880 |    0.9774 | 1.2980 |
|      14 |       0.0127 |       0.0847 |              |      0.1501 |    0.9766 | 1.3013 |
|      15 |       0.0107 |       0.0866 |              |      0.1236 |    0.9762 | 1.2992 |
|      16 |       0.0090 |       0.0878 |              |      0.1026 |    0.9763 | 1.2998 |
|      17 |       0.0076 |       0.0873 |              |      0.0874 |    0.9765 | 1.3034 |
|      18 |       0.0064 |       0.0861 |              |      0.0749 |    0.9771 | 1.3003 |
|      19 |       0.0055 |       0.0854 |              |      0.0640 |    0.9783 | 1.2992 |
|      20 |       0.0047 |       0.0846 |              |      0.0554 |    0.9786 | 1.3024 |
|      21 |       0.0040 |       0.0843 |              |      0.0476 |    0.9789 | 1.2993 |
|      22 |       0.0035 |       0.0845 |              |      0.0413 |    0.9790 | 1.3016 |
|      23 |       0.0030 |       0.0847 |              |      0.0360 |    0.9794 | 1.3527 |
|      24 |       0.0027 |       0.0852 |              |      0.0313 |    0.9793 | 1.3043 |
|      25 |       0.0023 |       0.0856 |              |      0.0274 |    0.9799 | 1.3127 |
|      26 |       0.0021 |       0.0862 |              |      0.0241 |    0.9800 | 1.3026 |
|      27 |       0.0019 |       0.0869 |              |      0.0214 |    0.9800 | 1.3041 |
|      28 |       0.0017 |       0.0875 |              |      0.0192 |    0.9801 | 1.3029 |
|      29 |       0.0015 |       0.0881 |              |      0.0173 |    0.9800 | 1.3009 |
|      30 |       0.0014 |       0.0888 |              |      0.0157 |    0.9799 | 1.3270 |
|      31 |       0.0013 |       0.0893 |              |      0.0144 |    0.9801 | 1.3010 |
|      32 |       0.0012 |       0.0898 |              |      0.0132 |    0.9801 | 1.3042 |
|      33 |       0.0011 |       0.0902 |              |      0.0123 |    0.9801 | 1.3025 |
|      34 |       0.0010 |       0.0906 |              |      0.0114 |    0.9800 | 1.3055 |
|      35 |       0.0010 |       0.0910 |              |      0.0107 |    0.9801 | 1.3018 |
|      36 |       0.0009 |       0.0914 |              |      0.0100 |    0.9802 | 1.2988 |
|      37 |       0.0009 |       0.0917 |              |      0.0094 |    0.9801 | 1.3011 |
|      38 |       0.0008 |       0.0921 |              |      0.0089 |    0.9801 | 1.2996 |
|      39 |       0.0008 |       0.0924 |              |      0.0084 |    0.9801 | 1.3137 |
|      40 |       0.0007 |       0.0928 |              |      0.0079 |    0.9803 | 1.3017 |
|      41 |       0.0007 |       0.0932 |              |      0.0075 |    0.9802 | 1.2993 |
|      42 |       0.0007 |       0.0935 |              |      0.0072 |    0.9803 | 1.2990 |
|      43 |       0.0006 |       0.0939 |              |      0.0068 |    0.9804 | 1.3220 |
|      44 |       0.0006 |       0.0943 |              |      0.0065 |    0.9804 | 1.3023 |
|      45 |       0.0006 |       0.0948 |              |      0.0063 |    0.9806 | 1.3299 |
|      46 |       0.0006 |       0.0952 |              |      0.0061 |    0.9805 | 1.3005 |
|      47 |       0.0006 |       0.0957 |              |      0.0061 |    0.9804 | 1.3004 |
|      48 |       0.0006 |       0.0964 |              |      0.0061 |    0.9804 | 1.3023 |
|      49 |       0.0006 |       0.0979 |              |      0.0063 |    0.9795 | 1.3014 |
|      50 |       0.0010 |       0.0983 |              |      0.0100 |    0.9794 | 1.2996 |






    
Out[12]:





NeuralNet(X_tensor_type=<function matrix at 0x7f31503d0320>,
     batch_iterator_test=<nolearn.lasagne.base.BatchIterator object at 0x7f313618b390>,
     batch_iterator_train=<nolearn.lasagne.base.BatchIterator object at 0x7f313618b850>,
     custom_score=None, eval_size=0.2,
     hidden1_nonlinearity=<function rectify at 0x7f3137aff758>,
     hidden1_num_units=200,
     hidden2_nonlinearity=<function rectify at 0x7f3137aff758>,
     hidden2_num_units=200, input_shape=(None, 784),
     layers=[('input', <class 'lasagne.layers.input.InputLayer'>), ('hidden1', <class 'lasagne.layers.dense.DenseLayer'>), ('hidden2', <class 'lasagne.layers.dense.DenseLayer'>), ('output', <class 'lasagne.layers.dense.DenseLayer'>)],
     loss=None, max_epochs=50, more_params={},
     objective=<class 'lasagne.objectives.Objective'>,
     objective_loss_function=<function categorical_crossentropy at 0x7f314ee94758>,
     on_epoch_finished=[<__main__.AdjustVariable object at 0x7f3135f99ed0>, <__main__.AdjustVariable object at 0x7f313593d9d0>],
     on_training_finished=(),
     output_nonlinearity=<function softmax at 0x7f3137aff668>,
     output_num_units=10, regression=False,
     update=<function nesterov_momentum at 0x7f31372412a8>,
     update_learning_rate=<CudaNdarrayType(float32, scalar)>,
     update_momentum=<CudaNdarrayType(float32, scalar)>,
     use_label_encoder=False, verbose=1,
     y_tensor_type=TensorType(int32, vector))

In [ ]:

    




    

In [13]:

    

convNet1 = NeuralNet(
    layers=[
        ('input', layers.InputLayer),
        ('conv1', layers.Conv2DLayer),      #Convolutional layer.  Params defined below
        ('pool1', layers.MaxPool2DLayer),   #Like downsampling, for execution speed
        ('conv2', layers.Conv2DLayer),
        ('hidden3', layers.DenseLayer),
        ('output', layers.DenseLayer),
        ],

        input_shape=(None, 1, image_width, image_height),
        conv1_num_filters=7, conv1_filter_size=(3, 3), conv1_nonlinearity=rectify,
        pool1_pool_size=(3, 3),
        conv2_num_filters=12, conv2_filter_size=(2, 2), conv2_nonlinearity=rectify,
        hidden3_num_units=50,
        output_num_units=10, output_nonlinearity=softmax,
    
        update_learning_rate=theano.shared(float32(0.05)),
        update_momentum=theano.shared(float32(0.7)),

        on_epoch_finished=[
            AdjustVariable('update_learning_rate', start=0.05, stop=0.0001),
            AdjustVariable('update_momentum', start=0.7, stop=0.999),
            ],     
    
        regression=False,
        max_epochs=5,
        verbose=1,
)
convNet1.fit(X_train_reshaped, y_train)


    

    




  input             	(None, 1, 28, 28)   	produces     784 outputs
  conv1             	(None, 7, 26, 26)   	produces    4732 outputs
  pool1             	(None, 7, 9, 9)     	produces     567 outputs
  conv2             	(None, 12, 8, 8)    	produces     768 outputs
  hidden3           	(None, 50)          	produces      50 outputs
  output            	(None, 10)          	produces      10 outputs
|   epoch |   train loss |   valid loss |   valid best |   train/val |   val acc |     dur |
|--------:|-------------:|-------------:|-------------:|------------:|----------:|--------:|
|       1 |       0.2966 |       0.1125 |       0.1125 |      2.6353 |    0.9652 | 11.1504 |
|       2 |       0.0972 |       0.0802 |       0.0802 |      1.2110 |    0.9747 | 11.1565 |
|       3 |       0.0697 |       0.0742 |       0.0742 |      0.9387 |    0.9760 | 11.1568 |
|       4 |       0.0567 |       0.0698 |       0.0698 |      0.8119 |    0.9779 | 11.4567 |
|       5 |       0.0491 |       0.0666 |       0.0666 |      0.7370 |    0.9794 | 11.4369 |






    
Out[13]:





NeuralNet(X_tensor_type=<function tensor4 at 0x7f31503d06e0>,
     batch_iterator_test=<nolearn.lasagne.base.BatchIterator object at 0x7f313618b390>,
     batch_iterator_train=<nolearn.lasagne.base.BatchIterator object at 0x7f313618b850>,
     conv1_filter_size=(3, 3),
     conv1_nonlinearity=<function rectify at 0x7f3137aff758>,
     conv1_num_filters=7, conv2_filter_size=(2, 2),
     conv2_nonlinearity=<function rectify at 0x7f3137aff758>,
     conv2_num_filters=12, custom_score=None, eval_size=0.2,
     hidden3_num_units=50, input_shape=(None, 1, 28, 28),
     layers=[('input', <class 'lasagne.layers.input.InputLayer'>), ('conv1', <class 'lasagne.layers.conv.Conv2DLayer'>), ('pool1', <class 'lasagne.layers.pool.MaxPool2DLayer'>), ('conv2', <class 'lasagne.layers.conv.Conv2DLayer'>), ('hidden3', <class 'lasagne.layers.dense.DenseLayer'>), ('output', <class 'lasagne.layers.dense.DenseLayer'>)],
     loss=None, max_epochs=5, more_params={},
     objective=<class 'lasagne.objectives.Objective'>,
     objective_loss_function=<function categorical_crossentropy at 0x7f314ee94758>,
     on_epoch_finished=[<__main__.AdjustVariable object at 0x7f3135ef04d0>, <__main__.AdjustVariable object at 0x7f3135baee50>],
     on_training_finished=(),
     output_nonlinearity=<function softmax at 0x7f3137aff668>,
     output_num_units=10, pool1_pool_size=(3, 3), regression=False,
     update=<function nesterov_momentum at 0x7f31372412a8>,
     update_learning_rate=<CudaNdarrayType(float32, scalar)>,
     update_momentum=<CudaNdarrayType(float32, scalar)>,
     use_label_encoder=False, verbose=1,
     y_tensor_type=TensorType(int32, vector))

In [ ]:

    




    

In [14]:

    

convNet11 = NeuralNet(
    layers=[
        ('input', layers.InputLayer),
        ('conv1', layers.Conv2DLayer),
        ('pool1', layers.MaxPool2DLayer),
        ('dropout1', layers.DropoutLayer),  # !
        ('conv2', layers.Conv2DLayer),
        ('pool2', layers.MaxPool2DLayer),
        ('dropout2', layers.DropoutLayer),  # !
        ('hidden4', layers.DenseLayer),
        ('hidden5', layers.DenseLayer),
        ('output', layers.DenseLayer),
        ],
    input_shape=(None, 1, image_width, image_height),
    conv1_num_filters=4, conv1_filter_size=(4, 4), conv1_nonlinearity=rectify,
    pool1_pool_size=(3, 3),
    dropout1_p=0.1,  # !
    conv2_num_filters=8, conv2_filter_size=(3, 3), conv2_nonlinearity=rectify,
    pool2_pool_size=(2, 2),
    dropout2_p=0.1,  # !
    hidden4_num_units=500, 
    hidden5_num_units=500,
    output_num_units=10, output_nonlinearity=softmax,

    update=nesterov_momentum,
    
    update_learning_rate=theano.shared(float32(0.07)),
    update_momentum=theano.shared(float32(0.4)),

    on_epoch_finished=[
        AdjustVariable('update_learning_rate', start=0.07, stop=0.0001),
        AdjustVariable('update_momentum', start=0.4, stop=0.999),
        ],  
    
    regression=False,
    max_epochs=25,
    verbose=1,
    )
convNet11.fit(X_train_reshaped, y_train)


    

    




  input             	(None, 1, 28, 28)   	produces     784 outputs
  conv1             	(None, 4, 25, 25)   	produces    2500 outputs
  pool1             	(None, 4, 9, 9)     	produces     324 outputs
  dropout1          	(None, 4, 9, 9)     	produces     324 outputs
  conv2             	(None, 8, 7, 7)     	produces     392 outputs
  pool2             	(None, 8, 4, 4)     	produces     128 outputs
  dropout2          	(None, 8, 4, 4)     	produces     128 outputs
  hidden4           	(None, 500)         	produces     500 outputs
  hidden5           	(None, 500)         	produces     500 outputs
  output            	(None, 10)          	produces      10 outputs
|   epoch |   train loss |   valid loss |   valid best |   train/val |   val acc |     dur |
|--------:|-------------:|-------------:|-------------:|------------:|----------:|--------:|
|       1 |       0.4632 |       0.1273 |       0.1273 |      3.6377 |    0.9594 | 12.6152 |
|       2 |       0.1562 |       0.0811 |       0.0811 |      1.9255 |    0.9737 | 12.6264 |
|       3 |       0.1173 |       0.0698 |       0.0698 |      1.6806 |    0.9782 | 12.5477 |
|       4 |       0.1039 |       0.0603 |       0.0603 |      1.7245 |    0.9801 | 12.5479 |
|       5 |       0.0950 |       0.0626 |              |      1.5168 |    0.9790 | 12.5457 |
|       6 |       0.0866 |       0.0528 |       0.0528 |      1.6422 |    0.9830 | 12.5525 |
|       7 |       0.0816 |       0.0486 |       0.0486 |      1.6810 |    0.9841 | 12.5534 |
|       8 |       0.0779 |       0.0474 |       0.0474 |      1.6437 |    0.9842 | 12.5532 |
|       9 |       0.0746 |       0.0490 |              |      1.5218 |    0.9839 | 12.5498 |
|      10 |       0.0706 |       0.0446 |       0.0446 |      1.5815 |    0.9854 | 12.5519 |
|      11 |       0.0700 |       0.0455 |              |      1.5367 |    0.9848 | 12.5526 |
|      12 |       0.0662 |       0.0469 |              |      1.4124 |    0.9848 | 12.6866 |
|      13 |       0.0644 |       0.0465 |              |      1.3843 |    0.9840 | 12.6811 |
|      14 |       0.0625 |       0.0426 |       0.0426 |      1.4667 |    0.9859 | 12.5528 |
|      15 |       0.0595 |       0.0434 |              |      1.3713 |    0.9862 | 12.5695 |
|      16 |       0.0585 |       0.0404 |       0.0404 |      1.4479 |    0.9866 | 12.5568 |
|      17 |       0.0582 |       0.0434 |              |      1.3412 |    0.9863 | 12.5736 |
|      18 |       0.0563 |       0.0423 |              |      1.3311 |    0.9858 | 12.5404 |
|      19 |       0.0545 |       0.0400 |       0.0400 |      1.3629 |    0.9869 | 12.5335 |
|      20 |       0.0541 |       0.0422 |              |      1.2807 |    0.9864 | 12.5556 |
|      21 |       0.0538 |       0.0385 |       0.0385 |      1.3955 |    0.9871 | 12.5542 |
|      22 |       0.0517 |       0.0406 |              |      1.2743 |    0.9866 | 12.5579 |
|      23 |       0.0521 |       0.0436 |              |      1.1953 |    0.9864 | 12.5352 |
|      24 |       0.0498 |       0.0376 |       0.0376 |      1.3266 |    0.9885 | 12.5369 |
|      25 |       0.0493 |       0.0376 |       0.0376 |      1.3114 |    0.9875 | 12.5313 |
|      26 |       0.0483 |       0.0393 |              |      1.2283 |    0.9877 | 12.5420 |
|      27 |       0.0470 |       0.0387 |              |      1.2142 |    0.9872 | 12.5509 |
|      28 |       0.0469 |       0.0375 |       0.0375 |      1.2489 |    0.9874 | 12.5326 |
|      29 |       0.0471 |       0.0383 |              |      1.2313 |    0.9871 | 12.5494 |
|      30 |       0.0442 |       0.0391 |              |      1.1305 |    0.9875 | 12.5429 |
|      31 |       0.0459 |       0.0377 |              |      1.2187 |    0.9876 | 12.5368 |
|      32 |       0.0439 |       0.0395 |              |      1.1126 |    0.9869 | 12.5407 |
|      33 |       0.0441 |       0.0362 |       0.0362 |      1.2174 |    0.9887 | 12.5425 |
|      34 |       0.0418 |       0.0350 |       0.0350 |      1.1932 |    0.9888 | 12.5333 |
|      35 |       0.0409 |       0.0363 |              |      1.1276 |    0.9882 | 12.5465 |
|      36 |       0.0411 |       0.0356 |              |      1.1540 |    0.9886 | 12.5399 |
|      37 |       0.0403 |       0.0377 |              |      1.0696 |    0.9885 | 12.5465 |
|      38 |       0.0420 |       0.0355 |              |      1.1846 |    0.9892 | 12.5341 |
|      39 |       0.0401 |       0.0367 |              |      1.0909 |    0.9886 | 12.5269 |
|      40 |       0.0403 |       0.0383 |              |      1.0526 |    0.9875 | 12.6627 |
|      41 |       0.0391 |       0.0370 |              |      1.0579 |    0.9879 | 12.7030 |
|      42 |       0.0399 |       0.0362 |              |      1.1031 |    0.9883 | 12.7675 |
|      43 |       0.0377 |       0.0371 |              |      1.0163 |    0.9879 | 12.5495 |
|      44 |       0.0392 |       0.0367 |              |      1.0681 |    0.9877 | 12.7555 |
|      45 |       0.0378 |       0.0367 |              |      1.0287 |    0.9881 | 12.9086 |
|      46 |       0.0376 |       0.0345 |       0.0345 |      1.0894 |    0.9889 | 12.6544 |
|      47 |       0.0374 |       0.0377 |              |      0.9936 |    0.9880 | 12.6939 |
|      48 |       0.0370 |       0.0345 |       0.0345 |      1.0740 |    0.9889 | 12.7490 |
|      49 |       0.0374 |       0.0348 |              |      1.0762 |    0.9889 | 12.7796 |
|      50 |       0.0371 |       0.0337 |       0.0337 |      1.0999 |    0.9889 | 12.5526 |






    
Out[14]:





NeuralNet(X_tensor_type=<function tensor4 at 0x7f31503d06e0>,
     batch_iterator_test=<nolearn.lasagne.base.BatchIterator object at 0x7f313618b390>,
     batch_iterator_train=<nolearn.lasagne.base.BatchIterator object at 0x7f313618b850>,
     conv1_filter_size=(4, 4),
     conv1_nonlinearity=<function rectify at 0x7f3137aff758>,
     conv1_num_filters=4, conv2_filter_size=(3, 3),
     conv2_nonlinearity=<function rectify at 0x7f3137aff758>,
     conv2_num_filters=8, custom_score=None, dropout1_p=0.1,
     dropout2_p=0.1, eval_size=0.2, hidden4_num_units=500,
     hidden5_num_units=500, input_shape=(None, 1, 28, 28),
     layers=[('input', <class 'lasagne.layers.input.InputLayer'>), ('conv1', <class 'lasagne.layers.conv.Conv2DLayer'>), ('pool1', <class 'lasagne.layers.pool.MaxPool2DLayer'>), ('dropout1', <class 'lasagne.layers.noise.DropoutLayer'>), ('conv2', <class 'lasagne.layers.conv.Conv2DLayer'>), ('pool2', <cla..., <class 'lasagne.layers.dense.DenseLayer'>), ('output', <class 'lasagne.layers.dense.DenseLayer'>)],
     loss=None, max_epochs=50, more_params={},
     objective=<class 'lasagne.objectives.Objective'>,
     objective_loss_function=<function categorical_crossentropy at 0x7f314ee94758>,
     on_epoch_finished=[<__main__.AdjustVariable object at 0x7f3135d7ff50>, <__main__.AdjustVariable object at 0x7f312e2a9ad0>],
     on_training_finished=(),
     output_nonlinearity=<function softmax at 0x7f3137aff668>,
     output_num_units=10, pool1_pool_size=(3, 3), pool2_pool_size=(2, 2),
     regression=False,
     update=<function nesterov_momentum at 0x7f31372412a8>,
     update_learning_rate=<CudaNdarrayType(float32, scalar)>,
     update_momentum=<CudaNdarrayType(float32, scalar)>,
     use_label_encoder=False, verbose=1,
     y_tensor_type=TensorType(int32, vector))

In [15]:

    

y_preds = convNet11.predict(X_test_reshaped)


    

In [16]:

    

pd.DataFrame(y_preds).to_csv('../../data/intermediate/no_trainRot_9889_pred.csv', index=None, header=None)


    

In [ ]:

    




    

In [ ]:

    




    

In [14]:

    

convNet2 = NeuralNet(
    layers=[
        ('input', layers.InputLayer),
        ('conv1', layers.Conv2DLayer),
        ('pool1', layers.MaxPool2DLayer),
        ('dropout1', layers.DropoutLayer),  # !
        ('conv2', layers.Conv2DLayer),
        ('pool2', layers.MaxPool2DLayer),
        ('dropout2', layers.DropoutLayer),  # !
        ('conv3', layers.Conv2DLayer),
        ('pool3', layers.MaxPool2DLayer),
        ('dropout3', layers.DropoutLayer),  # !
        ('hidden4', layers.DenseLayer),
        ('hidden5', layers.DenseLayer),
        ('output', layers.DenseLayer),
        ],
    input_shape=(None, 1, image_width, image_height),
    conv1_num_filters=4, conv1_filter_size=(4, 4), conv1_nonlinearity=rectify,
    pool1_pool_size=(3, 3),
    dropout1_p=0.1,  # !
    conv2_num_filters=8, conv2_filter_size=(3, 3), conv2_nonlinearity=rectify,
    pool2_pool_size=(2, 2),
    dropout2_p=0.1,  # !
    conv3_num_filters=16, conv3_filter_size=(2, 2), conv3_nonlinearity=rectify,
    pool3_pool_size=(2, 2),
    dropout3_p=0.1,  # !
    hidden4_num_units=100, 
    hidden5_num_units=100,
    output_num_units=10, output_nonlinearity=softmax,

    update=nesterov_momentum,
    
    update_learning_rate=theano.shared(float32(0.07)),
    update_momentum=theano.shared(float32(0.4)),

    on_epoch_finished=[
        AdjustVariable('update_learning_rate', start=0.07, stop=0.0001),
        AdjustVariable('update_momentum', start=0.4, stop=0.999),
        ],  
    
    regression=False,
    max_epochs=50,
    verbose=1,
    )
convNet2.fit(X_train_reshaped, y_train)


    

    




  input             	(None, 1, 28, 28)   	produces     784 outputs
  conv1             	(None, 4, 25, 25)   	produces    2500 outputs
  pool1             	(None, 4, 9, 9)     	produces     324 outputs
  dropout1          	(None, 4, 9, 9)     	produces     324 outputs
  conv2             	(None, 8, 7, 7)     	produces     392 outputs
  pool2             	(None, 8, 4, 4)     	produces     128 outputs
  dropout2          	(None, 8, 4, 4)     	produces     128 outputs
  conv3             	(None, 16, 3, 3)    	produces     144 outputs
  pool3             	(None, 16, 2, 2)    	produces      64 outputs
  dropout3          	(None, 16, 2, 2)    	produces      64 outputs
  hidden4           	(None, 100)         	produces     100 outputs
  hidden5           	(None, 100)         	produces     100 outputs
  output            	(None, 10)          	produces      10 outputs
|   epoch |   train loss |   valid loss |   valid best |   train/val |   val acc |    dur |
|--------:|-------------:|-------------:|-------------:|------------:|----------:|-------:|
|       1 |       1.8062 |       0.8608 |       0.8608 |      2.0982 |    0.7483 | 5.0279 |
|       2 |       0.8855 |       0.6061 |       0.6061 |      1.4609 |    0.7926 | 5.0242 |
|       3 |       0.6766 |       0.4469 |       0.4469 |      1.5138 |    0.8522 | 4.9939 |
|       4 |       0.5713 |       0.3705 |       0.3705 |      1.5416 |    0.8808 | 5.0022 |
|       5 |       0.5115 |       0.3501 |       0.3501 |      1.4611 |    0.8866 | 5.0291 |
|       6 |       0.4619 |       0.2905 |       0.2905 |      1.5901 |    0.9083 | 5.0135 |
|       7 |       0.4325 |       0.2628 |       0.2628 |      1.6458 |    0.9158 | 5.0075 |
|       8 |       0.4022 |       0.2510 |       0.2510 |      1.6021 |    0.9192 | 5.0117 |
|       9 |       0.3807 |       0.2069 |       0.2069 |      1.8402 |    0.9367 | 5.0088 |
|      10 |       0.3614 |       0.2574 |              |      1.4039 |    0.9152 | 5.0129 |
|      11 |       0.3424 |       0.1959 |       0.1959 |      1.7473 |    0.9397 | 5.0135 |
|      12 |       0.3238 |       0.2764 |              |      1.1714 |    0.9086 | 5.0100 |
|      13 |       0.3127 |       0.1657 |       0.1657 |      1.8870 |    0.9524 | 5.0123 |
|      14 |       0.3034 |       0.1560 |       0.1560 |      1.9448 |    0.9506 | 5.0086 |
|      15 |       0.3010 |       0.1658 |              |      1.8148 |    0.9496 | 5.1799 |
|      16 |       0.2904 |       0.1670 |              |      1.7382 |    0.9487 | 5.2096 |
|      17 |       0.2876 |       0.1727 |              |      1.6650 |    0.9484 | 5.1542 |
|      18 |       0.2791 |       0.1653 |              |      1.6889 |    0.9500 | 5.1316 |
|      19 |       0.2760 |       0.1441 |       0.1441 |      1.9152 |    0.9565 | 5.1440 |
|      20 |       0.2681 |       0.1565 |              |      1.7130 |    0.9521 | 5.1560 |
|      21 |       0.2649 |       0.1494 |              |      1.7727 |    0.9534 | 5.1376 |
|      22 |       0.2564 |       0.1366 |       0.1366 |      1.8769 |    0.9584 | 5.0909 |
|      23 |       0.2532 |       0.1459 |              |      1.7352 |    0.9534 | 5.0111 |
|      24 |       0.2546 |       0.1320 |       0.1320 |      1.9286 |    0.9595 | 5.0143 |
|      25 |       0.2456 |       0.1567 |              |      1.5667 |    0.9499 | 5.0057 |
|      26 |       0.2422 |       0.1390 |              |      1.7429 |    0.9570 | 5.0079 |
|      27 |       0.2394 |       0.1493 |              |      1.6031 |    0.9546 | 5.0117 |
|      28 |       0.2446 |       0.1358 |              |      1.8010 |    0.9590 | 5.0044 |
|      29 |       0.2386 |       0.1321 |              |      1.8060 |    0.9592 | 5.0151 |
|      30 |       0.2325 |       0.1277 |       0.1277 |      1.8208 |    0.9609 | 5.0255 |
|      31 |       0.2350 |       0.1309 |              |      1.7954 |    0.9605 | 5.0306 |
|      32 |       0.2280 |       0.1280 |              |      1.7817 |    0.9596 | 5.0061 |
|      33 |       0.2282 |       0.1249 |       0.1249 |      1.8270 |    0.9613 | 5.0262 |
|      34 |       0.2247 |       0.1260 |              |      1.7832 |    0.9592 | 5.0026 |
|      35 |       0.2214 |       0.1207 |       0.1207 |      1.8337 |    0.9632 | 5.0108 |
|      36 |       0.2213 |       0.1215 |              |      1.8206 |    0.9609 | 5.0146 |
|      37 |       0.2186 |       0.1304 |              |      1.6763 |    0.9592 | 5.0136 |
|      38 |       0.2188 |       0.1289 |              |      1.6971 |    0.9585 | 5.0144 |
|      39 |       0.2161 |       0.1383 |              |      1.5624 |    0.9564 | 5.0131 |
|      40 |       0.2135 |       0.1332 |              |      1.6023 |    0.9573 | 5.0153 |
|      41 |       0.2100 |       0.1235 |              |      1.7006 |    0.9624 | 5.0094 |
|      42 |       0.2117 |       0.1315 |              |      1.6097 |    0.9588 | 5.0206 |
|      43 |       0.2093 |       0.1155 |       0.1155 |      1.8111 |    0.9651 | 5.0167 |
|      44 |       0.2132 |       0.1184 |              |      1.8007 |    0.9637 | 5.0131 |
|      45 |       0.2050 |       0.1189 |              |      1.7248 |    0.9644 | 5.0277 |
|      46 |       0.1989 |       0.1159 |              |      1.7154 |    0.9646 | 5.0190 |
|      47 |       0.2067 |       0.1152 |       0.1152 |      1.7943 |    0.9658 | 5.0103 |
|      48 |       0.2028 |       0.1234 |              |      1.6436 |    0.9628 | 5.0101 |
|      49 |       0.2019 |       0.1189 |              |      1.6984 |    0.9639 | 5.0116 |
|      50 |       0.2014 |       0.1085 |       0.1085 |      1.8566 |    0.9674 | 5.0067 |






    
Out[14]:





NeuralNet(X_tensor_type=<function tensor4 at 0x7efc0405e6e0>,
     batch_iterator_test=<nolearn.lasagne.base.BatchIterator object at 0x7efbe9d726d0>,
     batch_iterator_train=<nolearn.lasagne.base.BatchIterator object at 0x7efbe9d722d0>,
     conv1_filter_size=(4, 4),
     conv1_nonlinearity=<function rectify at 0x7efbeb794398>,
     conv1_num_filters=4, conv2_filter_size=(3, 3),
     conv2_nonlinearity=<function rectify at 0x7efbeb794398>,
     conv2_num_filters=8, conv3_filter_size=(2, 2),
     conv3_nonlinearity=<function rectify at 0x7efbeb794398>,
     conv3_num_filters=16, custom_score=None, dropout1_p=0.1,
     dropout2_p=0.1, dropout3_p=0.1, eval_size=0.2, hidden4_num_units=100,
     hidden5_num_units=100, input_shape=(None, 1, 28, 28),
     layers=[('input', <class 'lasagne.layers.input.InputLayer'>), ('conv1', <class 'lasagne.layers.conv.Conv2DLayer'>), ('pool1', <class 'lasagne.layers.pool.MaxPool2DLayer'>), ('dropout1', <class 'lasagne.layers.noise.DropoutLayer'>), ('conv2', <class 'lasagne.layers.conv.Conv2DLayer'>), ('pool2', <cla..., <class 'lasagne.layers.dense.DenseLayer'>), ('output', <class 'lasagne.layers.dense.DenseLayer'>)],
     loss=None, max_epochs=50, more_params={},
     objective=<class 'lasagne.objectives.Objective'>,
     objective_loss_function=<function categorical_crossentropy at 0x7efc02b22758>,
     on_epoch_finished=[<__main__.AdjustVariable object at 0x7efbea5c42d0>, <__main__.AdjustVariable object at 0x7efbe2e43f50>],
     on_training_finished=(),
     output_nonlinearity=<function softmax at 0x7efbeb7942a8>,
     output_num_units=10, pool1_pool_size=(3, 3), pool2_pool_size=(2, 2),
     pool3_pool_size=(2, 2), regression=False,
     update=<function nesterov_momentum at 0x7efbeac0be60>,
     update_learning_rate=<CudaNdarrayType(float32, scalar)>,
     update_momentum=<CudaNdarrayType(float32, scalar)>,
     use_label_encoder=False, verbose=1,
     y_tensor_type=TensorType(int32, vector))

In [14]:

    

y_preds = convNet2.predict(X_test_reshaped)


    

In [15]:

    

pd.DataFrame(y_preds).to_csv('../../data/intermediate/no_pred.csv', index=None, header=None)


    

In [ ]: