In [1]:

    

import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
from sklearn.datasets import load_iris
from sklearn.preprocessing import LabelBinarizer
import pyprind
from pydeeptoy.networks import *
from pydeeptoy.optimizers import *
from pydeeptoy.losses import *
from sklearn.metrics import accuracy_score


    

    




/Users/kirill/anaconda/lib/python3.5/site-packages/sklearn/utils/fixes.py:64: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead
  if 'order' in inspect.getargspec(np.copy)[0]:

In [2]:

    

mnist = np.loadtxt('mnist/train.csv', delimiter=',', skiprows=1)


    

In [3]:

    

X = mnist[:, 1:]
y = mnist[:, 0]


    

In [4]:

    

X.shape


    

    
Out[4]:





(42000, 784)

In [5]:

    

mnistMean, mnistStd = np.mean(X), np.std(X)
X = (X - mnistMean)/mnistStd


    

In [6]:

    

one_hot_y = np.array(LabelBinarizer().fit_transform(y).T)


    

In [7]:

    

plt.imshow(X[9].reshape(28, 28), cmap='gray')
plt.title(np.argmax(one_hot_y[:, 9]))


    

    
Out[7]:





<matplotlib.text.Text at 0x14b123550>






    

In [ ]:

    

#X = X[:1000,:]
#one_hot_y = one_hot_y[:, :1000]
#y = y[:1000]


    

In [55]:

    

cg = ComputationalGraph()
#x_in = cg.constant(name="X")
Xr = X.reshape(-1, 1, 28, 28)
x_in = cg.constant(Xr, name="X")
w_in = 0.01*np.random.randn(16, 3)
conv = cg.reshape(cg.conv2d(x_in, cg.constant(w_in), receptive_field_size=4, filters_number=3, stride=2, padding=0), (-1, 507))

ctx.forward(cg, {x_in: Xr})

ctx[conv].value.shape


    

    




/Users/kirill/anaconda/lib/python3.5/site-packages/numpy/lib/shape_base.py:873: VisibleDeprecationWarning: using a non-integer number instead of an integer will result in an error in the future
  return c.reshape(shape_out)
/Users/kirill/documents/projects/DeepLearningToy/src/pydeeptoy/nodes.py:342: VisibleDeprecationWarning: using a non-integer number instead of an integer will result in an error in the future
  x_col = x_padded[idx].reshape(n, output_height * output_width, -1)






    
Out[55]:





(42000, 507)

In [26]:

    

X.shape


    

    
Out[26]:





(42000, 784)

In [64]:

    

len(Xr)


    

    
Out[64]:





42000

In [65]:

    

cg = ComputationalGraph()
#x_in = cg.constant(name="X")
Xr = X.reshape(-1, 1, 28, 28)
x_in = cg.constant(Xr, name="X")
w_in = 0.01*np.random.randn(16, 3)
conv = cg.reshape(cg.conv2d(x_in, cg.variable(init_value=w_in), receptive_field_size=4, filters_number=3, stride=2, padding=0), (-1, 507))

nn_output = neural_network(cg, cg.transpose(conv, 1, 0), 507, 10)
nn_output.name = "nn_output"
        
y_train = cg.constant(name="one_hot_y")
batch_size=256
loss = softmax(cg, nn_output, y_train, "loss_softmax")

ctx = SimulationContext()        

sgd = MomentumSgdOptimizer(learning_rate=0.01)
epochs = 1
bar = pyprind.ProgBar(epochs, bar_char='█', width=60, track_time=True, stream=1)
for epoch in range(0, epochs):
    indexes = np.arange(0, len(Xr))
    np.random.shuffle(indexes)
    train_x = Xr[indexes, :]
    train_y = one_hot_y[:, indexes]    
    for batch in range(0, len(train_x), batch_size):
        batch_x = train_x[batch:batch + batch_size]
        batch_y = train_y[:, batch:batch + batch_size]
        sgd.minimize(ctx, cg, {x_in: batch_x, y_train: batch_y})    
    bar.update(item_id = "loss = {0:.5f}".format(ctx[loss].value))

ctx.forward(cg, {x_in: Xr}, out=[nn_output])
y_pred = np.argmax(ctx[nn_output].value, axis=0)

accuracy = accuracy_score(y, y_pred)
accuracy, ctx[nn_output].value.shape


    

    




0%  100%
[ ]





    




/Users/kirill/anaconda/lib/python3.5/site-packages/numpy/lib/shape_base.py:873: VisibleDeprecationWarning: using a non-integer number instead of an integer will result in an error in the future
  return c.reshape(shape_out)
/Users/kirill/documents/projects/DeepLearningToy/src/pydeeptoy/nodes.py:342: VisibleDeprecationWarning: using a non-integer number instead of an integer will result in an error in the future
  x_col = x_padded[idx].reshape(n, output_height * output_width, -1)






    




---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-65-e4f4570cfac5> in <module>()
     26         batch_x = train_x[batch:batch + batch_size]
     27         batch_y = train_y[:, batch:batch + batch_size]
---> 28         sgd.minimize(ctx, cg, {x_in: batch_x, y_train: batch_y})
     29     bar.update(item_id = "loss = {0:.5f}".format(ctx[loss].value))
     30 

/Users/kirill/documents/projects/DeepLearningToy/src/pydeeptoy/optimizers.py in minimize(self, ctx, cg, params)
     19 
     20     def minimize(self, ctx: SimulationContext, cg: ComputationalGraph, params={}):
---> 21         ctx.forward_backward(cg, params)
     22         for v in cg.input_variables:
     23             if v not in self._vs:

/Users/kirill/documents/projects/DeepLearningToy/src/pydeeptoy/simulation.py in forward_backward(self, cg, params, reset_gradient, out)
     67     def forward_backward(self, cg: ComputationalGraph, params=dict(), reset_gradient=True, out=list()):
     68         self.forward(cg, params, out=out)
---> 69         self.backward(cg, reset_gradient=reset_gradient, out=out)
     70 
     71 

/Users/kirill/documents/projects/DeepLearningToy/src/pydeeptoy/simulation.py in backward(self, cg, reset_gradient, out)
     63             for i in cg.outputs:
     64                 self.get_data(i).reset_gradient(to_value=1)
---> 65         [node.backward(self) for node in reversed(self.sort_topologically(cg))]
     66 
     67     def forward_backward(self, cg: ComputationalGraph, params=dict(), reset_gradient=True, out=list()):

/Users/kirill/documents/projects/DeepLearningToy/src/pydeeptoy/simulation.py in <listcomp>(.0)
     63             for i in cg.outputs:
     64                 self.get_data(i).reset_gradient(to_value=1)
---> 65         [node.backward(self) for node in reversed(self.sort_topologically(cg))]
     66 
     67     def forward_backward(self, cg: ComputationalGraph, params=dict(), reset_gradient=True, out=list()):

/Users/kirill/documents/projects/DeepLearningToy/src/pydeeptoy/nodes.py in backward(self, data_bag)
    236     def backward(self, data_bag):
    237         data_bag[self.in1].gradient = data_bag[self.out].gradient.dot(data_bag[self.in2].value.T)
--> 238         data_bag[self.in2].gradient = data_bag[self.in1].value.T.dot(data_bag[self.out].gradient)
    239 
    240 

ValueError: shapes (16,169,256) and (256,169,3) not aligned: 256 (dim 2) != 169 (dim 1)

In [9]:

    

Xtest = np.loadtxt('mnist/test.csv', delimiter=',', skiprows=1)
Xtest = (Xtest - mnistMean)/mnistStd


    

In [10]:

    

Xtest.shape


    

    
Out[10]:





(28000, 784)

In [11]:

    

ctx.forward(cg, {x_in: Xtest.T, y_train: 1})
y_pred_test = np.argmax(ctx[nn_output].value, axis=0)


    

In [12]:

    

plt.imshow(Xtest[259].reshape(28, 28), cmap='gray')
plt.title(y_pred_test[259])


    

    
Out[12]:





<matplotlib.text.Text at 0x13e99c9e8>






    

In [13]:

    

np.savetxt('mnist/pred.csv', np.column_stack((np.arange(len(y_pred_test)) + 1, y_pred_test)), 
           delimiter=',', header='ImageId,Label', fmt='%.d', comments='')