$\newcommand{\xv}{\mathbf{x}} \newcommand{\Xv}{\mathbf{X}} \newcommand{\yv}{\mathbf{y}} \newcommand{\Yv}{\mathbf{Y}} \newcommand{\zv}{\mathbf{z}} \newcommand{\av}{\mathbf{a}} \newcommand{\Wv}{\mathbf{W}} \newcommand{\wv}{\mathbf{w}} \newcommand{\gv}{\mathbf{g}} \newcommand{\Hv}{\mathbf{H}} \newcommand{\dv}{\mathbf{d}} \newcommand{\Vv}{\mathbf{V}} \newcommand{\vv}{\mathbf{v}} \newcommand{\tv}{\mathbf{t}} \newcommand{\Tv}{\mathbf{T}} \newcommand{\zv}{\mathbf{z}} \newcommand{\Zv}{\mathbf{Z}} \newcommand{\muv}{\boldsymbol{\mu}} \newcommand{\sigmav}{\boldsymbol{\sigma}} \newcommand{\phiv}{\boldsymbol{\phi}} \newcommand{\Phiv}{\boldsymbol{\Phi}} \newcommand{\Sigmav}{\boldsymbol{\Sigma}} \newcommand{\Lambdav}{\boldsymbol{\Lambda}} \newcommand{\half}{\frac{1}{2}} \newcommand{\argmax}[1]{\underset{#1}{\operatorname{argmax}}} \newcommand{\argmin}[1]{\underset{#1}{\operatorname{argmin}}} \newcommand{\dimensionbar}[1]{\underset{#1}{\operatorname{|}}} $

Autoencoder Neural Networks



In [1]:

    
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import neuralnetworks as nn

import matplotlib.gridspec as gridspec
import pickle
import gzip

Get data from DeepLearning Tutorial.



In [2]:

    
with gzip.open('mnist.pkl.gz', 'rb') as f:
    train_set, valid_set, test_set = pickle.load(f, encoding='latin1')

Xorig = np.vstack([a.reshape((28, 28, 1))[np.newaxis, :, :, :] for a in train_set[0]])
Torig = np.array(train_set[1]).reshape((-1,1))

Xtest = np.vstack([a.reshape((28,28,1))[np.newaxis,:,:,:] for a in test_set[0]])
Ttest = np.array(test_set[1]).reshape((-1,1))

Xorig.shape, Torig.shape, Xtest.shape, Ttest.shape









    Out[2]:





((50000, 28, 28, 1), (50000, 1), (10000, 28, 28, 1), (10000, 1))



In [3]:

    
plt.figure(figsize=(10,10))
for i in range(100):
    plt.subplot(10,10,i+1)
    plt.imshow(-Xorig[i,:].reshape((28,28)),interpolation='nearest',cmap='gray')
    plt.axis('off')
    plt.title(str(Torig[i][0]))
plt.tight_layout()



In [4]:

    
if True:
    nEach = 100
    useThese = []
    for digit in range(10):
        useThese += np.where(Torig == digit)[0][:nEach].tolist()
    useThese = np.array(useThese)
    np.random.shuffle(useThese)
    X = Xorig[useThese,:]
    T = Torig[useThese,:]
    del Xorig # to save memory
    del Torig
else:
    X = Xorig
    T = Torig
X.shape, T.shape









    Out[4]:





((1000, 28, 28, 1), (1000, 1))

Flatten each 28x28 image into a 784 vector.



In [6]:

    
X = X.reshape((1000,784))
X.shape









    Out[6]:





(1000, 784)



In [7]:

    
rowsShuffled = np.arange(X.shape[0])
np.random.shuffle(rowsShuffled)
nTrain = int(X.shape[0] * 0.8)
Xtrain = X[rowsShuffled[:nTrain],:]
Ttrain = T[rowsShuffled[:nTrain],:]
Xtest = X[rowsShuffled[nTrain:],:]
Ttest = T[rowsShuffled[nTrain:],:]
Xtrain.shape,Ttrain.shape, Xtest.shape,Ttest.shape









    Out[7]:





((800, 784), (800, 1), (200, 784), (200, 1))



In [7]:

    
import time
startTime = time.time()
nnet = nn.NeuralNetwork(784, [100, 50, 2, 50, 100],784)
nnet.train(X, X, nIterations=6000, verbose=True)
print('Training took', (time.time() - startTime) / 60, 'minutes.')
plt.plot(nnet.getErrors());









    



SCG: Iteration 600 fValue Eval 0.570307072677 Scale 1e-15
SCG: Iteration 1200 fValue Eval 0.518529712354 Scale 1e-15
SCG: Iteration 1800 fValue Eval 0.494695097996 Scale 1e-15
SCG: Iteration 2400 fValue Eval 0.474274741627 Scale 1e-15
SCG: Iteration 3000 fValue Eval 0.466215465026 Scale 1e-15
SCG: Iteration 3600 fValue Eval 0.455275459378 Scale 1e-15
SCG: Iteration 4200 fValue Eval 0.447769776773 Scale 1e-15
SCG: Iteration 4800 fValue Eval 0.441382210428 Scale 1e-15
SCG: Iteration 5400 fValue Eval 0.435001179434 Scale 1e-15
SCG: Iteration 6000 fValue Eval 0.430543707808 Scale 1e-15
Training took 10.782583363850911 minutes.



In [8]:

    
Y,allOutputs = nnet.use(X,allOutputs=True)



In [9]:

    
plt.figure(figsize=(10,10))
for i in range(0,36,2):
    plt.subplot(6,6,i+1)
    plt.imshow(-X[i,:].reshape((28,28)), interpolation='nearest', cmap='gray')
    plt.axis('off')
    plt.subplot(6,6,i+2)
    plt.imshow(-Y[i,:].reshape((28,28)), interpolation='nearest', cmap='gray')
    plt.axis('off')



In [10]:

    
len(allOutputs)









    Out[10]:





5



In [11]:

    
allOutputs[2].shape









    Out[11]:





(1000, 2)



In [12]:

    
import time
startTime = time.time()
nnet = nn.NeuralNetwork(784, [100, 50, 2, 50, 100],784)
nnet.train(Xtrain, Xtrain, nIterations=40000, verbose=True)
print('Training took', (time.time() - startTime)/60, 'minutes.')
plt.plot(nnet.getErrors())









    



SCG: Iteration 4000 fValue Eval 0.417534223549 Scale 1e-15
SCG: Iteration 8000 fValue Eval 0.365798371953 Scale 1e-15
SCG: Iteration 12000 fValue Eval 0.340904241917 Scale 1e-15
SCG: Iteration 16000 fValue Eval 0.325064930466 Scale 1e-15
SCG: Iteration 20000 fValue Eval 0.313181775413 Scale 1e-15
SCG: Iteration 24000 fValue Eval 0.304330410994 Scale 1e-15
SCG: Iteration 28000 fValue Eval 0.297879815441 Scale 1e-15
SCG: Iteration 32000 fValue Eval 0.293500714584 Scale 1e-15
SCG: Iteration 36000 fValue Eval 0.289552550815 Scale 1e-15
SCG: Iteration 40000 fValue Eval 0.287055065362 Scale 1e-15
Training took 53.169415664672854 minutes.






    Out[12]:





[<matplotlib.lines.Line2D at 0x7f1af7c184e0>]



In [13]:

    
Y,allOutputs = nnet.use(Xtrain, allOutputs=True)
bottleNeck = allOutputs[2]
plt.figure(figsize=(10,10))
plt.scatter(bottleNeck[:,0], bottleNeck[:, 1], c=Ttrain.flat, alpha=0.5);



In [14]:

    
plt.figure(figsize=(10,10))
plt.xlim(-1,1)
plt.ylim(-1,1)
for i, txt in enumerate(Ttrain.flat):
    plt.annotate(txt, (bottleNeck[i, 0], bottleNeck[i, 1]))



In [15]:

    
plt.annotate?



In [16]:

    
plt.figure(figsize=(10,10))
plt.xlim(-1,1)
plt.ylim(-1,1)
for i, txt in enumerate(Ttrain.flat):
    plt.annotate(txt, (bottleNeck[i, 0], bottleNeck[i, 1]))
    
_,allOutputsTest = nnet.use(Xtest,allOutputs=True)
bottleTest = allOutputsTest[2]
for i, txt in enumerate(Ttest.flat):
    plt.annotate(txt, bottleTest[i,:], color='r');



In [ ]: