In [ ]:

    

import matplotlib as mpl
mpl.use('Agg')
%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
import os
import pandas as pd
import pickle
import seaborn as sns
import sys


    

In [ ]:

    

sys.path.append('../code')
sys.path.append('../../HW3/code/') # for PCA


    

In [ ]:

    

from NeuralNet import NeuralNet, make_dir
from TransferFunctions import ReLuTF, LinearTF
from neural_net_helpers import mnist_training, mnist_testing


    

In [ ]:

    

pca_training = pickle.load(file=open('../data/PCA_training_data_uncentered.pickle', "rb"))


    

In [ ]:

    

X_train_untransfored, y_train = mnist_training()
X_test_untransfored, y_test = mnist_testing()


    

In [ ]:

    

X_train_PCA50 = np.load("../data/X_transformed_by_50_components_uncentered.npy").T
X_test_PCA50 = np.load("../data/X_test_transformed_by_50_components_uncentered.npy").T


    

In [ ]:

    

# columns are data points and rows are features
d, N = np.shape(X_train_PCA50)
C = np.unique(y_train).shape[0]


    

In [ ]:

    

eta0 = 1e-3
n = NeuralNet(X=X_train_PCA50, y=y_train, 
              hidden_nodes=500, 
              hiddenTF=ReLuTF, outputTF=LinearTF,
              hiddenTF_kwargs={"scale_W1":5},  
              outputTF_kwargs={"scale_W2":1e7},
              minibatch_size=50,
              eta0 = eta0,
              convergence_delta = eta0/1000.,  # gets set to eta0/10
              X_test = X_test_PCA50, y_test = y_test,
              PCA = pca_training,
              verbose=True)


    

In [ ]:

    

print("N: {}".format(n.N))
print("d: {}".format(n.d))
print("C: {}".format(n.C))
print("X shape: {}".format(n.X.shape))
print("y shape: {}".format(n.y.shape))
print("Y shape: {}".format(n.Y.shape))


    

In [ ]:

    

n.W1[0:5, 0:3]
#np.round(n.W1[0:5, 0:5], 2)


    

In [ ]:

    

n.W2[0:5, 0:3]
#np.round(n.W2[0:5, 0:5], 2)


    

In [ ]:

    

n.run(epochs=3)


    

In [ ]:

    

n.results.tail()


    

In [ ]:

    

sl = n.plot_square_loss(logy=False)
sl.savefig(filename='../figures/Q-2-2-relu-linear/161211_relu_linear_square_loss.pdf')


    

In [ ]:

    

l01 = n.plot_01_loss()
l01.savefig(filename='../figures/Q-2-2-relu-linear/161211_relu_linear_01_loss.pdf')


    

In [ ]:

    

#p = n.plot_weight_evolution()


    

In [ ]:

    

p1 = n.plot_sum_of_weights('W1')
p2 = n.plot_sum_of_weights('W2')


    

In [ ]:

    

p1 = n.plot_norm_of_gradient('W1', logy=True)
p2 = n.plot_norm_of_gradient('W2', logy=True)


    

In [ ]:

    

p = n.plot_sample_dot_prods()


    

In [ ]:

    

n.results.tail(1).T


    

In [ ]:

    

image_dir = '../figures/Q-2-2-relu-linear'
make_dir(image_dir)


    

In [ ]:

    

n.visualize_10_W1_weights(image_dir)


    

In [ ]: