In [ ]:

    

import matplotlib as mpl
mpl.use('Agg')
%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
import os
import pandas as pd
import pickle
import seaborn as sns
import sys


    

In [ ]:

    

sys.path.append('../code')
sys.path.append('../../HW3/code/') # for PCA


    

In [ ]:

    

from NeuralNet import NeuralNet, make_dir
from TransferFunctions import TanhTF, LinearTF
from neural_net_helpers import mnist_training, mnist_testing


    

In [ ]:

    

pca_training = pickle.load(file=open('../data/PCA_training_data_uncentered.pickle', "rb"))


    

In [ ]:

    

X_train_untransfored, y_train = mnist_training()
X_test_untransfored, y_test = mnist_testing()


    

In [ ]:

    

X_train_PCA50 = np.load("../data/X_transformed_by_50_components_uncentered.npy").T
X_test_PCA50 = np.load("../data/X_test_transformed_by_50_components_uncentered.npy").T


    

In [ ]:

    

# columns are data points and rows are features
d, N = np.shape(X_train_PCA50)
C = np.unique(y_train).shape[0]


    

In [ ]:

    

n = NeuralNet(X=X_train_PCA50, y=y_train, 
              hidden_nodes=500, 
              hiddenTF=TanhTF, 
              #hiddenTF_kwargs={'W1_init_strategy':'Xavier'}, #"scale_W1":0.001},
              outputTF=TanhTF,
              minibatch_size=100,
              eta0 = 0.005, #1.0e-3 , #
              convergence_delta = None,  # gets set to eta0/10
              X_test = X_test_PCA50, y_test = y_test,
              PCA = pca_training,
              verbose=False)


    

In [ ]:

    

print("N: {}".format(n.N))
print("d: {}".format(n.d))
print("C: {}".format(n.C))
print("X shape: {}".format(n.X.shape))
print("y shape: {}".format(n.y.shape))
print("Y shape: {}".format(n.Y.shape))


    

In [ ]:

    

n.run(epochs=10)


    

In [ ]:

    

n.C


    

In [ ]:

    

n.results.tail()


    

In [ ]:

    

sl = n.plot_square_loss(logy=True)


    

In [ ]:

    

sl.axes[0].set_ylim([0, 1])
sl


    

In [ ]:

    

l01 = n.plot_01_loss()


    

In [ ]:

    

#p = n.plot_weight_evolution()


    

In [ ]:

    

import pprint


    

In [ ]:

    

print(n.W1.shape)
pprint.pprint(n.W1[0:2, :].sum(axis=1))
pprint.pprint(n.W1[0:2, :].sum(axis=1))


    

In [ ]:

    

p1 = n.plot_sum_of_weights('W1')
p2 = n.plot_sum_of_weights('W2')


    

In [ ]:

    

p1 = n.plot_norm_of_gradient('W1', logy=True)
p2 = n.plot_norm_of_gradient('W2', logy=True)


    

In [ ]:

    

p = n.plot_sample_dot_prods()


    

In [ ]:

    

n.results.tail(1).T


    

In [ ]:

    

assert False


    

In [ ]:

    

n.display_hidden_node_as_image(np.zeros(shape=(50,)))


    

In [ ]:

    

n.display_hidden_node_as_image(np.ones(shape=(50,)))


    

In [ ]:

    

n.display_hidden_node_as_image(n.W1[0,:])


    

In [ ]:

    

n.W1[1,:].shape


    

In [ ]:

    

n.display_hidden_node_as_image(n.W1[1,:])


    

In [ ]:

    

n.display_hidden_node_as_image(n.W1[10,:])


    

In [ ]:

    

image_dir = '../figures/Q-2-1_weight_visualization'
make_dir(image_dir)


    

In [ ]:

    

n.visualize_10_W1_weights(image_dir)


    

In [ ]:

    




    

In [ ]: