notebook.community

Edit and run



In [ ]:

    
import sys
print(sys.version)



In [ ]:

    
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
%matplotlib inline
import time

import pandas as pd
import seaborn as sns



In [ ]:

    
import sys
sys.path.append('../code/')



In [ ]:

    
! which python



In [ ]:

    
from least_squares_sgd import LeastSquaresSGD

from kernel import NoKernel, Fourier



In [ ]:

    
from sklearn.datasets import make_classification
X, y = make_classification(n_samples=100, n_features=60, 
                           n_informative=60, n_redundant=0, n_repeated=0, 
                           n_classes=5, n_clusters_per_class=1, 
                           weights=None, flip_y=0.001, class_sep=1.0, 
                           hypercube=True, shift=0.0, scale=1.0, 
                           shuffle=True, random_state=None)



In [ ]:

    
model = LeastSquaresSGD(X=X, y=y, batch_size=2, kernel=Fourier,
                        eta0_search_start=0.01,
                        max_epochs=20,
                        assess_test_data_during_fitting=False)

model.find_good_learning_rate()



In [ ]:

    
model.eta0



In [ ]:

    
model.eta0 # = model.eta0/10



In [ ]:

    
model.run()



In [ ]:

    
model.results



In [ ]:

    
model.plot_01_loss()



In [ ]:

    
model.plot_01_loss(logx=True)



In [ ]:

    
model.plot_square_loss(logx=False)



In [ ]:

    
fig, ax = plt.subplots(1, 1, figsize=(4,3))
plot_data = model.results[model.results['step'] > 1]
plot_x = 'step'
plot_y = 'training (0/1 loss)/N'
colors = ['gray']
plt.plot(plot_data[plot_x], plot_data[plot_y],
             linestyle='--', marker='o',
             color=colors[0])



In [ ]:

    
model.results.columns



In [ ]:

    
model.plot_loss_and_eta()



In [ ]:

    
## Add notion of model improving itself



In [ ]:

    
dp = model.delta_percent
dp



In [ ]:

    
model.run_longer(epochs=10, delta_percent=dp/10)



In [ ]:

    
model.plot_W_bar_update_variance()



In [ ]:

    
model.plot_loss_and_eta()

Should diverge:



In [ ]:

    
model_diverge = LeastSquaresSGD(X=X, y=y, batch_size=2, eta0 = model.eta0*10,
                                kernel=Fourier, 
                                max_epochs=100,
                                assess_test_data_during_fitting=False)
model_diverge.run()



In [ ]:

    
model_diverge.plot_square_loss()



In [ ]: