Q-2-1 Binary Logistic Regression

Janet Matsen

Code notes:



In [1]:

    
import numpy as np
import matplotlib as mpl
%matplotlib inline
import time

import pandas as pd
import seaborn as sns

from mnist import MNIST  # public package for making arrays out of MINST data.



In [2]:

    
import sys
sys.path.append('../code/')



In [3]:

    
from logistic_regression import LogisticRegression, LogisticRegressionBinary
from hyperparameter_explorer import HyperparameterExplorer



In [4]:

    
from mnist_helpers import mnist_training, mnist_testing, mnist_training_binary, mnist_testing_binary



In [5]:

    
import matplotlib.pyplot as plt
from pylab import rcParams
rcParams['figure.figsize'] = 4, 3

Prepare MNIST training data

from sklearn.datasets import make_classification all_X, all_y = make_classification(n_samples=300, n_features=15, n_redundant=5, n_repeated=0, n_classes=2, n_clusters_per_class=1, weights=None, flip_y=0.01, class_sep=1.0, hypercube=True, shift=0.0, scale=1.0, shuffle=True, random_state=None) train_X, train_y = all_X[30:, ], all_y[30:,] test_X, test_y = all_X[0:30, ], all_y[0:30,]

from mnist_helpers import binary_shuffled train_X, train_y, test_X, test_y = binary_shuffled(2)



In [6]:

    
train_X, train_y = mnist_training_binary(2, shuffled=True)
test_X, test_y = mnist_testing_binary(2)









    



[    0     1     2 ..., 59997 59998 59999]
[    0     1     2 ..., 59997 59998 59999]
[   0    1    2 ..., 9997 9998 9999]



In [7]:

    
print(train_X.shape, train_y.shape)
print(test_X.shape, test_y.shape)









    



(60000, 784) (60000,)
(10000, 784) (10000,)



In [8]:

    
print(np.var(train_y))
print(np.var(test_y))

Explore hyperparameters before training model on all of the training data.



In [9]:

    
hyper_explorer = HyperparameterExplorer(X=train_X, y=train_y, 
                                        model=LogisticRegressionBinary, 
                                        validation_split=0.1, 
                                        score_name = '-(log loss)/N, training', 
                                        use_prev_best_weights=True,
                                        test_X=test_X, test_y=test_y)  # need test_X, test_y for loss w/ fit plot









    



6000 of 60000 points from training are reserved for validation
variances of all training data: 0.08943950999999999
variances of split-off training & validation data: 0.08924355246913578, 0.09119774999999997



In [10]:

    
eta0=1e-4
hyper_explorer.train_model(lam=0, eta0=eta0, max_iter=10**2, delta_percent=eta0*10, verbose=True)









    



.....0.228878939774
.....0.146840970623
.....0.125649138876
.....0.116492642243
.....0.110910637592
.....0.107000618198
.....0.104042612147
.....0.101692908967
.....0.099762720292
.....0.09813797404
.....0.0967447832983
.....0.0955326879921
.....0.0944657625179
.....0.0935175571451
.....0.0926680547512
.....0.0919017538654
.....0.0912064138766
.....0.0905722064818
.....0.0899911255772
.....0.0894565667915
max iterations (100) reached.



In [11]:

    
hyper_explorer.models









    Out[11]:





{1: <logistic_regression.LogisticRegressionBinary at 0x7fd8bab78c50>}



In [12]:

    
hyper_explorer.models[1].plot_test_and_train_log_loss_during_fitting()









    Out[12]:



In [13]:

    
eta0=1e-4
hyper_explorer.train_model(lam=0, eta0=eta0, max_iter=10**2, delta_percent=eta0*10, verbose=True)









    



returning best weights for lambda = 0.  Corresponded to -(log loss)/N, validation = 0.09458159254219557
.....0.0871346583877
.....0.0859488674225
.....0.0851317191929
.....0.0845082296359
.....0.0840058396195
.....0.0835867357178
.....0.0832285068557
.....0.0829167091236
.....0.0826414640379
.....0.0823957086457
.....0.0821742154485
.....0.0819730067097
.....0.0817889860499
.....0.0816196968851
.....0.0814631584872
.....0.081317751447
.....0.0811821356434
.....0.081055190219
.....0.0809359688263
.....0.0808236657044
max iterations (100) reached.



In [14]:

    
eta0=1e-4
hyper_explorer.train_model(lam=0, eta0=eta0, max_iter=10**2, delta_percent=eta0*10, verbose=True)









    



returning best weights for lambda = 0.  Corresponded to -(log loss)/N, validation = 0.0857362025578528
.....0.0802859175926
.....0.0799862332641
.....0.0797667631811
.....0.079590632267
.....0.0794422895999
.....0.0793135096726
.....0.0791993493192
.....0.0790965800223
.....0.0790029650092
.....0.0789168846443
.....0.0788371251493
.....0.0787627514865
.....0.0786930268733
.....0.0786273596651
.....0.0785652670753
.....0.0785063496674
.....0.0784502729711
.....0.0783967539455
.....0.0783455508228
.....0.0782964553623
max iterations (100) reached.



In [15]:

    
eta0=1e-4
hyper_explorer.train_model(lam=0, eta0=eta0, max_iter=10**2, delta_percent=eta0*10, verbose=True)









    



returning best weights for lambda = 0.  Corresponded to -(log loss)/N, validation = 0.08350384104444057
.....0.0780497008123
.....0.0779020845723
.....0.0777890802171
.....0.0776952715233
.....0.0776140504525
.....0.077541870851
.....0.0774765729361
.....0.0774167282205
.....0.077361335118
.....0.0773096600568
.....0.0772611472389
.....0.0772153639771
.....0.077171965857
.....0.0771306735957
.....0.0770912571368
.....0.0770535243984
.....0.077017313114
.....0.0769824847896
.....0.0769489201423
.....0.076916515602
max iterations (100) reached.



In [16]:

    
eta0=1e-4
hyper_explorer.train_model(lam=0, eta0=eta0, max_iter=10**2, delta_percent=eta0*10, verbose=True)









    



returning best weights for lambda = 0.  Corresponded to -(log loss)/N, validation = 0.08240930245078756
.....0.0767506561293
.....0.0766488737147
.....0.0765697448516
.....0.0765032920869
.....0.0764452145281
.....0.0763931927883
.....0.0763458079181
.....0.0763021175137
.....0.0762614582246
.....0.0762233424108
.....0.0761873993102
.....0.07615333932
.....0.0761209311815
.....0.0760899867943
.....0.0760603507563
.....0.0760318929497
.....0.0760045031541
.....0.0759780870491
.....0.0759525631916
.....0.075927860693
max iterations (100) reached.



In [17]:

    
eta0=1e-4
hyper_explorer.train_model(lam=0, eta0=eta0, max_iter=10**2, delta_percent=eta0*10, verbose=True)









    



returning best weights for lambda = 0.  Corresponded to -(log loss)/N, validation = 0.08167289490921754
.....0.075800580633
.....0.0757217242953
.....0.0756600506271
.....0.0756080169475
.....0.0755623668911
.....0.0755213419968
.....0.0754838651442
.....0.0754492201258
.....0.0754169022527
.....0.0753865401291
.....0.0753578510902
.....0.075330614131
.....0.0753046526049
.....0.075279822701
.....0.0752560055029
.....0.0752331013567
.....0.0752110257775
.....0.0751897064089
.....0.075169080724
.....0.0751490942567
max iterations (100) reached.



In [18]:

    
eta0=1e-4
hyper_explorer.train_model(lam=0, eta0=eta0, max_iter=2*10**2, delta_percent=eta0*10, verbose=True)









    



returning best weights for lambda = 0.  Corresponded to -(log loss)/N, validation = 0.08112243724716188
.....0.0750457675006
.....0.0749814322987
.....0.0749309535911
.....0.0748882569367
.....0.0748507183464
.....0.0748169201347
.....0.0747859935138
.....0.0747573605469
.....0.0747306138074
.....0.0747054533514
.....0.0746816508041
.....0.0746590275392
.....0.0746374407288
.....0.0746167740517
.....0.0745969312876
.....0.0745778317742
.....0.0745594071032
.....0.0745415986674
.....0.0745243558039
.....0.0745076343671
.....0.0744913956143
.....0.0744756053252
.....0.0744602330972
.....0.0744452517762
.....0.0744306369915
.....0.0744163667733
.....0.0744024212351
.....0.0743887823077
.....0.0743754335159
.....0.0743623597886
.....0.0743495472978
.....0.0743369833196
.....0.0743246561155
.....0.0743125548288
.....0.0743006693948
.....0.0742889904631
.....0.0742775093283
.....0.0742662178697
.....0.0742551084984
.....0.0742441741093
max iterations (200) reached.



In [19]:

    
eta0=1e-4
hyper_explorer.train_model(lam=0, eta0=eta0, max_iter=2*10**2, delta_percent=eta0*10, verbose=True)









    



returning best weights for lambda = 0.  Corresponded to -(log loss)/N, validation = 0.08052110543278589
.....0.0741637436545
.....0.0741134373717
.....0.0740738518527
.....0.0740402917022
.....0.0740107278717
.....0.0739840637449
.....0.0739596270842
.....0.0739369705707
.....0.0739157787694
.....0.0738958193958
.....0.0738769155427
.....0.0738589288033
.....0.0738417484803
.....0.0738252843968
.....0.0738094619408
.....0.0737942185485
.....0.073779501148
.....0.0737652642598
.....0.0737514685606
.....0.0737380797769
.....0.0737250678234
.....0.0737124061202
.....0.0737000710476
.....0.0736880415047
.....0.0736762985496
.....0.0736648251024
.....0.0736536056995
.....0.0736426262875
.....0.07363187405
.....0.0736213372604
.....0.0736110051569
.....0.073600867835
.....0.0735909161554
.....0.0735811416636
.....0.0735715365204
.....0.0735620934411
.....0.0735528056425
.....0.0735436667956
.....0.0735346709847
.....0.0735258126702
max iterations (200) reached.



In [20]:

    
eta0=1e-4
hyper_explorer.train_model(lam=0, eta0=eta0, max_iter=2*10**2, delta_percent=eta0*10, verbose=True)









    



returning best weights for lambda = 0.  Corresponded to -(log loss)/N, validation = 0.08007759502311682
.....0.0734605403699
.....0.0734195958085
.....0.0733873153192
.....0.0733599061765
.....0.073335729083
.....0.0733138979151
.....0.0732938693836
.....0.0732752819192
.....0.0732578805181
.....0.0732414773697
.....0.0732259294186
.....0.0732111247269
.....0.0731969737553
.....0.0731834035531
.....0.0731703537528
.....0.073157773729
.....0.0731456205309
.....0.073133857348
.....0.0731224523481
.....0.0731113777836
.....0.0731006092946
.....0.0730901253569
.....0.0730799068408
.....0.073069936655
.....0.0730601994548
.....0.0730506814022
.....0.0730413699676
.....0.0730322537622
.....0.0730233223988
.....0.0730145663721
.....0.0730059769581
.....0.0729975461267
.....0.0729892664673
.....0.0729811311237
.....0.0729731337383
.....0.0729652684023
.....0.072957529613
.....0.0729499122359
.....0.0729424114709
.....0.0729350228228
max iterations (200) reached.



In [21]:

    
eta0=1e-4
hyper_explorer.train_model(lam=0, eta0=eta0, max_iter=10**2, delta_percent=eta0*10, verbose=True)









    



returning best weights for lambda = 0.  Corresponded to -(log loss)/N, validation = 0.07973803269346255
.....0.0728805124401
.....0.072846245466
.....0.0728191917547
.....0.0727961946701
.....0.0727758896773
.....0.0727575390979
.....0.0727406906112
.....0.0727250431616
.....0.0727103844046
.....0.0726965579347
.....0.0726834446075
.....0.0726709511873
.....0.0726590030884
.....0.0726475395384
.....0.0726365102435
.....0.0726258730226
.....0.0726155920868
.....0.0726056367607
.....0.0725959805156
.....0.0725866002252
max iterations (100) reached.



In [22]:

    
eta0=1e-4
hyper_explorer.train_model(lam=0, eta0=eta0, max_iter=2*10**2, delta_percent=eta0*10, verbose=True)









    



returning best weights for lambda = 0.  Corresponded to -(log loss)/N, validation = 0.07954922266861769
.....0.0725377741995
.....0.0725070501913
.....0.0724827750958
.....0.0724621272096
.....0.0724438867043
.....0.0724273940969
.....0.0724122449942
.....0.0723981702282
.....0.0723849799373
.....0.0723725342704
.....0.0723607266893
.....0.0723494738195
.....0.0723387089616
.....0.0723283777666
.....0.0723184352548
.....0.0723088437007
.....0.0722995710943
.....0.0722905899983
.....0.0722818766833
.....0.0722734104629
.....0.0722651731734
.....0.0722571487643
.....0.0722493229689
.....0.0722416830392
.....0.0722342175286
.....0.0722269161136
.....0.0722197694453
.....0.0722127690257
.....0.0722059071026
.....0.0721991765817
.....0.072192570951
.....0.0721860842157
.....0.072179710843
.....0.0721734457136
.....0.0721672840795
.....0.072161221528
.....0.0721552539488
.....0.0721493775064
.....0.0721435886148
.....0.0721378839153
max iterations (200) reached.



In [23]:

    
eta0=1e-4
hyper_explorer.train_model(lam=0, eta0=eta0, max_iter=2*10**2, delta_percent=eta0*10, verbose=True)









    



returning best weights for lambda = 0.  Corresponded to -(log loss)/N, validation = 0.07931918542830481
.....0.0720957361029
.....0.0720691714288
.....0.0720481632312
.....0.0720302807102
.....0.0720144729549
.....0.0720001717722
.....0.0719870287295
.....0.0719748118949
.....0.0719633576641
.....0.0719525455215
.....0.0719422836541
.....0.0719325002081
.....0.071923137698
.....0.0719141492806
.....0.0719054961862
.....0.0718971458966
.....0.0718890708189
.....0.0718812473009
.....0.0718736548861
.....0.07186627574
.....0.0718590942032
.....0.0718520964368
.....0.0718452701399
.....0.0718386043203
.....0.0718320891077
.....0.0718257155999
.....0.0718194757348
.....0.0718133621836
.....0.0718073682606
.....0.0718014878472
.....0.0717957153268
.....0.0717900455287
.....0.0717844736804
.....0.071778995366
.....0.0717736064897
.....0.0717683032448
.....0.0717630820854
.....0.0717579397023
.....0.0717528730017
.....0.0717478790857
max iterations (200) reached.



In [24]:

    
eta0=1e-4
hyper_explorer.train_model(lam=0, eta0=eta0, max_iter=2*10**2, delta_percent=eta0*10, verbose=True)









    



returning best weights for lambda = 0.  Corresponded to -(log loss)/N, validation = 0.0791316595100899
.....0.071710959823
.....0.0716876631944
.....0.0716692256423
.....0.0716535217357
.....0.0716396325362
.....0.0716270611826
.....0.0716155029732
.....0.071604755078
.....0.0715946744158
.....0.0715851555853
.....0.0715761182855
.....0.0715674996701
.....0.0715592494579
.....0.0715513266751
.....0.0715436974089
.....0.0715363332137
.....0.0715292099516
.....0.0715223069315
.....0.0715156062573
.....0.0715090923261
.....0.0715027514377
.....0.0714965714843
.....0.0714905417031
.....0.0714846524763
.....0.0714788951675
.....0.0714732619869
.....0.07146774588
.....0.0714623404332
.....0.0714570397962
.....0.0714518386143
.....0.071446731972
.....0.0714417153442
.....0.0714367845539
.....0.0714319357361
.....0.0714271653061
.....0.0714224699318
.....0.0714178465096
.....0.0714132921428
.....0.0714088041231
.....0.0714043799139
max iterations (200) reached.

Run for much longer...



In [25]:

    
eta0=1e-4
hyper_explorer.train_model(lam=0, eta0=eta0, max_iter=10**3, delta_percent=eta0*10, verbose=False)









    



returning best weights for lambda = 0.  Corresponded to -(log loss)/N, validation = 0.07897629896368533
.................................................................................................................................................................................................................................................Loss optimized.  Old/N: 0.0710686606963591, new/N:0.07106795124652061. Eta: 1.1904154565430094e-10



In [26]:

    
eta0=1e-4
hyper_explorer.train_model(lam=0, eta0=eta0, max_iter=10**2, delta_percent=eta0*10, verbose=False)









    



returning best weights for lambda = 0.  Corresponded to -(log loss)/N, validation = 0.07883319337753292
....................................................................................................max iterations (100) reached.



In [27]:

    
eta0=1e-4
hyper_explorer.train_model(lam=0, eta0=eta0, max_iter=10**2, delta_percent=eta0*10, verbose=False)









    



returning best weights for lambda = 0.  Corresponded to -(log loss)/N, validation = 0.0787570116830791
....................................................................................................max iterations (100) reached.



In [28]:

    
eta0=1e-4
hyper_explorer.train_model(lam=0, eta0=eta0, max_iter=3*10**2, delta_percent=eta0*10, verbose=False)









    



returning best weights for lambda = 0.  Corresponded to -(log loss)/N, validation = 0.078688465140148
..............................................................................................................................................................Loss optimized.  Old/N: 0.07049375138062645, new/N:0.07049304823466014. Eta: 1.468614047626193e-10



In [30]:

    
eta0=2*1e-4
hyper_explorer.train_model(lam=10, eta0=eta0, max_iter=10**3, delta_percent=eta0*10, verbose=False)









    



returning best weights for lambda = 0.  Corresponded to -(log loss)/N, validation = 0.0786097617585476
..........................................................................................................................Loss optimized.  Old/N: 0.07016605969425808, new/N:0.07016465721475812. Eta: 3.3395171654349344e-10



In [31]:

    
eta0=2*1e-4
hyper_explorer.train_model(lam=10, eta0=eta0, max_iter=10**3, delta_percent=eta0*10, verbose=False)









    



returning best weights for lambda = 10.  Corresponded to -(log loss)/N, validation = 0.07849489116688248
...................................................................................................Loss optimized.  Old/N: 0.069906434919012, new/N:0.06990503760151823. Eta: 3.703703703703704e-10



In [32]:

    
eta0=2*1e-4
hyper_explorer.train_model(lam=10, eta0=eta0, max_iter=10**3, delta_percent=eta0*10, verbose=False)









    



returning best weights for lambda = 10.  Corresponded to -(log loss)/N, validation = 0.07841134188698191
...................................................................................Loss optimized.  Old/N: 0.06969364433747575, new/N:0.0696922517548431. Eta: 4.0410720414072666e-10



In [34]:

    
eta0=2*1e-4
hyper_explorer.train_model(lam=10, eta0=eta0, max_iter=10**3, delta_percent=eta0*10, verbose=False)









    



returning best weights for lambda = 10.  Corresponded to -(log loss)/N, validation = 0.078346259057721
........Loss optimized.  Old/N: 0.06965205440709724, new/N:0.06965323033629169. Eta: 1.234567901234568e-09



In [35]:

    
eta0=2*1e-4
hyper_explorer.train_model(lam=1e2, eta0=eta0, max_iter=10**3, delta_percent=eta0*10, verbose=False)









    



returning best weights for lambda = 10.  Corresponded to -(log loss)/N, validation = 0.0782363106447062
...............................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................Loss optimized.  Old/N: 0.069357999427404, new/N:0.06935661318712855. Eta: 1.590876961435399e-10



In [36]:

    
eta0=2*1e-4
hyper_explorer.train_model(lam=1e2, eta0=eta0, max_iter=10**3, delta_percent=eta0*10, verbose=False)









    



returning best weights for lambda = 100.0.  Corresponded to -(log loss)/N, validation = 0.07885582344325766
..........Loss optimized.  Old/N: 0.06925203057280886, new/N:0.06925218069614712. Eta: 1.1167086836213468e-09



In [38]:

    
eta0=2*1e-4
hyper_explorer.train_model(lam=1e2, eta0=eta0, max_iter=10**3, delta_percent=eta0*10, verbose=False)









    



returning best weights for lambda = 100.0.  Corresponded to -(log loss)/N, validation = 0.07858404189294003
.................................................................................................................................Loss optimized.  Old/N: 0.06903532135408444, new/N:0.06903463313717496. Eta: 1.6241815172352394e-10



In [ ]:

    
hyper_explorerr_explorer.plot



In [39]:

    
eta0=2*1e-4
hyper_explorer.train_model(lam=100, eta0=eta0, max_iter=5*10**2, delta_percent=eta0*10, verbose=False)









    



returning best weights for lambda = 100.0.  Corresponded to -(log loss)/N, validation = 0.07858404189294003
....................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................max iterations (500) reached.



In [40]:

    
eta0=1*1e-4
hyper_explorer.train_model(lam=100, eta0=eta0, max_iter=5*10**2, delta_percent=eta0*10, verbose=False)









    



returning best weights for lambda = 100.0.  Corresponded to -(log loss)/N, validation = 0.07858404189294003
.................................................................................................................................Loss optimized.  Old/N: 0.06903532135399286, new/N:0.06903463313708263. Eta: 1.6241815172352394e-10



In [41]:

    
eta0=1*1e-4
hyper_explorer.train_model(lam=100, eta0=eta0, max_iter=5*10**2, delta_percent=eta0*10, verbose=False)









    



returning best weights for lambda = 100.0.  Corresponded to -(log loss)/N, validation = 0.07854219831489401
.................................................................................Loss optimized.  Old/N: 0.06892624365596425, new/N:0.06892555785894784. Eta: 2.0450282606453064e-10



In [42]:

    
eta0=1*1e-4
hyper_explorer.train_model(lam=100, eta0=eta0, max_iter=5*10**2, delta_percent=eta0*10, verbose=False)









    



returning best weights for lambda = 100.0.  Corresponded to -(log loss)/N, validation = 0.07845087757591102
.............................................................Loss optimized.  Old/N: 0.06884923859703629, new/N:0.0688485539799639. Eta: 2.351854203707232e-10



In [43]:

    
hyper_explorer.plot_best_fits()









    



../code/hyperparameter_explorer.py:183: FutureWarning: sort(columns=....) is deprecated, use sort_values(by=.....)
  plot_data = df.sort(x)



In [44]:

    
eta0=1*1e-4
hyper_explorer.train_model(lam=1e3, eta0=eta0, max_iter=5*10**2, delta_percent=eta0*10, verbose=False)









    



returning best weights for lambda = 1000.0.  Corresponded to -(log loss)/N, validation = 0.07854219831456688
.................................................................................Loss optimized.  Old/N: 0.06892624365621222, new/N:0.06892555785919703. Eta: 2.0450282606453064e-10



In [45]:

    
eta0=1*1e-4
hyper_explorer.train_model(lam=1e3, eta0=eta0, max_iter=5*10**2, delta_percent=eta0*10, verbose=False)









    



returning best weights for lambda = 1000.0.  Corresponded to -(log loss)/N, validation = 0.07845087757546047
.............................................................Loss optimized.  Old/N: 0.06884923859743809, new/N:0.06884855398036721. Eta: 2.351854203707232e-10



In [46]:

    
eta0=1*1e-4
hyper_explorer.train_model(lam=1e3, eta0=eta0, max_iter=5*10**2, delta_percent=eta0*10, verbose=False)









    



returning best weights for lambda = 1000.0.  Corresponded to -(log loss)/N, validation = 0.07839367688008966
..................................................Loss optimized.  Old/N: 0.06878822090813054, new/N:0.06878753420884505. Eta: 2.5931112667185367e-10



In [47]:

    
eta0=1*1e-4
hyper_explorer.train_model(lam=1e4, eta0=eta0, max_iter=5*10**2, delta_percent=eta0*10, verbose=False)









    



returning best weights for lambda = 1000.0.  Corresponded to -(log loss)/N, validation = 0.0783529752598479
............................................Loss optimized.  Old/N: 0.068736029005488, new/N:0.06873534657962829. Eta: 2.7605777499997404e-10



In [48]:

    
eta0=1*1e-4
hyper_explorer.train_model(lam=1e4, eta0=eta0, max_iter=5*10**2, delta_percent=eta0*10, verbose=False)









    



returning best weights for lambda = 10000.0.  Corresponded to -(log loss)/N, validation = 0.07832134630775965
........................................Loss optimized.  Old/N: 0.0686896016090476, new/N:0.06868892349610707. Eta: 2.8921067016408535e-10



In [49]:

    
eta0=1*1e-4
hyper_explorer.train_model(lam=1e4, eta0=eta0, max_iter=5*10**2, delta_percent=eta0*10, verbose=False)









    



returning best weights for lambda = 10000.0.  Corresponded to -(log loss)/N, validation = 0.07829552066587024
....................................Loss optimized.  Old/N: 0.06864802397047269, new/N:0.06864733931487374. Eta: 3.0444256908399504e-10



In [50]:

    
hyper_explorer.plot_best_fits()









    



../code/hyperparameter_explorer.py:183: FutureWarning: sort(columns=....) is deprecated, use sort_values(by=.....)
  plot_data = df.sort(x)



In [51]:

    
eta0=1*1e-4
hyper_explorer.train_model(lam=1e5, eta0=eta0, max_iter=5*10**2, delta_percent=eta0*10, verbose=False)









    



returning best weights for lambda = 10000.0.  Corresponded to -(log loss)/N, validation = 0.0782740807983206
..................................Loss optimized.  Old/N: 0.0686093454114367, new/N:0.06860866558380993. Eta: 3.1302009434389505e-10



In [52]:

    
eta0=1*1e-4
hyper_explorer.train_model(lam=1e5, eta0=eta0, max_iter=5*10**2, delta_percent=eta0*10, verbose=False)









    



returning best weights for lambda = 100000.0.  Corresponded to -(log loss)/N, validation = 0.07825544770448727
................................Loss optimized.  Old/N: 0.06857324535630785, new/N:0.06857256574072808. Eta: 3.223660295475886e-10



In [53]:

    
eta0=1e-4
hyper_explorer.train_model(lam=1e5, eta0=eta0, max_iter=10**2, delta_percent=eta0*10, verbose=True)









    



returning best weights for lambda = 100000.0.  Corresponded to -(log loss)/N, validation = 0.07823908549480486
.....0.0685623077617
.....0.0685558085734
.....0.0685506502449
.....0.0685462463079
.....0.0685423431886
.....0.068538803779
Loss optimized.  Old/N: 0.06853948710715517, new/N:0.06853880377899316. Eta: 3.326024111606943e-10



In [54]:

    
eta0=1e-4
hyper_explorer.train_model(lam=1e5, eta0=eta0, max_iter=10**2, delta_percent=eta0*10, verbose=True)









    



returning best weights for lambda = 100000.0.  Corresponded to -(log loss)/N, validation = 0.07822460124299482
.....0.0685288037083
.....0.0685224643715
.....0.0685174310381
.....0.0685131325496
.....0.0685093219407
....Loss optimized.  Old/N: 0.0685072116863605, new/N:0.06850653300304363. Eta: 3.3810034413899146e-10



In [55]:

    
hyper_explorer.plot_best_fits()









    



../code/hyperparameter_explorer.py:183: FutureWarning: sort(columns=....) is deprecated, use sort_values(by=.....)
  plot_data = df.sort(x)



In [56]:

    
eta0=1e-4
hyper_explorer.train_model(lam=1e6, eta0=eta0, max_iter=10**2, delta_percent=eta0*10, verbose=True)









    



returning best weights for lambda = 100000.0.  Corresponded to -(log loss)/N, validation = 0.07821143073936977
.....0.0684967598435
.....0.068490561595
.....0.0684856388524
.....0.0684814338419
.....0.0684777053774
...Loss optimized.  Old/N: 0.06847631617333368, new/N:0.06847564032691761. Eta: 3.438802558834294e-10



In [57]:

    
eta0=1e-4
hyper_explorer.train_model(lam=1e6, eta0=eta0, max_iter=10**2, delta_percent=eta0*10, verbose=True)









    



returning best weights for lambda = 1000000.0.  Corresponded to -(log loss)/N, validation = 0.07819938789277217
.....0.0684660695576
.....0.0684599975494
.....0.0684551739612
.....0.0684510528988
.....0.0684473983068
..Loss optimized.  Old/N: 0.06844671106002595, new/N:0.06844603649481965. Eta: 3.4996710463817337e-10



In [58]:

    
eta0=1e-4
hyper_explorer.train_model(lam=1e6, eta0=eta0, max_iter=10**2, delta_percent=eta0*10, verbose=True)









    



returning best weights for lambda = 1000000.0.  Corresponded to -(log loss)/N, validation = 0.07818832510965641
.....0.0684366484296
.....0.0684306906876
.....0.0684259570029
.....0.0684219121602
.....0.0684183247179
.Loss optimized.  Old/N: 0.06841832471792278, new/N:0.06841765005299363. Eta: 3.5638905505532455e-10



In [59]:

    
eta0=1e-4
hyper_explorer.train_model(lam=1e7, eta0=eta0, max_iter=10**2, delta_percent=eta0*10, verbose=True)









    



returning best weights for lambda = 1000000.0.  Corresponded to -(log loss)/N, validation = 0.07817812361756175
.....0.0684084288257
.....0.0684025756654
.....0.0683979243837
.....0.0683939494829
.....0.0683904237199
Loss optimized.  Old/N: 0.06839109973128206, new/N:0.06839042371992572. Eta: 3.6317802803367417e-10



In [60]:

    
eta0=1e-4
hyper_explorer.train_model(lam=1e7, eta0=eta0, max_iter=10**2, delta_percent=eta0*10, verbose=True)









    



returning best weights for lambda = 10000000.0.  Corresponded to -(log loss)/N, validation = 0.07816868588693919
.....0.0683813554819
.....0.0683755983998
.....0.0683710229082
.....0.0683671123956
....Loss optimized.  Old/N: 0.0683649871785017, new/N:0.06836430859510782. Eta: 3.703703703703704e-10



In [61]:

    
eta0=1e-4
hyper_explorer.train_model(lam=1e7, eta0=eta0, max_iter=10**2, delta_percent=eta0, verbose=True)









    



returning best weights for lambda = 10000000.0.  Corresponded to -(log loss)/N, validation = 0.07815993343927233
.....0.0683553817697
.....0.0683497136365
.....0.0683452083799
.....0.0683413575825
.....0.0683379413825
.....0.0683348408835
.....0.0683319831619
.....0.0683293196685
.....0.0683268161514
.....0.0683244473759
.....0.0683221941143
.....0.068320041317
.....0.0683179769417
.....0.0683159911745
.....0.0683140758916
.....0.0683122242782
.....0.0683104305511
.....0.0683086897522
.....0.0683069975927
.....0.0683053503331
max iterations (100) reached.



In [62]:

    
hyper_explorer.plot_best_fits()









    



../code/hyperparameter_explorer.py:183: FutureWarning: sort(columns=....) is deprecated, use sort_values(by=.....)
  plot_data = df.sort(x)



In [63]:

    
eta0=1e-4
hyper_explorer.train_model(lam=1e8, eta0=eta0, max_iter=3*10**2, delta_percent=eta0, verbose=True)









    



returning best weights for lambda = 10000000.0.  Corresponded to -(log loss)/N, validation = 0.07814116290495603
.....0.068296738305
.....0.068291263376
.....0.0682869108823
.....0.068283190107
.....0.0682798888393
.....0.0682768923192
.....0.0682741301588
.....0.0682715554967
.....0.0682691352714
.....0.0682668451248
.....0.0682646664972
.....0.0682625848614
.....0.0682605885934
.....0.0682586682194
.....0.0682568158978
.....0.0682550250501
.....0.0682532900935
.....0.0682516062417
.....0.0682499693548
.....0.0682483758228
.....0.0682468224755
.....0.0682453065109
.....0.0682438254381
.....0.0682423770309
.....0.0682409592901
.....0.068239570412
.....0.0682382087634
.....0.068236872859
.....0.0682355613439
.....0.0682342729779
.....0.0682330066224
.....0.068231761229
.....0.06823053583
.....0.0682293295297
.....0.0682281414975
.....0.068226970961
.....0.068225817201
.....0.0682246795458
.....0.0682235573677
.....0.0682224500784
.....0.0682213571262
.....0.0682202779922
.....0.0682192121884
.....0.0682181592547
.....0.0682171187567
.....0.0682160902842
.....0.0682150734486
.....0.0682140678822
.....0.0682130732358
.....0.0682120891783
.....0.0682111153946
.....0.0682101515851
.....0.0682091974643
.....0.0682082527603
.....0.0682073172135
.....0.0682063905759
.....0.0682054726106
.....0.0682045630911
.....0.0682036618003
.....0.0682027685305
max iterations (300) reached.



In [64]:

    
eta0=1e-4
hyper_explorer.train_model(lam=1e8, eta0=eta0, max_iter=3*10**2, delta_percent=eta0, verbose=False)









    



returning best weights for lambda = 100000000.0.  Corresponded to -(log loss)/N, validation = 0.07811137692953254
.....0.068194657713
.....0.0681894918963
.....0.0681853842043
.....0.0681818720548
.....0.0681787554011
.....0.0681759260596
.....0.0681733176751
.....0.0681708860708
.....0.0681686000784
.....0.0681664367348
.....0.0681643785427
.....0.0681624118065
.....0.0681605255674
.....0.0681587108938
.....0.0681569603926
.....0.0681552678616
.....0.0681536280373
.....0.0681520364073
.....0.0681504890681
.....0.0681489826161
.....0.0681475140623
.....0.068146080765
.....0.0681446803754
.....0.0681433107946
.....0.0681419701375
.....0.0681406567035
.....0.0681393689525
.....0.0681381054842
.....0.0681368650209
.....0.0681356463932
.....0.0681344485275
.....0.0681332704352
.....0.0681321112037
.....0.0681309699887
.....0.0681298460066
.....0.0681287385293
.....0.0681276468784
.....0.0681265704207
.....0.0681255085642
.....0.068124460754
.....0.0681234264699
.....0.0681224052227
.....0.068121396552
.....0.068120400024
.....0.0681194152292
.....0.0681184417804
.....0.0681174793114
.....0.0681165274752
.....0.0681155859428
.....0.0681146544016
.....0.0681137325546
.....0.0681128201193
.....0.0681119168264
.....0.0681110224193
.....0.0681101366533
.....0.0681092592945
.....0.0681083901195
.....0.0681075289146
.....0.0681066754751
.....0.068105829605
max iterations (300) reached.



In [65]:

    
eta0=1e-4
hyper_explorer.train_model(lam=1e8, eta0=eta0, max_iter=3*10**2, delta_percent=eta0, verbose=False)









    



returning best weights for lambda = 100000000.0.  Corresponded to -(log loss)/N, validation = 0.07808622009788238
............................................................................................................................................................................................................................................................................................................max iterations (300) reached.



In [66]:

    
eta0=2e-4
hyper_explorer.train_model(lam=1e8, eta0=eta0, max_iter=3*10**2, delta_percent=eta0, verbose=False)









    



returning best weights for lambda = 100000000.0.  Corresponded to -(log loss)/N, validation = 0.07806477464122763
............................................................................................................................................................................................................................................................................................................max iterations (300) reached.



In [67]:

    
eta0=2e-4
hyper_explorer.train_model(lam=1e8, eta0=eta0, max_iter=3*10**2, delta_percent=eta0, verbose=False)









    



returning best weights for lambda = 100000000.0.  Corresponded to -(log loss)/N, validation = 0.07803082747957293
............................................................................................................................................................................................................................................................................................................max iterations (300) reached.



In [68]:

    
eta0=2e-4
hyper_explorer.train_model(lam=1e8, eta0=eta0, max_iter=3*10**2, delta_percent=eta0*10, verbose=False)









    



returning best weights for lambda = 100000000.0.  Corresponded to -(log loss)/N, validation = 0.07800651836679405
......Loss optimized.  Old/N: 0.06767597826203207, new/N:0.06767517467270928. Eta: 1.3998684185526935e-09



In [69]:

    
hyper_explorer.plot_best_fits()









    



../code/hyperparameter_explorer.py:183: FutureWarning: sort(columns=....) is deprecated, use sort_values(by=.....)
  plot_data = df.sort(x)



In [73]:

    
eta0=1e-4
hyper_explorer.train_model(lam=1e9, eta0=eta0, max_iter=3*10**2, delta_percent=eta0*10, verbose=False)









    



returning best weights for lambda = 1000000000.0.  Corresponded to -(log loss)/N, validation = 0.0815273428562595
............................................................................................................................................................................................................................................................................................................max iterations (300) reached.



In [74]:

    
eta0=1e-4
hyper_explorer.train_model(lam=1e9, eta0=eta0, max_iter=3*10**2, delta_percent=eta0*5, verbose=False)









    



returning best weights for lambda = 1000000000.0.  Corresponded to -(log loss)/N, validation = 0.07940318861036862
............................................................................................................................................................................................................................................................................................................max iterations (300) reached.



In [75]:

    
eta0=1e-4
hyper_explorer.train_model(lam=1e9, eta0=eta0, max_iter=3*10**2, delta_percent=eta0*5, verbose=False)









    



returning best weights for lambda = 1000000000.0.  Corresponded to -(log loss)/N, validation = 0.07878530277954121
.................................................................................................................................................................................................................Loss optimized.  Old/N: 0.06742717867219988, new/N:0.06742684217529574. Eta: 1.2778991839673228e-10



In [76]:

    
eta0=1e-4
hyper_explorer.train_model(lam=1e9, eta0=eta0, max_iter=3*10**2, delta_percent=eta0*5, verbose=False)









    



returning best weights for lambda = 1000000000.0.  Corresponded to -(log loss)/N, validation = 0.07856518066800058
...............................................................................................................Loss optimized.  Old/N: 0.06734932595350092, new/N:0.06734898929075153. Eta: 1.7498355231908669e-10



In [77]:

    
eta0=1e-4
hyper_explorer.train_model(lam=1e9, eta0=eta0, max_iter=3*10**2, delta_percent=eta0*5, verbose=False)









    



returning best weights for lambda = 1000000000.0.  Corresponded to -(log loss)/N, validation = 0.07847134951692886
............................................................................Loss optimized.  Old/N: 0.0673004238411701, new/N:0.06730008840872094. Eta: 2.110381045548851e-10



In [78]:

    
eta0=1e-4
hyper_explorer.train_model(lam=1e9, eta0=eta0, max_iter=3*10**2, delta_percent=eta0*5, verbose=False)









    



returning best weights for lambda = 1000000000.0.  Corresponded to -(log loss)/N, validation = 0.07841582469421485
...........................................................Loss optimized.  Old/N: 0.06726436686098192, new/N:0.06726403289391442. Eta: 2.3907304606218624e-10



In [79]:

    
hyper_explorer.plot_best_fits()









    



../code/hyperparameter_explorer.py:183: FutureWarning: sort(columns=....) is deprecated, use sort_values(by=.....)
  plot_data = df.sort(x)



In [80]:

    
eta0=1e-4
hyper_explorer.train_model(lam=1e10, eta0=eta0, max_iter=3*10**2, delta_percent=eta0*5, verbose=False)









    



returning best weights for lambda = 1000000000.0.  Corresponded to -(log loss)/N, validation = 0.07837748272490987
..............................................Loss optimized.  Old/N: 0.06723705026770228, new/N:0.0672367147864651. Eta: 2.701203546257307e-10



In [81]:

    
eta0=1e-4
hyper_explorer.train_model(lam=1e10, eta0=eta0, max_iter=3*10**2, delta_percent=eta0*5, verbose=False)









    



returning best weights for lambda = 10000000000.0.  Corresponded to -(log loss)/N, validation = 0.07834729010085106
........................................Loss optimized.  Old/N: 0.06721399770696627, new/N:0.06721366469560247. Eta: 2.8921067016408535e-10



In [82]:

    
eta0=1e-4
hyper_explorer.train_model(lam=1e10, eta0=eta0, max_iter=3*10**2, delta_percent=eta0*5, verbose=False)









    



returning best weights for lambda = 10000000000.0.  Corresponded to -(log loss)/N, validation = 0.07832324308999172
...................................Loss optimized.  Old/N: 0.06719410319823614, new/N:0.06719376821639135. Eta: 3.08641975308642e-10



In [83]:

    
hyper_explorer.plot_best_fits()









    



../code/hyperparameter_explorer.py:183: FutureWarning: sort(columns=....) is deprecated, use sort_values(by=.....)
  plot_data = df.sort(x)



In [84]:

    
eta0=1e-4
hyper_explorer.train_model(lam=1e10, eta0=eta0, max_iter=3*10**2, delta_percent=eta0, verbose=False)









    



returning best weights for lambda = 10000000000.0.  Corresponded to -(log loss)/N, validation = 0.07830352397903066
............................................................................................................................................................................................................................................................................................................max iterations (300) reached.



In [85]:

    
eta0=1e-4
hyper_explorer.train_model(lam=1e11, eta0=eta0, max_iter=3*10**2, delta_percent=eta0*5, verbose=False)









    



returning best weights for lambda = 10000000000.0.  Corresponded to -(log loss)/N, validation = 0.07824995568632814
.......Loss optimized.  Old/N: 0.06713094025329445, new/N:0.06713061627407693. Eta: 6.547285010986551e-10



In [86]:

    
eta0=1e-4
hyper_explorer.train_model(lam=1e11, eta0=eta0, max_iter=3*10**2, delta_percent=eta0*5, verbose=False)









    



returning best weights for lambda = 100000000000.0.  Corresponded to -(log loss)/N, validation = 0.07823846057747147
......Loss optimized.  Old/N: 0.06712826335738181, new/N:0.06712793524342431. Eta: 6.999342092763467e-10



In [87]:

    
hyper_explorer.plot_best_fits()









    



../code/hyperparameter_explorer.py:183: FutureWarning: sort(columns=....) is deprecated, use sort_values(by=.....)
  plot_data = df.sort(x)



In [96]:

    
eta0=1e-20
hyper_explorer.train_model(lam=1e12, eta0=eta0, max_iter=3*10**2, delta_percent=eta0*5, verbose=False)









    



returning best weights for lambda = 100000000000.0.  Corresponded to -(log loss)/N, validation = 0.07822902535519212
.Loss optimized.  Old/N: 0.06712793739590155, new/N:0.06712793739590155. Eta: 1.30945700219731e-25



In [97]:

    
eta0=1e-18
hyper_explorer.train_model(lam=1e12, eta0=eta0, max_iter=3*10**2, delta_percent=eta0*5, verbose=False)









    



returning best weights for lambda = 1000000000000.0.  Corresponded to -(log loss)/N, validation = 0.07822902482033382
.Loss optimized.  Old/N: 0.06712793739590155, new/N:0.06712793739590155. Eta: 1.3094570021973102e-23



In [99]:

    
eta0=1e-15
hyper_explorer.train_model(lam=1e12, eta0=eta0, max_iter=3*10**2, delta_percent=eta0*5, verbose=False)









    



---------------------------------------------------------------------------
AssertionError                            Traceback (most recent call last)
<ipython-input-99-077bfe0a72a1> in <module>()
      1 eta0=1e-15
----> 2 hyper_explorer.train_model(lam=1e12, eta0=eta0, max_iter=3*10**2, delta_percent=eta0*5, verbose=False)

/home/jmatsen/Machine_Learning_CSE_546/HW2/code/hyperparameter_explorer.py in train_model(self, **kwargs)
     54             # Note: this is silly for non-iterative solvers like Ridge.
     55             if self.use_prev_best_weights:
---> 56                 best_weights = self.best_weights_given_lam(m.lam)
     57                 if "W" in m.__dict__.keys() and best_weights is not None:
     58                     # TODO: new fun best_weights_given_lam()

/home/jmatsen/Machine_Learning_CSE_546/HW2/code/hyperparameter_explorer.py in best_weights_given_lam(self, lam)
    165             closest_lambda, self.validation_score_name, closest_score))
    166 
--> 167         closest_row = \
    168             self.summary[(self.summary['lambda'] == closest_lambda) &
    169                          (self.summary[self.validation_score_name] ==

AssertionError:



In [104]:

    
pbf = hyper_explorer.plot_best_fits()









    



../code/hyperparameter_explorer.py:183: FutureWarning: sort(columns=....) is deprecated, use sort_values(by=.....)
  y2 = self.training_score_name

For hyper explorer's best model:

'lambda': [100000000.0]
'lambda normalized': [180.1107372127079]
'-(log loss)/N, training': [0.069521850509031063]
'training (0/1 loss)/N': [0.018574074074074073],
'log loss': [-3754.1799274876776],
'eta0': [0.0002],
'# nonzero weights': [267]}



In [107]:

    
hyper_explorer.train_on_whole_training_set(max_iter=1000)









    



getting best model.
{'lambda normalized': [180.1107372127079], 'iteration': [6], '-(log loss)/N, training': [0.069521850509031063], 'eta': [1.3998684185526935e-09], 'weights': [array([  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,
         0.00000000e+00,   0.00000000e+00,   0.00000000e+00,
         0.00000000e+00,   0.00000000e+00,   0.00000000e+00,
         0.00000000e+00,   0.00000000e+00,   0.00000000e+00,
        -5.58148779e-06,  -6.13285630e-06,   6.77553291e-06,
         2.82313871e-07,   0.00000000e+00,   0.00000000e+00,
         0.00000000e+00,   0.00000000e+00,   0.00000000e+00,
         0.00000000e+00,   0.00000000e+00,   0.00000000e+00,
         0.00000000e+00,   0.00000000e+00,   0.00000000e+00,
         0.00000000e+00,   0.00000000e+00,   0.00000000e+00,
         0.00000000e+00,   0.00000000e+00,  -1.30037105e-07,
        -4.93521904e-07,  -3.17312765e-05,  -8.06400552e-05,
        -2.33837336e-04,  -4.51900233e-04,  -5.46991084e-04,
        -3.60556778e-04,  -6.36516885e-04,  -6.41328235e-04,
        -6.08539709e-04,  -4.92555182e-04,  -3.99264872e-05,
        -5.81029040e-04,  -3.96035355e-04,  -4.74587791e-04,
        -3.24027451e-04,  -8.77740033e-05,  -1.82961043e-05,
        -7.25706927e-06,   0.00000000e+00,   0.00000000e+00,
         0.00000000e+00,   0.00000000e+00,   0.00000000e+00,
         0.00000000e+00,  -8.24550758e-08,  -3.33829833e-07,
        -2.32465721e-05,  -8.08645002e-06,  -4.20597500e-05,
        -3.09068554e-04,  -8.27459143e-04,  -8.26729183e-04,
        -1.90082725e-03,  -1.71988595e-03,  -1.81850044e-03,
        -2.41896452e-03,  -2.80398125e-03,  -2.30076694e-03,
        -2.59292371e-03,  -2.91929394e-03,  -3.29901161e-03,
        -3.03730492e-03,  -1.91086400e-03,  -1.08143181e-03,
        -7.45987456e-04,  -1.91747932e-04,   4.52707251e-06,
         2.39423970e-05,   0.00000000e+00,   0.00000000e+00,
         0.00000000e+00,   0.00000000e+00,  -1.15713298e-06,
        -1.97752211e-06,  -2.08648682e-05,  -6.81410968e-05,
        -8.11388077e-05,  -4.61877170e-06,  -2.74801456e-05,
         1.41801845e-03,   1.09858792e-03,   3.94991991e-04,
        -9.00290307e-05,   5.86352871e-04,  -6.64304109e-04,
        -1.37005957e-03,  -4.43914414e-04,  -1.16630444e-03,
        -2.09840230e-03,  -1.57120522e-03,  -1.48214953e-03,
        -1.51421951e-03,  -1.06802790e-03,  -7.94212737e-04,
        -2.24001866e-04,  -2.40646492e-06,  -2.45163216e-06,
         0.00000000e+00,   0.00000000e+00,  -1.14551932e-06,
        -1.99119336e-05,  -8.59114814e-05,  -3.37685749e-04,
        -1.22782485e-04,   4.70636387e-04,   2.90075250e-04,
         6.49636484e-04,   1.73667353e-03,   6.00600767e-04,
        -5.23983655e-06,   5.30736747e-04,   5.51578515e-04,
         9.90854041e-04,  -4.87044686e-04,   7.34062363e-04,
        -1.76637440e-03,  -5.39908855e-04,  -1.84655150e-03,
        -1.28591331e-03,  -3.13023037e-03,  -3.34261290e-03,
        -2.67060075e-03,  -1.53738570e-03,  -3.48872709e-04,
        -4.58066058e-05,  -4.01798152e-05,   0.00000000e+00,
         0.00000000e+00,  -1.18845962e-05,  -6.66365156e-05,
        -4.81271757e-05,  -6.17331958e-04,  -8.69168577e-04,
        -5.57962965e-05,   7.12545143e-04,  -7.20632861e-04,
         6.97085988e-04,   1.58363785e-03,   3.85506279e-04,
         1.10771724e-03,   3.84557238e-05,   7.34015697e-04,
         1.62050510e-03,   9.89399601e-04,   1.29156621e-03,
        -8.83774769e-04,   1.04432859e-05,  -1.61602608e-04,
        -7.19620893e-04,  -3.00218160e-03,  -2.37261198e-03,
        -4.99188684e-04,  -1.68874615e-04,  -5.93902899e-05,
         0.00000000e+00,  -3.42692488e-07,  -1.24845411e-04,
        -2.94561558e-04,  -9.04114741e-05,  -1.92625829e-04,
        -2.49400918e-04,   2.73336769e-04,  -1.03155628e-04,
         8.14564402e-04,   1.66801457e-04,   3.28224068e-04,
        -1.91116810e-05,   7.28630373e-04,   2.18745007e-04,
         8.13894584e-04,   4.42717495e-04,   4.06761555e-04,
        -6.42292888e-04,  -7.77505818e-05,  -1.39699142e-03,
         4.37245553e-05,  -7.67935198e-04,  -3.34895068e-03,
        -2.62556952e-03,  -7.85907135e-04,  -1.53095424e-04,
        -8.48348503e-06,  -4.54203422e-07,  -3.10208938e-05,
         1.48978280e-04,  -2.80845120e-04,   9.45279812e-05,
         2.32747674e-04,   7.48693478e-04,  -4.23575674e-04,
         1.21403072e-03,  -9.38973714e-04,  -1.08126644e-03,
         8.89145152e-04,  -5.68536470e-04,   9.43463325e-04,
         6.27432701e-04,  -1.32464451e-03,   6.65478769e-04,
        -2.05737746e-03,  -1.56089301e-04,  -1.06347020e-03,
        -6.94891165e-04,   9.48374449e-05,   1.39536812e-03,
        -3.60403859e-03,  -3.50248558e-03,  -1.01401067e-03,
        -2.48003843e-04,  -9.09510514e-06,  -7.86874635e-06,
        -4.57701903e-05,  -3.37163988e-04,  -9.26783901e-04,
         4.19261608e-04,  -4.52937060e-06,  -1.20417530e-03,
        -1.31343147e-04,   5.39800754e-04,   4.40941300e-04,
         1.65212585e-03,   5.51888801e-04,   2.39361739e-04,
        -4.83922972e-04,   4.27062773e-04,  -4.91714011e-04,
        -6.60053694e-04,   3.62753424e-04,  -1.00999406e-03,
         2.90191412e-04,   3.18080265e-04,  -4.39215231e-04,
         4.17528769e-05,  -1.24930111e-03,  -4.57834044e-03,
        -1.98170962e-03,  -2.45505105e-04,  -7.12471963e-06,
        -2.30175021e-06,  -1.05008060e-04,  -3.21781310e-04,
        -7.65008967e-04,  -5.23275104e-05,   1.10420292e-03,
        -1.28983083e-03,   2.02212534e-04,  -1.54347359e-03,
         7.72615762e-05,   4.24824943e-04,   2.01269842e-04,
         4.28631134e-04,  -9.61346641e-05,   2.63152771e-04,
        -3.25411587e-04,   2.52984043e-04,  -3.49549957e-04,
        -2.90413730e-04,  -8.81688355e-05,  -8.94875535e-04,
        -9.96552503e-04,   2.19907327e-04,  -8.15675413e-04,
        -4.69702992e-03,  -1.69069820e-03,  -1.66152987e-04,
        -1.15722801e-05,  -4.09713947e-06,  -8.76677363e-05,
        -3.87380384e-04,  -7.89799856e-04,  -8.35978190e-04,
         6.40731826e-04,   2.14355638e-03,  -5.66892773e-04,
        -2.29907008e-05,   9.47117397e-04,   4.92813271e-04,
         1.01559370e-03,  -5.26519498e-05,  -8.43010725e-04,
        -2.83826807e-04,  -1.06495353e-03,  -1.04857788e-03,
         8.90639982e-04,   1.02359727e-03,  -8.71891836e-04,
        -3.00996471e-04,  -5.16103317e-04,  -1.79355303e-04,
        -1.01900801e-03,  -2.37578641e-03,  -9.01647609e-04,
         1.20376112e-05,  -6.67081313e-06,  -5.89307846e-06,
        -2.39229862e-05,  -1.89259121e-04,  -1.38331299e-03,
        -1.29997417e-03,   3.36923682e-04,  -6.60241766e-04,
        -2.04085370e-03,  -2.69181281e-03,  -3.87171146e-03,
        -5.01698620e-03,  -4.22249355e-03,  -5.04325441e-03,
        -3.99925662e-03,  -3.20129500e-03,  -1.42705457e-03,
        -4.12477540e-04,  -1.28147299e-03,  -2.40082117e-04,
         5.56938954e-04,  -9.93774014e-04,  -2.32668915e-04,
        -6.67626964e-04,  -1.74216500e-03,  -7.47309680e-04,
         6.26692628e-04,   1.02678702e-04,  -8.11227346e-05,
        -4.52372252e-06,  -1.12454207e-05,  -7.42712392e-05,
        -2.06758278e-03,  -2.00747898e-03,  -2.35827320e-03,
        -6.49626684e-03,  -6.80864453e-03,  -4.09972266e-03,
        -4.61637601e-03,  -3.24598334e-03,  -4.21994035e-03,
        -4.03141661e-03,  -1.84555256e-03,  -5.07405818e-03,
        -2.99353573e-03,  -7.74228617e-04,  -2.21957541e-03,
        -9.33679391e-04,   1.35900181e-03,   6.22448387e-05,
         2.75852930e-04,  -2.23478103e-03,  -4.81155747e-03,
         2.11162788e-04,   2.64435476e-03,   1.41432847e-03,
         6.46455117e-05,  -7.84389065e-07,  -1.43319214e-05,
        -3.13186905e-05,  -1.75755038e-03,  -4.06184088e-03,
        -5.57670937e-03,  -5.57103674e-03,  -4.88540549e-03,
        -2.20750671e-03,  -1.25062644e-03,  -8.53348272e-04,
        -4.31085080e-04,   1.84890048e-04,  -1.38310140e-04,
        -1.75254830e-03,  -1.46079945e-03,  -1.41796410e-03,
        -5.39617858e-04,  -5.96171875e-04,  -1.33240195e-03,
        -9.89199680e-04,  -4.36983061e-04,  -1.38930702e-03,
        -2.24239419e-03,   8.37327440e-04,   3.63365315e-03,
         2.45815567e-03,   8.84038343e-04,  -1.92573291e-09,
        -1.29444217e-04,  -1.12406309e-04,  -1.46357174e-03,
        -2.44156663e-03,  -4.10456790e-03,  -2.47958896e-03,
         2.37230240e-05,  -9.37950321e-04,  -1.04405715e-03,
        -1.03715680e-03,  -3.23526772e-04,   6.82889148e-04,
        -9.46146270e-04,  -6.26581121e-04,  -1.17471390e-03,
         1.40875610e-04,  -9.47924024e-04,   2.19453704e-04,
        -1.54614612e-03,  -3.75302884e-04,  -1.73266474e-03,
        -1.52587814e-03,  -1.00767249e-03,  -2.90072674e-04,
         2.28259559e-03,   1.91831664e-03,   6.40712674e-05,
        -5.26340702e-09,  -2.77089056e-04,  -3.94041913e-04,
        -3.02553899e-04,   9.19266555e-05,   1.94501133e-03,
         1.08773237e-03,  -2.53977695e-04,  -4.82409847e-04,
         1.36865383e-04,   4.26311784e-04,   6.48988140e-04,
         7.16369338e-04,   1.70754312e-04,  -8.65457171e-04,
        -6.79956335e-05,   8.98838675e-04,  -1.59637027e-03,
        -1.31295409e-03,   4.36417856e-04,  -1.20922304e-03,
        -1.38196636e-03,  -1.63693737e-03,  -2.56177850e-03,
         2.70391206e-04,   2.17901405e-03,   2.75967096e-03,
         3.88604583e-04,  -2.64734703e-08,  -1.43543270e-04,
        -8.22925066e-04,   7.74337653e-04,   1.28957542e-03,
         4.49604640e-04,   1.80909612e-03,   6.92921907e-04,
        -7.60633307e-04,   1.15070005e-03,   6.48344012e-04,
         2.82531997e-04,   1.42364840e-03,   1.03822392e-03,
        -1.36161018e-03,   3.96901247e-04,   9.77786680e-05,
         1.86797986e-03,  -9.54763770e-04,   3.31425172e-05,
        -1.25061677e-03,   1.69389983e-03,   8.11098365e-04,
         7.57135147e-04,   8.27599853e-04,   4.00123544e-03,
         3.67300379e-03,   4.06183941e-04,   0.00000000e+00,
        -5.18504483e-05,  -7.55792982e-04,   6.06457974e-04,
         1.44309305e-03,   1.21688436e-03,   1.86291313e-03,
         5.29267691e-04,   1.18511560e-03,   7.42782567e-04,
         3.80171010e-04,   5.74637678e-04,   1.63351249e-03,
         9.36243290e-04,   4.19589031e-04,   1.80308653e-03,
         1.78369486e-05,  -7.59729512e-04,   1.11694295e-04,
        -1.15933744e-03,  -2.19391934e-04,  -3.94377169e-04,
         6.48903469e-04,   1.11994938e-03,   1.57823458e-03,
         4.96842039e-03,   2.11480648e-03,   1.66971483e-04,
        -5.09629942e-07,  -4.49966560e-06,  -1.16269130e-03,
        -1.57179629e-03,   1.10012532e-03,   2.59296833e-03,
         1.78010626e-03,   1.96310729e-03,   1.03284498e-03,
         1.24077609e-04,   1.26313216e-03,  -9.25093229e-04,
         1.97749518e-03,   1.48692840e-04,   1.36694452e-03,
         3.19586159e-04,   7.04213094e-04,   1.07030063e-03,
        -8.18746357e-05,   4.13210895e-04,  -1.03027904e-05,
         3.11596929e-04,   8.58228173e-04,   1.66737158e-03,
         3.76212883e-03,   3.74642407e-03,   6.26321947e-04,
         1.14304125e-04,  -1.49408358e-06,  -1.47583486e-05,
        -1.11316787e-03,  -2.11565205e-03,   5.61690804e-04,
        -2.82075732e-04,   5.70829255e-04,   1.28249871e-03,
         4.67708244e-04,   1.63703387e-03,   1.38791335e-03,
         1.19555467e-03,   5.12667319e-04,   1.51437282e-03,
         8.37303591e-04,   7.73822587e-04,   6.51291750e-04,
         4.67252369e-04,   8.68170718e-04,  -1.32183871e-03,
         7.52948225e-04,   1.12048230e-03,   8.48207380e-04,
         7.56901949e-04,   2.18632241e-03,   2.96763831e-03,
         7.38167633e-04,   1.69161747e-05,   0.00000000e+00,
        -1.25426161e-05,  -2.89361343e-04,  -9.73600309e-04,
         1.40672260e-03,   1.61940452e-03,   1.56936387e-03,
         2.97120373e-03,   1.54059571e-03,   1.28762332e-03,
         2.62809549e-04,   1.78581805e-03,  -5.76707707e-04,
        -8.20176617e-04,  -9.48359322e-04,  -2.00692516e-04,
         3.53610501e-04,  -1.75468428e-03,   8.02561709e-04,
         1.41976462e-03,   1.20933994e-03,   1.52390856e-03,
         1.73984988e-04,   2.52036146e-03,   3.95568689e-03,
         1.64960183e-03,   2.71784138e-04,  -8.35357991e-06,
        -4.67922155e-08,  -1.35317801e-06,  -1.94295375e-04,
        -4.30835276e-04,   1.08625759e-04,   1.25058534e-04,
         2.93283096e-04,   2.05996961e-04,  -1.37926699e-03,
         1.86372438e-03,   8.10633239e-05,   5.08792942e-04,
        -5.31693902e-04,  -1.01666232e-05,  -7.10165043e-04,
        -2.04910341e-03,   5.37532668e-04,   7.64872798e-04,
         5.42523784e-05,   1.85316647e-03,   4.69246751e-05,
         1.14452899e-03,   2.32947041e-03,   3.82485140e-03,
         1.65583386e-03,   1.09890401e-03,   1.01236196e-05,
        -1.54100563e-07,  -4.53299588e-08,  -1.03580175e-07,
        -3.01978475e-04,   1.14642678e-04,   1.00505475e-03,
         1.14835901e-04,   1.00674084e-03,   9.16055288e-04,
         8.36062379e-04,   1.22828763e-03,  -3.34981620e-04,
         9.76785718e-04,   2.24255675e-04,   2.05968336e-04,
        -8.60160413e-04,  -1.40565177e-03,  -5.47819467e-04,
        -4.58247262e-04,   1.33599702e-03,   6.15239630e-05,
         9.31285901e-04,   1.32273375e-03,   2.71768599e-03,
         1.86404635e-03,   1.27025294e-03,   4.72753533e-04,
        -2.30230337e-04,  -2.84493347e-07,   0.00000000e+00,
         0.00000000e+00,  -5.21002628e-05,  -4.99973813e-04,
        -2.70808648e-04,  -5.13971818e-04,  -6.34215300e-05,
        -6.70208438e-04,  -1.55038140e-03,  -5.63242927e-04,
        -9.73123673e-04,  -9.36824648e-04,  -5.10890718e-04,
        -2.07968750e-03,  -1.02380673e-03,  -1.36748448e-03,
        -8.64091319e-04,  -1.13719656e-03,   1.10302443e-03,
         1.49618440e-03,   8.83319128e-04,   2.08845015e-03,
         2.15596696e-03,   1.51406021e-03,   1.24345963e-03,
         6.89060487e-04,  -1.11075759e-04,   0.00000000e+00,
         0.00000000e+00,   0.00000000e+00,  -4.75688609e-05,
        -1.00560794e-03,  -1.82405506e-03,  -2.81025677e-03,
        -3.04859641e-03,  -2.59892601e-03,  -1.20559705e-03,
        -1.39922033e-03,  -1.79129133e-03,  -1.09478575e-03,
        -1.02277249e-03,  -4.42472248e-04,   1.05010289e-03,
        -8.53189950e-04,  -1.60995505e-03,  -7.87430126e-04,
         7.21081046e-06,  -4.28952881e-04,  -4.24791052e-04,
        -6.42580590e-05,   2.58480005e-04,   3.81113963e-04,
         1.50305550e-03,   9.38913122e-04,   2.87393513e-05,
         0.00000000e+00,   0.00000000e+00,   0.00000000e+00,
        -2.07745890e-06,  -2.96933605e-04,  -7.90985243e-04,
        -1.67175277e-03,  -2.16102096e-03,  -3.57439270e-03,
        -4.58904080e-03,  -5.34546574e-03,  -4.87383547e-03,
        -5.07472457e-03,  -4.84642852e-03,  -4.33330117e-03,
        -3.03036613e-03,  -2.35868297e-03,  -2.34807539e-03,
        -2.19181719e-03,  -2.94910447e-03,  -2.60357172e-03,
        -2.02431529e-03,  -2.25828139e-03,  -1.38553023e-03,
        -7.28500130e-04,  -2.28777811e-04,   2.55298618e-05,
         3.06928357e-05,   0.00000000e+00,   0.00000000e+00,
         0.00000000e+00,   0.00000000e+00,  -9.17968140e-08,
        -5.57273267e-05,  -2.76343990e-04,  -6.21071634e-04,
        -8.85173395e-04,  -1.03575593e-03,  -1.18175152e-03,
        -1.47210231e-03,  -2.01362348e-03,  -1.79642339e-03,
        -1.91771235e-03,  -1.97853633e-03,  -1.91977202e-03,
        -1.86630213e-03,  -1.52612435e-03,  -1.11819839e-03,
        -5.11656952e-04,  -4.92508669e-04,  -5.59542257e-04,
        -1.89741238e-04,  -3.10882991e-06,  -4.86974817e-08,
        -7.74364994e-07,   0.00000000e+00,   0.00000000e+00,
         0.00000000e+00,   0.00000000e+00,   0.00000000e+00,
         0.00000000e+00,  -1.71921033e-07,  -2.59551949e-06,
        -2.59444740e-05,  -5.58648947e-05,  -4.52057264e-05,
        -6.79606916e-05,  -2.25124768e-04,  -2.29415025e-04,
        -2.15009542e-04,  -2.24270253e-04,  -2.55413839e-04,
        -2.39597491e-04,  -1.95331905e-04,  -1.56292388e-04,
        -5.75300670e-05,  -1.96438420e-05,  -6.80962660e-05,
        -3.08618638e-05,  -1.79533612e-04,  -3.55163987e-08,
         0.00000000e+00,   0.00000000e+00,   0.00000000e+00,
         0.00000000e+00])], 'training 0/1 loss': [1003], '-(log loss), training': [3754.1799274876776], 'lambda': [100000000.0], 'training (0/1 loss)/N': [0.018574074074074073], 'log loss': [-3754.1799274876776], 'eta0': [0.0002], '# nonzero weights': [267]}
...................................................................................................................................................................................................................................Loss optimized.  Old/N: 0.06852264795867914, new/N:0.06852127964570084. Eta: 2.2075539284417398e-10



In [101]:

    
100000000 - 1e8









    Out[101]:





0.0



In [111]:

    
final_training_results = hyper_explorer.final_model.results_row()
del final_training_results['weights']
final_training_results









    Out[111]:





{'# nonzero weights': [252],
 '-(log loss), training': [4111.2767787420507],
 '-(log loss)/N, training': [0.068521279645700844],
 'eta': [2.2075539284417398e-10],
 'eta0': [0.0002],
 'iteration': [233],
 'lambda': [100000000.0],
 'lambda normalized': [180.1107372127079],
 'log loss': [-4111.2767787420507],
 'training (0/1 loss)/N': [0.018616666666666667],
 'training 0/1 loss': [1117]}



In [112]:

    
hyper_explorer.final_model.plot_test_and_train_01_loss_during_fitting()









    



---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-112-a13c1b53a929> in <module>()
----> 1 hyper_explorer.final_model.plot_test_and_train_01_loss_during_fitting()

/home/jmatsen/Machine_Learning_CSE_546/HW2/code/logistic_regression.py in plot_test_and_train_01_loss_during_fitting(self, filename, colors)
    202         fig = super(LogisticRegression, self).plot_ys(
    203             x='iteration', y1=train_y, y2=test_y, label="normalized 0/1 loss",
--> 204             logx=False, colors=colors)
    205         if filename is not None:
    206             fig.savefig(filename + '.pdf')

TypeError: plot_ys() got an unexpected keyword argument 'label'



In [127]:

    
fig, ax = plt.subplots(1, 1, figsize=(4, 3))
plot_data = hyper_explorer.final_model.results
print(plot_data.columns)
plt.plot(plot_data['iteration'], plot_data['-(log loss)/N, training'], linestyle='--', marker='o', color='c')
plt.plot(plot_data['iteration'], plot_data['-(log loss)/N, testing'], linestyle='--', marker='o', color='g')
plt.legend(loc='best')
plt.xlabel('iteration')
plt.ylabel('-(log loss)/N')
ax.axhline(y=0, color='k')
plt.tight_layout()
fig.savefig("161031_Q-2-1_norm_log_loss_during_fit.pdf")









    



Index(['# nonzero weights', '-(log loss), training', '-(log loss)/N, training',
       'eta', 'eta0', 'iteration', 'lambda', 'lambda normalized', 'log loss',
       'log loss percent change', 'training (0/1 loss)/N', 'training 0/1 loss',
       'weights', '-(log loss), testing', '-(log loss)/N, testing',
       'testing (0/1 loss)/N', 'testing 0/1 loss'],
      dtype='object')



In [129]:

    
fig, ax = plt.subplots(1, 1, figsize=(4, 3))
plot_data = hyper_explorer.final_model.results
plt.plot(plot_data['iteration'], plot_data['training (0/1 loss)/N'], linestyle='--', marker='o', color='c')
plt.plot(plot_data['iteration'], plot_data['testing (0/1 loss)/N'], linestyle='--', marker='o', color='g')
plt.legend(loc='best')
plt.xlabel('iteration')
plt.ylabel('(0/1 loss)/N')
ax.axhline(y=0, color='k')
plt.tight_layout()
fig.savefig("161031_Q-2-1_norm_01_loss_during_fit.pdf")



In [124]:

    
hyper_explorer.final_model.plot_ys(x='iteration', y1='-(log loss)/N, training', y2='-(log loss)/N, testing')









    Out[124]:



In [125]:

    
hyper_explorer.final_model.plot_log_loss_normalized_and_eta()



In [126]:

    
hyper_explorer.evaluate_test_data()









    



                                                                     0
# nonzero weights                                                  252
-(log loss), test                                               768.45
-(log loss)/N, test                                           0.076845
eta                                                        2.20755e-10
eta0                                                            0.0002
iteration                                                          233
lambda                                                           1e+08
lambda normalized                                              180.111
log loss                                                       -768.45
test (0/1 loss)/N                                               0.0187
test 0/1 loss                                                      187
weights              [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...



In [132]:

    
# For fun, train one model starting at zero.
naive_model = hyper_explorer.final_model.copy()
naive_model.w = naive_model.w.copy()  # just to be safe



In [134]:

    
# `b.base is a`
naive_model.w.base is hyper_explorer.final_model.w









    Out[134]:





False



In [137]:

    
naive_model.w[:]=0



In [138]:

    
naive_model.w









    Out[138]:





array([ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,
        0.,  0.,  0.,  0.])



In [139]:

    
naive_model.lam









    Out[139]:





100000000.0



In [143]:

    
naive_model.max_iter = 1000



In [145]:

    
naive_model.run()









    



............................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................Loss optimized.  Old/N: 0.07290634462084887, new/N:0.07290488697638506. Eta: 1.4331119346411592e-10



In [150]:

    
naive_model.X.shape









    Out[150]:





(60000, 784)



In [149]:

    
naive_model.results_row()









    Out[149]:





{'# nonzero weights': [133],
 '-(log loss), training': [4374.2932185831032],
 '-(log loss)/N, training': [0.072904886976385058],
 'eta': [1.4331119346411592e-10],
 'eta0': [0.0002],
 'iteration': [1773],
 'lambda': [100000000.0],
 'lambda normalized': [180.1107372127079],
 'log loss': [-4374.2932185831032],
 'training (0/1 loss)/N': [0.019883333333333333],
 'training 0/1 loss': [1193],
 'weights': [array([  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,
           0.00000000e+00,   0.00000000e+00,   0.00000000e+00,
           0.00000000e+00,   0.00000000e+00,   0.00000000e+00,
           0.00000000e+00,   0.00000000e+00,   0.00000000e+00,
          -1.23384174e-06,  -9.55809295e-07,   1.94282890e-06,
           8.09512040e-08,   0.00000000e+00,   0.00000000e+00,
           0.00000000e+00,   0.00000000e+00,   0.00000000e+00,
           0.00000000e+00,   0.00000000e+00,   0.00000000e+00,
           0.00000000e+00,   0.00000000e+00,   0.00000000e+00,
           0.00000000e+00,   0.00000000e+00,   0.00000000e+00,
           0.00000000e+00,   0.00000000e+00,  -3.55539652e-08,
          -1.22036530e-07,  -3.56393302e-06,  -1.15968858e-05,
          -3.36689581e-05,  -7.40219026e-05,  -1.15547330e-04,
          -1.04950288e-04,  -1.61850653e-04,  -1.65105401e-04,
          -2.20927908e-04,  -2.17476695e-04,  -1.18069482e-04,
          -2.07756405e-04,  -1.31017414e-04,  -1.28999068e-04,
          -9.03537506e-05,  -2.62364883e-05,  -4.36532057e-06,
          -1.31586384e-06,   0.00000000e+00,   0.00000000e+00,
           0.00000000e+00,   0.00000000e+00,   0.00000000e+00,
           0.00000000e+00,  -7.65527324e-08,  -7.25902037e-08,
          -3.91769031e-06,  -1.38573539e-06,  -1.11211153e-05,
          -4.54853493e-05,  -1.31906123e-04,  -1.73500863e-04,
          -3.88387251e-04,  -5.50240387e-04,  -7.35734692e-04,
          -1.04991787e-03,  -1.39222583e-03,  -1.33400633e-03,
          -1.35358401e-03,  -1.50156644e-03,  -1.47082092e-03,
          -1.11627527e-03,  -6.40130378e-04,  -3.66422522e-04,
          -2.41522665e-04,  -7.77302645e-05,  -9.91283495e-06,
           2.73821296e-06,   0.00000000e+00,   0.00000000e+00,
           0.00000000e+00,   0.00000000e+00,  -2.30814848e-07,
          -4.43175408e-07,  -2.86663709e-06,  -4.00605917e-06,
           5.10280228e-06,   8.85172138e-05,   2.42275275e-04,
           6.92845377e-04,   6.59008280e-04,   3.68755192e-04,
           1.58175427e-04,   2.13739565e-04,  -3.84859100e-04,
          -8.60309489e-04,  -7.34256225e-04,  -1.25858530e-03,
          -1.73671362e-03,  -1.44120616e-03,  -1.16079945e-03,
          -9.02408777e-04,  -5.62919531e-04,  -3.17001965e-04,
          -9.16526078e-05,  -6.27141457e-06,  -5.27306775e-07,
           0.00000000e+00,   0.00000000e+00,  -1.82660820e-07,
          -2.66411306e-06,  -1.71171212e-05,  -7.91504779e-05,
          -3.65111096e-05,   9.33041946e-05,   2.07950423e-04,
           3.58313300e-04,   7.86050992e-04,   5.09139286e-04,
           1.24753574e-04,   2.50159449e-04,   1.70768348e-04,
           2.39185661e-04,  -2.81013798e-04,  -1.42026761e-06,
          -9.69503192e-04,  -1.02196891e-03,  -1.37768915e-03,
          -1.17148906e-03,  -1.64583705e-03,  -1.49672828e-03,
          -1.00311019e-03,  -5.16847314e-04,  -1.09651112e-04,
          -1.01085700e-05,  -7.29910555e-06,   0.00000000e+00,
           0.00000000e+00,  -1.10612683e-06,  -1.24748105e-05,
          -1.13449049e-05,  -1.18971791e-04,  -9.79000696e-05,
           1.90259157e-04,   4.90885382e-04,   2.82408656e-04,
           5.61542982e-04,   8.96682162e-04,   7.45253613e-04,
           8.65164327e-04,   6.36055848e-04,   9.12867142e-04,
           1.23094560e-03,   1.01084440e-03,   8.32014340e-04,
          -3.19477648e-04,  -3.69880068e-04,  -6.79396421e-04,
          -1.10405305e-03,  -1.48136817e-03,  -9.62373252e-04,
          -2.15028520e-04,  -3.89332833e-05,  -1.08955338e-05,
           0.00000000e+00,  -3.69959435e-08,  -2.63780523e-05,
          -8.04459523e-05,  -9.42236896e-05,  -9.79573744e-05,
          -9.74444385e-06,   1.50821965e-04,   3.31294812e-04,
           3.66531116e-04,   3.20040389e-04,   2.42833135e-04,
           3.99359979e-04,   5.19160902e-04,   2.87085694e-04,
           4.52787005e-04,   5.24408742e-04,   1.57640815e-04,
          -1.33078198e-04,  -1.52958721e-04,  -4.80089810e-04,
          -2.20156114e-04,  -8.57331092e-04,  -1.70153982e-03,
          -1.09114250e-03,  -3.24819980e-04,  -4.83519170e-05,
          -3.48092161e-06,  -1.26522820e-07,  -9.57489861e-06,
           2.49800007e-05,  -1.10123411e-04,  -7.37374970e-05,
          -1.69949874e-05,   1.85586984e-04,  -7.08881634e-05,
           3.60084051e-04,  -1.57694056e-04,  -3.04014169e-04,
           1.60398982e-04,  -1.09070770e-04,   4.07860912e-04,
           2.71955246e-04,  -4.21369831e-04,  -1.83655592e-04,
          -1.05515139e-03,  -5.18830150e-04,  -6.62001346e-04,
          -3.44430224e-04,   1.95388148e-04,   4.06014506e-05,
          -1.64509428e-03,  -1.35080799e-03,  -4.54928758e-04,
          -7.34993822e-05,  -3.66577473e-06,  -9.66488590e-07,
          -2.13793009e-05,  -9.66168036e-05,  -2.85159513e-04,
          -2.49273933e-05,   6.57364053e-05,  -9.86443979e-05,
           8.65150637e-05,   2.86734168e-05,  -3.28379926e-05,
           7.00062277e-04,   2.97303241e-04,  -1.97775817e-04,
          -1.65489737e-04,  -2.53706918e-05,  -4.76661936e-04,
          -4.66062558e-04,  -3.39297290e-04,  -8.54380528e-04,
          -2.23279427e-04,  -2.12650133e-04,  -1.86003388e-04,
          -1.50657828e-04,  -1.03434731e-03,  -1.67318227e-03,
          -6.93033224e-04,  -7.02631010e-05,  -2.94541586e-06,
          -5.85714152e-07,  -3.00405777e-05,  -1.56146120e-04,
          -2.80795138e-04,   2.98751850e-05,   6.27212793e-04,
          -3.64985830e-05,  -5.79980330e-05,  -3.35140920e-04,
           3.60074350e-04,   6.35177424e-04,   5.09772290e-04,
           1.50923661e-04,   1.05563960e-04,   1.26158057e-04,
          -5.21748373e-05,   9.47885068e-05,  -1.51885402e-04,
          -1.47027191e-04,  -2.57636878e-04,  -6.68925498e-04,
          -7.30018433e-04,  -4.07459868e-04,  -9.07143372e-04,
          -1.53029820e-03,  -5.44753719e-04,  -5.78990541e-05,
          -5.82243060e-06,  -8.95028914e-07,  -2.18636483e-05,
          -1.14867112e-04,  -2.04622376e-04,  -2.09283188e-04,
           3.55452954e-04,   4.55353050e-04,  -3.72186481e-04,
          -4.06130439e-05,   1.61031115e-04,  -4.35715085e-05,
           7.54866313e-07,  -5.77430637e-04,  -7.60627677e-04,
          -5.70226570e-04,  -8.75774698e-04,  -4.96292787e-04,
           3.77915755e-04,   5.08205343e-04,  -4.14830042e-04,
          -2.85763171e-04,  -4.18353524e-04,  -5.20705941e-04,
          -8.17966782e-04,  -7.73839248e-04,  -2.81073649e-04,
          -2.01770768e-05,  -3.00227083e-06,  -9.40052591e-07,
          -1.09755426e-05,  -6.05315264e-05,  -3.70317561e-04,
          -4.50287715e-04,  -4.69358633e-04,  -1.29649158e-03,
          -2.13647795e-03,  -2.32594540e-03,  -2.53608722e-03,
          -3.05833852e-03,  -2.96673275e-03,  -3.31184385e-03,
          -2.90019833e-03,  -2.69992133e-03,  -1.73125836e-03,
          -5.37847247e-04,  -6.36859574e-04,  -2.42613201e-04,
           9.09000947e-05,  -2.64622076e-04,  -1.16275696e-04,
          -8.17187323e-04,  -1.13716537e-03,  -4.15213161e-04,
           1.33022202e-04,   1.80065099e-05,  -6.81619543e-06,
          -6.43845242e-07,  -4.91219277e-06,  -2.87402784e-05,
          -5.37171538e-04,  -8.47592248e-04,  -2.09928198e-03,
          -3.81315848e-03,  -4.21957556e-03,  -3.32137787e-03,
          -3.42899436e-03,  -3.14893630e-03,  -3.45129396e-03,
          -3.04145520e-03,  -2.25870802e-03,  -3.30185108e-03,
          -1.99499875e-03,  -9.60886785e-04,  -1.25203732e-03,
          -5.50234209e-04,   4.86167885e-04,  -3.75242443e-05,
          -6.42122608e-05,  -1.46894233e-03,  -2.07652191e-03,
          -1.90960613e-04,   6.77713452e-04,   3.18601251e-04,
           2.25184129e-05,  -1.54610230e-07,  -3.31733665e-06,
          -1.33609895e-05,  -4.77852329e-04,  -1.49779847e-03,
          -3.34112211e-03,  -3.88551583e-03,  -3.29679754e-03,
          -2.32602808e-03,  -1.62679394e-03,  -1.08348841e-03,
          -6.23314596e-04,  -1.01784705e-05,  -4.34313356e-04,
          -1.52309704e-03,  -1.26128955e-03,  -7.68860464e-04,
          -5.55297766e-04,  -6.36107673e-04,  -5.43251181e-04,
          -3.53194041e-04,  -5.83776530e-04,  -1.62348762e-03,
          -1.68456663e-03,   7.20951114e-05,   1.09330595e-03,
           6.28271423e-04,   1.81871331e-04,  -1.36926304e-07,
          -3.35205087e-05,  -2.99004466e-05,  -4.19465993e-04,
          -1.14231296e-03,  -2.80895949e-03,  -2.06622283e-03,
          -6.62328461e-04,  -4.60483943e-04,  -3.49573520e-04,
          -3.48884124e-04,   2.19431105e-04,   8.22423929e-04,
          -3.34750858e-04,  -7.21402977e-04,  -8.62089007e-04,
           7.58077566e-05,  -7.05573871e-04,  -6.61229301e-04,
          -8.34209590e-04,  -3.50086347e-04,  -1.10232425e-03,
          -1.47722439e-03,  -1.28161252e-03,  -1.26321247e-05,
           1.13575328e-03,   6.38072410e-04,   1.94212285e-05,
          -5.30863757e-08,  -6.75784910e-05,  -1.01956705e-04,
          -1.10171099e-04,  -1.82070428e-04,  -1.53616463e-04,
           3.23245148e-04,   2.84995530e-04,  -4.89093883e-05,
           9.59227625e-05,   7.19465663e-05,   2.90355388e-04,
           3.14556495e-04,  -1.46733587e-04,  -8.63540987e-04,
          -1.52962664e-04,   4.73302686e-04,  -7.79030840e-04,
          -8.63415580e-04,  -3.43430233e-04,  -9.42509889e-04,
          -8.50719619e-04,  -1.26698867e-03,  -1.29007048e-03,
           3.92194444e-04,   1.52215016e-03,   8.72317497e-04,
           8.91711849e-05,  -4.71548512e-08,  -3.36678421e-05,
          -1.84888751e-04,   2.31023453e-04,   6.53663327e-04,
           7.96100742e-04,   1.35994326e-03,   5.01781476e-04,
          -1.87264251e-04,   3.39047533e-04,   3.90020432e-04,
           5.48028136e-04,   1.02624007e-03,   4.84731717e-04,
          -7.27224014e-04,   3.15238670e-04,   4.63078309e-04,
           3.69720805e-04,  -1.06535585e-04,  -1.81755732e-04,
          -4.93208149e-04,   5.76894195e-04,   1.80176096e-04,
           6.18844306e-06,   9.01882868e-04,   2.16907441e-03,
           1.04000627e-03,   9.66365892e-05,   0.00000000e+00,
          -1.30485688e-05,  -1.79555507e-04,   2.36864336e-04,
           1.19847705e-03,   1.40381571e-03,   1.36557052e-03,
           5.88768043e-04,   7.48010486e-04,   5.67021089e-04,
           3.69420186e-04,   6.67943035e-04,   1.16255438e-03,
           4.82097542e-04,   2.04952336e-04,   8.78319047e-04,
           1.92952593e-04,  -2.09643208e-04,  -1.16802308e-05,
          -6.12522373e-04,  -1.95425042e-04,   2.40988536e-04,
           5.57373620e-04,   8.81927995e-04,   1.91582054e-03,
           2.35513582e-03,   6.21177920e-04,   3.29080167e-05,
          -2.04341349e-07,  -1.21112102e-06,  -3.39126315e-04,
          -2.85611281e-04,   1.05630952e-03,   1.64437996e-03,
           1.17974128e-03,   1.35180826e-03,   9.05338234e-04,
           7.43612980e-04,   7.26508604e-04,   6.97460517e-05,
           9.53255075e-04,   6.98349427e-04,   9.32032171e-04,
           4.09733059e-04,   5.35778173e-04,   5.29951016e-04,
          -1.24606070e-05,  -1.70529836e-04,  -1.26963607e-04,
           2.46140959e-04,   7.56308284e-04,   1.71712875e-03,
           2.65790746e-03,   1.67373482e-03,   1.76611668e-04,
           2.08946873e-05,  -3.08880710e-07,  -3.91143822e-06,
          -3.70638847e-04,  -6.29047886e-04,   6.84575541e-04,
           7.15493095e-04,   6.16554968e-04,   1.07760629e-03,
           7.32126240e-04,   1.00504368e-03,   1.14751400e-03,
           8.59663672e-04,   6.44229476e-04,   8.03059320e-04,
           5.85578904e-04,   6.33420237e-04,   5.41226865e-04,
           1.92424630e-04,   2.72783911e-04,  -1.36969982e-04,
           4.17135089e-04,   4.73472304e-04,   6.48165089e-04,
           1.40202369e-03,   1.79573154e-03,   1.04113362e-03,
           1.87845804e-04,   4.44988290e-06,   0.00000000e+00,
          -3.98737684e-06,  -1.59002018e-04,  -2.43084078e-04,
           7.83927965e-04,   7.55674582e-04,   9.53433601e-04,
           1.61575167e-03,   1.00786521e-03,   9.55649876e-04,
           7.88780810e-04,   7.88273079e-04,  -2.51785584e-04,
          -6.17764773e-04,  -4.12823305e-04,  -3.68538833e-04,
          -6.57321101e-05,  -5.98457021e-04,   4.87382594e-04,
           9.03571522e-04,   6.42021173e-04,   5.73229696e-04,
           6.77725406e-04,   1.78695880e-03,   1.71334182e-03,
           6.53629640e-04,   9.30504789e-05,  -1.67173112e-06,
          -1.16284176e-07,  -8.11819950e-07,  -9.35787521e-05,
          -4.69190182e-05,   3.36814704e-04,   4.06304108e-04,
           4.86719071e-04,   3.09747440e-04,   1.27641952e-04,
           9.08088186e-04,   2.45597053e-04,   2.25511521e-04,
          -2.21468956e-04,  -3.47552523e-04,  -7.00339560e-04,
          -9.62581839e-04,   1.47920921e-05,   4.63399914e-04,
           5.62795000e-04,   1.03531263e-03,   3.60661897e-04,
           8.50624473e-04,   1.79344046e-03,   2.21033358e-03,
           1.08830538e-03,   4.38727476e-04,   2.80622123e-05,
          -5.29127481e-08,  -1.12650296e-07,  -1.21550880e-07,
          -7.28236576e-05,   6.47414074e-05,   5.03216847e-04,
           1.72473750e-04,   6.73654140e-04,   5.55354651e-04,
           6.48841492e-04,   7.86989436e-04,   6.98964217e-05,
           6.30425056e-04,   3.06165120e-04,   1.46701057e-06,
          -6.67015907e-04,  -9.82668388e-04,  -5.59926405e-04,
          -1.87410478e-04,   7.37024355e-04,   6.45962796e-04,
           9.85125756e-04,   1.45878933e-03,   1.78943514e-03,
           1.33298883e-03,   6.44588400e-04,   1.92000148e-04,
          -3.64135070e-05,  -9.76850733e-08,   0.00000000e+00,
           0.00000000e+00,  -9.70511736e-06,  -1.14290926e-04,
          -3.39984357e-05,  -2.86656119e-04,  -3.25147384e-04,
          -6.40545643e-04,  -9.03000189e-04,  -6.19247658e-04,
          -8.48148121e-04,  -6.73804980e-04,  -8.08943441e-04,
          -1.33033217e-03,  -1.30040286e-03,  -1.26260099e-03,
          -9.03318425e-04,  -5.73452691e-04,   5.13705738e-04,
           8.22336985e-04,   6.58020511e-04,   1.01524494e-03,
           9.14854936e-04,   6.29589985e-04,   3.93796260e-04,
           1.72598690e-04,  -1.83429753e-05,   0.00000000e+00,
           0.00000000e+00,   0.00000000e+00,  -1.55990902e-05,
          -2.39168619e-04,  -4.66790641e-04,  -9.02372982e-04,
          -1.36410070e-03,  -1.64342506e-03,  -1.47568524e-03,
          -1.45148935e-03,  -1.62155492e-03,  -1.36228463e-03,
          -1.04958006e-03,  -7.59994488e-04,  -3.40595361e-04,
          -7.31696289e-04,  -9.46143900e-04,  -6.53614507e-04,
          -3.33449700e-04,  -4.49561468e-04,  -3.06251495e-04,
          -6.35623169e-05,   6.98028549e-05,   1.21931195e-04,
           3.68144469e-04,   1.93046407e-04,   8.81391148e-06,
           0.00000000e+00,   0.00000000e+00,   0.00000000e+00,
          -3.52127062e-07,  -7.34434489e-05,  -2.29241150e-04,
          -4.68716137e-04,  -8.04610603e-04,  -1.45430010e-03,
          -1.91171707e-03,  -2.19561023e-03,  -2.22538945e-03,
          -2.30246585e-03,  -2.17931686e-03,  -1.93985717e-03,
          -1.48707613e-03,  -1.22720042e-03,  -1.21449299e-03,
          -1.15712252e-03,  -1.19045357e-03,  -9.94929222e-04,
          -6.27219298e-04,  -5.77055567e-04,  -3.58073648e-04,
          -1.58283817e-04,  -4.03941948e-05,   7.70391186e-06,
           9.38897583e-06,   0.00000000e+00,   0.00000000e+00,
           0.00000000e+00,   0.00000000e+00,  -5.43742409e-08,
          -1.21259444e-05,  -6.68991769e-05,  -1.81510474e-04,
          -2.94356764e-04,  -3.96455317e-04,  -5.15533789e-04,
          -6.69897462e-04,  -8.39831731e-04,  -8.10125171e-04,
          -8.10632856e-04,  -7.59578124e-04,  -6.95134255e-04,
          -6.82959466e-04,  -5.49480733e-04,  -4.00297530e-04,
          -1.94956582e-04,  -1.46823010e-04,  -1.29481629e-04,
          -4.23234657e-05,  -1.44775652e-06,  -3.81928036e-08,
          -1.49668925e-07,   0.00000000e+00,   0.00000000e+00,
           0.00000000e+00,   0.00000000e+00,   0.00000000e+00,
           0.00000000e+00,  -1.88080173e-07,  -1.26303620e-06,
          -8.09232593e-06,  -1.69653137e-05,  -1.61164167e-05,
          -2.80068578e-05,  -6.81215199e-05,  -7.38608194e-05,
          -7.83250910e-05,  -8.74258846e-05,  -1.02021902e-04,
          -9.09233138e-05,  -7.71024044e-05,  -6.01051703e-05,
          -2.58551664e-05,  -9.67091192e-06,  -1.62935997e-05,
          -7.65980260e-06,  -4.00790962e-05,  -2.81422840e-07,
           0.00000000e+00,   0.00000000e+00,   0.00000000e+00,
           0.00000000e+00])]}



In [151]:

    
fig, ax = plt.subplots(1, 1, figsize=(4, 3))
plot_data = naive_model.results
plt.plot(plot_data['iteration'], plot_data['training (0/1 loss)/N'], linestyle='--', marker='o', color='c')
plt.plot(plot_data['iteration'], plot_data['testing (0/1 loss)/N'], linestyle='--', marker='o', color='g')
plt.legend(loc='best')
plt.xlabel('iteration')
plt.ylabel('(0/1 loss)/N')
ax.axhline(y=0, color='k')
plt.tight_layout()
fig.savefig("161031_Q-2-1_norm_01_loss_during_fit--no_starting_weights.pdf")



In [152]:

    
fig, ax = plt.subplots(1, 1, figsize=(4, 3))
plot_data = naive_model.results
print(plot_data.columns)
plt.plot(plot_data['iteration'], plot_data['-(log loss)/N, training'], linestyle='--', marker='o', color='c')
plt.plot(plot_data['iteration'], plot_data['-(log loss)/N, testing'], linestyle='--', marker='o', color='g')
plt.legend(loc='best')
plt.xlabel('iteration')
plt.ylabel('-(log loss)/N')
ax.axhline(y=0, color='k')
plt.tight_layout()
fig.savefig("161031_Q-2-1_norm_log_loss_during_fit--no_starting_weights.pdf")









    



Index(['# nonzero weights', '-(log loss), training', '-(log loss)/N, training',
       'eta', 'eta0', 'iteration', 'lambda', 'lambda normalized', 'log loss',
       'log loss percent change', 'training (0/1 loss)/N', 'training 0/1 loss',
       'weights', '-(log loss), testing', '-(log loss)/N, testing',
       'testing (0/1 loss)/N', 'testing 0/1 loss'],
      dtype='object')



In [ ]: