notebook.community

Edit and run 





In [3]:



    


%load_ext autoreload
%autoreload 2

import os
import numpy as np
import pandas as pd

from browser import *



    


















    






The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload

















In [4]:



    


exps = [
    'C100_DenseBest',
    'C100_DenseBest_Linear',
    'C100_SparseBest',
    'C100_SparseBest_Linear',
]
paths = [os.path.expanduser("~/nta/results/{}".format(e)) for e in exps]
df = load_many(paths)



    











In [5]:



    


df.shape



    


















    
Out[5]:








(40, 62)

















In [7]:



    


df.head()



    


















    
Out[7]:











  
    

      
      Experiment Name
      test_accuracy
      test_accuracy_max
      epoch_test_accuracy
      noise_accuracy
      noise_accuracy_max
      epoch_noise_accuracy
      mean_accuracy
      mean_accuracy_max
      epoch_mean_accuracy
      ...
      repetitions
      restore_supported
      stop
      sync_function
      test_batch_size
      test_batches_in_epoch
      upload_dir
      use_max_pooling
      weight_decay
      weight_sparsity
    

  
  
    

      0
      0
      0.5866
      0.6081
      56
      0.1426
      0.2249
      51
      0.36460
      0.40110
      51
      ...
      10
      False
      {'stop': 1}
      aws s3 sync `dirname {local_dir}` {remote_dir}...
      128
      100
      s3://lsouza/ray/results
      True
      0.0007
      1.0
    

    

      1
      1
      0.5894
      0.5910
      34
      0.1533
      0.2225
      40
      0.37135
      0.39765
      40
      ...
      10
      False
      {'stop': 1}
      aws s3 sync `dirname {local_dir}` {remote_dir}...
      128
      100
      s3://lsouza/ray/results
      True
      0.0007
      1.0
    

    

      2
      2
      0.5736
      0.6075
      32
      0.1573
      0.2093
      24
      0.36545
      0.39860
      47
      ...
      10
      False
      {'stop': 1}
      aws s3 sync `dirname {local_dir}` {remote_dir}...
      128
      100
      s3://lsouza/ray/results
      True
      0.0007
      1.0
    

    

      3
      3
      0.5413
      0.6024
      39
      0.1015
      0.2072
      12
      0.32140
      0.38710
      30
      ...
      10
      False
      {'stop': 1}
      aws s3 sync `dirname {local_dir}` {remote_dir}...
      128
      100
      s3://lsouza/ray/results
      True
      0.0007
      1.0
    

    

      4
      4
      0.5781
      0.6051
      35
      0.1633
      0.1999
      30
      0.37070
      0.39120
      35
      ...
      10
      False
      {'stop': 1}
      aws s3 sync `dirname {local_dir}` {remote_dir}...
      128
      100
      s3://lsouza/ray/results
      True
      0.0007
      1.0
    

  



5 rows × 62 columns



















In [14]:



    


(df.groupby(['dataset', 'name'])['test_accuracy_max', 'noise_accuracy_max', 
                                 'mean_accuracy_max', 'epochs']
                                 .max().round(4))



    


















    
Out[14]:











  
    

      
      
      test_accuracy_max
      noise_accuracy_max
      mean_accuracy_max
      epochs
    

    

      dataset
      name
      
      
      
      
    

  
  
    

      CIFAR100
      C100_DenseBest
      0.7240
      0.2252
      0.4684
      98
    

    

      C100_DenseBest_Linear
      0.6078
      0.2417
      0.4174
      66
    

    

      C100_SparseBest
      0.6885
      0.2926
      0.4644
      92
    

    

      C100_SparseBest_Linear
      0.6913
      0.2972
      0.4706
      93
    

  




















In [15]:



    


(df.groupby(['dataset', 'name'])['test_accuracy_max', 'noise_accuracy_max', 
                                 'mean_accuracy_max', 'epochs']
                                 .mean().round(4))



    


















    
Out[15]:











  
    

      
      
      test_accuracy_max
      noise_accuracy_max
      mean_accuracy_max
      epochs
    

    

      dataset
      name
      
      
      
      
    

  
  
    

      CIFAR100
      C100_DenseBest
      0.6132
      0.2089
      0.4009
      55.5
    

    

      C100_DenseBest_Linear
      0.5997
      0.2097
      0.3918
      50.6
    

    

      C100_SparseBest
      0.5459
      0.2600
      0.3893
      64.3
    

    

      C100_SparseBest_Linear
      0.5575
      0.2752
      0.3894
      65.6
    

  




















In [16]:



    


def stats(arr):
    mean = np.mean(arr)
    std = np.std(arr)
    return [round(v, 4) for v in [mean-std, mean, mean+std]]

tunable_params = ['linear_percent_on']



    











In [29]:



    


# CIFAR-10 SPARSE
filters = ((df['name']=='C100_SparseBest_Linear'))
         
(df[filters]
    .sort_values('mean_accuracy_max', ascending=False)[tunable_params]
    .head(4)
    .apply(stats))



    


















    
Out[29]:








linear_percent_on    [0.534, 0.6674, 0.8008]
dtype: object

















In [30]:



    


# CIFAR-10 SPARSE
filters = ((df['name']=='C100_SparseBest_Linear'))
         
(df[filters]
    .sort_values('mean_accuracy_max', ascending=False)[tunable_params]
    .tail(4)
    .apply(stats))



    


















    
Out[30]:








linear_percent_on    [0.4005, 0.4927, 0.5849]
dtype: object

Search of optimal linear weight sparsity





In [32]:



    


exps = [
    'C100_SparseBest_Linear1_T',
    'C100_SparseBest_Linear2_T',
]

paths = [os.path.expanduser("~/nta/results/{}".format(e)) for e in exps]
df = load_many(paths)



    











In [33]:



    


df.shape



    


















    
Out[33]:








(12, 62)

















In [42]:



    


df.head(20)



    


















    
Out[42]:











  
    

      
      Experiment Name
      test_accuracy
      test_accuracy_max
      epoch_test_accuracy
      noise_accuracy
      noise_accuracy_max
      epoch_noise_accuracy
      mean_accuracy
      mean_accuracy_max
      epoch_mean_accuracy
      ...
      repetitions
      restore_supported
      stop
      sync_function
      test_batch_size
      test_batches_in_epoch
      upload_dir
      use_max_pooling
      weight_decay
      weight_sparsity
    

  
  
    

      0
      0_weight_sparsity=[0.615606968378589]
      0.6942
      0.6968
      159
      0.2254
      0.2602
      33
      0.45980
      0.46735
      85
      ...
      6
      False
      {'stop': 1}
      aws s3 sync `dirname {local_dir}` {remote_dir}...
      128
      100
      s3://lsouza/ray/results
      True
      0.0007
      0.615607
    

    

      1
      1_weight_sparsity=[0.5108914633795211]
      0.6916
      0.6936
      141
      0.2281
      0.2698
      30
      0.45985
      0.46620
      89
      ...
      6
      False
      {'stop': 1}
      aws s3 sync `dirname {local_dir}` {remote_dir}...
      128
      100
      s3://lsouza/ray/results
      True
      0.0007
      0.510891
    

    

      2
      2_weight_sparsity=[0.7383259279890858]
      0.6997
      0.7012
      161
      0.2225
      0.2661
      24
      0.46110
      0.46405
      161
      ...
      6
      False
      {'stop': 1}
      aws s3 sync `dirname {local_dir}` {remote_dir}...
      128
      100
      s3://lsouza/ray/results
      True
      0.0007
      0.738326
    

    

      3
      3_weight_sparsity=[0.513634818975599]
      0.6932
      0.6944
      156
      0.2268
      0.2515
      122
      0.46000
      0.46825
      122
      ...
      6
      False
      {'stop': 1}
      aws s3 sync `dirname {local_dir}` {remote_dir}...
      128
      100
      s3://lsouza/ray/results
      True
      0.0007
      0.513635
    

    

      4
      4_weight_sparsity=[0.2995260895161448]
      0.6931
      0.6949
      103
      0.2291
      0.2681
      34
      0.46110
      0.46510
      117
      ...
      6
      False
      {'stop': 1}
      aws s3 sync `dirname {local_dir}` {remote_dir}...
      128
      100
      s3://lsouza/ray/results
      True
      0.0007
      0.299526
    

    

      5
      5_weight_sparsity=[0.3558040583351598]
      0.6996
      0.7018
      150
      0.2295
      0.2665
      69
      0.46455
      0.46750
      85
      ...
      6
      False
      {'stop': 1}
      aws s3 sync `dirname {local_dir}` {remote_dir}...
      128
      100
      s3://lsouza/ray/results
      True
      0.0007
      0.355804
    

    

      6
      0_weight_sparsity=[0.712460875922671, 0.712460...
      0.6952
      0.6967
      143
      0.2024
      0.2671
      67
      0.44880
      0.47050
      84
      ...
      6
      False
      {'stop': 1}
      aws s3 sync `dirname {local_dir}` {remote_dir}...
      128
      100
      s3://lsouza/ray/results
      True
      0.0007
      0.712461
    

    

      7
      1_weight_sparsity=[0.7331810324021211, 0.73318...
      0.6898
      0.6931
      160
      0.1850
      0.2418
      28
      0.43740
      0.45330
      97
      ...
      6
      False
      {'stop': 1}
      aws s3 sync `dirname {local_dir}` {remote_dir}...
      128
      100
      s3://lsouza/ray/results
      True
      0.0007
      0.733181
    

    

      8
      2_weight_sparsity=[0.7007782122493296, 0.70077...
      0.6943
      0.6959
      157
      0.2004
      0.2406
      34
      0.44735
      0.45670
      85
      ...
      6
      False
      {'stop': 1}
      aws s3 sync `dirname {local_dir}` {remote_dir}...
      128
      100
      s3://lsouza/ray/results
      True
      0.0007
      0.700778
    

    

      9
      3_weight_sparsity=[0.28786972025082924, 0.2878...
      0.6997
      0.7015
      136
      0.2078
      0.2798
      45
      0.45375
      0.45885
      125
      ...
      6
      False
      {'stop': 1}
      aws s3 sync `dirname {local_dir}` {remote_dir}...
      128
      100
      s3://lsouza/ray/results
      True
      0.0007
      0.287870
    

    

      10
      4_weight_sparsity=[0.32698651710912197, 0.3269...
      0.6964
      0.7003
      159
      0.2098
      0.2522
      35
      0.45310
      0.45990
      107
      ...
      6
      False
      {'stop': 1}
      aws s3 sync `dirname {local_dir}` {remote_dir}...
      128
      100
      s3://lsouza/ray/results
      True
      0.0007
      0.326987
    

    

      11
      5_weight_sparsity=[0.4413290626165175, 0.44132...
      0.6968
      0.7002
      161
      0.2487
      0.2671
      106
      0.47275
      0.47825
      106
      ...
      6
      False
      {'stop': 1}
      aws s3 sync `dirname {local_dir}` {remote_dir}...
      128
      100
      s3://lsouza/ray/results
      True
      0.0007
      0.441329
    

  



12 rows × 62 columns



















In [35]:



    


(df.groupby(['dataset', 'name'])['test_accuracy_max', 'noise_accuracy_max', 
                                 'mean_accuracy_max', 'epochs']
                                 .max().round(4))



    


















    
Out[35]:











  
    

      
      
      test_accuracy_max
      noise_accuracy_max
      mean_accuracy_max
      epochs
    

    

      dataset
      name
      
      
      
      
    

  
  
    

      CIFAR100
      C100_SparseBest_Linear1_T
      0.7018
      0.2698
      0.4683
      164
    

    

      C100_SparseBest_Linear2_T
      0.7015
      0.2798
      0.4782
      164
    

  




















In [36]:



    


(df.groupby(['dataset', 'name'])['test_accuracy_max', 'noise_accuracy_max', 
                                 'mean_accuracy_max', 'epochs']
                                 .mean().round(4))



    


















    
Out[36]:











  
    

      
      
      test_accuracy_max
      noise_accuracy_max
      mean_accuracy_max
      epochs
    

    

      dataset
      name
      
      
      
      
    

  
  
    

      CIFAR100
      C100_SparseBest_Linear1_T
      0.6971
      0.2637
      0.4664
      164
    

    

      C100_SparseBest_Linear2_T
      0.6980
      0.2581
      0.4629
      164
    

  




















In [46]:



    


tunable_params = ['weight_sparsity']



    











In [68]:



    


# CIFAR-10 SPARSE
filters = ((df['name']=='C100_SparseBest_Linear1_T'))
         
(df[filters]
    .sort_values('mean_accuracy_max', ascending=False)[tunable_params]
    .head(3)
    .apply(stats))



    


















    
Out[68]:











  
    

      
      weight_sparsity
    

  
  
    

      3
      0.3881
    

    

      5
      0.4950
    

    

      0
      0.6019
    

  




















In [69]:



    


# CIFAR-10 SPARSE
filters = ((df['name']=='C100_SparseBest_Linear1_T'))
         
(df[filters]
    .sort_values('mean_accuracy_max', ascending=False)[tunable_params]
    .tail(3)
    .apply(stats))



    


















    
Out[69]:











  
    

      
      weight_sparsity
    

  
  
    

      1
      0.3371
    

    

      4
      0.5162
    

    

      2
      0.6954
    

  




















In [70]:



    


# CIFAR-10 SPARSE
filters = ((df['name']=='C100_SparseBest_Linear2_T'))
         
(df[filters]
    .sort_values('mean_accuracy_max', ascending=False)[tunable_params]
    .head(3)
    .apply(stats))



    


















    
Out[70]:











  
    

      
      weight_sparsity
    

  
  
    

      11
      0.3319
    

    

      6
      0.4936
    

    

      10
      0.6552
    

  




















In [71]:



    


# CIFAR-10 SPARSE
filters = ((df['name']=='C100_SparseBest_Linear2_T'))
         
(df[filters]
    .sort_values('mean_accuracy_max', ascending=False)[tunable_params]
    .tail(3)
    .apply(stats))



    


















    
Out[71]:











  
    

      
      weight_sparsity
    

  
  
    

      9
      0.3712
    

    

      8
      0.5739
    

    

      7
      0.7767
    

  




















In [ ]:

Content source: numenta/nupic.research

Similar notebooks:

notebook.community | gallery | about