In [1]:
%load_ext autoreload
%autoreload 2
In [2]:
import sys
sys.path.append("../../")
In [3]:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import glob
import tabulate
import pprint
import click
import numpy as np
import pandas as pd
from ray.tune.commands import *
from nupic.research.frameworks.dynamic_sparse.common.browser import *
In [20]:
base = 'run_stochastic_synapses_gscvalidate'
exps = [
base
]
paths = [os.path.expanduser("~/nta/results/{}".format(e)) for e in exps]
for p in paths:
print(os.path.exists(p), p)
has_nz = lambda k: "nz" in k
has_sp = lambda k: "sparsity" in k
df = load_many(paths, raw_metrics=[has_nz, has_sp])
In [21]:
df
Out[21]:
In [22]:
df.columns
Out[22]:
In [16]:
# helper functions
def mean_and_std(s):
return "{:.3f} ± {:.3f}".format(s.mean(), s.std())
def round_mean(s):
return "{:.0f}".format(round(s.mean()))
stats = ['min', 'max', 'mean', 'std']
def agg(columns, filter=None, round=3):
if filter is None:
return (df.groupby(columns)
.agg({'val_acc_max_epoch': round_mean,
'val_acc_max': stats,
'model': ['count']})).round(round)
else:
return (df[filter].groupby(columns)
.agg({'val_acc_max_epoch': round_mean,
'val_acc_max': stats,
'model': ['count']})).round(round)
In [51]:
df['cnn1/expected_nz'].apply(lambda s: print(len(s)))
Out[51]:
In [59]:
df['cnn1/final_expected_nz'] = df['cnn1/expected_nz'].apply(lambda s: s.iloc[0])
df['cnn1/final_inference_nz'] = df['cnn1/inference_nz'].apply(lambda s: s.iloc[0])
df['cnn2/final_expected_nz'] = df['cnn2/expected_nz'].apply(lambda s: s.iloc[0])
df['cnn2/final_inference_nz'] = df['cnn2/inference_nz'].apply(lambda s: s.iloc[0])
df['fc1/final_expected_nz'] = df['fc1/expected_nz'].apply(lambda s: s.iloc[0])
df['fc1/final_inference_nz'] = df['fc1/inference_nz'].apply(lambda s: s.iloc[0])
df['fc2/final_expected_nz'] = df['fc2/expected_nz'].apply(lambda s: s.iloc[0])
df['fc2/final_inference_nz'] = df['fc2/inference_nz'].apply(lambda s: s.iloc[0])
In [68]:
df.groupby(["network"]).agg({
"val_acc_max": ["mean", "std"],
})
Out[68]:
In [69]:
df.groupby(["network"]).agg({
'cnn1/final_expected_nz': "mean",
'cnn2/final_expected_nz': "mean",
'fc1/final_expected_nz': "mean",
'fc2/final_expected_nz': "mean",
})
Out[69]:
In [70]:
df.groupby(["network"]).agg({
'cnn1/final_inference_nz': "mean",
'cnn2/final_inference_nz': "mean",
'fc1/final_inference_nz': "mean",
'fc2/final_inference_nz': "mean",
})
Out[70]: