notebook.community

Edit and run



In [1]:

    
%matplotlib inline
import pandas as pd
import numpy as np
from sklearn.metrics.pairwise import cosine_similarity
from matplotlib import pyplot as plt
import glob
import numba



In [2]:

    
prototype_indicies = range(65,85)
prototype_indicies

# index 5 - 24 are the activation values
activation_indicies = range(5, 25)  # returns 5 - 24
print(list(activation_indicies))









    



[5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]



In [3]:

    
@numba.jit('f8(f8[:])')
def calculate_sse(row_vector):
    '''Calculate the sum square error between the activation values and prototype
    
    Parameters
    ----------
    row_vector: pandas series
    
    Returns:
    --------
    sum_square_error: floating point of sse
    '''
    assert(len(activation_indicies) == 20)
    assert(len(prototype_indicies) == 20)

    activation_values = row_vector.iloc[activation_indicies].values
    prototype_values = row_vector.iloc[prototype_indicies].values

    error_vector = activation_values - prototype_values

    square_error_vector = error_vector ** 2
    sum_square_error = np.sum(square_error_vector)
   
    return(sum_square_error)



In [4]:

    
def test_calculate_sse():
    pout_files = glob.glob('02-lens_batch_2014-11-14_20:46:22/*/output/*.pout')
    pout_file = pout_files[0]
    data = pd.read_csv(pout_file, header=None)
    data['sse'] = data.apply(calculate_sse, axis=1)
    calculated = np.sort(data['sse'].unique())
    expected = np.sort(np.array([1.100000e+01, 0.000000e+00, 4.293802e-04, 1.198868e-02, 
                                 1.742523e-02, 9.875449e-03, 1.272909e-02, 3.948474e-03, 
                                 2.029364e-02, 9.931646e-03, 7.607752e-03, 3.174525e-02, 
                                 6.270994e-04, 1.662769e-02, 2.289100e-02, 1.357461e-02, 
                                 3.237630e-04, 6.436357e-02, 7.514400e-04, 2.992966e-02, 
                                 6.378620e-02, 1.634918e-02, 1.794454e-02, 4.571180e-04, 
                                 3.821324e-04, 5.305328e-04, 1.435284e-02, 8.429803e-02, 
                                 1.080844e-02, 5.324111e-04, 1.824554e-02, 3.258835e-04, 
                                 5.280545e-04, 2.791479e-04]))
    assert(calculated.shape == expected.shape)
    np.testing.assert_array_almost_equal(calculated, expected)
test_calculate_sse()



In [5]:

    
def calculate_cosine_sim(row_vector):
    '''Calculate the cosine similarity between the activation values and prototype
    
    Parameters
    ----------
    row_vector: pandas series
    
    Returns:
    --------
    cos_sim: floating point of cosine similarity
    '''
    assert(len(activation_indicies) == 20)
    assert(len(prototype_indicies) == 20)

    activation_values = np.array(row_vector.iloc[activation_indicies])
    prototype_values = np.array(row_vector.iloc[prototype_indicies])

    activation_values_adj = activation_values + 1 
    prototype_values_adj = prototype_values + 1
    
    cos_sim = cosine_similarity(activation_values_adj, prototype_values_adj)
    assert(cos_sim.shape == (1, 1))
    return(cos_sim[0][0])



In [6]:

    
def test_calculate_cosine_sim():
    pout_files = glob.glob('02-lens_batch_2014-11-14_20:46:22/*/output/*.pout')
    pout_file = pout_files[0]
    print(pout_file)
    data = pd.read_csv(pout_file, header=None)
    data['cos'] = data.apply(calculate_cosine_sim, axis=1)
    calculated = data['cos'].values
    # above is same as:
    # calculated = np.array(data['cos'])
  
    expected = np.array([0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 1.0000000, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999965, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999965,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999040, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9999965, 0.9998430, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999040,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9999965, 0.9998430, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999137, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9999040, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999965, 0.9998430, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999137, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9999040, 0.9521574, 0.9521574, 0.9521574, 0.9998968, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999965, 0.9998430, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999137, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999040, 0.9521574, 0.9521574, 0.9521574, 0.9998968, 0.9521574, 0.9521574,
    0.9521574, 0.9999719, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999965, 0.9998430, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999137, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999040, 0.9521574, 0.9521574, 0.9521574, 0.9998968,
    0.9521574, 0.9521574, 0.9998336, 0.9999719, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999965,
    0.9998430, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9999140, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9999137, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999040, 0.9521574, 0.9521574,
    0.9521574, 0.9998968, 0.9521574, 0.9521574, 0.9998336, 0.9999719, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9999965, 0.9998430, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999140, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9999137, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999040,
    0.9521574, 0.9999503, 0.9521574, 0.9998968, 0.9521574, 0.9521574, 0.9998336, 0.9999719, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9999965, 0.9998430, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999140, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999137, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9999040, 0.9521574, 0.9999503, 0.9521574, 0.9998968, 0.9521574, 0.9521574, 0.9998336, 0.9999719, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999965, 0.9998430, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9997087, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999140, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999137, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9999040, 0.9521574, 0.9999503, 0.9521574, 0.9998968, 0.9521574, 0.9521574, 0.9998336, 0.9999719,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999965, 0.9998430, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9997087, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999140, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999137, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999040, 0.9521574, 0.9999503, 0.9521574, 0.9998968, 0.9521574, 0.9521574,
    0.9998336, 0.9999955, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999965, 0.9998430, 0.9521574,
    0.9521574, 0.9998704, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9997087, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9999140, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999137, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999040, 0.9521574, 0.9999503, 0.9521574, 0.9998968,
    0.9521574, 0.9521574, 0.9998336, 0.9999955, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999965,
    0.9998430, 0.9521574, 0.9521574, 0.9998704, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9997087, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9997983, 0.9999140, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9999137, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999040, 0.9521574, 0.9999503,
    0.9521574, 0.9998968, 0.9521574, 0.9521574, 0.9998336, 0.9999955, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9999965, 0.9998430, 0.9521574, 0.9521574, 0.9998704, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9997087, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9997983, 0.9999140, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9999137, 0.9998956, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999040,
    0.9521574, 0.9999503, 0.9521574, 0.9998968, 0.9521574, 0.9521574, 0.9998336, 0.9999955, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9999965, 0.9998430, 0.9521574, 0.9521574, 0.9998704, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9997087, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9997983, 0.9999140, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999137, 0.9998956, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9999040, 0.9999977, 0.9999503, 0.9521574, 0.9998968, 0.9521574, 0.9521574, 0.9998336, 0.9999955, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999965, 0.9998430, 0.9521574, 0.9521574, 0.9998704, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9997087, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9997983, 0.9999140, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999137, 0.9998956, 0.9521574, 0.9521574, 0.9521574, 0.9994081,
    0.9521574, 0.9521574, 0.9521574, 0.9999040, 0.9999977, 0.9999503, 0.9521574, 0.9998968, 0.9521574, 0.9521574, 0.9998336, 0.9999955,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999965, 0.9998430, 0.9521574, 0.9521574, 0.9998704,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9997087, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9997983, 0.9999140, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999137, 0.9998956, 0.9521574, 0.9521574,
    0.9521574, 0.9999937, 0.9521574, 0.9521574, 0.9521574, 0.9999040, 0.9999977, 0.9999503, 0.9521574, 0.9998968, 0.9521574, 0.9521574,
    0.9998336, 0.9999955, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999965, 0.9998430, 0.9521574,
    0.9521574, 0.9998704, 0.9521574, 0.9997936, 0.9521574, 0.9521574, 0.9997087, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9997983,
    0.9999140, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999137, 0.9998956,
    0.9521574, 0.9521574, 0.9521574, 0.9999937, 0.9521574, 0.9521574, 0.9521574, 0.9999040, 0.9999977, 0.9999503, 0.9521574, 0.9998968,
    0.9521574, 0.9521574, 0.9998336, 0.9999955, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999965,
    0.9998430, 0.9521574, 0.9521574, 0.9998704, 0.9521574, 0.9997936, 0.9521574, 0.9521574, 0.9997087, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9997983, 0.9999140, 0.9521574, 0.9994543, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9999137, 0.9998956, 0.9521574, 0.9521574, 0.9521574, 0.9999937, 0.9521574, 0.9521574, 0.9521574, 0.9999040, 0.9999977, 0.9999503,
    0.9521574, 0.9998968, 0.9521574, 0.9521574, 0.9998336, 0.9999955, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9999965, 0.9998430, 0.9521574, 0.9521574, 0.9998704, 0.9521574, 0.9997936, 0.9521574, 0.9521574, 0.9997087, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9997983, 0.9999140, 0.9521574, 0.9994543, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9999137, 0.9998956, 0.9521574, 0.9521574, 0.9521574, 0.9999937, 0.9521574, 0.9521574, 0.9521574, 0.9999040,
    0.9999977, 0.9999503, 0.9521574, 0.9998968, 0.9521574, 0.9521574, 0.9998336, 0.9999955, 0.9998663, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9999965, 0.9998430, 0.9521574, 0.9521574, 0.9998704, 0.9521574, 0.9997936, 0.9521574, 0.9521574,
    0.9997087, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9997983, 0.9999140, 0.9521574, 0.9994543, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9521574, 0.9998522, 0.9521574, 0.9999137, 0.9998956, 0.9521574, 0.9521574, 0.9521574, 0.9999937, 0.9521574, 0.9521574,
    0.9521574, 0.9999040, 0.9999977, 0.9999503, 0.9521574, 0.9998968, 0.9521574, 0.9521574, 0.9998336, 0.9999955, 0.9998663, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999965, 0.9998430, 0.9521574, 0.9521574, 0.9998704, 0.9521574, 0.9997936,
    0.9521574, 0.9521574, 0.9997087, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9997983, 0.9999140, 0.9521574, 0.9994543, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9998522, 0.9521574, 0.9999137, 0.9998956, 0.9521574, 0.9521574, 0.9521574, 0.9999937,
    0.9521574, 0.9521574, 0.9521574, 0.9999040, 0.9999977, 0.9999503, 0.9521574, 0.9998968, 0.9521574, 0.9521574, 0.9998336, 0.9999955,
    0.9998663, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999965, 0.9998430, 0.9521574, 0.9521574, 0.9998704,
    0.9521574, 0.9997936, 0.9521574, 0.9521574, 0.9997087, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9997983, 0.9999140, 0.9521574,
    0.9994543, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9998522, 0.9521574, 0.9999137, 0.9998956, 0.9521574, 0.9521574,
    0.9521574, 0.9999937, 0.9521574, 0.9521574, 0.9521574, 0.9999964, 0.9999977, 0.9999503, 0.9521574, 0.9998968, 0.9521574, 0.9521574,
    0.9998336, 0.9999955, 0.9998663, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999970, 0.9521574, 0.9999965, 0.9998430, 0.9521574,
    0.9521574, 0.9998704, 0.9521574, 0.9997936, 0.9521574, 0.9521574, 0.9997087, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9997983,
    0.9999140, 0.9521574, 0.9994543, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9998522, 0.9521574, 0.9999137, 0.9998956,
    0.9521574, 0.9521574, 0.9521574, 0.9999937, 0.9521574, 0.9521574, 0.9521574, 0.9999964, 0.9999977, 0.9999503, 0.9521574, 0.9998968,
    0.9521574, 0.9521574, 0.9998336, 0.9999955, 0.9998663, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999970, 0.9521574, 0.9999965,
    0.9998430, 0.9521574, 0.9521574, 0.9998704, 0.9521574, 0.9997936, 0.9521574, 0.9521574, 0.9997087, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9997983, 0.9999140, 0.9521574, 0.9994543, 0.9521574, 0.9999956, 0.9521574, 0.9521574, 0.9521574, 0.9998522, 0.9521574,
    0.9999137, 0.9998956, 0.9521574, 0.9521574, 0.9521574, 0.9999937, 0.9521574, 0.9521574, 0.9521574, 0.9999964, 0.9999977, 0.9999503,
    0.9521574, 0.9998968, 0.9521574, 0.9521574, 0.9998336, 0.9999955, 0.9998663, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999970,
    0.9521574, 0.9999965, 0.9998430, 0.9521574, 0.9521574, 0.9998704, 0.9521574, 0.9997936, 0.9521574, 0.9521574, 0.9997087, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9997983, 0.9999140, 0.9521574, 0.9994543, 0.9521574, 0.9999956, 0.9521574, 0.9521574, 0.9521574,
    0.9998522, 0.9521574, 0.9999137, 0.9998956, 0.9521574, 0.9521574, 0.9521574, 0.9999937, 0.9521574, 0.9521574, 0.9521574, 0.9999964,
    0.9999977, 0.9999503, 0.9998781, 0.9998968, 0.9521574, 0.9521574, 0.9998336, 0.9999955, 0.9998663, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9999970, 0.9521574, 0.9999965, 0.9998430, 0.9521574, 0.9521574, 0.9998704, 0.9521574, 0.9997936, 0.9521574, 0.9521574,
    0.9997087, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9997983, 0.9999140, 0.9521574, 0.9994543, 0.9521574, 0.9999956, 0.9521574,
    0.9521574, 0.9521574, 0.9998522, 0.9521574, 0.9999137, 0.9998956, 0.9521574, 0.9521574, 0.9993189, 0.9999937, 0.9521574, 0.9521574,
    0.9521574, 0.9999964, 0.9999977, 0.9999503, 0.9998781, 0.9998968, 0.9521574, 0.9521574, 0.9998336, 0.9999955, 0.9998663, 0.9521574,
    0.9521574, 0.9521574, 0.9521574, 0.9999970, 0.9521574, 0.9999965, 0.9998430, 0.9521574, 0.9521574, 0.9998704, 0.9521574, 0.9997936,
    0.9521574, 0.9521574, 0.9997087, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9997983, 0.9999140, 0.9521574, 0.9994543, 0.9521574,
    0.9999956, 0.9521574, 0.9521574, 0.9521574, 0.9998522, 0.9521574, 0.9999137, 0.9998956, 0.9521574, 0.9521574, 0.9993189, 0.9999937,
    0.9521574, 0.9521574, 0.9521574, 0.9999964, 0.9999977, 0.9999503, 0.9998781, 0.9998968, 0.9521574, 0.9521574, 0.9998336, 0.9999955,
    0.9998663, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999970, 0.9521574, 0.9999965, 0.9998430, 0.9999115, 0.9521574, 0.9998704,
    0.9521574, 0.9997936, 0.9521574, 0.9521574, 0.9997087, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9997983, 0.9999140, 0.9521574,
    0.9994543, 0.9521574, 0.9999956, 0.9521574, 0.9521574, 0.9521574, 0.9998522, 0.9521574, 0.9999137, 0.9998956, 0.9521574, 0.9521574,
    0.9993189, 0.9999937, 0.9521574, 0.9521574, 0.9521574, 0.9999964, 0.9999977, 0.9999503, 0.9998781, 0.9998968, 0.9521574, 0.9521574,
    0.9998336, 0.9999955, 0.9998663, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999970, 0.9521574, 0.9999965, 0.9998430, 0.9999115,
    0.9521574, 0.9998704, 0.9521574, 0.9997936, 0.9521574, 0.9521574, 0.9997087, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999956,
    0.9999140, 0.9521574, 0.9994543, 0.9521574, 0.9999956, 0.9521574, 0.9521574, 0.9521574, 0.9998522, 0.9521574, 0.9999137, 0.9998956,
    0.9521574, 0.9521574, 0.9993189, 0.9999937, 0.9521574, 0.9521574, 0.9521574, 0.9999964, 0.9999977, 0.9999503, 0.9998781, 0.9998968,
    0.9521574, 0.9521574, 0.9998336, 0.9999955, 0.9998663, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999970, 0.9521574, 0.9999965,
    0.9998430, 0.9999115, 0.9521574, 0.9998704, 0.9521574, 0.9997936, 0.9521574, 0.9521574, 0.9997087, 0.9521574, 0.9998352, 0.9521574,
    0.9521574, 0.9999956, 0.9999140, 0.9521574, 0.9994543, 0.9521574, 0.9999956, 0.9521574, 0.9521574, 0.9521574, 0.9998522, 0.9521574,
    0.9999137, 0.9998956, 0.9521574, 0.9521574, 0.9993189, 0.9999937, 0.9521574, 0.9521574, 0.9521574, 0.9999964, 0.9999977, 0.9999503,
    0.9998781, 0.9998968, 0.9521574, 0.9521574, 0.9998336, 0.9999955, 0.9998663, 0.9521574, 0.9521574, 0.9521574, 0.9521574, 0.9999970,
    0.9521574, 0.9999965, 0.9998430, 0.9999115, 0.9521574, 0.9998704, 0.9521574, 0.9997936, 0.9521574, 0.9521574, 0.9997087, 0.9521574,
    0.9998352, 0.9521574, 0.9521574, 0.9999956, 0.9999140, 0.9521574, 0.9994543, 0.9521574, 0.9999956, 0.9521574, 0.9521574, 0.9521574,
    0.9998522, 0.9521574, 0.9999137, 0.9998956, 0.9521574, 0.9521574, 0.9993189, 0.9999937, 0.9521574, 0.9521574, 0.9521574, 0.9999964,
    0.9999977, 0.9999503, 0.9998781, 0.9998968, 0.9521574, 0.9521574, 0.9998336, 0.9999719, 0.9998663, 0.9521574, 0.9521574, 0.9521574,
    0.9521574, 0.9999970, 0.9521574, 0.9999965, 0.9998430, 0.9999115, 0.9521574, 0.9998704, 0.9521574, 0.9997936, 0.9521574, 0.9521574,
    0.9997087, 0.9521574, 0.9998352, 0.9521574, 0.9521574, 0.9999956, 0.9999140, 0.9521574, 0.9994543, 0.9521574, 0.9999956, 0.9521574,
    0.9521574, 0.9521574, 0.9998522, 0.9521574, 0.9999137, 0.9998956, 0.9521574, 0.9521574, 0.9993189, 0.9999937, 0.9521574, 0.9521574,
    0.9521574, 0.9999964, 0.9999977, 0.9999503, 0.9998781, 0.9998968, 0.9521574, 0.9521574, 0.9998336, 0.9999719, 0.9998663, 0.9521574,
    0.9521574, 0.9521574, 0.9999973, 0.9999970, 0.9521574, 0.9999965, 0.9998430, 0.9999115, 0.9521574, 0.9998704, 0.9521574, 0.9997936,
    0.9521574, 0.9521574, 0.9997087, 0.9521574, 0.9998352, 0.9521574, 0.9521574, 0.9999956, 0.9999140, 0.9521574, 0.9994543, 0.9521574,
    0.9999956, 0.9521574, 0.9521574, 0.9521574, 0.9998522, 0.9521574, 0.9999137, 0.9998956, 0.9521574, 0.9521574, 0.9993189, 0.9999937,
    0.9521574, 0.9521574, 0.9521574, 0.9999964, 0.9999977, 0.9999503, 0.9998781, 0.9998968, 0.9521574, 0.9521574, 0.9998336, 0.9999719,
    0.9998663, 0.9521574, 0.9521574, 0.9521574, 0.9999973, 0.9999970, 0.9521574, 0.9999965, 0.9998430, 0.9999115, 0.9521574, 0.9998704,
    0.9521574, 0.9997936, 0.9521574, 0.9521574, 0.9997087, 0.9521574, 0.9998352, 0.9521574, 0.9521574, 0.9999956, 0.9999140, 0.9521574,
    0.9994543, 0.9521574, 0.9999956, 0.9521574, 0.9999964, 0.9521574, 0.9998522, 0.9521574, 0.9999137, 0.9998956, 0.9521574, 0.9521574,
    0.9993189, 0.9999937, 0.9521574, 0.9521574, 0.9521574, 0.9999964, 0.9999977, 0.9999503, 0.9998781, 0.9998968, 0.9521574, 0.9521574,
    0.9998336, 0.9999719, 0.9998663, 0.9521574, 0.9521574, 0.9521574, 0.9999973, 0.9999970, 0.9521574, 0.9999965, 0.9998430, 0.9999115,
    0.9521574, 0.9998704, 0.9521574, 0.9997936, 0.9521574, 0.9521574, 0.9997087, 0.9521574, 0.9998352, 0.9521574, 0.9521574, 0.9999956,
    0.9999140, 0.9521574, 0.9994543, 0.9521574, 0.9999956, 0.9521574, 0.9999964, 0.9521574, 0.9999977, 0.9521574, 0.9999137, 0.9998956,
    0.9521574, 0.9521574, 0.9993189, 0.9999937, 0.9521574, 0.9521574, 0.9521574, 0.9999964, 0.9999977, 0.9999503, 0.9998781, 0.9998968,
    0.9521574, 0.9521574, 0.9998336, 0.9999719, 0.9998663, 0.9521574, 0.9521574, 0.9521574, 0.9999973, 0.9999970, 0.9521574, 0.9999965,
    0.9998430, 0.9999115, 0.9521574, 0.9998704, 0.9521574, 0.9997936, 0.9521574, 0.9521574, 0.9997087, 0.9521574, 0.9998352, 0.9521574,
    0.9521574, 0.9999956, 0.9999140, 0.9521574, 0.9994543, 0.9521574, 0.9999956, 0.9521574, 0.9999964, 0.9521574, 0.9999977, 0.9521574,
    0.9999137, 0.9998956, 0.9521574, 0.9521574, 0.9993189, 0.9999937, 0.9521574, 0.9521574, 0.9521574, 0.9999964, 0.9999977, 0.9999503,
    0.9998781, 0.9998968, 0.9521574, 0.9521574, 0.9998336, 0.9999719, 0.9998663, 0.9521574])
    assert(calculated.shape == expected.shape)
    np.testing.assert_array_almost_equal(calculated, expected)
test_calculate_cosine_sim()









    



02-lens_batch_2014-11-14_20:46:22/d000_c03_r00/output/network_of_agents.pout



In [7]:

    
def create_cosine_sim_cutoffs(dataframe, list_of_values):
    assert(isinstance(list_of_values, list))
    assert(isinstance(list_of_values[0], float))
    
    for value in list_of_values:
        # format the float for 2 decimals, and only keep the decimal portion
        new_column_name = 'cos' + ("{0:.2f}".format(value)).split('.')[1]
        dataframe[new_column_name] = np.where(dataframe['cos'] >= value, 1, 0)
    return dataframe



In [8]:

    
# pout_files = glob.glob('02-lens_batch_2014-11-14_20:46:22/*/output/*.pout')
pout_files = glob.glob('/home/dchen/Desktop/snowmane_pout/*/output/*.pout')
pout_files









    Out[8]:





['/home/dchen/Desktop/snowmane_pout/d020_c03_r00/output/network_of_agents.pout',
 '/home/dchen/Desktop/snowmane_pout/d000_c03_r00/output/network_of_agents.pout']



In [9]:

    
for pout_file in pout_files:
    pass

Run each part of the loop individually and time it



In [10]:

    
%%timeit
# 1 loops, best of 3: 28.3 s per loop

pout_file = pout_files[0]
data = pd.read_csv(pout_file, header=None)









    



1 loops, best of 3: 28.3 s per loop



In [13]:

    
data = pd.read_csv(pout_file, header=None)



In [14]:

    
data.shape









    Out[14]:





(7002100, 85)



In [ ]:

    
%%timeit
data['sse'] = data.apply(calculate_sse, axis=1)



In [ ]:

    
data



In [ ]:

    
data['sse'] = data.apply(calculate_sse, axis=1)

loop body all at once



In [129]:

    
# %%timeit
# inside the loop

# rel path of the .pout file
pout_file = pout_files[0]
# print(pout_file)

# name of the folder
folder = pout_file.split('/')[1]
# print(folder)

# load into dataframe
data = pd.read_csv(pout_file, header=None)
# print(data.head())

data['sse'] = data.apply(calculate_sse, axis=1)
data['cos'] = data.apply(calculate_cosine_sim, axis=1)

data['if_ever_updated'] = np.where(data[2] > 0, 1, 0)

cos_sim_thresholds = [0.80, 0.85, 0.90, 0.95, 0.97, 0.99]
assert(isinstance(cos_sim_thresholds, list))
for x in cos_sim_thresholds:
    assert(isinstance(x, float))

data = create_cosine_sim_cutoffs(data, list_of_values=cos_sim_thresholds)

# for cos_threshold in cos_sim_thresholds:
#     col_value = 'cos' + str(cos_sim_thresholds)
#     data.groupby([0])[col_value].









    



---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-129-9d3362a36c58> in <module>()
----> 1 get_ipython().run_cell_magic('timeit', '', "# inside the loop\n\n# rel path of the .pout file\npout_file = pout_files[0]\n# print(pout_file)\n\n# name of the folder\nfolder = pout_file.split('/')[1]\n# print(folder)\n\n# load into dataframe\ndata = pd.read_csv(pout_file, header=None)\n# print(data.head())\n\ndata['sse'] = data.apply(calculate_sse, axis=1)\ndata['cos'] = data.apply(calculate_cosine_sim, axis=1)\n\ndata['if_ever_updated'] = np.where(data[2] > 0, 1, 0)\n\ncos_sim_thresholds = [0.80, 0.85, 0.90, 0.95, 0.97, 0.99]\nassert(isinstance(cos_sim_thresholds, list))\nfor x in cos_sim_thresholds:\n    assert(isinstance(x, float))\n\ndata = create_cosine_sim_cutoffs(data, list_of_values=cos_sim_thresholds)\n\n# for cos_threshold in cos_sim_thresholds:\n#     col_value = 'cos' + str(cos_sim_thresholds)\n#     data.groupby([0])[col_value].")

/home/dchen/anaconda3/envs/mann/lib/python3.4/site-packages/IPython/core/interactiveshell.py in run_cell_magic(self, magic_name, line, cell)
   2160             magic_arg_s = self.var_expand(line, stack_depth)
   2161             with self.builtin_trap:
-> 2162                 result = fn(magic_arg_s, cell)
   2163             return result
   2164 

/home/dchen/anaconda3/envs/mann/lib/python3.4/site-packages/IPython/core/magics/execution.py in timeit(self, line, cell)

/home/dchen/anaconda3/envs/mann/lib/python3.4/site-packages/IPython/core/magic.py in <lambda>(f, *a, **k)
    191     # but it's overkill for just that one bit of state.
    192     def magic_deco(arg):
--> 193         call = lambda f, *a, **k: f(*a, **k)
    194 
    195         if callable(arg):

/home/dchen/anaconda3/envs/mann/lib/python3.4/site-packages/IPython/core/magics/execution.py in timeit(self, line, cell)
   1014                     break
   1015                 number *= 10
-> 1016         all_runs = timer.repeat(repeat, number)
   1017         best = min(all_runs) / number
   1018         if not quiet :

/home/dchen/anaconda3/envs/mann/lib/python3.4/timeit.py in repeat(self, repeat, number)
    204         r = []
    205         for i in range(repeat):
--> 206             t = self.timeit(number)
    207             r.append(t)
    208         return r

/home/dchen/anaconda3/envs/mann/lib/python3.4/timeit.py in timeit(self, number)
    176         gc.disable()
    177         try:
--> 178             timing = self.inner(it, self.timer)
    179         finally:
    180             if gcold:

<magic-timeit> in inner(_it, _timer)

/home/dchen/anaconda3/envs/mann/lib/python3.4/site-packages/pandas/core/frame.py in apply(self, func, axis, broadcast, raw, reduce, args, **kwds)
   3469                     if reduce is None:
   3470                         reduce = True
-> 3471                     return self._apply_standard(f, axis, reduce=reduce)
   3472             else:
   3473                 return self._apply_broadcast(f, axis)

/home/dchen/anaconda3/envs/mann/lib/python3.4/site-packages/pandas/core/frame.py in _apply_standard(self, func, axis, ignore_failures, reduce)
   3521                 labels = self._get_agg_axis(axis)
   3522                 result = lib.reduce(values, func, axis=axis, dummy=dummy,
-> 3523                                     labels=labels)
   3524                 return Series(result, index=labels)
   3525             except Exception:

/home/dchen/anaconda3/envs/mann/lib/python3.4/site-packages/pandas/lib.cpython-34m.so in pandas.lib.reduce (pandas/lib.c:37350)()

/home/dchen/anaconda3/envs/mann/lib/python3.4/site-packages/pandas/lib.cpython-34m.so in pandas.lib.Reducer.get_result (pandas/lib.c:28388)()

<ipython-input-90-8256a103286a> in calculate_sse(row_vector)
     14 
     15     activation_values = row_vector.iloc[activation_indicies].values
---> 16     prototype_values = row_vector.iloc[prototype_indicies].values
     17 
     18     error_vector = activation_values - prototype_values

/home/dchen/anaconda3/envs/mann/lib/python3.4/site-packages/pandas/core/indexing.py in __getitem__(self, key)
   1142             return self._getitem_tuple(key)
   1143         else:
-> 1144             return self._getitem_axis(key, axis=0)
   1145 
   1146     def _getitem_axis(self, key, axis=0, validate_iterable=False):

/home/dchen/anaconda3/envs/mann/lib/python3.4/site-packages/pandas/core/indexing.py in _getitem_axis(self, key, axis, validate_iterable)
   1400 
   1401                 # validate list bounds
-> 1402                 self._is_valid_list_like(key, axis)
   1403 
   1404                 # force an actual list

/home/dchen/anaconda3/envs/mann/lib/python3.4/site-packages/pandas/core/indexing.py in _is_valid_list_like(self, key, axis)
   1338         ax = self.obj._get_axis(axis)
   1339         l = len(ax)
-> 1340         if len(arr) and (arr.max() >= l or arr.min() <= -l):
   1341             raise IndexError("positional indexers are out-of-bounds")
   1342 

/home/dchen/anaconda3/envs/mann/lib/python3.4/site-packages/numpy/core/_methods.py in _amax(a, axis, out, keepdims)
     24 # small reductions
     25 def _amax(a, axis=None, out=None, keepdims=False):
---> 26     return umr_maximum(a, axis, None, out, keepdims)
     27 
     28 def _amin(a, axis=None, out=None, keepdims=False):

KeyboardInterrupt:



In [102]:

    
%%timeit
by_time = data.groupby([0])['cos99'].agg({'mean', 'count', 'sum'})
by_time['prop'] = by_time['sum']/by_time['count']
# by_update_status = data.groupby(['if_ever_updated'])['cos0.99'].agg(['mean', 'count', 'sum'])
# by_time.div(by_update_status, level='if_ever_updated')
by_time









    



1000 loops, best of 3: 1.66 ms per loop



In [104]:

    
%%timeit
plt.plot(by_time['prop'])









    



1000 loops, best of 3: 849 µs per loop



In [106]:

    
by_time_update = data.groupby([0, 'if_ever_updated'])['cos99'].agg({'mean', 'count', 'sum'})
by_time_update['prop'] = by_time_update['sum']/by_time_update['count']
# by_update_status = data.groupby(['if_ever_updated'])['cos0.99'].agg(['mean', 'count', 'sum'])
# by_time.div(by_update_status, level='if_ever_updated')
by_time_update









    Out[106]:






  
    
      
      
      sum
      mean
      count
      prop
    
    
      0
      if_ever_updated
      
      
      
      
    
  
  
    
      -3 
      0
        0
       0.000000
       50
       0.000000
    
    
      -2 
      0
        1
       0.020000
       50
       0.020000
    
    
      -1 
      0
        1
       0.020000
       50
       0.020000
    
    
       0 
      0
        1
       0.020408
       49
       0.020408
    
    
      1
        1
       1.000000
        1
       1.000000
    
    
       1 
      0
        1
       0.020833
       48
       0.020833
    
    
      1
        2
       1.000000
        2
       1.000000
    
    
       2 
      0
        1
       0.021277
       47
       0.021277
    
    
      1
        3
       1.000000
        3
       1.000000
    
    
       3 
      0
        1
       0.021739
       46
       0.021739
    
    
      1
        4
       1.000000
        4
       1.000000
    
    
       4 
      0
        1
       0.022222
       45
       0.022222
    
    
      1
        5
       1.000000
        5
       1.000000
    
    
       5 
      0
        1
       0.022727
       44
       0.022727
    
    
      1
        6
       1.000000
        6
       1.000000
    
    
       6 
      0
        1
       0.023256
       43
       0.023256
    
    
      1
        7
       1.000000
        7
       1.000000
    
    
       7 
      0
        1
       0.023810
       42
       0.023810
    
    
      1
        8
       1.000000
        8
       1.000000
    
    
       8 
      0
        1
       0.023810
       42
       0.023810
    
    
      1
        8
       1.000000
        8
       1.000000
    
    
       9 
      0
        1
       0.024390
       41
       0.024390
    
    
      1
        9
       1.000000
        9
       1.000000
    
    
       10
      0
        1
       0.024390
       41
       0.024390
    
    
      1
        9
       1.000000
        9
       1.000000
    
    
       11
      0
        1
       0.025000
       40
       0.025000
    
    
      1
       10
       1.000000
       10
       1.000000
    
    
       12
      0
        1
       0.025641
       39
       0.025641
    
    
      1
       11
       1.000000
       11
       1.000000
    
    
       13
      0
        1
       0.026316
       38
       0.026316
    
    
      ...
      ...
      ...
      ...
      ...
      ...
    
    
       22
      0
        1
       0.031250
       32
       0.031250
    
    
      1
       18
       1.000000
       18
       1.000000
    
    
       23
      0
        1
       0.032258
       31
       0.032258
    
    
      1
       19
       1.000000
       19
       1.000000
    
    
       24
      0
        1
       0.033333
       30
       0.033333
    
    
      1
       20
       1.000000
       20
       1.000000
    
    
       25
      0
        1
       0.034483
       29
       0.034483
    
    
      1
       21
       1.000000
       21
       1.000000
    
    
       26
      0
        1
       0.035714
       28
       0.035714
    
    
      1
       22
       1.000000
       22
       1.000000
    
    
       27
      0
        1
       0.035714
       28
       0.035714
    
    
      1
       22
       1.000000
       22
       1.000000
    
    
       28
      0
        1
       0.037037
       27
       0.037037
    
    
      1
       23
       1.000000
       23
       1.000000
    
    
       29
      0
        1
       0.037037
       27
       0.037037
    
    
      1
       23
       1.000000
       23
       1.000000
    
    
       30
      0
        1
       0.038462
       26
       0.038462
    
    
      1
       24
       1.000000
       24
       1.000000
    
    
       31
      0
        1
       0.038462
       26
       0.038462
    
    
      1
       24
       1.000000
       24
       1.000000
    
    
       32
      0
        1
       0.038462
       26
       0.038462
    
    
      1
       24
       1.000000
       24
       1.000000
    
    
       33
      0
        1
       0.040000
       25
       0.040000
    
    
      1
       25
       1.000000
       25
       1.000000
    
    
       34
      0
        1
       0.041667
       24
       0.041667
    
    
      1
       26
       1.000000
       26
       1.000000
    
    
       35
      0
        1
       0.041667
       24
       0.041667
    
    
      1
       26
       1.000000
       26
       1.000000
    
    
       36
      0
        1
       0.041667
       24
       0.041667
    
    
      1
       26
       1.000000
       26
       1.000000
    
  

77 rows × 4 columns



In [124]:

    
plt.plot(by_time_update.reset_index()[by_time_update.reset_index()['if_ever_updated'] == 1]['prop'])









    Out[124]:





[<matplotlib.lines.Line2D at 0x7f98e3dc0e10>]



In [120]:

    
by_time_update.reset_index().if_ever_updated









    Out[120]:





0     0
1     0
2     0
3     0
4     1
5     0
6     1
7     0
8     1
9     0
10    1
11    0
12    1
13    0
14    1
...
62    1
63    0
64    1
65    0
66    1
67    0
68    1
69    0
70    1
71    0
72    1
73    0
74    1
75    0
76    1
Name: if_ever_updated, Length: 77, dtype: int64



In [100]:

    
data.columns









    Out[100]:





Index([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 'sse', 'cos', 'if_ever_updated', 'cos80', 'cos85', 'cos90', 'cos95', 'cos97', 'cos99'], dtype='object')



In [101]:

    
# test calculate_sse
data.loc[0].loc[activation_indicies]

data.iloc[0].iloc[prototype_indicies]

type(data.loc[0].loc[activation_indicies])

diff = data.iloc[0].iloc[activation_indicies].reset_index(drop=True) -\
data.iloc[0].iloc[prototype_indicies].reset_index(drop=True)

assert(np.array_equal(np.array(diff),
                      np.array([-1, -1, -1, 0, -1, 0, -1, -1, -1, 0, 0, -1, -1, 0, -1, 0, -1, 0, 0, 0])
                      ))

diff = np.array(data.iloc[0].iloc[activation_indicies]) -\
np.array(data.iloc[0].iloc[prototype_indicies])

assert(np.array_equal(np.array(diff),
                      np.array([-1, -1, -1, 0, -1, 0, -1, -1, -1, 0, 0, -1, -1, 0, -1, 0, -1, 0, 0, 0])
                      ))

		sum	mean	count	prop
0	if_ever_updated
-3	0	0	0.000000	50	0.000000
-2	0	1	0.020000	50	0.020000
-1	0	1	0.020000	50	0.020000
0	0	1	0.020408	49	0.020408
0	1	1	1.000000	1	1.000000
1	0	1	0.020833	48	0.020833
1	1	2	1.000000	2	1.000000
2	0	1	0.021277	47	0.021277
2	1	3	1.000000	3	1.000000
3	0	1	0.021739	46	0.021739
3	1	4	1.000000	4	1.000000
4	0	1	0.022222	45	0.022222
4	1	5	1.000000	5	1.000000
5	0	1	0.022727	44	0.022727
5	1	6	1.000000	6	1.000000
6	0	1	0.023256	43	0.023256
6	1	7	1.000000	7	1.000000
7	0	1	0.023810	42	0.023810
7	1	8	1.000000	8	1.000000
8	0	1	0.023810	42	0.023810
8	1	8	1.000000	8	1.000000
9	0	1	0.024390	41	0.024390
9	1	9	1.000000	9	1.000000
10	0	1	0.024390	41	0.024390
10	1	9	1.000000	9	1.000000
11	0	1	0.025000	40	0.025000
11	1	10	1.000000	10	1.000000
12	0	1	0.025641	39	0.025641
12	1	11	1.000000	11	1.000000
13	0	1	0.026316	38	0.026316
...	...	...	...	...	...
22	0	1	0.031250	32	0.031250
22	1	18	1.000000	18	1.000000
23	0	1	0.032258	31	0.032258
23	1	19	1.000000	19	1.000000
24	0	1	0.033333	30	0.033333
24	1	20	1.000000	20	1.000000
25	0	1	0.034483	29	0.034483
25	1	21	1.000000	21	1.000000
26	0	1	0.035714	28	0.035714
26	1	22	1.000000	22	1.000000
27	0	1	0.035714	28	0.035714
27	1	22	1.000000	22	1.000000
28	0	1	0.037037	27	0.037037
28	1	23	1.000000	23	1.000000
29	0	1	0.037037	27	0.037037
29	1	23	1.000000	23	1.000000
30	0	1	0.038462	26	0.038462
30	1	24	1.000000	24	1.000000
31	0	1	0.038462	26	0.038462
31	1	24	1.000000	24	1.000000
32	0	1	0.038462	26	0.038462
32	1	24	1.000000	24	1.000000
33	0	1	0.040000	25	0.040000
33	1	25	1.000000	25	1.000000
34	0	1	0.041667	24	0.041667
34	1	26	1.000000	26	1.000000
35	0	1	0.041667	24	0.041667
35	1	26	1.000000	26	1.000000
36	0	1	0.041667	24	0.041667
36	1	26	1.000000	26	1.000000