In [1]:

    

import pandas as pd
from pandas.io.json import json_normalize #package for flattening json in pandas df
import numpy as np
import matplotlib.pyplot as plt
import matplotlib
import json
import os
%matplotlib inline


    

In [2]:

    

os.chdir("./runs/lagaris/1d_trapz_main_serv/")
origin_path = os.getcwd() 
runs_id = os.listdir("./")
runs_id = [int(item) for item in runs_id]
runs_id = sorted(runs_id)


    

In [3]:

    

df_list = []
for run_id in runs_id:
    os.chdir("./"+str(run_id))
    f_in = open('out.json', 'r')
    run_info = json.load(f_in)
    f_in.close()
    a = json_normalize(run_info)
    #a.set_index(pd.Index([run_id]))
    df_list.append(a)
    #a = pd.concat(a,b)
    os.chdir(origin_path)
res1 = pd.concat(df_list,ignore_index=True)


    

In [4]:

    

res_correct = res1[(res1['Out info.MSE'] < 0.01 ) | (res1['Model info.m_trapz'] < 10) ]
#res_correct = res1


    

In [5]:

    

#res_correct


    

In [6]:

    

#res_n_sig5 = res_correct[res_correct['Model info.n_sig'] == 5]

m_trapz_all = np.array(res_correct['Model info.m_trapz'].tolist())

mse_all = np.array(res_correct['Out info.MSE'].tolist())

mse_means = []
std_errs = []
h = 16

for i in range(119):
    ds = res_correct[0+h*i:h+h*i]
    mse_l = np.array(ds['Out info.MSE'].tolist())
    mse_means.append(np.mean(mse_l))
    std_errs.append(np.std(mse_l))

my_set = set(m_trapz_all)
m_trapz = sorted( list(my_set) ) 
#print(m_trapz)

figure = plt.figure(figsize=(12,8))
axes = figure.add_subplot (1, 1, 1)
plt.grid(True)
plt.title('MSE vs number of integration points. 5 sigmoids, 20 train points ', fontsize=15)
plt.xlabel('Number of integration points', fontsize=15)
plt.ylabel('MSE', fontsize=15)
#plt.scatter(m_trapz_all, mse_all, label = 'MSE for all points', marker = "D",s=40)

plt.plot(m_trapz, mse_means, color='black', marker='x', linestyle='dashed', linewidth=3, markersize=16, label = 'Mean MSE')
#plt.errorbar(m_trapz, mse_means, yerr=std_errs, ecolor='r', lw=2, capsize=15, mew = 3, zorder=3, label = 'Errorbar', linestyle='None')
axes.set_yscale ('log')#, nonposy='clip')
axes.set_xscale ('log')#, nonposy='clip')
plt.legend(loc=1, prop={'size': 25})
#print(std_errs)


    

    




/home/rodion/anaconda3/lib/python3.6/site-packages/numpy/core/fromnumeric.py:3118: RuntimeWarning: Mean of empty slice.
  out=out, **kwargs)
/home/rodion/anaconda3/lib/python3.6/site-packages/numpy/core/_methods.py:85: RuntimeWarning: invalid value encountered in double_scalars
  ret = ret.dtype.type(ret / rcount)
/home/rodion/anaconda3/lib/python3.6/site-packages/numpy/core/_methods.py:140: RuntimeWarning: Degrees of freedom <= 0 for slice
  keepdims=keepdims)
/home/rodion/anaconda3/lib/python3.6/site-packages/numpy/core/_methods.py:110: RuntimeWarning: invalid value encountered in true_divide
  arrmean, rcount, out=arrmean, casting='unsafe', subok=False)
/home/rodion/anaconda3/lib/python3.6/site-packages/numpy/core/_methods.py:132: RuntimeWarning: invalid value encountered in double_scalars
  ret = ret.dtype.type(ret / rcount)






    
Out[6]:





<matplotlib.legend.Legend at 0x7f09450328d0>






    

In [7]:

    

figure = plt.figure(figsize=(12,8))
axes = figure.add_subplot (1, 1, 1)
plt.grid(True)
plt.title('MSE vs number of integration points. 5 sigmoids, 20 train points ', fontsize=15)
plt.xlabel('Number of integration points', fontsize=15)
plt.ylabel('MSE', fontsize=15)
#plt.scatter(m_trapz_all, mse_all, label = 'MSE for all points', marker = "D",s=40)

plt.plot(m_trapz[:30], mse_means[:30], color='black', marker='x', linestyle='dashed', linewidth=3, markersize=16, label = 'Mean MSE')
plt.errorbar(m_trapz[:30], mse_means[:30], yerr=std_errs[:30], ecolor='r', lw=2, capsize=15, mew = 3, zorder=3, label = 'Std.error of MSE', linestyle='None')
axes.set_yscale ('log')#, nonposy='clip')
axes.set_xscale ('log')#, nonposy='clip')
plt.legend(loc=1, prop={'size': 25})
#print(std_errs)


    

    
Out[7]:





<matplotlib.legend.Legend at 0x7f09447ef668>






    

In [8]:

    

figure = plt.figure(figsize=(12,8))
axes = figure.add_subplot (1, 1, 1)
plt.grid(True)
plt.title('MSE vs number of integration points. 5 sigmoids, 20 train points ', fontsize=15)
plt.xlabel('Number of integration points', fontsize=15)
plt.ylabel('MSE', fontsize=15)
plt.scatter(m_trapz_all[:29*16], mse_all[:29*16], label = 'MSE for all points', marker = "D",s=40)

plt.plot(m_trapz[:30], mse_means[:30], color='black', marker='x', linestyle='dashed', linewidth=3, markersize=16, label = 'Mean MSE')
plt.errorbar(m_trapz[:30], mse_means[:30], yerr=std_errs[:30], ecolor='r', lw=2, capsize=15, mew = 3, zorder=3, label = 'Std.error of MSE', linestyle='None')
axes.set_yscale ('log')#, nonposy='clip')
#axes.set_xscale ('log')#, nonposy='clip')
plt.legend(loc=1, prop={'size': 25})
#print(std_errs)


    

    
Out[8]:





<matplotlib.legend.Legend at 0x7f09451c1f28>






    

In [9]:

    

m_trapz[:30]


    

    
Out[9]:





[2,
 3,
 4,
 5,
 6,
 7,
 8,
 9,
 10,
 11,
 12,
 13,
 14,
 15,
 16,
 17,
 18,
 19,
 20,
 21,
 22,
 23,
 24,
 25,
 26,
 27,
 28,
 29,
 30,
 31]

In [10]:

    

figure = plt.figure(figsize=(12,8))
axes = figure.add_subplot (1, 1, 1)
plt.grid(True)
plt.title('MSE vs number of integration points. 5 sigmoids, 20 train points ', fontsize=15)
plt.xlabel('Number of integration points', fontsize=15)
plt.ylabel('MSE', fontsize=15)
plt.scatter(m_trapz_all[30*16:-32], mse_all[30*16:-32], label = 'MSE for all points', marker = "D",s=40)

plt.plot(m_trapz[30:], mse_means[30:], color='black', marker='x', linestyle='dashed', linewidth=3, markersize=16, label = 'Mean MSE')
plt.errorbar(m_trapz[30:], mse_means[30:], yerr=std_errs[30:], ecolor='r', lw=2, capsize=15, mew = 3, zorder=3, label = 'Std.error of MSE', linestyle='None')
axes.set_yscale ('log')#, nonposy='clip')
#axes.set_xscale ('log')#, nonposy='clip')
plt.legend(loc=1, prop={'size': 25})
#print(std_errs)


    

    
Out[10]:





<matplotlib.legend.Legend at 0x7f0944dec5c0>






    

In [ ]:

    




    

In [ ]:

    




    

In [ ]:

    




    

In [11]:

    

figure = plt.figure(figsize=(12,8))
axes = figure.add_subplot (1, 1, 1)
plt.grid(True)
plt.title('MSE vs number of integration points. 5 sigmoids, 20 train points ', fontsize=15)
plt.xlabel('Number of integration points', fontsize=15)
plt.ylabel('MSE', fontsize=15)
#plt.scatter(m_trapz_all, mse_all, label = 'MSE for all points', marker = "D",s=40)

plt.plot(m_trapz[:15], mse_means[:15], color='black', marker='x', linestyle='dashed', linewidth=3, markersize=16, label = 'Mean MSE')
plt.errorbar(m_trapz[:15], mse_means[:15], yerr=std_errs[:15], ecolor='r', lw=2, capsize=15, mew = 3, zorder=3, label = 'Std.error of MSE', linestyle='None')
axes.set_yscale ('log')#, nonposy='clip')
axes.set_xscale ('log')#, nonposy='clip')
plt.legend(loc=1, prop={'size': 25})
#print(std_errs)


    

    
Out[11]:





<matplotlib.legend.Legend at 0x7f0944180208>






    

In [12]:

    

hui = 10
x = np.log10(m_trapz[:15])
y = np.log10(mse_means[:15])
der = []
for i in range(x.size-hui):
    der.append((y[i+1]-y[i])/(x[i+1]-x[i]))
    
    
rrr = np.zeros(shape = (np.array(der).size)) + 10
figure = plt.figure(figsize=(12,8))
axes = figure.add_subplot (1, 1, 1)
plt.grid(True)
plt.title('MSE vs number of integration points. 5 sigmoids, 20 train points ', fontsize=15)
plt.xlabel('Number of integration points', fontsize=15)
plt.ylabel('MSE', fontsize=15)
#plt.scatter(m_trapz_all, mse_all, label = 'MSE for all points', marker = "D",s=40)
    
plt.plot(m_trapz[:-hui], np.power(rrr,der), color='blue', marker='+', linestyle='dashed', linewidth=3, markersize=16, label = 'der MSE')   

plt.plot(m_trapz[:15-hui], mse_means[:15-hui], color='black', marker='x', linestyle='dashed', linewidth=3, markersize=16, label = 'Mean MSE')
plt.errorbar(m_trapz[:15-hui], mse_means[:15-hui], yerr=std_errs[:15-hui], ecolor='r', lw=2, capsize=15, mew = 3, zorder=3, label = 'Std.error of MSE', linestyle='None')
axes.set_yscale ('log')#, nonposy='clip')
axes.set_xscale ('log')#, nonposy='clip')
plt.legend(loc=1, prop={'size': 25})
#print(std_errs)


    

    




---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-12-64f2b48614f7> in <module>()
     16 #plt.scatter(m_trapz_all, mse_all, label = 'MSE for all points', marker = "D",s=40)
     17 
---> 18 plt.plot(m_trapz[:-hui], np.power(rrr,der), color='blue', marker='+', linestyle='dashed', linewidth=3, markersize=16, label = 'der MSE')
     19 
     20 plt.plot(m_trapz[:15-hui], mse_means[:15-hui], color='black', marker='x', linestyle='dashed', linewidth=3, markersize=16, label = 'Mean MSE')

/home/rodion/anaconda3/lib/python3.6/site-packages/matplotlib/pyplot.py in plot(*args, **kwargs)
   3356                       mplDeprecation)
   3357     try:
-> 3358         ret = ax.plot(*args, **kwargs)
   3359     finally:
   3360         ax._hold = washold

/home/rodion/anaconda3/lib/python3.6/site-packages/matplotlib/__init__.py in inner(ax, *args, **kwargs)
   1853                         "the Matplotlib list!)" % (label_namer, func.__name__),
   1854                         RuntimeWarning, stacklevel=2)
-> 1855             return func(ax, *args, **kwargs)
   1856 
   1857         inner.__doc__ = _add_data_doc(inner.__doc__,

/home/rodion/anaconda3/lib/python3.6/site-packages/matplotlib/axes/_axes.py in plot(self, *args, **kwargs)
   1525         kwargs = cbook.normalize_kwargs(kwargs, _alias_map)
   1526 
-> 1527         for line in self._get_lines(*args, **kwargs):
   1528             self.add_line(line)
   1529             lines.append(line)

/home/rodion/anaconda3/lib/python3.6/site-packages/matplotlib/axes/_base.py in _grab_next_args(self, *args, **kwargs)
    404                 this += args[0],
    405                 args = args[1:]
--> 406             for seg in self._plot_args(this, kwargs):
    407                 yield seg
    408 

/home/rodion/anaconda3/lib/python3.6/site-packages/matplotlib/axes/_base.py in _plot_args(self, tup, kwargs)
    381             x, y = index_of(tup[-1])
    382 
--> 383         x, y = self._xy_from_xy(x, y)
    384 
    385         if self.command == 'plot':

/home/rodion/anaconda3/lib/python3.6/site-packages/matplotlib/axes/_base.py in _xy_from_xy(self, x, y)
    240         if x.shape[0] != y.shape[0]:
    241             raise ValueError("x and y must have same first dimension, but "
--> 242                              "have shapes {} and {}".format(x.shape, y.shape))
    243         if x.ndim > 2 or y.ndim > 2:
    244             raise ValueError("x and y can be no greater than 2-D, but have "

ValueError: x and y must have same first dimension, but have shapes (109,) and (5,)





    

In [ ]:

    

x = np.log10(m_trapz[:15])
y = np.log10(mse_means[:15])


    

In [ ]:

    

np.power(rrr,der)


    

In [ ]:

    

der


    

In [ ]:

    

x = np.log10(m_trapz[:15])
y = np.log10(mse_means[:15])
der = []

hui = 10

for i in range(x.size-hui):
    der.append((y[i+1]-y[i])/(x[i+1]-x[i]))

plt.plot(x[:-hui], der)


    

In [ ]:

    

rrr = np.zeros(shape = (np.array(der).size)) + 10


    

In [27]:

    

hui = 4
x = np.log10(m_trapz[:15])
y = np.log10(mse_means[:15])
der = []
for i in range(x.size - hui):
    der.append((y[i+1]-y[i])/(x[i+1]-x[i]))
    

rrr = np.zeros(shape = (np.array(der).size)) + 10
figure = plt.figure(figsize=(12,8))
axes = figure.add_subplot (1, 1, 1)
plt.grid(True)
plt.title('Numerical derivative of MSE', fontsize=15)
plt.xlabel('Number of integration points', fontsize=15)
#plt.ylabel(, fontsize=15)
#plt.scatter(m_trapz_all, mse_all, label = 'MSE for all points', marker = "D",s=40)

#plt.scatter(m_trapz[4:15-hui], np.power(rrr,der), label = 'der', marker = "D", s=40)


plt.plot(m_trapz[:15-hui], np.power(rrr,der), color='blue', marker='x', linestyle='dashed', linewidth=0, markersize=16, label = 'der MSE')   

#plt.plot(m_trapz[:15-hui], mse_means[:15-hui], color='black', marker='x', linestyle='dashed', linewidth=3, markersize=16, label = 'Mean MSE')
#plt.errorbar(m_trapz[:15-hui], mse_means[:15-hui], yerr=std_errs[:15-hui], ecolor='r', lw=2, capsize=15, mew = 3, zorder=3, label = 'Std.error of MSE', linestyle='None')
axes.set_yscale ('log')#, nonposy='clip')
axes.set_xscale ('log')#, nonposy='clip')
#plt.legend(loc=1, prop={'size': 25})
#print(std_errs)


    

In [15]:

    

figure = plt.figure(figsize=(12,8))
axes = figure.add_subplot (1, 1, 1)
plt.grid(True)
plt.title('MSE vs number of integration points. 5 sigmoids, 20 train points ', fontsize=15)
plt.xlabel('Number of integration points', fontsize=15)
plt.ylabel('MSE', fontsize=15)
#plt.scatter(m_trapz_all, mse_all, label = 'MSE for all points', marker = "D",s=40)

plt.plot(m_trapz[:15], mse_means[:15], color='black', marker='x', linestyle='dashed', linewidth=3, markersize=16, label = 'Mean MSE')
plt.errorbar(m_trapz[:15], mse_means[:15], yerr=std_errs[:15], ecolor='r', lw=2, capsize=15, mew = 3, zorder=3, label = 'Std.error of MSE', linestyle='None')
#axes.set_yscale ('log')#, nonposy='clip')
#axes.set_xscale ('log')#, nonposy='clip')
plt.legend(loc=1, prop={'size': 25})
#print(std_errs)


    

    
Out[15]:





<matplotlib.legend.Legend at 0x7f0944367978>






    

In [16]:

    

x = m_trapz[:15]
y = mse_means[:15]
p = np.polyfit(m_trapz[:15], mse_means[:15], 3)

f = np.poly1d(p)

figure = plt.figure(figsize=(12,8))
axes = figure.add_subplot (1, 1, 1)
plt.grid(True)
plt.plot(x, y)
plt.plot(x, f(x))

#axes.set_yscale ('log')#, nonposy='clip')
#axes.set_xscale ('log')#, nonposy='clip')


    

    
Out[16]:





[<matplotlib.lines.Line2D at 0x7f0942506908>]






    

In [ ]:

    




    

In [ ]: