In [41]:
# This line configures matplotlib to show figures embedded in the notebook, 
# instead of opening a new window for each figure. More about that later. 
# If you are using an old version of IPython, try using '%pylab inline' instead.
%matplotlib inline
from pylab import *
import matplotlib.pyplot as plt
import matplotlib
import numpy as np
from pylab import *
import matplotlib.pyplot as plt
%matplotlib inline

In [63]:
x=[5,0,0,0,2,3,2,0,2,2,2,3,1,1,1,15,1,5,1,0,3,4,3,11,0,7,0,1,1,0,0,1,0,6,8,4,0,2,4,2,0,1,1,2,4,3,1,0,6,2,0,2,3,4,2,6,1,2,1,0,0,4,0,0,2,1,6,0,3,0,4,7,6,2,0,4,2,1,6,0,0,1,1,5,11,2,1,2,3,0,0,0,4,2,3,0,1,0,1,1,1,2,1,6,9,9,2,0,1,4,7,2,2,1,3,2,0,1,1,7,1,7,0,0,2,5,4,0,6,2,0,2,5,7,6,6,2,0,3,1,2,4,1,1,2,0,2,1,1,2,0,0,3,3,2,3,6,0,4,0,2,8,6,1,2,5,3,2,0,1,0,0,0,0,3,1,4,2,0,0,7,6,2,0,1,1,0,0,2,0,3,7,1,0,1,1,2,1,0,8,0,4,1,5,4,2,0,0,0,0,0,9,2,0,5,4,0,1,0,0,4,0,2,1,2,2,0,3,3,3,6,2,4,1,0,1,3,2,4,0,7,2,0,4,1,1,1,6,1,19,3,1,1,2,3,4,2,1,1,2,1,4,1,0,2,0,3,0,0,3,0,0,5,2,4,1,1,1,0,4,4,0,7,0,1,4,1,4,1,1,0,1,3,13,0,0,3,1,3,0,0,2,5,1,0,2,3,0,0,1,3,0,4,1,5,0,0,1,2,0,0,3,0,2,0,11,2,4,3,1,4,0,2,0,0,1,3,1,1,0,2,9,0,1,4,0,2,0,10,7,4,2,0,1,1,1,0,5,4,1,0,2,5,1,1]
x=x[:120]
ll = []

xx= 4
for i in range(xx):
    ll.append(sum(x[len(x)/xx*i:len(x)/xx*(i+1)]))


plot(ll)


Out[63]:
[<matplotlib.lines.Line2D at 0x7effc3155a90>]

In [36]:
run 8_accuracy.py


vw ./wnew/AAPL-250-4000-2013-2013-tfidf-x-test.vw -t -i ./wnew/AAPL-250-4000-2013-2013-tfidf-x-model.vw -p ./wnew/AAPL-250-4000-2013-2013-tfidf-x-pred.vw -r ./wnew/AAPL-250-4000-2013-2013-tfidf-x-raw.vw
vw ./wnew/AAPL-250-4000-2010-2014-tfidf-x-test.vw -t -i ./wnew/AAPL-250-4000-2010-2014-tfidf-x-model.vw -p ./wnew/AAPL-250-4000-2010-2014-tfidf-x-pred.vw -r ./wnew/AAPL-250-4000-2010-2014-tfidf-x-raw.vw
vw ./wnew/AAPL-250-4000-2012-2012-x-test.vw -t -i ./wnew/AAPL-250-4000-2012-2012-x-model.vw -p ./wnew/AAPL-250-4000-2012-2012-x-pred.vw -r ./wnew/AAPL-250-4000-2012-2012-x-raw.vw
vw ./wnew/AAPL-250-4000-2013-2013-log-x-test.vw -t -i ./wnew/AAPL-250-4000-2013-2013-log-x-model.vw -p ./wnew/AAPL-250-4000-2013-2013-log-x-pred.vw -r ./wnew/AAPL-250-4000-2013-2013-log-x-raw.vw
vw ./wnew/AAPL-250-4000-2012-2012-tfidf-x-test.vw -t -i ./wnew/AAPL-250-4000-2012-2012-tfidf-x-model.vw -p ./wnew/AAPL-250-4000-2012-2012-tfidf-x-pred.vw -r ./wnew/AAPL-250-4000-2012-2012-tfidf-x-raw.vw
vw ./wnew/AAPL-250-4000-2014-2014-log-x-test.vw -t -i ./wnew/AAPL-250-4000-2014-2014-log-x-model.vw -p ./wnew/AAPL-250-4000-2014-2014-log-x-pred.vw -r ./wnew/AAPL-250-4000-2014-2014-log-x-raw.vw
vw ./wnew/AAPL-250-4000-2010-2014-log-x-test.vw -t -i ./wnew/AAPL-250-4000-2010-2014-log-x-model.vw -p ./wnew/AAPL-250-4000-2010-2014-log-x-pred.vw -r ./wnew/AAPL-250-4000-2010-2014-log-x-raw.vw
vw ./wnew/AAPL-250-4000-2013-2013-x-test.vw -t -i ./wnew/AAPL-250-4000-2013-2013-x-model.vw -p ./wnew/AAPL-250-4000-2013-2013-x-pred.vw -r ./wnew/AAPL-250-4000-2013-2013-x-raw.vw
vw ./wnew/AAPL-250-4000-2010-2010-log-x-test.vw -t -i ./wnew/AAPL-250-4000-2010-2010-log-x-model.vw -p ./wnew/AAPL-250-4000-2010-2010-log-x-pred.vw -r ./wnew/AAPL-250-4000-2010-2010-log-x-raw.vw
vw ./wnew/AAPL-250-4000-2014-2014-tfidf-x-test.vw -t -i ./wnew/AAPL-250-4000-2014-2014-tfidf-x-model.vw -p ./wnew/AAPL-250-4000-2014-2014-tfidf-x-pred.vw -r ./wnew/AAPL-250-4000-2014-2014-tfidf-x-raw.vw
vw ./wnew/AAPL-250-4000-2012-2012-log-x-test.vw -t -i ./wnew/AAPL-250-4000-2012-2012-log-x-model.vw -p ./wnew/AAPL-250-4000-2012-2012-log-x-pred.vw -r ./wnew/AAPL-250-4000-2012-2012-log-x-raw.vw
vw ./wnew/AAPL-250-4000-2011-2011-tfidf-x-test.vw -t -i ./wnew/AAPL-250-4000-2011-2011-tfidf-x-model.vw -p ./wnew/AAPL-250-4000-2011-2011-tfidf-x-pred.vw -r ./wnew/AAPL-250-4000-2011-2011-tfidf-x-raw.vw
vw ./wnew/AAPL-250-4000-2010-2010-x-test.vw -t -i ./wnew/AAPL-250-4000-2010-2010-x-model.vw -p ./wnew/AAPL-250-4000-2010-2010-x-pred.vw -r ./wnew/AAPL-250-4000-2010-2010-x-raw.vw
vw ./wnew/AAPL-250-4000-2010-2015-x-test.vw -t -i ./wnew/AAPL-250-4000-2010-2015-x-model.vw -p ./wnew/AAPL-250-4000-2010-2015-x-pred.vw -r ./wnew/AAPL-250-4000-2010-2015-x-raw.vw
vw ./wnew/AAPL-250-4000-2010-2014-x-test.vw -t -i ./wnew/AAPL-250-4000-2010-2014-x-model.vw -p ./wnew/AAPL-250-4000-2010-2014-x-pred.vw -r ./wnew/AAPL-250-4000-2010-2014-x-raw.vw
vw ./wnew/AAPL-250-4000-2010-2010-tfidf-x-test.vw -t -i ./wnew/AAPL-250-4000-2010-2010-tfidf-x-model.vw -p ./wnew/AAPL-250-4000-2010-2010-tfidf-x-pred.vw -r ./wnew/AAPL-250-4000-2010-2010-tfidf-x-raw.vw
vw ./wnew/AAPL-250-4000-2011-2011-x-test.vw -t -i ./wnew/AAPL-250-4000-2011-2011-x-model.vw -p ./wnew/AAPL-250-4000-2011-2011-x-pred.vw -r ./wnew/AAPL-250-4000-2011-2011-x-raw.vw
vw ./wnew/AAPL-250-4000-2014-2014-x-test.vw -t -i ./wnew/AAPL-250-4000-2014-2014-x-model.vw -p ./wnew/AAPL-250-4000-2014-2014-x-pred.vw -r ./wnew/AAPL-250-4000-2014-2014-x-raw.vw
vw ./wnew/AAPL-250-4000-2010-2015-log-x-test.vw -t -i ./wnew/AAPL-250-4000-2010-2015-log-x-model.vw -p ./wnew/AAPL-250-4000-2010-2015-log-x-pred.vw -r ./wnew/AAPL-250-4000-2010-2015-log-x-raw.vw
vw ./wnew/AAPL-250-4000-2011-2011-log-x-test.vw -t -i ./wnew/AAPL-250-4000-2011-2011-log-x-model.vw -p ./wnew/AAPL-250-4000-2011-2011-log-x-pred.vw -r ./wnew/AAPL-250-4000-2011-2011-log-x-raw.vw
vw ./wnew/AAPL-250-4000-2010-2015-tfidf-x-test.vw -t -i ./wnew/AAPL-250-4000-2010-2015-tfidf-x-model.vw -p ./wnew/AAPL-250-4000-2010-2015-tfidf-x-pred.vw -r ./wnew/AAPL-250-4000-2010-2015-tfidf-x-raw.vw
just pass

In [37]:
hist([i[1] for i in losses])


Out[37]:
(array([  5.35000000e+02,   5.22800000e+03,   1.33870000e+04,
          1.26510000e+04,   6.26200000e+03,   1.28200000e+03,
          1.81000000e+02,   2.30000000e+01,   4.30000000e+01,
          2.00000000e+00]),
 array([  0.       ,   1.8402004,   3.6804008,   5.5206012,   7.3608016,
          9.201002 ,  11.0412024,  12.8814028,  14.7216032,  16.5618036,
         18.402004 ]),
 <a list of 10 Patch objects>)

In [39]:
hist([i[0] for i in losses])


Out[39]:
(array([  1.52550000e+04,   1.07310000e+04,   7.48800000e+03,
          3.65400000e+03,   7.89000000e+02,   7.35000000e+02,
          5.64000000e+02,   2.34000000e+02,   1.41000000e+02,
          3.00000000e+00]),
 array([  1. ,   2.8,   4.6,   6.4,   8.2,  10. ,  11.8,  13.6,  15.4,
         17.2,  19. ]),
 <a list of 10 Patch objects>)

In [14]:
losses[5]


Out[14]:
[4.0, 4.7121329999999997]

In [40]:
d={}
for i in losses:
    x, y = i[0], i[1]
    try:
        d[x].append(y)
    except:
        d[x]=[]
        d[x].append(y)
        
lengths = [len(j) for i, j in d.items()]
#plot(lengths)
#print lengths[30]

from sklearn.metrics import mean_squared_error as accuracy
from sklearn.metrics import mean_absolute_error as absolute
from math import sqrt
from random import shuffle

ys = []

for i, j in d.items()[:30]:
    if len(j) > 27:
        shuffle(j)
        #j=j[:27]
        #ys.append(sqrt(absolute(np.zeros(len(j))+i, j)))
        ys.append(sum(j) / float(len(j)))
plot(ys)

xs= range(len(ys))
yys = [math.exp(-x) for x in xs]
plot(xs)


Out[40]:
[<matplotlib.lines.Line2D at 0x7effc8bb0190>]

In [110]:
plt.ion()
plt.show()

for i in range(3,4):
    hist(d[i][:141], label='class '+str(i), normed=False,bins=100)
    plt.draw()
    
plt.legend(loc='lower left')


Out[110]:
<matplotlib.legend.Legend at 0x7fca81f7aed0>

In [94]:
plot([np.histogram(d[i][:141],normed =False)[0][-1] for i, j in d.items()[:30]])


Out[94]:
[<matplotlib.lines.Line2D at 0x7fca82e8c910>]

In [21]:
#x = np.linspace(0, 3*np.pi, 500)
x = np.array([i[0] for i in losses if i[0] != 0])
y = np.array([i[1] for i in losses if i[0] != 0])

fig, axes = plt.subplots(figsize=(16,16))
axes.scatter(x, y)


Out[21]:
<matplotlib.collections.PathCollection at 0x7fca8d6a8350>

In [67]:
len(x)


---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-67-e62f33f06233> in <module>()
----> 1 len(x)

TypeError: object of type 'numpy.float64' has no len()

In [10]:
len(y)


Out[10]:
378408

In [69]:
len(d[1])


Out[69]:
18474

In [33]:
import time
import numpy as np
import matplotlib.pyplot as plt

plt.axis([0, 10, 0, 1])
plt.ion()
plt.show()

for i in range(10):
    y = np.random.random()
    plt.scatter(i, y)
    plt.draw()
    time.sleep(0.05)



In [18]:
import math

math.exp(-2)


Out[18]:
0.1353352832366127

In [ ]:


In [38]:
changes = []
runs = []

pos = 0
for idx in xrange(1, len(maxes[:,0])-1):
        previous = maxes[idx-1][0]
        current = maxes[idx][0]

        if previous > current:
            changes.append(idx-1)
            runs.append(idx-1-pos)
            pos = idx-1


---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-38-f8e45ee77a73> in <module>()
      3 
      4 pos = 0
----> 5 for idx in xrange(1, len(maxes[:,0])-1):
      6         previous = maxes[idx-1][0]
      7         current = maxes[idx][0]

NameError: name 'maxes' is not defined