In [1]:

    

from pylab import *
import os.path
import glob
import ocrolib
import rnnscript,rnnlib
import argparse
from ocrolib.lineest import make_normalizer
from mpl_toolkits.mplot3d import Axes3D
from sklearn import manifold
from rnntests import *
from functools import *
import lstm; reload(lstm)
import lstm_c; reload(lstm_c)
figsize(12,8)


    

In [2]:

    

ns = 30
genseq = partial(generate_dmod,n=ns)
Ni = 2
Ns = 2
No = 1


    

In [3]:

    

print "c_lstm"
clstm = lstm_c.C_LSTM(Ni,Ns,No)
llayer = rnnscript.as_lstm2(clstm.net.hiddenLayers[0])
clstm.setLearningRate(1e-4,0.9)

errs = []
for i in range(40000):
    xs,ys = genseq()
    a = clstm.ctrain(xs,ys[:,0])
    err = sum((a-ys)**2)**.5
    errs.append(err)
    if i%10000==0:
        print i,mean(errs[-1000:])
        s = clstm.states()
        clf()
        subplot(121);
        plot(xs,linewidth=3,alpha=0.3,color='yellow')
        plot(ys,color='blue')
        plot(a.ravel(),linestyle='dashed',color='red')
        subplot(122); plot(s[:,0],s[:,1])
        draw()

test_xs,test_ys = genseq()
print "done"


    

    




c_lstm
0 2.72833125444
10000 2.5813184233
20000 2.57067867332
30000 2.53718237387
done






    

In [4]:

    

import lstm as pylstm; reload(pylstm)
import lstm_c; reload(lstm_c)

before = pylstm.LSTM1(Ni,Ns,No)
lstm_c.c_to_py(before,clstm.net)

clstm.setLearningRate(0.0,0.0)
clstm.trainer.learnRate = 0.0
clstm.trainer.momentum = 0.0
before.setLearningRate(0.0,0.0)


    

In [5]:

    

lstm_c.show_net(clstm.net)


    

In [6]:

    

a0 = clstm.ctrain(test_xs,test_ys[:,0],update=0)
s0 = clstm.states()
subplot(121);
plot(a0.ravel(),linewidth=5,color='red',alpha=0.2)
a1 = array(before.ctrain(test_xs,test_ys[:,0]))
s1 = array(before.states())
plot(a1.ravel(),linewidth=1,color='purple',linestyle='dashed')
subplot(122)
plot(s0[:,0],s0[:,1],linewidth=5,alpha=0.2,color='blue')
plot(s1[:,0],s1[:,1],color='green',linestyle='dashed')


    

    
Out[6]:





[<matplotlib.lines.Line2D at 0x4eb74d0>]






    

In [7]:

    

cnet = clstm.net
olayer = cnet.outputLayers[0]
deltas1 = lstm_c.sb2array(olayer.outputErrors)
deltas2 = lstm_c.sb2array(olayer.inputErrors)
plot(deltas1,linewidth=5,alpha=0.2,color='blue')
plot(deltas2,linewidth=5,alpha=0.3,color='cyan')
deltas3 = -array(before.ldeltas[-1])
plot(deltas3,color='green',linestyle='dashed')
deltas4 = -array(before.nets[1].dzspre)
plot(deltas4,color='black',linestyle='dashed')
print deltas4.shape


    

    




(30, 1)






    

In [8]:

    

clayer = rnnlib.as_lstm2(cnet.hiddenLayers[0])
olayer = cnet.outputLayers[0]
deltas1b = lstm_c.sb2array(clayer.outputErrors)
plot(deltas1b,linewidth=5,alpha=0.3,color='blue')
lr = before.nets[1]
deltas2b = lr.backward(deltas1)
plot(deltas2b,color='green',linestyle='dashed')


    

    
Out[8]:





[<matplotlib.lines.Line2D at 0x5843310>,
 <matplotlib.lines.Line2D at 0x54b1f10>]






    

In [9]:

    

m = {v.name:v for v in cnet.connections.values()}
h2o = m['hidden_0_0_to_output']
b2o = m['bias_to_output']
h2o.debug = True
b2o.debug = True
h2o.clear()
b2o.clear()
xs = arange(6).reshape(3,2)
ys = zeros(3)
clstm.ctrain(xs,ys,update=0)
h2o.debug = False
b2o.debug = False
print
print array(h2o.dinputs).reshape(3,2)
print array(h2o.dweights).reshape(3,2)
print
print array(b2o.dinputs).reshape(3)
print array(b2o.dweights).reshape(3)


    

    




[[-0.41343991 -0.11135256]
 [ 0.68053883  0.50121699]
 [ 0.76512815  0.72537126]]
[[ 2.79681971  1.77695312]
 [ 2.79681971  1.77695312]
 [ 2.79681971  1.77695312]]

[ 1.  1.  1.]
[ 0.22160357  0.22160357  0.22160357]

In [10]:

    

before.nets[1].W2


    

    
Out[10]:





array([[ 0.22160357,  2.79681971,  1.77695312]])

In [11]:

    

lstm_c.sb2array(h2o.frm.outputErrors)


    

    
Out[11]:





array([[ 0.81067991,  0.49461019],
       [ 2.70244646,  1.71964097],
       [ 2.72600174,  1.7319591 ]], dtype=float32)