In [40]:

    

#!export PYTHONPATH=./:$PYTHONPATH
import librnn
from librnn.pylearn2.datasets.music import MusicSequence


    

In [41]:

    

dataset = MusicSequence('jsb')
testset = MusicSequence('jsb', which_set='test')


    

In [42]:

    

for d,b in dataset.iterator(1, dataset.get_num_examples(), mode='sequential',
                 data_specs=dataset.get_data_specs()):
    d
    #print len(d)
    #print d.shape
    #print b.shape


    

In [4]:

    

from pylearn2.sandbox.rnn.models.rnn import RNN, GRU, Recurrent
from pylearn2.models.mlp import Sigmoid, CompositeLayer, Linear, MLP, Tanh, CompositeLayer, FlattenerLayer
from pylearn2.space import VectorSpace, CompositeSpace
from pylearn2.sandbox.rnn.space import SequenceDataSpace


    

In [5]:

    

dimZ = 10
dimX = 96
bsize = 1
dimH = 24
IH_model = MLP(
    batch_size=bsize,
    input_space=CompositeSpace(
        [VectorSpace(dimX), VectorSpace(dimZ)]
        ),
    input_source=['x', 'z'],
    layers=[
    #Tanh(layer_name='IH', dim=400+dimZ, irange=.01),
    FlattenerLayer(CompositeLayer(
        layer_name='rec',
        layers=[
            Tanh(layer_name='IHx', dim=dimH/2, irange=.01),
            Tanh(layer_name='IHz', dim=dimH/2, irange=.01)
        ]
        )),
    CompositeLayer(layer_name='mz and vz',
        layers=[
            Linear(layer_name='mz', dim=20, irange=.01),
            Linear(layer_name='vz', dim=20, irange=.01)
        ]
        )
    ]
)

HI_model = MLP(batch_size=bsize, nvis=dimZ, layers=[
    Tanh(layer_name='H_decoder', dim=dimH, irange=.01),
    Sigmoid(layer_name='x_hat', dim=dimX, irange=.01)
])


    

In [53]:

    

import vpcn_scan; reload(vpcn_scan)
from vpcn_scan import VPCN
import numpy as np
import time
import gzip, cPickle
from theano import tensor as T
import top; reload(top)

L = 1
learning_rate = .0001

encoder = vpcn_scan.VPCN(IH_model, HI_model, T.tensor3('x'), 'rmsprop', learning_rate)
encoder.initParams()
encoder.createGradientFunctions()

print "Initializing weights and biases"
lowerbound = np.array([])
testlowerbound = np.array([])

begin = time.time()
for j in xrange(50):
    encoder.lowerbound = 0
    print 'Iteration:', j
    encoder.iterate(dataset)
    end = time.time()
    print("Iteration %d, lower bound = %.2f,"
          " time = %.2fs"
          % (j, encoder.lowerbound*bsize/dataset.get_num_examples(), end - begin))
    begin = end

    if j % 5 == 0:
        #print "### Calculating test lowerbound "
        testlowerbound = np.append(testlowerbound,encoder.getLowerBound(testset))
        print "### Test lowerbound: %f ###" % testlowerbound[-1]
        print ">>> learning rate: %f" % encoder.opt.lr.get_value()


    

    




Initializing weights and biases
Iteration: 0
$> Compiling optimizer.
Iteration 0, lower bound = 568.13, time = 54.61s
### Test lowerbound: 580.448331 ###
>>> learning rate: 0.000100
Iteration: 1
Iteration 1, lower bound = 570.38, time = 28.01s
Iteration: 2
Iteration 2, lower bound = 568.27, time = 26.76s
Iteration: 3
Iteration 3, lower bound = 569.34, time = 26.74s
Iteration: 4
Iteration 4, lower bound = 568.48, time = 26.80s
Iteration: 5
Iteration 5, lower bound = 568.67, time = 26.73s
### Test lowerbound: 580.084088 ###
>>> learning rate: 0.000100
Iteration: 6
Iteration 6, lower bound = 569.07, time = 27.73s
Iteration: 7
Iteration 7, lower bound = 568.77, time = 26.86s
Iteration: 8
Iteration 8, lower bound = 568.47, time = 26.86s
Iteration: 9
Iteration 9, lower bound = 568.43, time = 26.92s
Iteration: 10
Iteration 10, lower bound = 568.38, time = 26.85s
### Test lowerbound: 578.504301 ###
>>> learning rate: 0.000100
Iteration: 11
Iteration 11, lower bound = 568.70, time = 27.79s
Iteration: 12
Iteration 12, lower bound = 569.66, time = 26.87s
Iteration: 13
Iteration 13, lower bound = 568.56, time = 26.81s
Iteration: 14
Iteration 14, lower bound = 568.59, time = 26.68s
Iteration: 15
Iteration 15, lower bound = 568.26, time = 26.67s
### Test lowerbound: 579.365832 ###
>>> learning rate: 0.000100
Iteration: 16
Iteration 16, lower bound = 568.51, time = 27.57s
Iteration: 17
Iteration 17, lower bound = 568.25, time = 26.62s
Iteration: 18
Iteration 18, lower bound = 568.16, time = 26.65s
Iteration: 19
Iteration 19, lower bound = 568.75, time = 26.64s
Iteration: 20
Iteration 20, lower bound = 568.35, time = 26.64s
### Test lowerbound: 579.728060 ###
>>> learning rate: 0.000100
Iteration: 21
Iteration 21, lower bound = 568.69, time = 27.66s
Iteration: 22
Iteration 22, lower bound = 568.43, time = 26.72s
Iteration: 23
Iteration 23, lower bound = 569.32, time = 26.75s
Iteration: 24
Iteration 24, lower bound = 568.67, time = 26.78s
Iteration: 25
Iteration 25, lower bound = 568.92, time = 26.81s
### Test lowerbound: 578.751264 ###
>>> learning rate: 0.000100
Iteration: 26
Iteration 26, lower bound = 568.40, time = 27.70s
Iteration: 27
Iteration 27, lower bound = 568.93, time = 26.73s
Iteration: 28
Iteration 28, lower bound = 569.28, time = 26.75s
Iteration: 29
Iteration 29, lower bound = 569.29, time = 26.76s
Iteration: 30
Iteration 30, lower bound = 567.99, time = 26.72s
### Test lowerbound: 580.887666 ###
>>> learning rate: 0.000100
Iteration: 31
Iteration 31, lower bound = 569.44, time = 27.61s
Iteration: 32
Iteration 32, lower bound = 567.83, time = 26.76s
Iteration: 33
Iteration 33, lower bound = 568.98, time = 26.73s
Iteration: 34
Iteration 34, lower bound = 568.56, time = 26.72s
Iteration: 35
Iteration 35, lower bound = 567.53, time = 26.72s
### Test lowerbound: 580.029139 ###
>>> learning rate: 0.000100
Iteration: 36
Iteration 36, lower bound = 568.90, time = 27.63s
Iteration: 37
Iteration 37, lower bound = 568.56, time = 26.69s
Iteration: 38
Iteration 38, lower bound = 569.33, time = 26.67s
Iteration: 39
Iteration 39, lower bound = 568.12, time = 26.69s
Iteration: 40
Iteration 40, lower bound = 568.48, time = 26.69s
### Test lowerbound: 579.070953 ###
>>> learning rate: 0.000100
Iteration: 41
Iteration 41, lower bound = 568.40, time = 27.64s
Iteration: 42
Iteration 42, lower bound = 568.97, time = 26.74s
Iteration: 43
Iteration 43, lower bound = 568.29, time = 26.73s
Iteration: 44
Iteration 44, lower bound = 568.47, time = 26.94s
Iteration: 45
Iteration 45, lower bound = 569.51, time = 26.89s
### Test lowerbound: 579.536246 ###
>>> learning rate: 0.000100
Iteration: 46
Iteration 46, lower bound = 568.52, time = 27.66s
Iteration: 47
Iteration 47, lower bound = 568.66, time = 26.70s
Iteration: 48
Iteration 48, lower bound = 568.33, time = 26.75s
Iteration: 49
Iteration 49, lower bound = 568.88, time = 26.66s

In [54]:

    

%matplotlib inline
from pylab import imshow, plot
imshow(-d[:,0,:].T, cmap='gray')


    

    
Out[54]:





<matplotlib.image.AxesImage at 0x7fed86ca68d0>






    

In [55]:

    

import theano
def decoder(HI_model):
    Z = T.tensor3()
    def step(z):
        return HI_model.fprop(z)
    X,_ = theano.scan(step, sequences=[Z])
    return theano.function([Z], X)

def encoder(IH_model, dimZ, deterministic=True):
    X     = T.tensor3()
    bsize = IH_model.batch_size
    def step(x, z_t):
        m, ls = IH_model.fprop((x, z_t))
        ls = 0.5*ls
        if deterministic:
            z = m
        else:
            assert 0
        return z
    zero = np.zeros((bsize, dimZ)).astype('float32')
    zero = theano.shared(zero,'0')
    Z,_ = theano.scan(step, sequences=[X], outputs_info=[zero])

    return theano.function([X], Z)

fdecoder = decoder(HI_model)
fencoder = encoder(IH_model, 20)


    

In [56]:

    

X = fdecoder(np.random.normal(0,1,(100,1,20)).astype('float32'))
Z = fencoder(d)
Xh = fdecoder(Z)


    

In [59]:

    

_ = plot(Z[:,0,:].T)


    

In [64]:

    

_ = imshow(-(Xh[:,0,:]>.5).T,cmap='gray')


    

In [ ]: