In [1]:

    

import pickle as pkl
import numpy as np
import theano
import theano.tensor as T
import lasagne 
from lasagne.layers import InputLayer, DenseLayer, RecurrentLayer, NonlinearityLayer, ReshapeLayer, get_output, get_all_params, get_all_param_values, ElemwiseSumLayer
import ctc_cost
from time import time
from TIMIT_utils import index2char_TIMIT
from special_activations import clipped_relu
import sys


    

    




/usr/local/lib/python2.7/dist-packages/theano/tensor/signal/downsample.py:6: UserWarning: downsample module has been moved to the theano.tensor.signal.pool module.
  "downsample module has been moved to the theano.tensor.signal.pool module.")

In [2]:

    

f = open('TIMIT_data_prepared_for_CTC.pkl','rb')
data = pkl.load(f)
inp = data['x']
inp1 = data['inputs']
msk = data['mask']
tgt = data['y_indices']
char = data['chars']


    

In [3]:

    

input_size = len(inp1[0][0])
hidden_size = 50
num_output_classes = len(char)
learning_rate = 0.001
output_size = num_output_classes+1
batch_size = None
input_seq_length = None
gradient_clipping = 5


    

In [5]:

    

y = T.imatrix('targets')


    

In [6]:

    

l_in = InputLayer(shape=(batch_size, input_seq_length, input_size))
n_batch, n_time_steps, n_features = l_in.input_var.shape #Unnecessary in this version. Just collecting the info so that we can reshape the output back to the original shape
# h_1 = DenseLayer(l_in, num_units=hidden_size, nonlinearity=clipped_relu)
l_rec_forward = RecurrentLayer(l_in, num_units=hidden_size, grad_clipping=gradient_clipping, nonlinearity=clipped_relu)
l_rec_backward = RecurrentLayer(l_in, num_units=hidden_size, grad_clipping=gradient_clipping, nonlinearity=clipped_relu, backwards=True)
l_rec_accumulation = ElemwiseSumLayer([l_rec_forward,l_rec_backward])
l_rec_reshaped = ReshapeLayer(l_rec_accumulation, (-1,hidden_size))
l_h2 = DenseLayer(l_rec_reshaped, num_units=hidden_size, nonlinearity=clipped_relu)
l_out = DenseLayer(l_h2, num_units=output_size, nonlinearity=lasagne.nonlinearities.linear)
l_out_reshaped = ReshapeLayer(l_out, (n_batch, n_time_steps, output_size))#Reshaping back
l_out_softmax = NonlinearityLayer(l_out, nonlinearity=lasagne.nonlinearities.softmax)
l_out_softmax_reshaped = ReshapeLayer(l_out_softmax, (n_batch, n_time_steps, output_size))


    

In [7]:

    

output_logits = get_output(l_out_reshaped)
output_softmax = get_output(l_out_softmax_reshaped)


    

In [8]:

    

all_params = get_all_params(l_out,trainable=True)
# print all_params==[l_rec.W_in_to_hid, l_rec.b, l_rec.W_hid_to_hid, l_out.W, l_out.b]


    

In [9]:

    

print 'Number of trainable parameters =', len(all_params)
print all_params==[l_rec_forward.W_in_to_hid, l_rec_forward.b, l_rec_forward.W_hid_to_hid, l_rec_backward.W_in_to_hid, l_rec_backward.b, l_rec_backward.W_hid_to_hid, l_h2.W, l_h2.b, l_out.W, l_out.b]


    

    




Number of trainable parameters = 10
True

In [10]:

    

pseudo_cost = ctc_cost.pseudo_cost(y, output_logits)


    

In [11]:

    

pseudo_cost_grad = T.grad(pseudo_cost.sum() / n_batch, all_params)


    

In [12]:

    

true_cost = ctc_cost.cost(y, output_softmax)
cost = T.mean(true_cost)


    

In [14]:

    

shared_learning_rate = theano.shared(lasagne.utils.floatX(0.01))
updates = lasagne.updates.rmsprop(pseudo_cost_grad, all_params, learning_rate=learning_rate)


    

In [15]:

    

theano.config.exception_verbosity='high'
train = theano.function([l_in.input_var,y], [output_logits, output_softmax, cost, pseudo_cost], updates=updates)


    

In [16]:

    

inp0 = inp1[0]
inp00= np.asarray([inp0],dtype=theano.config.floatX)
tgt0 = np.asarray(tgt[0],dtype=np.int16)
tgt00 = np.asarray([tgt0])
print inp00.shape, tgt00.shape


    

    




(1, 278, 39) (1, 52)

In [19]:

    

num_epochs = 100
# num_training_samples = len(inp1)
num_training_samples = 1
for epoch in range(num_epochs):
    t = time()
    cost = 0
    failures = []
    for i in range(num_training_samples):
        curr_inp = inp1[i]
#         curr_msk = msk[i].astype(np.bool)
#         curr_inp = curr_inp[curr_msk]
        curr_inp = np.asarray([curr_inp],dtype=theano.config.floatX)
        curr_tgt = np.asarray(tgt[i],dtype=np.int16)
        curr_tgt = np.asarray([curr_tgt])
        try:
            _,_,c,_=train(curr_inp,curr_tgt)
            cost += c
        except IndexError:
            failures.append(i)
            print 'Current input seq: ', curr_inp
            print 'Current output seq: ', curr_tgt
            sys.exit(IndexError)
    print 'Epoch: ', epoch, 'Cost: ', float(cost/(num_training_samples-len(failures))), ', time taken =', time()-t
#     print 'Exceptions: ', len(failures), 'Total examples: ', num_training_samples
    if epoch%10==0:        
        #Save the model
        np.savez('CTC_model_under_test.npz', *get_all_param_values(l_out_softmax_reshaped, trainable=True))
        for i in range(2):
            curr_inp = inp1[i]
            curr_inp = np.asarray([curr_inp],dtype=theano.config.floatX)
            curr_tgt = np.asarray(tgt[i],dtype=np.int16)
            curr_out = output_softmax.eval({l_in.input_var:curr_inp})
            print 'Predicted:', index2char_TIMIT(np.argmax(curr_out, axis=2)[0])
            print 'Target:', index2char_TIMIT(curr_tgt)


    

    




Epoch:  0 Cost:  695.258914005 , time taken = 0.464868068695
Predicted: ______________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________________
Target: she had your dark suit in greasy wash water all year
Predicted: _________________________________________________________________________________________________________________________________________________________
Target: i saw your horse outside
Epoch:  1 Cost:  342.928594065 , time taken = 0.416613101959
Epoch:  2 Cost:  179.489504172 , time taken = 0.405362129211
Epoch:  3 Cost:  170.353690783 , time taken = 0.440174102783
Epoch:  4 Cost:  163.192402963 , time taken = 0.405067920685
Epoch:  5 Cost:  156.962595127 , time taken = 0.410233020782
Epoch:  6 Cost:  150.732229863 , time taken = 0.410019159317
Epoch:  7 Cost:  144.213916376 , time taken = 0.402567863464
Epoch:  8 Cost:  137.57019083 , time taken = 0.404509067535
Epoch:  9 Cost:  131.081858182 , time taken = 0.413440942764
Epoch:  10 Cost:  124.487711411 , time taken = 0.414126157761
Predicted: ____________________________________________________________o_______________________ ___________________________________ _________________________________________________________________a____ _____________at_______________________________________________________________________
Target: she had your dark suit in greasy wash water all year
Predicted: _________________________________________ ___a________________________________________________a___ _____________________a________________________________
Target: i saw your horse outside
Epoch:  11 Cost:  117.754391567 , time taken = 0.447767972946
Epoch:  12 Cost:  110.836461606 , time taken = 0.428574085236
Epoch:  13 Cost:  105.898919285 , time taken = 0.416472911835
Epoch:  14 Cost:  107.680783819 , time taken = 0.422326803207
Epoch:  15 Cost:  106.533532606 , time taken = 0.419183969498
Epoch:  16 Cost:  100.035885964 , time taken = 0.418770074844
Epoch:  17 Cost:  87.1408354663 , time taken = 0.420788049698
Epoch:  18 Cost:  79.5227705391 , time taken = 0.425096035004
Epoch:  19 Cost:  73.4768237044 , time taken = 0.41675901413
Epoch:  20 Cost:  68.2099057461 , time taken = 0.411364793777
Predicted: _______________________________________________ _____y______ou__ d__________________ _________________________uu__i____t i______________________ ______________re_______a______y_____ ___aa___h ______w______at________________ __a_ __l _______________________________e_a___________
Target: she had your dark suit in greasy wash water all year
Predicted: _re__a___ar____a_______________________r_ ___ _______________________________________ ________a___ ____r ______r________ _______ _______________ ________
Target: i saw your horse outside
Epoch:  21 Cost:  63.5831362397 , time taken = 0.421214103699
Epoch:  22 Cost:  59.3020642967 , time taken = 0.406807899475
Epoch:  23 Cost:  55.4734078076 , time taken = 0.422957181931
Epoch:  24 Cost:  52.488289869 , time taken = 0.420311927795
Epoch:  25 Cost:  48.9821892326 , time taken = 0.412256002426
Epoch:  26 Cost:  45.9154323602 , time taken = 0.408935070038
Epoch:  27 Cost:  40.6925957131 , time taken = 0.417082071304
Epoch:  28 Cost:  35.7549082236 , time taken = 0.404206037521
Epoch:  29 Cost:  31.5326324974 , time taken = 0.412347793579
Epoch:  30 Cost:  27.8280499631 , time taken = 0.414273023605
Predicted: sh___ h____a___________________________________ _____y______our_ d___a__________r__k _________________________su__i____t i__________n___________ __g___________re_______a_____sy_____ ___wa__sh ______w______at________e____rr_ __all__l __________________________y____e_a_____r_____
Target: she had your dark suit in greasy wash water all year
Predicted: _re__a___ar____a____________________g__r_r___ _____________________e_________________ ________a___ ____r ______r_g____g_ _______ _______________ _s______
Target: i saw your horse outside
Epoch:  31 Cost:  24.8733769688 , time taken = 0.417771100998
Epoch:  32 Cost:  22.337057492 , time taken = 0.412835121155
Epoch:  33 Cost:  20.3623817962 , time taken = 0.427484989166
Epoch:  34 Cost:  18.4281522935 , time taken = 0.414781093597
Epoch:  35 Cost:  17.1687421698 , time taken = 0.417280912399
Epoch:  36 Cost:  15.6881284507 , time taken = 0.424530982971
Epoch:  37 Cost:  14.9465407564 , time taken = 0.4105489254
Epoch:  38 Cost:  13.4430649052 , time taken = 0.410006046295
Epoch:  39 Cost:  12.6493225725 , time taken = 0.416280984879
Epoch:  40 Cost:  11.4918678028 , time taken = 0.414304971695
Predicted: sh__e h____a___________________________________ _____y______our_ d___a__________r__k _________________________su__i____t i__________n___________ __g___________re_______a_____sy_____ ___wa__sh ______w______at________e____rr_ __all__l __________________________y____e_a_____r_____
Target: she had your dark suit in greasy wash water all year
Predicted: _re_____e rh___a____________________g__r_r___ _____________________e_ _______________ ________a___ _____ _____wr_g____g_ _____ _ _______________ ls______
Target: i saw your horse outside
Epoch:  41 Cost:  10.4982937984 , time taken = 0.420031070709
Epoch:  42 Cost:  9.33870641798 , time taken = 0.401381015778
Epoch:  43 Cost:  8.5642309527 , time taken = 0.39977812767
Epoch:  44 Cost:  7.6467728752 , time taken = 0.40585398674
Epoch:  45 Cost:  7.02417568064 , time taken = 0.409202814102
Epoch:  46 Cost:  6.21159696292 , time taken = 0.412712812424
Epoch:  47 Cost:  5.79530629742 , time taken = 0.412620782852
Epoch:  48 Cost:  5.10275867153 , time taken = 0.4261739254
Epoch:  49 Cost:  4.70748777423 , time taken = 0.417473077774
Epoch:  50 Cost:  4.2161527069 , time taken = 0.418891906738
Predicted: sh__e h____a____________________d______________ _____y______our_ d___a__________r__k _________________________su__i____t i__________n___________ __g___________re_______a_____sy_____ ___wa__sh ______w______at________e____rr_ __all__l __________________________y____e_a_____r_____
Target: she had your dark suit in greasy wash water all year
Predicted: _re_____e rh___a_______________________r_r___ _____________________e_ _______________ ________a___ _____ _____wr_g____g_y_____ _ ________________gs______
Target: i saw your horse outside
Epoch:  51 Cost:  3.92090622112 , time taken = 0.4078810215
Epoch:  52 Cost:  3.53588133573 , time taken = 0.411890983582
Epoch:  53 Cost:  3.27719304612 , time taken = 0.410335063934
Epoch:  54 Cost:  2.99158167047 , time taken = 0.411561965942
Epoch:  55 Cost:  2.80235490122 , time taken = 0.416101932526
Epoch:  56 Cost:  2.57297723797 , time taken = 0.407879114151
Epoch:  57 Cost:  2.41938355212 , time taken = 0.401879787445
Epoch:  58 Cost:  2.22885560338 , time taken = 0.399511098862
Epoch:  59 Cost:  2.1265490806 , time taken = 0.399022102356
Epoch:  60 Cost:  1.92352310151 , time taken = 0.407444953918
Predicted: sh__e h____a____________________d______________ _____y______our_ d___a__________r__k _________________________su__i____t i__________n___________ __g___________re_______a_____sy_____ ___wa__sh ______w______at________e____rr_ __all__l __________________________y____e_a_____r_____
Target: she had your dark suit in greasy wash water all year
Predicted: _re_h___e rh___a_______________________r_r___ _____________________e_ _______________ ________a___ _____ _____wr_g____g_y_____ _ ________________gs______
Target: i saw your horse outside
Epoch:  61 Cost:  1.80484279124 , time taken = 0.411237001419
Epoch:  62 Cost:  1.65171381186 , time taken = 0.398931980133
Epoch:  63 Cost:  1.56257511647 , time taken = 0.423418998718
Epoch:  64 Cost:  1.41884125149 , time taken = 0.407498121262
Epoch:  65 Cost:  1.3246180081 , time taken = 0.403403043747
Epoch:  66 Cost:  1.21549038112 , time taken = 0.416229963303
Epoch:  67 Cost:  1.14304921779 , time taken = 0.412912845612
Epoch:  68 Cost:  1.05509660723 , time taken = 0.417524814606
Epoch:  69 Cost:  0.995153813729 , time taken = 0.406607151031
Epoch:  70 Cost:  0.921401154805 , time taken = 0.402260065079
Predicted: sh__e h____a____________________d______________ _____y______our_ d___a__________r__k _________________________su__i____t i__________n___________ __g___________re_______a_____sy_____ ___wa__sh ______w______at________e____rr_ __all__l __________________________y____e_a_____r_____
Target: she had your dark suit in greasy wash water all year
Predicted: _re_h___e rh___a_______________________h_r___ _____________________e_ _______________ ________a___ _____ _____wg_g____g_y_____ _ ________________gs______
Target: i saw your horse outside
Epoch:  71 Cost:  0.869646956452 , time taken = 0.445788145065
Epoch:  72 Cost:  0.803373195665 , time taken = 0.420474052429
Epoch:  73 Cost:  0.758740814343 , time taken = 0.416527032852
Epoch:  74 Cost:  0.702201068917 , time taken = 0.415088891983
Epoch:  75 Cost:  0.664818493953 , time taken = 0.413058996201
Epoch:  76 Cost:  0.611278627215 , time taken = 0.415685176849
Epoch:  77 Cost:  0.576907684605 , time taken = 0.417546987534
Epoch:  78 Cost:  0.534437784294 , time taken = 0.419013023376
Epoch:  79 Cost:  0.508590999047 , time taken = 0.415677070618
Epoch:  80 Cost:  0.466608813544 , time taken = 0.421269893646
Predicted: sh__e h____a____________________d______________ _____y______our_ d___a__________r__k _________________________su__i____t i__________n___________ __g___________re_______a_____sy_____ ___wa__sh ______w______at________e____rr_ __all__l __________________________y____e_a_____r_____
Target: she had your dark suit in greasy wash water all year
Predicted: _re_h___e rh___a_______________________h_r___ _____________________e_ _l_____________ ________a___ _____ _____wg_g____g_y_______ ________________gs______
Target: i saw your horse outside
Epoch:  81 Cost:  0.440707552881 , time taken = 0.417589902878
Epoch:  82 Cost:  0.407251542954 , time taken = 0.422451972961
Epoch:  83 Cost:  0.386693420567 , time taken = 0.424895048141
Epoch:  84 Cost:  0.358527518907 , time taken = 0.417057991028
Epoch:  85 Cost:  0.339120320275 , time taken = 0.421405076981
Epoch:  86 Cost:  0.313970282001 , time taken = 0.413981199265
Epoch:  87 Cost:  0.297925901041 , time taken = 0.424841880798
Epoch:  88 Cost:  0.276489075815 , time taken = 0.41254401207
Epoch:  89 Cost:  0.264080459513 , time taken = 0.415322065353
Epoch:  90 Cost:  0.24550561974 , time taken = 0.410289049149
Predicted: sh__e h____a____________________d______________ _____y______our_ d___a__________r__k _________________________su__i____t i__________n___________ __g___________re_______a_____sy_____ ___wa__sh ______w______at________e____rr_ __all__l __________________________y____e_a_____r_____
Target: she had your dark suit in greasy wash water all year
Predicted: _re_h___e rh___a_______________________h_r___ _____________________e_ _l_____________ ________a___ _____ _____wg_g____g_y____a _ ________________gs______
Target: i saw your horse outside
Epoch:  91 Cost:  0.234048768906 , time taken = 0.406760931015
Epoch:  92 Cost:  0.216640805133 , time taken = 0.412416934967
Epoch:  93 Cost:  0.205755959314 , time taken = 0.412768125534
Epoch:  94 Cost:  0.189948261922 , time taken = 0.423375844955
Epoch:  95 Cost:  0.181333456368 , time taken = 0.409070968628
Epoch:  96 Cost:  0.167968282014 , time taken = 0.401962995529
Epoch:  97 Cost:  0.160850249575 , time taken = 0.412121772766
Epoch:  98 Cost:  0.14895047728 , time taken = 0.411957979202
Epoch:  99 Cost:  0.142852759076 , time taken = 0.41521692276

In [20]:

    




    

In [21]: