In [18]:
from fuel.datasets import Dataset
from librnn.pylearn2.datasets.music import MusicSequence
from blocks.bricks import Sigmoid, Tanh, MLP, Linear, Rectifier
from blocks.bricks.recurrent import SimpleRecurrent, GatedRecurrent, LSTM
from blocks.bricks import recurrent
from blocks.initialization import Orthogonal, IsotropicGaussian, Constant
from blocks.algorithms import GradientDescent, Scale, Adam
from fuel.streams import DataStream
from fuel.schemes import ConstantScheme
from theano import tensor
from blocks.bricks import WEIGHTS, BIASES
from blocks.graph import ComputationGraph
from blocks.filter import VariableFilter
from blocks.main_loop import MainLoop
from blocks.model import Model
from blocks.extensions import FinishAfter, Printing
#from blocks.extensions.saveload import SerializeMainLoop
from blocks.extensions.monitoring import TrainingDataMonitoring, DataStreamMonitoring
from blocks.bricks.cost import BinaryCrossEntropy
from blocks_contrib.bricks.recurrent import DelayLine
In [19]:
class MusicFuel(Dataset):
def __init__(self, which_set, which_dataset):
self.pylearn2_dataset = MusicSequence(which_set=which_set, which_dataset=which_dataset)
self.sources = self.pylearn2_dataset.get_data_specs()[-1]
def open(self):
num_examples = self.pylearn2_dataset.get_num_examples()
return self.pylearn2_dataset.iterator(1, num_examples, mode='sequential',
data_specs=self.pylearn2_dataset.get_data_specs(), return_tuple=True)
def get_data(self,state=None,request=None):
return next(state)
In [17]:
import theano
x = tensor.tensor3('features')
input_to_state = recurrent.Linear(name='input_to_state', input_dim=96, output_dim=48)
x_transform = input_to_state.apply(x)
mem_dim = 5
#RNN = DelayLine(48, mem_dim)
RNN = SimpleRecurrent(activation=Tanh(), dim=48, weights_init=Orthogonal(), name="RNN")
h = RNN.apply(x_transform) # h is hidden states in the RNN
state_to_output = Linear(name='state_to_output', input_dim=48, output_dim=96)
pre_out = state_to_output.apply(h)
y_hat = Sigmoid(name='last_layer').apply(pre_out)
y_hat.name = 'output_sequence'
predict = theano.function(inputs = [x, ], outputs = y_hat)
y = tensor.tensor3('targets')
#cost = BinaryCrossEntropy().apply(y, y_hat)
cost = tensor.nnet.binary_crossentropy(y_hat, y).sum(axis=2).mean()
cost.name = 'BCE'
state_to_output.weights_init = input_to_state.weights_init = IsotropicGaussian(0.01)
#RNN.weights_init = Constant(1.)
RNN.weights_init = IsotropicGaussian(.01)
RNN.biases_init = state_to_output.biases_init = input_to_state.biases_init = Constant(0)
RNN.initialize()
state_to_output.initialize()
input_to_state.initialize()
cg = ComputationGraph(cost)
params = VariableFilter(roles=[WEIGHTS, BIASES])(cg.variables)
In [7]:
trainset = DataStream(MusicFuel(which_set='train', which_dataset='jsb'))
testset = DataStream(MusicFuel(which_set='test', which_dataset='jsb'))
validset = DataStream(MusicFuel(which_set='valid', which_dataset='jsb'))
batch_size = 1
num_epochs = 100
cost.name = "sequence_log_likelihood"
algorithm = GradientDescent(
cost=cost, params=params,
step_rule=Adam(0.001))
main_loop = MainLoop(
algorithm=algorithm,
data_stream=trainset,
model=None,
extensions=[FinishAfter(after_n_epochs=num_epochs),
TrainingDataMonitoring([cost], prefix="train",
after_every_epoch=True),
DataStreamMonitoring([cost], validset, prefix="valid"),
DataStreamMonitoring([cost], testset, prefix="test"),
Printing()])
main_loop.run()
-------------------------------------------------------------------------------
BEFORE FIRST EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 0
iterations_done: 0
Log records from the iteration 0:
test_sequence_log_likelihood: 66.5422134399
valid_sequence_log_likelihood: 66.5423049927
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 1
iterations_done: 229
Log records from the iteration 229:
test_sequence_log_likelihood: 12.3917503357
train_sequence_log_likelihood: 23.2203674316
valid_sequence_log_likelihood: 12.4435710907
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 2
iterations_done: 458
Log records from the iteration 458:
test_sequence_log_likelihood: 11.5960769653
train_sequence_log_likelihood: 11.7109384537
valid_sequence_log_likelihood: 11.6284751892
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 3
iterations_done: 687
Log records from the iteration 687:
test_sequence_log_likelihood: 11.447063446
train_sequence_log_likelihood: 11.3389539719
valid_sequence_log_likelihood: 11.4748182297
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 4
iterations_done: 916
Log records from the iteration 916:
test_sequence_log_likelihood: 11.3869514465
train_sequence_log_likelihood: 11.2365179062
valid_sequence_log_likelihood: 11.412899971
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 5
iterations_done: 1145
Log records from the iteration 1145:
test_sequence_log_likelihood: 11.3467588425
train_sequence_log_likelihood: 11.1844978333
valid_sequence_log_likelihood: 11.3712720871
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 6
iterations_done: 1374
Log records from the iteration 1374:
test_sequence_log_likelihood: 11.3005485535
train_sequence_log_likelihood: 11.1413288116
valid_sequence_log_likelihood: 11.3233232498
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 7
iterations_done: 1603
Log records from the iteration 1603:
test_sequence_log_likelihood: 11.2215738297
train_sequence_log_likelihood: 11.0832176208
valid_sequence_log_likelihood: 11.2416753769
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 8
iterations_done: 1832
Log records from the iteration 1832:
test_sequence_log_likelihood: 11.0788593292
train_sequence_log_likelihood: 10.982421875
valid_sequence_log_likelihood: 11.0948495865
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 9
iterations_done: 2061
Log records from the iteration 2061:
test_sequence_log_likelihood: 10.8791627884
train_sequence_log_likelihood: 10.8212652206
valid_sequence_log_likelihood: 10.8903532028
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 10
iterations_done: 2290
Log records from the iteration 2290:
test_sequence_log_likelihood: 10.670340538
train_sequence_log_likelihood: 10.6209602356
valid_sequence_log_likelihood: 10.6766967773
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 11
iterations_done: 2519
Log records from the iteration 2519:
test_sequence_log_likelihood: 10.4783372879
train_sequence_log_likelihood: 10.4139738083
valid_sequence_log_likelihood: 10.4813518524
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 12
iterations_done: 2748
Log records from the iteration 2748:
test_sequence_log_likelihood: 10.3120241165
train_sequence_log_likelihood: 10.2251596451
valid_sequence_log_likelihood: 10.3148918152
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 13
iterations_done: 2977
Log records from the iteration 2977:
test_sequence_log_likelihood: 10.1671113968
train_sequence_log_likelihood: 10.0634279251
valid_sequence_log_likelihood: 10.1706151962
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 14
iterations_done: 3206
Log records from the iteration 3206:
test_sequence_log_likelihood: 10.0409088135
train_sequence_log_likelihood: 9.92346858978
valid_sequence_log_likelihood: 10.0427932739
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 15
iterations_done: 3435
Log records from the iteration 3435:
test_sequence_log_likelihood: 9.93277740479
train_sequence_log_likelihood: 9.80157852173
valid_sequence_log_likelihood: 9.93099880219
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 16
iterations_done: 3664
Log records from the iteration 3664:
test_sequence_log_likelihood: 9.83869552612
train_sequence_log_likelihood: 9.69463062286
valid_sequence_log_likelihood: 9.83348846436
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 17
iterations_done: 3893
Log records from the iteration 3893:
test_sequence_log_likelihood: 9.75417423248
train_sequence_log_likelihood: 9.59885883331
valid_sequence_log_likelihood: 9.7477273941
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 18
iterations_done: 4122
Log records from the iteration 4122:
test_sequence_log_likelihood: 9.67738342285
train_sequence_log_likelihood: 9.51193523407
valid_sequence_log_likelihood: 9.67248535156
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 19
iterations_done: 4351
Log records from the iteration 4351:
test_sequence_log_likelihood: 9.60901927948
train_sequence_log_likelihood: 9.43361759186
valid_sequence_log_likelihood: 9.60782718658
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 20
iterations_done: 4580
Log records from the iteration 4580:
test_sequence_log_likelihood: 9.55015182495
train_sequence_log_likelihood: 9.36464691162
valid_sequence_log_likelihood: 9.55357170105
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 21
iterations_done: 4809
Log records from the iteration 4809:
test_sequence_log_likelihood: 9.50051689148
train_sequence_log_likelihood: 9.30515480042
valid_sequence_log_likelihood: 9.50822639465
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 22
iterations_done: 5038
Log records from the iteration 5038:
test_sequence_log_likelihood: 9.45862770081
train_sequence_log_likelihood: 9.2540473938
valid_sequence_log_likelihood: 9.46950721741
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 23
iterations_done: 5267
Log records from the iteration 5267:
test_sequence_log_likelihood: 9.42270278931
train_sequence_log_likelihood: 9.20960998535
valid_sequence_log_likelihood: 9.43540668488
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 24
iterations_done: 5496
Log records from the iteration 5496:
test_sequence_log_likelihood: 9.39130592346
train_sequence_log_likelihood: 9.17027378082
valid_sequence_log_likelihood: 9.4046087265
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 25
iterations_done: 5725
Log records from the iteration 5725:
test_sequence_log_likelihood: 9.36340999603
train_sequence_log_likelihood: 9.13487625122
valid_sequence_log_likelihood: 9.37641429901
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 26
iterations_done: 5954
Log records from the iteration 5954:
test_sequence_log_likelihood: 9.33829975128
train_sequence_log_likelihood: 9.10264778137
valid_sequence_log_likelihood: 9.350440979
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 27
iterations_done: 6183
Log records from the iteration 6183:
test_sequence_log_likelihood: 9.31543636322
train_sequence_log_likelihood: 9.0730381012
valid_sequence_log_likelihood: 9.32640552521
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 28
iterations_done: 6412
Log records from the iteration 6412:
test_sequence_log_likelihood: 9.29437923431
train_sequence_log_likelihood: 9.04561710358
valid_sequence_log_likelihood: 9.30404567719
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 29
iterations_done: 6641
Log records from the iteration 6641:
test_sequence_log_likelihood: 9.27476215363
train_sequence_log_likelihood: 9.01999855042
valid_sequence_log_likelihood: 9.28309345245
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 30
iterations_done: 6870
Log records from the iteration 6870:
test_sequence_log_likelihood: 9.25632667542
train_sequence_log_likelihood: 8.99587059021
valid_sequence_log_likelihood: 9.26334095001
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 31
iterations_done: 7099
Log records from the iteration 7099:
test_sequence_log_likelihood: 9.23888397217
train_sequence_log_likelihood: 8.97299480438
valid_sequence_log_likelihood: 9.24462604523
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 32
iterations_done: 7328
Log records from the iteration 7328:
test_sequence_log_likelihood: 9.2223033905
train_sequence_log_likelihood: 8.95119667053
valid_sequence_log_likelihood: 9.22683048248
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 33
iterations_done: 7557
Log records from the iteration 7557:
test_sequence_log_likelihood: 9.20650863647
train_sequence_log_likelihood: 8.9303483963
valid_sequence_log_likelihood: 9.20986175537
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 34
iterations_done: 7786
Log records from the iteration 7786:
test_sequence_log_likelihood: 9.19141864777
train_sequence_log_likelihood: 8.91035366058
valid_sequence_log_likelihood: 9.19363880157
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 35
iterations_done: 8015
Log records from the iteration 8015:
test_sequence_log_likelihood: 9.17697525024
train_sequence_log_likelihood: 8.89113330841
valid_sequence_log_likelihood: 9.1780834198
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 36
iterations_done: 8244
Log records from the iteration 8244:
test_sequence_log_likelihood: 9.1631269455
train_sequence_log_likelihood: 8.87262821198
valid_sequence_log_likelihood: 9.16313838959
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 37
iterations_done: 8473
Log records from the iteration 8473:
test_sequence_log_likelihood: 9.14982414246
train_sequence_log_likelihood: 8.85477352142
valid_sequence_log_likelihood: 9.14874076843
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 38
iterations_done: 8702
Log records from the iteration 8702:
test_sequence_log_likelihood: 9.13701438904
train_sequence_log_likelihood: 8.83753108978
valid_sequence_log_likelihood: 9.13484382629
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 39
iterations_done: 8931
Log records from the iteration 8931:
test_sequence_log_likelihood: 9.12466526031
train_sequence_log_likelihood: 8.82086467743
valid_sequence_log_likelihood: 9.12140274048
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 40
iterations_done: 9160
Log records from the iteration 9160:
test_sequence_log_likelihood: 9.11274242401
train_sequence_log_likelihood: 8.80473423004
valid_sequence_log_likelihood: 9.10838985443
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 41
iterations_done: 9389
Log records from the iteration 9389:
test_sequence_log_likelihood: 9.1012096405
train_sequence_log_likelihood: 8.789103508
valid_sequence_log_likelihood: 9.09575653076
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 42
iterations_done: 9618
Log records from the iteration 9618:
test_sequence_log_likelihood: 9.09004116058
train_sequence_log_likelihood: 8.77394294739
valid_sequence_log_likelihood: 9.0834941864
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 43
iterations_done: 9847
Log records from the iteration 9847:
test_sequence_log_likelihood: 9.07921504974
train_sequence_log_likelihood: 8.75922584534
valid_sequence_log_likelihood: 9.07157325745
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 44
iterations_done: 10076
Log records from the iteration 10076:
test_sequence_log_likelihood: 9.06871414185
train_sequence_log_likelihood: 8.74491786957
valid_sequence_log_likelihood: 9.05997562408
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 45
iterations_done: 10305
Log records from the iteration 10305:
test_sequence_log_likelihood: 9.0585308075
train_sequence_log_likelihood: 8.73100757599
valid_sequence_log_likelihood: 9.04868602753
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 46
iterations_done: 10534
Log records from the iteration 10534:
test_sequence_log_likelihood: 9.04865550995
train_sequence_log_likelihood: 8.71745967865
valid_sequence_log_likelihood: 9.0376996994
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 47
iterations_done: 10763
Log records from the iteration 10763:
test_sequence_log_likelihood: 9.03908824921
train_sequence_log_likelihood: 8.70427322388
valid_sequence_log_likelihood: 9.02700519562
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 48
iterations_done: 10992
Log records from the iteration 10992:
test_sequence_log_likelihood: 9.02982711792
train_sequence_log_likelihood: 8.69142436981
valid_sequence_log_likelihood: 9.01660442352
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 49
iterations_done: 11221
Log records from the iteration 11221:
test_sequence_log_likelihood: 9.02087402344
train_sequence_log_likelihood: 8.67891120911
valid_sequence_log_likelihood: 9.00649929047
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 50
iterations_done: 11450
Log records from the iteration 11450:
test_sequence_log_likelihood: 9.01223373413
train_sequence_log_likelihood: 8.66671562195
valid_sequence_log_likelihood: 8.99668884277
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 51
iterations_done: 11679
Log records from the iteration 11679:
test_sequence_log_likelihood: 9.00390815735
train_sequence_log_likelihood: 8.65484523773
valid_sequence_log_likelihood: 8.98717689514
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 52
iterations_done: 11908
Log records from the iteration 11908:
test_sequence_log_likelihood: 8.99589824677
train_sequence_log_likelihood: 8.64328289032
valid_sequence_log_likelihood: 8.97796821594
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 53
iterations_done: 12137
Log records from the iteration 12137:
test_sequence_log_likelihood: 8.98820114136
train_sequence_log_likelihood: 8.63203239441
valid_sequence_log_likelihood: 8.96905899048
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 54
iterations_done: 12366
Log records from the iteration 12366:
test_sequence_log_likelihood: 8.98081302643
train_sequence_log_likelihood: 8.62107849121
valid_sequence_log_likelihood: 8.96045207977
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 55
iterations_done: 12595
Log records from the iteration 12595:
test_sequence_log_likelihood: 8.97372531891
train_sequence_log_likelihood: 8.61042404175
valid_sequence_log_likelihood: 8.95214176178
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 56
iterations_done: 12824
Log records from the iteration 12824:
test_sequence_log_likelihood: 8.96693706512
train_sequence_log_likelihood: 8.60005092621
valid_sequence_log_likelihood: 8.94412994385
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 57
iterations_done: 13053
Log records from the iteration 13053:
test_sequence_log_likelihood: 8.96043300629
train_sequence_log_likelihood: 8.58996295929
valid_sequence_log_likelihood: 8.93640422821
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 58
iterations_done: 13282
Log records from the iteration 13282:
test_sequence_log_likelihood: 8.95420837402
train_sequence_log_likelihood: 8.5801410675
valid_sequence_log_likelihood: 8.92896270752
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 59
iterations_done: 13511
Log records from the iteration 13511:
test_sequence_log_likelihood: 8.94824981689
train_sequence_log_likelihood: 8.57059192657
valid_sequence_log_likelihood: 8.92179679871
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 60
iterations_done: 13740
Log records from the iteration 13740:
test_sequence_log_likelihood: 8.94254970551
train_sequence_log_likelihood: 8.56129646301
valid_sequence_log_likelihood: 8.91489887238
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 61
iterations_done: 13969
Log records from the iteration 13969:
test_sequence_log_likelihood: 8.93709564209
train_sequence_log_likelihood: 8.5522441864
valid_sequence_log_likelihood: 8.90826225281
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 62
iterations_done: 14198
Log records from the iteration 14198:
test_sequence_log_likelihood: 8.9318780899
train_sequence_log_likelihood: 8.54343605042
valid_sequence_log_likelihood: 8.90187931061
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 63
iterations_done: 14427
Log records from the iteration 14427:
test_sequence_log_likelihood: 8.92688751221
train_sequence_log_likelihood: 8.53485965729
valid_sequence_log_likelihood: 8.89573955536
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 64
iterations_done: 14656
Log records from the iteration 14656:
test_sequence_log_likelihood: 8.92211055756
train_sequence_log_likelihood: 8.526512146
valid_sequence_log_likelihood: 8.88984012604
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 65
iterations_done: 14885
Log records from the iteration 14885:
test_sequence_log_likelihood: 8.91753959656
train_sequence_log_likelihood: 8.5183763504
valid_sequence_log_likelihood: 8.88416576385
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 66
iterations_done: 15114
Log records from the iteration 15114:
test_sequence_log_likelihood: 8.91316509247
train_sequence_log_likelihood: 8.51045894623
valid_sequence_log_likelihood: 8.87871265411
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 67
iterations_done: 15343
Log records from the iteration 15343:
test_sequence_log_likelihood: 8.90897464752
train_sequence_log_likelihood: 8.50274181366
valid_sequence_log_likelihood: 8.87347316742
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 68
iterations_done: 15572
Log records from the iteration 15572:
test_sequence_log_likelihood: 8.904961586
train_sequence_log_likelihood: 8.4952287674
valid_sequence_log_likelihood: 8.8684387207
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 69
iterations_done: 15801
Log records from the iteration 15801:
test_sequence_log_likelihood: 8.90111351013
train_sequence_log_likelihood: 8.48790073395
valid_sequence_log_likelihood: 8.86360168457
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 70
iterations_done: 16030
Log records from the iteration 16030:
test_sequence_log_likelihood: 8.89742565155
train_sequence_log_likelihood: 8.48076534271
valid_sequence_log_likelihood: 8.85895252228
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 71
iterations_done: 16259
Log records from the iteration 16259:
test_sequence_log_likelihood: 8.89388656616
train_sequence_log_likelihood: 8.47380161285
valid_sequence_log_likelihood: 8.85448932648
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 72
iterations_done: 16488
Log records from the iteration 16488:
test_sequence_log_likelihood: 8.8904914856
train_sequence_log_likelihood: 8.46702384949
valid_sequence_log_likelihood: 8.85020160675
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 73
iterations_done: 16717
Log records from the iteration 16717:
test_sequence_log_likelihood: 8.88723087311
train_sequence_log_likelihood: 8.4604101181
valid_sequence_log_likelihood: 8.84608459473
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 74
iterations_done: 16946
Log records from the iteration 16946:
test_sequence_log_likelihood: 8.88409805298
train_sequence_log_likelihood: 8.45395565033
valid_sequence_log_likelihood: 8.84212970734
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 75
iterations_done: 17175
Log records from the iteration 17175:
test_sequence_log_likelihood: 8.88108730316
train_sequence_log_likelihood: 8.44766044617
valid_sequence_log_likelihood: 8.83833026886
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 76
iterations_done: 17404
Log records from the iteration 17404:
test_sequence_log_likelihood: 8.87818527222
train_sequence_log_likelihood: 8.44151878357
valid_sequence_log_likelihood: 8.83468532562
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 77
iterations_done: 17633
Log records from the iteration 17633:
test_sequence_log_likelihood: 8.87539958954
train_sequence_log_likelihood: 8.43552684784
valid_sequence_log_likelihood: 8.83117866516
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 78
iterations_done: 17862
Log records from the iteration 17862:
test_sequence_log_likelihood: 8.87271595001
train_sequence_log_likelihood: 8.42967319489
valid_sequence_log_likelihood: 8.8278169632
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 79
iterations_done: 18091
Log records from the iteration 18091:
test_sequence_log_likelihood: 8.87013339996
train_sequence_log_likelihood: 8.4239616394
valid_sequence_log_likelihood: 8.82459068298
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 80
iterations_done: 18320
Log records from the iteration 18320:
test_sequence_log_likelihood: 8.86764240265
train_sequence_log_likelihood: 8.41838359833
valid_sequence_log_likelihood: 8.82149219513
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 81
iterations_done: 18549
Log records from the iteration 18549:
test_sequence_log_likelihood: 8.86524486542
train_sequence_log_likelihood: 8.41294288635
valid_sequence_log_likelihood: 8.81851959229
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 82
iterations_done: 18778
Log records from the iteration 18778:
test_sequence_log_likelihood: 8.8629322052
train_sequence_log_likelihood: 8.40762233734
valid_sequence_log_likelihood: 8.81566524506
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 83
iterations_done: 19007
Log records from the iteration 19007:
test_sequence_log_likelihood: 8.86070346832
train_sequence_log_likelihood: 8.4024181366
valid_sequence_log_likelihood: 8.81292724609
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 84
iterations_done: 19236
Log records from the iteration 19236:
test_sequence_log_likelihood: 8.85855102539
train_sequence_log_likelihood: 8.39733886719
valid_sequence_log_likelihood: 8.81029987335
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 85
iterations_done: 19465
Log records from the iteration 19465:
test_sequence_log_likelihood: 8.85647583008
train_sequence_log_likelihood: 8.392370224
valid_sequence_log_likelihood: 8.80777549744
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 86
iterations_done: 19694
Log records from the iteration 19694:
test_sequence_log_likelihood: 8.85447120667
train_sequence_log_likelihood: 8.38751316071
valid_sequence_log_likelihood: 8.80535697937
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 87
iterations_done: 19923
Log records from the iteration 19923:
test_sequence_log_likelihood: 8.85254096985
train_sequence_log_likelihood: 8.38276386261
valid_sequence_log_likelihood: 8.80303573608
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 88
iterations_done: 20152
Log records from the iteration 20152:
test_sequence_log_likelihood: 8.85067176819
train_sequence_log_likelihood: 8.37811851501
valid_sequence_log_likelihood: 8.80080699921
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 89
iterations_done: 20381
Log records from the iteration 20381:
test_sequence_log_likelihood: 8.84886837006
train_sequence_log_likelihood: 8.37357902527
valid_sequence_log_likelihood: 8.79866981506
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 90
iterations_done: 20610
Log records from the iteration 20610:
test_sequence_log_likelihood: 8.84712409973
train_sequence_log_likelihood: 8.36913013458
valid_sequence_log_likelihood: 8.79661655426
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 91
iterations_done: 20839
Log records from the iteration 20839:
test_sequence_log_likelihood: 8.84543895721
train_sequence_log_likelihood: 8.36478042603
valid_sequence_log_likelihood: 8.79464817047
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 92
iterations_done: 21068
Log records from the iteration 21068:
test_sequence_log_likelihood: 8.84381103516
train_sequence_log_likelihood: 8.36051654816
valid_sequence_log_likelihood: 8.792760849
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 93
iterations_done: 21297
Log records from the iteration 21297:
test_sequence_log_likelihood: 8.84223461151
train_sequence_log_likelihood: 8.35634040833
valid_sequence_log_likelihood: 8.79094791412
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 94
iterations_done: 21526
Log records from the iteration 21526:
test_sequence_log_likelihood: 8.84070968628
train_sequence_log_likelihood: 8.35225772858
valid_sequence_log_likelihood: 8.78921222687
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 95
iterations_done: 21755
Log records from the iteration 21755:
test_sequence_log_likelihood: 8.83923339844
train_sequence_log_likelihood: 8.34825325012
valid_sequence_log_likelihood: 8.78753948212
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 96
iterations_done: 21984
Log records from the iteration 21984:
test_sequence_log_likelihood: 8.83780384064
train_sequence_log_likelihood: 8.34433555603
valid_sequence_log_likelihood: 8.78593921661
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 97
iterations_done: 22213
Log records from the iteration 22213:
test_sequence_log_likelihood: 8.8364238739
train_sequence_log_likelihood: 8.34049034119
valid_sequence_log_likelihood: 8.78440093994
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 98
iterations_done: 22442
Log records from the iteration 22442:
test_sequence_log_likelihood: 8.83508300781
train_sequence_log_likelihood: 8.33672046661
valid_sequence_log_likelihood: 8.7829246521
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 99
iterations_done: 22671
Log records from the iteration 22671:
test_sequence_log_likelihood: 8.83378887177
train_sequence_log_likelihood: 8.33302688599
valid_sequence_log_likelihood: 8.78151035309
-------------------------------------------------------------------------------
AFTER ANOTHER EPOCH
-------------------------------------------------------------------------------
Training status:
epochs_done: 100
iterations_done: 22900
Log records from the iteration 22900:
test_sequence_log_likelihood: 8.83253288269
train_sequence_log_likelihood: 8.32940196991
training_finish_requested: True
valid_sequence_log_likelihood: 8.78014850616
-------------------------------------------------------------------------------
TRAINING HAS BEEN FINISHED:
-------------------------------------------------------------------------------
Training status:
epochs_done: 100
iterations_done: 22900
Log records from the iteration 22900:
test_sequence_log_likelihood: 8.83253288269
train_sequence_log_likelihood: 8.32940196991
training_finish_requested: True
training_finished: True
valid_sequence_log_likelihood: 8.78014850616
/Users/eder/python/blocks/blocks/log.py:147: FutureWarning: comparison to `None` will result in an elementwise object comparison in the future.
if value != default_value:
In [20]:
RNN.W.get_value()
Out[20]:
array([[ 0.00379554, 0.00304139, -0.00141121, ..., -0.00814169,
-0.00306936, -0.00491639],
[-0.00769605, -0.00094225, -0.01423664, ..., 0.01357418,
-0.02250058, 0.00086137],
[-0.00804271, -0.00742074, -0.0046008 , ..., -0.00231734,
0.00146121, 0.00114211],
...,
[ 0.0058826 , -0.0118331 , 0.00024153, ..., 0.01251155,
0.00976696, 0.01697472],
[-0.00867319, 0.00736092, -0.00528297, ..., -0.01506917,
0.02351925, -0.02167687],
[ 0.01323971, 0.00132816, -0.00656847, ..., 0.00588472,
0.00595488, 0.00471435]], dtype=float32)
In [ ]:
Content source: EderSantana/blocks_contrib
Similar notebooks: