In [1]:

    

import sys
sys.path.append('..')
%load_ext autoreload
%autoreload 2


    

In [2]:

    

import numpy as np
import math
import matplotlib.pyplot as plt
import random
import time
import dill as pickle

from IPython.display import clear_output

import dali.core as D
import dali
from dali.data import Lines, Process, DiscoverFiles, BatchBenefactor, IdentityReducer
from dali.data.batch import TranslationBatch
from dali.data.translation import TranslationFiles, TranslationMapper, build_vocabs, iterate_examples
from dali.utils.scoring import bleu, ErrorTracker
from dali.utils import (
    Vocab,
    Solver,
    median_smoothing,
    subsample,
    Throttled,
    pickle_globals,
    unpickle_globals,
)
from dali import beam_search

%matplotlib inline


    

In [3]:

    

GPU_ID = 0
D.config.default_device = 'gpu'
D.config.default_gpu    = GPU_ID
print(D.config.gpu_id_to_name(GPU_ID))


    

    




b'Quadro K2000M'

In [4]:

    

def generate_example(item_size, num_items, batch_size = 1):
    return [ D.Mat(np.random.binomial(1, 0.5, (batch_size, item_size)), dtype=np.float32) for _ in range(num_items) ]

def list_norm(xs):
    squared_sum = np.sqrt(sum((x.dw ** 2).sum() for x in xs))
    return squared_sum


def evaluate_copy(model, solver, input_size, max_train_seq_length, batch_size=1,
                  epochs=100000, curriculum_epochs=None, debug=False):    
    start_prediction = D.Mat.ones((1,input_size), constant=True)
    
    def test(model, max_train_seq_length):
        with D.NoBackprop():
            for test_len in [max_train_seq_length // 2, max_train_seq_length, max_train_seq_length * 2]:
                print ("    Testing sequence length %d: " % (test_len,))

                NUM_TRIES = 10

                num_bits_correct = 0

                for _ in range(NUM_TRIES):
                    example = generate_example(input_size, test_len)
                    state = model.initial_states()
                    for input_vec in example:
                        state = model.activate(input_vec, state)

                    state = model.activate(start_prediction, state)
                    for input_vec in example:
                        decoded = np.round(model.decode(state).sigmoid().w)
                        num_bits_correct += sum(sum(np.abs(decoded - input_vec.w)))
                        state = model.activate(start_prediction, state)

                        
                print('        Cost per sequence: %f (chance: %f)' % (num_bits_correct / NUM_TRIES, 
                                                                      test_len * input_size / 2))
    
    
    not_too_often = Throttled(1)
    test_freqency = Throttled(30)
    errors = []

    for epoch in range(epochs):
        try:
            if curriculum_epochs is not None:
                max_len = int(max_train_seq_length * (epoch / curriculum_epochs)) + 1
                max_len = max(1, min(max_train_seq_length, max_len))
            else:
                max_len = max_train_seq_length
            example = generate_example(input_size, random.randint(1, max_len), BATCH_SIZE)
            state = model.initial_states()
            for input_vec in example:
                state = model.activate(input_vec, state)

            state = model.activate(start_prediction, state)
            error = D.Mat.zeros((1,1), constant=True)
            for input_idx, input_vec in enumerate(example):
                decoded = model.decode(state)
                error = error + D.MatOps.sigmoid_binary_cross_entropy(decoded, input_vec).sum()
                #error = error + ((decoded.sigmoid() - input_vec)**2).sum()
                if input_idx + 1 != len(example):
                    state = model.activate(start_prediction, state)

            (error / BATCH_SIZE).grad()
            D.Graph.backward()
            solver.step()
            
            errors.append(error.w[0,0] / BATCH_SIZE)
            if debug:
                return errors
            if not_too_often.should_i_run():
                recent_error = np.median(errors[-100:])
                print('Epoch %d (%d processed examples, curriculum: %d), error: %f' % (epoch, 
                                                                                       epoch*BATCH_SIZE,
                                                                                       max_len,
                                                                                       recent_error))
            if test_freqency.should_i_run():
                test(model, max_train_seq_length)
            if solver.base.solver_type == 'adagrad' and epoch % 1000 == 0:
                solver.reset_caches()
            
        except KeyboardInterrupt:
            print("Early stopping")
            test(model, max_train_seq_length)
            break
    test(model, max_train_seq_length)
    return errors


    

In [5]:

    

class LSTMBaseline(object):
    def __init__(self, input_size, hiddens, output_size):
        self.lstm    = D.StackedLSTM(input_size, hiddens)
        self.decoder = D.Layer(sum(hiddens), output_size)
        
    def activate(self, input_vec, previous_state):
        return self.lstm.activate(input_vec, previous_state)
    
    def decode(self, state):
        return self.decoder.activate(D.MatOps.hstack([s.hidden for s in state]))
    
    def parameters(self):
        return self.lstm.parameters() + self.decoder.parameters()
    
    def initial_states(self):
        return self.lstm.initial_states()


    

In [6]:

    

INPUT_SIZE           = 4
MAX_TRAIN_SEQ_LENGTH = 5


HIDDENS  = [256, 256, 256]
BATCH_SIZE  = 20
baseline_model = LSTMBaseline(INPUT_SIZE, HIDDENS, INPUT_SIZE)
##solver = Solver("rmsprop", learning_rate=0.00003, decay_rate=0.9, smooth_eps=1e-4)
solver = Solver(baseline_model.parameters(), "rmsprop", learning_rate=0.0001, decay_rate=0.9, smooth_eps=1e-4, clipval=10.0)

errors = evaluate_copy(baseline_model, solver, INPUT_SIZE, MAX_TRAIN_SEQ_LENGTH, batch_size=BATCH_SIZE,
                      epochs=10000, curriculum_epochs=5000)


    

    




Epoch 0 (0 processed examples, curriculum: 1), error: 2.772408
    Testing sequence length 2: 
        Cost per sequence: 3.700000 (chance: 4.000000)
    Testing sequence length 5: 
        Cost per sequence: 8.800000 (chance: 10.000000)
    Testing sequence length 10: 
        Cost per sequence: 19.100000 (chance: 20.000000)
Epoch 1 (20 processed examples, curriculum: 1), error: 2.770235
Early stopping
    Testing sequence length 2: 
        Cost per sequence: 3.700000 (chance: 4.000000)
    Testing sequence length 5: 
        Cost per sequence: 10.100000 (chance: 10.000000)
    Testing sequence length 10: 
        Cost per sequence: 19.100000 (chance: 20.000000)
    Testing sequence length 2: 
        Cost per sequence: 3.800000 (chance: 4.000000)
    Testing sequence length 5: 
        Cost per sequence: 10.500000 (chance: 10.000000)
    Testing sequence length 10: 
        Cost per sequence: 19.300000 (chance: 20.000000)

In [175]:

    

class GRUBaseline(object):
    def __init__(self, input_size, hidden, output_size):
        self.seq    = D.GRU(input_size, hidden)
        self.decoder = D.Layer(hidden, output_size)
        
    def activate(self, input_vec, previous_state):
        return self.seq.activate(input_vec, previous_state)
    
    def decode(self, state):
        return self.decoder.activate(state)
    
    def parameters(self):
        return self.seq.parameters() + self.decoder.parameters()
    
    def initial_states(self):
        return self.seq.initial_states()


    

In [176]:

    

INPUT_SIZE           = 4
MAX_TRAIN_SEQ_LENGTH = 5


HIDDEN  = 256
BATCH_SIZE  = 20
baseline_model = GRUBaseline(INPUT_SIZE, HIDDEN, INPUT_SIZE)
##solver = Solver("rmsprop", learning_rate=0.00003, decay_rate=0.9, smooth_eps=1e-4)
solver = Solver("rmsprop", learning_rate=0.0001, decay_rate=0.9, smooth_eps=1e-4, clipval=10.0)

errors = evaluate_copy(baseline_model, solver, INPUT_SIZE, MAX_TRAIN_SEQ_LENGTH, batch_size=BATCH_SIZE,
                      epochs=10000, curriculum_epochs=5000)


    

    




Epoch 0 (0 processed examples, curriculum: 1), error: 1.007807
    Testing sequence length 2: 
        Cost per sequence: 3.900000 (chance: 4.000000)
    Testing sequence length 5: 
        Cost per sequence: 8.600000 (chance: 10.000000)
    Testing sequence length 10: 
        Cost per sequence: 20.400000 (chance: 20.000000)
Epoch 141 (2820 processed examples, curriculum: 1), error: 1.006281
Epoch 327 (6540 processed examples, curriculum: 1), error: 0.999587
Epoch 513 (10260 processed examples, curriculum: 1), error: 0.983540
Epoch 698 (13960 processed examples, curriculum: 1), error: 0.952478
Epoch 883 (17660 processed examples, curriculum: 1), error: 0.909743
Epoch 1055 (21100 processed examples, curriculum: 2), error: 0.864641
Epoch 1198 (23960 processed examples, curriculum: 2), error: 0.892634
Epoch 1337 (26740 processed examples, curriculum: 2), error: 0.900058
Epoch 1484 (29680 processed examples, curriculum: 2), error: 0.855162
Epoch 1628 (32560 processed examples, curriculum: 2), error: 0.876122
Epoch 1770 (35400 processed examples, curriculum: 2), error: 0.870549
Epoch 1910 (38200 processed examples, curriculum: 2), error: 0.881265
Epoch 2039 (40780 processed examples, curriculum: 3), error: 0.837259
Epoch 2155 (43100 processed examples, curriculum: 3), error: 0.865655
Epoch 2265 (45300 processed examples, curriculum: 3), error: 0.822387
Epoch 2377 (47540 processed examples, curriculum: 3), error: 0.781270
Epoch 2493 (49860 processed examples, curriculum: 3), error: 0.896521
Epoch 2607 (52140 processed examples, curriculum: 3), error: 0.918313
Epoch 2713 (54260 processed examples, curriculum: 3), error: 0.966033
Epoch 2824 (56480 processed examples, curriculum: 3), error: 1.084609
Epoch 2937 (58740 processed examples, curriculum: 3), error: 1.189178
Epoch 3044 (60880 processed examples, curriculum: 4), error: 1.370431
Epoch 3135 (62700 processed examples, curriculum: 4), error: 1.451816
Epoch 3238 (64760 processed examples, curriculum: 4), error: 1.547736
Epoch 3338 (66760 processed examples, curriculum: 4), error: 1.736140
Epoch 3438 (68760 processed examples, curriculum: 4), error: 1.907993
Epoch 3534 (70680 processed examples, curriculum: 4), error: 1.973196
Epoch 3631 (72620 processed examples, curriculum: 4), error: 2.107588
Epoch 3733 (74660 processed examples, curriculum: 4), error: 2.069136
    Testing sequence length 2: 
        Cost per sequence: 1.500000 (chance: 4.000000)
    Testing sequence length 5: 
        Cost per sequence: 9.000000 (chance: 10.000000)
    Testing sequence length 10: 
        Cost per sequence: 19.300000 (chance: 20.000000)
Epoch 3811 (76220 processed examples, curriculum: 4), error: 1.965709
Epoch 3905 (78100 processed examples, curriculum: 4), error: 2.093093
Epoch 4001 (80020 processed examples, curriculum: 5), error: 2.197829
Epoch 4081 (81620 processed examples, curriculum: 5), error: 2.308477
Epoch 4166 (83320 processed examples, curriculum: 5), error: 2.477029
Epoch 4253 (85060 processed examples, curriculum: 5), error: 2.630740
Epoch 4333 (86660 processed examples, curriculum: 5), error: 2.706822
Epoch 4411 (88220 processed examples, curriculum: 5), error: 2.842612
Epoch 4490 (89800 processed examples, curriculum: 5), error: 2.820822
Epoch 4575 (91500 processed examples, curriculum: 5), error: 2.964960
Epoch 4656 (93120 processed examples, curriculum: 5), error: 3.101658
Epoch 4730 (94600 processed examples, curriculum: 5), error: 3.146535
Epoch 4809 (96180 processed examples, curriculum: 5), error: 3.271729
Epoch 4887 (97740 processed examples, curriculum: 5), error: 3.179016
Epoch 4970 (99400 processed examples, curriculum: 5), error: 3.145066
Epoch 5054 (101080 processed examples, curriculum: 5), error: 3.259116
Epoch 5140 (102800 processed examples, curriculum: 5), error: 3.227135
Epoch 5224 (104480 processed examples, curriculum: 5), error: 3.331628
Epoch 5311 (106220 processed examples, curriculum: 5), error: 3.348537
Epoch 5387 (107740 processed examples, curriculum: 5), error: 3.181329
Epoch 5464 (109280 processed examples, curriculum: 5), error: 3.285903
Epoch 5537 (110740 processed examples, curriculum: 5), error: 3.230394
Epoch 5624 (112480 processed examples, curriculum: 5), error: 3.191896
Epoch 5705 (114100 processed examples, curriculum: 5), error: 3.074115
Epoch 5795 (115900 processed examples, curriculum: 5), error: 2.920653
Epoch 5876 (117520 processed examples, curriculum: 5), error: 3.028009
Epoch 5957 (119140 processed examples, curriculum: 5), error: 3.042806
Epoch 6039 (120780 processed examples, curriculum: 5), error: 2.890799
Epoch 6121 (122420 processed examples, curriculum: 5), error: 2.883843
Epoch 6199 (123980 processed examples, curriculum: 5), error: 2.975576
    Testing sequence length 2: 
        Cost per sequence: 1.200000 (chance: 4.000000)
    Testing sequence length 5: 
        Cost per sequence: 6.800000 (chance: 10.000000)
    Testing sequence length 10: 
        Cost per sequence: 18.400000 (chance: 20.000000)
Epoch 6258 (125160 processed examples, curriculum: 5), error: 2.922020
Epoch 6340 (126800 processed examples, curriculum: 5), error: 2.817849
Epoch 6421 (128420 processed examples, curriculum: 5), error: 2.929928
Epoch 6505 (130100 processed examples, curriculum: 5), error: 3.039675
Epoch 6586 (131720 processed examples, curriculum: 5), error: 3.006363
Epoch 6667 (133340 processed examples, curriculum: 5), error: 2.955905
Epoch 6749 (134980 processed examples, curriculum: 5), error: 2.852565
Epoch 6827 (136540 processed examples, curriculum: 5), error: 2.969970
Epoch 6904 (138080 processed examples, curriculum: 5), error: 3.000140
Epoch 6991 (139820 processed examples, curriculum: 5), error: 3.092276
Epoch 7074 (141480 processed examples, curriculum: 5), error: 3.096953
Epoch 7154 (143080 processed examples, curriculum: 5), error: 3.182487
Epoch 7241 (144820 processed examples, curriculum: 5), error: 3.143697
Epoch 7324 (146480 processed examples, curriculum: 5), error: 3.207701
Epoch 7410 (148200 processed examples, curriculum: 5), error: 3.214571
Epoch 7484 (149680 processed examples, curriculum: 5), error: 3.294930
Epoch 7562 (151240 processed examples, curriculum: 5), error: 3.296404
Epoch 7644 (152880 processed examples, curriculum: 5), error: 3.230621
Epoch 7727 (154540 processed examples, curriculum: 5), error: 3.238880
Epoch 7795 (155900 processed examples, curriculum: 5), error: 3.172456
Epoch 7884 (157680 processed examples, curriculum: 5), error: 3.264173
Epoch 7961 (159220 processed examples, curriculum: 5), error: 3.272541
Epoch 8040 (160800 processed examples, curriculum: 5), error: 3.323133
Epoch 8122 (162440 processed examples, curriculum: 5), error: 3.309240
Epoch 8198 (163960 processed examples, curriculum: 5), error: 3.371342
Epoch 8272 (165440 processed examples, curriculum: 5), error: 3.202815
Epoch 8349 (166980 processed examples, curriculum: 5), error: 3.271015
Epoch 8438 (168760 processed examples, curriculum: 5), error: 3.137296
Epoch 8518 (170360 processed examples, curriculum: 5), error: 3.060765
Epoch 8603 (172060 processed examples, curriculum: 5), error: 3.064581
    Testing sequence length 2: 
        Cost per sequence: 0.500000 (chance: 4.000000)
    Testing sequence length 5: 
        Cost per sequence: 6.100000 (chance: 10.000000)
    Testing sequence length 10: 
        Cost per sequence: 16.200000 (chance: 20.000000)
Epoch 8663 (173260 processed examples, curriculum: 5), error: 2.911374
Epoch 8737 (174740 processed examples, curriculum: 5), error: 2.864648
Epoch 8817 (176340 processed examples, curriculum: 5), error: 2.750140
Epoch 8888 (177760 processed examples, curriculum: 5), error: 2.873200
Epoch 8968 (179360 processed examples, curriculum: 5), error: 2.893510
Epoch 9049 (180980 processed examples, curriculum: 5), error: 2.748871
Epoch 9131 (182620 processed examples, curriculum: 5), error: 2.852300
Epoch 9208 (184160 processed examples, curriculum: 5), error: 2.854271
Epoch 9295 (185900 processed examples, curriculum: 5), error: 2.845394
Epoch 9379 (187580 processed examples, curriculum: 5), error: 2.840664
Epoch 9463 (189260 processed examples, curriculum: 5), error: 2.931120
Epoch 9543 (190860 processed examples, curriculum: 5), error: 2.784592
Epoch 9624 (192480 processed examples, curriculum: 5), error: 2.880349
Epoch 9704 (194080 processed examples, curriculum: 5), error: 2.879215
Epoch 9783 (195660 processed examples, curriculum: 5), error: 2.960717
Epoch 9864 (197280 processed examples, curriculum: 5), error: 2.812695
Epoch 9942 (198840 processed examples, curriculum: 5), error: 2.753465
    Testing sequence length 2: 
        Cost per sequence: 0.500000 (chance: 4.000000)
    Testing sequence length 5: 
        Cost per sequence: 4.700000 (chance: 10.000000)
    Testing sequence length 10: 
        Cost per sequence: 17.800000 (chance: 20.000000)

In [6]:

    

from collections import defaultdict

class Capture(object):
    instances = set()
    
    def __init__(self):
        self.state = defaultdict(lambda: [])
        
    @classmethod
    def add(cls, name, value):
        for instance in cls.instances:
            instance.state[name].append(value)
    
    def __enter__(self):
        Capture.instances.add(self)
    
    def __exit__(self, *args, **kwargs):
        Capture.instances.remove(self)


    

In [7]:

    

class MLP(object):
    def __init__(self, input_sizes, hiddens, nonlinearities):
        self.input_sizes = input_sizes
        self.hiddens = hiddens
        self.input_nonlinearity, self.layer_nonlinearities = nonlinearities[0], nonlinearities[1:]
        
        self.input_layer = D.StackedInputLayer(input_sizes, hiddens[0])
        self.layers = [D.Layer(h_from, h_to) for h_from, h_to in zip(hiddens[:-1], hiddens[1:])]
        
    def activate(self, inputs):
        assert len(self.layers) == len(self.layer_nonlinearities)
        hidden = self.input_nonlinearity(self.input_layer.activate(inputs))
        for l, nonlinearity in zip(self.layers, self.layer_nonlinearities):
            hidden = nonlinearity(l.activate(hidden))
        return hidden
    
    def parameters(self):
        ret = self.input_layer.parameters()
        for l in self.layers:
            ret.extend(l.parameters())
        return ret
    
    def name_parameters(self, prefix):
        self.input_layer.name_parameters(prefix + "_input_layer")
        for layer_idx, layer in enumerate(self.layers):
            layer.name_parameters(prefix + '_layer%d' % (layer_idx,))


    

In [8]:

    

def row_by_scalar(a,b):
    return D.MatOps.eltmul(a, b, broadcast=True, axis=0)

def col_by_scalar(a,b):
    return D.MatOps.eltmul(a, b, broadcast=True, axis=1)

class NTMAddressing(object):
    def __init__(self, input_sizes, memory_locations, memory_size, shift_mask='local'):
        self.memory_locations = memory_locations
        
        self.content_key          = D.StackedInputLayer(input_sizes, memory_size)
        self.content_key_strength = D.StackedInputLayer(input_sizes, 1)
        self.interpolation        = D.StackedInputLayer(input_sizes, 1)
        self.shift                = D.StackedInputLayer(input_sizes, memory_locations)
        self.focus                = D.StackedInputLayer(input_sizes, 1)
        
        self.shift_mask = np.zeros((1, memory_locations))
        if shift_mask == 'local':
            self.shift_mask[0, 0] = 1
            self.shift_mask[0, 1] = 1
            self.shift_mask[0, -1] = 1
        else:
            raise Exception("Unknown shift mask type: %s" % (shift_mask,))
        self.shift_mask = D.Mat(self.shift_mask)
        
        self.initial_locations = D.random.uniform(-0.1, 0.1, (1,self.memory_locations))        
        
    def name_parameters(self, prefix):
        self.content_key.name_parameters(prefix + "_content_key")
        self.content_key_strength.name_parameters(prefix + "_content_key_strength")
        self.interpolation.name_parameters(prefix + "_interpolation")
        self.shift.name_parameters(prefix + "_shift")
        self.focus.name_parameters(prefix + "_focus")
        self.initial_locations.name = prefix + "_initial_locations"

    def content_addressing_activation(self, key, key_strength, memory):
        # cosine distance essentially
        key_broadcasted   = D.MatOps.broadcast(key, axis=0, num_replicas=memory.shape[0])
        unnormalized_dot  = (key * memory).sum(axis=1)
        key_norm          = key_broadcasted.L2_norm(axis=1)
        memory_norm       = memory.L2_norm(axis=1)
        cosine_similarity = (unnormalized_dot / (key_norm * memory_norm + 1e-6)).T()
        presoftmax        = row_by_scalar(cosine_similarity, key_strength)
        
        return D.MatOps.softmax(presoftmax)
    
    def shift_activation(self, inputs, weights):
        shift                = D.MatOps.softmax(self.shift.activate(inputs))
        shift                = shift * self.shift_mask
        shift                = shift / (shift.sum() + 1e-6)
        
        return D.MatOps.circular_convolution(weights, shift)
        
    def address(self, inputs, memory, state):
        """Outputs memory location weights.
        
        Inputs:
        inputs -- set of vectors controlling the mechanism (e.g. LSTM output)
        state  -- weights from previous timestep.
        """
        # todo - should memory contents be tanhed?
        key                  = self.content_key.activate(inputs)
        key_strength         = self.content_key_strength.activate(inputs).softplus()
        # todo - make multiplication of similarity (vector) * key strength_scalrar is correctly broadcasted
        content_weights      = self.content_addressing_activation(key, key_strength, memory)
        interpolation_gate   = self.interpolation.activate(inputs).sigmoid()
        interpolated_weights = (row_by_scalar(content_weights, interpolation_gate) +
                                row_by_scalar(state,1.0 - interpolation_gate))        
        shifted_weighs       = self.shift_activate(inputs, interpolated_weights)
        focus                = self.focus.activate(inputs).softplus() + 1.
        focused_weights      = shifted_weighs ** focus
        # todo - make sure it's correctly broadcasted
        sum_focused          = focused_weights.sum(axis=1)
        focused_weights      = focused_weights / (sum_focused + 1e-6)
        
        return focused_weights
    
    def initial_states(self):
        return D.MatOps.softmax(self.initial_locations)
    
    def parameters(self):
        res = []
        res.extend(self.content_key.parameters())
        res.extend(self.content_key_strength.parameters())
        res.extend(self.interpolation.parameters())
        res.extend(self.shift.parameters())
        res.extend(self.focus.parameters())
        res.append(self.initial_locations)
        return res

class NTMReadHead(object):
    def __init__(self, input_sizes, memory_locations, memory_size):
        self.addressing = NTMAddressing(input_sizes, memory_locations, memory_size)
    
    def name_parameters(self, prefix):
        self.addressing.name_parameters(prefix + "_adressing")
    
    def read(self, inputs, memory, state):
        weights = self.addressing.address(inputs, memory, state)
        Capture.add("read_head_weights", weights)
        # todo - make sure it is correctly broadcasted
        pulled_from_memory = row_by_scalar(memory, weights.T()).sum(axis=0)
        Capture.add("read_head_content", pulled_from_memory)
        return pulled_from_memory, weights
    
    def initial_states(self):
        return self.addressing.initial_states()
    
    def parameters(self):
        return self.addressing.parameters()


class NTMWriteHead(object):
    def __init__(self, input_sizes, memory_locations, memory_size):
        self.addressing = NTMAddressing(input_sizes, memory_locations, memory_size)
        self.content    = D.StackedInputLayer(input_sizes, memory_size)
        self.erase      = D.StackedInputLayer(input_sizes, memory_size)
        
    def name_parameters(self, prefix):
        self.addressing.name_parameters(prefix + "_adressing")
        self.content.name_parameters(prefix + "_content")
        self.erase.name_parameters(prefix + "_erase")

        
    def write(self, inputs, memory, state):
        weights = self.addressing.address(inputs, memory, state)
        Capture.add("write_head_weights", weights)

        # todo - make sure it is correctly broadcasted
        new_content = self.content.activate(inputs)
        Capture.add("write_head_content", new_content)

        erase       = self.erase.activate(inputs).sigmoid()
        
        memory = memory * (1.0 - weights.T().dot(erase))
        # todo - make sure it is an outer product
        memory = memory + weights.T().dot(new_content)
        return memory, weights
    
    def initial_states(self):
        return self.addressing.initial_states()
    
    def parameters(self):
        return self.addressing.parameters()


    

In [9]:

    

class NTM(object):
    def __init__(self, input_size, hiddens, output_size, 
                 num_read_heads, num_write_heads,
                 memory_locations, memory_size,
                 controller_type='lstm'):
        self.memory_locations = memory_locations
        self.memory_size      = memory_size
        self.controller_type  = controller_type
        
        controller_inputs = [input_size] + [memory_size for _ in range(num_read_heads)]
        if controller_type   == 'mlp':
            nonlinearities = [D.MatOps.tanh for _ in range(len(hiddens))]
            self.controller = MLP(controller_inputs, hiddens, nonlinearities)
            self.controller_output_size = [hiddens[-1]]
        elif controller_type == 'lstm':
            self.controller = D.StackedLSTM(controller_inputs, hiddens, memory_feeds_gates=True)
            self.controller_output_size = hiddens
        else:
            raise Exception("Unrecognized controller %s, should be one of [mlp,lstm]" % (controller_type,))
    
        self.decoder      = D.StackedInputLayer(self.controller_output_size, output_size)

        self.read_heads   = [NTMReadHead (self.controller_output_size, memory_locations, memory_size) 
                                 for _ in range(num_read_heads) ]
        self.write_heads  = [NTMWriteHead(self.controller_output_size, memory_locations, memory_size) 
                                 for _ in range(num_write_heads)]

        # initialize memory to random values int range (-0.1, 0.1) not too close to zero
        self.memory_init = D.random.uniform(0.01, 0.1, (self.memory_locations, self.memory_size))
        self.memory_init.w *= np.random.binomial(1, 0.5, self.memory_init.shape) * 2 - 1
        
    def name_parameters(self, prefix):
        self.decoder.name_parameters(prefix + "_decoder")
        self.controller.name_parameters(prefix + "_controller_" + self.controller_type)
        for head_idx, head in enumerate(self.read_heads):
            head.name_parameters(prefix + "_read_head%d" % (head_idx,))
        for head_idx, head in enumerate(self.write_heads):
            head.name_parameters(prefix + "_write_head%d" % (head_idx,))
        self.memory_init.name = prefix + "_memory_init"
        
    def activate(self, input_vec, state):
        # read
        new_read_heads_state = []
        read_from_memory     = []
        for head, head_state in zip(self.read_heads, state['read_heads']):
            content, new_head_state = head.read(self.extract_hiddens(state['controller']), 
                                                state['memory'],
                                                head_state)
            read_from_memory.append(content)
            new_read_heads_state.append(new_head_state)
        # activate
        if self.controller_type   == 'mlp':
            new_controller_state = self.controller.activate([input_vec] + read_from_memory)
        elif self.controller_type == 'lstm':
            new_controller_state = self.controller.activate([input_vec] + read_from_memory, state['controller'])

        # write
        new_memory = state['memory']
        new_write_heads_state = []
        for head, head_state in zip(self.write_heads, state['write_heads']):
            new_memory, new_head_state = head.write(self.extract_hiddens(new_controller_state),
                                                    new_memory,
                                                    head_state)
            new_write_heads_state.append(new_head_state)
        # collect results
        new_state = {
            'read_heads'           : new_read_heads_state,
            'write_heads'          : new_write_heads_state,
            'memory'               : new_memory,
            'controller'           : new_controller_state
        }

        return new_state
    
    def decode(self, state):
        return self.decoder.activate(self.extract_hiddens(state['controller']))
        
    def parameters(self):
        res = []
        res.extend(self.controller.parameters())
        res.extend(self.decoder.parameters())
        for head in self.read_heads:
            res.extend(head.parameters())
        for head in self.write_heads:
            res.extend(head.parameters())
        res.append(self.memory_init)
        return res
    
    def extract_hiddens(self, state):
        if self.controller_type   == 'mlp':
            return [state]
        elif self.controller_type == 'lstm':
            return [s.hidden for s in state]
    
    def initial_states(self):
        ret = {
            'read_heads':            [head.initial_states() for head in self.read_heads],
            'write_heads':           [head.initial_states() for head in self.write_heads],
            'memory':                self.memory_init,           
        }
        if self.controller_type   == 'mlp':
            ret['controller'] = D.Mat.ones((1, self.controller_output_size[0]))
        elif self.controller_type == 'lstm':
            ret['controller'] = self.controller.initial_states()
        
        return ret


    

In [25]:

    

INPUT_SIZE           = 8
MAX_TRAIN_SEQ_LENGTH = 20


HIDDENS          = [100]

BATCH_SIZE       = 1
assert BATCH_SIZE == 1

MEMORY_LOCATIONS = 128
MEMORY_SIZE      = 20
ntm_model = NTM(input_size=INPUT_SIZE,
                hiddens=HIDDENS,
                output_size=INPUT_SIZE,
                num_read_heads=1,
                num_write_heads=1,
                memory_locations=MEMORY_LOCATIONS,
                memory_size=MEMORY_SIZE,
                controller_type='mlp')
ntm_model.name_parameters('lstm')

#solver = Solver("rmsprop", learning_rate=0.00003, decay_rate=0.9, smooth_eps=1e-4)
solver = Solver(ntm_model.parameters(), "rmsprop",  learning_rate=1e-4, 
                decay_rate=0.9, smooth_eps=1e-6, clipval=10.0)
# solver = Solver(ntm_model.parameters(), "rmsprop_momentum", 
#                 learning_rate=1e-5)#, regc=1e-4)
# solver = Solver(ntm_model.parameters(), "adadelta", regc=1e-3, clipval=1.0)
#olver = Solver(ntm_model.parameters(), "sgd", learning_rate=1e-2, clipval=10.0)
# solver = Solver("adagrad", learning_rate=0.00001, clipval=10.0)


    

In [26]:

    

errors = evaluate_copy(ntm_model, solver, INPUT_SIZE, MAX_TRAIN_SEQ_LENGTH, batch_size=BATCH_SIZE,
                      epochs=200000, curriculum_epochs = None, debug=False)


    

    




Epoch 0 (0 processed examples, curriculum: 20), error: 12.213409
    Testing sequence length 10: 
        Cost per sequence: 42.900000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 87.700000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 162.200000 (chance: 160.000000)
Epoch 1 (1 processed examples, curriculum: 20), error: 28.602974
Epoch 8 (8 processed examples, curriculum: 20), error: 88.615517
Epoch 19 (19 processed examples, curriculum: 20), error: 64.141041
Epoch 29 (29 processed examples, curriculum: 20), error: 61.829483
Epoch 36 (36 processed examples, curriculum: 20), error: 62.718616
Epoch 48 (48 processed examples, curriculum: 20), error: 61.023891
Epoch 58 (58 processed examples, curriculum: 20), error: 55.186947
Epoch 66 (66 processed examples, curriculum: 20), error: 60.684856
Epoch 77 (77 processed examples, curriculum: 20), error: 55.378727
Epoch 84 (84 processed examples, curriculum: 20), error: 55.756500
Epoch 91 (91 processed examples, curriculum: 20), error: 58.220678
Epoch 100 (100 processed examples, curriculum: 20), error: 60.726011
Epoch 109 (109 processed examples, curriculum: 20), error: 55.891321
Epoch 117 (117 processed examples, curriculum: 20), error: 60.726011
Epoch 129 (129 processed examples, curriculum: 20), error: 55.891321
Epoch 138 (138 processed examples, curriculum: 20), error: 55.743246
Epoch 149 (149 processed examples, curriculum: 20), error: 55.912865
Epoch 158 (158 processed examples, curriculum: 20), error: 60.496950
Epoch 170 (170 processed examples, curriculum: 20), error: 55.780247
Epoch 180 (180 processed examples, curriculum: 20), error: 55.912865
Epoch 187 (187 processed examples, curriculum: 20), error: 58.143847
Epoch 196 (196 processed examples, curriculum: 20), error: 55.599728
Epoch 207 (207 processed examples, curriculum: 20), error: 52.661646
Epoch 220 (220 processed examples, curriculum: 20), error: 49.547709
Epoch 229 (229 processed examples, curriculum: 20), error: 50.059578
Epoch 240 (240 processed examples, curriculum: 20), error: 50.059578
    Testing sequence length 10: 
        Cost per sequence: 38.900000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 78.900000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 162.400000 (chance: 160.000000)
Epoch 243 (243 processed examples, curriculum: 20), error: 50.059578
Epoch 249 (249 processed examples, curriculum: 20), error: 52.693041
Epoch 259 (259 processed examples, curriculum: 20), error: 55.167084
Epoch 270 (270 processed examples, curriculum: 20), error: 55.188150
Epoch 278 (278 processed examples, curriculum: 20), error: 55.226418
Epoch 287 (287 processed examples, curriculum: 20), error: 52.693041
Epoch 296 (296 processed examples, curriculum: 20), error: 52.693041
Epoch 303 (303 processed examples, curriculum: 20), error: 55.309944
Epoch 311 (311 processed examples, curriculum: 20), error: 55.620520
Epoch 321 (321 processed examples, curriculum: 20), error: 60.795248
Epoch 334 (334 processed examples, curriculum: 20), error: 55.754862
Epoch 343 (343 processed examples, curriculum: 20), error: 55.620520
Epoch 355 (355 processed examples, curriculum: 20), error: 52.410749
Epoch 366 (366 processed examples, curriculum: 20), error: 49.547764
Epoch 374 (374 processed examples, curriculum: 20), error: 52.539104
Epoch 382 (382 processed examples, curriculum: 20), error: 58.208366
Epoch 388 (388 processed examples, curriculum: 20), error: 60.916681
Epoch 396 (396 processed examples, curriculum: 20), error: 60.916681
Epoch 406 (406 processed examples, curriculum: 20), error: 50.219351
Epoch 415 (415 processed examples, curriculum: 20), error: 50.219351
Epoch 423 (423 processed examples, curriculum: 20), error: 60.973888
Epoch 432 (432 processed examples, curriculum: 20), error: 61.603878
Epoch 439 (439 processed examples, curriculum: 20), error: 63.970463
Epoch 447 (447 processed examples, curriculum: 20), error: 61.677830
Epoch 455 (455 processed examples, curriculum: 20), error: 66.526020
    Testing sequence length 10: 
        Cost per sequence: 40.100000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 80.600000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 162.300000 (chance: 160.000000)
Epoch 465 (465 processed examples, curriculum: 20), error: 66.546791
Epoch 476 (476 processed examples, curriculum: 20), error: 63.903465
Epoch 488 (488 processed examples, curriculum: 20), error: 61.439680
Epoch 495 (495 processed examples, curriculum: 20), error: 61.603878
Epoch 502 (502 processed examples, curriculum: 20), error: 61.603878
Epoch 508 (508 processed examples, curriculum: 20), error: 63.903465
Epoch 513 (513 processed examples, curriculum: 20), error: 66.575256
Epoch 523 (523 processed examples, curriculum: 20), error: 61.131441
Epoch 531 (531 processed examples, curriculum: 20), error: 61.131441
Epoch 537 (537 processed examples, curriculum: 20), error: 61.320063
Epoch 548 (548 processed examples, curriculum: 20), error: 60.859034
Epoch 559 (559 processed examples, curriculum: 20), error: 55.569368
Epoch 571 (571 processed examples, curriculum: 20), error: 52.875006
Epoch 583 (583 processed examples, curriculum: 20), error: 50.129688
Epoch 592 (592 processed examples, curriculum: 20), error: 49.965900
Epoch 600 (600 processed examples, curriculum: 20), error: 49.965900
Epoch 612 (612 processed examples, curriculum: 20), error: 49.644711
Epoch 620 (620 processed examples, curriculum: 20), error: 49.864042
Epoch 630 (630 processed examples, curriculum: 20), error: 50.129688
Epoch 639 (639 processed examples, curriculum: 20), error: 52.727924
Epoch 650 (650 processed examples, curriculum: 20), error: 52.727924
Epoch 658 (658 processed examples, curriculum: 20), error: 52.727924
Epoch 669 (669 processed examples, curriculum: 20), error: 55.636812
Epoch 675 (675 processed examples, curriculum: 20), error: 60.859673
Epoch 682 (682 processed examples, curriculum: 20), error: 60.859673
    Testing sequence length 10: 
        Cost per sequence: 40.500000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 78.800000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 155.700000 (chance: 160.000000)
Epoch 687 (687 processed examples, curriculum: 20), error: 60.892366
Epoch 697 (697 processed examples, curriculum: 20), error: 60.814999
Epoch 707 (707 processed examples, curriculum: 20), error: 60.892366
Epoch 715 (715 processed examples, curriculum: 20), error: 60.814999
Epoch 724 (724 processed examples, curriculum: 20), error: 58.260725
Epoch 733 (733 processed examples, curriculum: 20), error: 55.538708
Epoch 744 (744 processed examples, curriculum: 20), error: 58.185442
Epoch 752 (752 processed examples, curriculum: 20), error: 55.517841
Epoch 759 (759 processed examples, curriculum: 20), error: 55.407650
Epoch 765 (765 processed examples, curriculum: 20), error: 58.144285
Epoch 771 (771 processed examples, curriculum: 20), error: 58.144285
Epoch 777 (777 processed examples, curriculum: 20), error: 58.144285
Epoch 785 (785 processed examples, curriculum: 20), error: 58.144285
Epoch 792 (792 processed examples, curriculum: 20), error: 61.102875
Epoch 802 (802 processed examples, curriculum: 20), error: 60.796438
Epoch 809 (809 processed examples, curriculum: 20), error: 60.955338
Epoch 818 (818 processed examples, curriculum: 20), error: 63.855938
Epoch 825 (825 processed examples, curriculum: 20), error: 66.705910
Epoch 835 (835 processed examples, curriculum: 20), error: 63.909464
Epoch 843 (843 processed examples, curriculum: 20), error: 63.909464
Epoch 851 (851 processed examples, curriculum: 20), error: 66.657242
Epoch 861 (861 processed examples, curriculum: 20), error: 66.412739
Epoch 871 (871 processed examples, curriculum: 20), error: 61.044998
Epoch 880 (880 processed examples, curriculum: 20), error: 61.044998
Epoch 886 (886 processed examples, curriculum: 20), error: 60.955338
Epoch 894 (894 processed examples, curriculum: 20), error: 55.386953
    Testing sequence length 10: 
        Cost per sequence: 39.100000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 76.800000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 158.700000 (chance: 160.000000)
Epoch 895 (895 processed examples, curriculum: 20), error: 55.473177
Epoch 902 (902 processed examples, curriculum: 20), error: 55.473177
Epoch 912 (912 processed examples, curriculum: 20), error: 55.437113
Epoch 923 (923 processed examples, curriculum: 20), error: 50.006731
Epoch 932 (932 processed examples, curriculum: 20), error: 49.711060
Epoch 939 (939 processed examples, curriculum: 20), error: 49.711060
Epoch 947 (947 processed examples, curriculum: 20), error: 49.950928
Epoch 956 (956 processed examples, curriculum: 20), error: 44.551476
Epoch 963 (963 processed examples, curriculum: 20), error: 47.074600
Epoch 974 (974 processed examples, curriculum: 20), error: 44.702236
Epoch 982 (982 processed examples, curriculum: 20), error: 49.538685
Epoch 990 (990 processed examples, curriculum: 20), error: 49.950928
Epoch 1000 (1000 processed examples, curriculum: 20), error: 49.748955
Epoch 1005 (1005 processed examples, curriculum: 20), error: 49.950928
Epoch 1010 (1010 processed examples, curriculum: 20), error: 50.320114
Epoch 1017 (1017 processed examples, curriculum: 20), error: 55.787069
Epoch 1029 (1029 processed examples, curriculum: 20), error: 58.458374
Epoch 1038 (1038 processed examples, curriculum: 20), error: 55.787069
Epoch 1046 (1046 processed examples, curriculum: 20), error: 55.787069
Epoch 1055 (1055 processed examples, curriculum: 20), error: 55.910143
Epoch 1061 (1061 processed examples, curriculum: 20), error: 58.458374
Epoch 1070 (1070 processed examples, curriculum: 20), error: 55.675625
Epoch 1077 (1077 processed examples, curriculum: 20), error: 61.168001
Epoch 1085 (1085 processed examples, curriculum: 20), error: 55.995855
Epoch 1097 (1097 processed examples, curriculum: 20), error: 55.675625
    Testing sequence length 10: 
        Cost per sequence: 40.500000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 78.200000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 155.700000 (chance: 160.000000)
Epoch 1102 (1102 processed examples, curriculum: 20), error: 55.530964
Epoch 1109 (1109 processed examples, curriculum: 20), error: 55.665657
Epoch 1115 (1115 processed examples, curriculum: 20), error: 55.215683
Epoch 1121 (1121 processed examples, curriculum: 20), error: 55.215683
Epoch 1133 (1133 processed examples, curriculum: 20), error: 50.065977
Epoch 1139 (1139 processed examples, curriculum: 20), error: 55.215683
Epoch 1146 (1146 processed examples, curriculum: 20), error: 55.215683
Epoch 1152 (1152 processed examples, curriculum: 20), error: 58.342772
Epoch 1159 (1159 processed examples, curriculum: 20), error: 60.745993
Epoch 1168 (1168 processed examples, curriculum: 20), error: 60.796791
Epoch 1179 (1179 processed examples, curriculum: 20), error: 60.847763
Epoch 1188 (1188 processed examples, curriculum: 20), error: 60.913654
Epoch 1196 (1196 processed examples, curriculum: 20), error: 60.913654
Epoch 1201 (1201 processed examples, curriculum: 20), error: 66.496552
Epoch 1211 (1211 processed examples, curriculum: 20), error: 60.796791
Epoch 1220 (1220 processed examples, curriculum: 20), error: 60.745993
Epoch 1230 (1230 processed examples, curriculum: 20), error: 66.496552
Epoch 1239 (1239 processed examples, curriculum: 20), error: 63.646751
Epoch 1253 (1253 processed examples, curriculum: 20), error: 52.719053
Epoch 1264 (1264 processed examples, curriculum: 20), error: 50.122761
Epoch 1277 (1277 processed examples, curriculum: 20), error: 49.914188
Epoch 1286 (1286 processed examples, curriculum: 20), error: 49.914188
Epoch 1294 (1294 processed examples, curriculum: 20), error: 50.026758
Epoch 1302 (1302 processed examples, curriculum: 20), error: 49.913240
    Testing sequence length 10: 
        Cost per sequence: 41.400000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 82.500000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 155.400000 (chance: 160.000000)
Epoch 1309 (1309 processed examples, curriculum: 20), error: 50.184719
Epoch 1319 (1319 processed examples, curriculum: 20), error: 50.025810
Epoch 1329 (1329 processed examples, curriculum: 20), error: 50.350775
Epoch 1338 (1338 processed examples, curriculum: 20), error: 50.350775
Epoch 1345 (1345 processed examples, curriculum: 20), error: 55.306747
Epoch 1356 (1356 processed examples, curriculum: 20), error: 55.580042
Epoch 1366 (1366 processed examples, curriculum: 20), error: 55.780544
Epoch 1375 (1375 processed examples, curriculum: 20), error: 56.104794
Epoch 1384 (1384 processed examples, curriculum: 20), error: 58.452168
Epoch 1389 (1389 processed examples, curriculum: 20), error: 56.104794
Epoch 1398 (1398 processed examples, curriculum: 20), error: 55.793034
Epoch 1402 (1402 processed examples, curriculum: 20), error: 58.452168
Epoch 1411 (1411 processed examples, curriculum: 20), error: 60.882063
Epoch 1420 (1420 processed examples, curriculum: 20), error: 60.905716
Epoch 1427 (1427 processed examples, curriculum: 20), error: 61.009687
Epoch 1435 (1435 processed examples, curriculum: 20), error: 60.909416
Epoch 1442 (1442 processed examples, curriculum: 20), error: 60.809685
Epoch 1447 (1447 processed examples, curriculum: 20), error: 60.932917
Epoch 1454 (1454 processed examples, curriculum: 20), error: 61.001663
Epoch 1462 (1462 processed examples, curriculum: 20), error: 60.912275
Epoch 1470 (1470 processed examples, curriculum: 20), error: 60.932917
Epoch 1480 (1480 processed examples, curriculum: 20), error: 60.833338
Epoch 1487 (1487 processed examples, curriculum: 20), error: 60.912275
Epoch 1495 (1495 processed examples, curriculum: 20), error: 61.068296
Epoch 1505 (1505 processed examples, curriculum: 20), error: 60.912275
    Testing sequence length 10: 
        Cost per sequence: 38.100000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 81.800000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 159.500000 (chance: 160.000000)
Epoch 1507 (1507 processed examples, curriculum: 20), error: 60.912275
Epoch 1514 (1514 processed examples, curriculum: 20), error: 58.294317
Epoch 1524 (1524 processed examples, curriculum: 20), error: 55.533724
Epoch 1533 (1533 processed examples, curriculum: 20), error: 55.533724
Epoch 1540 (1540 processed examples, curriculum: 20), error: 56.009628
Epoch 1549 (1549 processed examples, curriculum: 20), error: 52.868761
Epoch 1554 (1554 processed examples, curriculum: 20), error: 55.804947
Epoch 1565 (1565 processed examples, curriculum: 20), error: 55.804947
Epoch 1576 (1576 processed examples, curriculum: 20), error: 55.667149
Epoch 1585 (1585 processed examples, curriculum: 20), error: 49.736242
Epoch 1592 (1592 processed examples, curriculum: 20), error: 49.736242
Epoch 1603 (1603 processed examples, curriculum: 20), error: 49.478657
Epoch 1612 (1612 processed examples, curriculum: 20), error: 49.736242
Epoch 1621 (1621 processed examples, curriculum: 20), error: 49.917192
Epoch 1629 (1629 processed examples, curriculum: 20), error: 52.652637
Epoch 1636 (1636 processed examples, curriculum: 20), error: 55.201542
Epoch 1646 (1646 processed examples, curriculum: 20), error: 55.201542
Epoch 1654 (1654 processed examples, curriculum: 20), error: 55.185558
Epoch 1661 (1661 processed examples, curriculum: 20), error: 55.201542
Epoch 1669 (1669 processed examples, curriculum: 20), error: 55.464556
Epoch 1675 (1675 processed examples, curriculum: 20), error: 58.102968
Epoch 1682 (1682 processed examples, curriculum: 20), error: 60.678160
Epoch 1692 (1692 processed examples, curriculum: 20), error: 60.993862
Epoch 1700 (1700 processed examples, curriculum: 20), error: 66.502335
Epoch 1709 (1709 processed examples, curriculum: 20), error: 66.970825
    Testing sequence length 10: 
        Cost per sequence: 39.100000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 80.000000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 159.800000 (chance: 160.000000)
Epoch 1713 (1713 processed examples, curriculum: 20), error: 66.970825
Epoch 1723 (1723 processed examples, curriculum: 20), error: 61.320599
Epoch 1733 (1733 processed examples, curriculum: 20), error: 60.890732
Epoch 1744 (1744 processed examples, curriculum: 20), error: 60.829214
Epoch 1752 (1752 processed examples, curriculum: 20), error: 58.113962
Epoch 1758 (1758 processed examples, curriculum: 20), error: 55.364656
Epoch 1767 (1767 processed examples, curriculum: 20), error: 50.185474
Epoch 1776 (1776 processed examples, curriculum: 20), error: 49.876640
Epoch 1786 (1786 processed examples, curriculum: 20), error: 49.876640
Epoch 1795 (1795 processed examples, curriculum: 20), error: 50.070824
Epoch 1804 (1804 processed examples, curriculum: 20), error: 49.926907
Epoch 1815 (1815 processed examples, curriculum: 20), error: 50.249605
Epoch 1828 (1828 processed examples, curriculum: 20), error: 49.913357
Epoch 1839 (1839 processed examples, curriculum: 20), error: 49.913357
Epoch 1848 (1848 processed examples, curriculum: 20), error: 49.913357
Epoch 1859 (1859 processed examples, curriculum: 20), error: 50.134954
Epoch 1870 (1870 processed examples, curriculum: 20), error: 49.842581
Epoch 1878 (1878 processed examples, curriculum: 20), error: 50.576574
Epoch 1882 (1882 processed examples, curriculum: 20), error: 55.144434
Epoch 1890 (1890 processed examples, curriculum: 20), error: 52.950323
Epoch 1900 (1900 processed examples, curriculum: 20), error: 52.784859
Epoch 1909 (1909 processed examples, curriculum: 20), error: 49.439814
Epoch 1918 (1918 processed examples, curriculum: 20), error: 49.498478
Epoch 1925 (1925 processed examples, curriculum: 20), error: 52.736715
Epoch 1932 (1932 processed examples, curriculum: 20), error: 50.016035
Epoch 1939 (1939 processed examples, curriculum: 20), error: 55.266191
    Testing sequence length 10: 
        Cost per sequence: 42.200000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 76.900000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 164.200000 (chance: 160.000000)
Epoch 1940 (1940 processed examples, curriculum: 20), error: 52.736715
Epoch 1951 (1951 processed examples, curriculum: 20), error: 55.162075
Epoch 1960 (1960 processed examples, curriculum: 20), error: 55.212877
Epoch 1966 (1966 processed examples, curriculum: 20), error: 55.785963
Epoch 1976 (1976 processed examples, curriculum: 20), error: 55.785963
Epoch 1985 (1985 processed examples, curriculum: 20), error: 55.317600
Epoch 1994 (1994 processed examples, curriculum: 20), error: 55.543245
Epoch 2001 (2001 processed examples, curriculum: 20), error: 55.785963
Epoch 2007 (2007 processed examples, curriculum: 20), error: 60.697712
Epoch 2012 (2012 processed examples, curriculum: 20), error: 61.216612
Epoch 2021 (2021 processed examples, curriculum: 20), error: 60.613928
Epoch 2032 (2032 processed examples, curriculum: 20), error: 58.228867
Epoch 2040 (2040 processed examples, curriculum: 20), error: 60.391609
Epoch 2047 (2047 processed examples, curriculum: 20), error: 61.038403
Epoch 2055 (2055 processed examples, curriculum: 20), error: 60.643599
Epoch 2061 (2061 processed examples, curriculum: 20), error: 60.713984
Epoch 2068 (2068 processed examples, curriculum: 20), error: 60.630199
Epoch 2076 (2076 processed examples, curriculum: 20), error: 60.630199
Epoch 2082 (2082 processed examples, curriculum: 20), error: 61.229460
Epoch 2089 (2089 processed examples, curriculum: 20), error: 61.229460
Epoch 2097 (2097 processed examples, curriculum: 20), error: 60.775608
Epoch 2105 (2105 processed examples, curriculum: 20), error: 60.775608
Epoch 2117 (2117 processed examples, curriculum: 20), error: 60.391609
Epoch 2128 (2128 processed examples, curriculum: 20), error: 61.015614
Epoch 2137 (2137 processed examples, curriculum: 20), error: 61.015614
    Testing sequence length 10: 
        Cost per sequence: 40.200000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 81.500000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 157.600000 (chance: 160.000000)
Epoch 2141 (2141 processed examples, curriculum: 20), error: 55.649948
Epoch 2151 (2151 processed examples, curriculum: 20), error: 55.869144
Epoch 2160 (2160 processed examples, curriculum: 20), error: 61.033936
Epoch 2166 (2166 processed examples, curriculum: 20), error: 58.612509
Epoch 2175 (2175 processed examples, curriculum: 20), error: 55.869144
Epoch 2191 (2191 processed examples, curriculum: 20), error: 49.843464
Epoch 2199 (2199 processed examples, curriculum: 20), error: 44.557447
Epoch 2207 (2207 processed examples, curriculum: 20), error: 49.948469
Epoch 2214 (2214 processed examples, curriculum: 20), error: 55.294266
Epoch 2220 (2220 processed examples, curriculum: 20), error: 55.711353
Epoch 2227 (2227 processed examples, curriculum: 20), error: 55.830055
Epoch 2235 (2235 processed examples, curriculum: 20), error: 55.830055
Epoch 2246 (2246 processed examples, curriculum: 20), error: 55.642704
Epoch 2253 (2253 processed examples, curriculum: 20), error: 58.247997
Epoch 2259 (2259 processed examples, curriculum: 20), error: 55.840519
Epoch 2269 (2269 processed examples, curriculum: 20), error: 60.617535
Epoch 2277 (2277 processed examples, curriculum: 20), error: 60.678221
Epoch 2283 (2283 processed examples, curriculum: 20), error: 60.944126
Epoch 2291 (2291 processed examples, curriculum: 20), error: 60.989908
Epoch 2297 (2297 processed examples, curriculum: 20), error: 61.279327
Epoch 2303 (2303 processed examples, curriculum: 20), error: 61.166513
Epoch 2308 (2308 processed examples, curriculum: 20), error: 61.597242
Epoch 2317 (2317 processed examples, curriculum: 20), error: 61.311598
Epoch 2325 (2325 processed examples, curriculum: 20), error: 61.166513
Epoch 2334 (2334 processed examples, curriculum: 20), error: 61.166513
    Testing sequence length 10: 
        Cost per sequence: 39.800000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 80.300000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 160.600000 (chance: 160.000000)
Epoch 2340 (2340 processed examples, curriculum: 20), error: 61.090242
Epoch 2350 (2350 processed examples, curriculum: 20), error: 60.878366
Epoch 2358 (2358 processed examples, curriculum: 20), error: 60.660858
Epoch 2362 (2362 processed examples, curriculum: 20), error: 60.878366
Epoch 2370 (2370 processed examples, curriculum: 20), error: 60.878366
Epoch 2379 (2379 processed examples, curriculum: 20), error: 61.166513
Epoch 2389 (2389 processed examples, curriculum: 20), error: 60.742661
Epoch 2395 (2395 processed examples, curriculum: 20), error: 60.942213
Epoch 2401 (2401 processed examples, curriculum: 20), error: 66.328739
Epoch 2410 (2410 processed examples, curriculum: 20), error: 60.742661
Epoch 2420 (2420 processed examples, curriculum: 20), error: 60.868004
Epoch 2431 (2431 processed examples, curriculum: 20), error: 61.164579
Epoch 2438 (2438 processed examples, curriculum: 20), error: 63.789417
Epoch 2445 (2445 processed examples, curriculum: 20), error: 66.821823
Epoch 2453 (2453 processed examples, curriculum: 20), error: 66.499859
Epoch 2458 (2458 processed examples, curriculum: 20), error: 67.128006
Epoch 2463 (2463 processed examples, curriculum: 20), error: 67.128006
Epoch 2472 (2472 processed examples, curriculum: 20), error: 69.278324
Epoch 2479 (2479 processed examples, curriculum: 20), error: 69.278324
Epoch 2488 (2488 processed examples, curriculum: 20), error: 74.700054
Epoch 2500 (2500 processed examples, curriculum: 20), error: 66.425636
Epoch 2509 (2509 processed examples, curriculum: 20), error: 66.425636
Epoch 2517 (2517 processed examples, curriculum: 20), error: 66.425636
Epoch 2526 (2526 processed examples, curriculum: 20), error: 69.278324
Epoch 2536 (2536 processed examples, curriculum: 20), error: 69.278324
    Testing sequence length 10: 
        Cost per sequence: 39.900000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 76.700000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 159.700000 (chance: 160.000000)
Epoch 2537 (2537 processed examples, curriculum: 20), error: 69.394981
Epoch 2547 (2547 processed examples, curriculum: 20), error: 69.090023
Epoch 2554 (2554 processed examples, curriculum: 20), error: 71.753498
Epoch 2564 (2564 processed examples, curriculum: 20), error: 66.135201
Epoch 2575 (2575 processed examples, curriculum: 20), error: 66.135201
Epoch 2582 (2582 processed examples, curriculum: 20), error: 66.135201
Epoch 2591 (2591 processed examples, curriculum: 20), error: 66.294399
Epoch 2601 (2601 processed examples, curriculum: 20), error: 66.294399
Epoch 2613 (2613 processed examples, curriculum: 20), error: 65.984486
Epoch 2622 (2622 processed examples, curriculum: 20), error: 65.984486
Epoch 2631 (2631 processed examples, curriculum: 20), error: 66.061565
Epoch 2640 (2640 processed examples, curriculum: 20), error: 66.061565
Epoch 2648 (2648 processed examples, curriculum: 20), error: 61.169798
Epoch 2658 (2658 processed examples, curriculum: 20), error: 60.894875
Epoch 2665 (2665 processed examples, curriculum: 20), error: 63.775530
Epoch 2673 (2673 processed examples, curriculum: 20), error: 66.323353
Epoch 2682 (2682 processed examples, curriculum: 20), error: 63.838129
Epoch 2690 (2690 processed examples, curriculum: 20), error: 68.691017
Epoch 2698 (2698 processed examples, curriculum: 20), error: 71.384506
Epoch 2707 (2707 processed examples, curriculum: 20), error: 68.691017
Epoch 2717 (2717 processed examples, curriculum: 20), error: 68.749142
Epoch 2724 (2724 processed examples, curriculum: 20), error: 71.168999
Epoch 2733 (2733 processed examples, curriculum: 20), error: 71.168999
Epoch 2741 (2741 processed examples, curriculum: 20), error: 71.506210
Epoch 2753 (2753 processed examples, curriculum: 20), error: 66.933437
    Testing sequence length 10: 
        Cost per sequence: 39.500000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 80.400000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 163.200000 (chance: 160.000000)
Epoch 2765 (2765 processed examples, curriculum: 20), error: 61.018696
Epoch 2775 (2775 processed examples, curriculum: 20), error: 60.968884
Epoch 2786 (2786 processed examples, curriculum: 20), error: 58.337593
Epoch 2795 (2795 processed examples, curriculum: 20), error: 56.237465
Epoch 2803 (2803 processed examples, curriculum: 20), error: 55.974722
Epoch 2810 (2810 processed examples, curriculum: 20), error: 56.237465
Epoch 2816 (2816 processed examples, curriculum: 20), error: 58.337593
Epoch 2828 (2828 processed examples, curriculum: 20), error: 52.804039
Epoch 2838 (2838 processed examples, curriculum: 20), error: 46.996309
Epoch 2847 (2847 processed examples, curriculum: 20), error: 49.331039
Epoch 2856 (2856 processed examples, curriculum: 20), error: 50.062065
Epoch 2865 (2865 processed examples, curriculum: 20), error: 55.790485
Epoch 2874 (2874 processed examples, curriculum: 20), error: 55.790485
Epoch 2883 (2883 processed examples, curriculum: 20), error: 58.368355
Epoch 2894 (2894 processed examples, curriculum: 20), error: 55.278074
Epoch 2902 (2902 processed examples, curriculum: 20), error: 55.826048
Epoch 2910 (2910 processed examples, curriculum: 20), error: 60.658667
Epoch 2923 (2923 processed examples, curriculum: 20), error: 55.826048
Epoch 2935 (2935 processed examples, curriculum: 20), error: 55.630728
Epoch 2944 (2944 processed examples, curriculum: 20), error: 55.630728
Epoch 2951 (2951 processed examples, curriculum: 20), error: 55.689650
Epoch 2956 (2956 processed examples, curriculum: 20), error: 55.762339
Epoch 2964 (2964 processed examples, curriculum: 20), error: 55.524487
Epoch 2973 (2973 processed examples, curriculum: 20), error: 55.230127
Epoch 2984 (2984 processed examples, curriculum: 20), error: 50.174158
Epoch 2995 (2995 processed examples, curriculum: 20), error: 49.982449
    Testing sequence length 10: 
        Cost per sequence: 38.700000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 79.400000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 156.200000 (chance: 160.000000)
Epoch 2997 (2997 processed examples, curriculum: 20), error: 49.906502
Epoch 3008 (3008 processed examples, curriculum: 20), error: 44.501030
Epoch 3018 (3018 processed examples, curriculum: 20), error: 49.845411
Epoch 3025 (3025 processed examples, curriculum: 20), error: 50.073557
Epoch 3035 (3035 processed examples, curriculum: 20), error: 49.926876
Epoch 3044 (3044 processed examples, curriculum: 20), error: 49.999954
Epoch 3053 (3053 processed examples, curriculum: 20), error: 49.924007
Epoch 3062 (3062 processed examples, curriculum: 20), error: 50.073557
Epoch 3073 (3073 processed examples, curriculum: 20), error: 49.909407
Epoch 3083 (3083 processed examples, curriculum: 20), error: 50.017984
Epoch 3093 (3093 processed examples, curriculum: 20), error: 50.017984
Epoch 3099 (3099 processed examples, curriculum: 20), error: 60.454552
Epoch 3109 (3109 processed examples, curriculum: 20), error: 60.515959
Epoch 3118 (3118 processed examples, curriculum: 20), error: 60.792101
Epoch 3127 (3127 processed examples, curriculum: 20), error: 60.870054
Epoch 3136 (3136 processed examples, curriculum: 20), error: 60.758751
Epoch 3144 (3144 processed examples, curriculum: 20), error: 60.758751
Epoch 3153 (3153 processed examples, curriculum: 20), error: 60.675919
Epoch 3161 (3161 processed examples, curriculum: 20), error: 60.515959
Epoch 3170 (3170 processed examples, curriculum: 20), error: 60.835888
Epoch 3180 (3180 processed examples, curriculum: 20), error: 61.056356
Epoch 3190 (3190 processed examples, curriculum: 20), error: 60.864347
Epoch 3200 (3200 processed examples, curriculum: 20), error: 60.851442
Epoch 3209 (3209 processed examples, curriculum: 20), error: 60.835888
Epoch 3219 (3219 processed examples, curriculum: 20), error: 60.238306
    Testing sequence length 10: 
        Cost per sequence: 39.900000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 77.500000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 161.800000 (chance: 160.000000)
Epoch 3223 (3223 processed examples, curriculum: 20), error: 57.687738
Epoch 3229 (3229 processed examples, curriculum: 20), error: 60.238306
Epoch 3237 (3237 processed examples, curriculum: 20), error: 60.851442
Epoch 3246 (3246 processed examples, curriculum: 20), error: 60.851442
Epoch 3252 (3252 processed examples, curriculum: 20), error: 61.010426
Epoch 3261 (3261 processed examples, curriculum: 20), error: 60.913105
Epoch 3269 (3269 processed examples, curriculum: 20), error: 61.021059
Epoch 3277 (3277 processed examples, curriculum: 20), error: 61.389795
Epoch 3287 (3287 processed examples, curriculum: 20), error: 61.084000
Epoch 3296 (3296 processed examples, curriculum: 20), error: 66.065113
Epoch 3306 (3306 processed examples, curriculum: 20), error: 68.725662
Epoch 3316 (3316 processed examples, curriculum: 20), error: 72.000259
Epoch 3324 (3324 processed examples, curriculum: 20), error: 71.651955
Epoch 3333 (3333 processed examples, curriculum: 20), error: 66.019249
Epoch 3343 (3343 processed examples, curriculum: 20), error: 66.019249
Epoch 3352 (3352 processed examples, curriculum: 20), error: 61.182732
Epoch 3359 (3359 processed examples, curriculum: 20), error: 65.963486
Epoch 3369 (3369 processed examples, curriculum: 20), error: 63.586809
Epoch 3379 (3379 processed examples, curriculum: 20), error: 63.732710
Epoch 3389 (3389 processed examples, curriculum: 20), error: 61.227520
Epoch 3395 (3395 processed examples, curriculum: 20), error: 63.751753
Epoch 3404 (3404 processed examples, curriculum: 20), error: 63.751753
Epoch 3410 (3410 processed examples, curriculum: 20), error: 66.114532
Epoch 3418 (3418 processed examples, curriculum: 20), error: 66.038292
Epoch 3427 (3427 processed examples, curriculum: 20), error: 63.751753
    Testing sequence length 10: 
        Cost per sequence: 39.300000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 80.600000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 160.900000 (chance: 160.000000)
Epoch 3429 (3429 processed examples, curriculum: 20), error: 61.383425
Epoch 3438 (3438 processed examples, curriculum: 20), error: 61.235868
Epoch 3445 (3445 processed examples, curriculum: 20), error: 61.383425
Epoch 3454 (3454 processed examples, curriculum: 20), error: 61.383425
Epoch 3462 (3462 processed examples, curriculum: 20), error: 58.427080
Epoch 3468 (3468 processed examples, curriculum: 20), error: 55.508060
Epoch 3474 (3474 processed examples, curriculum: 20), error: 55.508060
Epoch 3480 (3480 processed examples, curriculum: 20), error: 55.378643
Epoch 3486 (3486 processed examples, curriculum: 20), error: 55.974451
Epoch 3494 (3494 processed examples, curriculum: 20), error: 55.378643
Epoch 3501 (3501 processed examples, curriculum: 20), error: 55.845034
Epoch 3506 (3506 processed examples, curriculum: 20), error: 58.584126
Epoch 3511 (3511 processed examples, curriculum: 20), error: 55.553995
Epoch 3521 (3521 processed examples, curriculum: 20), error: 55.731848
Epoch 3530 (3530 processed examples, curriculum: 20), error: 56.018311
Epoch 3536 (3536 processed examples, curriculum: 20), error: 56.018311
Epoch 3542 (3542 processed examples, curriculum: 20), error: 55.731848
Epoch 3547 (3547 processed examples, curriculum: 20), error: 55.374067
Epoch 3554 (3554 processed examples, curriculum: 20), error: 55.553995
Epoch 3562 (3562 processed examples, curriculum: 20), error: 55.731848
Epoch 3568 (3568 processed examples, curriculum: 20), error: 55.731848
Epoch 3576 (3576 processed examples, curriculum: 20), error: 55.122326
Epoch 3582 (3582 processed examples, curriculum: 20), error: 55.054113
    Testing sequence length 10: 
        Cost per sequence: 38.700000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 81.700000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 162.200000 (chance: 160.000000)
Epoch 3583 (3583 processed examples, curriculum: 20), error: 55.054113
Epoch 3592 (3592 processed examples, curriculum: 20), error: 55.563902
Epoch 3603 (3603 processed examples, curriculum: 20), error: 50.353701
Epoch 3612 (3612 processed examples, curriculum: 20), error: 50.202662
Epoch 3622 (3622 processed examples, curriculum: 20), error: 49.962595
Epoch 3631 (3631 processed examples, curriculum: 20), error: 50.202662
Epoch 3638 (3638 processed examples, curriculum: 20), error: 52.651211
Epoch 3646 (3646 processed examples, curriculum: 20), error: 50.202662
Epoch 3653 (3653 processed examples, curriculum: 20), error: 52.558271
Epoch 3659 (3659 processed examples, curriculum: 20), error: 55.081573
Epoch 3665 (3665 processed examples, curriculum: 20), error: 58.213423
Epoch 3674 (3674 processed examples, curriculum: 20), error: 61.002619
Epoch 3680 (3680 processed examples, curriculum: 20), error: 60.868261
Epoch 3686 (3686 processed examples, curriculum: 20), error: 60.868261
Epoch 3694 (3694 processed examples, curriculum: 20), error: 60.748028
Epoch 3699 (3699 processed examples, curriculum: 20), error: 60.804876
Epoch 3707 (3707 processed examples, curriculum: 20), error: 60.757549
Epoch 3715 (3715 processed examples, curriculum: 20), error: 60.757549
Epoch 3724 (3724 processed examples, curriculum: 20), error: 60.757549
Epoch 3733 (3733 processed examples, curriculum: 20), error: 60.723633
Epoch 3740 (3740 processed examples, curriculum: 20), error: 60.657291
Epoch 3748 (3748 processed examples, curriculum: 20), error: 60.757549
Epoch 3756 (3756 processed examples, curriculum: 20), error: 60.571283
Epoch 3764 (3764 processed examples, curriculum: 20), error: 55.566362
    Testing sequence length 10: 
        Cost per sequence: 40.600000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 80.700000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 156.300000 (chance: 160.000000)
Epoch 3767 (3767 processed examples, curriculum: 20), error: 55.410606
Epoch 3775 (3775 processed examples, curriculum: 20), error: 55.566362
Epoch 3786 (3786 processed examples, curriculum: 20), error: 55.281645
Epoch 3798 (3798 processed examples, curriculum: 20), error: 52.481676
Epoch 3808 (3808 processed examples, curriculum: 20), error: 49.918430
Epoch 3817 (3817 processed examples, curriculum: 20), error: 49.918430
Epoch 3828 (3828 processed examples, curriculum: 20), error: 49.970474
Epoch 3839 (3839 processed examples, curriculum: 20), error: 49.970474
Epoch 3852 (3852 processed examples, curriculum: 20), error: 47.172598
Epoch 3860 (3860 processed examples, curriculum: 20), error: 49.797861
Epoch 3866 (3866 processed examples, curriculum: 20), error: 49.899815
Epoch 3878 (3878 processed examples, curriculum: 20), error: 47.295969
Epoch 3885 (3885 processed examples, curriculum: 20), error: 55.081017
Epoch 3893 (3893 processed examples, curriculum: 20), error: 55.338879
Epoch 3900 (3900 processed examples, curriculum: 20), error: 60.627190
Epoch 3908 (3908 processed examples, curriculum: 20), error: 60.627190
Epoch 3917 (3917 processed examples, curriculum: 20), error: 61.092171
Epoch 3926 (3926 processed examples, curriculum: 20), error: 63.845812
Epoch 3933 (3933 processed examples, curriculum: 20), error: 61.595507
Epoch 3942 (3942 processed examples, curriculum: 20), error: 66.084244
Epoch 3953 (3953 processed examples, curriculum: 20), error: 66.199478
Epoch 3964 (3964 processed examples, curriculum: 20), error: 61.674513
Epoch 3973 (3973 processed examples, curriculum: 20), error: 66.320057
Epoch 3984 (3984 processed examples, curriculum: 20), error: 63.690130
Epoch 3990 (3990 processed examples, curriculum: 20), error: 65.953632
    Testing sequence length 10: 
        Cost per sequence: 38.400000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 80.800000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 155.300000 (chance: 160.000000)
Epoch 3992 (3992 processed examples, curriculum: 20), error: 66.320057
Epoch 4001 (4001 processed examples, curriculum: 20), error: 66.459019
Epoch 4011 (4011 processed examples, curriculum: 20), error: 61.408476
Epoch 4024 (4024 processed examples, curriculum: 20), error: 60.767334
Epoch 4036 (4036 processed examples, curriculum: 20), error: 49.780375
Epoch 4043 (4043 processed examples, curriculum: 20), error: 49.866173
Epoch 4051 (4051 processed examples, curriculum: 20), error: 50.340073
Epoch 4058 (4058 processed examples, curriculum: 20), error: 60.119278
Epoch 4065 (4065 processed examples, curriculum: 20), error: 57.652344
Epoch 4072 (4072 processed examples, curriculum: 20), error: 60.119278
Epoch 4084 (4084 processed examples, curriculum: 20), error: 55.131197
Epoch 4090 (4090 processed examples, curriculum: 20), error: 54.959949
Epoch 4097 (4097 processed examples, curriculum: 20), error: 54.959949
Epoch 4107 (4107 processed examples, curriculum: 20), error: 54.959949
Epoch 4115 (4115 processed examples, curriculum: 20), error: 60.119278
Epoch 4128 (4128 processed examples, curriculum: 20), error: 60.745108
Epoch 4137 (4137 processed examples, curriculum: 20), error: 60.816525
Epoch 4146 (4146 processed examples, curriculum: 20), error: 55.233370
Epoch 4157 (4157 processed examples, curriculum: 20), error: 55.062878
Epoch 4162 (4162 processed examples, curriculum: 20), error: 55.278519
Epoch 4169 (4169 processed examples, curriculum: 20), error: 55.062122
Epoch 4175 (4175 processed examples, curriculum: 20), error: 55.422373
Epoch 4184 (4184 processed examples, curriculum: 20), error: 58.100676
Epoch 4189 (4189 processed examples, curriculum: 20), error: 58.132332
    Testing sequence length 10: 
        Cost per sequence: 37.700000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 77.600000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 160.000000 (chance: 160.000000)
Epoch 4192 (4192 processed examples, curriculum: 20), error: 58.132332
Epoch 4200 (4200 processed examples, curriculum: 20), error: 58.132332
Epoch 4207 (4207 processed examples, curriculum: 20), error: 55.520918
Epoch 4219 (4219 processed examples, curriculum: 20), error: 50.220724
Epoch 4228 (4228 processed examples, curriculum: 20), error: 52.689863
Epoch 4240 (4240 processed examples, curriculum: 20), error: 50.161360
Epoch 4247 (4247 processed examples, curriculum: 20), error: 55.010885
Epoch 4257 (4257 processed examples, curriculum: 20), error: 55.171751
Epoch 4263 (4263 processed examples, curriculum: 20), error: 55.449717
Epoch 4271 (4271 processed examples, curriculum: 20), error: 55.509638
Epoch 4279 (4279 processed examples, curriculum: 20), error: 55.171751
Epoch 4288 (4288 processed examples, curriculum: 20), error: 55.171751
Epoch 4297 (4297 processed examples, curriculum: 20), error: 55.171751
Epoch 4305 (4305 processed examples, curriculum: 20), error: 55.149490
Epoch 4315 (4315 processed examples, curriculum: 20), error: 55.509638
Epoch 4322 (4322 processed examples, curriculum: 20), error: 56.145878
Epoch 4331 (4331 processed examples, curriculum: 20), error: 55.678427
Epoch 4339 (4339 processed examples, curriculum: 20), error: 63.442583
Epoch 4346 (4346 processed examples, curriculum: 20), error: 63.442583
Epoch 4354 (4354 processed examples, curriculum: 20), error: 66.093750
Epoch 4362 (4362 processed examples, curriculum: 20), error: 66.074482
Epoch 4369 (4369 processed examples, curriculum: 20), error: 66.074482
Epoch 4379 (4379 processed examples, curriculum: 20), error: 61.149107
Epoch 4388 (4388 processed examples, curriculum: 20), error: 63.815367
    Testing sequence length 10: 
        Cost per sequence: 38.800000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 76.300000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 161.000000 (chance: 160.000000)
Epoch 4390 (4390 processed examples, curriculum: 20), error: 66.623451
Epoch 4397 (4397 processed examples, curriculum: 20), error: 69.476345
Epoch 4408 (4408 processed examples, curriculum: 20), error: 66.605465
Epoch 4414 (4414 processed examples, curriculum: 20), error: 72.211079
Epoch 4424 (4424 processed examples, curriculum: 20), error: 72.061420
Epoch 4431 (4431 processed examples, curriculum: 20), error: 72.211079
Epoch 4438 (4438 processed examples, curriculum: 20), error: 72.061420
Epoch 4449 (4449 processed examples, curriculum: 20), error: 66.605465
Epoch 4458 (4458 processed examples, curriculum: 20), error: 66.372932
Epoch 4466 (4466 processed examples, curriculum: 20), error: 66.186543
Epoch 4473 (4473 processed examples, curriculum: 20), error: 66.531132
Epoch 4480 (4480 processed examples, curriculum: 20), error: 66.774971
Epoch 4487 (4487 processed examples, curriculum: 20), error: 69.248730
Epoch 4494 (4494 processed examples, curriculum: 20), error: 66.531132
Epoch 4503 (4503 processed examples, curriculum: 20), error: 63.398819
Epoch 4512 (4512 processed examples, curriculum: 20), error: 60.961094
Epoch 4519 (4519 processed examples, curriculum: 20), error: 61.084782
Epoch 4525 (4525 processed examples, curriculum: 20), error: 65.875538
Epoch 4533 (4533 processed examples, curriculum: 20), error: 65.875538
Epoch 4543 (4543 processed examples, curriculum: 20), error: 66.375980
Epoch 4555 (4555 processed examples, curriculum: 20), error: 60.872040
Epoch 4568 (4568 processed examples, curriculum: 20), error: 55.758665
Epoch 4579 (4579 processed examples, curriculum: 20), error: 49.927349
Epoch 4587 (4587 processed examples, curriculum: 20), error: 50.187120
Epoch 4593 (4593 processed examples, curriculum: 20), error: 55.041861
    Testing sequence length 10: 
        Cost per sequence: 38.700000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 75.900000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 158.800000 (chance: 160.000000)
Epoch 4596 (4596 processed examples, curriculum: 20), error: 55.041861
Epoch 4606 (4606 processed examples, curriculum: 20), error: 50.187120
Epoch 4614 (4614 processed examples, curriculum: 20), error: 50.194107
Epoch 4621 (4621 processed examples, curriculum: 20), error: 50.155142
Epoch 4630 (4630 processed examples, curriculum: 20), error: 50.028881
Epoch 4637 (4637 processed examples, curriculum: 20), error: 50.155142
Epoch 4646 (4646 processed examples, curriculum: 20), error: 50.194107
Epoch 4654 (4654 processed examples, curriculum: 20), error: 55.482374
Epoch 4663 (4663 processed examples, curriculum: 20), error: 60.625587
Epoch 4668 (4668 processed examples, curriculum: 20), error: 66.539845
Epoch 4676 (4676 processed examples, curriculum: 20), error: 67.025414
Epoch 4684 (4684 processed examples, curriculum: 20), error: 66.746811
Epoch 4692 (4692 processed examples, curriculum: 20), error: 66.443111
Epoch 4703 (4703 processed examples, curriculum: 20), error: 60.893190
Epoch 4713 (4713 processed examples, curriculum: 20), error: 60.943752
Epoch 4720 (4720 processed examples, curriculum: 20), error: 60.838535
Epoch 4730 (4730 processed examples, curriculum: 20), error: 58.098270
Epoch 4737 (4737 processed examples, curriculum: 20), error: 55.522207
Epoch 4745 (4745 processed examples, curriculum: 20), error: 55.664066
Epoch 4751 (4751 processed examples, curriculum: 20), error: 58.248455
Epoch 4758 (4758 processed examples, curriculum: 20), error: 58.248455
Epoch 4765 (4765 processed examples, curriculum: 20), error: 55.535044
Epoch 4772 (4772 processed examples, curriculum: 20), error: 52.724131
Epoch 4784 (4784 processed examples, curriculum: 20), error: 47.397560
Epoch 4794 (4794 processed examples, curriculum: 20), error: 47.397560
    Testing sequence length 10: 
        Cost per sequence: 40.200000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 77.200000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 158.700000 (chance: 160.000000)
Epoch 4798 (4798 processed examples, curriculum: 20), error: 50.049002
Epoch 4805 (4805 processed examples, curriculum: 20), error: 50.049002
Epoch 4812 (4812 processed examples, curriculum: 20), error: 55.375677
Epoch 4820 (4820 processed examples, curriculum: 20), error: 52.495232
Epoch 4827 (4827 processed examples, curriculum: 20), error: 55.375677
Epoch 4836 (4836 processed examples, curriculum: 20), error: 55.662022
Epoch 4843 (4843 processed examples, curriculum: 20), error: 55.571007
Epoch 4852 (4852 processed examples, curriculum: 20), error: 52.640028
Epoch 4860 (4860 processed examples, curriculum: 20), error: 52.640028
Epoch 4866 (4866 processed examples, curriculum: 20), error: 54.591253
Epoch 4875 (4875 processed examples, curriculum: 20), error: 55.282614
Epoch 4882 (4882 processed examples, curriculum: 20), error: 58.141268
Epoch 4889 (4889 processed examples, curriculum: 20), error: 61.067755
Epoch 4896 (4896 processed examples, curriculum: 20), error: 60.958584
Epoch 4907 (4907 processed examples, curriculum: 20), error: 55.557137
Epoch 4917 (4917 processed examples, curriculum: 20), error: 55.224873
Epoch 4926 (4926 processed examples, curriculum: 20), error: 54.724958
Epoch 4936 (4936 processed examples, curriculum: 20), error: 54.717251
Epoch 4942 (4942 processed examples, curriculum: 20), error: 55.416319
Epoch 4952 (4952 processed examples, curriculum: 20), error: 55.350870
Epoch 4964 (4964 processed examples, curriculum: 20), error: 49.977570
Epoch 4977 (4977 processed examples, curriculum: 20), error: 49.839954
Epoch 4985 (4985 processed examples, curriculum: 20), error: 49.839954
Epoch 4995 (4995 processed examples, curriculum: 20), error: 44.399464
    Testing sequence length 10: 
        Cost per sequence: 37.000000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 81.600000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 159.300000 (chance: 160.000000)
Epoch 4996 (4996 processed examples, curriculum: 20), error: 46.786640
Epoch 5004 (5004 processed examples, curriculum: 20), error: 49.885141
Epoch 5010 (5010 processed examples, curriculum: 20), error: 50.058815
Epoch 5016 (5016 processed examples, curriculum: 20), error: 53.029385
Epoch 5025 (5025 processed examples, curriculum: 20), error: 54.922743
Epoch 5032 (5032 processed examples, curriculum: 20), error: 55.377798
Epoch 5041 (5041 processed examples, curriculum: 20), error: 50.414413
Epoch 5048 (5048 processed examples, curriculum: 20), error: 54.960379
Epoch 5054 (5054 processed examples, curriculum: 20), error: 55.493212
Epoch 5062 (5062 processed examples, curriculum: 20), error: 56.427242
Epoch 5069 (5069 processed examples, curriculum: 20), error: 58.315802
Epoch 5077 (5077 processed examples, curriculum: 20), error: 61.123909
Epoch 5084 (5084 processed examples, curriculum: 20), error: 61.123909
Epoch 5089 (5089 processed examples, curriculum: 20), error: 61.023712
Epoch 5092 (5092 processed examples, curriculum: 20), error: 61.472504
Epoch 5095 (5095 processed examples, curriculum: 20), error: 63.759468
Epoch 5100 (5100 processed examples, curriculum: 20), error: 61.881081
Epoch 5106 (5106 processed examples, curriculum: 20), error: 61.290230
Epoch 5114 (5114 processed examples, curriculum: 20), error: 60.741434
Epoch 5120 (5120 processed examples, curriculum: 20), error: 60.741434
Epoch 5124 (5124 processed examples, curriculum: 20), error: 60.616486
Epoch 5129 (5129 processed examples, curriculum: 20), error: 60.741434
Epoch 5137 (5137 processed examples, curriculum: 20), error: 60.616486
Epoch 5141 (5141 processed examples, curriculum: 20), error: 60.560850
    Testing sequence length 10: 
        Cost per sequence: 40.300000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 79.100000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 157.500000 (chance: 160.000000)
Epoch 5145 (5145 processed examples, curriculum: 20), error: 60.492853
Epoch 5153 (5153 processed examples, curriculum: 20), error: 57.725142
Epoch 5160 (5160 processed examples, curriculum: 20), error: 57.725142
Epoch 5165 (5165 processed examples, curriculum: 20), error: 57.725142
Epoch 5171 (5171 processed examples, curriculum: 20), error: 59.983191
Epoch 5179 (5179 processed examples, curriculum: 20), error: 57.725142
Epoch 5186 (5186 processed examples, curriculum: 20), error: 60.682154
Epoch 5193 (5193 processed examples, curriculum: 20), error: 55.751200
Epoch 5199 (5199 processed examples, curriculum: 20), error: 55.328371
Epoch 5204 (5204 processed examples, curriculum: 20), error: 55.468544
Epoch 5210 (5210 processed examples, curriculum: 20), error: 57.725142
Epoch 5216 (5216 processed examples, curriculum: 20), error: 60.048859
Epoch 5221 (5221 processed examples, curriculum: 20), error: 60.755083
Epoch 5228 (5228 processed examples, curriculum: 20), error: 58.302801
Epoch 5231 (5231 processed examples, curriculum: 20), error: 61.191914
Epoch 5236 (5236 processed examples, curriculum: 20), error: 61.191914
Epoch 5243 (5243 processed examples, curriculum: 20), error: 65.707073
Epoch 5248 (5248 processed examples, curriculum: 20), error: 66.597618
Epoch 5256 (5256 processed examples, curriculum: 20), error: 66.636627
Epoch 5265 (5265 processed examples, curriculum: 20), error: 66.636627
Epoch 5271 (5271 processed examples, curriculum: 20), error: 66.714233
Epoch 5279 (5279 processed examples, curriculum: 20), error: 66.873661
Epoch 5286 (5286 processed examples, curriculum: 20), error: 66.873661
    Testing sequence length 10: 
        Cost per sequence: 42.500000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 79.200000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 161.800000 (chance: 160.000000)
Epoch 5287 (5287 processed examples, curriculum: 20), error: 69.120487
Epoch 5293 (5293 processed examples, curriculum: 20), error: 71.910717
Epoch 5302 (5302 processed examples, curriculum: 20), error: 69.120487
Epoch 5307 (5307 processed examples, curriculum: 20), error: 69.120487
Epoch 5312 (5312 processed examples, curriculum: 20), error: 69.120487
Epoch 5318 (5318 processed examples, curriculum: 20), error: 66.641350
Epoch 5324 (5324 processed examples, curriculum: 20), error: 66.641350
Epoch 5330 (5330 processed examples, curriculum: 20), error: 69.120487
Epoch 5336 (5336 processed examples, curriculum: 20), error: 69.120487
Epoch 5341 (5341 processed examples, curriculum: 20), error: 69.239086
Epoch 5346 (5346 processed examples, curriculum: 20), error: 69.239086
Epoch 5355 (5355 processed examples, curriculum: 20), error: 61.066311
Epoch 5363 (5363 processed examples, curriculum: 20), error: 60.990389
Epoch 5368 (5368 processed examples, curriculum: 20), error: 61.066311
Epoch 5375 (5375 processed examples, curriculum: 20), error: 61.066311
Epoch 5379 (5379 processed examples, curriculum: 20), error: 63.815823
Epoch 5385 (5385 processed examples, curriculum: 20), error: 63.860880
Epoch 5391 (5391 processed examples, curriculum: 20), error: 66.446152
Epoch 5398 (5398 processed examples, curriculum: 20), error: 66.241123
Epoch 5405 (5405 processed examples, curriculum: 20), error: 66.446152
Epoch 5411 (5411 processed examples, curriculum: 20), error: 66.553055
Epoch 5415 (5415 processed examples, curriculum: 20), error: 66.679771
Epoch 5421 (5421 processed examples, curriculum: 20), error: 66.679771
Epoch 5430 (5430 processed examples, curriculum: 20), error: 66.241123
    Testing sequence length 10: 
        Cost per sequence: 40.600000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 80.000000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 163.800000 (chance: 160.000000)
Epoch 5433 (5433 processed examples, curriculum: 20), error: 66.446152
Epoch 5438 (5438 processed examples, curriculum: 20), error: 66.446152
Epoch 5447 (5447 processed examples, curriculum: 20), error: 63.809523
Epoch 5452 (5452 processed examples, curriculum: 20), error: 66.446152
Epoch 5459 (5459 processed examples, curriculum: 20), error: 66.446152
Epoch 5465 (5465 processed examples, curriculum: 20), error: 66.648182
Epoch 5474 (5474 processed examples, curriculum: 20), error: 66.241123
Epoch 5480 (5480 processed examples, curriculum: 20), error: 66.055595
Epoch 5486 (5486 processed examples, curriculum: 20), error: 66.610172
Epoch 5492 (5492 processed examples, curriculum: 20), error: 66.317741
Epoch 5498 (5498 processed examples, curriculum: 20), error: 66.648182
Epoch 5506 (5506 processed examples, curriculum: 20), error: 66.512825
Epoch 5512 (5512 processed examples, curriculum: 20), error: 66.225876
Epoch 5520 (5520 processed examples, curriculum: 20), error: 58.301100
Epoch 5527 (5527 processed examples, curriculum: 20), error: 66.512825
Epoch 5535 (5535 processed examples, curriculum: 20), error: 60.863770
Epoch 5542 (5542 processed examples, curriculum: 20), error: 58.259153
Epoch 5549 (5549 processed examples, curriculum: 20), error: 66.176350
Epoch 5557 (5557 processed examples, curriculum: 20), error: 66.176350
Epoch 5563 (5563 processed examples, curriculum: 20), error: 61.111784
Epoch 5568 (5568 processed examples, curriculum: 20), error: 66.176350
Epoch 5574 (5574 processed examples, curriculum: 20), error: 66.449966
Epoch 5579 (5579 processed examples, curriculum: 20), error: 66.176350
    Testing sequence length 10: 
        Cost per sequence: 41.300000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 76.100000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 156.600000 (chance: 160.000000)
Epoch 5585 (5585 processed examples, curriculum: 20), error: 61.111784
Epoch 5591 (5591 processed examples, curriculum: 20), error: 61.111784
Epoch 5597 (5597 processed examples, curriculum: 20), error: 60.990295
Epoch 5602 (5602 processed examples, curriculum: 20), error: 60.990295
Epoch 5613 (5613 processed examples, curriculum: 20), error: 55.563471
Epoch 5619 (5619 processed examples, curriculum: 20), error: 58.219799
Epoch 5623 (5623 processed examples, curriculum: 20), error: 60.948349
Epoch 5630 (5630 processed examples, curriculum: 20), error: 61.111784
Epoch 5637 (5637 processed examples, curriculum: 20), error: 61.289799
Epoch 5645 (5645 processed examples, curriculum: 20), error: 63.681322
Epoch 5653 (5653 processed examples, curriculum: 20), error: 58.419918
Epoch 5661 (5661 processed examples, curriculum: 20), error: 60.921457
Epoch 5664 (5664 processed examples, curriculum: 20), error: 60.921457
Epoch 5671 (5671 processed examples, curriculum: 20), error: 58.449883
Epoch 5680 (5680 processed examples, curriculum: 20), error: 55.506601
Epoch 5687 (5687 processed examples, curriculum: 20), error: 55.698034
Epoch 5693 (5693 processed examples, curriculum: 20), error: 55.698034
Epoch 5701 (5701 processed examples, curriculum: 20), error: 55.506601
Epoch 5708 (5708 processed examples, curriculum: 20), error: 55.874750
Epoch 5712 (5712 processed examples, curriculum: 20), error: 58.187670
Epoch 5717 (5717 processed examples, curriculum: 20), error: 60.595034
Epoch 5723 (5723 processed examples, curriculum: 20), error: 55.874750
    Testing sequence length 10: 
        Cost per sequence: 39.700000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 79.700000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 160.500000 (chance: 160.000000)
Epoch 5725 (5725 processed examples, curriculum: 20), error: 55.807823
Epoch 5731 (5731 processed examples, curriculum: 20), error: 55.807823
Epoch 5742 (5742 processed examples, curriculum: 20), error: 52.972637
Epoch 5747 (5747 processed examples, curriculum: 20), error: 55.477800
Epoch 5753 (5753 processed examples, curriculum: 20), error: 55.477800
Epoch 5767 (5767 processed examples, curriculum: 20), error: 50.075218
Epoch 5773 (5773 processed examples, curriculum: 20), error: 50.075218
Epoch 5780 (5780 processed examples, curriculum: 20), error: 55.313055
Epoch 5789 (5789 processed examples, curriculum: 20), error: 55.271715
Epoch 5798 (5798 processed examples, curriculum: 20), error: 55.271715
Epoch 5806 (5806 processed examples, curriculum: 20), error: 55.271715
Epoch 5817 (5817 processed examples, curriculum: 20), error: 50.259285
Epoch 5822 (5822 processed examples, curriculum: 20), error: 50.377069
Epoch 5829 (5829 processed examples, curriculum: 20), error: 50.374460
Epoch 5836 (5836 processed examples, curriculum: 20), error: 50.066563
Epoch 5845 (5845 processed examples, curriculum: 20), error: 49.867027
Epoch 5851 (5851 processed examples, curriculum: 20), error: 49.867027
Epoch 5858 (5858 processed examples, curriculum: 20), error: 49.867027
Epoch 5863 (5863 processed examples, curriculum: 20), error: 55.213480
Epoch 5869 (5869 processed examples, curriculum: 20), error: 55.289190
Epoch 5875 (5875 processed examples, curriculum: 20), error: 55.289190
Epoch 5883 (5883 processed examples, curriculum: 20), error: 55.213480
Epoch 5890 (5890 processed examples, curriculum: 20), error: 55.533312
    Testing sequence length 10: 
        Cost per sequence: 41.500000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 80.700000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 160.300000 (chance: 160.000000)
Epoch 5895 (5895 processed examples, curriculum: 20), error: 55.573565
Epoch 5900 (5900 processed examples, curriculum: 20), error: 55.666882
Epoch 5905 (5905 processed examples, curriculum: 20), error: 55.934978
Epoch 5911 (5911 processed examples, curriculum: 20), error: 56.999763
Epoch 5916 (5916 processed examples, curriculum: 20), error: 60.978273
Epoch 5926 (5926 processed examples, curriculum: 20), error: 55.615219
Epoch 5935 (5935 processed examples, curriculum: 20), error: 55.498039
Epoch 5941 (5941 processed examples, curriculum: 20), error: 59.360130
Epoch 5948 (5948 processed examples, curriculum: 20), error: 55.666882
Epoch 5954 (5954 processed examples, curriculum: 20), error: 56.750126
Epoch 5960 (5960 processed examples, curriculum: 20), error: 56.877193
Epoch 5966 (5966 processed examples, curriculum: 20), error: 55.906193
Epoch 5971 (5971 processed examples, curriculum: 20), error: 58.529200
Epoch 5980 (5980 processed examples, curriculum: 20), error: 55.779125
Epoch 5986 (5986 processed examples, curriculum: 20), error: 55.913380
Epoch 5994 (5994 processed examples, curriculum: 20), error: 55.913380
Epoch 6002 (6002 processed examples, curriculum: 20), error: 55.683739
Epoch 6012 (6012 processed examples, curriculum: 20), error: 55.304758
Epoch 6017 (6017 processed examples, curriculum: 20), error: 55.683739
Epoch 6025 (6025 processed examples, curriculum: 20), error: 55.880096
Epoch 6032 (6032 processed examples, curriculum: 20), error: 58.365299
Epoch 6041 (6041 processed examples, curriculum: 20), error: 55.913380
Epoch 6047 (6047 processed examples, curriculum: 20), error: 60.916773
Epoch 6052 (6052 processed examples, curriculum: 20), error: 61.685009
    Testing sequence length 10: 
        Cost per sequence: 38.500000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 84.000000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 162.100000 (chance: 160.000000)
Epoch 6055 (6055 processed examples, curriculum: 20), error: 61.685009
Epoch 6061 (6061 processed examples, curriculum: 20), error: 62.253265
Epoch 6069 (6069 processed examples, curriculum: 20), error: 66.208305
Epoch 6077 (6077 processed examples, curriculum: 20), error: 66.414654
Epoch 6087 (6087 processed examples, curriculum: 20), error: 64.452797
Epoch 6094 (6094 processed examples, curriculum: 20), error: 66.208305
Epoch 6099 (6099 processed examples, curriculum: 20), error: 66.353756
Epoch 6104 (6104 processed examples, curriculum: 20), error: 66.533615
Epoch 6113 (6113 processed examples, curriculum: 20), error: 66.439270
Epoch 6120 (6120 processed examples, curriculum: 20), error: 66.353756
Epoch 6131 (6131 processed examples, curriculum: 20), error: 66.283657
Epoch 6142 (6142 processed examples, curriculum: 20), error: 66.018398
Epoch 6150 (6150 processed examples, curriculum: 20), error: 63.748653
Epoch 6157 (6157 processed examples, curriculum: 20), error: 63.604321
Epoch 6162 (6162 processed examples, curriculum: 20), error: 63.604321
Epoch 6170 (6170 processed examples, curriculum: 20), error: 63.604321
Epoch 6177 (6177 processed examples, curriculum: 20), error: 61.291111
Epoch 6186 (6186 processed examples, curriculum: 20), error: 66.169186
Epoch 6195 (6195 processed examples, curriculum: 20), error: 66.169186
Epoch 6203 (6203 processed examples, curriculum: 20), error: 61.059975
Epoch 6214 (6214 processed examples, curriculum: 20), error: 58.144663
Epoch 6225 (6225 processed examples, curriculum: 20), error: 55.699562
Epoch 6239 (6239 processed examples, curriculum: 20), error: 55.228516
Epoch 6248 (6248 processed examples, curriculum: 20), error: 55.378672
Epoch 6258 (6258 processed examples, curriculum: 20), error: 52.804125
    Testing sequence length 10: 
        Cost per sequence: 39.500000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 78.800000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 156.100000 (chance: 160.000000)
Epoch 6263 (6263 processed examples, curriculum: 20), error: 50.091141
Epoch 6271 (6271 processed examples, curriculum: 20), error: 50.003466
Epoch 6279 (6279 processed examples, curriculum: 20), error: 49.920519
Epoch 6286 (6286 processed examples, curriculum: 20), error: 50.317572
Epoch 6294 (6294 processed examples, curriculum: 20), error: 52.804125
Epoch 6303 (6303 processed examples, curriculum: 20), error: 52.804125
Epoch 6310 (6310 processed examples, curriculum: 20), error: 55.450089
Epoch 6318 (6318 processed examples, curriculum: 20), error: 58.263725
Epoch 6325 (6325 processed examples, curriculum: 20), error: 63.786251
Epoch 6334 (6334 processed examples, curriculum: 20), error: 69.257370
Epoch 6346 (6346 processed examples, curriculum: 20), error: 69.257370
Epoch 6358 (6358 processed examples, curriculum: 20), error: 66.798756
Epoch 6367 (6367 processed examples, curriculum: 20), error: 66.629826
Epoch 6376 (6376 processed examples, curriculum: 20), error: 61.123493
Epoch 6384 (6384 processed examples, curriculum: 20), error: 61.134035
Epoch 6392 (6392 processed examples, curriculum: 20), error: 61.108341
Epoch 6400 (6400 processed examples, curriculum: 20), error: 61.131330
Epoch 6409 (6409 processed examples, curriculum: 20), error: 61.061794
Epoch 6417 (6417 processed examples, curriculum: 20), error: 61.131330
Epoch 6426 (6426 processed examples, curriculum: 20), error: 61.037037
Epoch 6435 (6435 processed examples, curriculum: 20), error: 58.236729
Epoch 6442 (6442 processed examples, curriculum: 20), error: 60.960529
Epoch 6448 (6448 processed examples, curriculum: 20), error: 61.131330
Epoch 6457 (6457 processed examples, curriculum: 20), error: 61.131330
Epoch 6466 (6466 processed examples, curriculum: 20), error: 61.070061
    Testing sequence length 10: 
        Cost per sequence: 40.100000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 80.100000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 163.000000 (chance: 160.000000)
Epoch 6468 (6468 processed examples, curriculum: 20), error: 60.993553
Epoch 6475 (6475 processed examples, curriculum: 20), error: 61.175846
Epoch 6481 (6481 processed examples, curriculum: 20), error: 61.298126
Epoch 6489 (6489 processed examples, curriculum: 20), error: 61.114576
Epoch 6496 (6496 processed examples, curriculum: 20), error: 61.298126
Epoch 6505 (6505 processed examples, curriculum: 20), error: 61.298126
Epoch 6512 (6512 processed examples, curriculum: 20), error: 61.114576
Epoch 6521 (6521 processed examples, curriculum: 20), error: 60.921185
Epoch 6530 (6530 processed examples, curriculum: 20), error: 60.852730
Epoch 6537 (6537 processed examples, curriculum: 20), error: 61.140650
Epoch 6546 (6546 processed examples, curriculum: 20), error: 60.951172
Epoch 6555 (6555 processed examples, curriculum: 20), error: 60.951172
Epoch 6563 (6563 processed examples, curriculum: 20), error: 61.392916
Epoch 6569 (6569 processed examples, curriculum: 20), error: 61.439163
Epoch 6578 (6578 processed examples, curriculum: 20), error: 61.392916
Epoch 6585 (6585 processed examples, curriculum: 20), error: 60.962955
Epoch 6594 (6594 processed examples, curriculum: 20), error: 61.235439
Epoch 6601 (6601 processed examples, curriculum: 20), error: 61.235439
Epoch 6607 (6607 processed examples, curriculum: 20), error: 66.349709
Epoch 6614 (6614 processed examples, curriculum: 20), error: 63.872787
Epoch 6621 (6621 processed examples, curriculum: 20), error: 63.872787
Epoch 6629 (6629 processed examples, curriculum: 20), error: 66.349709
Epoch 6635 (6635 processed examples, curriculum: 20), error: 66.418762
Epoch 6643 (6643 processed examples, curriculum: 20), error: 66.231873
    Testing sequence length 10: 
        Cost per sequence: 39.000000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 79.900000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 159.500000 (chance: 160.000000)
Epoch 6649 (6649 processed examples, curriculum: 20), error: 66.361958
Epoch 6657 (6657 processed examples, curriculum: 20), error: 66.349709
Epoch 6665 (6665 processed examples, curriculum: 20), error: 61.107018
Epoch 6672 (6672 processed examples, curriculum: 20), error: 66.349709
Epoch 6683 (6683 processed examples, curriculum: 20), error: 66.231873
Epoch 6691 (6691 processed examples, curriculum: 20), error: 63.768572
Epoch 6697 (6697 processed examples, curriculum: 20), error: 66.086521
Epoch 6706 (6706 processed examples, curriculum: 20), error: 66.012211
Epoch 6712 (6712 processed examples, curriculum: 20), error: 66.086521
Epoch 6721 (6721 processed examples, curriculum: 20), error: 63.606182
Epoch 6732 (6732 processed examples, curriculum: 20), error: 61.429937
Epoch 6742 (6742 processed examples, curriculum: 20), error: 61.212831
Epoch 6751 (6751 processed examples, curriculum: 20), error: 61.212831
Epoch 6761 (6761 processed examples, curriculum: 20), error: 60.923441
Epoch 6769 (6769 processed examples, curriculum: 20), error: 61.238476
Epoch 6779 (6779 processed examples, curriculum: 20), error: 60.501814
Epoch 6788 (6788 processed examples, curriculum: 20), error: 60.825722
Epoch 6799 (6799 processed examples, curriculum: 20), error: 60.653212
Epoch 6809 (6809 processed examples, curriculum: 20), error: 55.532589
Epoch 6818 (6818 processed examples, curriculum: 20), error: 60.653212
Epoch 6825 (6825 processed examples, curriculum: 20), error: 60.959658
Epoch 6835 (6835 processed examples, curriculum: 20), error: 61.007589
Epoch 6843 (6843 processed examples, curriculum: 20), error: 60.915901
Epoch 6851 (6851 processed examples, curriculum: 20), error: 60.915901
Epoch 6858 (6858 processed examples, curriculum: 20), error: 61.213207
    Testing sequence length 10: 
        Cost per sequence: 40.500000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 78.100000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 164.500000 (chance: 160.000000)
Epoch 6865 (6865 processed examples, curriculum: 20), error: 61.213207
Epoch 6874 (6874 processed examples, curriculum: 20), error: 61.213207
Epoch 6884 (6884 processed examples, curriculum: 20), error: 63.817678
Epoch 6893 (6893 processed examples, curriculum: 20), error: 61.051449
Epoch 6903 (6903 processed examples, curriculum: 20), error: 60.915901
Epoch 6911 (6911 processed examples, curriculum: 20), error: 63.941305
Epoch 6919 (6919 processed examples, curriculum: 20), error: 61.310019
Epoch 6924 (6924 processed examples, curriculum: 20), error: 61.310019
Epoch 6934 (6934 processed examples, curriculum: 20), error: 63.834417
Epoch 6945 (6945 processed examples, curriculum: 20), error: 60.762238
Epoch 6953 (6953 processed examples, curriculum: 20), error: 60.674845
Epoch 6963 (6963 processed examples, curriculum: 20), error: 60.674845
Epoch 6974 (6974 processed examples, curriculum: 20), error: 55.283640
Epoch 6984 (6984 processed examples, curriculum: 20), error: 52.732113
Epoch 6991 (6991 processed examples, curriculum: 20), error: 57.952137
Epoch 6998 (6998 processed examples, curriculum: 20), error: 61.070156
Epoch 7006 (7006 processed examples, curriculum: 20), error: 60.610001
Epoch 7016 (7016 processed examples, curriculum: 20), error: 57.981522
Epoch 7025 (7025 processed examples, curriculum: 20), error: 55.498913
Epoch 7035 (7035 processed examples, curriculum: 20), error: 55.498913
Epoch 7046 (7046 processed examples, curriculum: 20), error: 55.683968
Epoch 7054 (7054 processed examples, curriculum: 20), error: 55.683968
Epoch 7062 (7062 processed examples, curriculum: 20), error: 60.548529
Epoch 7069 (7069 processed examples, curriculum: 20), error: 61.099922
Epoch 7075 (7075 processed examples, curriculum: 20), error: 66.481548
    Testing sequence length 10: 
        Cost per sequence: 41.400000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 76.400000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 157.300000 (chance: 160.000000)
Epoch 7079 (7079 processed examples, curriculum: 20), error: 66.481548
Epoch 7088 (7088 processed examples, curriculum: 20), error: 66.358032
Epoch 7097 (7097 processed examples, curriculum: 20), error: 61.183851
Epoch 7107 (7107 processed examples, curriculum: 20), error: 61.183851
Epoch 7116 (7116 processed examples, curriculum: 20), error: 61.099922
Epoch 7125 (7125 processed examples, curriculum: 20), error: 63.809214
Epoch 7134 (7134 processed examples, curriculum: 20), error: 63.708033
Epoch 7143 (7143 processed examples, curriculum: 20), error: 66.296497
Epoch 7152 (7152 processed examples, curriculum: 20), error: 61.186472
Epoch 7162 (7162 processed examples, curriculum: 20), error: 61.025276
Epoch 7170 (7170 processed examples, curriculum: 20), error: 61.025276
Epoch 7179 (7179 processed examples, curriculum: 20), error: 60.858614
Epoch 7188 (7188 processed examples, curriculum: 20), error: 60.937252
Epoch 7199 (7199 processed examples, curriculum: 20), error: 60.650789
Epoch 7209 (7209 processed examples, curriculum: 20), error: 55.752529
Epoch 7215 (7215 processed examples, curriculum: 20), error: 60.924124
Epoch 7223 (7223 processed examples, curriculum: 20), error: 61.025276
Epoch 7231 (7231 processed examples, curriculum: 20), error: 61.152977
Epoch 7240 (7240 processed examples, curriculum: 20), error: 60.994322
Epoch 7251 (7251 processed examples, curriculum: 20), error: 60.994322
Epoch 7263 (7263 processed examples, curriculum: 20), error: 60.924124
Epoch 7272 (7272 processed examples, curriculum: 20), error: 55.593576
Epoch 7281 (7281 processed examples, curriculum: 20), error: 60.862326
Epoch 7289 (7289 processed examples, curriculum: 20), error: 55.427910
Epoch 7295 (7295 processed examples, curriculum: 20), error: 60.927275
    Testing sequence length 10: 
        Cost per sequence: 38.900000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 83.400000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 161.200000 (chance: 160.000000)
Epoch 7300 (7300 processed examples, curriculum: 20), error: 61.271975
Epoch 7311 (7311 processed examples, curriculum: 20), error: 61.196442
Epoch 7319 (7319 processed examples, curriculum: 20), error: 60.710108
Epoch 7329 (7329 processed examples, curriculum: 20), error: 60.515852
Epoch 7336 (7336 processed examples, curriculum: 20), error: 60.897261
Epoch 7343 (7343 processed examples, curriculum: 20), error: 60.764187
Epoch 7352 (7352 processed examples, curriculum: 20), error: 60.764187
Epoch 7359 (7359 processed examples, curriculum: 20), error: 61.049639
Epoch 7366 (7366 processed examples, curriculum: 20), error: 60.764187
Epoch 7372 (7372 processed examples, curriculum: 20), error: 61.196602
Epoch 7380 (7380 processed examples, curriculum: 20), error: 61.049639
Epoch 7388 (7388 processed examples, curriculum: 20), error: 61.004086
Epoch 7395 (7395 processed examples, curriculum: 20), error: 60.764187
Epoch 7403 (7403 processed examples, curriculum: 20), error: 60.708389
Epoch 7411 (7411 processed examples, curriculum: 20), error: 60.860767
Epoch 7419 (7419 processed examples, curriculum: 20), error: 61.011902
Epoch 7428 (7428 processed examples, curriculum: 20), error: 60.924381
Epoch 7435 (7435 processed examples, curriculum: 20), error: 60.924381
Epoch 7443 (7443 processed examples, curriculum: 20), error: 55.513575
Epoch 7449 (7449 processed examples, curriculum: 20), error: 60.868584
Epoch 7458 (7458 processed examples, curriculum: 20), error: 60.868584
Epoch 7463 (7463 processed examples, curriculum: 20), error: 61.151049
Epoch 7470 (7470 processed examples, curriculum: 20), error: 61.011902
Epoch 7478 (7478 processed examples, curriculum: 20), error: 66.360897
    Testing sequence length 10: 
        Cost per sequence: 37.900000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 77.500000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 156.500000 (chance: 160.000000)
Epoch 7485 (7485 processed examples, curriculum: 20), error: 61.345196
Epoch 7493 (7493 processed examples, curriculum: 20), error: 61.345196
Epoch 7499 (7499 processed examples, curriculum: 20), error: 66.149704
Epoch 7506 (7506 processed examples, curriculum: 20), error: 66.034706
Epoch 7513 (7513 processed examples, curriculum: 20), error: 66.034706
Epoch 7519 (7519 processed examples, curriculum: 20), error: 66.245899
Epoch 7526 (7526 processed examples, curriculum: 20), error: 66.429211
Epoch 7532 (7532 processed examples, curriculum: 20), error: 66.434822
Epoch 7538 (7538 processed examples, curriculum: 20), error: 69.163315
Epoch 7548 (7548 processed examples, curriculum: 20), error: 66.278946
Epoch 7556 (7556 processed examples, curriculum: 20), error: 66.145893
Epoch 7561 (7561 processed examples, curriculum: 20), error: 66.296986
Epoch 7571 (7571 processed examples, curriculum: 20), error: 63.371473
Epoch 7582 (7582 processed examples, curriculum: 20), error: 57.935471
Epoch 7594 (7594 processed examples, curriculum: 20), error: 50.113960
Epoch 7601 (7601 processed examples, curriculum: 20), error: 49.947098
Epoch 7609 (7609 processed examples, curriculum: 20), error: 49.947098
Epoch 7618 (7618 processed examples, curriculum: 20), error: 49.751652
Epoch 7625 (7625 processed examples, curriculum: 20), error: 49.864731
Epoch 7633 (7633 processed examples, curriculum: 20), error: 49.751652
Epoch 7640 (7640 processed examples, curriculum: 20), error: 47.116550
Epoch 7647 (7647 processed examples, curriculum: 20), error: 49.834019
Epoch 7655 (7655 processed examples, curriculum: 20), error: 50.179243
    Testing sequence length 10: 
        Cost per sequence: 40.200000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 79.000000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 162.700000 (chance: 160.000000)
Epoch 7657 (7657 processed examples, curriculum: 20), error: 49.834019
Epoch 7663 (7663 processed examples, curriculum: 20), error: 50.179243
Epoch 7670 (7670 processed examples, curriculum: 20), error: 52.594339
Epoch 7676 (7676 processed examples, curriculum: 20), error: 55.293451
Epoch 7685 (7685 processed examples, curriculum: 20), error: 55.453127
Epoch 7692 (7692 processed examples, curriculum: 20), error: 58.397518
Epoch 7697 (7697 processed examples, curriculum: 20), error: 58.397518
Epoch 7704 (7704 processed examples, curriculum: 20), error: 55.559660
Epoch 7712 (7712 processed examples, curriculum: 20), error: 55.559660
Epoch 7721 (7721 processed examples, curriculum: 20), error: 60.810513
Epoch 7728 (7728 processed examples, curriculum: 20), error: 55.578388
Epoch 7737 (7737 processed examples, curriculum: 20), error: 55.520159
Epoch 7746 (7746 processed examples, curriculum: 20), error: 52.981091
Epoch 7754 (7754 processed examples, curriculum: 20), error: 55.214741
Epoch 7762 (7762 processed examples, curriculum: 20), error: 55.214741
Epoch 7773 (7773 processed examples, curriculum: 20), error: 50.177958
Epoch 7781 (7781 processed examples, curriculum: 20), error: 50.476055
Epoch 7790 (7790 processed examples, curriculum: 20), error: 50.476055
Epoch 7798 (7798 processed examples, curriculum: 20), error: 52.788891
Epoch 7805 (7805 processed examples, curriculum: 20), error: 52.788891
Epoch 7812 (7812 processed examples, curriculum: 20), error: 55.201050
Epoch 7822 (7822 processed examples, curriculum: 20), error: 55.201050
Epoch 7829 (7829 processed examples, curriculum: 20), error: 55.379074
    Testing sequence length 10: 
        Cost per sequence: 38.400000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 81.800000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 162.200000 (chance: 160.000000)
Epoch 7831 (7831 processed examples, curriculum: 20), error: 55.421400
Epoch 7839 (7839 processed examples, curriculum: 20), error: 55.421400
Epoch 7846 (7846 processed examples, curriculum: 20), error: 60.806938
Epoch 7854 (7854 processed examples, curriculum: 20), error: 60.806938
Epoch 7862 (7862 processed examples, curriculum: 20), error: 60.806938
Epoch 7869 (7869 processed examples, curriculum: 20), error: 61.031555
Epoch 7878 (7878 processed examples, curriculum: 20), error: 61.031555
Epoch 7887 (7887 processed examples, curriculum: 20), error: 60.828085
Epoch 7896 (7896 processed examples, curriculum: 20), error: 58.215141
Epoch 7903 (7903 processed examples, curriculum: 20), error: 55.555630
Epoch 7910 (7910 processed examples, curriculum: 20), error: 55.349136
Epoch 7915 (7915 processed examples, curriculum: 20), error: 60.720877
Epoch 7921 (7921 processed examples, curriculum: 20), error: 66.529846
Epoch 7932 (7932 processed examples, curriculum: 20), error: 60.720877
Epoch 7938 (7938 processed examples, curriculum: 20), error: 60.944794
Epoch 7946 (7946 processed examples, curriculum: 20), error: 60.827505
Epoch 7954 (7954 processed examples, curriculum: 20), error: 58.236288
Epoch 7963 (7963 processed examples, curriculum: 20), error: 58.236288
Epoch 7973 (7973 processed examples, curriculum: 20), error: 55.474268
Epoch 7983 (7983 processed examples, curriculum: 20), error: 55.474268
Epoch 7992 (7992 processed examples, curriculum: 20), error: 55.527212
Epoch 8001 (8001 processed examples, curriculum: 20), error: 60.917253
Epoch 8009 (8009 processed examples, curriculum: 20), error: 60.917253
Epoch 8015 (8015 processed examples, curriculum: 20), error: 55.527212
    Testing sequence length 10: 
        Cost per sequence: 41.800000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 78.200000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 161.500000 (chance: 160.000000)
Epoch 8022 (8022 processed examples, curriculum: 20), error: 55.375904
Epoch 8030 (8030 processed examples, curriculum: 20), error: 55.521090
Epoch 8037 (8037 processed examples, curriculum: 20), error: 55.521090
Epoch 8048 (8048 processed examples, curriculum: 20), error: 55.371096
Epoch 8057 (8057 processed examples, curriculum: 20), error: 55.371096
Epoch 8064 (8064 processed examples, curriculum: 20), error: 58.342607
Epoch 8073 (8073 processed examples, curriculum: 20), error: 58.457100
Epoch 8084 (8084 processed examples, curriculum: 20), error: 55.861559
Epoch 8095 (8095 processed examples, curriculum: 20), error: 55.632723
Epoch 8102 (8102 processed examples, curriculum: 20), error: 55.861559
Epoch 8109 (8109 processed examples, curriculum: 20), error: 58.484602
Epoch 8117 (8117 processed examples, curriculum: 20), error: 58.484602
Epoch 8126 (8126 processed examples, curriculum: 20), error: 55.607943
Epoch 8135 (8135 processed examples, curriculum: 20), error: 55.861559
Epoch 8144 (8144 processed examples, curriculum: 20), error: 61.123043
Epoch 8152 (8152 processed examples, curriculum: 20), error: 61.163788
Epoch 8159 (8159 processed examples, curriculum: 20), error: 61.092056
Epoch 8164 (8164 processed examples, curriculum: 20), error: 61.163788
Epoch 8170 (8170 processed examples, curriculum: 20), error: 66.149963
Epoch 8178 (8178 processed examples, curriculum: 20), error: 66.415672
Epoch 8185 (8185 processed examples, curriculum: 20), error: 66.415672
Epoch 8191 (8191 processed examples, curriculum: 20), error: 66.966015
Epoch 8197 (8197 processed examples, curriculum: 20), error: 67.248413
Epoch 8204 (8204 processed examples, curriculum: 20), error: 66.775047
    Testing sequence length 10: 
        Cost per sequence: 41.700000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 82.900000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 163.000000 (chance: 160.000000)
Epoch 8211 (8211 processed examples, curriculum: 20), error: 66.415672
Epoch 8220 (8220 processed examples, curriculum: 20), error: 66.115482
Epoch 8227 (8227 processed examples, curriculum: 20), error: 66.234184
Epoch 8236 (8236 processed examples, curriculum: 20), error: 66.449890
Epoch 8242 (8242 processed examples, curriculum: 20), error: 66.449890
Epoch 8253 (8253 processed examples, curriculum: 20), error: 61.243790
Epoch 8261 (8261 processed examples, curriculum: 20), error: 61.095638
Epoch 8269 (8269 processed examples, curriculum: 20), error: 60.602999
Epoch 8275 (8275 processed examples, curriculum: 20), error: 60.827749
Epoch 8282 (8282 processed examples, curriculum: 20), error: 61.034288
Epoch 8293 (8293 processed examples, curriculum: 20), error: 55.203587
Epoch 8301 (8301 processed examples, curriculum: 20), error: 50.190060
Epoch 8309 (8309 processed examples, curriculum: 20), error: 50.190060
Epoch 8319 (8319 processed examples, curriculum: 20), error: 49.627653
Epoch 8328 (8328 processed examples, curriculum: 20), error: 49.960592
Epoch 8336 (8336 processed examples, curriculum: 20), error: 55.223673
Epoch 8346 (8346 processed examples, curriculum: 20), error: 50.321302
Epoch 8355 (8355 processed examples, curriculum: 20), error: 55.726112
Epoch 8362 (8362 processed examples, curriculum: 20), error: 55.508062
Epoch 8370 (8370 processed examples, curriculum: 20), error: 55.191191
Epoch 8377 (8377 processed examples, curriculum: 20), error: 50.253515
Epoch 8382 (8382 processed examples, curriculum: 20), error: 52.759829
Epoch 8392 (8392 processed examples, curriculum: 20), error: 52.759829
Epoch 8400 (8400 processed examples, curriculum: 20), error: 55.037384
    Testing sequence length 10: 
        Cost per sequence: 41.500000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 76.600000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 159.500000 (chance: 160.000000)
Epoch 8407 (8407 processed examples, curriculum: 20), error: 55.037384
Epoch 8414 (8414 processed examples, curriculum: 20), error: 55.191191
Epoch 8422 (8422 processed examples, curriculum: 20), error: 55.325251
Epoch 8433 (8433 processed examples, curriculum: 20), error: 52.671558
Epoch 8442 (8442 processed examples, curriculum: 20), error: 55.225170
Epoch 8449 (8449 processed examples, curriculum: 20), error: 55.037384
Epoch 8461 (8461 processed examples, curriculum: 20), error: 50.119398
Epoch 8468 (8468 processed examples, curriculum: 20), error: 50.259886
Epoch 8474 (8474 processed examples, curriculum: 20), error: 55.171444
Epoch 8481 (8481 processed examples, curriculum: 20), error: 52.822418
Epoch 8489 (8489 processed examples, curriculum: 20), error: 55.394508
Epoch 8498 (8498 processed examples, curriculum: 20), error: 55.487562
Epoch 8504 (8504 processed examples, curriculum: 20), error: 60.774448
Epoch 8510 (8510 processed examples, curriculum: 20), error: 60.774448
Epoch 8517 (8517 processed examples, curriculum: 20), error: 55.619543
Epoch 8524 (8524 processed examples, curriculum: 20), error: 55.555664
Epoch 8535 (8535 processed examples, curriculum: 20), error: 55.555664
Epoch 8541 (8541 processed examples, curriculum: 20), error: 55.621706
Epoch 8546 (8546 processed examples, curriculum: 20), error: 60.774448
Epoch 8555 (8555 processed examples, curriculum: 20), error: 58.141327
Epoch 8561 (8561 processed examples, curriculum: 20), error: 61.145481
Epoch 8568 (8568 processed examples, curriculum: 20), error: 55.621706
Epoch 8577 (8577 processed examples, curriculum: 20), error: 55.440947
Epoch 8583 (8583 processed examples, curriculum: 20), error: 55.555664
    Testing sequence length 10: 
        Cost per sequence: 39.500000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 82.500000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 164.500000 (chance: 160.000000)
Epoch 8588 (8588 processed examples, curriculum: 20), error: 60.660286
Epoch 8596 (8596 processed examples, curriculum: 20), error: 57.963285
Epoch 8603 (8603 processed examples, curriculum: 20), error: 55.719696
Epoch 8611 (8611 processed examples, curriculum: 20), error: 55.389002
Epoch 8620 (8620 processed examples, curriculum: 20), error: 50.102224
Epoch 8626 (8626 processed examples, curriculum: 20), error: 55.126684
Epoch 8631 (8631 processed examples, curriculum: 20), error: 58.125154
Epoch 8640 (8640 processed examples, curriculum: 20), error: 60.820461
Epoch 8647 (8647 processed examples, curriculum: 20), error: 58.125154
Epoch 8654 (8654 processed examples, curriculum: 20), error: 61.026731
Epoch 8661 (8661 processed examples, curriculum: 20), error: 58.125154
Epoch 8669 (8669 processed examples, curriculum: 20), error: 58.125154
Epoch 8674 (8674 processed examples, curriculum: 20), error: 60.641993
Epoch 8680 (8680 processed examples, curriculum: 20), error: 61.054319
Epoch 8691 (8691 processed examples, curriculum: 20), error: 55.470423
Epoch 8699 (8699 processed examples, curriculum: 20), error: 55.470423
Epoch 8705 (8705 processed examples, curriculum: 20), error: 58.107590
Epoch 8715 (8715 processed examples, curriculum: 20), error: 55.494530
Epoch 8723 (8723 processed examples, curriculum: 20), error: 58.107590
Epoch 8732 (8732 processed examples, curriculum: 20), error: 55.390448
Epoch 8741 (8741 processed examples, curriculum: 20), error: 55.494530
Epoch 8749 (8749 processed examples, curriculum: 20), error: 55.461355
Epoch 8757 (8757 processed examples, curriculum: 20), error: 58.088146
Epoch 8766 (8766 processed examples, curriculum: 20), error: 55.461355
    Testing sequence length 10: 
        Cost per sequence: 40.100000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 78.500000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 161.800000 (chance: 160.000000)
Epoch 8769 (8769 processed examples, curriculum: 20), error: 55.461355
Epoch 8778 (8778 processed examples, curriculum: 20), error: 50.268566
Epoch 8786 (8786 processed examples, curriculum: 20), error: 55.588514
Epoch 8794 (8794 processed examples, curriculum: 20), error: 60.774939
Epoch 8802 (8802 processed examples, curriculum: 20), error: 60.770369
Epoch 8809 (8809 processed examples, curriculum: 20), error: 60.874746
Epoch 8818 (8818 processed examples, curriculum: 20), error: 60.874746
Epoch 8825 (8825 processed examples, curriculum: 20), error: 60.970110
Epoch 8834 (8834 processed examples, curriculum: 20), error: 60.877581
Epoch 8843 (8843 processed examples, curriculum: 20), error: 58.290949
Epoch 8849 (8849 processed examples, curriculum: 20), error: 60.814699
Epoch 8860 (8860 processed examples, curriculum: 20), error: 60.814699
Epoch 8867 (8867 processed examples, curriculum: 20), error: 60.877581
Epoch 8876 (8876 processed examples, curriculum: 20), error: 60.959642
Epoch 8883 (8883 processed examples, curriculum: 20), error: 60.959642
Epoch 8890 (8890 processed examples, curriculum: 20), error: 63.786028
Epoch 8898 (8898 processed examples, curriculum: 20), error: 58.258984
Epoch 8908 (8908 processed examples, curriculum: 20), error: 55.388769
Epoch 8916 (8916 processed examples, curriculum: 20), error: 55.603699
Epoch 8923 (8923 processed examples, curriculum: 20), error: 55.388769
Epoch 8931 (8931 processed examples, curriculum: 20), error: 55.388769
Epoch 8936 (8936 processed examples, curriculum: 20), error: 55.388769
Epoch 8946 (8946 processed examples, curriculum: 20), error: 55.301432
Epoch 8953 (8953 processed examples, curriculum: 20), error: 55.301432
    Testing sequence length 10: 
        Cost per sequence: 38.200000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 80.900000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 163.000000 (chance: 160.000000)
Epoch 8960 (8960 processed examples, curriculum: 20), error: 55.306417
Epoch 8968 (8968 processed examples, curriculum: 20), error: 55.274946
Epoch 8974 (8974 processed examples, curriculum: 20), error: 55.384111
Epoch 8981 (8981 processed examples, curriculum: 20), error: 55.306417
Epoch 8989 (8989 processed examples, curriculum: 20), error: 55.368921
Epoch 8995 (8995 processed examples, curriculum: 20), error: 55.448648
Epoch 9000 (9000 processed examples, curriculum: 20), error: 55.579115
Epoch 9005 (9005 processed examples, curriculum: 20), error: 60.757660
Epoch 9009 (9009 processed examples, curriculum: 20), error: 61.073177
Epoch 9016 (9016 processed examples, curriculum: 20), error: 55.579115
Epoch 9025 (9025 processed examples, curriculum: 20), error: 55.370588
Epoch 9036 (9036 processed examples, curriculum: 20), error: 49.976473
Epoch 9042 (9042 processed examples, curriculum: 20), error: 55.169674
Epoch 9047 (9047 processed examples, curriculum: 20), error: 55.566921
Epoch 9052 (9052 processed examples, curriculum: 20), error: 58.221201
Epoch 9058 (9058 processed examples, curriculum: 20), error: 58.221201
Epoch 9064 (9064 processed examples, curriculum: 20), error: 60.838694
Epoch 9071 (9071 processed examples, curriculum: 20), error: 60.838694
Epoch 9078 (9078 processed examples, curriculum: 20), error: 58.312422
Epoch 9086 (9086 processed examples, curriculum: 20), error: 60.838694
Epoch 9094 (9094 processed examples, curriculum: 20), error: 60.838694
Epoch 9100 (9100 processed examples, curriculum: 20), error: 58.406101
    Testing sequence length 10: 
        Cost per sequence: 38.900000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 80.800000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 160.100000 (chance: 160.000000)
Epoch 9105 (9105 processed examples, curriculum: 20), error: 60.957348
Epoch 9112 (9112 processed examples, curriculum: 20), error: 60.957348
Epoch 9118 (9118 processed examples, curriculum: 20), error: 71.583218
Epoch 9125 (9125 processed examples, curriculum: 20), error: 72.004280
Epoch 9133 (9133 processed examples, curriculum: 20), error: 72.143024
Epoch 9141 (9141 processed examples, curriculum: 20), error: 72.004826
Epoch 9148 (9148 processed examples, curriculum: 20), error: 71.769836
Epoch 9157 (9157 processed examples, curriculum: 20), error: 69.176907
Epoch 9163 (9163 processed examples, curriculum: 20), error: 71.951733
Epoch 9171 (9171 processed examples, curriculum: 20), error: 71.918709
Epoch 9178 (9178 processed examples, curriculum: 20), error: 72.162228
Epoch 9185 (9185 processed examples, curriculum: 20), error: 72.514690
Epoch 9192 (9192 processed examples, curriculum: 20), error: 72.514690
Epoch 9198 (9198 processed examples, curriculum: 20), error: 72.215496
Epoch 9206 (9206 processed examples, curriculum: 20), error: 72.215496
Epoch 9213 (9213 processed examples, curriculum: 20), error: 72.215496
Epoch 9219 (9219 processed examples, curriculum: 20), error: 72.215496
Epoch 9228 (9228 processed examples, curriculum: 20), error: 72.069996
Epoch 9235 (9235 processed examples, curriculum: 20), error: 72.309616
Epoch 9242 (9242 processed examples, curriculum: 20), error: 72.309616
Epoch 9253 (9253 processed examples, curriculum: 20), error: 71.988876
Epoch 9261 (9261 processed examples, curriculum: 20), error: 66.504539
Epoch 9266 (9266 processed examples, curriculum: 20), error: 71.988876
Epoch 9276 (9276 processed examples, curriculum: 20), error: 66.318039
    Testing sequence length 10: 
        Cost per sequence: 40.200000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 82.100000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 162.400000 (chance: 160.000000)
Epoch 9287 (9287 processed examples, curriculum: 20), error: 61.049076
Epoch 9293 (9293 processed examples, curriculum: 20), error: 61.216036
Epoch 9305 (9305 processed examples, curriculum: 20), error: 60.808683
Epoch 9314 (9314 processed examples, curriculum: 20), error: 55.332855
Epoch 9325 (9325 processed examples, curriculum: 20), error: 49.855986
Epoch 9334 (9334 processed examples, curriculum: 20), error: 50.093691
Epoch 9341 (9341 processed examples, curriculum: 20), error: 49.855986
Epoch 9352 (9352 processed examples, curriculum: 20), error: 49.881388
Epoch 9360 (9360 processed examples, curriculum: 20), error: 49.881388
Epoch 9366 (9366 processed examples, curriculum: 20), error: 49.881388
Epoch 9376 (9376 processed examples, curriculum: 20), error: 49.660023
Epoch 9386 (9386 processed examples, curriculum: 20), error: 50.054096
Epoch 9393 (9393 processed examples, curriculum: 20), error: 49.873463
Epoch 9400 (9400 processed examples, curriculum: 20), error: 52.752752
Epoch 9410 (9410 processed examples, curriculum: 20), error: 55.308451
Epoch 9420 (9420 processed examples, curriculum: 20), error: 55.308451
Epoch 9431 (9431 processed examples, curriculum: 20), error: 55.266180
Epoch 9439 (9439 processed examples, curriculum: 20), error: 55.266180
Epoch 9449 (9449 processed examples, curriculum: 20), error: 55.427435
Epoch 9459 (9459 processed examples, curriculum: 20), error: 55.405462
Epoch 9466 (9466 processed examples, curriculum: 20), error: 52.717653
Epoch 9475 (9475 processed examples, curriculum: 20), error: 55.405462
Epoch 9484 (9484 processed examples, curriculum: 20), error: 55.445736
Epoch 9492 (9492 processed examples, curriculum: 20), error: 55.370136
Epoch 9501 (9501 processed examples, curriculum: 20), error: 52.614311
    Testing sequence length 10: 
        Cost per sequence: 38.100000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 81.800000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 156.400000 (chance: 160.000000)
Epoch 9502 (9502 processed examples, curriculum: 20), error: 52.614311
Epoch 9510 (9510 processed examples, curriculum: 20), error: 55.370136
Epoch 9519 (9519 processed examples, curriculum: 20), error: 55.253933
Epoch 9525 (9525 processed examples, curriculum: 20), error: 55.475775
Epoch 9534 (9534 processed examples, curriculum: 20), error: 55.445736
Epoch 9544 (9544 processed examples, curriculum: 20), error: 52.772049
Epoch 9552 (9552 processed examples, curriculum: 20), error: 52.772049
Epoch 9560 (9560 processed examples, curriculum: 20), error: 55.308886
Epoch 9569 (9569 processed examples, curriculum: 20), error: 55.562881
Epoch 9578 (9578 processed examples, curriculum: 20), error: 55.307119
Epoch 9585 (9585 processed examples, curriculum: 20), error: 58.150873
Epoch 9595 (9595 processed examples, curriculum: 20), error: 55.472618
Epoch 9603 (9603 processed examples, curriculum: 20), error: 58.150873
Epoch 9612 (9612 processed examples, curriculum: 20), error: 55.444870
Epoch 9622 (9622 processed examples, curriculum: 20), error: 55.424648
Epoch 9629 (9629 processed examples, curriculum: 20), error: 55.472618
Epoch 9637 (9637 processed examples, curriculum: 20), error: 58.226595
Epoch 9646 (9646 processed examples, curriculum: 20), error: 55.472618
Epoch 9655 (9655 processed examples, curriculum: 20), error: 55.472618
Epoch 9664 (9664 processed examples, curriculum: 20), error: 55.472618
Epoch 9671 (9671 processed examples, curriculum: 20), error: 55.525434
Epoch 9679 (9679 processed examples, curriculum: 20), error: 55.659960
Epoch 9690 (9690 processed examples, curriculum: 20), error: 52.770014
Epoch 9700 (9700 processed examples, curriculum: 20), error: 50.455778
    Testing sequence length 10: 
        Cost per sequence: 42.200000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 80.900000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 164.800000 (chance: 160.000000)
Epoch 9706 (9706 processed examples, curriculum: 20), error: 55.260662
Epoch 9714 (9714 processed examples, curriculum: 20), error: 55.659960
Epoch 9723 (9723 processed examples, curriculum: 20), error: 55.659960
Epoch 9732 (9732 processed examples, curriculum: 20), error: 52.770014
Epoch 9739 (9739 processed examples, curriculum: 20), error: 55.636356
Epoch 9746 (9746 processed examples, curriculum: 20), error: 60.755848
Epoch 9754 (9754 processed examples, curriculum: 20), error: 60.979700
Epoch 9765 (9765 processed examples, curriculum: 20), error: 55.636356
Epoch 9771 (9771 processed examples, curriculum: 20), error: 55.655037
Epoch 9780 (9780 processed examples, curriculum: 20), error: 55.655037
Epoch 9786 (9786 processed examples, curriculum: 20), error: 60.979700
Epoch 9795 (9795 processed examples, curriculum: 20), error: 60.870802
Epoch 9805 (9805 processed examples, curriculum: 20), error: 55.821255
Epoch 9811 (9811 processed examples, curriculum: 20), error: 60.870802
Epoch 9818 (9818 processed examples, curriculum: 20), error: 60.979700
Epoch 9824 (9824 processed examples, curriculum: 20), error: 61.094370
Epoch 9832 (9832 processed examples, curriculum: 20), error: 61.184122
Epoch 9840 (9840 processed examples, curriculum: 20), error: 61.127752
Epoch 9848 (9848 processed examples, curriculum: 20), error: 61.184122
Epoch 9857 (9857 processed examples, curriculum: 20), error: 61.115349
Epoch 9864 (9864 processed examples, curriculum: 20), error: 61.184122
Epoch 9872 (9872 processed examples, curriculum: 20), error: 61.115349
Epoch 9879 (9879 processed examples, curriculum: 20), error: 61.559412
Epoch 9889 (9889 processed examples, curriculum: 20), error: 61.280138
    Testing sequence length 10: 
        Cost per sequence: 38.800000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 79.300000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 163.000000 (chance: 160.000000)
Epoch 9896 (9896 processed examples, curriculum: 20), error: 61.280138
Epoch 9904 (9904 processed examples, curriculum: 20), error: 66.178753
Epoch 9910 (9910 processed examples, curriculum: 20), error: 63.807848
Epoch 9918 (9918 processed examples, curriculum: 20), error: 61.115349
Epoch 9928 (9928 processed examples, curriculum: 20), error: 60.971937
Epoch 9935 (9935 processed examples, curriculum: 20), error: 60.985762
Epoch 9946 (9946 processed examples, curriculum: 20), error: 60.971937
Epoch 9954 (9954 processed examples, curriculum: 20), error: 55.659748
Epoch 9965 (9965 processed examples, curriculum: 20), error: 55.610126
Epoch 9972 (9972 processed examples, curriculum: 20), error: 55.553642
Epoch 9976 (9976 processed examples, curriculum: 20), error: 55.553642
Epoch 9983 (9983 processed examples, curriculum: 20), error: 55.677477
Epoch 9990 (9990 processed examples, curriculum: 20), error: 55.553642
Epoch 10000 (10000 processed examples, curriculum: 20), error: 52.850941
Epoch 10010 (10010 processed examples, curriculum: 20), error: 49.881762
Epoch 10017 (10017 processed examples, curriculum: 20), error: 50.290689
Epoch 10023 (10023 processed examples, curriculum: 20), error: 52.850941
Epoch 10029 (10029 processed examples, curriculum: 20), error: 55.363014
Epoch 10038 (10038 processed examples, curriculum: 20), error: 50.025547
Epoch 10042 (10042 processed examples, curriculum: 20), error: 55.363014
Epoch 10048 (10048 processed examples, curriculum: 20), error: 52.850941
Epoch 10053 (10053 processed examples, curriculum: 20), error: 55.363014
Epoch 10058 (10058 processed examples, curriculum: 20), error: 55.589632
Epoch 10063 (10063 processed examples, curriculum: 20), error: 58.044901
    Testing sequence length 10: 
        Cost per sequence: 38.000000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 80.200000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 154.700000 (chance: 160.000000)
Epoch 10068 (10068 processed examples, curriculum: 20), error: 58.044901
Epoch 10073 (10073 processed examples, curriculum: 20), error: 52.955425
Epoch 10078 (10078 processed examples, curriculum: 20), error: 52.955425
Epoch 10082 (10082 processed examples, curriculum: 20), error: 55.553293
Epoch 10087 (10087 processed examples, curriculum: 20), error: 55.534435
Epoch 10092 (10092 processed examples, curriculum: 20), error: 55.534435
Epoch 10098 (10098 processed examples, curriculum: 20), error: 60.571259
Epoch 10105 (10105 processed examples, curriculum: 20), error: 60.712774
Epoch 10110 (10110 processed examples, curriculum: 20), error: 60.740700
Epoch 10117 (10117 processed examples, curriculum: 20), error: 60.571259
Epoch 10122 (10122 processed examples, curriculum: 20), error: 60.712774
Epoch 10129 (10129 processed examples, curriculum: 20), error: 58.044901
Epoch 10135 (10135 processed examples, curriculum: 20), error: 60.712774
Epoch 10140 (10140 processed examples, curriculum: 20), error: 60.746363
Epoch 10147 (10147 processed examples, curriculum: 20), error: 60.961948
Epoch 10154 (10154 processed examples, curriculum: 20), error: 60.983458
Epoch 10159 (10159 processed examples, curriculum: 20), error: 60.983458
Epoch 10166 (10166 processed examples, curriculum: 20), error: 61.094009
Epoch 10172 (10172 processed examples, curriculum: 20), error: 61.094009
Epoch 10180 (10180 processed examples, curriculum: 20), error: 61.033838
Epoch 10186 (10186 processed examples, curriculum: 20), error: 61.033838
    Testing sequence length 10: 
        Cost per sequence: 38.400000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 78.000000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 160.300000 (chance: 160.000000)
Epoch 10190 (10190 processed examples, curriculum: 20), error: 61.033838
Epoch 10198 (10198 processed examples, curriculum: 20), error: 61.030819
Epoch 10204 (10204 processed examples, curriculum: 20), error: 61.108385
Epoch 10212 (10212 processed examples, curriculum: 20), error: 61.108385
Epoch 10222 (10222 processed examples, curriculum: 20), error: 55.792738
Epoch 10228 (10228 processed examples, curriculum: 20), error: 58.415970
Epoch 10234 (10234 processed examples, curriculum: 20), error: 61.093309
Epoch 10239 (10239 processed examples, curriculum: 20), error: 61.064196
Epoch 10247 (10247 processed examples, curriculum: 20), error: 61.064196
Epoch 10252 (10252 processed examples, curriculum: 20), error: 61.093309
Epoch 10259 (10259 processed examples, curriculum: 20), error: 58.329395
Epoch 10266 (10266 processed examples, curriculum: 20), error: 55.564987
Epoch 10272 (10272 processed examples, curriculum: 20), error: 61.050903
Epoch 10282 (10282 processed examples, curriculum: 20), error: 58.422127
Epoch 10291 (10291 processed examples, curriculum: 20), error: 55.564987
Epoch 10299 (10299 processed examples, curriculum: 20), error: 58.328894
Epoch 10306 (10306 processed examples, curriculum: 20), error: 60.953959
Epoch 10313 (10313 processed examples, curriculum: 20), error: 60.953959
Epoch 10319 (10319 processed examples, curriculum: 20), error: 63.823875
Epoch 10326 (10326 processed examples, curriculum: 20), error: 61.200422
Epoch 10331 (10331 processed examples, curriculum: 20), error: 63.739952
Epoch 10338 (10338 processed examples, curriculum: 20), error: 63.739952
Epoch 10346 (10346 processed examples, curriculum: 20), error: 61.343313
    Testing sequence length 10: 
        Cost per sequence: 37.200000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 81.500000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 161.300000 (chance: 160.000000)
Epoch 10351 (10351 processed examples, curriculum: 20), error: 58.328894
Epoch 10359 (10359 processed examples, curriculum: 20), error: 63.882843
Epoch 10365 (10365 processed examples, curriculum: 20), error: 66.225651
Epoch 10371 (10371 processed examples, curriculum: 20), error: 60.972296
Epoch 10376 (10376 processed examples, curriculum: 20), error: 63.882843
Epoch 10381 (10381 processed examples, curriculum: 20), error: 63.882843
Epoch 10386 (10386 processed examples, curriculum: 20), error: 66.370350
Epoch 10391 (10391 processed examples, curriculum: 20), error: 63.882843
Epoch 10397 (10397 processed examples, curriculum: 20), error: 58.417288
Epoch 10400 (10400 processed examples, curriculum: 20), error: 63.882843
Epoch 10405 (10405 processed examples, curriculum: 20), error: 63.882843
Epoch 10410 (10410 processed examples, curriculum: 20), error: 63.882843
Epoch 10418 (10418 processed examples, curriculum: 20), error: 61.357939
Epoch 10424 (10424 processed examples, curriculum: 20), error: 61.357939
Epoch 10431 (10431 processed examples, curriculum: 20), error: 61.357939
Epoch 10438 (10438 processed examples, curriculum: 20), error: 60.809372
Epoch 10445 (10445 processed examples, curriculum: 20), error: 61.264484
Epoch 10454 (10454 processed examples, curriculum: 20), error: 60.992563
Epoch 10462 (10462 processed examples, curriculum: 20), error: 61.044384
Epoch 10468 (10468 processed examples, curriculum: 20), error: 60.809372
Epoch 10474 (10474 processed examples, curriculum: 20), error: 60.809372
Epoch 10485 (10485 processed examples, curriculum: 20), error: 52.320660
Epoch 10493 (10493 processed examples, curriculum: 20), error: 55.158407
    Testing sequence length 10: 
        Cost per sequence: 41.200000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 81.800000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 158.100000 (chance: 160.000000)
Epoch 10501 (10501 processed examples, curriculum: 20), error: 55.043846
Epoch 10507 (10507 processed examples, curriculum: 20), error: 55.043846
Epoch 10512 (10512 processed examples, curriculum: 20), error: 55.103451
Epoch 10519 (10519 processed examples, curriculum: 20), error: 58.188213
Epoch 10526 (10526 processed examples, curriculum: 20), error: 55.103451
Epoch 10532 (10532 processed examples, curriculum: 20), error: 55.218012
Epoch 10539 (10539 processed examples, curriculum: 20), error: 58.672163
Epoch 10545 (10545 processed examples, curriculum: 20), error: 55.815382
Epoch 10552 (10552 processed examples, curriculum: 20), error: 58.530134
Epoch 10558 (10558 processed examples, curriculum: 20), error: 58.530134
Epoch 10564 (10564 processed examples, curriculum: 20), error: 58.530134
Epoch 10571 (10571 processed examples, curriculum: 20), error: 60.902966
Epoch 10578 (10578 processed examples, curriculum: 20), error: 63.761133
Epoch 10587 (10587 processed examples, curriculum: 20), error: 63.699041
Epoch 10594 (10594 processed examples, curriculum: 20), error: 66.486206
Epoch 10601 (10601 processed examples, curriculum: 20), error: 66.486206
Epoch 10606 (10606 processed examples, curriculum: 20), error: 66.510517
Epoch 10611 (10611 processed examples, curriculum: 20), error: 66.486206
Epoch 10619 (10619 processed examples, curriculum: 20), error: 58.530134
Epoch 10625 (10625 processed examples, curriculum: 20), error: 60.848478
Epoch 10629 (10629 processed examples, curriculum: 20), error: 60.989008
Epoch 10634 (10634 processed examples, curriculum: 20), error: 60.989008
    Testing sequence length 10: 
        Cost per sequence: 40.800000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 79.500000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 161.500000 (chance: 160.000000)
Epoch 10638 (10638 processed examples, curriculum: 20), error: 60.989008
Epoch 10647 (10647 processed examples, curriculum: 20), error: 60.848478
Epoch 10653 (10653 processed examples, curriculum: 20), error: 60.989008
Epoch 10660 (10660 processed examples, curriculum: 20), error: 63.518808
Epoch 10669 (10669 processed examples, curriculum: 20), error: 55.327038
Epoch 10674 (10674 processed examples, curriculum: 20), error: 57.993738
Epoch 10683 (10683 processed examples, curriculum: 20), error: 55.215765
Epoch 10691 (10691 processed examples, curriculum: 20), error: 55.508167
Epoch 10697 (10697 processed examples, curriculum: 20), error: 55.716471
Epoch 10705 (10705 processed examples, curriculum: 20), error: 55.313248
Epoch 10711 (10711 processed examples, curriculum: 20), error: 55.552441
Epoch 10716 (10716 processed examples, curriculum: 20), error: 60.606398
Epoch 10723 (10723 processed examples, curriculum: 20), error: 60.606398
Epoch 10729 (10729 processed examples, curriculum: 20), error: 63.549469
Epoch 10737 (10737 processed examples, curriculum: 20), error: 63.549469
Epoch 10745 (10745 processed examples, curriculum: 20), error: 66.232368
Epoch 10751 (10751 processed examples, curriculum: 20), error: 66.475246
Epoch 10760 (10760 processed examples, curriculum: 20), error: 60.929232
Epoch 10766 (10766 processed examples, curriculum: 20), error: 61.098999
Epoch 10773 (10773 processed examples, curriculum: 20), error: 60.879930
Epoch 10780 (10780 processed examples, curriculum: 20), error: 61.045380
Epoch 10786 (10786 processed examples, curriculum: 20), error: 63.737633
Epoch 10794 (10794 processed examples, curriculum: 20), error: 61.098999
Epoch 10802 (10802 processed examples, curriculum: 20), error: 61.098999
    Testing sequence length 10: 
        Cost per sequence: 39.100000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 78.500000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 158.300000 (chance: 160.000000)
Epoch 10804 (10804 processed examples, curriculum: 20), error: 61.098999
Epoch 10808 (10808 processed examples, curriculum: 20), error: 61.085314
Epoch 10816 (10816 processed examples, curriculum: 20), error: 55.826963
Epoch 10822 (10822 processed examples, curriculum: 20), error: 61.036013
Epoch 10830 (10830 processed examples, curriculum: 20), error: 58.352118
Epoch 10836 (10836 processed examples, curriculum: 20), error: 60.902153
Epoch 10843 (10843 processed examples, curriculum: 20), error: 60.986916
Epoch 10851 (10851 processed examples, curriculum: 20), error: 60.795168
Epoch 10859 (10859 processed examples, curriculum: 20), error: 61.036013
Epoch 10870 (10870 processed examples, curriculum: 20), error: 58.383503
Epoch 10876 (10876 processed examples, curriculum: 20), error: 61.026850
Epoch 10884 (10884 processed examples, curriculum: 20), error: 60.861235
Epoch 10891 (10891 processed examples, curriculum: 20), error: 60.936834
Epoch 10897 (10897 processed examples, curriculum: 20), error: 61.149481
Epoch 10905 (10905 processed examples, curriculum: 20), error: 61.149481
Epoch 10912 (10912 processed examples, curriculum: 20), error: 61.032562
Epoch 10919 (10919 processed examples, curriculum: 20), error: 61.102972
Epoch 10926 (10926 processed examples, curriculum: 20), error: 61.149481
Epoch 10934 (10934 processed examples, curriculum: 20), error: 61.149481
Epoch 10943 (10943 processed examples, curriculum: 20), error: 60.921412
Epoch 10951 (10951 processed examples, curriculum: 20), error: 60.921412
Epoch 10958 (10958 processed examples, curriculum: 20), error: 60.921412
Epoch 10967 (10967 processed examples, curriculum: 20), error: 61.192881
Epoch 10973 (10973 processed examples, curriculum: 20), error: 66.456963
    Testing sequence length 10: 
        Cost per sequence: 38.900000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 78.800000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 157.800000 (chance: 160.000000)
Epoch 10974 (10974 processed examples, curriculum: 20), error: 66.456963
Epoch 10980 (10980 processed examples, curriculum: 20), error: 66.534340
Epoch 10990 (10990 processed examples, curriculum: 20), error: 66.313557
Epoch 10998 (10998 processed examples, curriculum: 20), error: 63.837069
Epoch 11005 (11005 processed examples, curriculum: 20), error: 63.837069
Epoch 11013 (11013 processed examples, curriculum: 20), error: 63.837069
Epoch 11022 (11022 processed examples, curriculum: 20), error: 61.023325
Epoch 11032 (11032 processed examples, curriculum: 20), error: 55.446817
Epoch 11038 (11038 processed examples, curriculum: 20), error: 55.446817
Epoch 11043 (11043 processed examples, curriculum: 20), error: 55.506449
Epoch 11048 (11048 processed examples, curriculum: 20), error: 58.227564
Epoch 11054 (11054 processed examples, curriculum: 20), error: 55.506449
Epoch 11063 (11063 processed examples, curriculum: 20), error: 58.227564
Epoch 11069 (11069 processed examples, curriculum: 20), error: 50.356016
Epoch 11078 (11078 processed examples, curriculum: 20), error: 49.912302
Epoch 11087 (11087 processed examples, curriculum: 20), error: 49.750898
Epoch 11091 (11091 processed examples, curriculum: 20), error: 50.037434
Epoch 11097 (11097 processed examples, curriculum: 20), error: 50.037434
Epoch 11102 (11102 processed examples, curriculum: 20), error: 50.037434
Epoch 11107 (11107 processed examples, curriculum: 20), error: 49.844652
Epoch 11113 (11113 processed examples, curriculum: 20), error: 49.848997
Epoch 11119 (11119 processed examples, curriculum: 20), error: 50.236752
Epoch 11128 (11128 processed examples, curriculum: 20), error: 58.023804
Epoch 11135 (11135 processed examples, curriculum: 20), error: 60.845901
Epoch 11142 (11142 processed examples, curriculum: 20), error: 60.845901
    Testing sequence length 10: 
        Cost per sequence: 39.000000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 80.400000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 162.600000 (chance: 160.000000)
Epoch 11143 (11143 processed examples, curriculum: 20), error: 60.862398
Epoch 11152 (11152 processed examples, curriculum: 20), error: 55.335428
Epoch 11159 (11159 processed examples, curriculum: 20), error: 60.865089
Epoch 11167 (11167 processed examples, curriculum: 20), error: 60.897905
Epoch 11175 (11175 processed examples, curriculum: 20), error: 63.962662
Epoch 11180 (11180 processed examples, curriculum: 20), error: 66.730820
Epoch 11185 (11185 processed examples, curriculum: 20), error: 66.730820
Epoch 11190 (11190 processed examples, curriculum: 20), error: 61.287481
Epoch 11194 (11194 processed examples, curriculum: 20), error: 60.988270
Epoch 11200 (11200 processed examples, curriculum: 20), error: 60.988270
Epoch 11205 (11205 processed examples, curriculum: 20), error: 60.918543
Epoch 11213 (11213 processed examples, curriculum: 20), error: 60.885765
Epoch 11220 (11220 processed examples, curriculum: 20), error: 60.885765
Epoch 11225 (11225 processed examples, curriculum: 20), error: 60.988270
Epoch 11234 (11234 processed examples, curriculum: 20), error: 58.200190
Epoch 11239 (11239 processed examples, curriculum: 20), error: 58.200190
Epoch 11243 (11243 processed examples, curriculum: 20), error: 60.918543
Epoch 11250 (11250 processed examples, curriculum: 20), error: 61.053610
Epoch 11257 (11257 processed examples, curriculum: 20), error: 61.067440
Epoch 11267 (11267 processed examples, curriculum: 20), error: 60.918543
Epoch 11273 (11273 processed examples, curriculum: 20), error: 58.353868
Epoch 11280 (11280 processed examples, curriculum: 20), error: 55.670639
Epoch 11289 (11289 processed examples, curriculum: 20), error: 55.670639
Epoch 11296 (11296 processed examples, curriculum: 20), error: 58.353868
    Testing sequence length 10: 
        Cost per sequence: 41.000000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 81.200000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 158.100000 (chance: 160.000000)
Epoch 11297 (11297 processed examples, curriculum: 20), error: 60.983883
Epoch 11303 (11303 processed examples, curriculum: 20), error: 61.098160
Epoch 11310 (11310 processed examples, curriculum: 20), error: 66.148418
Epoch 11317 (11317 processed examples, curriculum: 20), error: 63.660549
Epoch 11327 (11327 processed examples, curriculum: 20), error: 58.313921
Epoch 11334 (11334 processed examples, curriculum: 20), error: 60.947138
Epoch 11338 (11338 processed examples, curriculum: 20), error: 61.070642
Epoch 11343 (11343 processed examples, curriculum: 20), error: 58.313921
Epoch 11350 (11350 processed examples, curriculum: 20), error: 55.646580
Epoch 11355 (11355 processed examples, curriculum: 20), error: 58.344648
Epoch 11362 (11362 processed examples, curriculum: 20), error: 55.834370
Epoch 11369 (11369 processed examples, curriculum: 20), error: 60.947138
Epoch 11377 (11377 processed examples, curriculum: 20), error: 58.344648
Epoch 11382 (11382 processed examples, curriculum: 20), error: 60.947138
Epoch 11387 (11387 processed examples, curriculum: 20), error: 60.947138
Epoch 11390 (11390 processed examples, curriculum: 20), error: 63.605709
Epoch 11399 (11399 processed examples, curriculum: 20), error: 55.646450
Epoch 11405 (11405 processed examples, curriculum: 20), error: 58.344648
Epoch 11413 (11413 processed examples, curriculum: 20), error: 60.927168
Epoch 11417 (11417 processed examples, curriculum: 20), error: 60.927168
Epoch 11425 (11425 processed examples, curriculum: 20), error: 61.142017
Epoch 11432 (11432 processed examples, curriculum: 20), error: 61.329445
Epoch 11439 (11439 processed examples, curriculum: 20), error: 61.329445
    Testing sequence length 10: 
        Cost per sequence: 41.000000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 80.100000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 162.900000 (chance: 160.000000)
Epoch 11440 (11440 processed examples, curriculum: 20), error: 61.265667
Epoch 11448 (11448 processed examples, curriculum: 20), error: 61.329445
Epoch 11456 (11456 processed examples, curriculum: 20), error: 61.265667
Epoch 11463 (11463 processed examples, curriculum: 20), error: 61.329445
Epoch 11473 (11473 processed examples, curriculum: 20), error: 60.906675
Epoch 11482 (11482 processed examples, curriculum: 20), error: 60.995935
Epoch 11489 (11489 processed examples, curriculum: 20), error: 60.906675
Epoch 11499 (11499 processed examples, curriculum: 20), error: 60.823000
Epoch 11505 (11505 processed examples, curriculum: 20), error: 55.505371
Epoch 11510 (11510 processed examples, curriculum: 20), error: 55.505371
Epoch 11518 (11518 processed examples, curriculum: 20), error: 52.650650
Epoch 11524 (11524 processed examples, curriculum: 20), error: 55.405331
Epoch 11536 (11536 processed examples, curriculum: 20), error: 49.921284
Epoch 11543 (11543 processed examples, curriculum: 20), error: 47.218208
Epoch 11554 (11554 processed examples, curriculum: 20), error: 44.265585
Epoch 11562 (11562 processed examples, curriculum: 20), error: 44.155907
Epoch 11568 (11568 processed examples, curriculum: 20), error: 44.358721
Epoch 11575 (11575 processed examples, curriculum: 20), error: 44.426519
Epoch 11581 (11581 processed examples, curriculum: 20), error: 44.453793
Epoch 11588 (11588 processed examples, curriculum: 20), error: 44.453793
Epoch 11595 (11595 processed examples, curriculum: 20), error: 49.741293
Epoch 11604 (11604 processed examples, curriculum: 20), error: 49.862682
Epoch 11609 (11609 processed examples, curriculum: 20), error: 49.961737
Epoch 11613 (11613 processed examples, curriculum: 20), error: 52.757870
    Testing sequence length 10: 
        Cost per sequence: 38.100000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 78.900000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 156.300000 (chance: 160.000000)
Epoch 11614 (11614 processed examples, curriculum: 20), error: 52.757870
Epoch 11621 (11621 processed examples, curriculum: 20), error: 49.961737
Epoch 11626 (11626 processed examples, curriculum: 20), error: 49.862682
Epoch 11633 (11633 processed examples, curriculum: 20), error: 55.575533
Epoch 11639 (11639 processed examples, curriculum: 20), error: 58.198387
Epoch 11647 (11647 processed examples, curriculum: 20), error: 49.930943
Epoch 11654 (11654 processed examples, curriculum: 20), error: 52.793085
Epoch 11660 (11660 processed examples, curriculum: 20), error: 49.985571
Epoch 11667 (11667 processed examples, curriculum: 20), error: 55.585827
Epoch 11673 (11673 processed examples, curriculum: 20), error: 58.314421
Epoch 11681 (11681 processed examples, curriculum: 20), error: 58.324375
Epoch 11689 (11689 processed examples, curriculum: 20), error: 55.560377
Epoch 11695 (11695 processed examples, curriculum: 20), error: 55.562147
Epoch 11702 (11702 processed examples, curriculum: 20), error: 66.624935
Epoch 11714 (11714 processed examples, curriculum: 20), error: 55.562147
Epoch 11718 (11718 processed examples, curriculum: 20), error: 66.399231
Epoch 11722 (11722 processed examples, curriculum: 20), error: 60.979280
Epoch 11728 (11728 processed examples, curriculum: 20), error: 55.374474
Epoch 11732 (11732 processed examples, curriculum: 20), error: 55.654585
Epoch 11739 (11739 processed examples, curriculum: 20), error: 55.562147
Epoch 11747 (11747 processed examples, curriculum: 20), error: 58.144026
Epoch 11757 (11757 processed examples, curriculum: 20), error: 55.510592
    Testing sequence length 10: 
        Cost per sequence: 41.000000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 80.300000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 162.600000 (chance: 160.000000)
Epoch 11760 (11760 processed examples, curriculum: 20), error: 55.562147
Epoch 11766 (11766 processed examples, curriculum: 20), error: 58.144026
Epoch 11773 (11773 processed examples, curriculum: 20), error: 55.629135
Epoch 11780 (11780 processed examples, curriculum: 20), error: 60.728827
Epoch 11786 (11786 processed examples, curriculum: 20), error: 58.144026
Epoch 11791 (11791 processed examples, curriculum: 20), error: 60.728827
Epoch 11796 (11796 processed examples, curriculum: 20), error: 58.144026
Epoch 11804 (11804 processed examples, curriculum: 20), error: 52.737431
Epoch 11812 (11812 processed examples, curriculum: 20), error: 55.603029
Epoch 11817 (11817 processed examples, curriculum: 20), error: 60.670755
Epoch 11820 (11820 processed examples, curriculum: 20), error: 60.670755
Epoch 11823 (11823 processed examples, curriculum: 20), error: 60.670755
Epoch 11828 (11828 processed examples, curriculum: 20), error: 61.115000
Epoch 11835 (11835 processed examples, curriculum: 20), error: 60.833103
Epoch 11845 (11845 processed examples, curriculum: 20), error: 60.968576
Epoch 11849 (11849 processed examples, curriculum: 20), error: 60.833103
Epoch 11855 (11855 processed examples, curriculum: 20), error: 60.968576
Epoch 11859 (11859 processed examples, curriculum: 20), error: 61.215549
Epoch 11864 (11864 processed examples, curriculum: 20), error: 61.146528
Epoch 11868 (11868 processed examples, curriculum: 20), error: 60.833103
Epoch 11872 (11872 processed examples, curriculum: 20), error: 60.833103
Epoch 11879 (11879 processed examples, curriculum: 20), error: 58.118353
Epoch 11884 (11884 processed examples, curriculum: 20), error: 61.166534
    Testing sequence length 10: 
        Cost per sequence: 38.200000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 78.200000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 158.900000 (chance: 160.000000)
Epoch 11885 (11885 processed examples, curriculum: 20), error: 61.166534
Epoch 11892 (11892 processed examples, curriculum: 20), error: 61.272141
Epoch 11903 (11903 processed examples, curriculum: 20), error: 63.810894
Epoch 11913 (11913 processed examples, curriculum: 20), error: 61.193928
Epoch 11919 (11919 processed examples, curriculum: 20), error: 60.930412
Epoch 11928 (11928 processed examples, curriculum: 20), error: 55.339422
Epoch 11936 (11936 processed examples, curriculum: 20), error: 57.920391
Epoch 11944 (11944 processed examples, curriculum: 20), error: 57.920391
Epoch 11948 (11948 processed examples, curriculum: 20), error: 60.643309
Epoch 11952 (11952 processed examples, curriculum: 20), error: 60.864422
Epoch 11956 (11956 processed examples, curriculum: 20), error: 60.930412
Epoch 11962 (11962 processed examples, curriculum: 20), error: 60.994781
Epoch 11966 (11966 processed examples, curriculum: 20), error: 61.310148
Epoch 11973 (11973 processed examples, curriculum: 20), error: 60.949532
Epoch 11979 (11979 processed examples, curriculum: 20), error: 60.994781
Epoch 11985 (11985 processed examples, curriculum: 20), error: 60.949532
Epoch 11993 (11993 processed examples, curriculum: 20), error: 55.142357
Epoch 11999 (11999 processed examples, curriculum: 20), error: 60.967003
Epoch 12006 (12006 processed examples, curriculum: 20), error: 60.979588
Epoch 12012 (12012 processed examples, curriculum: 20), error: 61.123188
Epoch 12019 (12019 processed examples, curriculum: 20), error: 60.943272
Epoch 12028 (12028 processed examples, curriculum: 20), error: 60.964838
Epoch 12033 (12033 processed examples, curriculum: 20), error: 60.943272
    Testing sequence length 10: 
        Cost per sequence: 39.900000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 82.700000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 156.700000 (chance: 160.000000)
Epoch 12037 (12037 processed examples, curriculum: 20), error: 60.921753
Epoch 12044 (12044 processed examples, curriculum: 20), error: 60.964838
Epoch 12055 (12055 processed examples, curriculum: 20), error: 60.760805
Epoch 12060 (12060 processed examples, curriculum: 20), error: 58.133240
Epoch 12069 (12069 processed examples, curriculum: 20), error: 55.537846
Epoch 12075 (12075 processed examples, curriculum: 20), error: 55.537846
Epoch 12081 (12081 processed examples, curriculum: 20), error: 55.392887
Epoch 12086 (12086 processed examples, curriculum: 20), error: 55.537846
Epoch 12092 (12092 processed examples, curriculum: 20), error: 60.405159
Epoch 12102 (12102 processed examples, curriculum: 20), error: 49.937759
Epoch 12108 (12108 processed examples, curriculum: 20), error: 49.937759
Epoch 12115 (12115 processed examples, curriculum: 20), error: 49.937759
Epoch 12125 (12125 processed examples, curriculum: 20), error: 49.870708
Epoch 12133 (12133 processed examples, curriculum: 20), error: 49.975262
Epoch 12142 (12142 processed examples, curriculum: 20), error: 50.303398
Epoch 12149 (12149 processed examples, curriculum: 20), error: 50.700026
Epoch 12157 (12157 processed examples, curriculum: 20), error: 50.434647
Epoch 12164 (12164 processed examples, curriculum: 20), error: 50.718763
Epoch 12173 (12173 processed examples, curriculum: 20), error: 55.227104
Epoch 12182 (12182 processed examples, curriculum: 20), error: 55.551826
Epoch 12191 (12191 processed examples, curriculum: 20), error: 55.217817
Epoch 12199 (12199 processed examples, curriculum: 20), error: 55.692150
Epoch 12206 (12206 processed examples, curriculum: 20), error: 55.738647
    Testing sequence length 10: 
        Cost per sequence: 40.900000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 77.900000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 164.200000 (chance: 160.000000)
Epoch 12208 (12208 processed examples, curriculum: 20), error: 55.968800
Epoch 12216 (12216 processed examples, curriculum: 20), error: 55.968800
Epoch 12225 (12225 processed examples, curriculum: 20), error: 60.838741
Epoch 12232 (12232 processed examples, curriculum: 20), error: 60.838741
Epoch 12239 (12239 processed examples, curriculum: 20), error: 60.990677
Epoch 12249 (12249 processed examples, curriculum: 20), error: 60.826256
Epoch 12257 (12257 processed examples, curriculum: 20), error: 60.940939
Epoch 12263 (12263 processed examples, curriculum: 20), error: 60.990677
Epoch 12270 (12270 processed examples, curriculum: 20), error: 61.262327
Epoch 12277 (12277 processed examples, curriculum: 20), error: 61.262327
Epoch 12284 (12284 processed examples, curriculum: 20), error: 66.369991
Epoch 12294 (12294 processed examples, curriculum: 20), error: 61.060627
Epoch 12300 (12300 processed examples, curriculum: 20), error: 61.262327
Epoch 12302 (12302 processed examples, curriculum: 20), error: 63.695946
Epoch 12304 (12304 processed examples, curriculum: 20), error: 61.092253
Epoch 12308 (12308 processed examples, curriculum: 20), error: 58.280025
Epoch 12309 (12309 processed examples, curriculum: 20), error: 60.903910
Epoch 12311 (12311 processed examples, curriculum: 20), error: 60.935535
Epoch 12314 (12314 processed examples, curriculum: 20), error: 58.280025
Epoch 12316 (12316 processed examples, curriculum: 20), error: 58.280025
Epoch 12318 (12318 processed examples, curriculum: 20), error: 55.721819
Epoch 12319 (12319 processed examples, curriculum: 20), error: 55.721819
    Testing sequence length 10: 
        Cost per sequence: 39.200000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 79.200000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 156.000000 (chance: 160.000000)
Epoch 12320 (12320 processed examples, curriculum: 20), error: 58.384453
    Testing sequence length 10: 
        Cost per sequence: 40.800000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 83.100000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 161.600000 (chance: 160.000000)
Epoch 12321 (12321 processed examples, curriculum: 20), error: 55.721819
    Testing sequence length 10: 
        Cost per sequence: 39.300000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 77.700000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 162.600000 (chance: 160.000000)
Epoch 12322 (12322 processed examples, curriculum: 20), error: 55.454388
    Testing sequence length 10: 
        Cost per sequence: 40.600000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 78.000000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 156.900000 (chance: 160.000000)
Epoch 12323 (12323 processed examples, curriculum: 20), error: 55.190189
    Testing sequence length 10: 
        Cost per sequence: 38.400000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 81.300000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 154.800000 (chance: 160.000000)
Epoch 12324 (12324 processed examples, curriculum: 20), error: 55.454388
    Testing sequence length 10: 
        Cost per sequence: 36.600000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 80.800000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 162.100000 (chance: 160.000000)
Epoch 12325 (12325 processed examples, curriculum: 20), error: 55.190189
    Testing sequence length 10: 
        Cost per sequence: 39.100000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 79.400000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 160.400000 (chance: 160.000000)
Epoch 12326 (12326 processed examples, curriculum: 20), error: 55.454388
    Testing sequence length 10: 
        Cost per sequence: 39.800000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 77.100000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 163.800000 (chance: 160.000000)
Epoch 12327 (12327 processed examples, curriculum: 20), error: 55.454388
    Testing sequence length 10: 
        Cost per sequence: 39.900000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 79.400000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 162.800000 (chance: 160.000000)
Epoch 12328 (12328 processed examples, curriculum: 20), error: 55.190189
Epoch 12335 (12335 processed examples, curriculum: 20), error: 52.698467
Epoch 12341 (12341 processed examples, curriculum: 20), error: 46.926802
Epoch 12348 (12348 processed examples, curriculum: 20), error: 49.924314
Epoch 12352 (12352 processed examples, curriculum: 20), error: 49.456959
Epoch 12357 (12357 processed examples, curriculum: 20), error: 49.924314
Epoch 12363 (12363 processed examples, curriculum: 20), error: 49.456959
    Testing sequence length 10: 
        Cost per sequence: 41.100000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 78.200000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 159.700000 (chance: 160.000000)
Epoch 12365 (12365 processed examples, curriculum: 20), error: 49.924314
Epoch 12366 (12366 processed examples, curriculum: 20), error: 49.924314
Epoch 12367 (12367 processed examples, curriculum: 20), error: 52.762100
Epoch 12370 (12370 processed examples, curriculum: 20), error: 49.924314
Epoch 12371 (12371 processed examples, curriculum: 20), error: 52.762100
Epoch 12373 (12373 processed examples, curriculum: 20), error: 50.031683
Epoch 12375 (12375 processed examples, curriculum: 20), error: 49.625877
Epoch 12377 (12377 processed examples, curriculum: 20), error: 49.456959
Epoch 12380 (12380 processed examples, curriculum: 20), error: 46.919724
Epoch 12381 (12381 processed examples, curriculum: 20), error: 49.456959
Epoch 12382 (12382 processed examples, curriculum: 20), error: 49.456959
Epoch 12383 (12383 processed examples, curriculum: 20), error: 49.625877
Epoch 12385 (12385 processed examples, curriculum: 20), error: 49.456959
Epoch 12386 (12386 processed examples, curriculum: 20), error: 49.625877
Epoch 12387 (12387 processed examples, curriculum: 20), error: 49.625877
    Testing sequence length 10: 
        Cost per sequence: 39.400000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 77.000000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 159.300000 (chance: 160.000000)
Epoch 12388 (12388 processed examples, curriculum: 20), error: 49.889320
    Testing sequence length 10: 
        Cost per sequence: 41.500000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 81.500000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 157.000000 (chance: 160.000000)
Epoch 12389 (12389 processed examples, curriculum: 20), error: 50.187757
    Testing sequence length 10: 
        Cost per sequence: 38.300000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 77.700000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 157.600000 (chance: 160.000000)
Epoch 12390 (12390 processed examples, curriculum: 20), error: 49.889320
    Testing sequence length 10: 
        Cost per sequence: 39.700000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 82.300000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 159.600000 (chance: 160.000000)
Epoch 12391 (12391 processed examples, curriculum: 20), error: 49.889320
Epoch 12399 (12399 processed examples, curriculum: 20), error: 49.987387
Epoch 12407 (12407 processed examples, curriculum: 20), error: 49.831312
Epoch 12411 (12411 processed examples, curriculum: 20), error: 49.987387
Epoch 12416 (12416 processed examples, curriculum: 20), error: 52.919271
Epoch 12421 (12421 processed examples, curriculum: 20), error: 55.603798
Epoch 12429 (12429 processed examples, curriculum: 20), error: 55.603798
Epoch 12434 (12434 processed examples, curriculum: 20), error: 55.627535
Epoch 12440 (12440 processed examples, curriculum: 20), error: 55.707403
Epoch 12446 (12446 processed examples, curriculum: 20), error: 61.118090
Epoch 12452 (12452 processed examples, curriculum: 20), error: 58.356504
Epoch 12458 (12458 processed examples, curriculum: 20), error: 58.356504
Epoch 12464 (12464 processed examples, curriculum: 20), error: 58.356504
Epoch 12470 (12470 processed examples, curriculum: 20), error: 60.970181
Epoch 12481 (12481 processed examples, curriculum: 20), error: 60.970181
Epoch 12489 (12489 processed examples, curriculum: 20), error: 55.633028
Epoch 12496 (12496 processed examples, curriculum: 20), error: 58.231163
Epoch 12503 (12503 processed examples, curriculum: 20), error: 60.704443
Epoch 12509 (12509 processed examples, curriculum: 20), error: 60.704443
Epoch 12515 (12515 processed examples, curriculum: 20), error: 58.231163
Epoch 12522 (12522 processed examples, curriculum: 20), error: 58.231163
Epoch 12529 (12529 processed examples, curriculum: 20), error: 60.779593
    Testing sequence length 10: 
        Cost per sequence: 41.600000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 76.200000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 156.700000 (chance: 160.000000)
Epoch 12531 (12531 processed examples, curriculum: 20), error: 60.779593
Epoch 12537 (12537 processed examples, curriculum: 20), error: 60.891459
Epoch 12545 (12545 processed examples, curriculum: 20), error: 60.704443
Epoch 12550 (12550 processed examples, curriculum: 20), error: 60.833488
Epoch 12557 (12557 processed examples, curriculum: 20), error: 60.910398
Epoch 12563 (12563 processed examples, curriculum: 20), error: 60.876221
Epoch 12568 (12568 processed examples, curriculum: 20), error: 60.833488
Epoch 12575 (12575 processed examples, curriculum: 20), error: 60.833488
Epoch 12583 (12583 processed examples, curriculum: 20), error: 60.833488
Epoch 12590 (12590 processed examples, curriculum: 20), error: 63.801914
Epoch 12596 (12596 processed examples, curriculum: 20), error: 66.507923
Epoch 12603 (12603 processed examples, curriculum: 20), error: 63.801914
Epoch 12608 (12608 processed examples, curriculum: 20), error: 63.801914
Epoch 12614 (12614 processed examples, curriculum: 20), error: 61.076427
Epoch 12622 (12622 processed examples, curriculum: 20), error: 60.957045
Epoch 12630 (12630 processed examples, curriculum: 20), error: 60.897654
Epoch 12637 (12637 processed examples, curriculum: 20), error: 60.897654
Epoch 12644 (12644 processed examples, curriculum: 20), error: 60.957045
Epoch 12649 (12649 processed examples, curriculum: 20), error: 61.045258
Epoch 12657 (12657 processed examples, curriculum: 20), error: 60.957045
Epoch 12662 (12662 processed examples, curriculum: 20), error: 61.045258
Epoch 12667 (12667 processed examples, curriculum: 20), error: 61.082739
    Testing sequence length 10: 
        Cost per sequence: 40.300000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 79.200000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 159.800000 (chance: 160.000000)
Epoch 12668 (12668 processed examples, curriculum: 20), error: 61.082739
Epoch 12673 (12673 processed examples, curriculum: 20), error: 61.112839
Epoch 12680 (12680 processed examples, curriculum: 20), error: 61.191406
Epoch 12687 (12687 processed examples, curriculum: 20), error: 61.279301
Epoch 12692 (12692 processed examples, curriculum: 20), error: 61.190336
Epoch 12696 (12696 processed examples, curriculum: 20), error: 61.190336
Epoch 12701 (12701 processed examples, curriculum: 20), error: 61.190336
Epoch 12708 (12708 processed examples, curriculum: 20), error: 61.119177
Epoch 12713 (12713 processed examples, curriculum: 20), error: 61.225603
Epoch 12721 (12721 processed examples, curriculum: 20), error: 61.119177
Epoch 12726 (12726 processed examples, curriculum: 20), error: 61.119177
Epoch 12732 (12732 processed examples, curriculum: 20), error: 61.225603
Epoch 12737 (12737 processed examples, curriculum: 20), error: 61.225603
Epoch 12743 (12743 processed examples, curriculum: 20), error: 63.795242
Epoch 12748 (12748 processed examples, curriculum: 20), error: 66.411636
Epoch 12753 (12753 processed examples, curriculum: 20), error: 66.671764
Epoch 12759 (12759 processed examples, curriculum: 20), error: 66.671764
Epoch 12766 (12766 processed examples, curriculum: 20), error: 66.411636
Epoch 12772 (12772 processed examples, curriculum: 20), error: 61.133514
Epoch 12776 (12776 processed examples, curriculum: 20), error: 66.411636
Epoch 12783 (12783 processed examples, curriculum: 20), error: 66.584011
Epoch 12790 (12790 processed examples, curriculum: 20), error: 61.133514
Epoch 12797 (12797 processed examples, curriculum: 20), error: 61.085873
    Testing sequence length 10: 
        Cost per sequence: 38.600000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 83.200000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 162.500000 (chance: 160.000000)
Epoch 12801 (12801 processed examples, curriculum: 20), error: 61.117672
Epoch 12812 (12812 processed examples, curriculum: 20), error: 60.921005
Epoch 12819 (12819 processed examples, curriculum: 20), error: 60.994219
Epoch 12826 (12826 processed examples, curriculum: 20), error: 60.921005
Epoch 12831 (12831 processed examples, curriculum: 20), error: 60.994219
Epoch 12838 (12838 processed examples, curriculum: 20), error: 60.909788
Epoch 12846 (12846 processed examples, curriculum: 20), error: 60.721325
Epoch 12854 (12854 processed examples, curriculum: 20), error: 55.526213
Epoch 12861 (12861 processed examples, curriculum: 20), error: 55.526213
Epoch 12868 (12868 processed examples, curriculum: 20), error: 55.526213
Epoch 12877 (12877 processed examples, curriculum: 20), error: 55.577181
Epoch 12888 (12888 processed examples, curriculum: 20), error: 52.750893
Epoch 12895 (12895 processed examples, curriculum: 20), error: 55.265894
Epoch 12899 (12899 processed examples, curriculum: 20), error: 55.146194
Epoch 12901 (12901 processed examples, curriculum: 20), error: 55.146194
Epoch 12903 (12903 processed examples, curriculum: 20), error: 55.265894
Epoch 12906 (12906 processed examples, curriculum: 20), error: 55.265894
Epoch 12907 (12907 processed examples, curriculum: 20), error: 55.313828
Epoch 12910 (12910 processed examples, curriculum: 20), error: 55.265894
Epoch 12912 (12912 processed examples, curriculum: 20), error: 55.265894
Epoch 12913 (12913 processed examples, curriculum: 20), error: 55.313828
Epoch 12914 (12914 processed examples, curriculum: 20), error: 55.427458
    Testing sequence length 10: 
        Cost per sequence: 41.800000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 78.500000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 161.100000 (chance: 160.000000)
Epoch 12915 (12915 processed examples, curriculum: 20), error: 55.313828
    Testing sequence length 10: 
        Cost per sequence: 37.300000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 80.400000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 168.900000 (chance: 160.000000)
Epoch 12916 (12916 processed examples, curriculum: 20), error: 55.427458
    Testing sequence length 10: 
        Cost per sequence: 38.900000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 79.900000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 159.300000 (chance: 160.000000)
Epoch 12917 (12917 processed examples, curriculum: 20), error: 55.427458
    Testing sequence length 10: 
        Cost per sequence: 39.600000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 79.100000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 156.900000 (chance: 160.000000)
Epoch 12918 (12918 processed examples, curriculum: 20), error: 55.427458
    Testing sequence length 10: 
        Cost per sequence: 39.500000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 82.100000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 159.500000 (chance: 160.000000)
Epoch 12919 (12919 processed examples, curriculum: 20), error: 55.313828
    Testing sequence length 10: 
        Cost per sequence: 41.500000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 77.900000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 163.100000 (chance: 160.000000)
Epoch 12920 (12920 processed examples, curriculum: 20), error: 55.179190
    Testing sequence length 10: 
        Cost per sequence: 41.900000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 80.600000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 154.800000 (chance: 160.000000)
Epoch 12921 (12921 processed examples, curriculum: 20), error: 55.179190
Epoch 12923 (12923 processed examples, curriculum: 20), error: 55.179190
Epoch 12924 (12924 processed examples, curriculum: 20), error: 55.313828
Epoch 12925 (12925 processed examples, curriculum: 20), error: 55.427458
    Testing sequence length 10: 
        Cost per sequence: 38.700000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 76.800000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 162.800000 (chance: 160.000000)
Epoch 12926 (12926 processed examples, curriculum: 20), error: 55.427458
    Testing sequence length 10: 
        Cost per sequence: 42.000000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 79.900000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 157.600000 (chance: 160.000000)
Epoch 12927 (12927 processed examples, curriculum: 20), error: 55.427458
    Testing sequence length 10: 
        Cost per sequence: 39.000000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 79.300000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 158.000000 (chance: 160.000000)
Epoch 12928 (12928 processed examples, curriculum: 20), error: 55.313828
Epoch 12936 (12936 processed examples, curriculum: 20), error: 55.179190
Epoch 12943 (12943 processed examples, curriculum: 20), error: 55.102900
Epoch 12949 (12949 processed examples, curriculum: 20), error: 52.656410
Epoch 12953 (12953 processed examples, curriculum: 20), error: 55.102900
Epoch 12960 (12960 processed examples, curriculum: 20), error: 55.102900
Epoch 12966 (12966 processed examples, curriculum: 20), error: 55.237537
Epoch 12975 (12975 processed examples, curriculum: 20), error: 55.102900
Epoch 12982 (12982 processed examples, curriculum: 20), error: 55.102900
Epoch 12990 (12990 processed examples, curriculum: 20), error: 55.102900
    Testing sequence length 10: 
        Cost per sequence: 36.700000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 81.200000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 161.200000 (chance: 160.000000)
Epoch 12992 (12992 processed examples, curriculum: 20), error: 55.102900
Epoch 12998 (12998 processed examples, curriculum: 20), error: 55.680727
Epoch 13002 (13002 processed examples, curriculum: 20), error: 60.780357
Epoch 13011 (13011 processed examples, curriculum: 20), error: 60.860533
Epoch 13020 (13020 processed examples, curriculum: 20), error: 60.767593
Epoch 13029 (13029 processed examples, curriculum: 20), error: 58.187481
Epoch 13037 (13037 processed examples, curriculum: 20), error: 60.767593
Epoch 13049 (13049 processed examples, curriculum: 20), error: 55.628908
Epoch 13056 (13056 processed examples, curriculum: 20), error: 55.548191
Epoch 13062 (13062 processed examples, curriculum: 20), error: 55.628908
Epoch 13070 (13070 processed examples, curriculum: 20), error: 55.628908
Epoch 13077 (13077 processed examples, curriculum: 20), error: 55.642044
Epoch 13084 (13084 processed examples, curriculum: 20), error: 55.642044
Epoch 13092 (13092 processed examples, curriculum: 20), error: 58.218822
Epoch 13100 (13100 processed examples, curriculum: 20), error: 55.411936
Epoch 13110 (13110 processed examples, curriculum: 20), error: 55.232527
Epoch 13115 (13115 processed examples, curriculum: 20), error: 55.541107
Epoch 13120 (13120 processed examples, curriculum: 20), error: 55.411936
Epoch 13130 (13130 processed examples, curriculum: 20), error: 52.594063
Epoch 13137 (13137 processed examples, curriculum: 20), error: 55.232527
Epoch 13143 (13143 processed examples, curriculum: 20), error: 55.603655
Epoch 13149 (13149 processed examples, curriculum: 20), error: 58.482134
Epoch 13157 (13157 processed examples, curriculum: 20), error: 55.748936
Epoch 13164 (13164 processed examples, curriculum: 20), error: 58.495138
    Testing sequence length 10: 
        Cost per sequence: 37.500000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 79.500000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 161.600000 (chance: 160.000000)
Epoch 13168 (13168 processed examples, curriculum: 20), error: 61.165451
Epoch 13175 (13175 processed examples, curriculum: 20), error: 61.165451
Epoch 13183 (13183 processed examples, curriculum: 20), error: 58.495138
Epoch 13190 (13190 processed examples, curriculum: 20), error: 58.495138
Epoch 13199 (13199 processed examples, curriculum: 20), error: 58.495138
Epoch 13207 (13207 processed examples, curriculum: 20), error: 61.110470
Epoch 13216 (13216 processed examples, curriculum: 20), error: 56.138077
Epoch 13224 (13224 processed examples, curriculum: 20), error: 58.511868
Epoch 13233 (13233 processed examples, curriculum: 20), error: 58.511868
Epoch 13239 (13239 processed examples, curriculum: 20), error: 55.965021
Epoch 13246 (13246 processed examples, curriculum: 20), error: 56.138077
Epoch 13254 (13254 processed examples, curriculum: 20), error: 58.511868
Epoch 13263 (13263 processed examples, curriculum: 20), error: 56.115385
Epoch 13270 (13270 processed examples, curriculum: 20), error: 55.525425
Epoch 13278 (13278 processed examples, curriculum: 20), error: 55.525425
Epoch 13285 (13285 processed examples, curriculum: 20), error: 55.525425
Epoch 13294 (13294 processed examples, curriculum: 20), error: 55.359941
Epoch 13303 (13303 processed examples, curriculum: 20), error: 55.359941
Epoch 13310 (13310 processed examples, curriculum: 20), error: 55.944609
Epoch 13317 (13317 processed examples, curriculum: 20), error: 58.107779
Epoch 13323 (13323 processed examples, curriculum: 20), error: 60.675367
Epoch 13330 (13330 processed examples, curriculum: 20), error: 60.882797
Epoch 13336 (13336 processed examples, curriculum: 20), error: 60.882797
Epoch 13345 (13345 processed examples, curriculum: 20), error: 60.882797
    Testing sequence length 10: 
        Cost per sequence: 37.600000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 79.600000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 158.700000 (chance: 160.000000)
Epoch 13347 (13347 processed examples, curriculum: 20), error: 60.882797
Epoch 13354 (13354 processed examples, curriculum: 20), error: 60.778065
Epoch 13360 (13360 processed examples, curriculum: 20), error: 60.778065
Epoch 13369 (13369 processed examples, curriculum: 20), error: 60.888548
Epoch 13376 (13376 processed examples, curriculum: 20), error: 60.957331
Epoch 13381 (13381 processed examples, curriculum: 20), error: 63.672859
Epoch 13390 (13390 processed examples, curriculum: 20), error: 61.248081
Epoch 13397 (13397 processed examples, curriculum: 20), error: 61.327898
Epoch 13402 (13402 processed examples, curriculum: 20), error: 63.752676
Epoch 13407 (13407 processed examples, curriculum: 20), error: 63.752676
Epoch 13414 (13414 processed examples, curriculum: 20), error: 66.318378
Epoch 13423 (13423 processed examples, curriculum: 20), error: 61.141323
Epoch 13431 (13431 processed examples, curriculum: 20), error: 61.086355
Epoch 13436 (13436 processed examples, curriculum: 20), error: 61.086355
Epoch 13443 (13443 processed examples, curriculum: 20), error: 61.086355
Epoch 13451 (13451 processed examples, curriculum: 20), error: 61.090002
Epoch 13460 (13460 processed examples, curriculum: 20), error: 61.025854
Epoch 13468 (13468 processed examples, curriculum: 20), error: 58.311785
Epoch 13473 (13473 processed examples, curriculum: 20), error: 58.311785
Epoch 13480 (13480 processed examples, curriculum: 20), error: 55.498451
Epoch 13486 (13486 processed examples, curriculum: 20), error: 55.580185
Epoch 13492 (13492 processed examples, curriculum: 20), error: 58.313919
Epoch 13498 (13498 processed examples, curriculum: 20), error: 55.643274
    Testing sequence length 10: 
        Cost per sequence: 39.200000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 80.800000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 157.000000 (chance: 160.000000)
Epoch 13503 (13503 processed examples, curriculum: 20), error: 55.643274
Epoch 13511 (13511 processed examples, curriculum: 20), error: 55.643274
Epoch 13518 (13518 processed examples, curriculum: 20), error: 58.216513
Epoch 13525 (13525 processed examples, curriculum: 20), error: 58.182976
Epoch 13533 (13533 processed examples, curriculum: 20), error: 55.741108
Epoch 13541 (13541 processed examples, curriculum: 20), error: 55.682583
Epoch 13550 (13550 processed examples, curriculum: 20), error: 55.493408
Epoch 13558 (13558 processed examples, curriculum: 20), error: 55.493408
Epoch 13564 (13564 processed examples, curriculum: 20), error: 55.682583
Epoch 13572 (13572 processed examples, curriculum: 20), error: 55.583157
Epoch 13580 (13580 processed examples, curriculum: 20), error: 55.682583
Epoch 13586 (13586 processed examples, curriculum: 20), error: 55.682583
Epoch 13592 (13592 processed examples, curriculum: 20), error: 55.183092
Epoch 13598 (13598 processed examples, curriculum: 20), error: 55.183092
Epoch 13605 (13605 processed examples, curriculum: 20), error: 52.817904
Epoch 13613 (13613 processed examples, curriculum: 20), error: 50.157574
Epoch 13620 (13620 processed examples, curriculum: 20), error: 49.894817
Epoch 13625 (13625 processed examples, curriculum: 20), error: 50.157574
Epoch 13634 (13634 processed examples, curriculum: 20), error: 50.157574
Epoch 13640 (13640 processed examples, curriculum: 20), error: 50.157574
    Testing sequence length 10: 
        Cost per sequence: 41.500000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 78.400000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 157.900000 (chance: 160.000000)
Epoch 13643 (13643 processed examples, curriculum: 20), error: 52.770586
Epoch 13648 (13648 processed examples, curriculum: 20), error: 55.545357
Epoch 13657 (13657 processed examples, curriculum: 20), error: 58.014891
Epoch 13663 (13663 processed examples, curriculum: 20), error: 58.014891
Epoch 13670 (13670 processed examples, curriculum: 20), error: 58.014891
Epoch 13678 (13678 processed examples, curriculum: 20), error: 55.438278
Epoch 13685 (13685 processed examples, curriculum: 20), error: 55.438278
Epoch 13689 (13689 processed examples, curriculum: 20), error: 55.517935
Epoch 13696 (13696 processed examples, curriculum: 20), error: 55.517935
Epoch 13701 (13701 processed examples, curriculum: 20), error: 55.438278
Epoch 13705 (13705 processed examples, curriculum: 20), error: 55.366053
Epoch 13713 (13713 processed examples, curriculum: 20), error: 55.438278
Epoch 13719 (13719 processed examples, curriculum: 20), error: 55.517935
Epoch 13728 (13728 processed examples, curriculum: 20), error: 55.441999
Epoch 13733 (13733 processed examples, curriculum: 20), error: 58.296595
Epoch 13740 (13740 processed examples, curriculum: 20), error: 63.779512
Epoch 13745 (13745 processed examples, curriculum: 20), error: 58.244720
Epoch 13752 (13752 processed examples, curriculum: 20), error: 63.900732
Epoch 13761 (13761 processed examples, curriculum: 20), error: 63.770739
Epoch 13767 (13767 processed examples, curriculum: 20), error: 63.770739
Epoch 13775 (13775 processed examples, curriculum: 20), error: 66.686447
Epoch 13782 (13782 processed examples, curriculum: 20), error: 60.993252
Epoch 13786 (13786 processed examples, curriculum: 20), error: 60.996809
    Testing sequence length 10: 
        Cost per sequence: 42.100000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 78.800000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 160.800000 (chance: 160.000000)
Epoch 13788 (13788 processed examples, curriculum: 20), error: 60.966681
Epoch 13793 (13793 processed examples, curriculum: 20), error: 66.342358
Epoch 13799 (13799 processed examples, curriculum: 20), error: 66.631447
Epoch 13806 (13806 processed examples, curriculum: 20), error: 66.631447
Epoch 13813 (13813 processed examples, curriculum: 20), error: 66.631447
Epoch 13820 (13820 processed examples, curriculum: 20), error: 63.660963
Epoch 13825 (13825 processed examples, curriculum: 20), error: 66.326836
Epoch 13833 (13833 processed examples, curriculum: 20), error: 60.859352
Epoch 13839 (13839 processed examples, curriculum: 20), error: 60.840597
Epoch 13845 (13845 processed examples, curriculum: 20), error: 60.684593
Epoch 13850 (13850 processed examples, curriculum: 20), error: 60.598742
Epoch 13854 (13854 processed examples, curriculum: 20), error: 60.684593
Epoch 13858 (13858 processed examples, curriculum: 20), error: 60.944368
Epoch 13865 (13865 processed examples, curriculum: 20), error: 60.966681
Epoch 13873 (13873 processed examples, curriculum: 20), error: 60.969852
Epoch 13880 (13880 processed examples, curriculum: 20), error: 60.969852
Epoch 13886 (13886 processed examples, curriculum: 20), error: 60.769609
Epoch 13896 (13896 processed examples, curriculum: 20), error: 49.812677
Epoch 13903 (13903 processed examples, curriculum: 20), error: 49.919077
Epoch 13907 (13907 processed examples, curriculum: 20), error: 50.105843
Epoch 13912 (13912 processed examples, curriculum: 20), error: 50.105843
Epoch 13918 (13918 processed examples, curriculum: 20), error: 58.115923
Epoch 13927 (13927 processed examples, curriculum: 20), error: 52.658041
    Testing sequence length 10: 
        Cost per sequence: 40.700000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 80.700000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 160.400000 (chance: 160.000000)
Epoch 13934 (13934 processed examples, curriculum: 20), error: 60.947540
Epoch 13941 (13941 processed examples, curriculum: 20), error: 61.167507
Epoch 13948 (13948 processed examples, curriculum: 20), error: 61.183796
Epoch 13957 (13957 processed examples, curriculum: 20), error: 61.020592
Epoch 13964 (13964 processed examples, curriculum: 20), error: 61.100689
Epoch 13972 (13972 processed examples, curriculum: 20), error: 58.409336
Epoch 13978 (13978 processed examples, curriculum: 20), error: 58.409336
Epoch 13984 (13984 processed examples, curriculum: 20), error: 58.409336
Epoch 13991 (13991 processed examples, curriculum: 20), error: 61.100689
Epoch 14001 (14001 processed examples, curriculum: 20), error: 61.276781
Epoch 14011 (14011 processed examples, curriculum: 20), error: 55.455809
Epoch 14017 (14017 processed examples, curriculum: 20), error: 55.455809
Epoch 14029 (14029 processed examples, curriculum: 20), error: 55.307903
Epoch 14038 (14038 processed examples, curriculum: 20), error: 52.581448
Epoch 14046 (14046 processed examples, curriculum: 20), error: 52.581448
Epoch 14055 (14055 processed examples, curriculum: 20), error: 49.914387
Epoch 14063 (14063 processed examples, curriculum: 20), error: 52.581448
Epoch 14068 (14068 processed examples, curriculum: 20), error: 55.296099
Epoch 14074 (14074 processed examples, curriculum: 20), error: 55.377645
Epoch 14082 (14082 processed examples, curriculum: 20), error: 55.295502
Epoch 14087 (14087 processed examples, curriculum: 20), error: 55.419558
Epoch 14094 (14094 processed examples, curriculum: 20), error: 55.419558
Epoch 14100 (14100 processed examples, curriculum: 20), error: 58.244619
    Testing sequence length 10: 
        Cost per sequence: 42.300000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 81.000000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 158.500000 (chance: 160.000000)
Epoch 14104 (14104 processed examples, curriculum: 20), error: 60.849104
Epoch 14111 (14111 processed examples, curriculum: 20), error: 63.788162
Epoch 14122 (14122 processed examples, curriculum: 20), error: 61.160891
Epoch 14129 (14129 processed examples, curriculum: 20), error: 63.970091
Epoch 14134 (14134 processed examples, curriculum: 20), error: 66.520287
Epoch 14135 (14135 processed examples, curriculum: 20), error: 66.617630
Epoch 14138 (14138 processed examples, curriculum: 20), error: 63.970091
Epoch 14141 (14141 processed examples, curriculum: 20), error: 63.970091
Epoch 14142 (14142 processed examples, curriculum: 20), error: 66.520287
Epoch 14145 (14145 processed examples, curriculum: 20), error: 66.520287
Epoch 14146 (14146 processed examples, curriculum: 20), error: 66.520287
Epoch 14148 (14148 processed examples, curriculum: 20), error: 66.520287
Epoch 14149 (14149 processed examples, curriculum: 20), error: 69.090656
Epoch 14151 (14151 processed examples, curriculum: 20), error: 69.090656
Epoch 14155 (14155 processed examples, curriculum: 20), error: 66.520287
Epoch 14157 (14157 processed examples, curriculum: 20), error: 66.520287
Epoch 14161 (14161 processed examples, curriculum: 20), error: 63.970091
Epoch 14162 (14162 processed examples, curriculum: 20), error: 66.520287
Epoch 14163 (14163 processed examples, curriculum: 20), error: 63.970091
    Testing sequence length 10: 
        Cost per sequence: 39.800000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 79.900000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 157.700000 (chance: 160.000000)
Epoch 14164 (14164 processed examples, curriculum: 20), error: 63.970091
    Testing sequence length 10: 
        Cost per sequence: 41.300000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 79.700000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 162.500000 (chance: 160.000000)
Epoch 14165 (14165 processed examples, curriculum: 20), error: 58.667753
    Testing sequence length 10: 
        Cost per sequence: 40.100000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 80.200000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 162.600000 (chance: 160.000000)
Epoch 14166 (14166 processed examples, curriculum: 20), error: 55.619797
    Testing sequence length 10: 
        Cost per sequence: 37.900000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 77.300000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 161.100000 (chance: 160.000000)
Epoch 14167 (14167 processed examples, curriculum: 20), error: 55.619797
    Testing sequence length 10: 
        Cost per sequence: 39.300000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 77.100000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 163.200000 (chance: 160.000000)
Epoch 14168 (14168 processed examples, curriculum: 20), error: 55.246210
    Testing sequence length 10: 
        Cost per sequence: 38.900000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 81.900000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 160.300000 (chance: 160.000000)
Epoch 14169 (14169 processed examples, curriculum: 20), error: 52.709299
    Testing sequence length 10: 
        Cost per sequence: 41.400000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 77.900000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 159.900000 (chance: 160.000000)
Epoch 14170 (14170 processed examples, curriculum: 20), error: 52.709299
    Testing sequence length 10: 
        Cost per sequence: 40.300000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 79.500000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 160.000000 (chance: 160.000000)
Epoch 14171 (14171 processed examples, curriculum: 20), error: 50.238590
    Testing sequence length 10: 
        Cost per sequence: 39.300000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 79.300000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 157.900000 (chance: 160.000000)
Epoch 14172 (14172 processed examples, curriculum: 20), error: 50.238590
    Testing sequence length 10: 
        Cost per sequence: 38.800000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 80.700000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 153.700000 (chance: 160.000000)
Epoch 14173 (14173 processed examples, curriculum: 20), error: 50.136017
    Testing sequence length 10: 
        Cost per sequence: 39.900000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 81.900000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 159.200000 (chance: 160.000000)
Epoch 14174 (14174 processed examples, curriculum: 20), error: 50.047672
Epoch 14175 (14175 processed examples, curriculum: 20), error: 50.047672
Epoch 14178 (14178 processed examples, curriculum: 20), error: 50.238590
Epoch 14179 (14179 processed examples, curriculum: 20), error: 50.238590
Epoch 14182 (14182 processed examples, curriculum: 20), error: 50.136017
Epoch 14184 (14184 processed examples, curriculum: 20), error: 50.136017
Epoch 14185 (14185 processed examples, curriculum: 20), error: 50.136017
Epoch 14186 (14186 processed examples, curriculum: 20), error: 50.136017
Epoch 14188 (14188 processed examples, curriculum: 20), error: 50.047672
Epoch 14197 (14197 processed examples, curriculum: 20), error: 49.958559
Epoch 14204 (14204 processed examples, curriculum: 20), error: 49.715000
Epoch 14209 (14209 processed examples, curriculum: 20), error: 49.830965
Epoch 14214 (14214 processed examples, curriculum: 20), error: 49.962519
Epoch 14215 (14215 processed examples, curriculum: 20), error: 49.962519
Epoch 14218 (14218 processed examples, curriculum: 20), error: 49.962519
Epoch 14221 (14221 processed examples, curriculum: 20), error: 49.962519
Epoch 14222 (14222 processed examples, curriculum: 20), error: 49.962519
    Testing sequence length 10: 
        Cost per sequence: 40.000000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 81.300000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 157.100000 (chance: 160.000000)
Epoch 14223 (14223 processed examples, curriculum: 20), error: 49.894808
    Testing sequence length 10: 
        Cost per sequence: 38.700000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 80.400000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 158.300000 (chance: 160.000000)
Epoch 14224 (14224 processed examples, curriculum: 20), error: 49.894808
    Testing sequence length 10: 
        Cost per sequence: 39.300000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 82.500000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 161.500000 (chance: 160.000000)
Epoch 14225 (14225 processed examples, curriculum: 20), error: 49.962519
Epoch 14231 (14231 processed examples, curriculum: 20), error: 50.098011
Epoch 14240 (14240 processed examples, curriculum: 20), error: 49.962519
    Testing sequence length 10: 
        Cost per sequence: 41.000000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 80.100000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 159.300000 (chance: 160.000000)
Epoch 14246 (14246 processed examples, curriculum: 20), error: 49.894808
Epoch 14252 (14252 processed examples, curriculum: 20), error: 50.098011
Epoch 14260 (14260 processed examples, curriculum: 20), error: 52.611149
Epoch 14267 (14267 processed examples, curriculum: 20), error: 55.072525
Epoch 14272 (14272 processed examples, curriculum: 20), error: 66.068340
Epoch 14278 (14278 processed examples, curriculum: 20), error: 66.439995
Epoch 14287 (14287 processed examples, curriculum: 20), error: 60.980824
Epoch 14296 (14296 processed examples, curriculum: 20), error: 58.061115
Epoch 14301 (14301 processed examples, curriculum: 20), error: 55.375551
Epoch 14303 (14303 processed examples, curriculum: 20), error: 55.375551
Epoch 14304 (14304 processed examples, curriculum: 20), error: 55.375551
Epoch 14305 (14305 processed examples, curriculum: 20), error: 55.375551
Epoch 14307 (14307 processed examples, curriculum: 20), error: 55.224419
Epoch 14309 (14309 processed examples, curriculum: 20), error: 55.048550
Epoch 14312 (14312 processed examples, curriculum: 20), error: 52.460218
Epoch 14313 (14313 processed examples, curriculum: 20), error: 55.048550
Epoch 14314 (14314 processed examples, curriculum: 20), error: 55.048550
Epoch 14317 (14317 processed examples, curriculum: 20), error: 47.157757
Epoch 14318 (14318 processed examples, curriculum: 20), error: 52.460218
    Testing sequence length 10: 
        Cost per sequence: 39.200000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 80.900000 (chance: 80.000000)
    Testing sequence length 40: 
Early stopping
    Testing sequence length 10: 
        Cost per sequence: 37.100000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 80.200000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 159.800000 (chance: 160.000000)
    Testing sequence length 10: 
        Cost per sequence: 37.300000 (chance: 40.000000)
    Testing sequence length 20: 
        Cost per sequence: 78.800000 (chance: 80.000000)
    Testing sequence length 40: 
        Cost per sequence: 163.900000 (chance: 160.000000)

In [23]:

    

len(errors)


    

    
Out[23]:





163892

In [22]:

    

y = median_smoothing(errors[::100], window=1000)
plt.plot(range(len(y)), y)


    

    
Out[22]:





[<matplotlib.lines.Line2D at 0x7ff2c428f278>]






    

In [14]:

    

for param in ntm_model.parameters():
    nel = param.w.ravel().shape[0]
    avg = sum(sum(abs(param.w))) / nel
    nrm =  np.linalg.norm(param)
    print('%.3f\t%.3f\t%d\t%s' % (avg, nrm,nel, param.name))
# plt.colorbar()


    

    




0.104	3.452	800	ntm_controller_mlp_input_layer.matrices[0]
0.317	16.636	2000	ntm_controller_mlp_input_layer.matrices[1]
0.083	1.026	100	ntm_controller_mlp_input_layer.b
0.104	3.585	800	ntm_decoder.matrices[0]
0.065	0.226	8	ntm_decoder.b
0.375	19.656	2000	ntm_read_head0_adressing_content_key.matrices[0]
0.281	1.443	20	ntm_read_head0_adressing_content_key.b
0.377	4.489	100	ntm_read_head0_adressing_content_key_strength.matrices[0]
0.685	0.685	1	ntm_read_head0_adressing_content_key_strength.b
0.332	3.583	100	ntm_read_head0_adressing_interpolation.matrices[0]
0.254	0.254	1	ntm_read_head0_adressing_interpolation.b
0.055	8.181	12800	ntm_read_head0_adressing_shift.matrices[0]
0.053	0.770	128	ntm_read_head0_adressing_shift.b
0.287	3.255	100	ntm_read_head0_adressing_focus.matrices[0]
0.338	0.338	1	ntm_read_head0_adressing_focus.b
0.263	3.398	128	ntm_read_head0_adressing_initial_locations
0.353	19.784	2000	ntm_write_head0_adressing_content_key.matrices[0]
0.685	3.493	20	ntm_write_head0_adressing_content_key.b
0.316	3.854	100	ntm_write_head0_adressing_content_key_strength.matrices[0]
0.037	0.037	1	ntm_write_head0_adressing_content_key_strength.b
0.399	4.638	100	ntm_write_head0_adressing_interpolation.matrices[0]
0.036	0.036	1	ntm_write_head0_adressing_interpolation.b
0.055	8.688	12800	ntm_write_head0_adressing_shift.matrices[0]
0.063	1.213	128	ntm_write_head0_adressing_shift.b
0.269	3.347	100	ntm_write_head0_adressing_focus.matrices[0]
0.259	0.259	1	ntm_write_head0_adressing_focus.b
0.422	5.352	128	ntm_write_head0_adressing_initial_locations
0.098	6.231	2560	ntm_memory_init

In [30]:

    

x.ravel().shape[0]


    

    
Out[30]:





4

In [91]:

    

ntm_model.read_heads[0].addressing.initial_locations.w
# plt.colorbar()


    

    
Out[91]:





array([[ 0.8116377 ,  0.59566122,  0.49466214,  0.46039918,  0.61630201,
        -0.33986348,  0.12560712,  0.58555883,  0.5218761 ,  0.33090848,
         0.51518619,  0.8930847 ,  0.96688551,  0.63240188,  0.55799574,
         0.55299872,  0.61381924,  1.00473475,  1.09844375,  0.75794619,
         0.18987241, -0.09666262,  0.65739894,  0.60510671,  0.26888797,
         0.34444487,  0.34612277,  0.38025174,  0.6011349 ,  0.88652372,
         1.00608325,  0.89585954,  0.95290732,  0.79351598,  1.00413716,
         0.64029497, -0.45334938, -0.09486344,  0.01748658, -0.44612688,
        -0.57601643, -0.67443895, -0.90790713, -1.02970672, -0.94918191,
        -0.89001358, -0.98256266, -1.15215051, -1.4409796 , -1.41517532,
        -1.40730977, -1.27923059, -0.32512578,  0.26151612,  0.49421379,
         0.77482742,  0.5601601 ,  0.09498564,  0.7505185 ,  1.13843215,
         0.79770923, -0.0198104 , -0.3197937 , -0.27566588, -0.29326198,
         0.15073785,  0.06546709, -0.01024877, -0.91852152, -0.92018431,
        -0.67073995,  0.06388454, -1.44116163, -1.45558369, -1.472453  ,
        -1.29750919, -1.43031549, -0.92626733,  0.08306657,  0.34299102,
         0.45042664,  0.64116013,  0.49844611,  0.12246892,  0.51454318,
         0.29408371,  0.54854816,  0.61646897,  0.13600096, -0.44812098,
        -0.27566028, -0.41483358, -0.39711383, -0.27181363,  0.07092586,
        -0.53938186, -0.52850872, -0.43358845, -0.45923787, -0.39129627,
        -0.4587228 , -0.44044241, -0.36679566,  0.45176172,  1.24123621,
         0.64989012,  0.48658693,  0.3470304 ,  0.48657009,  0.51197916,
         0.49574515,  0.65736896,  0.1352496 , -1.07666636, -1.07678401,
        -1.07564914, -0.50606668,  0.09098486,  0.8166846 , -0.13583948,
        -0.17788452, -0.17440428, -0.05486675, -0.31219858,  0.44331852,
         1.38567758,  1.01353371,  1.01901221]], dtype=float32)

In [92]:

    

h = ntm_model.read_heads[0]

plt.matshow(h.addressing.shift)
plt.colorbar()


    

    




---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-92-84f4d1cbfb41> in <module>()
      1 h = ntm_model.read_heads[0]
      2 
----> 3 plt.matshow(h.addressing.shift)
      4 plt.colorbar()

/usr/lib64/python3.4/site-packages/matplotlib/pyplot.py in matshow(A, fignum, **kw)
   2215     else:
   2216         # Extract actual aspect ratio of array and make appropriately sized figure
-> 2217         fig = figure(fignum, figsize=figaspect(A))
   2218         ax  = fig.add_axes([0.15, 0.09, 0.775, 0.775])
   2219 

/usr/lib64/python3.4/site-packages/matplotlib/figure.py in figaspect(arg)
   1648     # Extract the aspect ratio of the array
   1649     if isarray:
-> 1650         nr, nc = arg.shape[:2]
   1651         arr_ratio = float(nr) / nc
   1652     else:

ValueError: need more than 0 values to unpack

In [15]:

    

plt.matshow(ntm_model.memory_init)
plt.colorbar()


    

    
Out[15]:





<matplotlib.colorbar.Colorbar at 0x7ff2d805d4a8>






    

In [24]:

    

def combine_access(accesses):
    num_accesses = len(accesses)
    assert(num_accesses > 0)
    mem_locations = accesses[0].shape[1]
    res = np.zeros((num_accesses, mem_locations))
    norms = []
    for idx, access in enumerate(accesses):
        assert(access.shape == (1, mem_locations))
        res[idx] = access.w[0]
        norms.append(np.linalg.norm(access.w[0]))
    return res, norms

def inspect(model):
    start_prediction = D.Mat.ones((1,INPUT_SIZE), constant=True)
    c = Capture()
    with D.NoBackprop():
        with c:
            example = generate_example(INPUT_SIZE, MAX_TRAIN_SEQ_LENGTH)
            state = model.initial_states()
            for input_vec in example:
                state = model.activate(input_vec, state)

            state = model.activate(start_prediction, state)
            for input_vec in example:
                decoded = np.round(model.decode(state).sigmoid().w)
                print("input_vec", input_vec.w[0])
                print("decoded  ",   decoded[0])
                print("")
                state = model.activate(start_prediction, state)
    reads, read_norms  = combine_access(c.state["read_head_weights"])
    writes, write_norms = combine_access(c.state["write_head_weights"])
    
    print(c.state.keys())
    
    plt.figure()
    r = plt.matshow(reads)
    plt.title("Read locations")
    plt.colorbar(r)
    
    read_contents, _ = combine_access(c.state["read_head_content"])
    plt.figure()
    r = plt.matshow(read_contents)
    plt.title("Read contents")
    plt.colorbar(r)
    
    plt.figure()
    w = plt.matshow(writes)
    plt.title("Write locations")
    plt.colorbar(w)
    
    write_contents, _ = combine_access(c.state["write_head_content"])
    plt.figure()
    r = plt.matshow(write_contents)
    plt.title("Write contents")
    plt.colorbar(r)
    
    for idx, (read_n, write_n) in enumerate(zip(read_norms, write_norms)):
        print("At tstep %d READ: %.3f, WRITE: %.3f" % (idx, read_n, write_n))

inspect(ntm_model)


    

    




input_vec [ 1.  1.  0.  0.  1.  0.  0.  1.]
decoded   [ 1.  1.  0.  1.  0.  1.  0.  0.]

input_vec [ 0.  0.  0.  0.  0.  0.  1.  1.]
decoded   [ 1.  0.  1.  0.  0.  0.  0.  1.]

input_vec [ 1.  0.  1.  0.  0.  0.  0.  1.]
decoded   [ 1.  0.  0.  0.  0.  0.  0.  1.]

input_vec [ 1.  1.  0.  0.  1.  1.  0.  1.]
decoded   [ 1.  1.  0.  0.  0.  1.  0.  1.]

input_vec [ 1.  0.  1.  0.  0.  1.  0.  1.]
decoded   [ 1.  0.  0.  0.  0.  1.  1.  1.]

input_vec [ 0.  0.  1.  1.  1.  1.  0.  1.]
decoded   [ 1.  0.  0.  0.  0.  1.  1.  0.]

input_vec [ 0.  1.  1.  1.  0.  1.  1.  0.]
decoded   [ 1.  0.  0.  0.  0.  1.  1.  0.]

input_vec [ 0.  1.  1.  1.  0.  1.  1.  0.]
decoded   [ 1.  0.  0.  0.  0.  1.  1.  0.]

input_vec [ 0.  1.  0.  0.  1.  0.  1.  1.]
decoded   [ 1.  0.  0.  0.  0.  1.  1.  0.]

input_vec [ 0.  0.  1.  1.  0.  1.  0.  0.]
decoded   [ 1.  0.  0.  0.  0.  1.  1.  1.]

input_vec [ 1.  1.  0.  0.  0.  1.  1.  0.]
decoded   [ 1.  0.  0.  0.  0.  1.  1.  1.]

input_vec [ 0.  1.  0.  1.  0.  0.  1.  1.]
decoded   [ 1.  0.  0.  0.  0.  1.  1.  1.]

input_vec [ 1.  0.  0.  0.  0.  1.  1.  1.]
decoded   [ 1.  0.  0.  0.  0.  1.  1.  1.]

input_vec [ 1.  0.  0.  1.  1.  0.  1.  1.]
decoded   [ 1.  0.  0.  0.  0.  1.  1.  1.]

input_vec [ 1.  1.  0.  0.  1.  0.  1.  0.]
decoded   [ 1.  0.  0.  0.  0.  1.  1.  1.]

input_vec [ 1.  0.  0.  0.  0.  1.  1.  1.]
decoded   [ 1.  0.  0.  0.  0.  1.  1.  1.]

input_vec [ 0.  0.  0.  1.  1.  0.  1.  1.]
decoded   [ 1.  0.  0.  0.  0.  1.  1.  1.]

input_vec [ 1.  0.  1.  0.  1.  1.  0.  1.]
decoded   [ 1.  0.  0.  0.  0.  1.  1.  1.]

input_vec [ 1.  0.  0.  0.  0.  1.  1.  0.]
decoded   [ 1.  0.  0.  0.  0.  1.  1.  1.]

input_vec [ 1.  0.  0.  0.  0.  1.  1.  0.]
decoded   [ 1.  0.  0.  0.  0.  1.  1.  1.]

dict_keys(['write_head_content', 'read_head_content', 'read_head_weights', 'write_head_weights'])
At tstep 0 READ: 0.308, WRITE: 0.997
At tstep 1 READ: 0.533, WRITE: 0.988
At tstep 2 READ: 0.676, WRITE: 0.991
At tstep 3 READ: 0.870, WRITE: 0.840
At tstep 4 READ: 0.998, WRITE: 0.537
At tstep 5 READ: 1.000, WRITE: 0.746
At tstep 6 READ: 0.999, WRITE: 0.750
At tstep 7 READ: 1.000, WRITE: 0.750
At tstep 8 READ: 1.000, WRITE: 0.683
At tstep 9 READ: 0.998, WRITE: 0.717
At tstep 10 READ: 0.999, WRITE: 0.736
At tstep 11 READ: 1.000, WRITE: 0.551
At tstep 12 READ: 0.998, WRITE: 0.796
At tstep 13 READ: 1.000, WRITE: 0.764
At tstep 14 READ: 0.998, WRITE: 0.708
At tstep 15 READ: 0.999, WRITE: 0.798
At tstep 16 READ: 1.000, WRITE: 0.551
At tstep 17 READ: 0.993, WRITE: 0.658
At tstep 18 READ: 1.000, WRITE: 0.546
At tstep 19 READ: 0.999, WRITE: 0.546
At tstep 20 READ: 0.999, WRITE: 0.909
At tstep 21 READ: 0.482, WRITE: 0.978
At tstep 22 READ: 0.448, WRITE: 0.566
At tstep 23 READ: 0.360, WRITE: 0.088
At tstep 24 READ: 0.233, WRITE: 0.088
At tstep 25 READ: 0.179, WRITE: 0.088
At tstep 26 READ: 0.205, WRITE: 0.088
At tstep 27 READ: 0.189, WRITE: 0.088
At tstep 28 READ: 0.196, WRITE: 0.088
At tstep 29 READ: 0.191, WRITE: 0.088
At tstep 30 READ: 0.192, WRITE: 0.088
At tstep 31 READ: 0.190, WRITE: 0.088
At tstep 32 READ: 0.190, WRITE: 0.088
At tstep 33 READ: 0.189, WRITE: 0.088
At tstep 34 READ: 0.188, WRITE: 0.088
At tstep 35 READ: 0.187, WRITE: 0.088
At tstep 36 READ: 0.186, WRITE: 0.088
At tstep 37 READ: 0.185, WRITE: 0.088
At tstep 38 READ: 0.184, WRITE: 0.088
At tstep 39 READ: 0.183, WRITE: 0.088
At tstep 40 READ: 0.182, WRITE: 0.088






    






<matplotlib.figure.Figure at 0x7ff2bdfb9390>






    













    






<matplotlib.figure.Figure at 0x7ff2bdf0a630>






    













    






<matplotlib.figure.Figure at 0x7ff2bdeaca20>






    













    






<matplotlib.figure.Figure at 0x7ff2bddd1a90>






    

In [ ]:

    

c.state["read_head"][1].shape


    

In [ ]:

    

ntm_model.__class__ = NTM
for h in ntm_model.read_heads:
    h.__class__ = NTMReadHead
for h in ntm_model.write_heads:
    h.__class__ = NTMWriteHead


    

In [ ]:

    

ntm_model.write_heads[0].__class__ = NTMWriteHead


    

In [ ]:

    

A = D.Mat([[1,1,1],[2,2,2]])
B = D.Mat([[1,2,3]])


    

In [ ]:

    

print(A.w)
print(B.w)
A/ D.MatOps.broadcast(B, axis=0, num_replicas=2)


    

In [ ]:

    

A.w.mean(axis=0)


    

In [ ]:

    

A.softplus()


    

In [ ]:

    

A = D.Mat([1,2,3])


    

In [ ]:

    

D.Mat([1,2,3]) * D.Mat([2])


    

In [ ]:

    

x = D.random.uniform(-1,1, (1,1)).sum(0)


    

In [ ]:

    

D.MatOps.broadcast(x, axis=0, num_replicas=20)


    

In [ ]:

    

x = D.random.uniform(-1,1, (1,2))
y = D.random.uniform(-1,1, (1,3))


    

In [ ]:

    

x


    

In [ ]:

    

y


    

In [ ]:

    




    

In [ ]:

    

y.dot(x)


    

In [ ]:

    

ax = plt.subplots(2,2)


    

In [ ]:

    

ax


    

In [ ]:

    

1.0 - D.Mat.ones((2,3))


    

In [ ]: