In [39]:

    

import gzip
import itertools
import pickle
import os
import sys
import numpy as np
import lasagne
import theano
import theano.tensor as T
import time


    

In [40]:

    

import pandas as pd
import numpy as np
import random


    

In [41]:

    

from preprocess_data_lib import *


    

In [42]:

    

(X_train, y_train, X_valid, y_valid, X_test, y_test) = getData2(pct=0, cast=False)


    

In [43]:

    

(X_train.shape, y_train.shape, X_valid.shape, y_valid.shape, X_test.shape, y_test.shape)


    

    
Out[43]:





((42000, 784), (42000,), (0, 784), (0,), (28000, 784), (28000,))

In [ ]:

    




    

In [ ]:

    




    

In [44]:

    

BATCH_SIZE = 500
NUM_HIDDEN_UNITS = 1024
LEARNING_RATE = 0.02
MOMENTUM = 0.9


    

In [45]:

    

data_dic = dict(
        X_train=theano.shared(lasagne.utils.floatX(X_train)),
        y_train=T.cast(theano.shared(y_train), 'int32'),
        X_valid=theano.shared(lasagne.utils.floatX(X_valid)),
        y_valid=T.cast(theano.shared(y_valid), 'int32'),
        X_test=theano.shared(lasagne.utils.floatX(X_test)),
        y_test=T.cast(theano.shared(y_test), 'int32'),
        num_examples_train=X_train.shape[0],
        num_examples_valid=X_valid.shape[0],
        num_examples_test=X_test.shape[0],
        input_dim=X_train.shape[1],
        output_dim=10,
    )


    

In [46]:

    

def build_model(input_dim, output_dim,
                batch_size=BATCH_SIZE, num_hidden_units=NUM_HIDDEN_UNITS):
    """Create a symbolic representation of a neural network with `intput_dim`
    input nodes, `output_dim` output nodes and `num_hidden_units` per hidden
    layer.

    The training function of this model must have a mini-batch size of
    `batch_size`.

    A theano expression which represents such a network is returned.
    """
    l_in = lasagne.layers.InputLayer(
        shape=(batch_size, input_dim),
    )
    l_hidden1 = lasagne.layers.DenseLayer(
        l_in,
        num_units=num_hidden_units,
        nonlinearity=lasagne.nonlinearities.rectify,
    )
    l_hidden1_dropout = lasagne.layers.DropoutLayer(
        l_hidden1,
        p=0.5,
    )
    l_hidden2 = lasagne.layers.DenseLayer(
        l_hidden1_dropout,
        num_units=num_hidden_units,
        nonlinearity=lasagne.nonlinearities.rectify,
    )
    l_hidden2_dropout = lasagne.layers.DropoutLayer(
        l_hidden2,
        p=0.5,
    )
    l_out = lasagne.layers.DenseLayer(
        l_hidden2_dropout,
        num_units=output_dim,
        nonlinearity=lasagne.nonlinearities.softmax,
    )
    return l_out


    

In [47]:

    

def create_iter_functions(dataset, output_layer,
                          X_tensor_type=T.matrix,
                          batch_size=BATCH_SIZE,
                          learning_rate=LEARNING_RATE, momentum=MOMENTUM):
    """Create functions for training, validation and testing to iterate one
       epoch.
    """
    batch_index = T.iscalar('batch_index')
    X_batch = X_tensor_type('x')
    y_batch = T.ivector('y')
    batch_slice = slice(batch_index * batch_size,
                        (batch_index + 1) * batch_size)

    objective = lasagne.objectives.Objective(output_layer,
        loss_function=lasagne.objectives.categorical_crossentropy)

    loss_train = objective.get_loss(X_batch, target=y_batch)
    loss_eval = objective.get_loss(X_batch, target=y_batch,
                                   deterministic=True)

    pred = T.argmax(
        output_layer.get_output(X_batch, deterministic=True), axis=1)
    accuracy = T.mean(T.eq(pred, y_batch), dtype=theano.config.floatX)

    all_params = lasagne.layers.get_all_params(output_layer)
    updates = lasagne.updates.nesterov_momentum(
        loss_train, all_params, learning_rate, momentum)

    iter_train = theano.function(
        [batch_index], loss_train,
        updates=updates,
        givens={
            X_batch: dataset['X_train'][batch_slice],
            y_batch: dataset['y_train'][batch_slice],
        },
    )

    iter_valid = theano.function(
        [batch_index], [loss_eval, accuracy],
        givens={
            X_batch: dataset['X_valid'][batch_slice],
            y_batch: dataset['y_valid'][batch_slice],
        },
    )

    iter_test = theano.function(
        [batch_index], [pred],
        givens={
            X_batch: dataset['X_test'][batch_slice],
        },
    )

    return dict(
        train=iter_train,
        valid=iter_valid,
        test=iter_test,
    )


    

In [48]:

    

def train(iter_funcs, dataset, batch_size=BATCH_SIZE):
    """Train the model with `dataset` with mini-batch training. Each
       mini-batch has `batch_size` recordings.
    """
    num_batches_train = dataset['num_examples_train'] // batch_size
    num_batches_valid = dataset['num_examples_valid'] // batch_size

    for epoch in itertools.count(1):
        batch_train_losses = []
        for b in range(num_batches_train):
            batch_train_loss = iter_funcs['train'](b)
            batch_train_losses.append(batch_train_loss)

        avg_train_loss = np.mean(batch_train_losses)

        batch_valid_losses = []
        batch_valid_accuracies = []
        for b in range(num_batches_valid):
            batch_valid_loss, batch_valid_accuracy = iter_funcs['valid'](b)
            batch_valid_losses.append(batch_valid_loss)
            batch_valid_accuracies.append(batch_valid_accuracy)

        avg_valid_loss = np.mean(batch_valid_losses)
        avg_valid_accuracy = np.mean(batch_valid_accuracies)

        yield {
            'number': epoch,
            'train_loss': avg_train_loss,
            'valid_loss': avg_valid_loss,
            'valid_accuracy': avg_valid_accuracy,
        }


    

In [49]:

    

def test(iter_funcs, dataset, batch_size=BATCH_SIZE):
    num_batches_test = dataset['num_examples_test'] // batch_size

    for b in range(num_batches_test):
        yield iter_funcs['test'](b)
    yield iter_funcs['test'](num_batches_test)


    

In [ ]:

    




    

In [50]:

    

NUM_EPOCHS = 100


    

In [51]:

    

print("Loading data...")
dataset = data_dic

print("Building model and compiling functions...")
output_layer = build_model(
    input_dim=dataset['input_dim'],
    output_dim=dataset['output_dim'],
)
iter_funcs = create_iter_functions(dataset, output_layer)

print("Starting training...")
now = time.time()
try:
    for epoch in train(iter_funcs, dataset):
        print("Epoch {} of {} took {:.3f}s".format(
            epoch['number'], NUM_EPOCHS, time.time() - now))
        now = time.time()
        print("  training loss:\t\t{:.6f}".format(epoch['train_loss']))
        print("  validation loss:\t\t{:.6f}".format(epoch['valid_loss']))
        print("  validation accuracy:\t\t{:.2f} %%".format(
            epoch['valid_accuracy'] * 100))

        if epoch['number'] >= NUM_EPOCHS:
            break

except KeyboardInterrupt:
    pass


    

    




Loading data...
Building model and compiling functions...
Starting training...
Epoch 1 of 100 took 0.635s
  training loss:		1.042905
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 2 of 100 took 0.617s
  training loss:		0.452068
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 3 of 100 took 0.617s
  training loss:		0.359954
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 4 of 100 took 0.618s
  training loss:		0.304467
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 5 of 100 took 0.618s
  training loss:		0.265816
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 6 of 100 took 0.619s
  training loss:		0.237025
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 7 of 100 took 0.631s
  training loss:		0.216539
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 8 of 100 took 0.617s
  training loss:		0.197566
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 9 of 100 took 0.618s
  training loss:		0.180497
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 10 of 100 took 0.618s
  training loss:		0.170757
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 11 of 100 took 0.619s
  training loss:		0.156771
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 12 of 100 took 0.639s
  training loss:		0.147721
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 13 of 100 took 0.651s
  training loss:		0.137998
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 14 of 100 took 0.620s
  training loss:		0.130063
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 15 of 100 took 0.621s
  training loss:		0.126899
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 16 of 100 took 0.619s
  training loss:		0.118380
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 17 of 100 took 0.616s
  training loss:		0.113466
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 18 of 100 took 0.617s
  training loss:		0.108915
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 19 of 100 took 0.616s
  training loss:		0.104899
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 20 of 100 took 0.617s
  training loss:		0.098360
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 21 of 100 took 0.618s
  training loss:		0.093693
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 22 of 100 took 0.619s
  training loss:		0.090711
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 23 of 100 took 0.616s
  training loss:		0.084015
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 24 of 100 took 0.618s
  training loss:		0.081906
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 25 of 100 took 0.617s
  training loss:		0.080908
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 26 of 100 took 0.618s
  training loss:		0.076557
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 27 of 100 took 0.620s
  training loss:		0.074194
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 28 of 100 took 0.617s
  training loss:		0.070215
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 29 of 100 took 0.618s
  training loss:		0.066142
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 30 of 100 took 0.619s
  training loss:		0.065825
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 31 of 100 took 0.616s
  training loss:		0.063669
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 32 of 100 took 0.617s
  training loss:		0.061454
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 33 of 100 took 0.617s
  training loss:		0.059140
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 34 of 100 took 0.616s
  training loss:		0.058484
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 35 of 100 took 0.617s
  training loss:		0.057202
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 36 of 100 took 0.626s
  training loss:		0.054489
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 37 of 100 took 0.615s
  training loss:		0.051364
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 38 of 100 took 0.618s
  training loss:		0.050878
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 39 of 100 took 0.617s
  training loss:		0.049513
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 40 of 100 took 0.616s
  training loss:		0.047243
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 41 of 100 took 0.616s
  training loss:		0.046713
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 42 of 100 took 0.616s
  training loss:		0.044816
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 43 of 100 took 0.618s
  training loss:		0.043358
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 44 of 100 took 0.616s
  training loss:		0.042406
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 45 of 100 took 0.616s
  training loss:		0.040528
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 46 of 100 took 0.617s
  training loss:		0.041039
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 47 of 100 took 0.617s
  training loss:		0.038347
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 48 of 100 took 0.618s
  training loss:		0.037510
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 49 of 100 took 0.617s
  training loss:		0.036908
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 50 of 100 took 0.615s
  training loss:		0.035715
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 51 of 100 took 0.619s
  training loss:		0.035605
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 52 of 100 took 0.623s
  training loss:		0.034216
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 53 of 100 took 0.627s
  training loss:		0.032846
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 54 of 100 took 0.626s
  training loss:		0.031702
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 55 of 100 took 0.626s
  training loss:		0.030943
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 56 of 100 took 0.627s
  training loss:		0.029911
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 57 of 100 took 0.627s
  training loss:		0.030752
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 58 of 100 took 0.629s
  training loss:		0.030667
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 59 of 100 took 0.627s
  training loss:		0.028342
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 60 of 100 took 0.630s
  training loss:		0.027601
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 61 of 100 took 0.626s
  training loss:		0.027024
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 62 of 100 took 0.628s
  training loss:		0.025029
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 63 of 100 took 0.626s
  training loss:		0.026850
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 64 of 100 took 0.627s
  training loss:		0.026394
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 65 of 100 took 0.619s
  training loss:		0.025868
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 66 of 100 took 0.616s
  training loss:		0.024428
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 67 of 100 took 0.616s
  training loss:		0.023751
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 68 of 100 took 0.617s
  training loss:		0.023392
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 69 of 100 took 0.617s
  training loss:		0.022423
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 70 of 100 took 0.616s
  training loss:		0.022982
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 71 of 100 took 0.616s
  training loss:		0.021808
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 72 of 100 took 0.617s
  training loss:		0.022462
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 73 of 100 took 0.615s
  training loss:		0.021161
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 74 of 100 took 0.616s
  training loss:		0.020636
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 75 of 100 took 0.615s
  training loss:		0.021639
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 76 of 100 took 0.621s
  training loss:		0.019931
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 77 of 100 took 0.616s
  training loss:		0.018964
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 78 of 100 took 0.618s
  training loss:		0.018928
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 79 of 100 took 0.617s
  training loss:		0.019161
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 80 of 100 took 0.616s
  training loss:		0.018364
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 81 of 100 took 0.616s
  training loss:		0.017803
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 82 of 100 took 0.616s
  training loss:		0.017143
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 83 of 100 took 0.614s
  training loss:		0.016800
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 84 of 100 took 0.616s
  training loss:		0.018071
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 85 of 100 took 0.615s
  training loss:		0.018001
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 86 of 100 took 0.615s
  training loss:		0.017060
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 87 of 100 took 0.615s
  training loss:		0.018432
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 88 of 100 took 0.615s
  training loss:		0.017277
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 89 of 100 took 0.615s
  training loss:		0.016085
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 90 of 100 took 0.616s
  training loss:		0.014659
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 91 of 100 took 0.613s
  training loss:		0.016130
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 92 of 100 took 0.615s
  training loss:		0.015415
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 93 of 100 took 0.613s
  training loss:		0.013986
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 94 of 100 took 0.627s
  training loss:		0.015158
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 95 of 100 took 0.618s
  training loss:		0.015210
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 96 of 100 took 0.616s
  training loss:		0.014247
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 97 of 100 took 0.628s
  training loss:		0.014476
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 98 of 100 took 0.617s
  training loss:		0.013397
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 99 of 100 took 0.614s
  training loss:		0.014550
  validation loss:		nan
  validation accuracy:		nan %%
Epoch 100 of 100 took 0.617s
  training loss:		0.011932
  validation loss:		nan
  validation accuracy:		nan %%

In [ ]:

    




    

In [52]:

    

print("Loading data...")
dataset = data_dic

iter_funcs = create_iter_functions(dataset, output_layer)

print("Starting training...")
now = time.time()
try:
    preds = []
    for pred in test(iter_funcs, dataset):
        preds.extend(pred[0])
        
except KeyboardInterrupt:
    pass


    

    




Loading data...
Starting training...

In [ ]:

    




    

In [53]:

    

pd.DataFrame(preds).to_csv('../../data/intermediate/la4_pred.csv', index=None, header=None)


    

In [ ]: