In [102]:

    

#%sudo ldconfig /usr/local/cuda/lib64
%matplotlib inline
import theano
import numpy as np
import matplotlib.pyplot as plt

from sklearn.datasets import load_digits
from sklearn.preprocessing import LabelBinarizer
from lasagne import layers
from lasagne.updates import nesterov_momentum
from lasagne.nonlinearities import softmax
from nolearn.lasagne import NeuralNet
Conv2DLayer = layers.cuda_convnet.Conv2DCCLayer
MaxPool2DLayer = layers.cuda_convnet.MaxPool2DCCLayer
import pandas as pd


    

In [103]:

    

import time
def timestamp():
    return "_".join([str(k).zfill(2) for k in time.localtime()][0:5])

def show_image(img):
    plt.axis('off')
    plt.imshow(img, cmap='gray', interpolation='nearest')


    

In [3]:

    

def float32(k):
    return np.cast['float32'](k)

class AdjustVariable(object):
    def __init__(self, name, start=0.03, stop=0.001):
        self.name = name
        self.start, self.stop = start, stop
        self.ls = None

    def __call__(self, nn, train_history):
        if self.ls is None:
            self.ls = np.linspace(self.start, self.stop, nn.max_epochs)

        epoch = train_history[-1]['epoch']
        new_value = float32(self.ls[epoch - 1])
        setattr(nn, self.name, new_value)

class EarlyStopping(object):
    def __init__(self, patience=100):
        self.patience = patience
        self.best_valid = np.inf
        self.best_valid_epoch = 0
        self.best_weights = None

    def __call__(self, nn, train_history):
        current_valid = train_history[-1]['valid_loss']
        current_epoch = train_history[-1]['epoch']
        if current_valid < self.best_valid:
            self.best_valid = current_valid
            self.best_valid_epoch = current_epoch
            self.best_weights = [w.get_value() for w in nn.get_all_params()]
        elif self.best_valid_epoch + self.patience < current_epoch:
            print("Early stopping.")
            print("Best valid loss was {:.6f} at epoch {}.".format(
                self.best_valid, self.best_valid_epoch))
            nn.load_weights_from(self.best_weights)
            raise StopIteration()


    

In [4]:

    

import glob
import os
import matplotlib.pyplot as plt

classes = np.asarray([d.split('/')[1] for d in glob.glob('train/*')])
def load_filenames(base_dir, class_name):
    return [f.split('/')[2] for f in glob.glob(os.path.join(base_dir, class_name, '*.jpg'))]

def count_files(set_name, class_name):
    return len(load_filenames(set_name, class_name))

num_files = sum(count_files('train', c) for c in classes)
num_classes = len(classes)

print 'Number of classes:', num_classes
print 'Number of images:', num_files


    

    




Number of classes: 121
Number of images: 30336

In [127]:

    

%matplotlib inline
IMG_SIZE = 96
IMG_DIM = IMG_SIZE, IMG_SIZE

from skimage.filter import threshold_adaptive
from skimage.exposure import equalize_adapthist
from skimage.restoration import denoise_tv_chambolle, denoise_bilateral
from skimage.filter import sobel
from skimage.io import imread
from skimage.transform import resize
from skimage.filter.rank import median
from skimage.morphology import disk
#from PIL import ImageFilter

def place_center(img, shape):
    w, h = img.shape
    nw, nh = shape
    new_img = np.zeros(shape, dtype=img.dtype)
    x = nw / 2 - w / 2
    y = nh / 2 - h / 2
    #print 'reshaping to', (nw, nh), 'origin at', (x, y), (w, h), new_img.shape
    new_img[x:x+w, y:y+h] = img
    return new_img

def pre_process(img):
    #img = denoise_bilateral(img, sigma_range=0.1)
    
    #img = median(img, disk(1))
    img = (255 - img)
    
    #img = img.filter(ImageFilter.SHARPEN())
    
    s = max(img.shape)
    img = place_center(img, (s, s))
    img = resize(img, IMG_DIM)
    
    #img = (img - img.min()) / (img.max() - img.min())
    #img[img < 0.05] = 0.0
    #img = (img * 1000).astype(np.int8)
    return img.astype(np.float32)

img = imread('train/hydromedusae_typeD_bell_and_tentacles/1021.jpg')
img = pre_process(img)
print img.shape, img.max(), img.min()
show_image(img)
img.dtype, img.max(), img.min()


    

    




(96, 96) 0.854433 0.0






    
Out[127]:





(dtype('float32'), 0.85443264, 0.0)






    

In [128]:

    

import os, fnmatch
def find_files(directory, pattern):
    for root, dirs, files in os.walk(directory):
        for basename in files:
            if fnmatch.fnmatch(basename, pattern):
                filename = os.path.join(root, basename)
                yield filename


    

In [129]:

    

def load_train_data():
    for f in list(find_files('train/', '*.jpg')):
        class_name = f.split('/')[1]
        img = imread(f)
        yield class_name, pre_process(img)
           
def load_train():
    train = []
    train_label = []
    for i, (label, img) in enumerate(load_train_data()):
        train.append(img)
        train_label.append(label)
        if i % 2000 == 0:
            print i,

    train = np.asarray(train)
    train_label = np.asarray(train_label)
    
    return train, train_label

train, train_label = load_train()

idx = np.arange(len(train))
np.random.shuffle(idx)
X = (train.reshape(-1, 1, IMG_SIZE, IMG_SIZE)).astype(np.float32)[idx]
y = train_label[idx]

from sklearn.preprocessing import StandardScaler

#scaler = StandardScaler()
#scaler.fit(X.reshape(X.shape[0], -1))

#def picture_space(scaled_data):
#    Xt = scaler.inverse_transform(scaled_data.reshape(scaled_data.shape[0], -1))
#    return Xt.reshape(-1, 1, IMG_SIZE, IMG_SIZE)

#def input_space(unscaled_data):
#    Xt = scaler.transform(unscaled_data.reshape(unscaled_data.shape[0], -1))
#    return Xt.reshape(-1, 1, IMG_SIZE, IMG_SIZE)

#X_mean = X.reshape(X.shape[0], 96, 96).mean(axis=0)

print X.shape, X.dtype, X.max(), X.min(), y


    

    




0 2000 4000 6000 8000 10000 12000 14000 16000 18000 20000 22000 24000 26000 28000 30000 (30336, 1, 96, 96) float32 1.0 0.0 ['tunicate_doliolid' 'trichodesmium_puff' 'chaetognath_other' ...,
 'trichodesmium_puff' 'protist_noctiluca' 'copepod_cyclopoid_oithona']

In [148]:

    

from os.path import basename

def load_test_data():
    for f in glob.glob('test/*.jpg'):
        img = imread(f)
        id = basename(f).split('.jpg')[0]
        yield id, pre_process(img)

def load_test():
    data = []
    ids = []
    for i, (id, img) in enumerate(load_test_data()):
        ids.append(id)
        data.append(img)
        if i % 1000 == 0:
            print i,

    return (np.asarray(ids), np.asarray(data).reshape(-1, 1, IMG_SIZE, IMG_SIZE))

ids, X_test = load_test()
X_test.shape, X_test.dtype


    

    




0 1000 2000 3000 4000 5000 6000 7000 8000 9000 10000 11000 12000 13000 14000 15000 16000 17000 18000 19000 20000 21000 22000 23000 24000 25000 26000 27000 28000 29000 30000 31000 32000 33000 34000 35000 36000 37000 38000 39000 40000 41000 42000 43000 44000 45000 46000 47000 48000 49000 50000 51000 52000 53000 54000 55000 56000 57000 58000 59000 60000 61000 62000 63000 64000 65000 66000 67000 68000 69000 70000 71000 72000 73000 74000 75000 76000 77000 78000 79000 80000 81000 82000 83000 84000 85000 86000 87000 88000 89000 90000 91000 92000 93000 94000 95000 96000 97000 98000 99000 100000 101000 102000 103000 104000 105000 106000 107000 108000 109000 110000 111000 112000 113000 114000 115000 116000 117000 118000 119000 120000 121000 122000 123000 124000 125000 126000 127000 128000 129000 130000





    
Out[148]:





((130400, 1, 96, 96), dtype('float32'))






    

In [138]:

    

import math
from random import random
from skimage import transform

def rotate(img, angle):
    return transform.rotate(img, angle, resize=False)

def randomize_image(img):
    tf = transform.SimilarityTransform(scale=random() * 0.4 + 0.8,
                                       translation=((random() * 3), (random() * 3)))
    img = rotate(img, random() * 360)
    return transform.warp(img, tf)

img = imread('train/echinoderm_larva_pluteus_typeC/108510.jpg')
img = pre_process(img)
img = randomize_image(img)
plt.imshow(img, cmap='gray')
print img.shape, img.max(), img.min()


    

    




(96, 96) 0.830131736399 0.0






    

In [139]:

    

img = imread('train/hydromedusae_narcomedusae/102836.jpg')
plt.figure(figsize=(16, 5))
for i in range(10):
    plt.subplot(1, 10, i + 1)
    plt.axis('off')
    plt.imshow(randomize_image(pre_process(img)), cmap='gray')


    

In [140]:

    

plt.figure(figsize=(16, 5))
for i in range(10):
    plt.subplot(1, 10, i)
    plt.axis('off')
    plt.imshow(X_test[i][0], cmap='gray')


    

In [141]:

    

import skimage.transform as transform
import skimage

def fast_warp(img, tf, output_shape=IMG_DIM, mode='nearest'):
    """
    This wrapper function is about five times faster than skimage.transform.warp, for our use case.
    """
    m = tf.params
    img_wf = np.empty((output_shape[0], output_shape[1]), dtype='float32')
    img_wf = skimage.transform._warps_cy._warp_fast(img, m, output_shape=output_shape, mode=mode)
    return img_wf

def random_perturbation_transform(zoom_range, rotation_range, shear_range, translation_range, do_flip=False):
    # random shift [-4, 4] - shift no longer needs to be integer!
    shift_x = np.random.uniform(*translation_range)
    shift_y = np.random.uniform(*translation_range)
    translation = (shift_x, shift_y)

    # random rotation [0, 360]
    rotation = np.random.uniform(*rotation_range) # there is no post-augmentation, so full rotations here!

    # random shear [0, 5]
    shear = np.random.uniform(low=shear_range[0], high=shear_range[1])

    # random zoom [0.9, 1.1]
    # zoom = np.random.uniform(*zoom_range)
    log_zoom_range = [np.log(z) for z in zoom_range]
    zoom = np.exp(np.random.uniform(*log_zoom_range)) # for a zoom factor this sampling approach makes more sense.
    # the range should be multiplicatively symmetric, so [1/1.1, 1.1] instead of [0.9, 1.1] makes more sense.

    # flip
    if do_flip:
        shear += 180
        rotation += 180

    return build_augmentation_transform(zoom, rotation, shear, translation)


def build_augmentation_transform(zoom=1.0, rotation=0, shear=0, translation=(0, 0)):
    center_shift = np.array(IMG_DIM) / 2. - 0.5
    tform_center = transform.SimilarityTransform(translation=-center_shift)
    tform_uncenter = transform.SimilarityTransform(translation=center_shift)

    tform_augment = transform.AffineTransform(scale=(1/zoom, 1/zoom), 
                                              rotation=np.deg2rad(rotation), 
                                              shear=np.deg2rad(shear), 
                                              translation=translation)
    tform = tform_center + tform_augment + tform_uncenter 
    return tform


augmentation_params = {
            'zoom_range': (1.0, 1.1),
            'rotation_range': (0, 360),
            'shear_range': (0, 5),
            'translation_range': (-4, 4),
        }

def transform_randomly(X, plot = False):
    X = X.copy()
    tform_augment = random_perturbation_transform(**augmentation_params)
    tform_identity = skimage.transform.AffineTransform()
    tform_ds = skimage.transform.AffineTransform()

    for i in range(X.shape[0]):
        new = fast_warp(X[i][0], tform_ds + tform_augment + tform_identity, 
                             output_shape=IMG_DIM, mode='nearest').astype('float32')
        X[i,:] = new

    return X


    

In [142]:

    

from nolearn.lasagne import BatchIterator

class FlipBatchIterator(BatchIterator):
    def transform(self, Xb, yb):
        #Xb = Xb.copy()
        Xb, yb = super(FlipBatchIterator, self).transform(Xb, yb)
        #print 'shape', Xb.shape, Xb.min(), Xb.max()

        #bs = Xb.shape[0]
        #indices = np.random.choice(bs, bs / 2, replace=False)
        #Xb[indices] = Xb[indices, :, :, ::-1]
        #indices = np.random.choice(bs, bs / 2, replace=False)
        #Xb[indices] = Xb[indices, :, ::-1, :]
        
        #for i in range(bs):
        #    Xb[i][0] = randomize_image(Xb[i][0])
        #Xb += X_mean
        Xb = transform_randomly(Xb)
        #Xb -= X_mean
        
        return Xb, yb


    

In [143]:

    

import lasagne 
net2 = NeuralNet(
    layers=[
        ('input', layers.InputLayer),
        ('conv1', Conv2DLayer),
        ('pool1', MaxPool2DLayer),
        ('conv2', Conv2DLayer),
        ('conv3', Conv2DLayer),
        ('pool3', MaxPool2DLayer),
        ('hidden1', layers.DenseLayer),
        ('dropout1', layers.DropoutLayer),
        ('hidden2', layers.DenseLayer),
        ('dropout2', layers.DropoutLayer),
        ('hidden3', layers.DenseLayer),
        ('dropout3', layers.DropoutLayer),
        ('output', layers.DenseLayer),
        ],
    input_shape=(None, 1, IMG_SIZE, IMG_SIZE),

    conv1_num_filters=128, 
    conv1_filter_size=(5, 5), 
    conv1_pad=2, 
    conv1_strides=(4,4), 
    conv1_nonlinearity=lasagne.nonlinearities.rectify, 

    pool1_ds=(3, 3),
    pool1_strides=(2,2),

    conv2_num_filters=128, 
    conv2_filter_size=(3, 3), 
    conv2_pad=2, 
    conv2_nonlinearity=lasagne.nonlinearities.rectify,

    conv3_num_filters=256, 
    conv3_filter_size=(3, 3),
    conv3_pad=1, 
    conv3_nonlinearity=lasagne.nonlinearities.rectify, 

    pool3_ds=(3, 3),
    pool3_strides=(2,2),

    hidden1_num_units=512,
    hidden1_nonlinearity=lasagne.nonlinearities.rectify,

    dropout1_p=0.3,

    hidden2_num_units=1024,
    hidden2_nonlinearity=lasagne.nonlinearities.rectify, 

    dropout2_p=0.5,

    hidden3_num_units=1024,
    hidden3_nonlinearity=lasagne.nonlinearities.rectify, 

    dropout3_p=0.5,

    output_num_units=121, 
    output_nonlinearity=lasagne.nonlinearities.softmax,

    update_learning_rate=theano.shared(float32(0.01)),
    update_momentum=theano.shared(float32(0.9)),

    regression=False,
    #loss=lasagne.objectives.multinomial_nll,
    use_label_encoder=True,
    batch_iterator_train=FlipBatchIterator(batch_size=256),
    
    on_epoch_finished=[
        AdjustVariable('update_learning_rate', start=0.01, stop=0.0001),
        AdjustVariable('update_momentum', start=0.9, stop=0.999),
        EarlyStopping(patience=50),
        #SavePoint(interval=10, after=150)
        ],
    max_epochs=500,
    verbose=2,
    test_size=0.1
    )

net2.fit(X, y)


    

    




  InputLayer        	(None, 1, 96, 96)   	produces    9216 outputs
  Conv2DCCLayer     	(None, 128, 24, 24) 	produces   73728 outputs
  MaxPool2DCCLayer  	(None, 128, 12, 12) 	produces   18432 outputs
  Conv2DCCLayer     	(None, 128, 14, 14) 	produces   25088 outputs
  Conv2DCCLayer     	(None, 256, 14, 14) 	produces   50176 outputs
  MaxPool2DCCLayer  	(None, 256, 7, 7)   	produces   12544 outputs
  DenseLayer        	(None, 512)         	produces     512 outputs
  DropoutLayer      	(None, 512)         	produces     512 outputs
  DenseLayer        	(None, 1024)        	produces    1024 outputs
  DropoutLayer      	(None, 1024)        	produces    1024 outputs
  DenseLayer        	(None, 1024)        	produces    1024 outputs
  DropoutLayer      	(None, 1024)        	produces    1024 outputs
  DenseLayer        	(None, 121)         	produces     121 outputs

 Epoch  |  Train loss  |  Valid loss  |  Train / Val  |  Valid acc  |  Dur
--------|--------------|--------------|---------------|-------------|-------
     1  |    4.514816  |    4.163301  |     1.084432  |      8.27%  |  49.5s
     2  |    3.980086  |    3.597655  |     1.106300  |     16.93%  |  49.5s
     3  |    3.567194  |    3.284719  |     1.085997  |     21.65%  |  49.4s
     4  |    3.315604  |    3.060953  |     1.083193  |     26.87%  |  49.4s
     5  |    3.099026  |    2.872228  |     1.078962  |     28.84%  |  49.5s
     6  |    2.910819  |    2.697009  |     1.079277  |     29.85%  |  49.7s
     7  |    2.821706  |    2.616247  |     1.078532  |     31.24%  |  49.5s
     8  |    2.703946  |    2.498064  |     1.082417  |     33.68%  |  49.4s
     9  |    2.616390  |    2.512062  |     1.041531  |     32.31%  |  49.4s
    10  |    2.549234  |    2.428952  |     1.049520  |     34.68%  |  49.4s
    11  |    2.491193  |    2.337574  |     1.065717  |     36.35%  |  49.4s
    12  |    2.422464  |    2.215787  |     1.093275  |     39.40%  |  49.4s
    13  |    2.366790  |    2.208795  |     1.071530  |     38.61%  |  49.5s
    14  |    2.308735  |    2.175928  |     1.061035  |     39.54%  |  49.5s
    15  |    2.265619  |    2.120879  |     1.068245  |     40.65%  |  49.4s
    16  |    2.221924  |    2.070891  |     1.072931  |     42.37%  |  49.4s
    17  |    2.183815  |    2.083005  |     1.048396  |     42.21%  |  49.4s
    18  |    2.132203  |    2.009348  |     1.061142  |     42.72%  |  49.5s
    19  |    2.100908  |    2.011510  |     1.044443  |     43.03%  |  49.4s
    20  |    2.076659  |    1.900964  |     1.092424  |     45.94%  |  49.4s
    21  |    2.037948  |    1.920455  |     1.061180  |     45.34%  |  49.4s
    22  |    2.018811  |    1.872297  |     1.078253  |     46.65%  |  49.5s
    23  |    1.967410  |    1.841147  |     1.068579  |     47.85%  |  49.4s
    24  |    1.942533  |    1.810295  |     1.073048  |     47.55%  |  49.5s
    25  |    1.921008  |    1.832116  |     1.048519  |     46.90%  |  49.4s
    26  |    1.905511  |    1.769671  |     1.076760  |     48.84%  |  49.4s
    27  |    1.871097  |    1.755242  |     1.066005  |     49.27%  |  49.5s
    28  |    1.849135  |    1.720815  |     1.074569  |     50.50%  |  49.4s
    29  |    1.819282  |    1.692050  |     1.075194  |     51.02%  |  49.5s
    30  |    1.812130  |    1.743172  |     1.039559  |     49.66%  |  49.5s
    31  |    1.793987  |    1.666084  |     1.076768  |     51.08%  |  49.4s
    32  |    1.776803  |    1.634412  |     1.087121  |     52.22%  |  49.5s
    33  |    1.750452  |    1.658430  |     1.055488  |     51.63%  |  49.4s
    34  |    1.742827  |    1.563013  |     1.115043  |     54.19%  |  49.4s
    35  |    1.711854  |    1.563566  |     1.094839  |     53.69%  |  49.4s
    36  |    1.698909  |    1.568497  |     1.083145  |     54.32%  |  49.4s
    37  |    1.682404  |    1.561713  |     1.077281  |     54.15%  |  49.4s
    38  |    1.670981  |    1.534934  |     1.088634  |     55.15%  |  49.4s
    39  |    1.652180  |    1.513844  |     1.091381  |     55.01%  |  49.5s
    40  |    1.641673  |    1.533495  |     1.070543  |     54.49%  |  49.4s
    41  |    1.633144  |    1.561326  |     1.045998  |     54.19%  |  49.4s
    42  |    1.615360  |    1.474950  |     1.095196  |     56.76%  |  49.5s
    43  |    1.597133  |    1.485237  |     1.075339  |     56.76%  |  49.4s
    44  |    1.593419  |    1.486545  |     1.071894  |     56.42%  |  49.5s
    45  |    1.561915  |    1.470089  |     1.062463  |     57.16%  |  49.4s
    46  |    1.554206  |    1.424547  |     1.091017  |     58.14%  |  49.4s
    47  |    1.549561  |    1.440954  |     1.075372  |     57.50%  |  49.5s
    48  |    1.532824  |    1.434566  |     1.068493  |     57.32%  |  49.4s
    49  |    1.516609  |    1.440470  |     1.052857  |     57.80%  |  49.5s
    50  |    1.506098  |    1.389755  |     1.083715  |     59.09%  |  49.4s
    51  |    1.491354  |    1.414077  |     1.054649  |     58.27%  |  49.4s
    52  |    1.483318  |    1.390689  |     1.066607  |     58.30%  |  49.4s
    53  |    1.467695  |    1.367602  |     1.073188  |     59.12%  |  49.4s
    54  |    1.452139  |    1.332706  |     1.089617  |     60.34%  |  49.4s
    55  |    1.446781  |    1.362945  |     1.061511  |     59.53%  |  49.4s
    56  |    1.431749  |    1.339082  |     1.069202  |     59.51%  |  49.4s
    57  |    1.430642  |    1.317417  |     1.085945  |     60.37%  |  49.4s
    58  |    1.419246  |    1.328529  |     1.068284  |     59.43%  |  49.5s
    59  |    1.398970  |    1.301874  |     1.074582  |     60.64%  |  49.5s
    60  |    1.385206  |    1.287436  |     1.075942  |     60.89%  |  49.4s
    61  |    1.377407  |    1.355822  |     1.015920  |     59.37%  |  49.5s
    62  |    1.372244  |    1.283601  |     1.069058  |     60.47%  |  49.4s
    63  |    1.363415  |    1.304996  |     1.044766  |     60.41%  |  49.4s
    64  |    1.363615  |    1.266258  |     1.076885  |     61.52%  |  49.4s
    65  |    1.344508  |    1.276009  |     1.053682  |     61.16%  |  49.4s
    66  |    1.349081  |    1.292838  |     1.043503  |     60.92%  |  49.4s
    67  |    1.333125  |    1.252468  |     1.064398  |     61.63%  |  49.4s
    68  |    1.322692  |    1.252394  |     1.056131  |     61.88%  |  49.4s
    69  |    1.319835  |    1.241417  |     1.063169  |     61.86%  |  49.4s
    70  |    1.319091  |    1.222712  |     1.078824  |     62.63%  |  49.4s
    71  |    1.296077  |    1.212942  |     1.068540  |     63.10%  |  49.4s
    72  |    1.297146  |    1.209261  |     1.072677  |     62.80%  |  49.4s
    73  |    1.291823  |    1.204978  |     1.072072  |     62.76%  |  49.4s
    74  |    1.278415  |    1.220816  |     1.047181  |     62.70%  |  49.4s
    75  |    1.272310  |    1.190547  |     1.068677  |     64.07%  |  49.4s
    76  |    1.262495  |    1.176762  |     1.072855  |     64.41%  |  49.4s
    77  |    1.254775  |    1.195535  |     1.049551  |     63.75%  |  49.4s
    78  |    1.247010  |    1.215342  |     1.026057  |     63.13%  |  49.4s
    79  |    1.253050  |    1.172953  |     1.068287  |     64.25%  |  49.4s
    80  |    1.235529  |    1.167482  |     1.058286  |     64.17%  |  49.4s
    81  |    1.227560  |    1.155867  |     1.062025  |     64.68%  |  49.4s
    82  |    1.222169  |    1.151243  |     1.061608  |     64.90%  |  49.4s
    83  |    1.222224  |    1.131508  |     1.080172  |     65.53%  |  49.4s
    84  |    1.204587  |    1.143734  |     1.053206  |     64.70%  |  49.4s
    85  |    1.207969  |    1.139976  |     1.059644  |     65.00%  |  49.4s
    86  |    1.199705  |    1.156605  |     1.037264  |     64.44%  |  49.4s
    87  |    1.202236  |    1.136478  |     1.057861  |     65.82%  |  49.4s
    88  |    1.183083  |    1.120658  |     1.055705  |     65.63%  |  49.4s
    89  |    1.182217  |    1.133777  |     1.042724  |     65.14%  |  49.4s
    90  |    1.172806  |    1.126961  |     1.040680  |     65.63%  |  49.4s
    91  |    1.169853  |    1.135410  |     1.030335  |     65.51%  |  49.4s
    92  |    1.168894  |    1.122702  |     1.041143  |     65.59%  |  49.4s
    93  |    1.157378  |    1.134094  |     1.020531  |     65.09%  |  49.4s
    94  |    1.160542  |    1.110024  |     1.045511  |     66.03%  |  49.4s
    95  |    1.145131  |    1.111658  |     1.030111  |     66.05%  |  49.4s
    96  |    1.166569  |    1.112682  |     1.048430  |     65.72%  |  49.4s
    97  |    1.140121  |    1.094416  |     1.041762  |     66.60%  |  49.4s
    98  |    1.136276  |    1.109716  |     1.023934  |     65.88%  |  49.4s
    99  |    1.129624  |    1.085541  |     1.040609  |     67.23%  |  49.4s
   100  |    1.128190  |    1.098800  |     1.026747  |     66.54%  |  49.4s
   101  |    1.121748  |    1.105049  |     1.015112  |     66.31%  |  49.4s
   102  |    1.117422  |    1.062774  |     1.051420  |     67.20%  |  49.4s
   103  |    1.117758  |    1.091395  |     1.024155  |     66.05%  |  49.4s
   104  |    1.112636  |    1.078133  |     1.032003  |     66.78%  |  49.4s
   105  |    1.097550  |    1.094728  |     1.002578  |     66.12%  |  49.4s
   106  |    1.096879  |    1.072264  |     1.022956  |     66.69%  |  49.4s
   107  |    1.086308  |    1.081636  |     1.004319  |     66.56%  |  49.4s
   108  |    1.085203  |    1.064435  |     1.019510  |     68.03%  |  49.4s
   109  |    1.080925  |    1.062571  |     1.017273  |     67.44%  |  49.4s
   110  |    1.082615  |    1.055512  |     1.025677  |     67.39%  |  49.4s
   111  |    1.082033  |    1.054505  |     1.026105  |     67.73%  |  49.4s
   112  |    1.076614  |    1.056937  |     1.018617  |     67.47%  |  49.4s
   113  |    1.068702  |    1.047788  |     1.019961  |     67.83%  |  49.4s
   114  |    1.065820  |    1.105656  |     0.963971  |     66.12%  |  49.4s
   115  |    1.051182  |    1.040323  |     1.010438  |     68.14%  |  49.4s
   116  |    1.057008  |    1.043031  |     1.013400  |     67.89%  |  49.4s
   117  |    1.057013  |    1.035396  |     1.020878  |     68.50%  |  49.4s
   118  |    1.045406  |    1.045215  |     1.000182  |     68.08%  |  49.4s
   119  |    1.040881  |    1.042378  |     0.998564  |     67.67%  |  49.4s
   120  |    1.040673  |    1.055302  |     0.986138  |     67.36%  |  49.4s
   121  |    1.034022  |    1.042256  |     0.992100  |     67.76%  |  49.4s
   122  |    1.038296  |    1.020504  |     1.017434  |     68.84%  |  49.4s
   123  |    1.034292  |    1.021986  |     1.012040  |     68.81%  |  49.4s
   124  |    1.028819  |    1.032608  |     0.996330  |     67.83%  |  49.4s
   125  |    1.017421  |    1.032486  |     0.985409  |     68.41%  |  49.4s
   126  |    1.016683  |    1.041234  |     0.976422  |     67.94%  |  49.4s
   127  |    1.017793  |    1.019769  |     0.998062  |     68.66%  |  49.4s
   128  |    1.021532  |    1.014278  |     1.007151  |     69.05%  |  49.4s
   129  |    1.008226  |    1.035713  |     0.973460  |     68.47%  |  49.4s
   130  |    1.007548  |    1.025495  |     0.982499  |     68.16%  |  49.4s
   131  |    0.998978  |    1.039770  |     0.960768  |     68.13%  |  49.4s
   132  |    1.002085  |    1.004231  |     0.997863  |     69.27%  |  49.4s
   133  |    0.999713  |    0.995762  |     1.003968  |     69.31%  |  49.4s
   134  |    0.992301  |    1.014942  |     0.977693  |     68.29%  |  49.4s
   135  |    0.993338  |    1.006361  |     0.987059  |     69.37%  |  49.4s
   136  |    0.981874  |    1.016890  |     0.965566  |     68.75%  |  49.4s
   137  |    0.987003  |    1.010746  |     0.976510  |     68.98%  |  49.4s
   138  |    0.972870  |    0.998955  |     0.973887  |     69.11%  |  49.4s
   139  |    0.980951  |    1.007243  |     0.973897  |     68.84%  |  49.4s
   140  |    0.969003  |    1.032295  |     0.938688  |     68.03%  |  49.4s
   141  |    0.962753  |    0.999522  |     0.963213  |     69.40%  |  49.4s
   142  |    0.976385  |    0.989934  |     0.986314  |     70.01%  |  49.4s
   143  |    0.961898  |    1.007589  |     0.954654  |     69.32%  |  49.4s
   144  |    0.954035  |    1.007027  |     0.947379  |     68.95%  |  49.4s
   145  |    0.954553  |    1.015339  |     0.940132  |     68.46%  |  49.4s
   146  |    0.961166  |    1.009790  |     0.951848  |     69.49%  |  49.4s
   147  |    0.949982  |    0.979855  |     0.969513  |     69.79%  |  49.4s
   148  |    0.953713  |    0.996810  |     0.956765  |     69.22%  |  49.4s
   149  |    0.949701  |    0.992667  |     0.956716  |     69.33%  |  49.4s
   150  |    0.937505  |    0.991999  |     0.945066  |     69.68%  |  49.4s
   151  |    0.950444  |    0.983379  |     0.966509  |     69.80%  |  49.4s
   152  |    0.934313  |    0.986467  |     0.947130  |     69.08%  |  49.4s
   153  |    0.940119  |    1.002522  |     0.937754  |     68.86%  |  49.4s
   154  |    0.930459  |    1.001948  |     0.928650  |     69.45%  |  49.4s
   155  |    0.935676  |    1.017651  |     0.919446  |     68.25%  |  49.4s
   156  |    0.930803  |    0.982519  |     0.947364  |     70.17%  |  49.4s
   157  |    0.919376  |    0.992877  |     0.925971  |     69.36%  |  49.4s
   158  |    0.922103  |    1.002446  |     0.919853  |     69.40%  |  49.4s
   159  |    0.929952  |    0.972304  |     0.956442  |     70.27%  |  49.4s
   160  |    0.919674  |    0.973126  |     0.945072  |     70.14%  |  49.4s
   161  |    0.913589  |    0.984945  |     0.927554  |     69.76%  |  49.4s
   162  |    0.911624  |    0.966275  |     0.943441  |     70.41%  |  49.4s
   163  |    0.904873  |    0.977203  |     0.925982  |     70.15%  |  49.4s
   164  |    0.901595  |    0.967338  |     0.932037  |     70.72%  |  49.4s
   165  |    0.900213  |    0.974013  |     0.924231  |     70.36%  |  49.4s
   166  |    0.901677  |    0.985186  |     0.915235  |     69.36%  |  49.4s
   167  |    0.903380  |    0.970022  |     0.931298  |     69.96%  |  49.4s
   168  |    0.902221  |    0.973682  |     0.926608  |     70.12%  |  49.4s
   169  |    0.898907  |    0.985716  |     0.911933  |     69.75%  |  49.4s
   170  |    0.887822  |    0.990335  |     0.896487  |     69.43%  |  49.4s
   171  |    0.883402  |    0.959814  |     0.920389  |     70.57%  |  49.4s
   172  |    0.895198  |    0.953028  |     0.939321  |     70.61%  |  49.4s
   173  |    0.889065  |    0.949948  |     0.935909  |     70.64%  |  49.4s
   174  |    0.892756  |    0.965353  |     0.924798  |     70.09%  |  49.4s
   175  |    0.885764  |    0.954814  |     0.927682  |     70.57%  |  49.4s
   176  |    0.883839  |    0.966246  |     0.914714  |     70.60%  |  49.4s
   177  |    0.879234  |    0.950466  |     0.925055  |     70.73%  |  49.4s
   178  |    0.877990  |    0.940062  |     0.933971  |     71.60%  |  49.4s
   179  |    0.881069  |    0.939933  |     0.937374  |     71.27%  |  49.4s
   180  |    0.874736  |    0.957897  |     0.913183  |     70.41%  |  49.4s
   181  |    0.872765  |    0.970584  |     0.899216  |     69.96%  |  49.4s
   182  |    0.864182  |    1.019106  |     0.847980  |     68.87%  |  49.4s
   183  |    0.868964  |    0.967845  |     0.897833  |     69.87%  |  49.4s
   184  |    0.865975  |    0.949692  |     0.911848  |     70.29%  |  49.4s
   185  |    0.870170  |    0.960285  |     0.906158  |     70.49%  |  49.4s
   186  |    0.862089  |    0.951914  |     0.905637  |     70.45%  |  49.4s
   187  |    0.856168  |    0.963459  |     0.888640  |     70.48%  |  49.4s
   188  |    0.855411  |    0.944844  |     0.905346  |     70.97%  |  49.4s
   189  |    0.850957  |    0.955848  |     0.890264  |     70.50%  |  49.4s
   190  |    0.856053  |    0.961424  |     0.890401  |     70.92%  |  49.4s
   191  |    0.845816  |    0.960988  |     0.880153  |     70.13%  |  49.4s
   192  |    0.854358  |    0.965689  |     0.884713  |     70.26%  |  49.4s
   193  |    0.847524  |    0.967536  |     0.875961  |     70.26%  |  49.4s
   194  |    0.851652  |    0.998878  |     0.852609  |     69.20%  |  49.4s
   195  |    0.847535  |    0.966850  |     0.876594  |     70.20%  |  49.4s
   196  |    0.840495  |    0.955465  |     0.879672  |     70.55%  |  49.4s
   197  |    0.840563  |    0.951415  |     0.883487  |     71.17%  |  49.4s
   198  |    0.837756  |    0.950321  |     0.881550  |     71.45%  |  49.4s
   199  |    0.837865  |    0.961456  |     0.871455  |     70.33%  |  49.4s
   200  |    0.842373  |    0.965378  |     0.872583  |     70.59%  |  49.4s
   201  |    0.826729  |    0.947607  |     0.872439  |     71.09%  |  49.4s
   202  |    0.827910  |    0.949629  |     0.871824  |     71.05%  |  49.4s
   203  |    0.830272  |    0.944420  |     0.879134  |     71.07%  |  49.4s
   204  |    0.826034  |    0.930714  |     0.887526  |     71.41%  |  49.4s
   205  |    0.817770  |    0.948090  |     0.862545  |     70.97%  |  49.4s
   206  |    0.831199  |    0.945710  |     0.878916  |     71.11%  |  49.4s
   207  |    0.829552  |    0.948100  |     0.874962  |     71.26%  |  49.4s
   208  |    0.819805  |    0.939484  |     0.872613  |     71.45%  |  49.4s
   209  |    0.817773  |    0.938349  |     0.871502  |     71.43%  |  49.4s
   210  |    0.814414  |    0.941101  |     0.865384  |     71.11%  |  49.4s
   211  |    0.815203  |    0.942973  |     0.864503  |     70.93%  |  49.4s
   212  |    0.810585  |    0.951296  |     0.852085  |     70.84%  |  49.4s
   213  |    0.809185  |    0.936687  |     0.863879  |     71.45%  |  49.4s
   214  |    0.807195  |    0.932417  |     0.865702  |     71.59%  |  49.4s
   215  |    0.796035  |    0.936942  |     0.849610  |     71.47%  |  49.4s
   216  |    0.805593  |    0.949054  |     0.848838  |     70.65%  |  49.4s
   217  |    0.798731  |    0.939910  |     0.849795  |     71.10%  |  49.4s
   218  |    0.798498  |    0.948333  |     0.842002  |     70.86%  |  49.4s
   219  |    0.801256  |    0.941369  |     0.851160  |     71.75%  |  49.4s
   220  |    0.808488  |    0.943726  |     0.856698  |     71.04%  |  49.4s
   221  |    0.802212  |    0.931109  |     0.861566  |     71.82%  |  49.4s
   222  |    0.796021  |    0.932008  |     0.854093  |     71.72%  |  49.4s
   223  |    0.795592  |    0.930342  |     0.855161  |     71.90%  |  49.4s
   224  |    0.796487  |    0.932502  |     0.854140  |     72.03%  |  49.4s
   225  |    0.788895  |    0.924884  |     0.852966  |     71.76%  |  49.4s
   226  |    0.792307  |    0.950835  |     0.833275  |     70.72%  |  49.4s
   227  |    0.783191  |    0.919881  |     0.851404  |     72.49%  |  49.4s
   228  |    0.792024  |    0.924706  |     0.856515  |     71.85%  |  49.4s
   229  |    0.788496  |    0.936540  |     0.841924  |     71.91%  |  49.4s
   230  |    0.787978  |    0.932609  |     0.844918  |     71.56%  |  49.4s
   231  |    0.785036  |    0.936635  |     0.838145  |     71.57%  |  49.4s
   232  |    0.779073  |    0.922242  |     0.844760  |     71.97%  |  49.4s
   233  |    0.778738  |    0.939362  |     0.829007  |     71.74%  |  49.4s
   234  |    0.777392  |    0.918409  |     0.846454  |     72.26%  |  49.4s
   235  |    0.763842  |    0.937082  |     0.815129  |     71.58%  |  49.4s
   236  |    0.781248  |    0.938149  |     0.832754  |     71.51%  |  49.4s
   237  |    0.770802  |    0.959116  |     0.803659  |     70.90%  |  49.4s
   238  |    0.766570  |    0.926491  |     0.827391  |     71.82%  |  49.4s
   239  |    0.772932  |    0.926745  |     0.834028  |     71.85%  |  49.4s
   240  |    0.763671  |    0.926758  |     0.824024  |     71.74%  |  49.4s
   241  |    0.766188  |    0.925938  |     0.827472  |     71.93%  |  49.4s
   242  |    0.773151  |    0.934637  |     0.827220  |     71.95%  |  49.4s
   243  |    0.761318  |    0.928313  |     0.820110  |     71.99%  |  49.4s
   244  |    0.766596  |    0.931126  |     0.823300  |     71.83%  |  49.4s
   245  |    0.765821  |    0.934513  |     0.819486  |     71.22%  |  49.4s
   246  |    0.758088  |    0.934026  |     0.811636  |     71.61%  |  49.4s
   247  |    0.758844  |    0.920860  |     0.824060  |     71.77%  |  49.4s
   248  |    0.757697  |    0.926107  |     0.818152  |     72.00%  |  49.4s
   249  |    0.753380  |    0.943285  |     0.798678  |     71.54%  |  49.4s
   250  |    0.765321  |    0.938549  |     0.815430  |     71.23%  |  49.4s
   251  |    0.745445  |    0.943875  |     0.789771  |     71.66%  |  49.4s
   252  |    0.744768  |    0.937517  |     0.794405  |     71.71%  |  49.4s
   253  |    0.746841  |    0.938869  |     0.795470  |     71.57%  |  49.4s
   254  |    0.747362  |    0.931301  |     0.802492  |     71.69%  |  49.4s
   255  |    0.742702  |    0.943024  |     0.787575  |     71.54%  |  49.4s
   256  |    0.752451  |    0.916810  |     0.820728  |     72.34%  |  49.4s
   257  |    0.750356  |    0.931699  |     0.805363  |     71.66%  |  49.4s
   258  |    0.736891  |    0.923043  |     0.798328  |     72.34%  |  49.4s
   259  |    0.736678  |    0.921539  |     0.799399  |     72.34%  |  49.4s
   260  |    0.745727  |    0.926625  |     0.804777  |     71.86%  |  49.4s
   261  |    0.741144  |    0.922486  |     0.803421  |     72.36%  |  49.4s
   262  |    0.730796  |    0.932365  |     0.783808  |     72.03%  |  49.4s
   263  |    0.728777  |    0.919385  |     0.792679  |     71.99%  |  49.4s
   264  |    0.732199  |    0.936415  |     0.781918  |     71.76%  |  49.4s
   265  |    0.732964  |    0.920892  |     0.795928  |     71.97%  |  49.4s
   266  |    0.732527  |    0.932582  |     0.785482  |     71.32%  |  49.4s
   267  |    0.733064  |    0.921251  |     0.795726  |     72.28%  |  49.4s
   268  |    0.733052  |    0.934296  |     0.784603  |     72.20%  |  49.4s
   269  |    0.726792  |    0.918438  |     0.791335  |     72.68%  |  49.4s
   270  |    0.720705  |    0.922346  |     0.781382  |     72.33%  |  49.4s
   271  |    0.725622  |    0.921778  |     0.787199  |     72.17%  |  49.4s
   272  |    0.733478  |    0.909071  |     0.806843  |     72.30%  |  49.4s
   273  |    0.721379  |    0.930017  |     0.775662  |     72.04%  |  49.4s
   274  |    0.716054  |    0.918043  |     0.779978  |     72.47%  |  49.4s
   275  |    0.720020  |    0.916101  |     0.785962  |     72.33%  |  49.4s
   276  |    0.706735  |    0.920212  |     0.768014  |     72.36%  |  49.4s
   277  |    0.710928  |    0.927330  |     0.766640  |     72.04%  |  49.4s
   278  |    0.706938  |    0.909599  |     0.777197  |     72.62%  |  49.4s
   279  |    0.707781  |    0.923929  |     0.766056  |     71.89%  |  49.4s
   280  |    0.717188  |    0.915233  |     0.783613  |     72.30%  |  49.4s
   281  |    0.702395  |    0.918526  |     0.764698  |     72.49%  |  49.4s
   282  |    0.710904  |    0.909827  |     0.781362  |     72.82%  |  49.4s
   283  |    0.715301  |    0.915396  |     0.781411  |     72.40%  |  49.4s
   284  |    0.720404  |    0.917644  |     0.785058  |     71.96%  |  49.4s
   285  |    0.708330  |    0.922918  |     0.767490  |     71.87%  |  49.4s
   286  |    0.700401  |    0.913698  |     0.766556  |     72.37%  |  49.4s
   287  |    0.705097  |    0.925862  |     0.761557  |     72.25%  |  49.4s
   288  |    0.709369  |    0.921165  |     0.770079  |     72.21%  |  49.4s
   289  |    0.698956  |    0.932391  |     0.749638  |     71.93%  |  49.4s
   290  |    0.711080  |    0.923316  |     0.770137  |     72.13%  |  49.4s
   291  |    0.700229  |    0.945503  |     0.740589  |     71.81%  |  49.4s
   292  |    0.710864  |    0.927015  |     0.766831  |     72.58%  |  49.4s
   293  |    0.704391  |    0.923846  |     0.762454  |     72.34%  |  49.4s
   294  |    0.691201  |    0.921059  |     0.750442  |     72.51%  |  49.4s
   295  |    0.688865  |    0.913954  |     0.753719  |     72.76%  |  49.4s
   296  |    0.685017  |    0.922960  |     0.742196  |     72.05%  |  49.4s
   297  |    0.696606  |    0.924606  |     0.753409  |     72.43%  |  49.4s
   298  |    0.690129  |    0.928493  |     0.743278  |     72.02%  |  49.4s
   299  |    0.687595  |    0.919014  |     0.748188  |     72.34%  |  49.4s
   300  |    0.685325  |    0.918171  |     0.746402  |     72.27%  |  49.4s
   301  |    0.691333  |    0.924638  |     0.747679  |     72.54%  |  49.4s
   302  |    0.681726  |    0.922700  |     0.738838  |     72.80%  |  49.4s
   303  |    0.696184  |    0.921536  |     0.755460  |     72.42%  |  49.4s
   304  |    0.686320  |    0.922102  |     0.744299  |     72.67%  |  49.4s
   305  |    0.689981  |    0.946737  |     0.728800  |     71.58%  |  49.4s
   306  |    0.687565  |    0.906253  |     0.758690  |     73.12%  |  49.4s
   307  |    0.679431  |    0.915929  |     0.741795  |     72.69%  |  49.4s
   308  |    0.678204  |    0.910859  |     0.744577  |     73.08%  |  49.4s
   309  |    0.679538  |    0.910822  |     0.746071  |     72.32%  |  49.4s
   310  |    0.682977  |    0.924808  |     0.738507  |     72.06%  |  49.4s
   311  |    0.677376  |    0.922763  |     0.734073  |     72.87%  |  49.4s
   312  |    0.672514  |    0.909086  |     0.739769  |     72.50%  |  49.4s
   313  |    0.675404  |    0.906137  |     0.745366  |     72.37%  |  49.4s
   314  |    0.678743  |    0.953911  |     0.711537  |     71.26%  |  49.4s
   315  |    0.683877  |    0.924296  |     0.739890  |     72.37%  |  49.4s
   316  |    0.669211  |    0.905331  |     0.739189  |     72.76%  |  49.4s
   317  |    0.676231  |    0.917757  |     0.736830  |     72.75%  |  49.4s
   318  |    0.668742  |    0.928285  |     0.720406  |     72.33%  |  49.4s
   319  |    0.669340  |    0.921558  |     0.726314  |     72.83%  |  49.4s
   320  |    0.661386  |    0.916987  |     0.721261  |     72.34%  |  49.4s
   321  |    0.664787  |    0.921216  |     0.721640  |     72.54%  |  49.4s
   322  |    0.669071  |    0.935918  |     0.714882  |     71.89%  |  49.4s
   323  |    0.673991  |    0.912290  |     0.738790  |     72.23%  |  49.4s
   324  |    0.673876  |    0.920255  |     0.732272  |     72.57%  |  49.4s
   325  |    0.663447  |    0.933546  |     0.710674  |     72.52%  |  49.4s
   326  |    0.658156  |    0.929423  |     0.708134  |     72.50%  |  49.4s
   327  |    0.660325  |    0.923813  |     0.714782  |     72.58%  |  49.4s
   328  |    0.658969  |    0.913435  |     0.721419  |     72.77%  |  49.4s
   329  |    0.667764  |    0.926042  |     0.721094  |     72.56%  |  49.4s
   330  |    0.651988  |    0.921675  |     0.707395  |     72.71%  |  49.4s
   331  |    0.651169  |    0.938713  |     0.693682  |     72.31%  |  49.4s
   332  |    0.657655  |    0.935980  |     0.702638  |     72.57%  |  49.4s
   333  |    0.651096  |    0.923542  |     0.704998  |     72.64%  |  49.4s
   334  |    0.656845  |    0.913370  |     0.719144  |     72.57%  |  49.4s
   335  |    0.643925  |    0.934483  |     0.689071  |     72.79%  |  49.4s
   336  |    0.656832  |    0.928107  |     0.707712  |     72.36%  |  49.4s
   337  |    0.658005  |    0.933003  |     0.705255  |     72.31%  |  49.4s
   338  |    0.645088  |    0.915861  |     0.704352  |     72.96%  |  49.4s
   339  |    0.641145  |    0.926035  |     0.692355  |     72.89%  |  49.4s
   340  |    0.645164  |    0.932061  |     0.692190  |     72.52%  |  49.4s
   341  |    0.640423  |    0.929777  |     0.688792  |     72.67%  |  49.4s
   342  |    0.648243  |    0.919813  |     0.704755  |     73.20%  |  49.4s
   343  |    0.636766  |    0.931616  |     0.683507  |     73.43%  |  49.4s
   344  |    0.642386  |    0.958868  |     0.669942  |     71.66%  |  49.4s
   345  |    0.645138  |    0.921027  |     0.700455  |     72.63%  |  49.4s
   346  |    0.635598  |    0.916588  |     0.693439  |     72.76%  |  49.4s
   347  |    0.638965  |    0.927532  |     0.688888  |     73.06%  |  49.4s
   348  |    0.635730  |    0.927994  |     0.685059  |     72.81%  |  49.4s
   349  |    0.642340  |    0.923824  |     0.695305  |     72.18%  |  49.4s
   350  |    0.638139  |    0.937873  |     0.680410  |     72.56%  |  49.4s
   351  |    0.634168  |    0.922569  |     0.687393  |     72.74%  |  49.4s
   352  |    0.635287  |    0.926964  |     0.685342  |     72.71%  |  49.4s
   353  |    0.637509  |    0.937422  |     0.680067  |     72.57%  |  49.4s
   354  |    0.629893  |    0.939596  |     0.670387  |     72.67%  |  49.4s
   355  |    0.632785  |    0.950380  |     0.665824  |     72.58%  |  49.4s
   356  |    0.643508  |    0.934564  |     0.688565  |     72.59%  |  49.4s
   357  |    0.627625  |    0.917793  |     0.683841  |     73.47%  |  49.4s
   358  |    0.632957  |    0.951601  |     0.665149  |     71.97%  |  49.4s
   359  |    0.635932  |    0.947261  |     0.671338  |     72.52%  |  49.4s
   360  |    0.627176  |    0.924132  |     0.678665  |     72.83%  |  49.4s
   361  |    0.628446  |    0.923385  |     0.680589  |     73.06%  |  49.4s
   362  |    0.621080  |    0.976203  |     0.636220  |     72.29%  |  49.4s
   363  |    0.627363  |    0.929566  |     0.674899  |     73.27%  |  49.4s
   364  |    0.625007  |    0.944542  |     0.661704  |     72.76%  |  49.4s
   365  |    0.622927  |    0.934442  |     0.666630  |     72.83%  |  49.4s
   366  |    0.623295  |    0.924217  |     0.674403  |     73.09%  |  49.4s
   367  |    0.627894  |    0.934297  |     0.672050  |     72.63%  |  49.4s
Early stopping.
Best valid loss was 0.905331 at epoch 316.






    
Out[143]:





NeuralNet(X_tensor_type=<function tensor4 at 0x7f65461392a8>,
     batch_iterator_test=<nolearn.lasagne.BatchIterator object at 0x7f6539004590>,
     batch_iterator_train=<__main__.FlipBatchIterator object at 0x7f64ed4bbfd0>,
     conv1_filter_size=(5, 5),
     conv1_nonlinearity=<function rectify at 0x7f65394a6398>,
     conv1_num_filters=128, conv1_pad=2, conv1_strides=(4, 4),
     conv2_filter_size=(3, 3),
     conv2_nonlinearity=<function rectify at 0x7f65394a6398>,
     conv2_num_filters=128, conv2_pad=2, conv3_filter_size=(3, 3),
     conv3_nonlinearity=<function rectify at 0x7f65394a6398>,
     conv3_num_filters=256, conv3_pad=1, dropout1_p=0.3, dropout2_p=0.5,
     dropout3_p=0.5, eval_size=0.2,
     hidden1_nonlinearity=<function rectify at 0x7f65394a6398>,
     hidden1_num_units=512,
     hidden2_nonlinearity=<function rectify at 0x7f65394a6398>,
     hidden2_num_units=1024,
     hidden3_nonlinearity=<function rectify at 0x7f65394a6398>,
     hidden3_num_units=1024, input_shape=(None, 1, 96, 96),
     layers=[('input', <class 'lasagne.layers.input.InputLayer'>), ('conv1', <class 'lasagne.layers.cuda_convnet.Conv2DCCLayer'>), ('pool1', <class 'lasagne.layers.cuda_convnet.MaxPool2DCCLayer'>), ('conv2', <class 'lasagne.layers.cuda_convnet.Conv2DCCLayer'>), ('conv3', <class 'lasagne.layers.cuda_convn...<class 'lasagne.layers.noise.DropoutLayer'>), ('output', <class 'lasagne.layers.dense.DenseLayer'>)],
     loss=<function negative_log_likelihood at 0x7f6539028de8>,
     max_epochs=500, more_params={},
     on_epoch_finished=[<__main__.AdjustVariable object at 0x7f64ed4972d0>, <__main__.AdjustVariable object at 0x7f64ed497310>, <__main__.EarlyStopping object at 0x7f64ed497390>],
     on_training_finished=(),
     output_nonlinearity=<theano.tensor.nnet.nnet.Softmax object at 0x7f6545d81b50>,
     output_num_units=121, pool1_ds=(3, 3), pool1_strides=(2, 2),
     pool3_ds=(3, 3), pool3_strides=(2, 2), regression=False,
     test_size=0.1, update=<function nesterov_momentum at 0x7f6539028aa0>,
     update_learning_rate=array(0.002738677430897951, dtype=float32),
     update_momentum=array(0.9726132154464722, dtype=float32),
     use_label_encoder=True, verbose=2,
     y_tensor_type=TensorType(int32, vector))

In [144]:

    

import sys
sys.setrecursionlimit(10000)
import cPickle as pickle
dump_name = 'net_{}.pickle'.format(min([d['valid_loss'] for d in net2.train_history_]))
with open(dump_name, 'wb') as f:
    pickle.dump(net2, f, -1)
dump_name


    

    
Out[144]:





'net_0.905331134796.pickle'

In [145]:

    

%matplotlib inline
import pandas as pd
train_history = pd.DataFrame(net2.train_history_)
train_history[['train_loss', 'valid_loss']].plot(figsize=(15,10))


    

    
Out[145]:





<matplotlib.axes._subplots.AxesSubplot at 0x7f64e36986d0>






    

In [146]:

    

net2.predict(X_test[100:110])


    

    
Out[146]:





array(['detritus_other', 'hydromedusae_narcomedusae',
       'unknown_blobs_and_smudges', 'detritus_other',
       'tunicate_doliolid_nurse', 'ctenophore_cestid',
       'trichodesmium_puff', 'copepod_calanoid_flatheads',
       'siphonophore_calycophoran_rocketship_adult', 'trichodesmium_puff'], 
      dtype='|S45')

In [147]:

    

plt.figure(figsize=(20,20))
c = 1
for label, id, img in zip(net2.predict(X_test[100:110]), ids, X_test[100:110]):
    plt.subplot(10, 10, c)
    plt.axis('off')
    plt.imshow(img[0], cmap='hot')
    c +=1
    for i, img in enumerate(X[y == label][0:9]):
        plt.subplot(10, 10, c)
        plt.axis('off')
        plt.imshow(img[0], cmap='gray')
        c += 1


    

In [ ]:

    

#y_test = net2.predict_proba(X_test)
#y_test += net2.predict_proba(X_test[:, :, :, ::-1])
#y_test += net2.predict_proba(X_test[:, :, ::-1, :])

#for i in [45, 90, 135, 180, 225, 270, 315]:
#    print i
#    Xt = X_test.copy()
#    for idx in range(Xt.shape[0]):
#        Xt[idx][0] = rotate(Xt[idx][0], i)
#    y_test += net2.predict_proba(Xt)
#    
#y_test /= 10.0


    

In [149]:

    

y_test = net2.predict_proba(X_test)
for i in range(20):
    print i, 'T',
    Xt = transform_randomly(X_test)
    print 'P',
    y_test += net2.predict_proba(Xt)

y_test /= 20.0


    

    




0 T P 1 T P 2 T P 3 T P 4 T P 5 T P 6 T P 7 T P 8 T P 9 T P 10 T P 11 T P 12 T P 13 T P 14 T P 15 T P 16 T P 17 T P 18 T P 19 T P

In [150]:

    

submission_filename = 'submission_{}'.format(timestamp())
submission = pd.DataFrame(y_test, index=[id + '.jpg' for id in ids], columns=net2.enc_.classes_)
submission.to_csv(submission_filename + '.csv', index_label='image')
print submission_filename


    

    




submission_2015_03_10_07_19

In [ ]:

    

!zip -u {submission_filename}.zip {submission_filename}.csv