Train sea lion classifier with a convnet


In [1]:
INPUT_DIR = '../../input/kaggle-sea-lion/02/'
OUTPUT_DIR = '../../output/kaggle-sea-lion/05/'
IMAGE_DIMS = (84,84,3)#class0
#IMAGE_DIMS = (84,84,3)#class1
#IMAGE_DIMS = (56,56,3)#class2
#IMAGE_DIMS = (42,42,3)#class3
#IMAGE_DIMS = (26,26,3)#class4

INPUT_DATASET_NAME = 'lion-patches-0px'
SAVE_WEIGHTS_FILE = OUTPUT_DIR + 'last-weights-simple-84-84.h5'
SAVE_MODEL_FILE = None
SAVE_MODEL_FILE = OUTPUT_DIR + 'last-model-simple-84-84.yml'
LOAD_WEIGHTS_FILE = SAVE_WEIGHTS_FILE
LOAD_MODEL_FILE = None

RECREATE_OUTPUT_DIR = False
RUN_TRAINING = True

TRAIN_WEIGHT = (1,1,1,1,1,1)
TEST_WEIGHT = (1,1,1,1,1,1)
TRAIN_EPOCHS = 10
INPUT_RANGE = 1
DEBUG = False

BATCH_SIZE=48

In [2]:
%matplotlib inline
import numpy as np
import pandas as pd
import h5py
import matplotlib.pyplot as plt
import sklearn
import os
import glob

import keras
from keras.preprocessing.image import ImageDataGenerator
from keras import models

from modules.logging import logger
import modules.utils as utils
from modules.utils import Timer
import modules.logging
import modules.cnn as cnn
import modules.lions as lions


Using TensorFlow backend.

Prepare

Prepare output dir


In [3]:
utils.mkdirs(OUTPUT_DIR, dirs=['tf-logs','weights'], recreate=RECREATE_OUTPUT_DIR)
modules.logging.setup_file_logger(OUTPUT_DIR + 'out.log')
TF_LOGS_DIR = OUTPUT_DIR + 'tf-logs/'
WEIGHTS_DIR = OUTPUT_DIR + 'weights/'
input_dataset_path = INPUT_DIR + utils.dataset_name(INPUT_DATASET_NAME, IMAGE_DIMS)

logger.info('Output dirs created')


2017-06-11 13:44:06,452 INFO Output dirs created

Prepare train, validate and test data flows


In [4]:
logger.info('Using dataset ' + input_dataset_path + ' as input')
h5file = h5py.File(input_dataset_path, 'r')

#used for image augmentation (creating new images for balancing)
image_augmentation_generator = ImageDataGenerator(
        featurewise_center=False,
        samplewise_center=False,
        featurewise_std_normalization=False,
        samplewise_std_normalization=False,
        zca_whitening=False,
        rotation_range=359,
        width_shift_range=0,
        height_shift_range=0,
        shear_range=0,
        horizontal_flip=True,
        vertical_flip=True)

#applied to all images during training
image_randomize_generator = ImageDataGenerator(
        featurewise_center=False,
        samplewise_center=False,
        featurewise_std_normalization=False,
        samplewise_std_normalization=False,
        zca_whitening=False,
        rotation_range=359,
        width_shift_range=0,
        height_shift_range=0,
        shear_range=0,
        horizontal_flip=True,
        vertical_flip=True)

logger.info('preparing train data')
train_batch_generator = utils.BatchGeneratorXYH5(h5file, start_ratio=0, end_ratio=INPUT_RANGE)
train_balance_generator = utils.ClassBalancerGeneratorXY(train_batch_generator,
                                                         image_augmentation=image_augmentation_generator,
                                                         output_weight=TRAIN_WEIGHT,
                                                         max_augmentation_ratio=1,
                                                         max_undersampling_ratio=1,
                                                         enforce_max_ratios=False,
                                                         batch_size=BATCH_SIZE,
                                                         start_ratio=0, end_ratio=0.7)
train_generator = utils.ImageAugmentationXYGenerator(train_balance_generator, image_randomize_generator)
logger.info('train size=' + str(train_generator.size) + ' batches=' + str(train_generator.nr_batches))


logger.info('preparing valid data')
valid_batch_generator = utils.BatchGeneratorXYH5(h5file, start_ratio=0, end_ratio=INPUT_RANGE)
valid_generator = utils.ClassBalancerGeneratorXY(valid_batch_generator,
                                                         image_augmentation=image_augmentation_generator,
                                                         output_weight=TEST_WEIGHT,
                                                         max_augmentation_ratio=1,
                                                         max_undersampling_ratio=1,
                                                         enforce_max_ratios=False,
                                                         batch_size=BATCH_SIZE,
                                                         start_ratio=0.7, end_ratio=0.85)
logger.info('valid size=' + str(valid_generator.size) + ' batches=' + str(valid_generator.nr_batches))



logger.info('preparing test data')
test_batch_generator = utils.BatchGeneratorXYH5(h5file, start_ratio=0, end_ratio=INPUT_RANGE)
test_generator = utils.ClassBalancerGeneratorXY(test_batch_generator,
                                                         image_augmentation=image_augmentation_generator,
                                                         output_weight=TEST_WEIGHT,
                                                         max_augmentation_ratio=1,
                                                         max_undersampling_ratio=1,
                                                         enforce_max_ratios=False,
                                                         batch_size=BATCH_SIZE,
                                                         start_ratio=0.85, end_ratio=1)
logger.info('test size=' + str(test_generator.size) + ' batches=' + str(test_generator.nr_batches))

#FIXME when using 1 on end ratio size and nr_batches gets negative (h5 batch generator, not balancer...)


2017-06-11 13:44:06,485 INFO Using dataset ../../input/kaggle-sea-lion/02/lion-patches-0px-84-84.h5 as input
2017-06-11 13:44:06,498 INFO preparing train data
2017-06-11 13:44:06,501 INFO loading input data for class distribution analysis...
2017-06-11 13:44:06,502 INFO loading Y from raw dataset
2017-06-11 13:44:06,504 INFO > [started] generator dump...
1600/2263
2017-06-11 13:44:06,920 INFO > [done]    generator dump (416.161 ms)
2017-06-11 13:44:06,923 INFO raw sample class distribution
2017-06-11 13:44:06,924 INFO 0: 78
2017-06-11 13:44:06,926 INFO 1: 45
2017-06-11 13:44:06,927 INFO 2: 675
2017-06-11 13:44:06,928 INFO 3: 150
2017-06-11 13:44:06,930 INFO 4: 281
2017-06-11 13:44:06,931 INFO 5: 1034
2017-06-11 13:44:06,932 INFO overall output samples per class: 90
2017-06-11 13:44:06,934 INFO augmentation/undersampling ratio per class
2017-06-11 13:44:06,935 INFO SETUP FLOW 0 0.7
2017-06-11 13:44:06,936 INFO calculating source range according to start/end range of the desired output..
2017-06-11 13:44:06,938 INFO output distribution for this flow
2017-06-11 13:44:06,939 INFO 0: 62 (1.15)
2017-06-11 13:44:06,940 INFO 1: 62 (2.00)
2017-06-11 13:44:06,941 INFO 2: 62 (0.13)
2017-06-11 13:44:06,943 INFO 3: 62 (0.60)
2017-06-11 13:44:06,944 INFO 4: 62 (0.32)
2017-06-11 13:44:06,946 INFO 5: 62 (0.09)
2017-06-11 13:44:06,949 INFO source range: 0-1598 (1598)
2017-06-11 13:44:06,950 INFO output range: 0-377 (377)
2017-06-11 13:44:06,951 INFO train size=377 batches=8
2017-06-11 13:44:06,953 INFO preparing valid data
2017-06-11 13:44:06,954 INFO loading input data for class distribution analysis...
2017-06-11 13:44:06,956 INFO loading Y from raw dataset
2017-06-11 13:44:06,957 INFO > [started] generator dump...
1600/2263
2017-06-11 13:44:07,284 INFO > [done]    generator dump (326.767 ms)
2017-06-11 13:44:07,287 INFO raw sample class distribution
2017-06-11 13:44:07,288 INFO 0: 78
2017-06-11 13:44:07,290 INFO 1: 45
2017-06-11 13:44:07,291 INFO 2: 675
2017-06-11 13:44:07,292 INFO 3: 150
2017-06-11 13:44:07,294 INFO 4: 281
2017-06-11 13:44:07,295 INFO 5: 1034
2017-06-11 13:44:07,296 INFO overall output samples per class: 90
2017-06-11 13:44:07,297 INFO augmentation/undersampling ratio per class
2017-06-11 13:44:07,299 INFO SETUP FLOW 0.7 0.85
2017-06-11 13:44:07,301 INFO calculating source range according to start/end range of the desired output..
2017-06-11 13:44:07,303 INFO output distribution for this flow
2017-06-11 13:44:07,305 INFO 0: 13 (1.15)
2017-06-11 13:44:07,306 INFO 1: 13 (2.00)
2017-06-11 13:44:07,307 INFO 2: 13 (0.13)
2017-06-11 13:44:07,308 INFO 3: 13 (0.60)
2017-06-11 13:44:07,310 INFO 4: 13 (0.32)
2017-06-11 13:44:07,311 INFO 5: 13 (0.09)
2017-06-11 13:44:07,314 INFO source range: 1835-2069 (234)
2017-06-11 13:44:07,315 INFO output range: 378-458 (80)
2017-06-11 13:44:07,317 INFO valid size=80 batches=2
2017-06-11 13:44:07,318 INFO preparing test data
2017-06-11 13:44:07,320 INFO loading input data for class distribution analysis...
2017-06-11 13:44:07,321 INFO loading Y from raw dataset
2017-06-11 13:44:07,322 INFO > [started] generator dump...
1600/2263
2017-06-11 13:44:07,644 INFO > [done]    generator dump (321.468 ms)
2017-06-11 13:44:07,646 INFO raw sample class distribution
2017-06-11 13:44:07,648 INFO 0: 78
2017-06-11 13:44:07,649 INFO 1: 45
2017-06-11 13:44:07,651 INFO 2: 675
2017-06-11 13:44:07,652 INFO 3: 150
2017-06-11 13:44:07,653 INFO 4: 281
2017-06-11 13:44:07,655 INFO 5: 1034
2017-06-11 13:44:07,656 INFO overall output samples per class: 90
2017-06-11 13:44:07,657 INFO augmentation/undersampling ratio per class
2017-06-11 13:44:07,659 INFO SETUP FLOW 0.85 1
2017-06-11 13:44:07,660 INFO calculating source range according to start/end range of the desired output..
2017-06-11 13:44:07,662 INFO output distribution for this flow
2017-06-11 13:44:07,663 INFO 0: 13 (1.15)
2017-06-11 13:44:07,664 INFO 1: 13 (2.00)
2017-06-11 13:44:07,666 INFO 2: 13 (0.13)
2017-06-11 13:44:07,667 INFO 3: 13 (0.60)
2017-06-11 13:44:07,668 INFO 4: 13 (0.32)
2017-06-11 13:44:07,669 INFO 5: 13 (0.09)
2017-06-11 13:44:07,672 INFO source range: 1905-2230 (325)
2017-06-11 13:44:07,674 INFO output range: 459-539 (80)
2017-06-11 13:44:07,675 INFO test size=80 batches=2

In [5]:
#logger.info('INPUT DATASET DATA')
#dataset_path = INPUT_DIR + utils.dataset_name(INPUT_DATASET_NAME, IMAGE_DIMS)
#with h5py.File(input_dataset_path, 'r') as h5file:
#    logger.info('generator')
#    input_generator = utils.BatchGeneratorXYH5(h5file, start_ratio=0.001, end_ratio=0.0012, batch_size=64)
#    X, Y = utils.dump_xy_to_array(input_generator.flow(), input_generator.size, x=True, y=True)
#    utils.show_images(X, image_labels=utils.onehot_to_label(Y), group_by_label=False, cols=10, is_bgr=True, size=2)
#
#    logger.info('x ' + str(np.shape(X)))
#    logger.info('y ' + str(np.shape(Y)))
#    logger.info(str(utils.class_distribution(Y)))

if(DEBUG):
    logger.info('BALANCE GENERATOR DATA')
    #dataset_path = INPUT_DIR + utils.dataset_name(INPUT_DATASET_NAME, IMAGE_DIMS)
    X_train, Y_train = utils.dump_xy_to_array(train_generator, train_generator.size, x=False, y=True)
    logger.info('y ' + str(np.shape(Y_train)))
    logger.info(str(utils.class_distribution(Y_train)))

    for xs,ys in train_generator.flow():
        utils.show_images(xs, image_labels=utils.onehot_to_label(ys), cols=10, is_bgr=True, size=2)
        break

Prepare CNN model


In [6]:
logger.info('Load CNN model')
#model = lions.convnet_alexnet2_lion_keras(IMAGE_DIMS)

model = None
if(LOAD_MODEL_FILE!=None and os.path.isfile(LOAD_MODEL_FILE)):
    with open(LOAD_MODEL_FILE, 'r') as model_file:
        my = model_file.read()
        model = models.model_from_yaml(my)
        logger.info('loaded model from file ' + LOAD_MODEL_FILE)
else:
    model = lions.convnet_medium1_lion_keras(IMAGE_DIMS)
    logger.info('loaded model from function convnet_medium1_lion_keras')
    

if(LOAD_WEIGHTS_FILE!=None and os.path.isfile(LOAD_WEIGHTS_FILE)):
    model.load_weights(LOAD_WEIGHTS_FILE)
    logger.info('Loaded model weights from ' + LOAD_WEIGHTS_FILE)

logger.info('Model prepared')


2017-06-11 13:44:07,714 INFO Load CNN model
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:69: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(64, (3, 3), padding="same", activation="relu", kernel_initializer="glorot_uniform")`
  model.add(convolutional.Conv2D(64, (3, 3), activation='relu', padding='same', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:71: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(128, (3, 3), padding="same", activation="relu", kernel_initializer="glorot_uniform")`
  model.add(convolutional.Conv2D(128, (3, 3), activation='relu', padding='same', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:73: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(256, (3, 3), padding="same", activation="relu", kernel_initializer="glorot_uniform")`
  model.add(convolutional.Conv2D(256, (3, 3), activation='relu', padding='same', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:78: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(1024, activation="relu", kernel_initializer="glorot_uniform")`
  model.add(core.Dense(1024, activation='relu', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:80: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(1024, activation="relu", kernel_initializer="glorot_uniform")`
  model.add(core.Dense(1024, activation='relu', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:82: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(6, activation="softmax", kernel_initializer="glorot_uniform")`
  model.add(core.Dense(6, activation='softmax', init='glorot_uniform'))
2017-06-11 13:44:07,886 INFO loaded model from function convnet_medium1_lion_keras
2017-06-11 13:44:07,890 INFO Model prepared

Train model


In [7]:
if(RUN_TRAINING):
    logger.info('Starting CNN training...')
    history = model.fit_generator(train_generator.flow(),
                  steps_per_epoch = train_generator.nr_batches,
                  nb_epoch = TRAIN_EPOCHS,
                  callbacks = cnn.get_callbacks_keras(model, WEIGHTS_DIR, TF_LOGS_DIR),
                  validation_data = valid_generator.flow(), 
                  validation_steps = valid_generator.nr_batches,
                  verbose = 1)

    if(SAVE_MODEL_FILE!=None):
        with open(SAVE_MODEL_FILE, 'w') as model_file:
            model_file.write(model.to_yaml())
            logger.info('Saved last model to ' + SAVE_MODEL_FILE)
    
    if(SAVE_WEIGHTS_FILE!=None):
        model.save_weights(SAVE_WEIGHTS_FILE)
        logger.info('Saved last weights to ' + SAVE_WEIGHTS_FILE)


2017-06-11 13:44:07,901 INFO Starting CNN training...
INFO:tensorflow:Summary name conv2d_1/kernel:0 is illegal; using conv2d_1/kernel_0 instead.
2017-06-11 13:44:08,098 INFO Summary name conv2d_1/kernel:0 is illegal; using conv2d_1/kernel_0 instead.
INFO:tensorflow:Summary name conv2d_1/kernel:0 is illegal; using conv2d_1/kernel_0 instead.
2017-06-11 13:44:08,104 INFO Summary name conv2d_1/kernel:0 is illegal; using conv2d_1/kernel_0 instead.
INFO:tensorflow:Summary name conv2d_1/bias:0 is illegal; using conv2d_1/bias_0 instead.
2017-06-11 13:44:08,107 INFO Summary name conv2d_1/bias:0 is illegal; using conv2d_1/bias_0 instead.
INFO:tensorflow:Summary name conv2d_1/bias:0 is illegal; using conv2d_1/bias_0 instead.
2017-06-11 13:44:08,114 INFO Summary name conv2d_1/bias:0 is illegal; using conv2d_1/bias_0 instead.
INFO:tensorflow:Summary name conv2d_2/kernel:0 is illegal; using conv2d_2/kernel_0 instead.
2017-06-11 13:44:08,119 INFO Summary name conv2d_2/kernel:0 is illegal; using conv2d_2/kernel_0 instead.
INFO:tensorflow:Summary name conv2d_2/kernel:0 is illegal; using conv2d_2/kernel_0 instead.
2017-06-11 13:44:08,125 INFO Summary name conv2d_2/kernel:0 is illegal; using conv2d_2/kernel_0 instead.
INFO:tensorflow:Summary name conv2d_2/bias:0 is illegal; using conv2d_2/bias_0 instead.
2017-06-11 13:44:08,128 INFO Summary name conv2d_2/bias:0 is illegal; using conv2d_2/bias_0 instead.
INFO:tensorflow:Summary name conv2d_2/bias:0 is illegal; using conv2d_2/bias_0 instead.
2017-06-11 13:44:08,134 INFO Summary name conv2d_2/bias:0 is illegal; using conv2d_2/bias_0 instead.
INFO:tensorflow:Summary name conv2d_3/kernel:0 is illegal; using conv2d_3/kernel_0 instead.
2017-06-11 13:44:08,140 INFO Summary name conv2d_3/kernel:0 is illegal; using conv2d_3/kernel_0 instead.
INFO:tensorflow:Summary name conv2d_3/kernel:0 is illegal; using conv2d_3/kernel_0 instead.
2017-06-11 13:44:08,146 INFO Summary name conv2d_3/kernel:0 is illegal; using conv2d_3/kernel_0 instead.
INFO:tensorflow:Summary name conv2d_3/bias:0 is illegal; using conv2d_3/bias_0 instead.
2017-06-11 13:44:08,150 INFO Summary name conv2d_3/bias:0 is illegal; using conv2d_3/bias_0 instead.
INFO:tensorflow:Summary name conv2d_3/bias:0 is illegal; using conv2d_3/bias_0 instead.
2017-06-11 13:44:08,157 INFO Summary name conv2d_3/bias:0 is illegal; using conv2d_3/bias_0 instead.
INFO:tensorflow:Summary name dense_1/kernel:0 is illegal; using dense_1/kernel_0 instead.
2017-06-11 13:44:08,163 INFO Summary name dense_1/kernel:0 is illegal; using dense_1/kernel_0 instead.
INFO:tensorflow:Summary name dense_1/kernel:0 is illegal; using dense_1/kernel_0 instead.
2017-06-11 13:44:08,172 INFO Summary name dense_1/kernel:0 is illegal; using dense_1/kernel_0 instead.
INFO:tensorflow:Summary name dense_1/bias:0 is illegal; using dense_1/bias_0 instead.
2017-06-11 13:44:08,175 INFO Summary name dense_1/bias:0 is illegal; using dense_1/bias_0 instead.
INFO:tensorflow:Summary name dense_1/bias:0 is illegal; using dense_1/bias_0 instead.
2017-06-11 13:44:08,182 INFO Summary name dense_1/bias:0 is illegal; using dense_1/bias_0 instead.
INFO:tensorflow:Summary name dense_2/kernel:0 is illegal; using dense_2/kernel_0 instead.
2017-06-11 13:44:08,186 INFO Summary name dense_2/kernel:0 is illegal; using dense_2/kernel_0 instead.
INFO:tensorflow:Summary name dense_2/kernel:0 is illegal; using dense_2/kernel_0 instead.
2017-06-11 13:44:08,192 INFO Summary name dense_2/kernel:0 is illegal; using dense_2/kernel_0 instead.
INFO:tensorflow:Summary name dense_2/bias:0 is illegal; using dense_2/bias_0 instead.
2017-06-11 13:44:08,195 INFO Summary name dense_2/bias:0 is illegal; using dense_2/bias_0 instead.
INFO:tensorflow:Summary name dense_2/bias:0 is illegal; using dense_2/bias_0 instead.
2017-06-11 13:44:08,202 INFO Summary name dense_2/bias:0 is illegal; using dense_2/bias_0 instead.
INFO:tensorflow:Summary name dense_3/kernel:0 is illegal; using dense_3/kernel_0 instead.
2017-06-11 13:44:08,207 INFO Summary name dense_3/kernel:0 is illegal; using dense_3/kernel_0 instead.
INFO:tensorflow:Summary name dense_3/kernel:0 is illegal; using dense_3/kernel_0 instead.
2017-06-11 13:44:08,215 INFO Summary name dense_3/kernel:0 is illegal; using dense_3/kernel_0 instead.
INFO:tensorflow:Summary name dense_3/bias:0 is illegal; using dense_3/bias_0 instead.
2017-06-11 13:44:08,219 INFO Summary name dense_3/bias:0 is illegal; using dense_3/bias_0 instead.
INFO:tensorflow:Summary name dense_3/bias:0 is illegal; using dense_3/bias_0 instead.
2017-06-11 13:44:08,225 INFO Summary name dense_3/bias:0 is illegal; using dense_3/bias_0 instead.
/usr/local/lib/python3.4/dist-packages/ipykernel/__main__.py:9: UserWarning: Update your `fit_generator` call to the Keras 2 API: `fit_generator(<generator..., validation_steps=2, validation_data=<generator..., verbose=1, steps_per_epoch=8, callbacks=[<keras.ca..., epochs=10)`
2017-06-11 13:44:08,835 INFO starting new flow...
Epoch 1/10
Epoch 1/10
7/8 [=========================>....] - ETA: 1s - loss: 1.9127 - acc: 0.1280 
2017-06-11 13:44:24,734 INFO starting new flow...
Epoch 00000: val_acc improved from -inf to 0.10417, saving model to ../../output/kaggle-sea-lion/05/weights/weights-00-0.10.h5
8/8 [==============================] - 19s - loss: 1.8962 - acc: 0.1354 - val_loss: 1.8101 - val_acc: 0.1042
8/8 [==============================] - 19s - loss: 1.8962 - acc: 0.1354 - val_loss: 1.8101 - val_acc: 0.1042
Epoch 2/10
Epoch 2/10
7/8 [=========================>....] - ETA: 2s - loss: 1.7758 - acc: 0.1905 Epoch 00001: val_acc improved from 0.10417 to 0.13542, saving model to ../../output/kaggle-sea-lion/05/weights/weights-01-0.14.h5
8/8 [==============================] - 19s - loss: 1.7797 - acc: 0.1823 - val_loss: 1.9446 - val_acc: 0.1354
8/8 [==============================] - 20s - loss: 1.7797 - acc: 0.1823 - val_loss: 1.9446 - val_acc: 0.1354
Epoch 3/10
Epoch 3/10
7/8 [=========================>....] - ETA: 2s - loss: 1.6673 - acc: 0.3839 Epoch 00002: val_acc improved from 0.13542 to 0.17708, saving model to ../../output/kaggle-sea-lion/05/weights/weights-02-0.18.h5
8/8 [==============================] - 18s - loss: 1.6870 - acc: 0.3620 - val_loss: 1.9915 - val_acc: 0.1771
8/8 [==============================] - 19s - loss: 1.6870 - acc: 0.3620 - val_loss: 1.9915 - val_acc: 0.1771
Epoch 4/10
Epoch 4/10
7/8 [=========================>....] - ETA: 2s - loss: 1.6388 - acc: 0.3333 Epoch 00003: val_acc improved from 0.17708 to 0.31250, saving model to ../../output/kaggle-sea-lion/05/weights/weights-03-0.31.h5
8/8 [==============================] - 19s - loss: 1.5943 - acc: 0.3646 - val_loss: 1.7443 - val_acc: 0.3125
8/8 [==============================] - 19s - loss: 1.5943 - acc: 0.3646 - val_loss: 1.7443 - val_acc: 0.3125
Epoch 5/10
Epoch 5/10
7/8 [=========================>....] - ETA: 2s - loss: 1.4959 - acc: 0.5060 Epoch 00004: val_acc did not improve
8/8 [==============================] - 17s - loss: 1.4751 - acc: 0.4948 - val_loss: 1.6428 - val_acc: 0.3021
8/8 [==============================] - 17s - loss: 1.4751 - acc: 0.4948 - val_loss: 1.6428 - val_acc: 0.3021
Epoch 6/10
Epoch 6/10
7/8 [=========================>....] - ETA: 1s - loss: 1.2654 - acc: 0.4762 Epoch 00005: val_acc did not improve
8/8 [==============================] - 17s - loss: 1.2408 - acc: 0.4870 - val_loss: 1.7121 - val_acc: 0.2917
8/8 [==============================] - 17s - loss: 1.2408 - acc: 0.4870 - val_loss: 1.7121 - val_acc: 0.2917
Epoch 7/10
Epoch 7/10
7/8 [=========================>....] - ETA: 1s - loss: 1.2492 - acc: 0.5030 Epoch 00006: val_acc improved from 0.31250 to 0.33333, saving model to ../../output/kaggle-sea-lion/05/weights/weights-06-0.33.h5
8/8 [==============================] - 18s - loss: 1.2051 - acc: 0.5339 - val_loss: 1.5690 - val_acc: 0.3333
8/8 [==============================] - 18s - loss: 1.2051 - acc: 0.5339 - val_loss: 1.5690 - val_acc: 0.3333
Epoch 8/10
Epoch 8/10
7/8 [=========================>....] - ETA: 1s - loss: 1.0808 - acc: 0.5327 Epoch 00007: val_acc improved from 0.33333 to 0.34375, saving model to ../../output/kaggle-sea-lion/05/weights/weights-07-0.34.h5
8/8 [==============================] - 19s - loss: 1.0783 - acc: 0.5417 - val_loss: 1.7100 - val_acc: 0.3438
8/8 [==============================] - 19s - loss: 1.0783 - acc: 0.5417 - val_loss: 1.7100 - val_acc: 0.3438
Epoch 9/10
Epoch 9/10
7/8 [=========================>....] - ETA: 1s - loss: 1.0803 - acc: 0.5774 Epoch 00008: val_acc did not improve
8/8 [==============================] - 17s - loss: 1.0473 - acc: 0.5859 - val_loss: 1.5907 - val_acc: 0.2917
8/8 [==============================] - 17s - loss: 1.0473 - acc: 0.5859 - val_loss: 1.5907 - val_acc: 0.2917
Epoch 10/10
Epoch 10/10
7/8 [=========================>....] - ETA: 1s - loss: 1.1082 - acc: 0.5774 Epoch 00009: val_acc improved from 0.34375 to 0.36458, saving model to ../../output/kaggle-sea-lion/05/weights/weights-09-0.36.h5
8/8 [==============================] - 18s - loss: 1.0650 - acc: 0.5911 - val_loss: 1.4418 - val_acc: 0.3646
8/8 [==============================] - 19s - loss: 1.0650 - acc: 0.5911 - val_loss: 1.4418 - val_acc: 0.3646
2017-06-11 13:47:16,794 INFO Saved last model to ../../output/kaggle-sea-lion/05/last-model-simple-84-84.yml

2017-06-11 13:47:17,305 INFO Saved last weights to ../../output/kaggle-sea-lion/05/last-weights-simple-84-84.h5

Epoch accuracy/loss


In [8]:
if(RUN_TRAINING):
    logger.info('Training info')
    cnn.show_training_info_keras(history)


2017-06-11 13:47:17,317 INFO Training info

Confusion matrix


In [9]:
cnn.evaluate_dataset_keras(test_generator.flow(), 
                       test_generator.nr_batches, 
                       test_generator.size, 
                       model, 
                       class_labels=lions.CLASS_LABELS)


2017-06-11 13:47:17,606 INFO Evaluating model performance (80 samples)...
2017-06-11 13:47:17,610 INFO starting new flow...
2017-06-11 13:47:18,988 INFO Accuracy: 0.229166664183 - Loss: 1.655192554
2017-06-11 13:47:18,992 INFO Predicting Y for detailed analysis...
/notebooks/datascience-snippets/kaggle-sea-lion/modules/cnn.py:59: FutureWarning: comparison to `None` will result in an elementwise object comparison in the future.
  if(self.y_ds==None):
2017-06-11 13:47:20,975 INFO Accuracy: 0.25
2017-06-11 13:47:20,978 INFO Number of test samples: 80
2017-06-11 13:47:20,982 INFO Kappa score: 0.154035953472 (-1 bad; 0 just luck; 1 great)
/usr/local/lib/python3.4/dist-packages/sklearn/metrics/classification.py:1113: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
/usr/local/lib/python3.4/dist-packages/sklearn/metrics/classification.py:1115: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.
  'recall', 'true', average, warn_for)
2017-06-11 13:47:20,986 INFO 
                 precision    recall  f1-score   support

   0-adult_male       0.12      0.83      0.22         6
1-subadult_male       0.00      0.00      0.00        25
 2-adult_female       0.22      0.55      0.32        11
     3-juvenile       0.33      0.04      0.08        23
          4-pup       0.00      0.00      0.00         0
     5-non lion       1.00      0.53      0.70        15

    avg / total       0.32      0.25      0.21        80

/notebooks/datascience-snippets/kaggle-sea-lion/modules/utils.py:828: RuntimeWarning: invalid value encountered in true_divide
  acc_class = cm.diagonal()/np.sum(cm, axis=0)
2017-06-11 13:47:20,992 INFO Accuracy per class:
2017-06-11 13:47:20,993 INFO 0-adult_male: 12.5%
2017-06-11 13:47:20,995 INFO 1-subadult_male: nan%
2017-06-11 13:47:20,996 INFO 2-adult_female: 22.2%
2017-06-11 13:47:20,997 INFO 3-juvenile: 33.3%
2017-06-11 13:47:20,999 INFO 4-pup: 0.0%
2017-06-11 13:47:21,000 INFO 5-non lion: 100.0%
2017-06-11 13:47:21,002 INFO Confusion matrix:
2017-06-11 13:47:21,004 INFO 
[[ 5  0  0  1  0  0]
 [20  0  5  0  0  0]
 [ 5  0  6  0  0  0]
 [ 6  0 16  1  0  0]
 [ 0  0  0  0  0  0]
 [ 4  0  0  1  2  8]]
/notebooks/datascience-snippets/kaggle-sea-lion/modules/utils.py:678: RuntimeWarning: invalid value encountered in true_divide
  cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]

In [10]:
if(DEBUG):
    a = test_generator.flow()
    cnn.show_predictions(a, 50, model)

In [ ]:


In [ ]:


In [ ]:


In [ ]:


In [ ]:


In [ ]: