exact repeat of Classifier in keras1 while Classifier_rep4 in keras2 weights.084-0.0739.hdf5 15616/15616 [==============================] - 423s - loss: 0.1111 - acc: 0.9880 - val_loss: 0.0739 - val_acc: 0.9839 train loss: 0.03436334160963875 train_woNoF loss: 0.020864595229428972 crop_class ALB 0.021373 BET 0.019553 DOL 0.000657 LAG 0.000273 OTHER 0.014748 SHARK 0.000621 YFT 0.032967 Name: logloss_woNoF, dtype: float64 valid loss: 0.09322671996697018 valid_woNoF loss: 0.16453690212776534 crop_class ALB 0.110811 BET 0.327438 DOL 0.523910 LAG 0.004440 OTHER 0.255691 SHARK 0.002541 YFT 0.236282 Name: logloss_woNoF, dtype: float64 RFCN_AGONOSTICnms_resnet101_rfcn_ohem_iter_30000_resnet50_FT38_Classifier_Rep5_weights.084-0.0739.hdf5_clsMaxAve_conf0.80_croploss0.1645_imageloss0.0646_T1.csv

In [1]:
import os, random, glob, pickle, collections, math, json, time
import numpy as np
import pandas as pd
from __future__ import division
from __future__ import print_function
from PIL import Image
from sklearn.model_selection import train_test_split
from sklearn.metrics import log_loss
from sklearn.preprocessing import LabelEncoder

import matplotlib.pyplot as plt
%matplotlib inline 

from keras.models import Sequential, Model, load_model
from keras.layers import GlobalAveragePooling2D, Flatten, Dropout, Dense, LeakyReLU
from keras.optimizers import Adam
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau, TensorBoard
from keras.preprocessing.image import ImageDataGenerator
from keras.utils import np_utils
from keras.preprocessing import image
from keras import backend as K
K.set_image_dim_ordering('tf')


Using TensorFlow backend.

In [2]:
TRAIN_DIR = '../data/train/'
TEST_DIR = '../RFCN/JPEGImages/'
TRAIN_CROP_DIR = '../data/train_crop/'
TEST_CROP_DIR = '../data/test_stg1_crop/'
RFCN_MODEL = 'resnet101_rfcn_ohem_iter_30000'
CROP_MODEL = 'resnet50_FT38_Classifier_Rep5'
if not os.path.exists('./' + CROP_MODEL):
    os.mkdir('./' + CROP_MODEL)
CHECKPOINT_DIR = './' + CROP_MODEL + '/checkpoint/'
if not os.path.exists(CHECKPOINT_DIR):
    os.mkdir(CHECKPOINT_DIR)
LOG_DIR = './' + CROP_MODEL + '/log/'
if not os.path.exists(LOG_DIR):
    os.mkdir(LOG_DIR)
OUTPUT_DIR = './' + CROP_MODEL + '/output/'
if not os.path.exists(OUTPUT_DIR):
    os.mkdir(OUTPUT_DIR)
FISH_CLASSES = ['NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']
# CROP_CLASSES=FISH_CLASSES[:]
# CROP_CLASSES.remove('NoF')
CONF_THRESH = 0.8
ROWS = 224
COLS = 224
BATCHSIZE = 128
LEARNINGRATE = 1e-4
def imagewise_center(x):
    mean = np.mean(x, axis=0, keepdims=True)
    x_centered = x - mean
    return x_centered

def channelwise_center(x):
    mean = np.mean(x, axis=0, keepdims=True)
    mean = np.mean(mean, axis=(1,2), keepdims=True)
    x_centered = x - mean
    return x_centered    

def imagewise_mean(x):
    mean = np.mean(x, axis=0)
    return mean

def channelwise_mean(x):
    mean = np.mean(x, axis=0)
    mean = np.mean(mean, axis=(0,1))
    return mean

def preprocess_imagewise(x, imagewise_mean):
    #resnet50 image preprocessing
#     'RGB'->'BGR'
#     x = x[:, :, ::-1]
#     x /= 255.
    x -= imagewise_mean
    return x

def preprocess_channelwise(x, channelwise_mean):
    #resnet50 image preprocessing
#     'RGB'->'BGR'
#     x = x[:, :, ::-1]
#     x /= 255.
    x -= np.reshape(channelwise_mean, [1, 1, 3])
    return x

def load_img(path, bbox, target_size=None):
    img = Image.open(path)
#     img = img.convert('RGB')
    cropped = img.crop((bbox[0],bbox[1],bbox[2],bbox[3]))
    width_cropped, height_cropped = cropped.size
    if height_cropped > width_cropped: cropped = cropped.transpose(method=2)
    image_name = 'temp_{:f}.jpg'.format(time.time())
    cropped.save(image_name)
    cropped = Image.open(image_name)   
    if target_size:
        cropped = cropped.resize((target_size[1], target_size[0]), Image.BILINEAR)
    os.remove(image_name)
    return cropped

def get_best_model(checkpoint_dir = CHECKPOINT_DIR):
    files = glob.glob(checkpoint_dir+'*')
    val_losses = [float(f.split('-')[-1][:-5]) for f in files]
    index = val_losses.index(min(val_losses))
    print('Loading model from checkpoint file ' + files[index])
    model = load_model(files[index])
    model_name = files[index].split('/')[-1]
    print('Loading model Done!')
    return (model, model_name)

def data_from_df(df):
    X = np.ndarray((df.shape[0], ROWS, COLS, 3), dtype=np.uint8)
    y = np.zeros((df.shape[0], len(FISH_CLASSES)), dtype=K.floatx())
    i = 0
    for index,row in df.iterrows():
        image_file = row['image_file']
        fish = row['crop_class']
        bbox = [row['xmin'],row['ymin'],row['xmax'],row['ymax']]
        cropped = load_img(TEST_DIR+image_file,bbox,target_size=(ROWS,COLS))
        X[i] = np.asarray(cropped)
        y[i,FISH_CLASSES.index(fish)] = 1
        i += 1
    return (X, y)

def data_load(name):
    file_name = 'data_'+name+'_{}_{}.pickle'.format(ROWS, COLS)
    if os.path.exists(OUTPUT_DIR+file_name):
        print ('Loading from file '+file_name)
        with open(OUTPUT_DIR+file_name, 'rb') as f:
            data = pickle.load(f)
        X = data['X']
        y = data['y']
    else:
        print ('Generating file '+file_name)
        
        if name=='train' or name=='valid': 
            df = GTbbox_df[GTbbox_df['split']==name]
        elif name=='all':
            df = GTbbox_df
        else:
            print('Invalid name '+name)
    
        X, y = data_from_df(df)

        data = {'X': X,'y': y}
        with open(OUTPUT_DIR+file_name, 'wb') as f:
            pickle.dump(data, f)
    return (X, y)

In [3]:
# GTbbox_df = ['image_file','crop_index','crop_class','xmin',''ymin','xmax','ymax','split']

file_name = 'GTbbox_df.pickle'
if os.path.exists(OUTPUT_DIR+file_name):
    print ('Loading from file '+file_name)
    GTbbox_df = pd.read_pickle(OUTPUT_DIR+file_name)
else:
    print ('Generating file '+file_name)       
    GTbbox_df = pd.DataFrame(columns=['image_file','crop_index','crop_class','xmin','ymin','xmax','ymax'])  

    crop_classes=FISH_CLASSES[:]
    crop_classes.remove('NoF')
    
    for c in crop_classes:
        print(c)
        j = json.load(open('../data/BBannotations/{}.json'.format(c), 'r'))
        for l in j: 
            filename = l["filename"]
            head, image_file = os.path.split(filename)
            basename, file_extension = os.path.splitext(image_file) 
            image = Image.open(TEST_DIR+'/'+image_file)
            width_image, height_image = image.size
            for i in range(len(l["annotations"])):
                a = l["annotations"][i]
                xmin = (a["x"])
                ymin = (a["y"])
                width = (a["width"])
                height = (a["height"])
                xmax = xmin + width
                ymax = ymin + height
                assert max(xmin,0)<min(xmax,width_image)
                assert max(ymin,0)<min(ymax,height_image)
                GTbbox_df.loc[len(GTbbox_df)]=[image_file,i,a["class"],max(xmin,0),max(ymin,0),min(xmax,width_image),min(ymax,height_image)]
                if a["class"] != c: print(GTbbox_df.tail(1))  
        
        
    num_NoF = GTbbox_df.shape[0]*3
    RFCN_MODEL = 'resnet101_rfcn_ohem_iter_30000'
    with open('../data/RFCN_detections/detections_full_AGNOSTICnms_'+RFCN_MODEL+'.pkl','rb') as f:
        detections_full_AGNOSTICnms = pickle.load(f, encoding='latin1') 
    train_detections_full_AGNOSTICnms = detections_full_AGNOSTICnms[1000:]
#     with open("../RFCN/ImageSets/Main/test.txt","r") as f:
#         test_files = f.readlines()
#     train_files = test_files[1000:]
    with open("../RFCN/ImageSets/Main/train_test.txt","r") as f:
        train_files = f.readlines()
    num_NoF_perIm = int(math.ceil(num_NoF / len(train_detections_full_AGNOSTICnms)))

    for im in range(len(train_files)):
        image_file = train_files[im][:9]+'.jpg'
        image = Image.open(TEST_DIR+image_file)
#         width_image, height_image = image.size
        detects_im = train_detections_full_AGNOSTICnms[im]
        detects_im = detects_im[np.where(detects_im[:,4] >= 0.999)]
        np.random.seed(1986)
        bboxes = detects_im[np.random.choice(detects_im.shape[0], num_NoF_perIm, replace=False), :]
        for j in range(bboxes.shape[0]):    
            bbox = bboxes[j]
            xmin = bbox[0]
            ymin = bbox[1]
            xmax = bbox[2]
            ymax = bbox[3]
#             assert max(xmin,0)<min(xmax,width_image)
#             assert max(ymin,0)<min(ymax,height_image)
            GTbbox_df.loc[len(GTbbox_df)]=[image_file,j,'NoF',xmin,ymin,xmax,ymax]

    test_size = GTbbox_df.shape[0]-int(math.ceil(GTbbox_df.shape[0]*0.8/128)*128)
    train_ind, valid_ind = train_test_split(range(GTbbox_df.shape[0]), test_size=test_size, random_state=1986, stratify=GTbbox_df['crop_class'])
    GTbbox_df['split'] = ['train' if i in train_ind else 'valid' for i in range(GTbbox_df.shape[0])]
    GTbbox_df.to_pickle(OUTPUT_DIR+file_name)


Loading from file GTbbox_df.pickle

In [4]:
#Load data
X_train, y_train = data_load('train')
X_valid, y_valid = data_load('valid')
       
print('Loading data done.')
print('train sample', X_train.shape[0])
print('valid sample', X_valid.shape[0])
X_train = X_train.astype(np.float32)
X_valid = X_valid.astype(np.float32)
print('Convert to float32 done.')
X_train /= 255.
X_valid /= 255.
print('Rescale by 255 done.')
print('mean of X_train is', mean(X_train))
print('mean of X_valid is', mean(X_valid))
# X_train_centered = featurewise_center(X_train)
# X_valid_centered = featurewise_center(X_valid)
# print('Featurewise centered done.')


Loading from file data_train_224_224.pickle
Loading from file data_valid_224_224.pickle
Loading data done.
train sample 15616
valid sample 3863
Convert to float32 done.
Rescale by 255 done.
mean of X_train is [ 0.37469822  0.41190618  0.38363659]
mean of X_valid is [ 0.37204078  0.41050324  0.38143244]

In [5]:
# #class weight = n_samples / (n_classes * np.bincount(y))
# class_weight_fish = dict(GTbbox_df.groupby('crop_class').size())
# class_weight = {}
# n_samples = GTbbox_df.shape[0]
# for key,value in class_weight_fish.items():
#         class_weight[CROP_CLASSES.index(key)] = n_samples / (len(CROP_CLASSES)*value)
# class_weight

class_weight_fish = dict(GTbbox_df.groupby('crop_class').size())
class_weight = {}
ref = max(class_weight_fish.values())
for key,value in class_weight_fish.items():
    class_weight[FISH_CLASSES.index(key)] = ref/value
class_weight


Out[5]:
{0: 1.0,
 1: 6.0119379228014322,
 2: 49.372549019607845,
 3: 119.9047619047619,
 4: 143.88571428571427,
 5: 45.369369369369366,
 6: 79.936507936507937,
 7: 18.908635794743429}

In [6]:
#data augmentation

train_datagen = ImageDataGenerator(
    featurewise_center=True,
    rotation_range=180,
    shear_range=0.2,
    zoom_range=0.1,
    width_shift_range=0.1,
    height_shift_range=0.1,
    horizontal_flip=True,
    vertical_flip=True)
train_datagen.fit(X_train) 
train_generator = train_datagen.flow(X_train, y_train, batch_size=BATCHSIZE, shuffle=True, seed=None)

valid_datagen = ImageDataGenerator(featurewise_center=True)
valid_datagen.fit(X_valid)   
valid_generator = valid_datagen.flow(X_valid, y_valid, batch_size=BATCHSIZE, shuffle=True, seed=None)
# assert X_train_centered.shape[0]%BATCHSIZE==0
# steps_per_epoch = int(X_train_centered.shape[0]/BATCHSIZE)

In [7]:
#callbacks

early_stopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=10, verbose=1, mode='auto')        

model_checkpoint = ModelCheckpoint(filepath=CHECKPOINT_DIR+'weights.{epoch:03d}-{val_loss:.4f}.hdf5', monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=False, mode='auto')
        
learningrate_schedule = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=5, verbose=1, mode='auto', epsilon=0.001, cooldown=0, min_lr=0)

tensorboard = TensorBoard(log_dir=LOG_DIR, histogram_freq=0, write_graph=False, write_images=True)

In [9]:
#Resnet50
#top layer training

from keras.applications.resnet50 import ResNet50

base_model = ResNet50(weights='imagenet', include_top=False)
x = base_model.output
x = GlobalAveragePooling2D()(x)
# x = Flatten()(x)
# x = Dense(256)(x)
# x = LeakyReLU(alpha=0.33)(x)
# x = Dropout(0.5)(x)
predictions = Dense(len(FISH_CLASSES), init='glorot_normal', activation='softmax')(x)

model = Model(input=base_model.input, output=predictions)

# first: train only the top layers (which were randomly initialized)
for layer in base_model.layers:
    layer.trainable = False

# compile the model (should be done *after* setting layers to non-trainable)
optimizer = Adam(lr=LEARNINGRATE)
model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])

# train the model on the new data for a few epochs
model.fit_generator(train_generator, samples_per_epoch=len(X_train), nb_epoch=30, verbose=1, 
                    callbacks=[early_stopping, model_checkpoint, learningrate_schedule, tensorboard], 
                    validation_data=valid_generator, nb_val_samples=len(X_valid), 
                    class_weight=class_weight, nb_worker=3, pickle_safe=True)


Epoch 1/30
15488/15616 [============================>.] - ETA: 1s - loss: 13.4195 - acc: 0.3020Epoch 00000: val_loss improved from inf to 1.22347, saving model to ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.000-1.2235.hdf5
15616/15616 [==============================] - 256s - loss: 13.4529 - acc: 0.3039 - val_loss: 1.2235 - val_acc: 0.7762
Epoch 2/30
15488/15616 [============================>.] - ETA: 1s - loss: 10.2357 - acc: 0.5668Epoch 00001: val_loss improved from 1.22347 to 1.06890, saving model to ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.001-1.0689.hdf5
15616/15616 [==============================] - 244s - loss: 10.2616 - acc: 0.5674 - val_loss: 1.0689 - val_acc: 0.7681
Epoch 3/30
15488/15616 [============================>.] - ETA: 1s - loss: 9.1324 - acc: 0.6437Epoch 00002: val_loss did not improve
15616/15616 [==============================] - 244s - loss: 9.1543 - acc: 0.6440 - val_loss: 1.0909 - val_acc: 0.7752
Epoch 4/30
15488/15616 [============================>.] - ETA: 1s - loss: 8.2175 - acc: 0.6792Epoch 00003: val_loss did not improve
15616/15616 [==============================] - 245s - loss: 8.2398 - acc: 0.6789 - val_loss: 1.1254 - val_acc: 0.7807
Epoch 5/30
15488/15616 [============================>.] - ETA: 1s - loss: 7.2257 - acc: 0.7045Epoch 00004: val_loss did not improve
15616/15616 [==============================] - 243s - loss: 7.2322 - acc: 0.7050 - val_loss: 1.1603 - val_acc: 0.7742
Epoch 6/30
15488/15616 [============================>.] - ETA: 1s - loss: 7.0307 - acc: 0.7189Epoch 00005: val_loss did not improve
15616/15616 [==============================] - 242s - loss: 7.0415 - acc: 0.7188 - val_loss: 1.0711 - val_acc: 0.7714
Epoch 7/30
15488/15616 [============================>.] - ETA: 1s - loss: 6.4765 - acc: 0.7350Epoch 00006: val_loss improved from 1.06890 to 0.84479, saving model to ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.006-0.8448.hdf5
15616/15616 [==============================] - 244s - loss: 6.4902 - acc: 0.7354 - val_loss: 0.8448 - val_acc: 0.7787
Epoch 8/30
15488/15616 [============================>.] - ETA: 1s - loss: 6.0286 - acc: 0.7470Epoch 00007: val_loss improved from 0.84479 to 0.61040, saving model to ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.007-0.6104.hdf5
15616/15616 [==============================] - 245s - loss: 6.0327 - acc: 0.7476 - val_loss: 0.6104 - val_acc: 0.8122
Epoch 9/30
15488/15616 [============================>.] - ETA: 1s - loss: 6.1144 - acc: 0.7466Epoch 00008: val_loss improved from 0.61040 to 0.53957, saving model to ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.008-0.5396.hdf5
15616/15616 [==============================] - 244s - loss: 6.1359 - acc: 0.7465 - val_loss: 0.5396 - val_acc: 0.8415
Epoch 10/30
15488/15616 [============================>.] - ETA: 1s - loss: 5.7115 - acc: 0.7636Epoch 00009: val_loss did not improve
15616/15616 [==============================] - 243s - loss: 5.7366 - acc: 0.7633 - val_loss: 0.5685 - val_acc: 0.8221
Epoch 11/30
15488/15616 [============================>.] - ETA: 1s - loss: 5.2956 - acc: 0.7648Epoch 00010: val_loss did not improve
15616/15616 [==============================] - 244s - loss: 5.2870 - acc: 0.7650 - val_loss: 0.5537 - val_acc: 0.8206
Epoch 12/30
15488/15616 [============================>.] - ETA: 1s - loss: 5.5585 - acc: 0.7716Epoch 00011: val_loss did not improve
15616/15616 [==============================] - 244s - loss: 5.5708 - acc: 0.7713 - val_loss: 0.6180 - val_acc: 0.7944
Epoch 13/30
15488/15616 [============================>.] - ETA: 1s - loss: 5.2370 - acc: 0.7732Epoch 00012: val_loss did not improve
15616/15616 [==============================] - 244s - loss: 5.2485 - acc: 0.7733 - val_loss: 0.5902 - val_acc: 0.8085
Epoch 14/30
15488/15616 [============================>.] - ETA: 1s - loss: 4.9715 - acc: 0.7845Epoch 00013: val_loss improved from 0.53957 to 0.52944, saving model to ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.013-0.5294.hdf5
15616/15616 [==============================] - 245s - loss: 4.9616 - acc: 0.7849 - val_loss: 0.5294 - val_acc: 0.8191
Epoch 15/30
15488/15616 [============================>.] - ETA: 1s - loss: 5.1189 - acc: 0.7825Epoch 00014: val_loss did not improve
15616/15616 [==============================] - 244s - loss: 5.1414 - acc: 0.7820 - val_loss: 0.5734 - val_acc: 0.8102
Epoch 16/30
15488/15616 [============================>.] - ETA: 1s - loss: 4.8546 - acc: 0.7853Epoch 00015: val_loss did not improve
15616/15616 [==============================] - 244s - loss: 4.8600 - acc: 0.7855 - val_loss: 0.5602 - val_acc: 0.8175
Epoch 17/30
15488/15616 [============================>.] - ETA: 1s - loss: 4.6045 - acc: 0.7914Epoch 00016: val_loss improved from 0.52944 to 0.49176, saving model to ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.016-0.4918.hdf5
15616/15616 [==============================] - 244s - loss: 4.6007 - acc: 0.7919 - val_loss: 0.4918 - val_acc: 0.8352
Epoch 18/30
15488/15616 [============================>.] - ETA: 1s - loss: 4.8776 - acc: 0.7905Epoch 00017: val_loss did not improve
15616/15616 [==============================] - 243s - loss: 4.8841 - acc: 0.7906 - val_loss: 0.5559 - val_acc: 0.8105
Epoch 19/30
15488/15616 [============================>.] - ETA: 1s - loss: 4.5189 - acc: 0.7931Epoch 00018: val_loss did not improve
15616/15616 [==============================] - 245s - loss: 4.5200 - acc: 0.7935 - val_loss: 0.5182 - val_acc: 0.8309
Epoch 20/30
15488/15616 [============================>.] - ETA: 1s - loss: 4.4674 - acc: 0.8011Epoch 00019: val_loss improved from 0.49176 to 0.46069, saving model to ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.019-0.4607.hdf5
15616/15616 [==============================] - 245s - loss: 4.4551 - acc: 0.8015 - val_loss: 0.4607 - val_acc: 0.8478
Epoch 21/30
15488/15616 [============================>.] - ETA: 1s - loss: 4.5700 - acc: 0.8021Epoch 00020: val_loss did not improve
15616/15616 [==============================] - 244s - loss: 4.5818 - acc: 0.8017 - val_loss: 0.5109 - val_acc: 0.8281
Epoch 22/30
15488/15616 [============================>.] - ETA: 1s - loss: 4.3096 - acc: 0.8082Epoch 00021: val_loss did not improve
15616/15616 [==============================] - 243s - loss: 4.3181 - acc: 0.8082 - val_loss: 0.4996 - val_acc: 0.8349
Epoch 23/30
15488/15616 [============================>.] - ETA: 1s - loss: 4.2421 - acc: 0.8081Epoch 00022: val_loss did not improve
15616/15616 [==============================] - 244s - loss: 4.2407 - acc: 0.8087 - val_loss: 0.5015 - val_acc: 0.8369
Epoch 24/30
15488/15616 [============================>.] - ETA: 1s - loss: 4.3800 - acc: 0.8035Epoch 00023: val_loss did not improve
15616/15616 [==============================] - 245s - loss: 4.3904 - acc: 0.8032 - val_loss: 0.5025 - val_acc: 0.8231
Epoch 25/30
15488/15616 [============================>.] - ETA: 1s - loss: 4.2377 - acc: 0.8088Epoch 00024: val_loss improved from 0.46069 to 0.44290, saving model to ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.024-0.4429.hdf5
15616/15616 [==============================] - 243s - loss: 4.2432 - acc: 0.8088 - val_loss: 0.4429 - val_acc: 0.8584
Epoch 26/30
15488/15616 [============================>.] - ETA: 1s - loss: 4.0804 - acc: 0.8122Epoch 00025: val_loss improved from 0.44290 to 0.43199, saving model to ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.025-0.4320.hdf5
15616/15616 [==============================] - 244s - loss: 4.0700 - acc: 0.8126 - val_loss: 0.4320 - val_acc: 0.8692
Epoch 27/30
15488/15616 [============================>.] - ETA: 1s - loss: 4.2407 - acc: 0.8171Epoch 00026: val_loss did not improve
15616/15616 [==============================] - 244s - loss: 4.2456 - acc: 0.8169 - val_loss: 0.4849 - val_acc: 0.8440
Epoch 28/30
15488/15616 [============================>.] - ETA: 1s - loss: 4.0389 - acc: 0.8139Epoch 00027: val_loss did not improve
15616/15616 [==============================] - 245s - loss: 4.0429 - acc: 0.8138 - val_loss: 0.4666 - val_acc: 0.8463
Epoch 29/30
15488/15616 [============================>.] - ETA: 1s - loss: 3.8876 - acc: 0.8235Epoch 00028: val_loss improved from 0.43199 to 0.40779, saving model to ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.028-0.4078.hdf5
15616/15616 [==============================] - 244s - loss: 3.8857 - acc: 0.8239 - val_loss: 0.4078 - val_acc: 0.8727
Epoch 30/30
15488/15616 [============================>.] - ETA: 1s - loss: 4.1514 - acc: 0.8124Epoch 00029: val_loss did not improve
15616/15616 [==============================] - 244s - loss: 4.1600 - acc: 0.8122 - val_loss: 0.4558 - val_acc: 0.8478
Out[9]:
<keras.callbacks.History at 0x7fde5866eb00>

In [ ]:
### Resnet50
# fine tuning
# 164 conv5c+top
# 142 conv5+top
# 80 conv4+conv5+top
# 38 conv3+conv4+conv5+top
start_layer = 38

model, model_name = get_best_model()
# print('Loading model from weights.004-0.0565.hdf5')
# model = load_model(CHECHPOINT_DIR+'weights.004-0.0565.hdf5')

for layer in model.layers[:start_layer]:
   layer.trainable = False
for layer in model.layers[start_layer:]:
   layer.trainable = True

# we need to recompile the model for these modifications to take effect
# we use SGD with a low learning rate
optimizer = Adam(lr=1e-5)
model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])

model.fit_generator(train_generator, samples_per_epoch=len(X_train), nb_epoch=300, verbose=1, 
                    callbacks=[early_stopping, model_checkpoint, learningrate_schedule, tensorboard], 
                    validation_data=valid_generator, nb_val_samples=len(X_valid), 
                    class_weight=class_weight, nb_worker=3, pickle_safe=True, initial_epoch=29)


Loading model from checkpoint file ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.028-0.4078.hdf5
Loading model Done!
Epoch 30/300
15488/15616 [============================>.] - ETA: 3s - loss: 3.4825 - acc: 0.8378Epoch 00029: val_loss improved from inf to 0.37224, saving model to ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.029-0.3722.hdf5
15616/15616 [==============================] - 457s - loss: 3.4662 - acc: 0.8379 - val_loss: 0.3722 - val_acc: 0.8795
Epoch 31/300
15488/15616 [============================>.] - ETA: 3s - loss: 2.9071 - acc: 0.8583Epoch 00030: val_loss improved from 0.37224 to 0.30681, saving model to ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.030-0.3068.hdf5
15616/15616 [==============================] - 436s - loss: 2.9191 - acc: 0.8584 - val_loss: 0.3068 - val_acc: 0.9037
Epoch 32/300
15488/15616 [============================>.] - ETA: 3s - loss: 2.6294 - acc: 0.8740Epoch 00031: val_loss did not improve
15616/15616 [==============================] - 435s - loss: 2.6321 - acc: 0.8740 - val_loss: 0.3109 - val_acc: 0.8969
Epoch 33/300
15488/15616 [============================>.] - ETA: 3s - loss: 2.0985 - acc: 0.8920Epoch 00032: val_loss improved from 0.30681 to 0.27225, saving model to ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.032-0.2723.hdf5
15616/15616 [==============================] - 435s - loss: 2.0904 - acc: 0.8919 - val_loss: 0.2723 - val_acc: 0.9123
Epoch 34/300
15488/15616 [============================>.] - ETA: 3s - loss: 1.9942 - acc: 0.8986Epoch 00033: val_loss improved from 0.27225 to 0.24721, saving model to ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.033-0.2472.hdf5
15616/15616 [==============================] - 435s - loss: 1.9995 - acc: 0.8987 - val_loss: 0.2472 - val_acc: 0.9216
Epoch 35/300
15488/15616 [============================>.] - ETA: 3s - loss: 1.7864 - acc: 0.9101Epoch 00034: val_loss did not improve
15616/15616 [==============================] - 435s - loss: 1.7906 - acc: 0.9097 - val_loss: 0.2522 - val_acc: 0.9199
Epoch 36/300
15488/15616 [============================>.] - ETA: 3s - loss: 1.5109 - acc: 0.9203Epoch 00035: val_loss improved from 0.24721 to 0.23140, saving model to ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.035-0.2314.hdf5
15616/15616 [==============================] - 435s - loss: 1.5058 - acc: 0.9203 - val_loss: 0.2314 - val_acc: 0.9287
Epoch 37/300
15488/15616 [============================>.] - ETA: 3s - loss: 1.4573 - acc: 0.9245Epoch 00036: val_loss improved from 0.23140 to 0.21609, saving model to ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.036-0.2161.hdf5
15616/15616 [==============================] - 434s - loss: 1.4592 - acc: 0.9246 - val_loss: 0.2161 - val_acc: 0.9443
Epoch 38/300
 3456/15616 [=====>........................] - ETA: 299s - loss: 1.2652 - acc: 0.9404

In [8]:
#resume training

model, model_name = get_best_model()
# print('Loading model from weights.004-0.0565.hdf5')
# model = load_model(CHECHPOINT_DIR+'weights.004-0.0565.hdf5')

# #try increasing learningrate
# optimizer = Adam(lr=1e-4)
# model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])

model.fit_generator(train_generator, samples_per_epoch=len(X_train), nb_epoch=300, verbose=1, 
                    callbacks=[early_stopping, model_checkpoint, learningrate_schedule, tensorboard], 
                    validation_data=valid_generator, nb_val_samples=len(X_valid), 
                    class_weight=class_weight, nb_worker=3, pickle_safe=True, initial_epoch=46)


Loading model from checkpoint file ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.045-0.1470.hdf5
Loading model Done!
Epoch 47/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.5758 - acc: 0.9640Epoch 00046: val_loss improved from inf to 0.15212, saving model to ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.046-0.1521.hdf5
15616/15616 [==============================] - 449s - loss: 0.5742 - acc: 0.9639 - val_loss: 0.1521 - val_acc: 0.9544
Epoch 48/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.5925 - acc: 0.9622Epoch 00047: val_loss improved from 0.15212 to 0.14692, saving model to ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.047-0.1469.hdf5
15616/15616 [==============================] - 425s - loss: 0.5904 - acc: 0.9623 - val_loss: 0.1469 - val_acc: 0.9592
Epoch 49/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.5258 - acc: 0.9645Epoch 00048: val_loss improved from 0.14692 to 0.13391, saving model to ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.048-0.1339.hdf5
15616/15616 [==============================] - 424s - loss: 0.5257 - acc: 0.9646 - val_loss: 0.1339 - val_acc: 0.9627
Epoch 50/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.4133 - acc: 0.9702Epoch 00049: val_loss improved from 0.13391 to 0.13323, saving model to ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.049-0.1332.hdf5
15616/15616 [==============================] - 426s - loss: 0.4136 - acc: 0.9702 - val_loss: 0.1332 - val_acc: 0.9680
Epoch 51/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.4320 - acc: 0.9703Epoch 00050: val_loss improved from 0.13323 to 0.11295, saving model to ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.050-0.1129.hdf5
15616/15616 [==============================] - 425s - loss: 0.4303 - acc: 0.9705 - val_loss: 0.1129 - val_acc: 0.9685
Epoch 52/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.4226 - acc: 0.9681Epoch 00051: val_loss did not improve
15616/15616 [==============================] - 426s - loss: 0.4220 - acc: 0.9683 - val_loss: 0.1298 - val_acc: 0.9698
Epoch 53/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.3730 - acc: 0.9736Epoch 00052: val_loss did not improve
15616/15616 [==============================] - 425s - loss: 0.3723 - acc: 0.9736 - val_loss: 0.1178 - val_acc: 0.9728
Epoch 54/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.3812 - acc: 0.9725Epoch 00053: val_loss improved from 0.11295 to 0.11036, saving model to ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.053-0.1104.hdf5
15616/15616 [==============================] - 427s - loss: 0.3794 - acc: 0.9726 - val_loss: 0.1104 - val_acc: 0.9720
Epoch 55/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.3422 - acc: 0.9727Epoch 00054: val_loss did not improve
15616/15616 [==============================] - 424s - loss: 0.3416 - acc: 0.9728 - val_loss: 0.1104 - val_acc: 0.9655
Epoch 56/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.2666 - acc: 0.9780Epoch 00055: val_loss did not improve
15616/15616 [==============================] - 425s - loss: 0.2662 - acc: 0.9779 - val_loss: 0.1334 - val_acc: 0.9637
Epoch 57/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.2859 - acc: 0.9777Epoch 00056: val_loss did not improve
15616/15616 [==============================] - 425s - loss: 0.2849 - acc: 0.9778 - val_loss: 0.1140 - val_acc: 0.9700
Epoch 58/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.2557 - acc: 0.9769Epoch 00057: val_loss improved from 0.11036 to 0.09976, saving model to ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.057-0.0998.hdf5
15616/15616 [==============================] - 425s - loss: 0.2547 - acc: 0.9770 - val_loss: 0.0998 - val_acc: 0.9715
Epoch 59/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.2138 - acc: 0.9809Epoch 00058: val_loss did not improve
15616/15616 [==============================] - 423s - loss: 0.2127 - acc: 0.9809 - val_loss: 0.1004 - val_acc: 0.9751
Epoch 60/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.2387 - acc: 0.9810Epoch 00059: val_loss did not improve
15616/15616 [==============================] - 424s - loss: 0.2378 - acc: 0.9810 - val_loss: 0.1127 - val_acc: 0.9740
Epoch 61/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.2378 - acc: 0.9813Epoch 00060: val_loss improved from 0.09976 to 0.09788, saving model to ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.060-0.0979.hdf5
15616/15616 [==============================] - 426s - loss: 0.2376 - acc: 0.9814 - val_loss: 0.0979 - val_acc: 0.9753
Epoch 62/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.2059 - acc: 0.9817Epoch 00061: val_loss improved from 0.09788 to 0.09110, saving model to ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.061-0.0911.hdf5
15616/15616 [==============================] - 424s - loss: 0.2048 - acc: 0.9819 - val_loss: 0.0911 - val_acc: 0.9733
Epoch 63/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.2071 - acc: 0.9837Epoch 00062: val_loss did not improve
15616/15616 [==============================] - 425s - loss: 0.2075 - acc: 0.9837 - val_loss: 0.1077 - val_acc: 0.9723
Epoch 64/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1838 - acc: 0.9825Epoch 00063: val_loss did not improve
15616/15616 [==============================] - 423s - loss: 0.1838 - acc: 0.9825 - val_loss: 0.0961 - val_acc: 0.9735
Epoch 65/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1825 - acc: 0.9839Epoch 00064: val_loss did not improve
15616/15616 [==============================] - 426s - loss: 0.1821 - acc: 0.9839 - val_loss: 0.1118 - val_acc: 0.9743
Epoch 66/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1553 - acc: 0.9836Epoch 00065: val_loss did not improve
15616/15616 [==============================] - 424s - loss: 0.1545 - acc: 0.9837 - val_loss: 0.1118 - val_acc: 0.9730
Epoch 67/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1514 - acc: 0.9853Epoch 00066: val_loss did not improve
15616/15616 [==============================] - 424s - loss: 0.1508 - acc: 0.9855 - val_loss: 0.0985 - val_acc: 0.9771
Epoch 68/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1360 - acc: 0.9886Epoch 00067: val_loss did not improve

Epoch 00067: reducing learning rate to 9.999999747378752e-07.
15616/15616 [==============================] - 425s - loss: 0.1356 - acc: 0.9885 - val_loss: 0.1074 - val_acc: 0.9763
Epoch 69/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1447 - acc: 0.9872Epoch 00068: val_loss improved from 0.09110 to 0.08343, saving model to ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.068-0.0834.hdf5
15616/15616 [==============================] - 424s - loss: 0.1440 - acc: 0.9873 - val_loss: 0.0834 - val_acc: 0.9776
Epoch 70/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1341 - acc: 0.9867Epoch 00069: val_loss did not improve
15616/15616 [==============================] - 424s - loss: 0.1339 - acc: 0.9866 - val_loss: 0.1024 - val_acc: 0.9768
Epoch 71/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1245 - acc: 0.9874Epoch 00070: val_loss did not improve
15616/15616 [==============================] - 422s - loss: 0.1250 - acc: 0.9873 - val_loss: 0.1029 - val_acc: 0.9753
Epoch 72/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1371 - acc: 0.9873Epoch 00071: val_loss did not improve
15616/15616 [==============================] - 424s - loss: 0.1365 - acc: 0.9874 - val_loss: 0.1075 - val_acc: 0.9756
Epoch 73/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1202 - acc: 0.9866Epoch 00072: val_loss did not improve
15616/15616 [==============================] - 423s - loss: 0.1199 - acc: 0.9867 - val_loss: 0.0965 - val_acc: 0.9771
Epoch 74/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1014 - acc: 0.9875Epoch 00073: val_loss improved from 0.08343 to 0.07913, saving model to ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.073-0.0791.hdf5
15616/15616 [==============================] - 425s - loss: 0.1022 - acc: 0.9874 - val_loss: 0.0791 - val_acc: 0.9816
Epoch 75/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1145 - acc: 0.9887Epoch 00074: val_loss did not improve
15616/15616 [==============================] - 423s - loss: 0.1145 - acc: 0.9887 - val_loss: 0.1014 - val_acc: 0.9761
Epoch 76/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1286 - acc: 0.9871Epoch 00075: val_loss did not improve
15616/15616 [==============================] - 422s - loss: 0.1282 - acc: 0.9872 - val_loss: 0.0979 - val_acc: 0.9773
Epoch 77/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1041 - acc: 0.9883Epoch 00076: val_loss did not improve
15616/15616 [==============================] - 423s - loss: 0.1043 - acc: 0.9882 - val_loss: 0.0990 - val_acc: 0.9791
Epoch 78/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1171 - acc: 0.9868Epoch 00077: val_loss did not improve
15616/15616 [==============================] - 423s - loss: 0.1171 - acc: 0.9868 - val_loss: 0.1018 - val_acc: 0.9796
Epoch 79/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1022 - acc: 0.9881Epoch 00078: val_loss did not improve
15616/15616 [==============================] - 424s - loss: 0.1024 - acc: 0.9882 - val_loss: 0.0820 - val_acc: 0.9819
Epoch 80/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1094 - acc: 0.9890Epoch 00079: val_loss did not improve

Epoch 00079: reducing learning rate to 9.999999974752428e-08.
15616/15616 [==============================] - 423s - loss: 0.1088 - acc: 0.9890 - val_loss: 0.1018 - val_acc: 0.9768
Epoch 81/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1201 - acc: 0.9874Epoch 00080: val_loss improved from 0.07913 to 0.07598, saving model to ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.080-0.0760.hdf5
15616/15616 [==============================] - 423s - loss: 0.1196 - acc: 0.9875 - val_loss: 0.0760 - val_acc: 0.9829
Epoch 82/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1114 - acc: 0.9873Epoch 00081: val_loss did not improve
15616/15616 [==============================] - 422s - loss: 0.1110 - acc: 0.9874 - val_loss: 0.1018 - val_acc: 0.9776
Epoch 83/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1081 - acc: 0.9886Epoch 00082: val_loss did not improve
15616/15616 [==============================] - 423s - loss: 0.1080 - acc: 0.9887 - val_loss: 0.0979 - val_acc: 0.9783
Epoch 84/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1148 - acc: 0.9886Epoch 00083: val_loss did not improve
15616/15616 [==============================] - 423s - loss: 0.1142 - acc: 0.9887 - val_loss: 0.0913 - val_acc: 0.9808
Epoch 85/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1114 - acc: 0.9879Epoch 00084: val_loss improved from 0.07598 to 0.07387, saving model to ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.084-0.0739.hdf5
15616/15616 [==============================] - 423s - loss: 0.1111 - acc: 0.9880 - val_loss: 0.0739 - val_acc: 0.9839
Epoch 86/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1012 - acc: 0.9895Epoch 00085: val_loss did not improve
15616/15616 [==============================] - 428s - loss: 0.1012 - acc: 0.9895 - val_loss: 0.0935 - val_acc: 0.9798
Epoch 87/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1239 - acc: 0.9885Epoch 00086: val_loss did not improve
15616/15616 [==============================] - 426s - loss: 0.1231 - acc: 0.9886 - val_loss: 0.0989 - val_acc: 0.9766
Epoch 88/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1055 - acc: 0.9881Epoch 00087: val_loss did not improve
15616/15616 [==============================] - 426s - loss: 0.1059 - acc: 0.9881 - val_loss: 0.0826 - val_acc: 0.9819
Epoch 89/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.0973 - acc: 0.9882Epoch 00088: val_loss did not improve
15616/15616 [==============================] - 427s - loss: 0.1012 - acc: 0.9881 - val_loss: 0.1025 - val_acc: 0.9786
Epoch 90/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1146 - acc: 0.9890Epoch 00089: val_loss did not improve
15616/15616 [==============================] - 425s - loss: 0.1142 - acc: 0.9890 - val_loss: 0.0953 - val_acc: 0.9773
Epoch 91/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1111 - acc: 0.9878Epoch 00090: val_loss did not improve

Epoch 00090: reducing learning rate to 1.0000000116860975e-08.
15616/15616 [==============================] - 426s - loss: 0.1116 - acc: 0.9878 - val_loss: 0.0795 - val_acc: 0.9814
Epoch 92/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1105 - acc: 0.9886Epoch 00091: val_loss did not improve
15616/15616 [==============================] - 427s - loss: 0.1107 - acc: 0.9885 - val_loss: 0.0769 - val_acc: 0.9793
Epoch 93/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1201 - acc: 0.9874Epoch 00092: val_loss did not improve
15616/15616 [==============================] - 427s - loss: 0.1193 - acc: 0.9875 - val_loss: 0.0837 - val_acc: 0.9816
Epoch 94/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1119 - acc: 0.9880Epoch 00093: val_loss did not improve
15616/15616 [==============================] - 426s - loss: 0.1115 - acc: 0.9881 - val_loss: 0.0895 - val_acc: 0.9791
Epoch 95/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.0950 - acc: 0.9885Epoch 00094: val_loss did not improve
15616/15616 [==============================] - 427s - loss: 0.0952 - acc: 0.9885 - val_loss: 0.0983 - val_acc: 0.9801
Epoch 96/300
15488/15616 [============================>.] - ETA: 3s - loss: 0.1165 - acc: 0.9878Epoch 00095: val_loss did not improve

Epoch 00095: reducing learning rate to 9.999999939225292e-10.
15616/15616 [==============================] - 426s - loss: 0.1158 - acc: 0.9879 - val_loss: 0.0852 - val_acc: 0.9808
Epoch 00095: early stopping
Out[8]:
<keras.callbacks.History at 0x7f17f0055898>

In [49]:
#test prepare

test_model, test_model_name = get_best_model()
# test_model = load_model('./resnet50_FT38_CW_STGTrain/checkpoint/weights.000-0.0327.hdf5')
# test_model_name = 'weights.000-0.0327.hdf5'
# print('test_model_name', test_model_name)

def test_generator(df, mean, datagen = None, batch_size = BATCHSIZE):
    n = df.shape[0]
    batch_index = 0
    while 1:
        current_index = batch_index * batch_size
        if n >= current_index + batch_size:
            current_batch_size = batch_size
            batch_index += 1    
        else:
            current_batch_size = n - current_index
            batch_index = 0        
        batch_df = df[current_index:current_index+current_batch_size]
        batch_x = np.zeros((batch_df.shape[0], ROWS, COLS, 3), dtype=K.floatx())
        i = 0
        for index,row in batch_df.iterrows():
            image_file = row['image_file']
            bbox = [row['xmin'],row['ymin'],row['xmax'],row['ymax']]
            cropped = load_img(TEST_DIR+image_file,bbox,target_size=(ROWS,COLS))
            x = np.asarray(cropped, dtype=K.floatx())
            x /= 255.
            if datagen is not None: x = datagen.random_transform(x)            
            x = preprocess_imagewise(x, mean)
            batch_x[i] = x
            i += 1
        if batch_index%50 == 0: print('batch_index', batch_index)
        yield(batch_x)
        
test_aug_datagen = ImageDataGenerator(
    rotation_range=180,
    shear_range=0.2,
    zoom_range=0.1,
    width_shift_range=0.1,
    height_shift_range=0.1,
    horizontal_flip=True,
    vertical_flip=True)

# train_mean = [0.37698776,  0.41491762,  0.38681713]
train_mean = imagewise_mean(X_train)
print('train_mean:', train_mean.shape)


Loading model from checkpoint file ./resnet50_FT38_Classifier_Rep5/checkpoint/weights.084-0.0739.hdf5
Loading model Done!
train_mean: (224, 224, 3)

In [3]:
test_model_name = 'weights.084-0.0739.hdf5'

In [4]:
#GTbbox_CROPpred_df = ['image_file','crop_index','crop_class','xmin','ymin','xmax','ymax','split'
#                      'NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT', 'logloss',
#                      'ALB_woNoF', 'BET_woNoF', 'DOL_woNoF', 'LAG_woNoF', 'OTHER_woNoF', 'SHARK_woNoF', 'YFT_woNoF', 'logloss_woNoF']

file_name = 'GTbbox_CROPpred_df_'+test_model_name+'_.pickle'
if os.path.exists(OUTPUT_DIR+file_name):
    print ('Loading from file '+file_name)
    GTbbox_CROPpred_df = pd.read_pickle(OUTPUT_DIR+file_name)
else:
    print ('Generating file '+file_name) 
    nb_augmentation = 1
    if nb_augmentation ==1:
        test_preds = test_model.predict_generator(test_generator(df=GTbbox_df, mean=train_mean), 
                                                  val_samples=GTbbox_df.shape[0], nb_worker=1, pickle_safe=False)
#         test_preds = test_model.predict_generator(test_generator(df=GTbbox_df, mean=train_mean), 
#                                                   steps=int(math.ceil(GTbbox_df.shape[0]/BATCHSIZE)), workers=1, pickle_safe=False)
    else:
        test_preds = np.zeros((GTbbox_df.shape[0], len(FISH_CLASSES)), dtype=K.floatx())
        for idx in range(nb_augmentation):
            print('{}th augmentation for testing ...'.format(idx+1))
            test_preds += test_model.predict_generator(test_generator(df=GTbbox_df, mean=train_mean, datagen=test_aug_datagen), 
                                                       val_samples=GTbbox_df.shape[0], nb_worker=1, pickle_safe=False)
#             test_preds += test_model.predict_generator(test_generator(df=GTbbox_df, mean=train_mean, datagen=test_aug_datagen), 
#                                                        steps=GTbbox_df.shape[0], workers=1, pickle_safe=False)
        test_preds /= nb_augmentation

    CROPpred_df = pd.DataFrame(test_preds, columns=FISH_CLASSES)
    GTbbox_CROPpred_df = pd.concat([GTbbox_df,CROPpred_df], axis=1)
    GTbbox_CROPpred_df['logloss'] = GTbbox_CROPpred_df.apply(lambda row: -math.log(row[row['crop_class']]), axis=1)
    
    for fish in FISH_CLASSES:
        GTbbox_CROPpred_df[fish+'_woNoF'] = GTbbox_CROPpred_df.apply(lambda row: row[fish]/(1-row['NoF']) if fish!='NoF' else np.inf , axis=1)
    GTbbox_CROPpred_df['logloss_woNoF'] = GTbbox_CROPpred_df.apply(lambda row: -math.log(row[row['crop_class']+'_woNoF']), axis=1)
    
    GTbbox_CROPpred_df.to_pickle(OUTPUT_DIR+file_name) 
    
valid_CROPpred_df = GTbbox_CROPpred_df[GTbbox_CROPpred_df['split']=='valid']
crop_valid_woNoF_logloss = valid_CROPpred_df[valid_CROPpred_df['crop_class']!='NoF']['logloss_woNoF'].mean()
print('crop_valid_woNoF_logloss:', crop_valid_woNoF_logloss)


Loading from file GTbbox_CROPpred_df_weights.084-0.0739.hdf5_.pickle
crop_valid_woNoF_logloss: 0.16453690212776534

In [5]:
# print('all loss:', GTbbox_CROPpred_df['logloss'].mean())
# print('all fish loss:', GTbbox_CROPpred_df[GTbbox_CROPpred_df['crop_class']!='NoF']['logloss'].mean())
# print(GTbbox_CROPpred_df.groupby(['crop_class'])['logloss'].mean())
# print('all_woNoF loss:', GTbbox_CROPpred_df[GTbbox_CROPpred_df['crop_class']!='NoF']['logloss_woNoF'].mean())
# print(GTbbox_CROPpred_df[GTbbox_CROPpred_df['crop_class']!='NoF'].groupby(['crop_class'])['logloss_woNoF'].mean())

train_CROPpred_df = GTbbox_CROPpred_df[GTbbox_CROPpred_df['split']=='train']
print('train loss:', train_CROPpred_df['logloss'].mean())
# print('train fish loss:', train_CROPpred_df[train_CROPpred_df['crop_class']!='NoF']['logloss'].mean())
# print(train_CROPpred_df.groupby(['crop_class'])['logloss'].mean())
print('train_woNoF loss:', train_CROPpred_df[train_CROPpred_df['crop_class']!='NoF']['logloss_woNoF'].mean())
print(train_CROPpred_df[train_CROPpred_df['crop_class']!='NoF'].groupby(['crop_class'])['logloss_woNoF'].mean())

valid_CROPpred_df = GTbbox_CROPpred_df[GTbbox_CROPpred_df['split']=='valid']
print('valid loss:', valid_CROPpred_df['logloss'].mean())
# print('valid fish loss:', valid_CROPpred_df[valid_CROPpred_df['crop_class']!='NoF']['logloss'].mean())
# print(valid_CROPpred_df.groupby(['crop_class'])['logloss'].mean())
print('valid_woNoF loss:', valid_CROPpred_df[valid_CROPpred_df['crop_class']!='NoF']['logloss_woNoF'].mean())
print(valid_CROPpred_df[valid_CROPpred_df['crop_class']!='NoF'].groupby(['crop_class'])['logloss_woNoF'].mean())


train loss: 0.03436334160963875
train_woNoF loss: 0.020864595229428972
crop_class
ALB      0.021373
BET      0.019553
DOL      0.000657
LAG      0.000273
OTHER    0.014748
SHARK    0.000621
YFT      0.032967
Name: logloss_woNoF, dtype: float64
valid loss: 0.09322671996697018
valid_woNoF loss: 0.16453690212776534
crop_class
ALB      0.110811
BET      0.327438
DOL      0.523910
LAG      0.004440
OTHER    0.255691
SHARK    0.002541
YFT      0.236282
Name: logloss_woNoF, dtype: float64
fish_loglosses = {} for fish in FISH_CLASSES: fish_loglosses[fish] = GTbbox_CROPpred_df[GTbbox_CROPpred_df['crop_class']==fish]['logloss'] fish_loglosses['ALB'].plot.box()

In [6]:
# RFCNbbox_RFCNpred_df = ['image_class','image_file','crop_index','xmin','ymin','xmax','ymax',
#                          'NoF_RFCN', 'ALB_RFCN', 'BET_RFCN', 'DOL_RFCN',
#                          'LAG_RFCN', 'OTHER_RFCN', 'SHARK_RFCN', 'YFT_RFCN']
# select fish_conf >= CONF_THRESH

file_name = 'RFCNbbox_RFCNpred_df_conf{:.2f}.pickle'.format(CONF_THRESH)
if os.path.exists(OUTPUT_DIR+file_name):
    print ('Loading from file '+file_name)
    RFCNbbox_RFCNpred_df = pd.read_pickle(OUTPUT_DIR+file_name)
else:
    print ('Generating file '+file_name)        
    RFCNbbox_RFCNpred_df = pd.DataFrame(columns=['image_class','image_file','crop_index','xmin','ymin','xmax','ymax',
                                                  'NoF_RFCN', 'ALB_RFCN', 'BET_RFCN', 'DOL_RFCN',
                                                  'LAG_RFCN', 'OTHER_RFCN', 'SHARK_RFCN', 'YFT_RFCN']) 

    with open('../data/RFCN_detections/detections_full_AGNOSTICnms_'+RFCN_MODEL+'.pkl','rb') as f:
        detections_full_AGNOSTICnms = pickle.load(f, encoding='latin1') 
    with open("../RFCN/ImageSets/Main/test.txt","r") as f:
        test_files = f.readlines()
    with open("../RFCN/ImageSets/Main/train_test.txt","r") as f:
        train_file_labels = f.readlines()
    assert len(detections_full_AGNOSTICnms) == len(test_files)
    
    count = np.zeros(len(detections_full_AGNOSTICnms))
    
    for im in range(len(detections_full_AGNOSTICnms)):
        if im%1000 == 0: print(im)
        basename = test_files[im][:9]
        if im<1000:
            image_class = '--'
        else:
            for i in range(len(train_file_labels)):
                if train_file_labels[i][:9] == basename:
                    image_class = train_file_labels[i][10:-1]
                    break
        image = Image.open(TEST_DIR+'/'+basename+'.jpg')
        width_image, height_image = image.size
        
        bboxes = []
        detects_im = detections_full_AGNOSTICnms[im]
        for i in range(len(detects_im)):
            if np.sum(detects_im[i,5:]) >= CONF_THRESH:
#             if np.max(detects_im[i,5:]) >= CONF_THRESH:
                bboxes.append(detects_im[i,:]) 
        count[im] = len(bboxes)
        if len(bboxes) == 0:
            ind = np.argmax(np.sum(detects_im[:,5:], axis=1))
            bboxes.append(detects_im[ind,:])
        bboxes = np.asarray(bboxes)

        for j in range(len(bboxes)):    
            bbox = bboxes[j]
            xmin = bbox[0]
            ymin = bbox[1]
            xmax = bbox[2]
            ymax = bbox[3]
            assert max(xmin,0)<min(xmax,width_image)
            assert max(ymin,0)<min(ymax,height_image)
            RFCNbbox_RFCNpred_df.loc[len(RFCNbbox_RFCNpred_df)]=[image_class,basename+'.jpg',j,max(xmin,0),max(ymin,0),
                                                                   min(xmax,width_image),min(ymax,height_image),
                                                                   bbox[4],bbox[5],bbox[6],bbox[7],bbox[8],bbox[9],bbox[10],bbox[11]]   
    
    RFCNbbox_RFCNpred_df.to_pickle(OUTPUT_DIR+file_name)


Loading from file RFCNbbox_RFCNpred_df_conf0.80.pickle

In [7]:
# RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df = ['image_class', 'image_file','crop_index','xmin','ymin','xmax','ymax',
#                                    'NoF_RFCN', 'ALB_RFCN', 'BET_RFCN', 'DOL_RFCN',
#                                    'LAG_RFCN', 'OTHER_RFCN', 'SHARK_RFCN', 'YFT_RFCN',
#                                    'NoF_CROP', 'ALB_CROP', 'BET_CROP', 'DOL_CROP',
#                                    'LAG_CROP', 'OTHER_CROP', 'SHARK_CROP', 'YFT_CROP',
#                                    'NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']

file_name = 'RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df_'+test_model_name+'_.pickle'
if os.path.exists(OUTPUT_DIR+file_name):
    print ('Loading from file '+file_name)
    RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df = pd.read_pickle(OUTPUT_DIR+file_name)
else:
    print ('Generating file '+file_name)  
    nb_augmentation = 1
    if nb_augmentation ==1:
        test_preds = test_model.predict_generator(test_generator(df=RFCNbbox_RFCNpred_df, mean=train_mean), 
                                                  val_samples=RFCNbbox_RFCNpred_df.shape[0], nb_worker=1, pickle_safe=False)
    else:
        test_preds = np.zeros((RFCNbbox_RFCNpred_df.shape[0], len(FISH_CLASSES)), dtype=K.floatx())
        for idx in range(nb_augmentation):
            print('{}th augmentation for testing ...'.format(idx+1))
            test_preds += test_model.predict_generator(test_generator(df=RFCNbbox_RFCNpred_df, mean=train_mean, datagen=test_aug_datagen), 
                                                       val_samples=RFCNbbox_RFCNpred_df.shape[0], nb_worker=1, pickle_safe=False)
        test_preds /= nb_augmentation

    CROPpred_df = pd.DataFrame(test_preds, columns=['NoF_CROP', 'ALB_CROP', 'BET_CROP', 'DOL_CROP', 'LAG_CROP', 'OTHER_CROP', 'SHARK_CROP', 'YFT_CROP'])
    RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df = pd.concat([RFCNbbox_RFCNpred_df,CROPpred_df], axis=1)
    
    RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df['NoF'] = RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df['NoF_RFCN']
    for fish in ['ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']:
        RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df[fish] = RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df.apply(lambda row: (1-row['NoF_RFCN'])*row[fish+'_CROP']/(1-row['NoF_CROP']), axis=1)

#     for fish in FISH_CLASSES:
#         RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df[fish] = RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df[fish+'_CROP']

    RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df.to_pickle(OUTPUT_DIR+file_name)


Loading from file RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df_weights.084-0.0739.hdf5_.pickle

In [8]:
# clsMaxAve and hybrid RFCNpred&CROPpred such that RFCNpred for NoF and CROPpred for fish
# test_pred_df = ['logloss','image_class','image_file','NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']
# RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df = ['image_class', 'image_file','crop_index','xmin','ymin','xmax','ymax',
#                                    'NoF_RFCN', 'ALB_RFCN', 'BET_RFCN', 'DOL_RFCN',
#                                    'LAG_RFCN', 'OTHER_RFCN', 'SHARK_RFCN', 'YFT_RFCN',
#                                    'ALB_CROP', 'BET_CROP', 'DOL_CROP',
#                                    'LAG_CROP', 'OTHER_CROP', 'SHARK_CROP', 'YFT_CROP',
#                                    'NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']

file_name = 'test_pred_df_Hybrid_'+test_model_name+'_.pickle'
if os.path.exists(OUTPUT_DIR+file_name):
    print ('Loading from file '+file_name)
    test_pred_df = pd.read_pickle(OUTPUT_DIR+file_name)
else:
    print ('Generating file '+file_name)  
    with open("../RFCN/ImageSets/Main/test.txt","r") as f:
        test_files = f.readlines()
    
    test_pred_df = pd.DataFrame(columns=['logloss','image_class','image_file','NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT'])  
    for j in range(len(test_files)): 
        image_file = test_files[j][:-1]+'.jpg'
        test_pred_im_df = RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df.loc[RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df['image_file'] == image_file,
                                                                       ['image_class', 'NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']]
        image_class = test_pred_im_df.iloc[0]['image_class']
        test_pred_im_df.drop('image_class', axis=1, inplace=True)
        max_score = test_pred_im_df.max(axis=1)
        max_cls = test_pred_im_df.idxmax(axis=1)
        test_pred_im_df['max_score'] = max_score
        test_pred_im_df['max_cls'] = max_cls
        test_pred_im_df['Count'] = test_pred_im_df.groupby(['max_cls'])['max_cls'].transform('count')
        idx = test_pred_im_df.groupby(['max_cls'])['max_score'].transform(max) == test_pred_im_df['max_score']
        test_pred_im_clsMax_df = test_pred_im_df.loc[idx,['NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT', 'Count']]
        test_pred_im_clsMax_array = test_pred_im_clsMax_df.values
        pred = np.average(test_pred_im_clsMax_array[:,:-1], axis=0, weights=test_pred_im_clsMax_array[:,-1], returned=False).tolist()
        if image_class!='--':
            ind = FISH_CLASSES.index(image_class)
            logloss = -math.log(pred[ind]) 
        else:
            logloss = np.nan
        test_pred_im_clsMaxAve = [logloss,image_class,image_file]
        test_pred_im_clsMaxAve.extend(pred)
        test_pred_df.loc[len(test_pred_df)]=test_pred_im_clsMaxAve

    test_pred_df.to_pickle(OUTPUT_DIR+file_name) 
    

image_all_logloss = test_pred_df[test_pred_df['image_class']!='--']['logloss'].mean()
print('imag_all_logloss:', image_all_logloss)

# print(test_pred_df[test_pred_df['image_class']!='--'].groupby('image_class')['logloss'].mean())


Loading from file test_pred_df_Hybrid_weights.084-0.0739.hdf5_.pickle
imag_all_logloss: 0.06455932285959344

In [ ]:
#### visualization
# RFCNbbox_RFCNpred_CROPpred_df = ['image_class', 'image_file','crop_index','x_min','y_min','x_max','ymax',
#                                    'NoF_RFCN', 'ALB_RFCN', 'BET_RFCN', 'DOL_RFCN',
#                                    'LAG_RFCN', 'OTHER_RFCN', 'SHARK_RFCN', 'YFT_RFCN'
#                                    'NoF_CROP', 'ALB_CROP', 'BET_CROP', 'DOL_CROP',
#                                    'LAG_CROP', 'OTHER_CROP', 'SHARK_CROP', 'YFT_CROP']
#GTbbox_CROPpred_df = ['image_file','crop_index','crop_class','xmin','ymin','xmax','ymax',
#                      'NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT', 'logloss']
# test_pred_df = ['logloss','image_class','image_file','NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']

for j in range(test_pred_df.shape[0]):
    image_logloss = test_pred_df.iat[j,0]
    image_class = test_pred_df.iat[j,1]
    image_file = test_pred_df.iat[j,2]
    if j<1000 and j%30== 0:
        pass
    else: 
        continue
    im = Image.open('../RFCN/JPEGImages/'+image_file)
    im = np.asarray(im)
    fig, ax = plt.subplots(figsize=(10, 8))
    ax.imshow(im, aspect='equal')
    RFCN_dets = RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df.loc[RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df['image_file']==image_file]
    for index,row in RFCN_dets.iterrows():
        bbox = [row['xmin'],row['ymin'],row['xmax'],row['ymax']]
        RFCN = [row['NoF_RFCN'],row['ALB_RFCN'],row['BET_RFCN'],row['DOL_RFCN'],row['LAG_RFCN'],row['OTHER_RFCN'],row['SHARK_RFCN'],row['YFT_RFCN']]
        CROP = [row['NoF'],row['ALB'],row['BET'],row['DOL'],row['LAG'],row['OTHER'],row['SHARK'],row['YFT']]
        score_RFCN = max(RFCN)
        score_CROP = max(CROP)
        index_RFCN = RFCN.index(score_RFCN)
        index_CROP = CROP.index(score_CROP)
        class_RFCN = FISH_CLASSES[index_RFCN]
        class_CROP = FISH_CLASSES[index_CROP]
        ax.add_patch(plt.Rectangle((bbox[0], bbox[1]), bbox[2] - bbox[0], bbox[3] - bbox[1], fill=False, edgecolor='red', linewidth=2))
        ax.text(bbox[0], bbox[1] - 2, 'RFCN_{:s} {:.3f} \nHYBRID_{:s} {:.3f}'.format(class_RFCN, score_RFCN, class_CROP, score_CROP), bbox=dict(facecolor='red', alpha=0.5), fontsize=8, color='white')       
    GT_dets = GTbbox_CROPpred_df.loc[GTbbox_CROPpred_df['image_file']==image_file]
    for index,row in GT_dets.iterrows():
        bbox = [row['xmin'],row['ymin'],row['xmax'],row['ymax']]
        CROP = [row['NoF'],row['ALB'],row['BET'],row['DOL'],row['LAG'],row['OTHER'],row['SHARK'],row['YFT']]
        score_CROP = max(CROP)
        index_CROP = CROP.index(score_CROP)
        class_CROP = FISH_CLASSES[index_CROP]
        ax.add_patch(plt.Rectangle((bbox[0], bbox[1]), bbox[2] - bbox[0], bbox[3] - bbox[1], fill=False, edgecolor='green', linewidth=2))
        ax.text(bbox[0], bbox[3] + 40, 'GT_{:s} \nCROP_{:s} {:.3f}'.format(row['crop_class'], class_CROP, score_CROP), bbox=dict(facecolor='green', alpha=0.5), fontsize=8, color='white')
    ax.set_title(('Image {:s}    FISH {:s}    logloss {}').format(image_file, image_class, image_logloss), fontsize=10) 
    plt.axis('off')
    plt.tight_layout()
    plt.draw()

In [9]:
#temperature
T = 2.5
test_pred_array = test_pred_df[FISH_CLASSES].values
test_pred_T_array = np.exp(np.log(test_pred_array)/T)
test_pred_T_array = test_pred_T_array/np.sum(test_pred_T_array, axis=1, keepdims=True)
test_pred_T_df = pd.DataFrame(test_pred_T_array, columns=FISH_CLASSES)
test_pred_T_df = pd.concat([test_pred_df[['image_class','image_file']],test_pred_T_df], axis=1)

#test submission
submission = test_pred_T_df.loc[:999,['image_file','NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']]
submission.rename(columns={'image_file':'image'}, inplace=True)
sub_file = 'RFCN_AGONOSTICnms_'+RFCN_MODEL+'_'+CROP_MODEL+'_'+test_model_name+'_clsMaxAve_conf{:.2f}_cropvalloss{:.4f}_imageallloss{:.4f}_T{}.csv'.format(CONF_THRESH, crop_valid_woNoF_logloss, image_all_logloss, T)
submission.to_csv(sub_file, index=False)
submission.to_csv(OUTPUT_DIR + sub_file, index=False)
print('Done!'+sub_file)


Done!RFCN_AGONOSTICnms_resnet101_rfcn_ohem_iter_30000_resnet50_FT38_Classifier_Rep5_weights.084-0.0739.hdf5_clsMaxAve_conf0.80_cropvalloss0.1645_imageallloss0.0646_T2.5.csv

In [ ]: