In [4]:

    

import os, random, glob, pickle, collections, math, json
import numpy as np
import pandas as pd
from __future__ import division
from __future__ import print_function
from PIL import Image
from sklearn.model_selection import train_test_split
from sklearn.metrics import log_loss
from sklearn.preprocessing import LabelEncoder

import matplotlib.pyplot as plt
%matplotlib inline 

from keras.models import Sequential, Model, load_model
from keras.layers import GlobalAveragePooling2D, Flatten, Dropout, Dense, LeakyReLU
from keras.optimizers import Adam
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau, TensorBoard
from keras.preprocessing.image import ImageDataGenerator
from keras.utils import np_utils
from keras.preprocessing import image
from keras import backend as K
K.set_image_dim_ordering('tf')


    

In [5]:

    

TRAIN_DIR = '../data/train/'
TEST_DIR = '../RFCN/JPEGImages/'
TRAIN_CROP_DIR = '../data/train_crop/'
TEST_CROP_DIR = '../data/test_stg1_crop/'
RFCN_MODEL = 'resnet101_rfcn_ohem_iter_30000'
CROP_MODEL = 'resnet50_FT38_Classifier_Rep2'
if not os.path.exists('./' + CROP_MODEL):
    os.mkdir('./' + CROP_MODEL)
CHECKPOINT_DIR = './' + CROP_MODEL + '/checkpoint/'
if not os.path.exists(CHECKPOINT_DIR):
    os.mkdir(CHECKPOINT_DIR)
LOG_DIR = './' + CROP_MODEL + '/log/'
if not os.path.exists(LOG_DIR):
    os.mkdir(LOG_DIR)
OUTPUT_DIR = './' + CROP_MODEL + '/output/'
if not os.path.exists(OUTPUT_DIR):
    os.mkdir(OUTPUT_DIR)
FISH_CLASSES = ['NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']
# CROP_CLASSES=FISH_CLASSES[:]
# CROP_CLASSES.remove('NoF')
CONF_THRESH = 0.8
ROWS = 224
COLS = 224
BATCHSIZE = 128
LEARNINGRATE = 1e-4
def featurewise_center(x):
    mean = np.mean(x, axis=0, keepdims=True)
    mean = np.mean(mean, axis=(1,2), keepdims=True)
    x_centered = x - mean
    return x_centered

def mean(x):
    mean = np.mean(x, axis=0)
    mean = np.mean(mean, axis=(0,1))
    return mean

def load_img(path, bbox, target_size=None):
    img = Image.open(path)
#     img = img.convert('RGB')
    cropped = img.crop((bbox[0],bbox[1],bbox[2],bbox[3]))
    width_cropped, height_cropped = cropped.size
    if height_cropped > width_cropped: cropped = cropped.transpose(method=2)
#     cropped.save('try.jpg')
#     cropped = Image.open('try.jpg')   
    if target_size:
        cropped = cropped.resize((target_size[1], target_size[0]), Image.BILINEAR)
    return cropped

def preprocess_input(x, mean):
    #resnet50 image preprocessing
#     'RGB'->'BGR'
#     x = x[:, :, ::-1]
#     x /= 255.
    x[:, :, 0] -= mean[0]
    x[:, :, 1] -= mean[1]
    x[:, :, 2] -= mean[2]
    return x

def get_best_model(checkpoint_dir = CHECKPOINT_DIR):
    files = glob.glob(checkpoint_dir+'*')
    val_losses = [float(f.split('-')[-1][:-5]) for f in files]
    index = val_losses.index(min(val_losses))
    print('Loading model from checkpoint file ' + files[index])
    model = load_model(files[index])
    model_name = files[index].split('/')[-1]
    print('Loading model Done!')
    return (model, model_name)


    

In [6]:

    

# GTbbox_df = ['image_file','crop_index','crop_class','xmin',''ymin','xmax','ymax','split']

file_name = 'GTbbox_df.pickle'
if os.path.exists(OUTPUT_DIR+file_name):
    print ('Loading from file '+file_name)
    GTbbox_df = pd.read_pickle(OUTPUT_DIR+file_name)
else:
    print ('Generating file '+file_name)       
    GTbbox_df = pd.DataFrame(columns=['image_file','crop_index','crop_class','xmin','ymin','xmax','ymax'])  

    crop_classes=FISH_CLASSES[:]
    crop_classes.remove('NoF')
    
    for c in crop_classes:
        print(c)
        j = json.load(open('../data/BBannotations/{}.json'.format(c), 'r'))
        for l in j: 
            filename = l["filename"]
            head, image_file = os.path.split(filename)
            basename, file_extension = os.path.splitext(image_file) 
            image = Image.open(TEST_DIR+'/'+image_file)
            width_image, height_image = image.size
            for i in range(len(l["annotations"])):
                a = l["annotations"][i]
                xmin = (a["x"])
                ymin = (a["y"])
                width = (a["width"])
                height = (a["height"])
                xmax = xmin + width
                ymax = ymin + height
                assert max(xmin,0)<min(xmax,width_image)
                assert max(ymin,0)<min(ymax,height_image)
                GTbbox_df.loc[len(GTbbox_df)]=[image_file,i,a["class"],max(xmin,0),max(ymin,0),min(xmax,width_image),min(ymax,height_image)]
                if a["class"] != c: print(GTbbox_df.tail(1))  
        
        
    num_NoF = GTbbox_df.shape[0]*3
    RFCN_MODEL = 'resnet101_rfcn_ohem_iter_30000'
    with open('../data/RFCN_detections/detections_full_AGNOSTICnms_'+RFCN_MODEL+'.pkl','rb') as f:
        detections_full_AGNOSTICnms = pickle.load(f) 
    train_detections_full_AGNOSTICnms = detections_full_AGNOSTICnms[1000:]
    with open("../RFCN/ImageSets/Main/test.txt","r") as f:
        test_files = f.readlines()
    train_files = test_files[1000:]
    num_NoF_perIm = int(math.ceil(num_NoF / len(train_detections_full_AGNOSTICnms)))

    for im in range(len(train_files)):
        image_file = train_files[im][:-1]+'.jpg'
        image = Image.open(TEST_DIR+image_file)
        width_image, height_image = image.size
        detects_im = train_detections_full_AGNOSTICnms[im]
        detects_im = detects_im[np.where(detects_im[:,4] >= 0.999)]
        np.random.seed(1986)
        bboxes = detects_im[np.random.choice(detects_im.shape[0], num_NoF_perIm, replace=False), :]
        for j in range(bboxes.shape[0]):    
            bbox = bboxes[j]
            xmin = bbox[0]
            ymin = bbox[1]
            xmax = bbox[2]
            ymax = bbox[3]
            assert max(xmin,0)<min(xmax,width_image)
            assert max(ymin,0)<min(ymax,height_image)
            GTbbox_df.loc[len(GTbbox_df)]=[image_file,j,'NoF',max(xmin,0),max(ymin,0),min(xmax,width_image),min(ymax,height_image)]

    test_size = GTbbox_df.shape[0]-int(math.ceil(GTbbox_df.shape[0]*0.8/128)*128)
    train_ind, valid_ind = train_test_split(range(GTbbox_df.shape[0]), test_size=test_size, random_state=1986, stratify=GTbbox_df['crop_class'])
    GTbbox_df['split'] = ['train' if i in train_ind else 'valid' for i in range(GTbbox_df.shape[0])]
    GTbbox_df.to_pickle(OUTPUT_DIR+file_name)


    

    




Loading from file GTbbox_df.pickle

In [14]:

    

#Load data

def data_from_df(df):
    X = np.ndarray((df.shape[0], ROWS, COLS, 3), dtype=np.uint8)
    y = np.zeros((df.shape[0], len(FISH_CLASSES)), dtype=K.floatx())
    i = 0
    for index,row in df.iterrows():
        image_file = row['image_file']
        fish = row['crop_class']
        bbox = [row['xmin'],row['ymin'],row['xmax'],row['ymax']]
        cropped = load_img(TEST_DIR+image_file,bbox,target_size=(ROWS,COLS))
        X[i] = np.asarray(cropped)
        y[i,FISH_CLASSES.index(fish)] = 1
        i += 1
    return (X, y)

def data_load(name):
    file_name = 'data_'+name+'_{}_{}.pickle'.format(ROWS, COLS)
    if os.path.exists(OUTPUT_DIR+file_name):
        print ('Loading from file '+file_name)
        with open(OUTPUT_DIR+file_name, 'rb') as f:
            data = pickle.load(f)
        X = data['X']
        y = data['y']
    else:
        print ('Generating file '+file_name)
        
        if name=='train' or name=='valid': 
            df = GTbbox_df[GTbbox_df['split']==name]
        elif name=='all':
            df = GTbbox_df
        else:
            print('Invalid name '+name)
    
        X, y = data_from_df(df)

        data = {'X': X,'y': y}
        with open(OUTPUT_DIR+file_name, 'wb') as f:
            pickle.dump(data, f)
    return (X, y)
X_train, y_train = data_load('train')
X_valid, y_valid = data_load('valid')
       
print('Loading data done.')
print('train sample', X_train.shape[0])
print('valid sample', X_valid.shape[0])
X_train = X_train.astype(np.float32)
X_valid = X_valid.astype(np.float32)
print('Convert to float32 done.')
X_train /= 255.
X_valid /= 255.
print('Rescale by 255 done.')
X_train_centered = featurewise_center(X_train)
print('mean of X_train is', mean(X_train))
X_valid_centered = featurewise_center(X_valid)
print('mean of X_valid is', mean(X_valid))
print('Featurewise centered done.')


    

    




Loading from file data_train_224_224.pickle
Loading from file data_valid_224_224.pickle
Loading data done.
train sample 15616
valid sample 3863
Convert to float32 done.
Rescale by 255 done.
mean of X_train is [ 0.40802017  0.44779599  0.41339037]
mean of X_valid is [ 0.41244254  0.45302087  0.41779673]
Featurewise centered done.

In [8]:

    

# #class weight = n_samples / (n_classes * np.bincount(y))
# class_weight_fish = dict(GTbbox_df.groupby('crop_class').size())
# class_weight = {}
# n_samples = GTbbox_df.shape[0]
# for key,value in class_weight_fish.items():
#         class_weight[CROP_CLASSES.index(key)] = n_samples / (len(CROP_CLASSES)*value)
# class_weight

class_weight_fish = dict(GTbbox_df.groupby('crop_class').size())
class_weight = {}
ref = max(class_weight_fish.values())
for key,value in class_weight_fish.items():
    class_weight[FISH_CLASSES.index(key)] = ref/value
class_weight


    

    
Out[8]:





{0: 1.0,
 1: 6.0119379228014322,
 2: 49.372549019607845,
 3: 119.9047619047619,
 4: 143.88571428571427,
 5: 45.369369369369366,
 6: 79.936507936507937,
 7: 18.908635794743429}

In [9]:

    

#data augmentation

train_datagen = ImageDataGenerator(
    rotation_range=180,
    shear_range=0.2,
    zoom_range=0.1,
    width_shift_range=0.1,
    height_shift_range=0.1,
    horizontal_flip=True,
    vertical_flip=True)
train_generator = train_datagen.flow(X_train, y_train, batch_size=BATCHSIZE, shuffle=True, seed=None)
assert X_train.shape[0]%BATCHSIZE==0
steps_per_epoch = int(X_train.shape[0]/BATCHSIZE)


    

In [10]:

    

#callbacks

early_stopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=10, verbose=1, mode='auto')        

model_checkpoint = ModelCheckpoint(filepath=CHECKPOINT_DIR+'weights.{epoch:03d}-{val_loss:.4f}.hdf5', monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=False, mode='auto')
        
learningrate_schedule = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=5, verbose=1, mode='auto', epsilon=0.001, cooldown=0, min_lr=0)

tensorboard = TensorBoard(log_dir=LOG_DIR, histogram_freq=0, write_graph=False, write_images=True)


    

In [28]:

    

#Resnet50
#top layer training

from keras.applications.resnet50 import ResNet50

base_model = ResNet50(weights='imagenet', include_top=False)
x = base_model.output
x = GlobalAveragePooling2D()(x)
# x = Flatten()(x)
# x = Dense(256)(x)
# x = LeakyReLU(alpha=0.33)(x)
# x = Dropout(0.5)(x)
predictions = Dense(len(FISH_CLASSES), init='glorot_normal', activation='softmax')(x)

model = Model(inputs=base_model.input, outputs=predictions)

# first: train only the top layers (which were randomly initialized)
for layer in base_model.layers:
    layer.trainable = False

# compile the model (should be done *after* setting layers to non-trainable)
optimizer = Adam(lr=LEARNINGRATE)
model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])

# train the model on the new data for a few epochs
model.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=30, verbose=1, 
                    callbacks=[early_stopping, model_checkpoint, learningrate_schedule, tensorboard], 
                    validation_data=(X_valid,y_valid), class_weight=class_weight, 
                    workers=3, pickle_safe=True, initial_epoch=0)


    

    




Downloading data from https://github.com/fchollet/deep-learning-models/releases/download/v0.2/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5






    




/usr/lib/python2.7/dist-packages/ipykernel/__main__.py:13: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(8, activation="softmax", kernel_initializer="glorot_normal")`






    




Epoch 1/30
121/122 [============================>.] - ETA: 1s - loss: 12.1726 - acc: 0.3901   Epoch 00000: val_loss improved from inf to 1.11952, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.000-1.1195.hdf5
122/122 [==============================] - 249s - loss: 12.1760 - acc: 0.3918 - val_loss: 1.1195 - val_acc: 0.7756
Epoch 2/30
121/122 [============================>.] - ETA: 1s - loss: 10.0166 - acc: 0.6117  Epoch 00001: val_loss improved from 1.11952 to 1.01355, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.001-1.0136.hdf5
122/122 [==============================] - 216s - loss: 10.0170 - acc: 0.6121 - val_loss: 1.0136 - val_acc: 0.7756
Epoch 3/30
121/122 [============================>.] - ETA: 1s - loss: 8.4796 - acc: 0.6795  Epoch 00002: val_loss improved from 1.01355 to 0.96510, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.002-0.9651.hdf5
122/122 [==============================] - 216s - loss: 8.4765 - acc: 0.6795 - val_loss: 0.9651 - val_acc: 0.7756
Epoch 4/30
121/122 [============================>.] - ETA: 1s - loss: 7.4992 - acc: 0.7161  Epoch 00003: val_loss did not improve
122/122 [==============================] - 217s - loss: 7.4948 - acc: 0.7161 - val_loss: 0.9873 - val_acc: 0.7756
Epoch 5/30
121/122 [============================>.] - ETA: 1s - loss: 7.2107 - acc: 0.7237  Epoch 00004: val_loss did not improve
122/122 [==============================] - 216s - loss: 7.2319 - acc: 0.7233 - val_loss: 0.9785 - val_acc: 0.7756
Epoch 6/30
121/122 [============================>.] - ETA: 1s - loss: 6.9087 - acc: 0.7350  Epoch 00005: val_loss improved from 0.96510 to 0.85294, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.005-0.8529.hdf5
122/122 [==============================] - 216s - loss: 6.8812 - acc: 0.7351 - val_loss: 0.8529 - val_acc: 0.7761
Epoch 7/30
121/122 [============================>.] - ETA: 1s - loss: 6.3763 - acc: 0.7543  Epoch 00006: val_loss improved from 0.85294 to 0.69586, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.006-0.6959.hdf5
122/122 [==============================] - 217s - loss: 6.3712 - acc: 0.7542 - val_loss: 0.6959 - val_acc: 0.7849
Epoch 8/30
121/122 [============================>.] - ETA: 1s - loss: 6.0321 - acc: 0.7612  Epoch 00007: val_loss improved from 0.69586 to 0.58757, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.007-0.5876.hdf5
122/122 [==============================] - 217s - loss: 6.0139 - acc: 0.7610 - val_loss: 0.5876 - val_acc: 0.8144
Epoch 9/30
121/122 [============================>.] - ETA: 1s - loss: 5.8884 - acc: 0.7680  Epoch 00008: val_loss improved from 0.58757 to 0.57193, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.008-0.5719.hdf5
122/122 [==============================] - 217s - loss: 5.8796 - acc: 0.7680 - val_loss: 0.5719 - val_acc: 0.8328
Epoch 10/30
121/122 [============================>.] - ETA: 1s - loss: 5.5829 - acc: 0.7719  Epoch 00009: val_loss did not improve
122/122 [==============================] - 217s - loss: 5.5704 - acc: 0.7720 - val_loss: 0.5915 - val_acc: 0.8206
Epoch 11/30
121/122 [============================>.] - ETA: 1s - loss: 5.4747 - acc: 0.7658  Epoch 00010: val_loss improved from 0.57193 to 0.55983, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.010-0.5598.hdf5
122/122 [==============================] - 217s - loss: 5.4529 - acc: 0.7662 - val_loss: 0.5598 - val_acc: 0.8253
Epoch 12/30
121/122 [============================>.] - ETA: 1s - loss: 5.1792 - acc: 0.7834  Epoch 00011: val_loss improved from 0.55983 to 0.54541, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.011-0.5454.hdf5
122/122 [==============================] - 217s - loss: 5.1944 - acc: 0.7832 - val_loss: 0.5454 - val_acc: 0.8286
Epoch 13/30
121/122 [============================>.] - ETA: 1s - loss: 4.8591 - acc: 0.7897  Epoch 00012: val_loss improved from 0.54541 to 0.53937, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.012-0.5394.hdf5
122/122 [==============================] - 217s - loss: 4.8573 - acc: 0.7898 - val_loss: 0.5394 - val_acc: 0.8310
Epoch 14/30
121/122 [============================>.] - ETA: 1s - loss: 5.1746 - acc: 0.7935  Epoch 00013: val_loss did not improve
122/122 [==============================] - 216s - loss: 5.1702 - acc: 0.7934 - val_loss: 0.5606 - val_acc: 0.8136
Epoch 15/30
121/122 [============================>.] - ETA: 1s - loss: 4.7411 - acc: 0.8003  Epoch 00014: val_loss improved from 0.53937 to 0.48448, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.014-0.4845.hdf5
122/122 [==============================] - 216s - loss: 4.7394 - acc: 0.8005 - val_loss: 0.4845 - val_acc: 0.8478
Epoch 16/30
121/122 [============================>.] - ETA: 1s - loss: 4.7897 - acc: 0.8009  Epoch 00015: val_loss did not improve
122/122 [==============================] - 216s - loss: 4.7830 - acc: 0.8012 - val_loss: 0.4860 - val_acc: 0.8455
Epoch 17/30
121/122 [============================>.] - ETA: 1s - loss: 4.6513 - acc: 0.8004  Epoch 00016: val_loss did not improve
122/122 [==============================] - 217s - loss: 4.6519 - acc: 0.8000 - val_loss: 0.4938 - val_acc: 0.8429
Epoch 18/30
121/122 [============================>.] - ETA: 1s - loss: 4.3667 - acc: 0.8058  Epoch 00017: val_loss improved from 0.48448 to 0.46331, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.017-0.4633.hdf5
122/122 [==============================] - 217s - loss: 4.3759 - acc: 0.8058 - val_loss: 0.4633 - val_acc: 0.8491
Epoch 19/30
121/122 [============================>.] - ETA: 1s - loss: 4.2284 - acc: 0.8171  Epoch 00018: val_loss improved from 0.46331 to 0.46199, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.018-0.4620.hdf5
122/122 [==============================] - 217s - loss: 4.2497 - acc: 0.8165 - val_loss: 0.4620 - val_acc: 0.8548
Epoch 20/30
121/122 [============================>.] - ETA: 1s - loss: 4.4280 - acc: 0.8045  Epoch 00019: val_loss did not improve
122/122 [==============================] - 216s - loss: 4.4333 - acc: 0.8044 - val_loss: 0.4707 - val_acc: 0.8512
Epoch 21/30
121/122 [============================>.] - ETA: 1s - loss: 4.3111 - acc: 0.8153  Epoch 00020: val_loss did not improve
122/122 [==============================] - 216s - loss: 4.2933 - acc: 0.8152 - val_loss: 0.4961 - val_acc: 0.8385
Epoch 22/30
121/122 [============================>.] - ETA: 1s - loss: 4.0651 - acc: 0.8172  Epoch 00021: val_loss improved from 0.46199 to 0.42621, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.021-0.4262.hdf5
122/122 [==============================] - 217s - loss: 4.0676 - acc: 0.8178 - val_loss: 0.4262 - val_acc: 0.8620
Epoch 23/30
121/122 [============================>.] - ETA: 1s - loss: 4.2346 - acc: 0.8130  Epoch 00022: val_loss did not improve
122/122 [==============================] - 216s - loss: 4.2203 - acc: 0.8135 - val_loss: 0.4347 - val_acc: 0.8535
Epoch 24/30
121/122 [============================>.] - ETA: 1s - loss: 4.1358 - acc: 0.8177  Epoch 00023: val_loss did not improve
122/122 [==============================] - 217s - loss: 4.1179 - acc: 0.8182 - val_loss: 0.4477 - val_acc: 0.8566
Epoch 25/30
121/122 [============================>.] - ETA: 1s - loss: 4.0492 - acc: 0.8203  Epoch 00024: val_loss did not improve
122/122 [==============================] - 216s - loss: 4.0402 - acc: 0.8206 - val_loss: 0.4305 - val_acc: 0.8607
Epoch 26/30
121/122 [============================>.] - ETA: 1s - loss: 3.7497 - acc: 0.8304  Epoch 00025: val_loss did not improve
122/122 [==============================] - 216s - loss: 3.7602 - acc: 0.8304 - val_loss: 0.4273 - val_acc: 0.8600
Epoch 27/30
121/122 [============================>.] - ETA: 1s - loss: 3.8716 - acc: 0.8268  Epoch 00026: val_loss did not improve
122/122 [==============================] - 216s - loss: 3.8618 - acc: 0.8274 - val_loss: 0.4350 - val_acc: 0.8574
Epoch 28/30
121/122 [============================>.] - ETA: 1s - loss: 3.8608 - acc: 0.8286  Epoch 00027: val_loss did not improve

Epoch 00027: reducing learning rate to 9.99999974738e-06.
122/122 [==============================] - 217s - loss: 3.8563 - acc: 0.8284 - val_loss: 0.4278 - val_acc: 0.8566
Epoch 29/30
121/122 [============================>.] - ETA: 1s - loss: 3.7149 - acc: 0.8332  Epoch 00028: val_loss did not improve
122/122 [==============================] - 217s - loss: 3.7281 - acc: 0.8332 - val_loss: 0.4350 - val_acc: 0.8574
Epoch 30/30
121/122 [============================>.] - ETA: 1s - loss: 3.8132 - acc: 0.8302  Epoch 00029: val_loss did not improve
122/122 [==============================] - 217s - loss: 3.8121 - acc: 0.8299 - val_loss: 0.4331 - val_acc: 0.8579






    
Out[28]:





<keras.callbacks.History at 0x7f3a2da23510>

In [10]:

    

### Resnet50
# fine tuning
# 164 conv5c+top
# 142 conv5+top
# 80 conv4+conv5+top
# 38 conv3+conv4+conv5+top
start_layer = 38

model, model_name = get_best_model()
# print('Loading model from weights.004-0.0565.hdf5')
# model = load_model('./checkpoints/checkpoint3/weights.004-0.0565.hdf5')

for layer in model.layers[:start_layer]:
   layer.trainable = False
for layer in model.layers[start_layer:]:
   layer.trainable = True

# we need to recompile the model for these modifications to take effect
# we use SGD with a low learning rate
optimizer = Adam(lr=1e-5)
model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])

model.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=200, verbose=1, 
                    callbacks=[early_stopping, model_checkpoint, learningrate_schedule, tensorboard], 
                    validation_data=(X_valid,y_valid), class_weight=class_weight, 
                    workers=3, pickle_safe=True, initial_epoch=22)


    

    




Loading model from checkpoint file ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.021-0.4262.hdf5
Loading model Done!
Epoch 23/200
121/122 [============================>.] - ETA: 2s - loss: 3.6884 - acc: 0.8367  Epoch 00022: val_loss improved from inf to 0.37782, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.022-0.3778.hdf5
122/122 [==============================] - 424s - loss: 3.6982 - acc: 0.8364 - val_loss: 0.3778 - val_acc: 0.8858
Epoch 24/200
121/122 [============================>.] - ETA: 2s - loss: 3.0563 - acc: 0.8547  Epoch 00023: val_loss improved from 0.37782 to 0.35813, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.023-0.3581.hdf5
122/122 [==============================] - 396s - loss: 3.0474 - acc: 0.8544 - val_loss: 0.3581 - val_acc: 0.8853
Epoch 25/200
121/122 [============================>.] - ETA: 2s - loss: 2.6702 - acc: 0.8743  Epoch 00024: val_loss improved from 0.35813 to 0.31100, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.024-0.3110.hdf5
122/122 [==============================] - 397s - loss: 2.6562 - acc: 0.8745 - val_loss: 0.3110 - val_acc: 0.9029
Epoch 26/200
121/122 [============================>.] - ETA: 2s - loss: 2.2594 - acc: 0.8893  Epoch 00025: val_loss improved from 0.31100 to 0.29977, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.025-0.2998.hdf5
122/122 [==============================] - 397s - loss: 2.2547 - acc: 0.8892 - val_loss: 0.2998 - val_acc: 0.9130
Epoch 27/200
121/122 [============================>.] - ETA: 2s - loss: 1.9029 - acc: 0.9009  Epoch 00026: val_loss improved from 0.29977 to 0.27509, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.026-0.2751.hdf5
122/122 [==============================] - 397s - loss: 1.9033 - acc: 0.9008 - val_loss: 0.2751 - val_acc: 0.9120
Epoch 28/200
121/122 [============================>.] - ETA: 2s - loss: 1.7054 - acc: 0.9139  Epoch 00027: val_loss improved from 0.27509 to 0.24825, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.027-0.2482.hdf5
122/122 [==============================] - 396s - loss: 1.6986 - acc: 0.9143 - val_loss: 0.2482 - val_acc: 0.9273
Epoch 29/200
121/122 [============================>.] - ETA: 2s - loss: 1.5197 - acc: 0.9194  Epoch 00028: val_loss improved from 0.24825 to 0.23384, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.028-0.2338.hdf5
122/122 [==============================] - 396s - loss: 1.5127 - acc: 0.9196 - val_loss: 0.2338 - val_acc: 0.9298
Epoch 30/200
121/122 [============================>.] - ETA: 2s - loss: 1.2687 - acc: 0.9314  Epoch 00029: val_loss improved from 0.23384 to 0.21322, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.029-0.2132.hdf5
122/122 [==============================] - 397s - loss: 1.2748 - acc: 0.9315 - val_loss: 0.2132 - val_acc: 0.9394
Epoch 31/200
121/122 [============================>.] - ETA: 2s - loss: 1.1911 - acc: 0.9333  Epoch 00030: val_loss did not improve
122/122 [==============================] - 396s - loss: 1.1950 - acc: 0.9334 - val_loss: 0.2213 - val_acc: 0.9384
Epoch 32/200
121/122 [============================>.] - ETA: 2s - loss: 1.0873 - acc: 0.9366  Epoch 00031: val_loss improved from 0.21322 to 0.20865, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.031-0.2086.hdf5
122/122 [==============================] - 397s - loss: 1.0859 - acc: 0.9364 - val_loss: 0.2086 - val_acc: 0.9366
Epoch 33/200
121/122 [============================>.] - ETA: 2s - loss: 0.9980 - acc: 0.9432  Epoch 00032: val_loss improved from 0.20865 to 0.18935, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.032-0.1893.hdf5
122/122 [==============================] - 396s - loss: 0.9942 - acc: 0.9436 - val_loss: 0.1893 - val_acc: 0.9454
Epoch 34/200
121/122 [============================>.] - ETA: 2s - loss: 0.9025 - acc: 0.9482  Epoch 00033: val_loss improved from 0.18935 to 0.18435, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.033-0.1844.hdf5
122/122 [==============================] - 397s - loss: 0.9022 - acc: 0.9483 - val_loss: 0.1844 - val_acc: 0.9454
Epoch 35/200
121/122 [============================>.] - ETA: 2s - loss: 0.7573 - acc: 0.9522  Epoch 00034: val_loss improved from 0.18435 to 0.18373, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.034-0.1837.hdf5
122/122 [==============================] - 397s - loss: 0.7561 - acc: 0.9520 - val_loss: 0.1837 - val_acc: 0.9472
Epoch 36/200
121/122 [============================>.] - ETA: 2s - loss: 0.7396 - acc: 0.9542  Epoch 00035: val_loss improved from 0.18373 to 0.17409, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.035-0.1741.hdf5
122/122 [==============================] - 397s - loss: 0.7373 - acc: 0.9540 - val_loss: 0.1741 - val_acc: 0.9526
Epoch 37/200
121/122 [============================>.] - ETA: 2s - loss: 0.7014 - acc: 0.9553  Epoch 00036: val_loss improved from 0.17409 to 0.17403, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.036-0.1740.hdf5
122/122 [==============================] - 397s - loss: 0.7020 - acc: 0.9552 - val_loss: 0.1740 - val_acc: 0.9526
Epoch 38/200
121/122 [============================>.] - ETA: 2s - loss: 0.6516 - acc: 0.9578  Epoch 00037: val_loss improved from 0.17403 to 0.16262, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.037-0.1626.hdf5
122/122 [==============================] - 397s - loss: 0.6487 - acc: 0.9579 - val_loss: 0.1626 - val_acc: 0.9542
Epoch 39/200
121/122 [============================>.] - ETA: 2s - loss: 0.6168 - acc: 0.9606  Epoch 00038: val_loss improved from 0.16262 to 0.15674, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.038-0.1567.hdf5
122/122 [==============================] - 397s - loss: 0.6163 - acc: 0.9607 - val_loss: 0.1567 - val_acc: 0.9560
Epoch 40/200
121/122 [============================>.] - ETA: 2s - loss: 0.5454 - acc: 0.9647  Epoch 00039: val_loss improved from 0.15674 to 0.14514, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.039-0.1451.hdf5
122/122 [==============================] - 397s - loss: 0.5436 - acc: 0.9646 - val_loss: 0.1451 - val_acc: 0.9586
Epoch 41/200
121/122 [============================>.] - ETA: 2s - loss: 0.5332 - acc: 0.9651  Epoch 00040: val_loss improved from 0.14514 to 0.14024, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.040-0.1402.hdf5
122/122 [==============================] - 397s - loss: 0.5315 - acc: 0.9651 - val_loss: 0.1402 - val_acc: 0.9607
Epoch 42/200
121/122 [============================>.] - ETA: 2s - loss: 0.4407 - acc: 0.9693  Epoch 00041: val_loss did not improve
122/122 [==============================] - 396s - loss: 0.4457 - acc: 0.9689 - val_loss: 0.1523 - val_acc: 0.9583
Epoch 43/200
121/122 [============================>.] - ETA: 2s - loss: 0.4091 - acc: 0.9680  Epoch 00042: val_loss improved from 0.14024 to 0.13319, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.042-0.1332.hdf5
122/122 [==============================] - 397s - loss: 0.4092 - acc: 0.9679 - val_loss: 0.1332 - val_acc: 0.9630
Epoch 44/200
121/122 [============================>.] - ETA: 2s - loss: 0.3964 - acc: 0.9720  Epoch 00043: val_loss did not improve
122/122 [==============================] - 396s - loss: 0.3948 - acc: 0.9720 - val_loss: 0.1352 - val_acc: 0.9609
Epoch 45/200
121/122 [============================>.] - ETA: 2s - loss: 0.3366 - acc: 0.9747  Epoch 00044: val_loss improved from 0.13319 to 0.13070, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.044-0.1307.hdf5
122/122 [==============================] - 397s - loss: 0.3349 - acc: 0.9747 - val_loss: 0.1307 - val_acc: 0.9651
Epoch 46/200
121/122 [============================>.] - ETA: 2s - loss: 0.3189 - acc: 0.9759  Epoch 00045: val_loss improved from 0.13070 to 0.12413, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.045-0.1241.hdf5
122/122 [==============================] - 397s - loss: 0.3178 - acc: 0.9758 - val_loss: 0.1241 - val_acc: 0.9653
Epoch 47/200
121/122 [============================>.] - ETA: 2s - loss: 0.2903 - acc: 0.9760  Epoch 00046: val_loss improved from 0.12413 to 0.12202, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.046-0.1220.hdf5
122/122 [==============================] - 397s - loss: 0.2929 - acc: 0.9761 - val_loss: 0.1220 - val_acc: 0.9666
Epoch 48/200
121/122 [============================>.] - ETA: 2s - loss: 0.3255 - acc: 0.9754  Epoch 00047: val_loss did not improve
122/122 [==============================] - 396s - loss: 0.3256 - acc: 0.9754 - val_loss: 0.1239 - val_acc: 0.9679
Epoch 49/200
121/122 [============================>.] - ETA: 2s - loss: 0.2920 - acc: 0.9771  Epoch 00048: val_loss improved from 0.12202 to 0.11761, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.048-0.1176.hdf5
122/122 [==============================] - 397s - loss: 0.2928 - acc: 0.9770 - val_loss: 0.1176 - val_acc: 0.9689
Epoch 50/200
121/122 [============================>.] - ETA: 2s - loss: 0.2751 - acc: 0.9786  Epoch 00049: val_loss improved from 0.11761 to 0.11230, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.049-0.1123.hdf5
122/122 [==============================] - 397s - loss: 0.2745 - acc: 0.9787 - val_loss: 0.1123 - val_acc: 0.9718
Epoch 51/200
121/122 [============================>.] - ETA: 2s - loss: 0.2211 - acc: 0.9799  Epoch 00050: val_loss did not improve
122/122 [==============================] - 397s - loss: 0.2205 - acc: 0.9800 - val_loss: 0.1146 - val_acc: 0.9700
Epoch 52/200
121/122 [============================>.] - ETA: 2s - loss: 0.2761 - acc: 0.9773  Epoch 00051: val_loss improved from 0.11230 to 0.11150, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.051-0.1115.hdf5
122/122 [==============================] - 397s - loss: 0.2763 - acc: 0.9771 - val_loss: 0.1115 - val_acc: 0.9702
Epoch 53/200
121/122 [============================>.] - ETA: 2s - loss: 0.2081 - acc: 0.9809  Epoch 00052: val_loss improved from 0.11150 to 0.10400, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.052-0.1040.hdf5
122/122 [==============================] - 397s - loss: 0.2071 - acc: 0.9810 - val_loss: 0.1040 - val_acc: 0.9736
Epoch 54/200
121/122 [============================>.] - ETA: 2s - loss: 0.2081 - acc: 0.9810  Epoch 00053: val_loss did not improve
122/122 [==============================] - 397s - loss: 0.2109 - acc: 0.9807 - val_loss: 0.1044 - val_acc: 0.9723
Epoch 55/200
121/122 [============================>.] - ETA: 2s - loss: 0.2108 - acc: 0.9822  Epoch 00054: val_loss improved from 0.10400 to 0.10020, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.054-0.1002.hdf5
122/122 [==============================] - 397s - loss: 0.2099 - acc: 0.9823 - val_loss: 0.1002 - val_acc: 0.9736
Epoch 56/200
121/122 [============================>.] - ETA: 2s - loss: 0.1709 - acc: 0.9827  Epoch 00055: val_loss did not improve
122/122 [==============================] - 396s - loss: 0.1698 - acc: 0.9828 - val_loss: 0.1004 - val_acc: 0.9728
Epoch 57/200
121/122 [============================>.] - ETA: 2s - loss: 0.1777 - acc: 0.9835  Epoch 00056: val_loss improved from 0.10020 to 0.09666, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.056-0.0967.hdf5
122/122 [==============================] - 397s - loss: 0.1772 - acc: 0.9834 - val_loss: 0.0967 - val_acc: 0.9749
Epoch 58/200
121/122 [============================>.] - ETA: 2s - loss: 0.1632 - acc: 0.9842  Epoch 00057: val_loss did not improve
122/122 [==============================] - 396s - loss: 0.1630 - acc: 0.9844 - val_loss: 0.1017 - val_acc: 0.9739
Epoch 59/200
121/122 [============================>.] - ETA: 2s - loss: 0.1509 - acc: 0.9848  Epoch 00058: val_loss did not improve
122/122 [==============================] - 396s - loss: 0.1506 - acc: 0.9847 - val_loss: 0.1038 - val_acc: 0.9739
Epoch 60/200
121/122 [============================>.] - ETA: 2s - loss: 0.1520 - acc: 0.9853  Epoch 00059: val_loss improved from 0.09666 to 0.09607, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.059-0.0961.hdf5
122/122 [==============================] - 397s - loss: 0.1524 - acc: 0.9853 - val_loss: 0.0961 - val_acc: 0.9751
Epoch 61/200
121/122 [============================>.] - ETA: 2s - loss: 0.1319 - acc: 0.9874  Epoch 00060: val_loss did not improve
122/122 [==============================] - 396s - loss: 0.1313 - acc: 0.9874 - val_loss: 0.0966 - val_acc: 0.9744
Epoch 62/200
121/122 [============================>.] - ETA: 2s - loss: 0.1256 - acc: 0.9875  Epoch 00061: val_loss did not improve
122/122 [==============================] - 396s - loss: 0.1263 - acc: 0.9873 - val_loss: 0.1010 - val_acc: 0.9728
Epoch 63/200
121/122 [============================>.] - ETA: 2s - loss: 0.1472 - acc: 0.9870  Epoch 00062: val_loss improved from 0.09607 to 0.08979, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.062-0.0898.hdf5
122/122 [==============================] - 397s - loss: 0.1488 - acc: 0.9869 - val_loss: 0.0898 - val_acc: 0.9770
Epoch 64/200
121/122 [============================>.] - ETA: 2s - loss: 0.1399 - acc: 0.9877  Epoch 00063: val_loss did not improve
122/122 [==============================] - 397s - loss: 0.1401 - acc: 0.9878 - val_loss: 0.0914 - val_acc: 0.9764
Epoch 65/200
121/122 [============================>.] - ETA: 2s - loss: 0.1343 - acc: 0.9873  Epoch 00064: val_loss did not improve
122/122 [==============================] - 396s - loss: 0.1339 - acc: 0.9873 - val_loss: 0.0916 - val_acc: 0.9739
Epoch 66/200
121/122 [============================>.] - ETA: 2s - loss: 0.1315 - acc: 0.9866  Epoch 00065: val_loss did not improve
122/122 [==============================] - 397s - loss: 0.1308 - acc: 0.9867 - val_loss: 0.0924 - val_acc: 0.9762
Epoch 67/200
121/122 [============================>.] - ETA: 2s - loss: 0.1119 - acc: 0.9879  Epoch 00066: val_loss did not improve
122/122 [==============================] - 397s - loss: 0.1119 - acc: 0.9878 - val_loss: 0.0967 - val_acc: 0.9757
Epoch 68/200
121/122 [============================>.] - ETA: 2s - loss: 0.1074 - acc: 0.9894  Epoch 00067: val_loss did not improve
122/122 [==============================] - 397s - loss: 0.1070 - acc: 0.9894 - val_loss: 0.0947 - val_acc: 0.9764
Epoch 69/200
121/122 [============================>.] - ETA: 2s - loss: 0.1033 - acc: 0.9893  Epoch 00068: val_loss did not improve

Epoch 00068: reducing learning rate to 9.99999974738e-07.
122/122 [==============================] - 399s - loss: 0.1030 - acc: 0.9894 - val_loss: 0.0951 - val_acc: 0.9754
Epoch 70/200
121/122 [============================>.] - ETA: 2s - loss: 0.0917 - acc: 0.9904  Epoch 00069: val_loss improved from 0.08979 to 0.08944, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.069-0.0894.hdf5
122/122 [==============================] - 397s - loss: 0.0921 - acc: 0.9905 - val_loss: 0.0894 - val_acc: 0.9770
Epoch 71/200
121/122 [============================>.] - ETA: 2s - loss: 0.0805 - acc: 0.9902  Epoch 00070: val_loss improved from 0.08944 to 0.08801, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.070-0.0880.hdf5
122/122 [==============================] - 397s - loss: 0.0805 - acc: 0.9901 - val_loss: 0.0880 - val_acc: 0.9770
Epoch 72/200
121/122 [============================>.] - ETA: 2s - loss: 0.0823 - acc: 0.9915  Epoch 00071: val_loss did not improve
122/122 [==============================] - 396s - loss: 0.0818 - acc: 0.9915 - val_loss: 0.0881 - val_acc: 0.9777
Epoch 73/200
121/122 [============================>.] - ETA: 2s - loss: 0.0839 - acc: 0.9910  Epoch 00072: val_loss did not improve
122/122 [==============================] - 397s - loss: 0.0840 - acc: 0.9910 - val_loss: 0.0882 - val_acc: 0.9777
Epoch 74/200
121/122 [============================>.] - ETA: 2s - loss: 0.0727 - acc: 0.9905  Epoch 00073: val_loss did not improve
122/122 [==============================] - 397s - loss: 0.0722 - acc: 0.9906 - val_loss: 0.0888 - val_acc: 0.9775
Epoch 75/200
121/122 [============================>.] - ETA: 2s - loss: 0.0984 - acc: 0.9903  Epoch 00074: val_loss improved from 0.08801 to 0.08757, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.074-0.0876.hdf5
122/122 [==============================] - 397s - loss: 0.1006 - acc: 0.9903 - val_loss: 0.0876 - val_acc: 0.9772
Epoch 76/200
121/122 [============================>.] - ETA: 2s - loss: 0.0776 - acc: 0.9901  Epoch 00075: val_loss did not improve
122/122 [==============================] - 396s - loss: 0.0794 - acc: 0.9901 - val_loss: 0.0895 - val_acc: 0.9767
Epoch 77/200
121/122 [============================>.] - ETA: 2s - loss: 0.0727 - acc: 0.9916  Epoch 00076: val_loss did not improve

Epoch 00076: reducing learning rate to 9.99999997475e-08.
122/122 [==============================] - 396s - loss: 0.0725 - acc: 0.9915 - val_loss: 0.0881 - val_acc: 0.9767
Epoch 78/200
121/122 [============================>.] - ETA: 2s - loss: 0.0819 - acc: 0.9893  Epoch 00077: val_loss did not improve
122/122 [==============================] - 396s - loss: 0.0825 - acc: 0.9892 - val_loss: 0.0882 - val_acc: 0.9770
Epoch 79/200
121/122 [============================>.] - ETA: 2s - loss: 0.0702 - acc: 0.9913  Epoch 00078: val_loss did not improve
122/122 [==============================] - 397s - loss: 0.0701 - acc: 0.9912 - val_loss: 0.0886 - val_acc: 0.9767
Epoch 80/200
121/122 [============================>.] - ETA: 2s - loss: 0.0747 - acc: 0.9919  Epoch 00079: val_loss improved from 0.08757 to 0.08747, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.079-0.0875.hdf5
122/122 [==============================] - 397s - loss: 0.0744 - acc: 0.9920 - val_loss: 0.0875 - val_acc: 0.9770
Epoch 81/200
121/122 [============================>.] - ETA: 2s - loss: 0.0825 - acc: 0.9911  Epoch 00080: val_loss did not improve
122/122 [==============================] - 396s - loss: 0.0820 - acc: 0.9912 - val_loss: 0.0880 - val_acc: 0.9770
Epoch 82/200
121/122 [============================>.] - ETA: 2s - loss: 0.0825 - acc: 0.9903  Epoch 00081: val_loss did not improve

Epoch 00081: reducing learning rate to 1.00000001169e-08.
122/122 [==============================] - 396s - loss: 0.0827 - acc: 0.9901 - val_loss: 0.0892 - val_acc: 0.9770
Epoch 83/200
121/122 [============================>.] - ETA: 2s - loss: 0.0844 - acc: 0.9916  Epoch 00082: val_loss improved from 0.08747 to 0.08736, saving model to ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.082-0.0874.hdf5
122/122 [==============================] - 397s - loss: 0.0843 - acc: 0.9916 - val_loss: 0.0874 - val_acc: 0.9770
Epoch 84/200
121/122 [============================>.] - ETA: 2s - loss: 0.0740 - acc: 0.9921  Epoch 00083: val_loss did not improve
122/122 [==============================] - 396s - loss: 0.0747 - acc: 0.9917 - val_loss: 0.0895 - val_acc: 0.9770
Epoch 85/200
121/122 [============================>.] - ETA: 2s - loss: 0.0728 - acc: 0.9917  Epoch 00084: val_loss did not improve
122/122 [==============================] - 397s - loss: 0.0732 - acc: 0.9916 - val_loss: 0.0885 - val_acc: 0.9770
Epoch 86/200
108/122 [=========================>....] - ETA: 40s - loss: 0.0720 - acc: 0.9899 





    




---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-10-2274f7d71d67> in <module>()
     24                     callbacks=[early_stopping, model_checkpoint, learningrate_schedule, tensorboard],
     25                     validation_data=(X_valid,y_valid), class_weight=class_weight,
---> 26                     workers=3, pickle_safe=True, initial_epoch=22)

/usr/local/lib64/python2.7/site-packages/keras/legacy/interfaces.pyc in wrapper(*args, **kwargs)
     86                 warnings.warn('Update your `' + object_name +
     87                               '` call to the Keras 2 API: ' + signature, stacklevel=2)
---> 88             return func(*args, **kwargs)
     89         wrapper._legacy_support_signature = inspect.getargspec(func)
     90         return wrapper

/usr/local/lib64/python2.7/site-packages/keras/engine/training.pyc in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_q_size, workers, pickle_safe, initial_epoch)
   1874                     outs = self.train_on_batch(x, y,
   1875                                                sample_weight=sample_weight,
-> 1876                                                class_weight=class_weight)
   1877 
   1878                     if not isinstance(outs, list):

/usr/local/lib64/python2.7/site-packages/keras/engine/training.pyc in train_on_batch(self, x, y, sample_weight, class_weight)
   1618             ins = x + y + sample_weights
   1619         self._make_train_function()
-> 1620         outputs = self.train_function(ins)
   1621         if len(outputs) == 1:
   1622             return outputs[0]

/usr/local/lib64/python2.7/site-packages/keras/backend/tensorflow_backend.pyc in __call__(self, inputs)
   2071         session = get_session()
   2072         updated = session.run(self.outputs + [self.updates_op],
-> 2073                               feed_dict=feed_dict)
   2074         return updated[:len(self.outputs)]
   2075 

/usr/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
    765     try:
    766       result = self._run(None, fetches, feed_dict, options_ptr,
--> 767                          run_metadata_ptr)
    768       if run_metadata:
    769         proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)

/usr/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
    963     if final_fetches or final_targets:
    964       results = self._do_run(handle, final_targets, final_fetches,
--> 965                              feed_dict_string, options, run_metadata)
    966     else:
    967       results = []

/usr/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
   1013     if handle is None:
   1014       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
-> 1015                            target_list, options, run_metadata)
   1016     else:
   1017       return self._do_call(_prun_fn, self._session, handle, feed_dict,

/usr/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
   1020   def _do_call(self, fn, *args):
   1021     try:
-> 1022       return fn(*args)
   1023     except errors.OpError as e:
   1024       message = compat.as_text(e.message)

/usr/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc in _run_fn(session, feed_dict, fetch_list, target_list, options, run_metadata)
   1002         return tf_session.TF_Run(session, options,
   1003                                  feed_dict, fetch_list, target_list,
-> 1004                                  status, run_metadata)
   1005 
   1006     def _prun_fn(session, handle, feed_dict, fetch_list):

KeyboardInterrupt: 

In [ ]:

    

#resume training

model, model_name = get_best_model()
# print('Loading model from weights.004-0.0565.hdf5')
# model = load_model('./checkpoints/checkpoint3/weights.004-0.0565.hdf5')

#remenber to set initial_epoch
optimizer = Adam(lr=1e-4)
model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])

model.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=200, verbose=1, 
                    callbacks=[early_stopping, model_checkpoint, learningrate_schedule, tensorboard], 
                    validation_data=(X_valid,y_valid), class_weight=class_weight, 
                    workers=3, pickle_safe=True, initial_epoch=83)


    

In [12]:

    

#test prepare

test_model, test_model_name = get_best_model()
# print('Loading model from weights.004-0.0565.hdf5')
# test_model = load_model('./checkpoints/checkpoint2/weights.004-0.0565.hdf5')

def test_generator(df, mean, datagen = None, batch_size = BATCHSIZE):
    n = df.shape[0]
    batch_index = 0
    while 1:
        current_index = batch_index * batch_size
        if n >= current_index + batch_size:
            current_batch_size = batch_size
            batch_index += 1    
        else:
            current_batch_size = n - current_index
            batch_index = 0        
        batch_df = df[current_index:current_index+current_batch_size]
        batch_x = np.zeros((batch_df.shape[0], ROWS, COLS, 3), dtype=K.floatx())
        i = 0
        for index,row in batch_df.iterrows():
            image_file = row['image_file']
            bbox = [row['xmin'],row['ymin'],row['xmax'],row['ymax']]
            cropped = load_img(TEST_DIR+image_file,bbox,target_size=(ROWS,COLS))
            x = np.asarray(cropped, dtype=K.floatx())
            x /= 255.
            if datagen is not None: x = datagen.random_transform(x)            
            x = preprocess_input(x, mean)
            batch_x[i] = x
            i += 1
        if batch_index%50 == 0: print('batch_index', batch_index)
        yield(batch_x)
        
test_aug_datagen = ImageDataGenerator(
    rotation_range=180,
    shear_range=0.2,
    zoom_range=0.1,
    width_shift_range=0.1,
    height_shift_range=0.1,
    horizontal_flip=True,
    vertical_flip=True)

train_mean = mean(X_valid)
print('train_mean', train_mean)


    

    




Loading model from checkpoint file ./resnet50_FT38_Classifier_Rep2/checkpoint/weights.082-0.0874.hdf5
Loading model Done!

In [17]:

    

# train_mean = [0.37698776,  0.41491762,  0.38681713]
train_mean = mean(X_valid)


    

In [26]:

    

#validation data fish logloss
 
valid_pred = test_model.predict(X_valid, batch_size=BATCHSIZE, verbose=1)
# valid_pred = test_model.predict_generator(test_generator(df=valid_df, mean=valid_mean),
#                                           val_samples=valid_df.shape[0], nb_worker=1, pickle_safe=False)
valid_logloss_df = pd.DataFrame(columns=['logloss','class'])
for i in range(y_valid.shape[0]):
    index = np.argmax(y_valid[i,:])
    fish = FISH_CLASSES[index]
    logloss = -math.log(valid_pred[i,index])
    valid_logloss_df.loc[len(valid_logloss_df)]=[logloss,fish]                                       
print(valid_logloss_df.groupby(['class'])['logloss'].mean())
print(valid_logloss_df['logloss'].mean())

train_pred = test_model.predict(X_train, batch_size=BATCHSIZE, verbose=1)
# train_pred = test_model.predict_generator(test_generator(df=train_df, ),
#                                           val_samples=train_df.shape[0], nb_worker=1, pickle_safe=False)
train_logloss_df = pd.DataFrame(columns=['logloss','class'])
for i in range(y_train.shape[0]):
    index = np.argmax(y_train[i,:])
    fish = FISH_CLASSES[index]
    logloss = -math.log(train_pred[i,index])
    train_logloss_df.loc[len(train_logloss_df)]=[logloss,fish]                                       
print(train_logloss_df.groupby(['class'])['logloss'].mean())
print(train_logloss_df['logloss'].mean())


    

    




3863/3863 [==============================] - 40s     
class
ALB      0.124283
BET      0.442586
DOL      0.678694
LAG      0.000818
NoF      0.061490
OTHER    0.289067
SHARK    0.000414
YFT      0.178863
Name: logloss, dtype: float64
0.0873555796313
15616/15616 [==============================] - 164s     
class
ALB      0.026392
BET      0.003495
DOL      0.001125
LAG      0.002566
NoF      0.026310
OTHER    0.009264
SHARK    0.006521
YFT      0.017942
Name: logloss, dtype: float64
0.0248458333809

In [ ]:

    

#GTbbox_CROPpred_df = ['image_file','crop_index','crop_class','xmin','ymin','xmax','ymax','split'
#                      'NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT', 'logloss']

file_name = 'GTbbox_CROPpred_df_'+test_model_name+'_.pickle'
if os.path.exists(OUTPUT_DIR+file_name):
    print ('Loading from file '+file_name)
    GTbbox_CROPpred_df = pd.read_pickle(OUTPUT_DIR+file_name)
else:
    print ('Generating file '+file_name) 
    nb_augmentation = 1
    if nb_augmentation ==1:
        test_preds = test_model.predict_generator(test_generator(df=GTbbox_df, mean=train_mean), 
                                                  steps=int(math.ceil(GTbbox_df.shape[0]/BATCHSIZE)), workers=1, pickle_safe=False)
    else:
        test_preds = np.zeros((GTbbox_df.shape[0], len(FISH_CLASSES)), dtype=K.floatx())
        for idx in range(nb_augmentation):
            print('{}th augmentation for testing ...'.format(idx+1))
            test_preds += test_model.predict_generator(test_generator(df=GTbbox_df, mean=train_mean, datagen=test_aug_datagen), 
                                                       steps=GTbbox_df.shape[0], workers=1, pickle_safe=False)
        test_preds /= nb_augmentation

    CROPpred_df = pd.DataFrame(test_preds, columns=FISH_CLASSES)
    GTbbox_CROPpred_df = pd.concat([GTbbox_df,CROPpred_df], axis=1)
    GTbbox_CROPpred_df['logloss'] = GTbbox_CROPpred_df.apply(lambda row: -math.log(row[row['crop_class']]), axis=1)
    GTbbox_CROPpred_df.to_pickle(OUTPUT_DIR+file_name) 

#logloss of every fish class
print(GTbbox_CROPpred_df.groupby(['crop_class'])['logloss'].mean())
print(GTbbox_CROPpred_df['logloss'].mean())


    

In [ ]:

    

Loading from file GTbbox_CROPpred_df_weights.000-0.0327.hdf5_.pickle
crop_class
ALB      0.076577
BET      0.139025
DOL      0.126520
LAG      0.000761
NoF      0.051943
OTHER    0.133949
SHARK    0.018328
YFT      0.090739
Name: logloss, dtype: float64
0.05936252677814113


    

In [6]:

    

# RFCNbbox_RFCNpred_df = ['image_class','image_file','crop_index','xmin','ymin','xmax','ymax',
#                          'NoF_RFCN', 'ALB_RFCN', 'BET_RFCN', 'DOL_RFCN',
#                          'LAG_RFCN', 'OTHER_RFCN', 'SHARK_RFCN', 'YFT_RFCN']
# select fish_conf >= CONF_THRESH

file_name = 'RFCNbbox_RFCNpred_df_conf{:.2f}.pickle'.format(CONF_THRESH)
if os.path.exists(OUTPUT_DIR+file_name):
    print ('Loading from file '+file_name)
    RFCNbbox_RFCNpred_df = pd.read_pickle(OUTPUT_DIR+file_name)
else:
    print ('Generating file '+file_name)        
    RFCNbbox_RFCNpred_df = pd.DataFrame(columns=['image_class','image_file','crop_index','xmin','ymin','xmax','ymax',
                                                  'NoF_RFCN', 'ALB_RFCN', 'BET_RFCN', 'DOL_RFCN',
                                                  'LAG_RFCN', 'OTHER_RFCN', 'SHARK_RFCN', 'YFT_RFCN']) 

    with open('../data/RFCN_detections/detections_full_AGNOSTICnms_'+RFCN_MODEL+'.pkl','rb') as f:
        detections_full_AGNOSTICnms = pickle.load(f, encoding='latin1') 
    with open("../RFCN/ImageSets/Main/test.txt","r") as f:
        test_files = f.readlines()
    with open("../RFCN/ImageSets/Main/train_test.txt","r") as f:
        train_file_labels = f.readlines()
    assert len(detections_full_AGNOSTICnms) == len(test_files)
    
    count = np.zeros(len(detections_full_AGNOSTICnms))
    
    for im in range(len(detections_full_AGNOSTICnms)):
        if im%1000 == 0: print(im)
        basename = test_files[im][:9]
        if im<1000:
            image_class = '--'
        else:
            for i in range(len(train_file_labels)):
                if train_file_labels[i][:9] == basename:
                    image_class = train_file_labels[i][10:-1]
                    break
        image = Image.open(TEST_DIR+'/'+basename+'.jpg')
        width_image, height_image = image.size
        
        bboxes = []
        detects_im = detections_full_AGNOSTICnms[im]
        for i in range(len(detects_im)):
#             if np.sum(detects_im[i,5:]) >= CONF_THRESH:
            if np.max(detects_im[i,5:]) >= CONF_THRESH:
                bboxes.append(detects_im[i,:]) 
        count[im] = len(bboxes)
        if len(bboxes) == 0:
            ind = np.argmax(np.sum(detects_im[:,5:], axis=1))
            bboxes.append(detects_im[ind,:])
        bboxes = np.asarray(bboxes)

        for j in range(len(bboxes)):    
            bbox = bboxes[j]
            xmin = bbox[0]
            ymin = bbox[1]
            xmax = bbox[2]
            ymax = bbox[3]
            assert max(xmin,0)<min(xmax,width_image)
            assert max(ymin,0)<min(ymax,height_image)
            RFCNbbox_RFCNpred_df.loc[len(RFCNbbox_RFCNpred_df)]=[image_class,basename+'.jpg',j,max(xmin,0),max(ymin,0),
                                                                   min(xmax,width_image),min(ymax,height_image),
                                                                   bbox[4],bbox[5],bbox[6],bbox[7],bbox[8],bbox[9],bbox[10],bbox[11]]   
    
    RFCNbbox_RFCNpred_df.to_pickle(OUTPUT_DIR+file_name)


    

    




Loading from file RFCNbbox_RFCNpred_df_conf0.80.pickle

In [32]:

    

file_name = 'data_trainfish_Crop_{}_{}.pickle'.format(ROWS, COLS)
if os.path.exists(OUTPUT_DIR+file_name):
    print ('Loading from file '+file_name)
    with open(OUTPUT_DIR+file_name, 'rb') as f:
        data_trainfish = pickle.load(f)
    X_trainfish_crop = data_train['X_trainfish_crop']
else:
    print ('Generating file '+file_name)

    GTbbox_CROPpred_fish_df = GTbbox_CROPpred_df.loc[GTbbox_CROPpred_df['crop_class']!='NoF']
    X_trainfish_crop = np.ndarray((GTbbox_CROPpred_fish_df.shape[0], ROWS, COLS, 3), dtype=np.uint8)
    i = 0
    for index,row in GTbbox_CROPpred_fish_df.iterrows():
        image_file = row['image_file']
        bbox = [row['xmin'],row['ymin'],row['xmax'],row['ymax']]
        cropped = load_img(TEST_DIR+image_file,bbox,target_size=(ROWS,COLS))
        X_trainfish_crop[i] = np.asarray(cropped)
        i += 1
   
    #save data to file
    data_trainfish = {'X_trainfish_crop': X_trainfish_crop}
    with open(OUTPUT_DIR+file_name, 'wb') as f:
        pickle.dump(data_trainfish, f)
        
print('Loading data done.')
X_trainfish_crop = X_trainfish_crop.astype(np.float32)
print('Convert to float32 done.')
X_trainfish_crop /= 255.
print('Rescale by 255 done.')


    

    




Generating file data_trainfish_Crop_224_224.pickle
Loading data done.
Convert to float32 done.
Rescale by 255 done.

In [33]:

    

mean(X_trainfish_crop)


    

    
Out[33]:





array([ 0.40706199,  0.4373979 ,  0.39489502], dtype=float32)

In [28]:

    

mean(X_test_crop[1251:])


    

    
Out[28]:





array([ 0.41078389,  0.43895897,  0.39912957], dtype=float32)

In [7]:

    

# test_mean = [0.41019869,  0.43978861,  0.39873621]
test_mean = [0.37698776,  0.41491762,  0.38681713]


    

In [8]:

    

# RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df = ['image_class', 'image_file','crop_index','xmin','ymin','xmax','ymax',
#                                    'NoF_RFCN', 'ALB_RFCN', 'BET_RFCN', 'DOL_RFCN',
#                                    'LAG_RFCN', 'OTHER_RFCN', 'SHARK_RFCN', 'YFT_RFCN',
#                                    'NoF_CROP', 'ALB_CROP', 'BET_CROP', 'DOL_CROP',
#                                    'LAG_CROP', 'OTHER_CROP', 'SHARK_CROP', 'YFT_CROP',
#                                    'NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']

file_name = 'RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df_'+test_model_name+'_.pickle'
if os.path.exists(OUTPUT_DIR+file_name):
    print ('Loading from file '+file_name)
    RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df = pd.read_pickle(OUTPUT_DIR+file_name)
else:
    print ('Generating file '+file_name)  
    nb_augmentation = 1
    if nb_augmentation ==1:
        test_preds = test_model.predict_generator(test_generator(df=RFCNbbox_RFCNpred_df, mean=test_mean), 
                                                  val_samples=RFCNbbox_RFCNpred_df.shape[0], nb_worker=1, pickle_safe=False)
    else:
        test_preds = np.zeros((RFCNbbox_RFCNpred_df.shape[0], len(FISH_CLASSES)), dtype=K.floatx())
        for idx in range(nb_augmentation):
            print('{}th augmentation for testing ...'.format(idx+1))
            test_preds += test_model.predict_generator(test_generator(df=RFCNbbox_RFCNpred_df, mean=test_mean, datagen=test_aug_datagen), 
                                                       val_samples=RFCNbbox_RFCNpred_df.shape[0], nb_worker=1, pickle_safe=False)
        test_preds /= nb_augmentation

    CROPpred_df = pd.DataFrame(test_preds, columns=['ALB_CROP', 'BET_CROP', 'DOL_CROP', 'LAG_CROP', 'NoF_CROP', 'OTHER_CROP', 'SHARK_CROP', 'YFT_CROP'])
    RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df = pd.concat([RFCNbbox_RFCNpred_df,CROPpred_df], axis=1)
    
    RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df['NoF'] = RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df['NoF_RFCN']
    for fish in ['ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']:
        RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df[fish] = RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df.apply(lambda row: (1-row['NoF_RFCN'])*row[[fish+'_CROP']]/(1-row['NoF_CROP']) if row['NoF_CROP']!=1 else 0, axis=1)
#     for fish in FISH_CLASSES:
#         RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df[fish] = RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df[fish+'_CROP']

    RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df.to_pickle(OUTPUT_DIR+file_name)


    

    




Loading from file RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df_weights.000-0.0327.hdf5_.pickle

In [9]:

    

# clsMaxAve and hybrid RFCNpred&CROPpred such that RFCNpred for NoF and CROPpred for fish
# test_pred_df = ['logloss','image_class','image_file','NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']
# RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df = ['image_class', 'image_file','crop_index','xmin','ymin','xmax','ymax',
#                                    'NoF_RFCN', 'ALB_RFCN', 'BET_RFCN', 'DOL_RFCN',
#                                    'LAG_RFCN', 'OTHER_RFCN', 'SHARK_RFCN', 'YFT_RFCN',
#                                    'ALB_CROP', 'BET_CROP', 'DOL_CROP',
#                                    'LAG_CROP', 'OTHER_CROP', 'SHARK_CROP', 'YFT_CROP',
#                                    'NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']

file_name = 'test_pred_df_Hybrid_'+test_model_name+'_.pickle'
if os.path.exists(OUTPUT_DIR+file_name):
    print ('Loading from file '+file_name)
    test_pred_df = pd.read_pickle(OUTPUT_DIR+file_name)
else:
    print ('Generating file '+file_name)  
    with open("../RFCN/ImageSets/Main/test.txt","r") as f:
        test_files = f.readlines()
    
    test_pred_df = pd.DataFrame(columns=['logloss','image_class','image_file','NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT'])  
    for j in range(len(test_files)): 
        image_file = test_files[j][:-1]+'.jpg'
        test_pred_im_df = RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df.loc[RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df['image_file'] == image_file,
                                                                       ['image_class', 'NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']]
        image_class = test_pred_im_df.iloc[0]['image_class']
        test_pred_im_df.drop('image_class', axis=1, inplace=True)
        max_score = test_pred_im_df.max(axis=1)
        max_cls = test_pred_im_df.idxmax(axis=1)
        test_pred_im_df['max_score'] = max_score
        test_pred_im_df['max_cls'] = max_cls
        test_pred_im_df['Count'] = test_pred_im_df.groupby(['max_cls'])['max_cls'].transform('count')
        idx = test_pred_im_df.groupby(['max_cls'])['max_score'].transform(max) == test_pred_im_df['max_score']
        test_pred_im_clsMax_df = test_pred_im_df.loc[idx,['NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT', 'Count']]
        test_pred_im_clsMax_array = test_pred_im_clsMax_df.values
        pred = np.average(test_pred_im_clsMax_array[:,:-1], axis=0, weights=test_pred_im_clsMax_array[:,-1], returned=False).tolist()
        if image_class!='--':
            ind = FISH_CLASSES.index(image_class)
            logloss = -math.log(pred[ind]) 
        else:
            logloss = np.nan
        test_pred_im_clsMaxAve = [logloss,image_class,image_file]
        test_pred_im_clsMaxAve.extend(pred)
        test_pred_df.loc[len(test_pred_df)]=test_pred_im_clsMaxAve

    test_pred_df.to_pickle(OUTPUT_DIR+file_name)


    

    




Loading from file test_pred_df_Hybrid_weights.000-0.0327.hdf5_.pickle

In [ ]:

    

#### visualization
# RFCNbbox_RFCNpred_CROPpred_df = ['image_class', 'image_file','crop_index','x_min','y_min','x_max','ymax',
#                                    'NoF_RFCN', 'ALB_RFCN', 'BET_RFCN', 'DOL_RFCN',
#                                    'LAG_RFCN', 'OTHER_RFCN', 'SHARK_RFCN', 'YFT_RFCN'
#                                    'NoF_CROP', 'ALB_CROP', 'BET_CROP', 'DOL_CROP',
#                                    'LAG_CROP', 'OTHER_CROP', 'SHARK_CROP', 'YFT_CROP']
#GTbbox_CROPpred_df = ['image_file','crop_index','crop_class','xmin','ymin','xmax','ymax',
#                      'NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT', 'logloss']
# test_pred_df = ['logloss','image_class','image_file','NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']

for j in range(test_pred_df.shape[0]):
    image_logloss = test_pred_df.iat[j,0]
    image_class = test_pred_df.iat[j,1]
    image_file = test_pred_df.iat[j,2]
    if j<1000 and j%30== 0:
        pass
    else: 
        continue
    im = Image.open('../RFCN/JPEGImages/'+image_file)
    im = np.asarray(im)
    fig, ax = plt.subplots(figsize=(10, 8))
    ax.imshow(im, aspect='equal')
    RFCN_dets = RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df.loc[RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df['image_file']==image_file]
    for index,row in RFCN_dets.iterrows():
        bbox = [row['xmin'],row['ymin'],row['xmax'],row['ymax']]
        RFCN = [row['NoF_RFCN'],row['ALB_RFCN'],row['BET_RFCN'],row['DOL_RFCN'],row['LAG_RFCN'],row['OTHER_RFCN'],row['SHARK_RFCN'],row['YFT_RFCN']]
        CROP = [row['NoF'],row['ALB'],row['BET'],row['DOL'],row['LAG'],row['OTHER'],row['SHARK'],row['YFT']]
        score_RFCN = max(RFCN)
        score_CROP = max(CROP)
        index_RFCN = RFCN.index(score_RFCN)
        index_CROP = CROP.index(score_CROP)
        class_RFCN = FISH_CLASSES[index_RFCN]
        class_CROP = FISH_CLASSES[index_CROP]
        ax.add_patch(plt.Rectangle((bbox[0], bbox[1]), bbox[2] - bbox[0], bbox[3] - bbox[1], fill=False, edgecolor='red', linewidth=2))
        ax.text(bbox[0], bbox[1] - 2, 'RFCN_{:s} {:.3f} \nHYBRID_{:s} {:.3f}'.format(class_RFCN, score_RFCN, class_CROP, score_CROP), bbox=dict(facecolor='red', alpha=0.5), fontsize=8, color='white')       
    GT_dets = GTbbox_CROPpred_df.loc[GTbbox_CROPpred_df['image_file']==image_file]
    for index,row in GT_dets.iterrows():
        bbox = [row['xmin'],row['ymin'],row['xmax'],row['ymax']]
        CROP = [row['NoF'],row['ALB'],row['BET'],row['DOL'],row['LAG'],row['OTHER'],row['SHARK'],row['YFT']]
        score_CROP = max(CROP)
        index_CROP = CROP.index(score_CROP)
        class_CROP = FISH_CLASSES[index_CROP]
        ax.add_patch(plt.Rectangle((bbox[0], bbox[1]), bbox[2] - bbox[0], bbox[3] - bbox[1], fill=False, edgecolor='green', linewidth=2))
        ax.text(bbox[0], bbox[3] + 40, 'GT_{:s} \nCROP_{:s} {:.3f}'.format(row['crop_class'], class_CROP, score_CROP), bbox=dict(facecolor='green', alpha=0.5), fontsize=8, color='white')
    ax.set_title(('Image {:s}    FISH {:s}    logloss {}').format(image_file, image_class, image_logloss), fontsize=10) 
    plt.axis('off')
    plt.tight_layout()
    plt.draw()


    

In [11]:

    

#temperature
T = 2.5
test_pred_array = test_pred_df[FISH_CLASSES].values
test_pred_T_array = np.exp(np.log(test_pred_array)/T)
test_pred_T_array = test_pred_T_array/np.sum(test_pred_T_array, axis=1, keepdims=True)
test_pred_T_df = pd.DataFrame(test_pred_T_array, columns=FISH_CLASSES)
test_pred_T_df = pd.concat([test_pred_df[['image_class','image_file']],test_pred_T_df], axis=1)

#add logloss
test_pred_T_df['logloss'] = test_pred_T_df.apply(lambda row: -math.log(row[row['image_class']]) if row['image_class']!='--' else np.nan, axis=1)

#calculate train logloss
print(test_pred_T_df.groupby(['image_class'])['logloss'].mean())
train_logloss = test_pred_T_df['logloss'].mean()
print('logloss of train is', train_logloss )


    

    




image_class
--            NaN
ALB      0.151489
BET      0.204419
DOL      0.141192
LAG      0.052902
NoF      0.125777
OTHER    0.128937
SHARK    0.180230
YFT      0.147513
Name: logloss, dtype: float64
logloss of train is 0.14783966909064958

In [ ]:

    

#test submission
submission = test_pred_T_df.loc[:999,['image_file','NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']]
submission.rename(columns={'image_file':'image'}, inplace=True)
sub_file = 'RFCN_AGONOSTICnms_'+RFCN_MODEL+'_'+CROP_MODEL+'_clsMaxAve_conf{:.2f}_T{}_'.format(CONF_THRESH, T)+'{:.4f}'.format(train_logloss)+'.csv'
submission.to_csv(sub_file, index=False)
print('Done!'+sub_file)


    

    




Done!RFCN_AGONOSTICnms_resnet101_rfcn_ohem_iter_30000_resnet50_FT38_Classifier_Rep_clsMaxAve_conf0.80_T2.5_0.1478.csv

In [ ]: