In [1]:
import os, random, glob, pickle, collections, math, json
import numpy as np
import pandas as pd
from __future__ import division
from __future__ import print_function
from PIL import Image
from sklearn.model_selection import train_test_split
from sklearn.metrics import log_loss
from sklearn.preprocessing import LabelEncoder
import matplotlib.pyplot as plt
%matplotlib inline
from keras.models import Sequential, Model, load_model, model_from_json
from keras import layers
from keras.layers import GlobalAveragePooling2D, Flatten, Dropout, Dense, LeakyReLU, Conv2D, Input, BatchNormalization, Activation
from keras.regularizers import l2
from keras.optimizers import Adam, RMSprop
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau, TensorBoard
from keras.preprocessing.image import ImageDataGenerator
from keras.utils import np_utils
from keras.preprocessing import image
from keras import backend as K
K.set_image_dim_ordering('tf')
In [2]:
TRAIN_DIR = '../data/train/'
TEST_DIR = '../RFCN/JPEGImages/'
TRAIN_CROP_DIR = '../data/train_crop/'
TEST_CROP_DIR = '../data/test_stg1_crop/'
RFCN_MODEL = 'resnet101_rfcn_ohem_iter_30000'
CROP_MODEL = 'resnet55ss3_Hybrid_woNoF'
if not os.path.exists('./' + CROP_MODEL):
os.mkdir('./' + CROP_MODEL)
CHECKPOINT_DIR = './' + CROP_MODEL + '/checkpoint/'
if not os.path.exists(CHECKPOINT_DIR):
os.mkdir(CHECKPOINT_DIR)
LOG_DIR = './' + CROP_MODEL + '/log/'
if not os.path.exists(LOG_DIR):
os.mkdir(LOG_DIR)
OUTPUT_DIR = './' + CROP_MODEL + '/output/'
if not os.path.exists(OUTPUT_DIR):
os.mkdir(OUTPUT_DIR)
FISH_CLASSES = ['NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']
CROP_CLASSES=FISH_CLASSES[:]
CROP_CLASSES.remove('NoF')
CONF_THRESH = 0.8
ROWS = 224
COLS = 224
BATCHSIZE = 128
LEARNINGRATE = 1e-4
def featurewise_center(x):
mean = np.mean(x, axis=0, keepdims=True)
mean = np.mean(mean, axis=(1,2), keepdims=True)
x_centered = x - mean
return x_centered
def featurewise_mean(x):
mean = np.mean(x, axis=0)
mean = np.mean(mean, axis=(0,1))
return mean
def preprocess_featurewise(x, featurewise_mean):
x -= np.reshape(featurewise_mean, [1, 1, 3])
return x
def load_img(path, bbox, target_size=None):
img = Image.open(path)
cropped = img.crop((bbox[0],bbox[1],bbox[2],bbox[3]))
width_cropped, height_cropped = cropped.size
if height_cropped > width_cropped: cropped = cropped.transpose(method=2)
if target_size:
cropped = cropped.resize((target_size[1], target_size[0]), Image.BILINEAR)
return cropped
def get_best_model(checkpoint_dir = CHECKPOINT_DIR):
files = glob.glob(checkpoint_dir+'*')
val_losses = [float(f.split('-')[-1][:-5]) for f in files]
index = val_losses.index(min(val_losses))
print('Loading model from checkpoint file ' + files[index])
model = load_model(files[index])
model_name = files[index].split('/')[-1]
print('Loading model Done!')
return (model, model_name)
def data_from_df(df):
X = np.ndarray((df.shape[0], ROWS, COLS, 3), dtype=np.uint8)
y = np.zeros((df.shape[0], len(CROP_CLASSES)), dtype=K.floatx())
i = 0
for index,row in df.iterrows():
image_file = row['image_file']
fish = row['crop_class']
bbox = [row['xmin'],row['ymin'],row['xmax'],row['ymax']]
cropped = load_img(TEST_DIR+image_file,bbox,target_size=(ROWS,COLS))
X[i] = np.asarray(cropped)
y[i,CROP_CLASSES.index(fish)] = 1
i += 1
return (X, y)
def data_load(name):
file_name = 'data_'+name+'_{}_{}.pickle'.format(ROWS, COLS)
if os.path.exists(OUTPUT_DIR+file_name):
print ('Loading from file '+file_name)
with open(OUTPUT_DIR+file_name, 'rb') as f:
data = pickle.load(f)
X = data['X']
y = data['y']
else:
print ('Generating file '+file_name)
if name=='train' or name=='valid':
df = GTbbox_df[GTbbox_df['split']==name]
elif name=='all':
df = GTbbox_df
else:
print('Invalid name '+name)
X, y = data_from_df(df)
data = {'X': X,'y': y}
with open(OUTPUT_DIR+file_name, 'wb') as f:
pickle.dump(data, f)
X = X.astype(np.float32)
X /= 255.
return (X, y)
In [3]:
# GTbbox_df = ['image_file','image_class','crop_index','crop_class','xmin',''ymin','xmax','ymax','split']
file_name = 'GTbbox_df.pickle'
if os.path.exists(OUTPUT_DIR+file_name):
print ('Loading from file '+file_name)
GTbbox_df = pd.read_pickle(OUTPUT_DIR+file_name)
else:
print ('Generating file '+file_name)
GTbbox_df = pd.DataFrame(columns=['image_file','image_class','crop_index','crop_class','xmin','ymin','xmax','ymax'])
crop_classes=FISH_CLASSES[:]
crop_classes.remove('NoF')
with open("../RFCN/ImageSets/Main/train_test.txt","r") as f:
train_file_labels = f.readlines()
for c in crop_classes:
print(c)
j = json.load(open('../data/BBannotations/{}.json'.format(c), 'r'))
for l in j:
filename = l["filename"]
head, image_file = os.path.split(filename)
basename, file_extension = os.path.splitext(image_file)
for i in range(len(train_file_labels)):
if train_file_labels[i][:9] == basename:
image_class = train_file_labels[i][10:-1]
break
image = Image.open(TEST_DIR+'/'+image_file)
width_image, height_image = image.size
for i in range(len(l["annotations"])):
a = l["annotations"][i]
xmin = (a["x"])
ymin = (a["y"])
width = (a["width"])
height = (a["height"])
xmax = xmin + width
ymax = ymin + height
assert max(xmin,0)<min(xmax,width_image)
assert max(ymin,0)<min(ymax,height_image)
GTbbox_df.loc[len(GTbbox_df)]=[image_file,image_class,i,a["class"],max(xmin,0),max(ymin,0),min(xmax,width_image),min(ymax,height_image)]
# test_size = GTbbox_df.shape[0]-int(math.ceil(GTbbox_df.shape[0]*0.8/128)*128)
# train_ind, valid_ind = train_test_split(range(GTbbox_df.shape[0]), test_size=test_size, random_state=1986, stratify=GTbbox_df['crop_class'])
# GTbbox_df['split'] = ['train' if i in train_ind else 'valid' for i in range(GTbbox_df.shape[0])]
image_files_df = GTbbox_df[['image_file','image_class']]
image_files_df.drop_duplicates(inplace = True)
test_file_nb = image_files_df.shape[0] - int(math.ceil(image_files_df.shape[0]*0.9))
train_files_df, valid_files_df = train_test_split(image_files_df, test_size=test_file_nb, random_state=1986, stratify=image_files_df.image_class)
GTbbox_df['split'] = ['train' if image_file in train_files_df.image_file.tolist() else 'valid' for image_file in GTbbox_df.image_file]
GTbbox_df.to_pickle(OUTPUT_DIR+file_name)
print('Done')
# GTbbox_df.loc[(GTbbox_df['image_class']!='NoF') & (GTbbox_df['crop_class']!='NoF') & (GTbbox_df['image_class']!=GTbbox_df['crop_class'])]
In [4]:
#Load data
X_all, y_all = data_load('all')
X_train, y_train = data_load('train')
X_valid, y_valid = data_load('valid')
print('Loading data done.')
print('all sample ', X_all.shape[0])
print('train sample ', X_train.shape[0])
print('valid sample ', X_valid.shape[0])
print('featurewise mean of X_all is ', featurewise_mean(X_all))
print('featurewise mean of X_train is ', featurewise_mean(X_train))
print('featurewise mean of X_valid is ', featurewise_mean(X_valid))
In [5]:
# #class weight = n_samples / (n_classes * np.bincount(y))
# class_weight_fish = dict(GTbbox_df.groupby('crop_class').size())
# class_weight = {}
# n_samples = GTbbox_df.shape[0]
# for key,value in class_weight_fish.items():
# class_weight[CROP_CLASSES.index(key)] = n_samples / (len(CROP_CLASSES)*value)
# class_weight
train_df = GTbbox_df[GTbbox_df['split']=='train']
class_weight_fish = dict(train_df.groupby('crop_class').size())
class_weight = {}
ref = max(class_weight_fish.values())
for key,value in class_weight_fish.items():
class_weight[CROP_CLASSES.index(key)] = ref/value
class_weight
Out[5]:
In [6]:
#data preprocessing
all_mean = featurewise_mean(X_all)
def preprocessing_function(x):
x -= np.reshape(all_mean, [1, 1, 3])
return x
train_datagen = ImageDataGenerator(
preprocessing_function=preprocessing_function,
rotation_range=180,
shear_range=0.2,
zoom_range=0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True,
vertical_flip=True)
train_generator = train_datagen.flow(X_train, y_train, batch_size=BATCHSIZE, shuffle=True, seed=None)
# assert X_train.shape[0]%BATCHSIZE==0
steps_per_epoch = int(math.floor(X_train.shape[0]/BATCHSIZE))
X_valid_centered = X_valid - np.reshape(all_mean, [1, 1, 1, 3])
In [7]:
#callbacks
early_stopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=250, verbose=1, mode='auto')
model_checkpoint = ModelCheckpoint(filepath=CHECKPOINT_DIR+'weights.{epoch:03d}-{val_loss:.4f}.hdf5', monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=False, mode='auto')
learningrate_schedule = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=100, verbose=1, mode='auto', epsilon=0.001, cooldown=0, min_lr=0)
tensorboard = TensorBoard(log_dir=LOG_DIR, histogram_freq=0, write_graph=False, write_images=True)
In [8]:
def identity_block(input_tensor, kernel_size, filters, stage, block):
"""The identity block is the block that has no conv layer at shortcut.
# Arguments
input_tensor: input tensor
kernel_size: defualt 3, the kernel size of middle conv layer at main path
filters: list of integers, the filterss of 3 conv layer at main path
stage: integer, current stage label, used for generating layer names
block: 'a','b'..., current block label, used for generating layer names
# Returns
Output tensor for the block.
"""
filters = filters
if K.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
x = Conv2D(filters, kernel_size, padding='same', name=conv_name_base + '2a',
kernel_initializer="he_normal", kernel_regularizer=l2(0.0001))(input_tensor)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x)
x = Activation('relu')(x)
x = Conv2D(filters, kernel_size, padding='same', name=conv_name_base + '2b',
kernel_initializer="he_normal", kernel_regularizer=l2(0.0001))(x)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x)
x = Activation('relu')(x)
x = layers.add([x, input_tensor])
x = Activation('relu')(x)
return x
def conv_block(input_tensor, kernel_size, filters, stage, block, strides=(2, 2)):
"""conv_block is the block that has a conv layer at shortcut
# Arguments
input_tensor: input tensor
kernel_size: defualt 3, the kernel size of middle conv layer at main path
filters: list of integers, the filterss of 3 conv layer at main path
stage: integer, current stage label, used for generating layer names
block: 'a','b'..., current block label, used for generating layer names
# Returns
Output tensor for the block.
Note that from stage 3, the first conv layer at main path is with strides=(2,2)
And the shortcut should have strides=(2,2) as well
"""
filters = filters
if K.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
x = Conv2D(filters, kernel_size, padding='same', strides=strides, name=conv_name_base + '2a',
kernel_initializer="he_normal", kernel_regularizer=l2(0.0001))(input_tensor)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x)
x = Activation('relu')(x)
x = Conv2D(filters, kernel_size, padding='same', name=conv_name_base + '2b',
kernel_initializer="he_normal", kernel_regularizer=l2(0.0001))(x)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x)
x = Activation('relu')(x)
shortcut = Conv2D(filters, (1, 1), strides=strides, name=conv_name_base + '1',
kernel_initializer="he_normal", kernel_regularizer=l2(0.0001))(input_tensor)
shortcut = BatchNormalization(axis=bn_axis, name=bn_name_base + '1')(shortcut)
x = layers.add([x, shortcut])
x = Activation('relu')(x)
return x
def create_model_resnet55ss():
img_input = Input(shape=(ROWS, COLS, 3))
x = Conv2D(16, (3, 3), strides=(2, 2), name='conv1', padding='same',
kernel_initializer="he_normal", kernel_regularizer=l2(0.0001))(img_input)
x = BatchNormalization(name='bn_conv1')(x)
x = Activation('relu')(x)
x = conv_block(x, 3, 16, stage=2, block='a')
x = identity_block(x, 3, 16, stage=2, block='b')
x = identity_block(x, 3, 16, stage=2, block='c')
x = identity_block(x, 3, 16, stage=2, block='d')
x = identity_block(x, 3, 16, stage=2, block='e')
x = identity_block(x, 3, 16, stage=2, block='f')
x = identity_block(x, 3, 16, stage=2, block='g')
x = identity_block(x, 3, 16, stage=2, block='h')
x = identity_block(x, 3, 16, stage=2, block='i')
x = conv_block(x, 3, 32, stage=3, block='a')
x = identity_block(x, 3, 32, stage=3, block='b')
x = identity_block(x, 3, 32, stage=3, block='c')
x = identity_block(x, 3, 32, stage=3, block='d')
x = identity_block(x, 3, 32, stage=3, block='e')
x = identity_block(x, 3, 32, stage=3, block='f')
x = identity_block(x, 3, 32, stage=3, block='g')
x = identity_block(x, 3, 32, stage=3, block='h')
x = identity_block(x, 3, 32, stage=3, block='i')
x = conv_block(x, 3, 64, stage=4, block='a')
x = identity_block(x, 3, 64, stage=4, block='b')
x = identity_block(x, 3, 64, stage=4, block='c')
x = identity_block(x, 3, 64, stage=4, block='d')
x = identity_block(x, 3, 64, stage=4, block='e')
x = identity_block(x, 3, 64, stage=4, block='f')
x = identity_block(x, 3, 64, stage=4, block='g')
x = identity_block(x, 3, 64, stage=4, block='h')
x = identity_block(x, 3, 64, stage=4, block='i')
# x = conv_block(x, 3, 128, stage=5, block='a')
# x = identity_block(x, 3, 128, stage=5, block='b')
# x = identity_block(x, 3, 128, stage=5, block='c')
x = GlobalAveragePooling2D()(x)
# model.add(Dropout(0.8))
x = Dense(len(CROP_CLASSES), activation='softmax')(x)
model = Model(img_input, x)
return model
In [ ]:
#train from scratch
model = create_model_resnet55ss()
# compile the model (should be done *after* setting layers to non-trainable)
optimizer = Adam(lr=1e-4)
model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])
# train the model on the new data for a few epochs
model.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=2000, verbose=1,
callbacks=[early_stopping, model_checkpoint, learningrate_schedule, tensorboard],
validation_data=(X_valid_centered,y_valid),
class_weight=class_weight, workers=3, pickle_safe=True)
In [9]:
#resume training
model, model_name = get_best_model()
# print('Loading model from weights.004-0.0565.hdf5')
# model = load_model(CHECKPOINT_DIR + 'weights.011-1.7062.hdf5')
# optimizer = Adam(lr=1e-4)
# model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])
model.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=2000, verbose=1,
callbacks=[early_stopping, model_checkpoint, learningrate_schedule, tensorboard],
validation_data=(X_valid_centered,y_valid),
class_weight=class_weight, workers=3, pickle_safe=True, initial_epoch=454)
Out[9]:
In [33]:
#test prepare
test_model, test_model_name = get_best_model()
# print('Loading model from weights.004-0.0565.hdf5')
# test_model = load_model('./checkpoints/checkpoint2/weights.004-0.0565.hdf5')
def test_generator(df, mean, datagen = None, batch_size = BATCHSIZE):
n = df.shape[0]
batch_index = 0
while 1:
current_index = batch_index * batch_size
if n >= current_index + batch_size:
current_batch_size = batch_size
batch_index += 1
else:
current_batch_size = n - current_index
batch_index = 0
batch_df = df[current_index:current_index+current_batch_size]
batch_x = np.zeros((batch_df.shape[0], ROWS, COLS, 3), dtype=K.floatx())
i = 0
for index,row in batch_df.iterrows():
image_file = row['image_file']
bbox = [row['xmin'],row['ymin'],row['xmax'],row['ymax']]
cropped = load_img(TEST_DIR+image_file,bbox,target_size=(ROWS,COLS))
x = np.asarray(cropped, dtype=K.floatx())
x /= 255.
if datagen is not None: x = datagen.random_transform(x)
x = preprocess_input(x, mean)
batch_x[i] = x
i += 1
if batch_index%50 == 0: print('batch_index', batch_index)
yield(batch_x)
test_aug_datagen = ImageDataGenerator(
rotation_range=180,
shear_range=0.2,
zoom_range=0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True,
vertical_flip=True)
# # train_mean = [0.37698776, 0.41491762, 0.38681713]
# X_train, _ = data_load('train')
# train_mean = mean(X_train)
# print('train_mean:', train_mean)
In [34]:
#validation data fish logloss
# X_valid_centered = X_valid - np.reshape(train_mean, [1, 1, 1, 3])
valid_pred = test_model.predict(X_valid, batch_size=BATCHSIZE, verbose=1)
# valid_pred = test_model.predict_generator(test_generator(df=valid_df, mean=valid_mean),
# val_samples=valid_df.shape[0], nb_worker=1, pickle_safe=False)
valid_logloss_df = pd.DataFrame(columns=['logloss','class'])
for i in range(y_valid.shape[0]):
index = np.argmax(y_valid[i,:])
fish = CROP_CLASSES[index]
logloss = -math.log(valid_pred[i,index])
valid_logloss_df.loc[len(valid_logloss_df)]=[logloss,fish]
print(valid_logloss_df.groupby(['class'])['logloss'].mean())
print(valid_logloss_df['logloss'].mean())
# X_train_centered = X_train - np.reshape(train_mean, [1, 1, 1, 3])
train_pred = test_model.predict(X_train, batch_size=BATCHSIZE, verbose=1)
# train_pred = test_model.predict_generator(test_generator(df=train_df, ),
# val_samples=train_df.shape[0], nb_worker=1, pickle_safe=False)
train_logloss_df = pd.DataFrame(columns=['logloss','class'])
for i in range(y_train.shape[0]):
index = np.argmax(y_train[i,:])
fish = CROP_CLASSES[index]
logloss = -math.log(train_pred[i,index])
train_logloss_df.loc[len(train_logloss_df)]=[logloss,fish]
print(train_logloss_df.groupby(['class'])['logloss'].mean())
print(train_logloss_df['logloss'].mean())
In [9]:
#GTbbox_CROPpred_df = ['image_file','crop_index','crop_class','xmin','ymin','xmax','ymax',
# 'NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT', 'logloss']
file_name = 'GTbbox_CROPpred_df_'+test_model_name+'_.pickle'
if os.path.exists(OUTPUT_DIR+file_name):
print ('Loading from file '+file_name)
GTbbox_CROPpred_df = pd.read_pickle(OUTPUT_DIR+file_name)
else:
print ('Generating file '+file_name)
nb_augmentation = 1
if nb_augmentation ==1:
test_preds = test_model.predict_generator(test_generator(df=GTbbox_df, mean=train_mean),
steps=int(math.ceil(GTbbox_df.shape[0]/BATCHSIZE)), workers=1, pickle_safe=False)
else:
test_preds = np.zeros((GTbbox_df.shape[0], len(FISH_CLASSES)), dtype=K.floatx())
for idx in range(nb_augmentation):
print('{}th augmentation for testing ...'.format(idx+1))
test_preds += test_model.predict_generator(test_generator(df=GTbbox_df, mean=train_mean, datagen=test_aug_datagen),
steps=int(math.ceil(GTbbox_df.shape[0]/BATCHSIZE)), workers=1, pickle_safe=False)
test_preds /= nb_augmentation
CROPpred_df = pd.DataFrame(test_preds, columns=['ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT'])
GTbbox_CROPpred_df = pd.concat([GTbbox_df,CROPpred_df], axis=1)
GTbbox_CROPpred_df['logloss'] = GTbbox_CROPpred_df.apply(lambda row: -math.log(row[row['crop_class']]), axis=1)
GTbbox_CROPpred_df.to_pickle(OUTPUT_DIR+file_name)
#logloss of every fish class
print(GTbbox_CROPpred_df.groupby(['crop_class'])['logloss'].mean())
print(GTbbox_CROPpred_df['logloss'].mean())
In [10]:
# RFCNbbox_RFCNpred_df = ['image_class','image_file','crop_index','xmin','ymin','xmax','ymax',
# 'NoF_RFCN', 'ALB_RFCN', 'BET_RFCN', 'DOL_RFCN',
# 'LAG_RFCN', 'OTHER_RFCN', 'SHARK_RFCN', 'YFT_RFCN']
# select fish_conf >= CONF_THRESH
file_name = 'RFCNbbox_RFCNpred_df_conf{:.2f}.pickle'.format(CONF_THRESH)
if os.path.exists(OUTPUT_DIR+file_name):
print ('Loading from file '+file_name)
RFCNbbox_RFCNpred_df = pd.read_pickle(OUTPUT_DIR+file_name)
else:
print ('Generating file '+file_name)
RFCNbbox_RFCNpred_df = pd.DataFrame(columns=['image_class','image_file','crop_index','xmin','ymin','xmax','ymax',
'NoF_RFCN', 'ALB_RFCN', 'BET_RFCN', 'DOL_RFCN',
'LAG_RFCN', 'OTHER_RFCN', 'SHARK_RFCN', 'YFT_RFCN'])
with open('../data/RFCN_detections/detections_full_AGNOSTICnms_'+RFCN_MODEL+'.pkl','rb') as f:
detections_full_AGNOSTICnms = pickle.load(f)
with open("../RFCN/ImageSets/Main/test.txt","r") as f:
test_files = f.readlines()
with open("../RFCN/ImageSets/Main/train_test.txt","r") as f:
train_file_labels = f.readlines()
assert len(detections_full_AGNOSTICnms) == len(test_files)
count = np.zeros(len(detections_full_AGNOSTICnms))
for im in range(len(detections_full_AGNOSTICnms)):
if im%1000 == 0: print(im)
basename = test_files[im][:9]
if im<1000:
image_class = '--'
else:
for i in range(len(train_file_labels)):
if train_file_labels[i][:9] == basename:
image_class = train_file_labels[i][10:-1]
break
image = Image.open(TEST_DIR+'/'+basename+'.jpg')
width_image, height_image = image.size
bboxes = []
detects_im = detections_full_AGNOSTICnms[im]
for i in range(len(detects_im)):
# if np.sum(detects_im[i,5:]) >= CONF_THRESH:
if np.max(detects_im[i,5:]) >= CONF_THRESH:
bboxes.append(detects_im[i,:])
count[im] = len(bboxes)
if len(bboxes) == 0:
ind = np.argmax(np.sum(detects_im[:,5:], axis=1))
bboxes.append(detects_im[ind,:])
bboxes = np.asarray(bboxes)
for j in range(len(bboxes)):
bbox = bboxes[j]
xmin = bbox[0]
ymin = bbox[1]
xmax = bbox[2]
ymax = bbox[3]
assert max(xmin,0)<min(xmax,width_image)
assert max(ymin,0)<min(ymax,height_image)
RFCNbbox_RFCNpred_df.loc[len(RFCNbbox_RFCNpred_df)]=[image_class,basename+'.jpg',j,max(xmin,0),max(ymin,0),
min(xmax,width_image),min(ymax,height_image),
bbox[4],bbox[5],bbox[6],bbox[7],bbox[8],bbox[9],bbox[10],bbox[11]]
RFCNbbox_RFCNpred_df.to_pickle(OUTPUT_DIR+file_name)
In [12]:
# RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df = ['image_class', 'image_file','crop_index','xmin','ymin','xmax','ymax',
# 'NoF_RFCN', 'ALB_RFCN', 'BET_RFCN', 'DOL_RFCN',
# 'LAG_RFCN', 'OTHER_RFCN', 'SHARK_RFCN', 'YFT_RFCN',
# 'ALB_CROP', 'BET_CROP', 'DOL_CROP',
# 'LAG_CROP', 'OTHER_CROP', 'SHARK_CROP', 'YFT_CROP',
# 'NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']
file_name = 'RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df_'+test_model_name+'_.pickle'
if os.path.exists(OUTPUT_DIR+file_name):
print ('Loading from file '+file_name)
RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df = pd.read_pickle(OUTPUT_DIR+file_name)
else:
print ('Generating file '+file_name)
nb_augmentation = 1
if nb_augmentation ==1:
test_preds = test_model.predict_generator(test_generator(df=RFCNbbox_RFCNpred_df, mean=train_mean),
steps=int(math.ceil(RFCNbbox_RFCNpred_df.shape[0]/BATCHSIZE)), workers=1, pickle_safe=False)
else:
test_preds = np.zeros((RFCNbbox_RFCNpred_df.shape[0], len(CROP_CLASSES)), dtype=K.floatx())
for idx in range(nb_augmentation):
print('{}th augmentation for testing ...'.format(idx+1))
test_preds += test_model.predict_generator(test_generator(df=RFCNbbox_RFCNpred_df, mean=train_mean, datagen=test_aug_datagen),
steps=int(math.ceil(RFCNbbox_RFCNpred_df.shape[0]/BATCHSIZE)), workers=1, pickle_safe=False)
test_preds /= nb_augmentation
CROPpred_df = pd.DataFrame(test_preds, columns=['ALB_CROP', 'BET_CROP', 'DOL_CROP', 'LAG_CROP', 'OTHER_CROP', 'SHARK_CROP', 'YFT_CROP'])
RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df = pd.concat([RFCNbbox_RFCNpred_df,CROPpred_df], axis=1)
RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df['NoF'] = RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df['NoF_RFCN']
for fish in ['ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']:
RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df[fish] = RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df.apply(lambda row: (1-row['NoF_RFCN'])*row[[fish+'_CROP']], axis=1)
# for fish in FISH_CLASSES:
# RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df[fish] = RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df[fish+'_CROP']
RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df.to_pickle(OUTPUT_DIR+file_name)
In [13]:
# clsMaxAve and hybrid RFCNpred&CROPpred such that RFCNpred for NoF and CROPpred for fish
# test_pred_df = ['logloss','image_class','image_file','NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']
# RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df = ['image_class', 'image_file','crop_index','xmin','ymin','xmax','ymax',
# 'NoF_RFCN', 'ALB_RFCN', 'BET_RFCN', 'DOL_RFCN',
# 'LAG_RFCN', 'OTHER_RFCN', 'SHARK_RFCN', 'YFT_RFCN',
# 'ALB_CROP', 'BET_CROP', 'DOL_CROP',
# 'LAG_CROP', 'OTHER_CROP', 'SHARK_CROP', 'YFT_CROP',
# 'NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']
file_name = 'test_pred_df_Hybrid_'+test_model_name+'_.pickle'
if os.path.exists(OUTPUT_DIR+file_name):
print ('Loading from file '+file_name)
test_pred_df = pd.read_pickle(OUTPUT_DIR+file_name)
else:
print ('Generating file '+file_name)
with open("../RFCN/ImageSets/Main/test.txt","r") as f:
test_files = f.readlines()
test_pred_df = pd.DataFrame(columns=['logloss','image_class','image_file','NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT'])
for j in range(len(test_files)):
image_file = test_files[j][:-1]+'.jpg'
test_pred_im_df = RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df.loc[RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df['image_file'] == image_file,
['image_class', 'NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']]
image_class = test_pred_im_df.iloc[0]['image_class']
test_pred_im_df.drop('image_class', axis=1, inplace=True)
max_score = test_pred_im_df.max(axis=1)
max_cls = test_pred_im_df.idxmax(axis=1)
test_pred_im_df['max_score'] = max_score
test_pred_im_df['max_cls'] = max_cls
test_pred_im_df['Count'] = test_pred_im_df.groupby(['max_cls'])['max_cls'].transform('count')
idx = test_pred_im_df.groupby(['max_cls'])['max_score'].transform(max) == test_pred_im_df['max_score']
test_pred_im_clsMax_df = test_pred_im_df.loc[idx,['NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT', 'Count']]
test_pred_im_clsMax_array = test_pred_im_clsMax_df.values
pred = np.average(test_pred_im_clsMax_array[:,:-1], axis=0, weights=test_pred_im_clsMax_array[:,-1], returned=False).tolist()
if image_class!='--':
ind = FISH_CLASSES.index(image_class)
logloss = -math.log(pred[ind])
else:
logloss = np.nan
test_pred_im_clsMaxAve = [logloss,image_class,image_file]
test_pred_im_clsMaxAve.extend(pred)
test_pred_df.loc[len(test_pred_df)]=test_pred_im_clsMaxAve
test_pred_df.to_pickle(OUTPUT_DIR+file_name)
In [ ]:
#### visualization
# RFCNbbox_RFCNpred_CROPpred_df = ['image_class', 'image_file','crop_index','x_min','y_min','x_max','ymax',
# 'NoF_RFCN', 'ALB_RFCN', 'BET_RFCN', 'DOL_RFCN',
# 'LAG_RFCN', 'OTHER_RFCN', 'SHARK_RFCN', 'YFT_RFCN'
# 'NoF_CROP', 'ALB_CROP', 'BET_CROP', 'DOL_CROP',
# 'LAG_CROP', 'OTHER_CROP', 'SHARK_CROP', 'YFT_CROP']
#GTbbox_CROPpred_df = ['image_file','crop_index','crop_class','xmin','ymin','xmax','ymax',
# 'NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT', 'logloss']
# test_pred_df = ['logloss','image_class','image_file','NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']
for j in range(test_pred_df.shape[0]):
image_logloss = test_pred_df.iat[j,0]
image_class = test_pred_df.iat[j,1]
image_file = test_pred_df.iat[j,2]
if j<1000 and j%30== 0:
pass
else:
continue
im = Image.open('../RFCN/JPEGImages/'+image_file)
im = np.asarray(im)
fig, ax = plt.subplots(figsize=(10, 8))
ax.imshow(im, aspect='equal')
RFCN_dets = RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df.loc[RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df['image_file']==image_file]
for index,row in RFCN_dets.iterrows():
bbox = [row['xmin'],row['ymin'],row['xmax'],row['ymax']]
RFCN = [row['NoF_RFCN'],row['ALB_RFCN'],row['BET_RFCN'],row['DOL_RFCN'],row['LAG_RFCN'],row['OTHER_RFCN'],row['SHARK_RFCN'],row['YFT_RFCN']]
CROP = [row['NoF'],row['ALB'],row['BET'],row['DOL'],row['LAG'],row['OTHER'],row['SHARK'],row['YFT']]
score_RFCN = max(RFCN)
score_CROP = max(CROP)
index_RFCN = RFCN.index(score_RFCN)
index_CROP = CROP.index(score_CROP)
class_RFCN = FISH_CLASSES[index_RFCN]
class_CROP = FISH_CLASSES[index_CROP]
ax.add_patch(plt.Rectangle((bbox[0], bbox[1]), bbox[2] - bbox[0], bbox[3] - bbox[1], fill=False, edgecolor='red', linewidth=2))
ax.text(bbox[0], bbox[1] - 2, 'RFCN_{:s} {:.3f} \nHYBRID_{:s} {:.3f}'.format(class_RFCN, score_RFCN, class_CROP, score_CROP), bbox=dict(facecolor='red', alpha=0.5), fontsize=8, color='white')
GT_dets = GTbbox_CROPpred_df.loc[GTbbox_CROPpred_df['image_file']==image_file]
for index,row in GT_dets.iterrows():
bbox = [row['xmin'],row['ymin'],row['xmax'],row['ymax']]
CROP = [row['NoF'],row['ALB'],row['BET'],row['DOL'],row['LAG'],row['OTHER'],row['SHARK'],row['YFT']]
score_CROP = max(CROP)
index_CROP = CROP.index(score_CROP)
class_CROP = FISH_CLASSES[index_CROP]
ax.add_patch(plt.Rectangle((bbox[0], bbox[1]), bbox[2] - bbox[0], bbox[3] - bbox[1], fill=False, edgecolor='green', linewidth=2))
ax.text(bbox[0], bbox[3] + 40, 'GT_{:s} \nCROP_{:s} {:.3f}'.format(row['crop_class'], class_CROP, score_CROP), bbox=dict(facecolor='green', alpha=0.5), fontsize=8, color='white')
ax.set_title(('Image {:s} FISH {:s} logloss {}').format(image_file, image_class, image_logloss), fontsize=10)
plt.axis('off')
plt.tight_layout()
plt.draw()
In [14]:
#temperature
T = 1
test_pred_array = test_pred_df[FISH_CLASSES].values
test_pred_T_array = np.exp(np.log(test_pred_array)/T)
test_pred_T_array = test_pred_T_array/np.sum(test_pred_T_array, axis=1, keepdims=True)
test_pred_T_df = pd.DataFrame(test_pred_T_array, columns=FISH_CLASSES)
test_pred_T_df = pd.concat([test_pred_df[['image_class','image_file']],test_pred_T_df], axis=1)
#add logloss
test_pred_T_df['logloss'] = test_pred_T_df.apply(lambda row: -math.log(row[row['image_class']]) if row['image_class']!='--' else np.nan, axis=1)
#calculate train logloss
print(test_pred_T_df.groupby(['image_class'])['logloss'].mean())
train_logloss = test_pred_T_df['logloss'].mean()
print('logloss of train is', train_logloss )
In [15]:
#test submission
submission = test_pred_T_df.loc[:999,['image_file','NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']]
submission.rename(columns={'image_file':'image'}, inplace=True)
sub_file = 'RFCN_AGONOSTICnms_'+RFCN_MODEL+'_'+CROP_MODEL+'_clsMaxAve_conf{:.2f}_T{}_'.format(CONF_THRESH, T)+'{:.4f}'.format(train_logloss)+'.csv'
submission.to_csv(sub_file, index=False)
submission.to_csv(OUTPUT_DIR + sub_file, index=False)
print('Done!'+sub_file)
In [ ]: