In [1]:
import os, random, glob, pickle, collections, math, json, time
import numpy as np
import pandas as pd
from __future__ import division
from __future__ import print_function
from PIL import Image
from sklearn.model_selection import train_test_split
from sklearn.metrics import log_loss
from sklearn.preprocessing import LabelEncoder
import matplotlib.pyplot as plt
%matplotlib inline
from keras.models import Sequential, Model, load_model
from keras.layers import GlobalAveragePooling2D, Flatten, Dropout, Dense, LeakyReLU
from keras.optimizers import Adam
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau, TensorBoard
from keras.preprocessing.image import ImageDataGenerator
from keras.utils import np_utils
from keras.preprocessing import image
from keras import backend as K
K.set_image_dim_ordering('tf')
In [144]:
TEST_DIR = '../RFCN/JPEGImages/'
RFCN_MODEL = 'resnet101_rfcn_ohem_iter_30000'
CROP_MODEL = 'resnet50_FT38_Hybrid_wNoF'
if os.getcwd().split('/')[-1] != CROP_MODEL:
print('WRONG CROP_MODEL DIR!!!')
CHECKPOINT_DIR = './checkpoint/'
if not os.path.exists(CHECKPOINT_DIR):
os.mkdir(CHECKPOINT_DIR)
LOG_DIR = './log/'
if not os.path.exists(LOG_DIR):
os.mkdir(LOG_DIR)
OUTPUT_DIR = './output/'
if not os.path.exists(OUTPUT_DIR):
os.mkdir(OUTPUT_DIR)
FISH_CLASSES = ['NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']
CONF_THRESH = 0.8
ROWS = 224
COLS = 224
BATCHSIZE = 128
LEARNINGRATE = 1e-4
def featurewise_center(x):
mean = np.mean(x, axis=0, keepdims=True)
mean = np.mean(mean, axis=(1,2), keepdims=True)
x_centered = x - mean
return x_centered
def featurewise_mean(x):
mean = np.mean(x, axis=0)
mean = np.mean(mean, axis=(0,1))
return mean
def preprocess_featurewise(x, featurewise_mean):
# resnet50 image preprocessing
# 'RGB'->'BGR'
x = x[:, :, ::-1]
x -= np.reshape(featurewise_mean, [1, 1, 3])
return x
def load_img(path, bbox, target_size=None):
img = Image.open(path)
# img = img.convert('RGB')
cropped = img.crop((bbox[0],bbox[1],bbox[2],bbox[3]))
width_cropped, height_cropped = cropped.size
if height_cropped > width_cropped: cropped = cropped.transpose(method=2)
image_name = 'temp_{:f}.jpg'.format(time.time())
cropped.save(image_name)
cropped = Image.open(image_name)
if target_size:
cropped = cropped.resize((target_size[1], target_size[0]), Image.BILINEAR)
os.remove(image_name)
return cropped
def get_best_model(checkpoint_dir = CHECKPOINT_DIR):
files = glob.glob(checkpoint_dir+'*')
val_losses = [float(f.split('-')[-1][:-5]) for f in files]
index = val_losses.index(min(val_losses))
print('Loading model from checkpoint file ' + files[index])
model = load_model(files[index])
model_name = files[index].split('/')[-1]
print('Loading model Done!')
return (model, model_name)
def data_from_df(df):
X = np.ndarray((df.shape[0], ROWS, COLS, 3), dtype=np.uint8)
y = np.zeros((df.shape[0], len(FISH_CLASSES)), dtype=K.floatx())
i = 0
for index,row in df.iterrows():
image_file = row['image_file']
fish = row['crop_class']
bbox = [row['xmin'],row['ymin'],row['xmax'],row['ymax']]
cropped = load_img(TEST_DIR+image_file,bbox,target_size=(ROWS,COLS))
X[i] = np.asarray(cropped)
y[i,FISH_CLASSES.index(fish)] = 1
i += 1
return (X, y)
def data_load(name):
file_name = 'data_'+name+'_{}_{}.pickle'.format(ROWS, COLS)
if os.path.exists(OUTPUT_DIR+file_name):
print ('Loading from file '+file_name)
with open(OUTPUT_DIR+file_name, 'rb') as f:
data = pickle.load(f)
X = data['X']
y = data['y']
else:
print ('Generating file '+file_name)
if name=='train' or name=='valid':
df = GTbbox_df[GTbbox_df['split']==name]
elif name=='all':
df = GTbbox_df
else:
print('Invalid name '+name)
X, y = data_from_df(df)
data = {'X': X,'y': y}
with open(OUTPUT_DIR+file_name, 'wb') as f:
pickle.dump(data, f)
X = X.astype(np.float32)
X /= 255.
return (X, y)
In [136]:
# GTbbox_df = ['image_file','image_class','crop_index','crop_class','xmin',''ymin','xmax','ymax','split']
file_name = 'GTbbox_df.pickle'
if os.path.exists(OUTPUT_DIR+file_name):
print ('Loading from file '+file_name)
GTbbox_df = pd.read_pickle(OUTPUT_DIR+file_name)
else:
print ('Generating file '+file_name)
GTbbox_df = pd.DataFrame(columns=['image_file','image_class','crop_index','crop_class','xmin','ymin','xmax','ymax'])
crop_classes=FISH_CLASSES[:]
crop_classes.remove('NoF')
with open("../RFCN/ImageSets/Main/train_test.txt","r") as f:
train_file_labels = f.readlines()
for c in crop_classes:
print(c)
j = json.load(open('../data/BBannotations/{}.json'.format(c), 'r'))
for l in j:
filename = l["filename"]
head, image_file = os.path.split(filename)
basename, file_extension = os.path.splitext(image_file)
for i in range(len(train_file_labels)):
if train_file_labels[i][:9] == basename:
image_class = train_file_labels[i][10:-1]
break
image = Image.open(TEST_DIR+'/'+image_file)
width_image, height_image = image.size
for i in range(len(l["annotations"])):
a = l["annotations"][i]
xmin = (a["x"])
ymin = (a["y"])
width = (a["width"])
height = (a["height"])
xmax = xmin + width
ymax = ymin + height
assert max(xmin,0)<min(xmax,width_image)
assert max(ymin,0)<min(ymax,height_image)
GTbbox_df.loc[len(GTbbox_df)]=[image_file,image_class,i,a["class"],max(xmin,0),max(ymin,0),min(xmax,width_image),min(ymax,height_image)]
print('NoF')
num_NoF = GTbbox_df.shape[0]*3
RFCN_MODEL = 'resnet101_rfcn_ohem_iter_30000'
with open('../data/RFCN_detections/detections_full_AGNOSTICnms_'+RFCN_MODEL+'.pkl','rb') as f:
detections_full_AGNOSTICnms = pickle.load(f)
train_detections_full_AGNOSTICnms = detections_full_AGNOSTICnms[1000:]
with open("../RFCN/ImageSets/Main/test.txt","r") as f:
test_files = f.readlines()
train_files = test_files[1000:]
num_NoF_perIm = int(math.ceil(num_NoF / len(train_detections_full_AGNOSTICnms)))
for im in range(len(train_files)):
image_file = train_files[im][:9]+'.jpg'
for i in range(len(train_file_labels)):
if train_file_labels[i][:9] == train_files[im][:9]:
image_class = train_file_labels[i][10:-1]
break
image = Image.open(TEST_DIR+image_file)
width_image, height_image = image.size
detects_im = train_detections_full_AGNOSTICnms[im]
detects_im = detects_im[np.where(detects_im[:,4] >= 0.999)]
np.random.seed(1986)
bboxes = detects_im[np.random.choice(detects_im.shape[0], num_NoF_perIm, replace=False), :]
for j in range(bboxes.shape[0]):
bbox = bboxes[j]
xmin = bbox[0]
ymin = bbox[1]
xmax = bbox[2]
ymax = bbox[3]
assert max(xmin,0)<min(xmax,width_image)
assert max(ymin,0)<min(ymax,height_image)
GTbbox_df.loc[len(GTbbox_df)]=[image_file,image_class,j,'NoF',xmin,ymin,xmax,ymax]
# test_size = GTbbox_df.shape[0]-int(math.ceil(GTbbox_df.shape[0]*0.8/128)*128)
# train_ind, valid_ind = train_test_split(range(GTbbox_df.shape[0]), test_size=test_size, random_state=1986, stratify=GTbbox_df['crop_class'])
# GTbbox_df['split'] = ['train' if i in train_ind else 'valid' for i in range(GTbbox_df.shape[0])]
image_files_df = GTbbox_df[['image_file','image_class']]
image_files_df.drop_duplicates(inplace = True)
test_file_nb = image_files_df.shape[0] - int(math.ceil(image_files_df.shape[0]*0.9))
train_files_df, valid_files_df = train_test_split(image_files_df, test_size=test_file_nb, random_state=1986, stratify=image_files_df.image_class)
GTbbox_df['split'] = ['train' if image_file in train_files_df.image_file.tolist() else 'valid' for image_file in GTbbox_df.image_file]
# train2valid_nb = GTbbox_df.loc[GTbbox_df['split']=='train'].shape[0] - int(math.floor(GTbbox_df.loc[GTbbox_df['split']=='train'].shape[0]/128)*128)
# train_df = GTbbox_df.loc[GTbbox_df['split']=='train']
# np.random.seed(1986)
# inds = np.random.choice(train_df.index, train2valid_nb)
# GTbbox_df.loc[inds, 'split'] = 'valid'
GTbbox_df.to_pickle(OUTPUT_DIR+file_name)
print('Done')
# GTbbox_df.loc[(GTbbox_df['image_class']!='NoF') & (GTbbox_df['crop_class']!='NoF') & (GTbbox_df['image_class']!=GTbbox_df['crop_class'])]
In [145]:
#Load data
X_all, y_all = data_load('all')
X_train, y_train = data_load('train')
X_valid, y_valid = data_load('valid')
print('Loading data done.')
print('all sample ', X_all.shape[0])
print('train sample ', X_train.shape[0])
print('valid sample ', X_valid.shape[0])
print('featurewise mean of X_all is ', featurewise_mean(X_all))
print('featurewise mean of X_train is ', featurewise_mean(X_train))
print('featurewise mean of X_valid is ', featurewise_mean(X_valid))
In [146]:
# #class weight = n_samples / (n_classes * np.bincount(y))
# class_weight_fish = dict(GTbbox_df.groupby('crop_class').size())
# class_weight = {}
# n_samples = GTbbox_df.shape[0]
# for key,value in class_weight_fish.items():
# class_weight[CROP_CLASSES.index(key)] = n_samples / (len(CROP_CLASSES)*value)
# class_weight
class_weight_fish = dict(GTbbox_df.groupby('crop_class').size())
class_weight = {}
ref = max(class_weight_fish.values())
for key,value in class_weight_fish.items():
class_weight[FISH_CLASSES.index(key)] = ref/value
class_weight
Out[146]:
In [147]:
#data augmentation
all_mean = featurewise_mean(X_all)
def preprocessing_function(x):
# resnet50 image preprocessing
# 'RGB'->'BGR'
x = x[:, :, ::-1]
x -= np.reshape(all_mean, [1, 1, 3])
return x
train_datagen = ImageDataGenerator(
preprocessing_function=preprocessing_function,
rotation_range=180,
shear_range=0.2,
zoom_range=0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True,
vertical_flip=True)
train_generator = train_datagen.flow(X_train, y_train, batch_size=BATCHSIZE, shuffle=True, seed=None)
steps_per_epoch = int(math.ceil(X_train.shape[0]/BATCHSIZE))
X_valid_centered = X_valid - np.reshape(all_mean, [1, 1, 1, 3])
In [148]:
#callbacks
early_stopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=25, verbose=1, mode='auto')
model_checkpoint = ModelCheckpoint(filepath=CHECKPOINT_DIR+'weights.{epoch:03d}-{val_loss:.4f}.hdf5', monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=False, mode='auto')
learningrate_schedule = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=10, verbose=1, mode='auto', epsilon=0.001, cooldown=0, min_lr=0)
tensorboard = TensorBoard(log_dir=LOG_DIR, histogram_freq=0, write_graph=False, write_images=True)
In [149]:
#Resnet50
#top layer training
from keras.applications.resnet50 import ResNet50
base_model = ResNet50(weights='imagenet', include_top=False)
x = base_model.output
x = GlobalAveragePooling2D()(x)
# x = Dropout(0.5)(x)
predictions = Dense(len(FISH_CLASSES), kernel_initializer='glorot_normal', activation='softmax')(x)
model = Model(inputs=base_model.input, outputs=predictions)
# first: train only the top layers (which were randomly initialized)
for layer in base_model.layers:
layer.trainable = False
# compile the model (should be done *after* setting layers to non-trainable)
optimizer = Adam(lr=1e-4)
model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])
# train the model on the new data for a few epochs
model.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=30, verbose=1,
callbacks=[early_stopping, model_checkpoint, learningrate_schedule, tensorboard],
validation_data=(X_valid_centered, y_valid),
class_weight=class_weight, workers=3, pickle_safe=True)
Out[149]:
In [150]:
### Resnet50
# fine tuning
# 164 conv5c+top
# 142 conv5+top
# 80 conv4+conv5+top
# 38 conv3+conv4+conv5+top
start_layer = 38
model, model_name = get_best_model()
# print('Loading model from weights.004-0.0565.hdf5')
# model = load_model(CHECHPOINT_DIR+'weights.004-0.0565.hdf5')
for layer in model.layers[:start_layer]:
layer.trainable = False
for layer in model.layers[start_layer:]:
layer.trainable = True
# we need to recompile the model for these modifications to take effect
# we use SGD with a low learning rate
optimizer = Adam(lr=1e-5)
model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])
model.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=300, verbose=1,
callbacks=[early_stopping, model_checkpoint, learningrate_schedule, tensorboard],
validation_data=(X_valid_centered, y_valid),
class_weight=class_weight, workers=3, pickle_safe=True, initial_epoch=27)
Out[150]:
In [ ]:
#resume training
model, model_name = get_best_model()
# print('Loading model from weights.004-0.0565.hdf5')
# model = load_model(CHECHPOINT_DIR+'weights.004-0.0565.hdf5')
# #try increasing learningrate
# optimizer = Adam(lr=1e-4)
# model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])
model.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=300, verbose=1,
callbacks=[early_stopping, model_checkpoint, learningrate_schedule, tensorboard],
validation_data=(X_valid_centered, y_valid),
class_weight=class_weight, workers=3, pickle_safe=True, initial_epoch=)
In [49]:
#test prepare
test_model, test_model_name = get_best_model()
# test_model = load_model('./resnet50_FT38_CW_STGTrain/checkpoint/weights.000-0.0327.hdf5')
# test_model_name = 'weights.000-0.0327.hdf5'
# print('test_model_name', test_model_name)
def test_generator(df, mean, datagen = None, batch_size = BATCHSIZE):
n = df.shape[0]
batch_index = 0
while 1:
current_index = batch_index * batch_size
if n >= current_index + batch_size:
current_batch_size = batch_size
batch_index += 1
else:
current_batch_size = n - current_index
batch_index = 0
batch_df = df[current_index:current_index+current_batch_size]
batch_x = np.zeros((batch_df.shape[0], ROWS, COLS, 3), dtype=K.floatx())
i = 0
for index,row in batch_df.iterrows():
image_file = row['image_file']
bbox = [row['xmin'],row['ymin'],row['xmax'],row['ymax']]
cropped = load_img(TEST_DIR+image_file,bbox,target_size=(ROWS,COLS))
x = np.asarray(cropped, dtype=K.floatx())
x /= 255.
if datagen is not None: x = datagen.random_transform(x)
x = preprocess_imagewise(x, mean)
batch_x[i] = x
i += 1
if batch_index%50 == 0: print('batch_index', batch_index)
yield(batch_x)
test_aug_datagen = ImageDataGenerator(
rotation_range=180,
shear_range=0.2,
zoom_range=0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True,
vertical_flip=True)
# train_mean = [0.37698776, 0.41491762, 0.38681713]
train_mean = imagewise_mean(X_train)
print('train_mean:', train_mean.shape)
In [3]:
test_model_name = 'weights.084-0.0739.hdf5'
In [4]:
#GTbbox_CROPpred_df = ['image_file','crop_index','crop_class','xmin','ymin','xmax','ymax','split'
# 'NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT', 'logloss',
# 'ALB_woNoF', 'BET_woNoF', 'DOL_woNoF', 'LAG_woNoF', 'OTHER_woNoF', 'SHARK_woNoF', 'YFT_woNoF', 'logloss_woNoF']
file_name = 'GTbbox_CROPpred_df_'+test_model_name+'_.pickle'
if os.path.exists(OUTPUT_DIR+file_name):
print ('Loading from file '+file_name)
GTbbox_CROPpred_df = pd.read_pickle(OUTPUT_DIR+file_name)
else:
print ('Generating file '+file_name)
nb_augmentation = 1
if nb_augmentation ==1:
test_preds = test_model.predict_generator(test_generator(df=GTbbox_df, mean=train_mean),
val_samples=GTbbox_df.shape[0], nb_worker=1, pickle_safe=False)
# test_preds = test_model.predict_generator(test_generator(df=GTbbox_df, mean=train_mean),
# steps=int(math.ceil(GTbbox_df.shape[0]/BATCHSIZE)), workers=1, pickle_safe=False)
else:
test_preds = np.zeros((GTbbox_df.shape[0], len(FISH_CLASSES)), dtype=K.floatx())
for idx in range(nb_augmentation):
print('{}th augmentation for testing ...'.format(idx+1))
test_preds += test_model.predict_generator(test_generator(df=GTbbox_df, mean=train_mean, datagen=test_aug_datagen),
val_samples=GTbbox_df.shape[0], nb_worker=1, pickle_safe=False)
# test_preds += test_model.predict_generator(test_generator(df=GTbbox_df, mean=train_mean, datagen=test_aug_datagen),
# steps=GTbbox_df.shape[0], workers=1, pickle_safe=False)
test_preds /= nb_augmentation
CROPpred_df = pd.DataFrame(test_preds, columns=FISH_CLASSES)
GTbbox_CROPpred_df = pd.concat([GTbbox_df,CROPpred_df], axis=1)
GTbbox_CROPpred_df['logloss'] = GTbbox_CROPpred_df.apply(lambda row: -math.log(row[row['crop_class']]), axis=1)
for fish in FISH_CLASSES:
GTbbox_CROPpred_df[fish+'_woNoF'] = GTbbox_CROPpred_df.apply(lambda row: row[fish]/(1-row['NoF']) if fish!='NoF' else np.inf , axis=1)
GTbbox_CROPpred_df['logloss_woNoF'] = GTbbox_CROPpred_df.apply(lambda row: -math.log(row[row['crop_class']+'_woNoF']), axis=1)
GTbbox_CROPpred_df.to_pickle(OUTPUT_DIR+file_name)
valid_CROPpred_df = GTbbox_CROPpred_df[GTbbox_CROPpred_df['split']=='valid']
crop_valid_woNoF_logloss = valid_CROPpred_df[valid_CROPpred_df['crop_class']!='NoF']['logloss_woNoF'].mean()
print('crop_valid_woNoF_logloss:', crop_valid_woNoF_logloss)
In [5]:
# print('all loss:', GTbbox_CROPpred_df['logloss'].mean())
# print('all fish loss:', GTbbox_CROPpred_df[GTbbox_CROPpred_df['crop_class']!='NoF']['logloss'].mean())
# print(GTbbox_CROPpred_df.groupby(['crop_class'])['logloss'].mean())
# print('all_woNoF loss:', GTbbox_CROPpred_df[GTbbox_CROPpred_df['crop_class']!='NoF']['logloss_woNoF'].mean())
# print(GTbbox_CROPpred_df[GTbbox_CROPpred_df['crop_class']!='NoF'].groupby(['crop_class'])['logloss_woNoF'].mean())
train_CROPpred_df = GTbbox_CROPpred_df[GTbbox_CROPpred_df['split']=='train']
print('train loss:', train_CROPpred_df['logloss'].mean())
# print('train fish loss:', train_CROPpred_df[train_CROPpred_df['crop_class']!='NoF']['logloss'].mean())
# print(train_CROPpred_df.groupby(['crop_class'])['logloss'].mean())
print('train_woNoF loss:', train_CROPpred_df[train_CROPpred_df['crop_class']!='NoF']['logloss_woNoF'].mean())
print(train_CROPpred_df[train_CROPpred_df['crop_class']!='NoF'].groupby(['crop_class'])['logloss_woNoF'].mean())
valid_CROPpred_df = GTbbox_CROPpred_df[GTbbox_CROPpred_df['split']=='valid']
print('valid loss:', valid_CROPpred_df['logloss'].mean())
# print('valid fish loss:', valid_CROPpred_df[valid_CROPpred_df['crop_class']!='NoF']['logloss'].mean())
# print(valid_CROPpred_df.groupby(['crop_class'])['logloss'].mean())
print('valid_woNoF loss:', valid_CROPpred_df[valid_CROPpred_df['crop_class']!='NoF']['logloss_woNoF'].mean())
print(valid_CROPpred_df[valid_CROPpred_df['crop_class']!='NoF'].groupby(['crop_class'])['logloss_woNoF'].mean())
In [6]:
# RFCNbbox_RFCNpred_df = ['image_class','image_file','crop_index','xmin','ymin','xmax','ymax',
# 'NoF_RFCN', 'ALB_RFCN', 'BET_RFCN', 'DOL_RFCN',
# 'LAG_RFCN', 'OTHER_RFCN', 'SHARK_RFCN', 'YFT_RFCN']
# select fish_conf >= CONF_THRESH
file_name = 'RFCNbbox_RFCNpred_df_conf{:.2f}.pickle'.format(CONF_THRESH)
if os.path.exists(OUTPUT_DIR+file_name):
print ('Loading from file '+file_name)
RFCNbbox_RFCNpred_df = pd.read_pickle(OUTPUT_DIR+file_name)
else:
print ('Generating file '+file_name)
RFCNbbox_RFCNpred_df = pd.DataFrame(columns=['image_class','image_file','crop_index','xmin','ymin','xmax','ymax',
'NoF_RFCN', 'ALB_RFCN', 'BET_RFCN', 'DOL_RFCN',
'LAG_RFCN', 'OTHER_RFCN', 'SHARK_RFCN', 'YFT_RFCN'])
with open('../data/RFCN_detections/detections_full_AGNOSTICnms_'+RFCN_MODEL+'.pkl','rb') as f:
detections_full_AGNOSTICnms = pickle.load(f, encoding='latin1')
with open("../RFCN/ImageSets/Main/test.txt","r") as f:
test_files = f.readlines()
with open("../RFCN/ImageSets/Main/train_test.txt","r") as f:
train_file_labels = f.readlines()
assert len(detections_full_AGNOSTICnms) == len(test_files)
count = np.zeros(len(detections_full_AGNOSTICnms))
for im in range(len(detections_full_AGNOSTICnms)):
if im%1000 == 0: print(im)
basename = test_files[im][:9]
if im<1000:
image_class = '--'
else:
for i in range(len(train_file_labels)):
if train_file_labels[i][:9] == basename:
image_class = train_file_labels[i][10:-1]
break
image = Image.open(TEST_DIR+'/'+basename+'.jpg')
width_image, height_image = image.size
bboxes = []
detects_im = detections_full_AGNOSTICnms[im]
for i in range(len(detects_im)):
if np.sum(detects_im[i,5:]) >= CONF_THRESH:
# if np.max(detects_im[i,5:]) >= CONF_THRESH:
bboxes.append(detects_im[i,:])
count[im] = len(bboxes)
if len(bboxes) == 0:
ind = np.argmax(np.sum(detects_im[:,5:], axis=1))
bboxes.append(detects_im[ind,:])
bboxes = np.asarray(bboxes)
for j in range(len(bboxes)):
bbox = bboxes[j]
xmin = bbox[0]
ymin = bbox[1]
xmax = bbox[2]
ymax = bbox[3]
assert max(xmin,0)<min(xmax,width_image)
assert max(ymin,0)<min(ymax,height_image)
RFCNbbox_RFCNpred_df.loc[len(RFCNbbox_RFCNpred_df)]=[image_class,basename+'.jpg',j,max(xmin,0),max(ymin,0),
min(xmax,width_image),min(ymax,height_image),
bbox[4],bbox[5],bbox[6],bbox[7],bbox[8],bbox[9],bbox[10],bbox[11]]
RFCNbbox_RFCNpred_df.to_pickle(OUTPUT_DIR+file_name)
In [7]:
# RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df = ['image_class', 'image_file','crop_index','xmin','ymin','xmax','ymax',
# 'NoF_RFCN', 'ALB_RFCN', 'BET_RFCN', 'DOL_RFCN',
# 'LAG_RFCN', 'OTHER_RFCN', 'SHARK_RFCN', 'YFT_RFCN',
# 'NoF_CROP', 'ALB_CROP', 'BET_CROP', 'DOL_CROP',
# 'LAG_CROP', 'OTHER_CROP', 'SHARK_CROP', 'YFT_CROP',
# 'NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']
file_name = 'RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df_'+test_model_name+'_.pickle'
if os.path.exists(OUTPUT_DIR+file_name):
print ('Loading from file '+file_name)
RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df = pd.read_pickle(OUTPUT_DIR+file_name)
else:
print ('Generating file '+file_name)
nb_augmentation = 1
if nb_augmentation ==1:
test_preds = test_model.predict_generator(test_generator(df=RFCNbbox_RFCNpred_df, mean=train_mean),
val_samples=RFCNbbox_RFCNpred_df.shape[0], nb_worker=1, pickle_safe=False)
else:
test_preds = np.zeros((RFCNbbox_RFCNpred_df.shape[0], len(FISH_CLASSES)), dtype=K.floatx())
for idx in range(nb_augmentation):
print('{}th augmentation for testing ...'.format(idx+1))
test_preds += test_model.predict_generator(test_generator(df=RFCNbbox_RFCNpred_df, mean=train_mean, datagen=test_aug_datagen),
val_samples=RFCNbbox_RFCNpred_df.shape[0], nb_worker=1, pickle_safe=False)
test_preds /= nb_augmentation
CROPpred_df = pd.DataFrame(test_preds, columns=['NoF_CROP', 'ALB_CROP', 'BET_CROP', 'DOL_CROP', 'LAG_CROP', 'OTHER_CROP', 'SHARK_CROP', 'YFT_CROP'])
RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df = pd.concat([RFCNbbox_RFCNpred_df,CROPpred_df], axis=1)
RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df['NoF'] = RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df['NoF_RFCN']
for fish in ['ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']:
RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df[fish] = RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df.apply(lambda row: (1-row['NoF_RFCN'])*row[fish+'_CROP']/(1-row['NoF_CROP']), axis=1)
# for fish in FISH_CLASSES:
# RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df[fish] = RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df[fish+'_CROP']
RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df.to_pickle(OUTPUT_DIR+file_name)
In [8]:
# clsMaxAve and hybrid RFCNpred&CROPpred such that RFCNpred for NoF and CROPpred for fish
# test_pred_df = ['logloss','image_class','image_file','NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']
# RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df = ['image_class', 'image_file','crop_index','xmin','ymin','xmax','ymax',
# 'NoF_RFCN', 'ALB_RFCN', 'BET_RFCN', 'DOL_RFCN',
# 'LAG_RFCN', 'OTHER_RFCN', 'SHARK_RFCN', 'YFT_RFCN',
# 'ALB_CROP', 'BET_CROP', 'DOL_CROP',
# 'LAG_CROP', 'OTHER_CROP', 'SHARK_CROP', 'YFT_CROP',
# 'NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']
file_name = 'test_pred_df_Hybrid_'+test_model_name+'_.pickle'
if os.path.exists(OUTPUT_DIR+file_name):
print ('Loading from file '+file_name)
test_pred_df = pd.read_pickle(OUTPUT_DIR+file_name)
else:
print ('Generating file '+file_name)
with open("../RFCN/ImageSets/Main/test.txt","r") as f:
test_files = f.readlines()
test_pred_df = pd.DataFrame(columns=['logloss','image_class','image_file','NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT'])
for j in range(len(test_files)):
image_file = test_files[j][:-1]+'.jpg'
test_pred_im_df = RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df.loc[RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df['image_file'] == image_file,
['image_class', 'NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']]
image_class = test_pred_im_df.iloc[0]['image_class']
test_pred_im_df.drop('image_class', axis=1, inplace=True)
max_score = test_pred_im_df.max(axis=1)
max_cls = test_pred_im_df.idxmax(axis=1)
test_pred_im_df['max_score'] = max_score
test_pred_im_df['max_cls'] = max_cls
test_pred_im_df['Count'] = test_pred_im_df.groupby(['max_cls'])['max_cls'].transform('count')
idx = test_pred_im_df.groupby(['max_cls'])['max_score'].transform(max) == test_pred_im_df['max_score']
test_pred_im_clsMax_df = test_pred_im_df.loc[idx,['NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT', 'Count']]
test_pred_im_clsMax_array = test_pred_im_clsMax_df.values
pred = np.average(test_pred_im_clsMax_array[:,:-1], axis=0, weights=test_pred_im_clsMax_array[:,-1], returned=False).tolist()
if image_class!='--':
ind = FISH_CLASSES.index(image_class)
logloss = -math.log(pred[ind])
else:
logloss = np.nan
test_pred_im_clsMaxAve = [logloss,image_class,image_file]
test_pred_im_clsMaxAve.extend(pred)
test_pred_df.loc[len(test_pred_df)]=test_pred_im_clsMaxAve
test_pred_df.to_pickle(OUTPUT_DIR+file_name)
image_all_logloss = test_pred_df[test_pred_df['image_class']!='--']['logloss'].mean()
print('imag_all_logloss:', image_all_logloss)
# print(test_pred_df[test_pred_df['image_class']!='--'].groupby('image_class')['logloss'].mean())
In [ ]:
#### visualization
# RFCNbbox_RFCNpred_CROPpred_df = ['image_class', 'image_file','crop_index','x_min','y_min','x_max','ymax',
# 'NoF_RFCN', 'ALB_RFCN', 'BET_RFCN', 'DOL_RFCN',
# 'LAG_RFCN', 'OTHER_RFCN', 'SHARK_RFCN', 'YFT_RFCN'
# 'NoF_CROP', 'ALB_CROP', 'BET_CROP', 'DOL_CROP',
# 'LAG_CROP', 'OTHER_CROP', 'SHARK_CROP', 'YFT_CROP']
#GTbbox_CROPpred_df = ['image_file','crop_index','crop_class','xmin','ymin','xmax','ymax',
# 'NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT', 'logloss']
# test_pred_df = ['logloss','image_class','image_file','NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']
for j in range(test_pred_df.shape[0]):
image_logloss = test_pred_df.iat[j,0]
image_class = test_pred_df.iat[j,1]
image_file = test_pred_df.iat[j,2]
if j<1000 and j%30== 0:
pass
else:
continue
im = Image.open('../RFCN/JPEGImages/'+image_file)
im = np.asarray(im)
fig, ax = plt.subplots(figsize=(10, 8))
ax.imshow(im, aspect='equal')
RFCN_dets = RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df.loc[RFCNbbox_RFCNpred_CROPpred_HYBRIDpred_df['image_file']==image_file]
for index,row in RFCN_dets.iterrows():
bbox = [row['xmin'],row['ymin'],row['xmax'],row['ymax']]
RFCN = [row['NoF_RFCN'],row['ALB_RFCN'],row['BET_RFCN'],row['DOL_RFCN'],row['LAG_RFCN'],row['OTHER_RFCN'],row['SHARK_RFCN'],row['YFT_RFCN']]
CROP = [row['NoF'],row['ALB'],row['BET'],row['DOL'],row['LAG'],row['OTHER'],row['SHARK'],row['YFT']]
score_RFCN = max(RFCN)
score_CROP = max(CROP)
index_RFCN = RFCN.index(score_RFCN)
index_CROP = CROP.index(score_CROP)
class_RFCN = FISH_CLASSES[index_RFCN]
class_CROP = FISH_CLASSES[index_CROP]
ax.add_patch(plt.Rectangle((bbox[0], bbox[1]), bbox[2] - bbox[0], bbox[3] - bbox[1], fill=False, edgecolor='red', linewidth=2))
ax.text(bbox[0], bbox[1] - 2, 'RFCN_{:s} {:.3f} \nHYBRID_{:s} {:.3f}'.format(class_RFCN, score_RFCN, class_CROP, score_CROP), bbox=dict(facecolor='red', alpha=0.5), fontsize=8, color='white')
GT_dets = GTbbox_CROPpred_df.loc[GTbbox_CROPpred_df['image_file']==image_file]
for index,row in GT_dets.iterrows():
bbox = [row['xmin'],row['ymin'],row['xmax'],row['ymax']]
CROP = [row['NoF'],row['ALB'],row['BET'],row['DOL'],row['LAG'],row['OTHER'],row['SHARK'],row['YFT']]
score_CROP = max(CROP)
index_CROP = CROP.index(score_CROP)
class_CROP = FISH_CLASSES[index_CROP]
ax.add_patch(plt.Rectangle((bbox[0], bbox[1]), bbox[2] - bbox[0], bbox[3] - bbox[1], fill=False, edgecolor='green', linewidth=2))
ax.text(bbox[0], bbox[3] + 40, 'GT_{:s} \nCROP_{:s} {:.3f}'.format(row['crop_class'], class_CROP, score_CROP), bbox=dict(facecolor='green', alpha=0.5), fontsize=8, color='white')
ax.set_title(('Image {:s} FISH {:s} logloss {}').format(image_file, image_class, image_logloss), fontsize=10)
plt.axis('off')
plt.tight_layout()
plt.draw()
In [9]:
#temperature
T = 2.5
test_pred_array = test_pred_df[FISH_CLASSES].values
test_pred_T_array = np.exp(np.log(test_pred_array)/T)
test_pred_T_array = test_pred_T_array/np.sum(test_pred_T_array, axis=1, keepdims=True)
test_pred_T_df = pd.DataFrame(test_pred_T_array, columns=FISH_CLASSES)
test_pred_T_df = pd.concat([test_pred_df[['image_class','image_file']],test_pred_T_df], axis=1)
#test submission
submission = test_pred_T_df.loc[:999,['image_file','NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']]
submission.rename(columns={'image_file':'image'}, inplace=True)
sub_file = 'RFCN_AGONOSTICnms_'+RFCN_MODEL+'_'+CROP_MODEL+'_'+test_model_name+'_clsMaxAve_conf{:.2f}_cropvalloss{:.4f}_imageallloss{:.4f}_T{}.csv'.format(CONF_THRESH, crop_valid_woNoF_logloss, image_all_logloss, T)
submission.to_csv(sub_file, index=False)
submission.to_csv(OUTPUT_DIR + sub_file, index=False)
print('Done!'+sub_file)
In [ ]: