In [1]:
import os, random, glob, pickle, collections
import numpy as np
import pandas as pd
import ujson as json
from PIL import Image
from sklearn.model_selection import train_test_split
from sklearn.metrics import log_loss
from sklearn.preprocessing import LabelEncoder
import matplotlib.pyplot as plt
from matplotlib import ticker
import seaborn as sns
%matplotlib inline
from keras.models import Sequential, Model, load_model, model_from_json
from keras.layers import GlobalAveragePooling2D, Flatten, Dropout, Dense
from keras.optimizers import Adam
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau, TensorBoard
from keras.preprocessing.image import ImageDataGenerator
from keras.utils import np_utils
from keras import backend as K
K.set_image_dim_ordering('tf')
from skimage.data import imread
from skimage.io import imshow,imsave
import cv2
from skimage.util import crop
from skimage.transform import rotate
from skimage.transform import resize
import math
In [2]:
TRAIN_DIR = '../data/train/'
TEST_DIR = '../RFCN/JPEGImages/'
TRAIN_CROP_DIR = '../data/train_crop/'
TEST_CROP_DIR = '../data/test_stg1_crop/'
CHECKPOINT_DIR = './checkpoints/checkpoint3/'
FISH_CLASSES = ['ALB', 'BET', 'DOL', 'LAG', 'NoF', 'OTHER', 'SHARK', 'YFT']
CONF_THRESH = 0.8
ROWS = 224
COLS = 224
BatchSize = 128
LearningRate = 1e-4
le = LabelEncoder()
le.fit(FISH_CLASSES)
print(le.transform(FISH_CLASSES))
def featurewise_center(x):
mean = np.mean(x, axis=0, keepdims=True)
mean = np.mean(mean, axis=(1,2), keepdims=True)
x_centered = x - mean
return x_centered
In [31]:
#crop and cache to TRAIN_CROP_DIR by BBannotations
if not os.path.exists(TRAIN_CROP_DIR):
os.mkdir(TRAIN_CROP_DIR)
for c in FISH_CLASSES:
TRAIN_CROP_DIR_c = TRAIN_CROP_DIR + '{}/'.format(c)
if not os.path.exists(TRAIN_CROP_DIR_c):
os.mkdir(TRAIN_CROP_DIR_c)
files = glob.glob(TRAIN_CROP_DIR_c+'*')
for f in files:
os.remove(f)
GT_crop_bboxs_df = pd.DataFrame(columns=['GT_crop_files', 'xmin', 'ymin', 'xmax', 'ymax'])
crop_classes=FISH_CLASSES[:]
crop_classes.remove('NoF')
count = {}
for c in crop_classes:
j = json.load(open('../data/BBannotations/{}.json'.format(c), 'r'))
for l in j:
filename = l["filename"]
head, tail = os.path.split(filename)
basename, file_extension = os.path.splitext(tail)
image = Image.open(TRAIN_DIR+c+'/'+tail)
for i in range(len(l["annotations"])):
a = l["annotations"][i]
file_crop = TRAIN_CROP_DIR + '{}/'.format(a["class"])+c+'_'+basename+'_{}_'.format(i)+a["class"]+'.jpg'
xmin = (a["x"])
ymin = (a["y"])
width = (a["width"])
height = (a["height"])
xmax = xmin + width
ymax = ymin + height
#save cropped img
cropped = image.crop((max(xmin,0), max(ymin,0), xmax, ymax))
width_cropped, height_cropped = cropped.size
if height_cropped > width_cropped: cropped = cropped.transpose(method=2)
cropped.save(file_crop)
if a["class"] != c: print(file_crop)
GT_crop_bboxs_df.loc[len(GT_crop_bboxs_df)]=[file_crop.split('/')[-1],max(xmin,0),max(ymin,0),xmax,ymax]
count[c] = len(os.listdir(TRAIN_CROP_DIR+c))
num_NoF = sum(count.values())*3
#crop and cache to TRAIN_CROP_DIR/NoF by RFCN
#crop images by detections_full_AGNOSTICnms.pkl
RFCN_MODEL = 'resnet101_rfcn_ohem_iter_30000'
with open('../data/RFCN_detections/detections_full_AGNOSTICnms_'+RFCN_MODEL+'.pkl','rb') as f:
detections_full_AGNOSTICnms = pickle.load(f, encoding='latin1')
train_detections_full_AGNOSTICnms = detections_full_AGNOSTICnms[1000:]
num_NoF_perIm = math.ceil(num_NoF / len(train_detections_full_AGNOSTICnms))
outputs = []
for im in range(len(train_detections_full_AGNOSTICnms)):
#for im in range(1):
outputs_im = []
detects_im = train_detections_full_AGNOSTICnms[im]
for i in range(len(detects_im)):
if detects_im[i,4] >= 0.999:
outputs_im.append(detects_im[i,:])
outputs_im = np.asarray(outputs_im)
outputs_im = outputs_im[np.random.choice(outputs_im.shape[0], num_NoF_perIm, replace=False), :]
outputs.append(outputs_im)
train_outputs = outputs
with open("../RFCN/ImageSets/Main/train_test.txt","r") as f:
train_files = f.readlines()
for i in range(len(train_outputs)):
basename = train_files[i][:9]
bboxes = train_outputs[i]
image = Image.open(TEST_DIR+basename+'.jpg')
for j in range(len(bboxes)):
bbox = bboxes[j]
xmin = bbox[0]
ymin = bbox[1]
xmax = bbox[2]
ymax = bbox[3]
file_crop = TRAIN_CROP_DIR+'NoF/'+train_files[i][10:-1]+'_'+basename+'_{}_NoF'.format(j)+'.jpg'
cropped = image.crop((xmin, ymin, xmax, ymax))
width_cropped, height_cropped = cropped.size
if height_cropped > width_cropped: cropped = cropped.transpose(method=2)
cropped.save(file_crop)
GT_crop_bboxs_df.loc[len(GT_crop_bboxs_df)]=[file_crop.split('/')[-1],xmin,ymin,xmax,ymax]
GT_crop_bboxs_df.to_pickle('../data/train_crop/GT_crop_files_BBox.pickle')
count['NoF'] = len(os.listdir(TRAIN_CROP_DIR+'NoF'))
print(count)
In [21]:
#visualize
ALB_images = os.listdir(TRAIN_CROP_DIR+'ALB')
for j in range(1000,1010):
im = Image.open(TRAIN_CROP_DIR+'ALB/'+ALB_images[j])
im = np.asarray(im)
fig, ax = plt.subplots()
ax.imshow(im, aspect='equal')
ax.set_title(('{:s}').format(ALB_images[j]),fontsize=14)
plt.axis('off')
plt.tight_layout()
plt.draw()
In [3]:
#Loading data
import pickle
def read_image(src):
"""Read and resize individual images"""
im = Image.open(src)
im = im.resize((COLS, ROWS), Image.BILINEAR)
im = np.asarray(im)
return im
if os.path.exists('../data/data_train_BBCrop_{}_{}.pickle'.format(ROWS, COLS)):
print ('Exist data_train_BBCrop_{}_{}.pickle. Loading data from file.'.format(ROWS, COLS))
with open('../data/data_train_BBCrop_{}_{}.pickle'.format(ROWS, COLS), 'rb') as f:
data_train = pickle.load(f)
X_train_crop = data_train['X_train_crop']
y_train_crop = data_train['y_train_crop']
train_crop_files = data_train['train_crop_files']
class_weight = data_train['class_weight']
else:
print ('Loading data from original images. Generating data_train_BBCrop_{}_{}.pickle.'.format(ROWS, COLS))
y_train_crop = []
train_crop_files = []
for fish in FISH_CLASSES:
fish_dir = TRAIN_CROP_DIR+'{}'.format(fish)
fish_files = [fish+'/'+im for im in os.listdir(fish_dir)]
train_crop_files.extend(fish_files)
y_fish = np.tile(fish, len(fish_files))
y_train_crop.extend(y_fish)
y_train_crop = np.array(y_train_crop)
X_train_crop = np.ndarray((len(train_crop_files), ROWS, COLS, 3), dtype=np.uint8)
for i, im in enumerate(train_crop_files):
X_train_crop[i] = read_image(TRAIN_CROP_DIR+im)
if i%1000 == 0: print('Processed {} of {}'.format(i, len(train_crop_files)))
# class_weight
y_train_crop = le.transform(y_train_crop)
class_weight = dict(collections.Counter(y_train_crop))
ref = max(class_weight.values())
for key,value in class_weight.items():
class_weight[key] = ref/value
# One Hot Encoding Labels
y_train_crop = np_utils.to_categorical(y_train_crop)
train_crop_files = [file.split('/')[-1] for file in train_crop_files]
#save data to file
data_train = {'X_train_crop': X_train_crop,'y_train_crop': y_train_crop,'train_crop_files': train_crop_files,'class_weight':class_weight}
with open('../data/data_train_BBCrop_{}_{}.pickle'.format(ROWS, COLS), 'wb') as f:
pickle.dump(data_train, f)
#rescale
# print('Loading data done.')
# X_train_crop = X_train_crop.astype(np.float32)
# print('Convert to float32 done.')
# X_train_crop /= 255.
# print('Rescale by 255 done.')
# #traing stg1 and stg2 and resume should have the same train test split!!! Remenber to set the random_state!
# X_train, X_valid, y_train, y_valid = train_test_split(X_train_crop, y_train_crop, test_size=0.2, random_state=1986, stratify=y_train_crop)
# del X_train_crop
In [23]:
y_train_crop = []
train_crop_files = []
crop_classes=FISH_CLASSES[:]
crop_classes.remove('NoF')
count = {}
for fish in crop_classes:
fish_dir = TRAIN_CROP_DIR+'{}'.format(fish)
fish_files = [fish+'/'+im for im in os.listdir(fish_dir)]
train_crop_files.extend(fish_files)
y_fish = np.tile(fish, len(fish_files))
y_train_crop.extend(y_fish)
y_train_crop = np.array(y_train_crop)
X_train_crop = np.ndarray((len(train_crop_files), ROWS, COLS, 3), dtype=np.uint8)
for i, im in enumerate(train_crop_files):
X_train_crop[i] = read_image(TRAIN_CROP_DIR+im)
if i%1000 == 0: print('Processed {} of {}'.format(i, len(train_crop_files)))
In [4]:
#data preprocessing
train_datagen = ImageDataGenerator(
featurewise_center=True,
#featurewise_std_normalization=True,
#rescale=1./255,
rotation_range=180,
shear_range=0.2,
zoom_range=0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True,
vertical_flip=True)
train_datagen.fit(X_train)
train_generator = train_datagen.flow(X_train, y_train, batch_size=BatchSize, shuffle=True, seed=None)
valid_datagen = ImageDataGenerator(
featurewise_center=True)
#featurewise_std_normalization=True)
#rescale=1./255
valid_datagen.fit(X_valid)
valid_generator = valid_datagen.flow(X_valid, y_valid, batch_size=BatchSize, shuffle=True, seed=None)
In [5]:
#callbacks
early_stopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=10, verbose=1, mode='auto')
model_checkpoint = ModelCheckpoint(filepath=CHECKPOINT_DIR+'weights.{epoch:03d}-{val_loss:.4f}.hdf5', monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=False, mode='auto')
learningrate_schedule = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=5, verbose=1, mode='auto', epsilon=0.001, cooldown=0, min_lr=0)
tensorboard = TensorBoard(log_dir='./logs/log3', histogram_freq=0, write_graph=True, write_images=True)
In [8]:
#Resnet50
#stg1 training
from keras.applications.resnet50 import ResNet50
base_model = ResNet50(weights='imagenet', include_top=False)
x = base_model.output
x = GlobalAveragePooling2D()(x)
#x = Flatten()(x)
#x = Dense(256, init='glorot_normal', activation='relu')(x)
#x = LeakyReLU(alpha=0.33)(x)
#x = Dropout(0.5)(x)
#x = Dense(256, init='glorot_normal', activation='relu')(x)
#x = LeakyReLU(alpha=0.33)(x)
#x = Dropout(0.5)(x)
predictions = Dense(len(FISH_CLASSES), init='glorot_normal', activation='softmax')(x)
# this is the model we will train
model = Model(input=base_model.input, output=predictions)
# first: train only the top layers (which were randomly initialized)
# i.e. freeze all convolutional VGG16 layers
for layer in base_model.layers:
layer.trainable = False
# compile the model (should be done *after* setting layers to non-trainable)
optimizer = Adam(lr=LearningRate)
model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])
# train the model on the new data for a few epochs
model.fit_generator(train_generator, samples_per_epoch=len(X_train), nb_epoch=30, verbose=1,
callbacks=[early_stopping, model_checkpoint, learningrate_schedule, tensorboard],
validation_data=valid_generator, nb_val_samples=len(X_valid), class_weight=class_weight, nb_worker=3, pickle_safe=True)
Out[8]:
In [ ]:
#Resnet50
#stg2 training
files = glob.glob(CHECKPOINT_DIR+'*')
val_losses = [float(f.split('-')[-1][:-5]) for f in files]
index = val_losses.index(min(val_losses))
print('Loading model from checkpoints file ' + files[index])
model = load_model(files[index])
# print('Loading model from weights.004-0.0565.hdf5')
# model = load_model('./checkpoints/checkpoint3/weights.004-0.0565.hdf5')
from keras.applications.resnet50 import ResNet50
base_model = ResNet50(weights='imagenet', include_top=False)
# at this point, the top layers are well trained and we can start fine-tuning
# convolutional layers from inception V3. We will freeze the bottom N layers
# and train the remaining top layers.
# let's visualize layer names and layer indices to see how many layers
# we should freeze:
# for i, layer in enumerate(base_model.layers):
# print(i, layer.name)
# we chose to train the top 2 inception blocks, i.e. we will freeze
# the first 172 layers and unfreeze the rest:
#164
for layer in modael.layers[:142]:
layer.trainable = False
for layer in model.layers[142:]:
layer.trainable = True
# we need to recompile the model for these modifications to take effect
# we use SGD with a low learning rate
optimizer = Adam(lr=LearningRate)
model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])
model.fit_generator(train_generator, samples_per_epoch=len(X_train), nb_epoch=300, verbose=1,
callbacks=[early_stopping, model_checkpoint, learningrate_schedule, tensorboard],
validation_data=valid_generator, nb_val_samples=len(X_valid), class_weight=class_weight, nb_worker=3, pickle_safe=True)
In [8]:
#Resnet50
#stg3 training
files = glob.glob(CHECKPOINT_DIR+'*')
val_losses = [float(f.split('-')[-1][:-5]) for f in files]
index = val_losses.index(min(val_losses))
print('Loading model from checkpoints file ' + files[index])
model = load_model(files[index])
# print('Loading model from weights.004-0.0565.hdf5')
# model = load_model('./checkpoints/checkpoint3/weights.004-0.0565.hdf5')
# from keras.applications.resnet50 import ResNet50
# base_model = ResNet50(weights='imagenet', include_top=False)
# at this point, the top layers are well trained and we can start fine-tuning
# convolutional layers from inception V3. We will freeze the bottom N layers
# and train the remaining top layers.
# let's visualize layer names and layer indices to see how many layers
# we should freeze:
# for i, layer in enumerate(base_model.layers):
# print(i, layer.name)
# we chose to train the top 2 inception blocks, i.e. we will freeze
# the first 172 layers and unfreeze the rest:
#164,142,80
for layer in model.layers[:80]:
layer.trainable = False
for layer in model.layers[80:]:
layer.trainable = True
# we need to recompile the model for these modifications to take effect
# we use SGD with a low learning rate
optimizer = Adam(lr=1e-5)
model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])
model.fit_generator(train_generator, samples_per_epoch=len(X_train), nb_epoch=300, verbose=1,
callbacks=[early_stopping, model_checkpoint, learningrate_schedule, tensorboard],
validation_data=valid_generator, nb_val_samples=len(X_valid), class_weight=class_weight, nb_worker=3, pickle_safe=True)
Out[8]:
In [7]:
#Resnet50
#stg4 training
files = glob.glob(CHECKPOINT_DIR+'*')
val_losses = [float(f.split('-')[-1][:-5]) for f in files]
index = val_losses.index(min(val_losses))
print('Loading model from checkpoints file ' + files[index])
model = load_model(files[index])
# print('Loading model from weights.004-0.0565.hdf5')
# model = load_model('./checkpoints/checkpoint3/weights.004-0.0565.hdf5')
# from keras.applications.resnet50 import ResNet50
# base_model = ResNet50(weights='imagenet', include_top=False)
# at this point, the top layers are well trained and we can start fine-tuning
# convolutional layers from inception V3. We will freeze the bottom N layers
# and train the remaining top layers.
# let's visualize layer names and layer indices to see how many layers
# we should freeze:
# for i, layer in enumerate(base_model.layers):
# print(i, layer.name)
# we chose to train the top 2 inception blocks, i.e. we will freeze
# the first 172 layers and unfreeze the rest:
#164,142,80,38
# for layer in model.layers[:38]:
# layer.trainable = False
# for layer in model.layers[38:]:
# layer.trainable = True
# we need to recompile the model for these modifications to take effect
# we use SGD with a low learning rate
optimizer = Adam(lr=1e-5)
model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])
model.fit_generator(train_generator, samples_per_epoch=len(X_train), nb_epoch=300, verbose=1,
callbacks=[early_stopping, model_checkpoint, learningrate_schedule, tensorboard],
validation_data=valid_generator, nb_val_samples=len(X_valid), class_weight=class_weight, nb_worker=3, pickle_safe=True)
Out[7]:
In [ ]:
#resume training
files = glob.glob(CHECKPOINT_DIR+'*')
val_losses = [float(f.split('-')[-1][:-5]) for f in files]
index = val_losses.index(min(val_losses))
print('Loading model from checkpoints file ' + files[index])
model = load_model(files[index])
# print('Loading model from weights.004-0.0565.hdf5')
# model = load_model('./checkpoints/checkpoint3/weights.004-0.0565.hdf5')
model.fit_generator(train_generator, samples_per_epoch=len(X_train), nb_epoch=30, verbose=1,
callbacks=[early_stopping, model_checkpoint, learningrate_schedule, tensorboard],
validation_data=valid_generator, nb_val_samples=len(X_valid), class_weight=class_weight, nb_worker=3, pickle_safe=True)
In [18]:
#test-time augmentation
# print ('Exist data_train_BBCrop_{}_{}.pickle. Loading data from file.'.format(ROWS, COLS))
# with open('../data/data_train_BBCrop_{}_{}.pickle'.format(ROWS, COLS), 'rb') as f:
# data_train = pickle.load(f)
# X_train_crop = data_train['X_train_crop']
# y_train_crop = data_train['y_train_crop']
# GT_crop_files = data_train['train_crop_files']
# class_weight = data_train['class_weight']
# print('Loading data done.')
# X_train_crop = X_train_crop.astype(np.float32)
# print('Convert to float32 done.')
# X_train_crop /= 255.
# print('Rescale by 255 done.')
# #featurewise_center
# mean = np.mean(X_train_crop, axis=0, keepdims=True)
# mean = np.mean(mean, axis=(1,2), keepdims=True)
# X_train_crop_centered = X_train_crop - mean
# #pad to can be divided by BatchSize
# X_pad_array = np.zeros((math.ceil(len(X_train_crop)/BatchSize)*BatchSize-len(X_train_crop),ROWS,COLS,3), dtype=np.float32)
# X_train_crop_centered = np.concatenate((X_train_crop_centered, X_pad_array), axis=0)
# y_pad_array = np.zeros((math.ceil(len(y_train_crop)/BatchSize)*BatchSize-len(y_train_crop),len(FISH_CLASSES)), dtype=np.float64)
# y_train_crop = np.concatenate((y_train_crop, y_pad_array), axis=0)
# files = glob.glob(CHECKPOINT_DIR+'*')
# val_losses = [float(f.split('-')[-1][:-5]) for f in files]
# index = val_losses.index(min(val_losses))
# print('Loading model from checkpoints file ' + files[index])
# model = load_model(files[index])
# # print('Loading model from weights.004-0.0565.hdf5')
# # model = load_model('./checkpoints/checkpoint2/weights.004-0.0565.hdf5')
test_aug_datagen = ImageDataGenerator(
# featurewise_center=True,
rotation_range=0,
shear_range=0,
zoom_range=0,
width_shift_range=0,
height_shift_range=0,
horizontal_flip=False,
vertical_flip=False)
# test_aug_datagen.fit(X_train_crop)
nbr_augmentation = 1
random_seed = [1986,8,22,7,13]
for idx in range(nbr_augmentation):
print('{}th augmentation for testing ...'.format(idx))
#shuffle=False!!!
test_aug_generator = test_aug_datagen.flow(X_train_crop_centered, y_train_crop, batch_size=BatchSize, shuffle=False, seed=random_seed[idx])
print('Begin to predict for testing data ...')
if idx == 0:
GT_crop_preds = model.predict_generator(test_aug_generator, val_samples=len(X_train_crop_centered), nb_worker=3, pickle_safe=True)
else:
GT_crop_preds += model.predict_generator(test_aug_generator, val_samples=len(X_train_crop_centered), nb_worker=3, pickle_safe=True)
GT_crop_preds /= nbr_augmentation
In [19]:
GT_crop_preds[19479:,:]
Out[19]:
In [21]:
GT_crop_preds[3000:3005]
Out[21]:
In [23]:
GT_crop_preds = GT_crop_preds[:19479,:]
In [9]:
#get GT_crop_BBClassifier_preds
print ('Exist data_train_BBCrop_{}_{}.pickle. Loading data from file.'.format(ROWS, COLS))
with open('../data/data_train_BBCrop_{}_{}.pickle'.format(ROWS, COLS), 'rb') as f:
data_train = pickle.load(f)
X_train_crop = data_train['X_train_crop']
y_train_crop = data_train['y_train_crop']
GT_crop_files = data_train['train_crop_files']
class_weight = data_train['class_weight']
print('Loading data done.')
X_train_crop = X_train_crop.astype(np.float32)
print('Convert to float32 done.')
X_train_crop /= 255.
print('Rescale by 255 done.')
#featurewise_center
mean = np.mean(X_train_crop, axis=0, keepdims=True)
mean = np.mean(mean, axis=(1,2), keepdims=True)
X_train_crop_centered = X_train_crop - mean
files = glob.glob(CHECKPOINT_DIR+'*')
val_losses = [float(f.split('-')[-1][:-5]) for f in files]
index = val_losses.index(min(val_losses))
print('Loading model from checkpoints file ' + files[index])
model = load_model(files[index])
# print('Loading model from weights.004-0.0565.hdf5')
# model = load_model('./checkpoints/checkpoint2/weights.004-0.0565.hdf5')
print(model.evaluate(X_train_crop_centered, y_train_crop, batch_size=BatchSize, verbose=1))
GT_crop_preds = model.predict(X_train_crop_centered, batch_size=BatchSize, verbose=1)
GT_crop_preds_df = pd.DataFrame(GT_crop_preds, columns=['ALB', 'BET', 'DOL', 'LAG', 'NoF', 'OTHER', 'SHARK', 'YFT'])
GT_crop_preds_df.insert(0,'GT_crop_files',GT_crop_files)
#FISH_CLASSES = ['ALB', 'BET', 'DOL', 'LAG', 'NoF', 'OTHER', 'SHARK', 'YFT']
#GT_crop_preds_df.insert(0,'y_train_crop',[FISH_CLASSES[i] for i in np.argmax(y_train_crop, axis=1)])
GT_crop_bboxs_df = pd.read_pickle('../data/train_crop/GT_crop_files_BBox.pickle')
GT_crop_BBClassifier_preds = pd.merge(GT_crop_bboxs_df, GT_crop_preds_df)
GT_crop_BBClassifier_preds['image_files'] = GT_crop_BBClassifier_preds.GT_crop_files.apply(lambda x: 'img_'+x.split('_')[2])
GT_crop_BBClassifier_preds['gt'] = GT_crop_BBClassifier_preds.GT_crop_files.apply(lambda x: x.split('_')[-1][:-4])
#add logloss
columns_reorder = ['image_files', 'GT_crop_files', 'gt', 'xmin', 'ymin', 'xmax', 'ymax', 'NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']
GT_crop_BBClassifier_preds = GT_crop_BBClassifier_preds[columns_reorder]
def f_logloss(row):
fish = row[2]
ind = ['NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT'].index(fish)
return -math.log(row[7+ind])
GT_crop_BBClassifier_preds['logloss'] = GT_crop_BBClassifier_preds.apply(f_logloss, axis=1)
columns_reorder = ['image_files', 'GT_crop_files', 'gt', 'logloss', 'xmin', 'ymin', 'xmax', 'ymax', 'NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']
GT_crop_BBClassifier_preds = GT_crop_BBClassifier_preds[columns_reorder]
GT_crop_BBClassifier_preds[5000:5005]
Out[9]:
In [10]:
#class_weight stg3 weights.014-0.0369.hdf5
GT_crop_BBClassifier_preds.groupby(['gt'])['logloss'].mean()
Out[10]:
In [27]:
#class_weight stg2
GT_crop_BBClassifier_preds.groupby(['gt'])['logloss'].mean()
Out[27]:
In [12]:
#before class_weight
GT_crop_BBClassifier_preds.groupby(['gt'])['logloss'].mean()
Out[12]:
In [11]:
#get RFCN bbox from detections_full_AGNOSTICnms.pkl
RFCN_MODEL = 'resnet101_rfcn_ohem_iter_30000'
import pickle
with open('../data/RFCN_detections/detections_full_AGNOSTICnms_'+RFCN_MODEL+'.pkl','rb') as f:
detections_full_AGNOSTICnms = pickle.load(f, encoding='latin1')
outputs = []
count = np.zeros(len(detections_full_AGNOSTICnms))
for im in range(len(detections_full_AGNOSTICnms)):
outputs_im = []
detects_im = detections_full_AGNOSTICnms[im]
for i in range(len(detects_im)):
if np.max(detects_im[i,5:]) >= CONF_THRESH:
outputs_im.append(detects_im[i,:])
count[im] = len(outputs_im)
if len(outputs_im) == 0:
ind = np.argmax(np.max(detects_im[:,5:], axis=1))
outputs_im.append(detects_im[ind,:])
outputs_im = np.asarray(outputs_im)
outputs.append(outputs_im)
#crop test images and cache to TEST_CROP_DIR
# if not os.path.exists(TEST_CROP_DIR):
# os.mkdir(TEST_CROP_DIR)
# files = glob.glob(TEST_CROP_DIR+'*')
# for f in files:
# os.remove(f)
# with open("../RFCN/ImageSets/Main/test.txt","r") as f:
# ims = f.readlines()
# test_files = [im[:-1]+'.jpg' for im in ims]
# for i in range(len(outputs)):
# if i%1000 == 0:
# print(i)
# filename = test_files[i]
# bboxes = outputs[i]
# basename, file_extension = os.path.splitext(filename)
# image = Image.open(TEST_DIR+filename)
# for j in range(len(bboxes)):
# bbox = bboxes[j]
# xmin = bbox[0]
# ymin = bbox[1]
# xmax = bbox[2]
# ymax = bbox[3]
# file_crop = TEST_CROP_DIR+basename+'_{}'.format(j)+'.jpg'
# cropped = image.crop((xmin, ymin, xmax, ymax))
# width_cropped, height_cropped = cropped.size
# if height_cropped > width_cropped: cropped = cropped.transpose(method=2)
# cropped.save(file_crop)
print('train_image:{} RFCN>conf_crop:{} RFCN_total_crop:{}'.format(len(outputs)-1000,int(sum(count[1000:])), int(sum([outputs[i].shape[0] for i in range(1000,len(outputs))]))))
print('test_image: {} RFCN>conf_crop:{} RFCN_total_crop:{}'.format(1000,int(sum(count[:1000])), int(sum([outputs[i].shape[0] for i in range(1000)]))))
In [12]:
#get RFCN_crop_RFCN_preds
test_crop_preds = np.vstack(outputs)[:,:]
columns = ['x0', 'y0', 'x1', 'y1','NoF_RFCN', 'ALB_RFCN', 'BET_RFCN', 'DOL_RFCN', 'LAG_RFCN', 'OTHER_RFCN', 'SHARK_RFCN', 'YFT_RFCN']
RFCN_preds_df = pd.DataFrame(test_crop_preds, columns=columns)
with open("../RFCN/ImageSets/Main/test.txt","r") as f:
ims = f.readlines()
test_files = [im[:-1]+'.jpg' for im in ims]
test_crop_files_RFCN = []
for i in range(len(outputs)):
filename = test_files[i]
basename, file_extension = os.path.splitext(filename)
for j in range(len(outputs[i])):
file_crop = basename+'_{}_'.format(j)+'.jpg'
test_crop_files_RFCN.append(file_crop)
RFCN_preds_df.insert(0, 'test_crop_files', test_crop_files_RFCN)
In [ ]:
#Load test data
import datetime
def read_image(src):
"""Read and resize individual images"""
im = Image.open(src)
im = im.resize((COLS, ROWS), Image.BILINEAR)
im = np.asarray(im)
return im
if os.path.exists('../data/data_test_BBCrop_{}_{}.pickle'.format(ROWS, COLS)):
print ('Exist data_test_BBCrop_{}_{}.pickle. Loading test data from file.'.format(ROWS, COLS))
with open('../data/data_test_BBCrop_{}_{}.pickle'.format(ROWS, COLS), 'rb') as f:
data_test = pickle.load(f)
X_test_crop = data_test['X_test_crop']
test_crop_files = data_test['test_crop_files']
else:
print ('Loading test data from original images. Generating data_test_BBCrop_{}_{}.pickle.'.format(ROWS, COLS))
test_crop_files = sorted([im for im in os.listdir(TEST_CROP_DIR)])
X_test_crop = np.ndarray((len(test_crop_files), ROWS, COLS, 3), dtype=np.uint8)
for i, im in enumerate(test_crop_files):
X_test_crop[i] = read_image(TEST_CROP_DIR+im)
if i%1000 == 0: print('Processed {} of {}'.format(i, len(test_crop_files)))
data_test = {'X_test_crop': X_test_crop,'test_crop_files': test_crop_files }
with open('../data/data_test_BBCrop_{}_{}.pickle'.format(ROWS, COLS), 'wb') as f:
pickle.dump(data_test, f, protocol=4)
print('Loading data done.')
X_test_crop = X_test_crop.astype(np.float32)
print('Convert to float32 done.')
X_test_crop /= 255.
print('Rescale by 255 done.')
In [14]:
#get RFCN_crop_BBClassifier_preds
files = glob.glob(CHECKPOINT_DIR+'*')
val_losses = [float(f.split('-')[-1][:-5]) for f in files]
index = val_losses.index(min(val_losses))
print('Loading model from checkpoints file ' + files[index])
model = load_model(files[index])
# print('Loading model from weights.004-0.0565.hdf5')
# model = load_model('./checkpoints/checkpoint2/weights.004-0.0565.hdf5')
X_test_crop_centered = featurewise_center(X_test_crop)
test_crop_preds = model.predict(X_test_crop_centered, batch_size=BatchSize, verbose=1)
columns = ['ALB_BBCROP', 'BET_BBCROP', 'DOL_BBCROP', 'LAG_BBCROP', 'NoF_BBCROP', 'OTHER_BBCROP', 'SHARK_BBCROP', 'YFT_BBCROP']
BBCROP_preds_df = pd.DataFrame(test_crop_preds, columns=columns)
test_crop_files_BBCROP = test_crop_files
BBCROP_preds_df.insert(0, 'test_crop_files', test_crop_files_BBCROP)
In [15]:
#get RFCN_crop_RFCN_BBClassifier_preds
test_preds_df = pd.merge(RFCN_preds_df, BBCROP_preds_df)
test_preds_df['test_files'] = test_preds_df.test_crop_files.apply(lambda x: 'img_'+x.split('_')[1])
#add ground truth from ImageSets/Main/train_test.txt to test_preds_df
with open("../RFCN/ImageSets/Main/train_test.txt","r") as f:
train_file_labels = f.readlines()
for index, row in test_preds_df.iterrows():
im = row['test_files']
gt = 'nan'
logloss_RFCN = -np.inf
logloss_BBCROP = -np.inf
for im_label in train_file_labels:
if im_label[:9] == im:
gt = im_label[10:-1]
logloss_RFCN = -math.log(row[gt+'_RFCN'])
logloss_BBCROP = -math.log(row[gt+'_BBCROP'])
test_preds_df.set_value(index,'gt',gt)
test_preds_df.set_value(index,'logloss_RFCN',logloss_RFCN)
test_preds_df.set_value(index,'logloss_BBCROP',logloss_BBCROP)
columns_reorder = ['test_files', 'gt', 'logloss_RFCN', 'logloss_BBCROP', 'test_crop_files', 'x0', 'y0', 'x1', 'y1']
FISH_CLASSES = ['NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']
for c in FISH_CLASSES:
columns_reorder.append(c+'_RFCN')
columns_reorder.append(c+'_BBCROP')
test_preds_df = test_preds_df[columns_reorder]
test_preds_df.head()
Out[15]:
In [35]:
#visualization
#import xml.etree.ElementTree
FISH_CLASSES = ['NoF', 'ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT']
with open("../RFCN/ImageSets/Main/test.txt","r") as f:
ims = f.readlines()
test_files = [im[:-1] for im in ims][1000:]
for j in range(20):
RFCN_dets = test_preds_df.loc[test_preds_df['test_files']==test_files[j]]
im = Image.open('../RFCN/JPEGImages/'+test_files[j]+'.jpg')
im = np.asarray(im)
fig, ax = plt.subplots(figsize=(10, 8))
ax.imshow(im, aspect='equal')
for index,row in RFCN_dets.iterrows():
row = row.tolist()
bbox = row[5:9]
RFCN = [row[i] for i in [9,11,13,15,17,19,21,23]]
BBCROP = [row[i] for i in [10,12,14,16,18,20,22,24]]
score_RFCN = max(RFCN)
score_BBCROP = max(BBCROP)
index_RFCN = RFCN.index(score_RFCN)
index_BBCROP = BBCROP.index(score_BBCROP)
class_RFCN = FISH_CLASSES[index_RFCN]
class_BBCROP = FISH_CLASSES[index_BBCROP]
ax.add_patch(plt.Rectangle((bbox[0], bbox[1]), bbox[2] - bbox[0], bbox[3] - bbox[1], fill=False, edgecolor='red', linewidth=2))
ax.text(bbox[0], bbox[1] - 2, 'RFCN_{:s} {:.3f} \nCROP_{:s} {:.3f}'.format(class_RFCN, score_RFCN, class_BBCROP, score_BBCROP), bbox=dict(facecolor='red', alpha=0.5), fontsize=8, color='white')
GT_dets = GT_crop_BBClassifier_preds.loc[GT_crop_BBClassifier_preds['image_files']==test_files[j]]
for index,row in GT_dets.iterrows():
row = row.tolist()
bbox = row[4:8]
BBCROP = row[8:]
score_BBCROP = max(BBCROP)
index_BBCROP = BBCROP.index(score_BBCROP)
class_BBCROP = FISH_CLASSES[index_BBCROP]
ax.add_patch(plt.Rectangle((bbox[0], bbox[1]), bbox[2] - bbox[0], bbox[3] - bbox[1], fill=False, edgecolor='green', linewidth=2))
ax.text(bbox[0], bbox[3] + 40, 'GT_{:s} \nCROP_{:s} {:.3f}'.format(row[2], class_BBCROP, score_BBCROP), bbox=dict(facecolor='green', alpha=0.5), fontsize=8, color='white')
# root = xml.etree.ElementTree.parse('../RFCN/Annotations/'+test_files[j]+'.xml').getroot()
# for child in root.findall('object'):
# bbox = [child.find('bndbox').find('xmin').text, child.find('bndbox').find('ymin').text, child.find('bndbox').find('xmax').text,child.find('bndbox').find('ymax').text]
# bbox = [float(x) for x in bbox]
# ax.add_patch(plt.Rectangle((bbox[0], bbox[1]), bbox[2] - bbox[0], bbox[3] - bbox[1], fill=False, edgecolor='green', linewidth=2))
# ax.text(bbox[0], bbox[3] - 2, 'GT_{:s}'.format(child.find('name').text), bbox=dict(facecolor='green', alpha=0.5), fontsize=8, color='white')
ax.set_title(('Image {:s} {:s}').format(test_files[j], row[1]), fontsize=10)
plt.axis('off')
plt.tight_layout()
plt.draw()