In [1]:
# Try predicting the augmented versions as well
# aggregate the results from all augmentations of 1 image
# Average/weighted average the predictions from an ensemble of nets
In [2]:
import numpy as np
import time
import sys
import tools.my_io as my_io
import caffe
import os
import pickle
import itertools
from scipy.misc import imresize
mode = 'normal'
# mode = 'aug'
# Set the right path to your model definition file, pretrained model weights,
# and the image you would like to classify.
# MODEL_FILE = './deploy_vanilla.prototxt'
# MODEL_FILE = './deploy_deeper.prototxt'
MODEL_FILE = '/media/raid_arr/data/ndsb/config/deploy_shallow_googlenet.prototxt'
# PRETRAINED = './models/vanilla/vanilla_iter_20000.caffemodel'
# PRETRAINED = '/media/raid_arr/data/ndsb/models/pl_iter_60000.caffemodel'
PRETRAINED = '/media/raid_arr/data/ndsb/models/shallow_gnet_iter_88000.caffemodel'
MEAN_FILE = '/media/raid_arr/data/ndsb/augment/testaug_mean.npy'
# TEST_FILE = './data/test_final.txt'
if mode == 'aug':
TEST_DB = '/data/ndsb/augment/ndsb_testaug_lmdb/'
else:
TEST_DB = '/data/ndsb/ndsb_test_lmdb'
N = 20000 # Chunk size
model_name = os.path.splitext(os.path.basename(PRETRAINED))[0]
In [3]:
# # Loading From Database
# print 'Loading data...'
# tic = time.time()
# data = my_io.load_lmdb(TEST_DB)
# print "Done in %.2f s." % (time.time() - tic)
# # test_files_all, images, labels = zip(*data)
# # test_labels = labels
In [4]:
# image_dims = data[0][1].shape[:2]
caffe.set_mode_gpu()
# caffe.set_phase_test()
image_dims = (64, 64)
crop_dims = np.array([57, 57])
mean=np.load(MEAN_FILE)
mean.shape
mean_resized = caffe.io.resize_image(mean.transpose((1,2,0)), crop_dims).transpose((2,0,1))
net = caffe.Classifier(MODEL_FILE, PRETRAINED,
mean=mean_resized,
raw_scale=1.0, # 255 if load from caffe.io, 1.0 if load from my_io lmdb
image_dims=image_dims)
In [5]:
# PREDICTION TIME
print 'Predicting...', TEST_DB
prediction_list = []
test_files_list = []
next_key = ''
first_run = True
while next_key or first_run:
first_run = False
print 'Starting at key: ', next_key
read_start = time.time()
data_chunk, next_key = my_io.load_lmdb_chunk(TEST_DB, next_key, N)
print "Read done in %.2f s." % (time.time() - read_start)
print 'Chunk size:', len(data_chunk)
sys.stdout.flush()
pred_start = time.time()
test_files_chunk, images_chunk, fake_labels = zip(*data_chunk)
prediction = net.predict(images_chunk)
# prediction = np.array([1]) # for testing db read
prediction_list.append(prediction)
test_files_list.append(test_files_chunk)
print "Pred done in %.2f s." % (time.time() - pred_start)
sys.stdout.flush()
predictions = np.concatenate(prediction_list)
test_files = list(itertools.chain(*test_files_list))
print "Done predicting"
In [6]:
## Saving predictions
# pred_save_path = '/media/raid_arr/data/ndsb/saved_preds/pred.p'
# pickle.dump(predictions, open(pred_save_path, 'wb'))
# print 'Saved predictions:', pred_save_path
# # Saving test_file_paths
# test_f_save_path = '/media/raid_arr/data/ndsb/saved_preds/test_files.p'
# pickle.dump(test_files, open(test_f_save_path, 'wb'))
# print 'Saved predictions:', test_f_save_path
In [7]:
if mode == 'aug':
# Averaging over predictions for augmentations
test_files_arr = np.array(test_files)
# Setting up base names
base_im = lambda f_path: os.path.basename(f_path).split('_')[0]
start = time.time()
base_names = np.array([base_im(f_path) for f_path in test_files_arr])
print "Base names done in %.2f s." % (time.time() - start)
In [8]:
if mode == 'aug':
unique_im_names = list(set(base_names))
predictions_agg = np.zeros((len(unique_ims), predictions.shape[1]))
for ii, im_name in enumerate(unique_im_names):
inds = base_names == im_name
p_mean = np.mean(predictions[inds, :], axis=0)
predictions_agg[ii, :] = p_mean
print 'Finished Aggregation'
unique_im_f = [im_n + '.jpg' for im_n in unique_im_names]
In [9]:
# SUBMISSION CREATION
# test_files_all, images, labels = zip(*data)
import tools.submission as sub
f_name='SUBMISSION_%s.csv' % model_name
if mode == 'aug':
sub.make_submission(unique_im_f, predictions_agg, f_name=f_name)
else:
sub.make_submission(test_files, predictions, f_name=f_name)
print 'Submission created:', f_name
In [10]:
# np.sum(predictions_agg)
# predictions_agg.shape
# test_files
print 1
In [11]:
for k, v in net.blobs.items():
print k, v.data.shape