In [1]:
import matplotlib
matplotlib.use('agg')
import cPickle as pickle
import os; import sys; sys.path.append('../../')
import gp
import gp.nets as nets
In [2]:
from sklearn.metrics import classification_report, accuracy_score, roc_curve, auc, precision_recall_fscore_support, f1_score, precision_recall_curve, average_precision_score, zero_one_loss
import numpy as np
import time
In [3]:
def r_d(cnn):
# load dojo data
input_image, input_prob, input_gold, input_rhoana, dojo_bbox = gp.Legacy.read_dojo_data()
original_mean_VI, original_median_VI, original_VI_s = gp.Legacy.VI(input_gold, input_rhoana)
# output folder for anything to store
output_folder = '/home/d/netstatsCVPR2018/'+cnn.uuid+'/'
if not os.path.exists(output_folder):
os.makedirs(output_folder)
# # find merge errors, if we did not generate them before
# merge_error_file = output_folder+'/merges_new_cnn.p'
# if os.path.exists(merge_error_file):
# print 'Loading merge errors from file..'
# with open(merge_error_file, 'rb') as f:
# merge_errors = pickle.load(f)
# else:
# print 'Finding Top 5 merge errors..'
# merge_errors = gp.Legacy.get_top5_merge_errors(cnn, input_image, input_prob, input_rhoana)
# with open(merge_error_file, 'wb') as f:
# pickle.dump(merge_errors, f)
# print len(merge_errors), ' merge errors found.'
# we need to create a bigM for the dojo volume
bigM_dojo_file = output_folder + '/bigM_dojo_test2.p'
if os.path.exists(bigM_dojo_file):
print 'Loading dojo bigM from file..'
with open(bigM_dojo_file, 'rb') as f:
bigM_dojo = pickle.load(f)
else:
print 'Creating dojo bigM..'
bigM_dojo = gp.Legacy.create_bigM_without_mask(cnn, input_image, input_prob, input_rhoana, verbose=False)
with open(bigM_dojo_file, 'wb') as f:
pickle.dump(bigM_dojo, f)
print
dojo_vi_95_file = output_folder + '/dojo_vi_95_t6.p'
dojo_merge_vis = output_folder + '/dojo_merge_auto95_vis.p'
dojo_split_vis = output_folder + '/dojo_split_auto95_vis.p'
dojo_merge_fixes = output_folder + '/dojo_merge_auto95_fixes.p'
dojo_split_fixes = output_folder + '/dojo_split_auto95_fixes.p'
dojo_output_95 = output_folder + '/dojo_auto95_output.p'
if os.path.exists(dojo_vi_95_file):
print 'Loading merge errors p < .05 and split errors p > .95 from file..'
with open(dojo_vi_95_file, 'rb') as f:
dojo_vi_95 = pickle.load(f)
else:
# #
# # perform merge correction with p < .05
# #
# print 'Correcting merge errors with p < .05'
# bigM_dojo_05, corrected_rhoana_05, dojo_auto_merge_fixes, vi_s_per_step = gp.Legacy.perform_auto_merge_correction(cnn, bigM_dojo, input_image, input_prob, input_rhoana, merge_errors, .05, input_gold=input_gold)
# print ' Mean VI improvement', original_mean_VI-gp.Legacy.VI(input_gold, corrected_rhoana_05)[0]
# print ' Median VI improvement', original_median_VI-gp.Legacy.VI(input_gold, corrected_rhoana_05)[1]
# with open(dojo_merge_vis, 'wb') as f:
# pickle.dump(vi_s_per_step, f)
# with open(dojo_merge_fixes, 'wb') as f:
# pickle.dump(dojo_auto_merge_fixes, f)
#
# perform split correction with p > .95
#
print 'Correcting split errors with p > .95'
bigM_dojo_after_95, out_dojo_volume_after_auto_95, dojo_auto_fixes_95, dojo_auto_vi_s_95, vi_s_per_step2 = gp.Legacy.splits_global_from_M_automatic(cnn, bigM_dojo, input_image, input_prob, input_rhoana, input_gold, sureness_threshold=.95)
dojo_vi_95 = gp.Legacy.VI(input_gold, out_dojo_volume_after_auto_95)
with open(dojo_vi_95_file, 'wb') as f:
pickle.dump(dojo_vi_95, f)
with open(dojo_split_vis, 'wb') as f:
pickle.dump(vi_s_per_step2, f)
with open(dojo_split_fixes, 'wb') as f:
pickle.dump(dojo_auto_fixes_95, f)
with open(dojo_output_95, 'wb') as f:
pickle.dump(out_dojo_volume_after_auto_95, f)
print ' Mean VI improvement', original_mean_VI-dojo_vi_95[0]
print ' Median VI improvement', original_median_VI-dojo_vi_95[1]
In [ ]:
In [ ]:
In [ ]:
In [4]:
NETS = []
NETS.append('../../nets/IP_FULL.p') # image + prob
NETS.append('../../nets/IPLB_FULL.p') # image + prob + large border
NETS.append('../../nets/IPM_FULL.p') # image + prob + binary
NETS.append('../../nets/IPMB_FULL.p') # image + prob + binary + border
NETS.append('../../nets/IPMLB_FULL.p') # image + prob + binary + large border
print NETS
In [5]:
roc1 = []
roc2 = []
In [6]:
#
# image + prob
#
network_path = NETS[0]
with open(network_path, 'rb') as f:
net = pickle.load(f)
X_test, y_test = gp.Patch.load_rgb_test_only('ipm')
X_test = X_test[:,:2,:,:]
test_prediction = net.predict(X_test)
test_prediction_prob = net.predict_proba(X_test)
print
print 'Precision/Recall:'
print classification_report(y_test, test_prediction)
test_acc = net.score(X_test, y_test)
acc_score = accuracy_score(y_test, test_prediction)
print 'Test Accuracy:', test_acc
print 'Accuracy Score:', acc_score
fpr, tpr, thresholds = roc_curve(y_test, test_prediction)
area = auc(fpr, tpr)
print 'AUC', area
fpr2, tpr2, thresholds = roc_curve(y_test, test_prediction_prob[:,1])
area2 = auc(fpr2, tpr2)
print 'AUC2', area2
roc1.append({'ip':[fpr,tpr,area]})
roc2.append({'ip':[fpr2,tpr2,area2]})
t0 = time.time()
net.uuid = 'IP'
r_d(net)
print time.time()-t0, 'seconds'
In [7]:
#
# image + prob + large border
#
network_path = NETS[1]
with open(network_path, 'rb') as f:
net = pickle.load(f)
X_test, y_test = gp.Patch.load_rgba_test_only('ipmlb')
X_test = np.delete(X_test, [2], axis=1) ### Remove binary mask
test_prediction = net.predict(X_test)
test_prediction_prob = net.predict_proba(X_test)
print
print 'Precision/Recall:'
print classification_report(y_test, test_prediction)
test_acc = net.score(X_test, y_test)
acc_score = accuracy_score(y_test, test_prediction)
print 'Test Accuracy:', test_acc
print 'Accuracy Score:', acc_score
fpr, tpr, thresholds = roc_curve(y_test, test_prediction)
area = auc(fpr, tpr)
print 'AUC', area
fpr2, tpr2, thresholds = roc_curve(y_test, test_prediction_prob[:,1])
area2 = auc(fpr2, tpr2)
print 'AUC2', area2
roc1.append({'iplb':[fpr,tpr,area]})
roc2.append({'iplb':[fpr2,tpr2,area2]})
t0 = time.time()
net.uuid = 'IPLB'
r_d(net)
print time.time()-t0, 'seconds'
In [8]:
#
# image + prob + binary
#
network_path = NETS[2]
with open(network_path, 'rb') as f:
net = pickle.load(f)
X_test, y_test = gp.Patch.load_rgb_test_only('ipm')
test_prediction = net.predict(X_test)
test_prediction_prob = net.predict_proba(X_test)
print
print 'Precision/Recall:'
print classification_report(y_test, test_prediction)
test_acc = net.score(X_test, y_test)
acc_score = accuracy_score(y_test, test_prediction)
print 'Test Accuracy:', test_acc
print 'Accuracy Score:', acc_score
fpr, tpr, thresholds = roc_curve(y_test, test_prediction)
area = auc(fpr, tpr)
print 'AUC', area
fpr2, tpr2, thresholds = roc_curve(y_test, test_prediction_prob[:,1])
area2 = auc(fpr2, tpr2)
print 'AUC2', area2
roc1.append({'ipm':[fpr,tpr,area]})
roc2.append({'ipm':[fpr2,tpr2,area2]})
t0 = time.time()
net.uuid = 'IPM'
r_d(net)
print time.time()-t0, 'seconds'
In [9]:
#
# image + prob + binary + border
#
network_path = NETS[3]
with open(network_path, 'rb') as f:
net = pickle.load(f)
X_test, y_test = gp.Patch.load_rgba_test_only('ipmb')
test_prediction = net.predict(X_test)
test_prediction_prob = net.predict_proba(X_test)
print
print 'Precision/Recall:'
print classification_report(y_test, test_prediction)
test_acc = net.score(X_test, y_test)
acc_score = accuracy_score(y_test, test_prediction)
print 'Test Accuracy:', test_acc
print 'Accuracy Score:', acc_score
fpr, tpr, thresholds = roc_curve(y_test, test_prediction)
area = auc(fpr, tpr)
print 'AUC', area
fpr2, tpr2, thresholds = roc_curve(y_test, test_prediction_prob[:,1])
area2 = auc(fpr2, tpr2)
print 'AUC2', area2
roc1.append({'ipmb':[fpr,tpr,area]})
roc2.append({'ipmb':[fpr2,tpr2,area2]})
t0 = time.time()
net.uuid = 'IPMB'
r_d(net)
print time.time()-t0, 'seconds'
In [10]:
#
# image + prob + binary + large border
#
network_path = NETS[4]
with open(network_path, 'rb') as f:
net = pickle.load(f)
X_test, y_test = gp.Patch.load_rgba_test_only('ipmlb')
test_prediction = net.predict(X_test)
test_prediction_prob = net.predict_proba(X_test)
print
print 'Precision/Recall:'
print classification_report(y_test, test_prediction)
test_acc = net.score(X_test, y_test)
acc_score = accuracy_score(y_test, test_prediction)
print 'Test Accuracy:', test_acc
print 'Accuracy Score:', acc_score
fpr, tpr, thresholds = roc_curve(y_test, test_prediction)
area = auc(fpr, tpr)
print 'AUC', area
fpr2, tpr2, thresholds = roc_curve(y_test, test_prediction_prob[:,1])
area2 = auc(fpr2, tpr2)
print 'AUC2', area2
roc1.append({'ipmlb':[fpr,tpr,area]})
roc2.append({'ipmlb':[fpr2,tpr2,area2]})
t0 = time.time()
net.uuid = 'IPMLB'
r_d(net)
print time.time()-t0, 'seconds'
In [11]:
net.input_names
In [ ]:
%pylab inline
import time
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]: