In [1]:
# remove after testing
%load_ext autoreload
%autoreload 2
In [2]:
import pickle
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import PolynomialFeatures
from sklearn.cross_validation import StratifiedShuffleSplit
from sklearn.ensemble import BaggingClassifier
from sklearn.svm import SVC
from mclearn.active import run_active_learning_expt
from mclearn.heuristics import (random_h, margin_h, entropy_h,
qbb_margin_h, qbb_kl_h, pool_entropy_h, pool_variance_h)
from mclearn.tools import results_exist, load_results
from mclearn.preprocessing import balanced_train_test_split
from mclearn.viz import plot_learning_curve_df, plot_final_accuracy, order_learning_curves
%matplotlib inline
sns.set_style('ticks')
sns.set_palette('Dark2', 8)
In [3]:
running_expt = False # set to False if only want to load in results
fig_dir = '../thesis/figures/'
target_col = 'class'
vstatlas_features = ['rmagC', 'umg', 'gmr', 'rmi', 'imz', 'rmw1', 'w1m2']
xrange = np.arange(50, 301)
colors = {'Random': '#e6ab02',
'Entropy': '#e7298a',
'Margin': '#d95f02',
'QBB Margin': '#1b9e77',
'QBB KL': '#66a61e',
'Pool Variance': '#7570b3',
'Pool Entropy': '#a6761d',
'Thompson': '#666666'}
linestyles = {'Random': '-',
'Entropy': '-',
'Margin': '-',
'QBB Margin': '-',
'QBB KL': '-',
'Pool Variance': '-',
'Pool Entropy': '-',
'Thompson': '-'}
labels = ['Random', 'Entropy', 'Margin', 'QBB Margin', 'QBB KL', 'Pool Variance', 'Pool Entropy', 'Thompson']
upper_l = 0.97918682
upper_r = 0.99091814
In [4]:
if running_expt:
vstatlas = pd.read_hdf('../data/vstatlas.h5', 'vstatlas')
transformer = PolynomialFeatures(degree=2, interaction_only=False, include_bias=True)
X = np.array(vstatlas[vstatlas_features])
X_poly = transformer.fit_transform(X)
y = np.array(vstatlas[target_col])
In [5]:
if running_expt:
logistic = LogisticRegression(multi_class='ovr', penalty='l1', C=100, random_state=2, class_weight='auto')
logistic_committee = BaggingClassifier(logistic, n_estimators=11, n_jobs=-1, max_samples=300)
X_pool, _, y_pool, _ = balanced_train_test_split(X_poly, y, train_size=2360, test_size=0, random_state=11)
kfold = StratifiedShuffleSplit(y_pool, n_iter=10, test_size=0.3, train_size=0.7, random_state=19)
In [6]:
heuristics = [random_h, entropy_h, margin_h, qbb_margin_h, qbb_kl_h, pool_variance_h, pool_entropy_h]
thompson_path = '../pickle/07_thompson_sampling/vstatlas_balanced_logistic_thompson.pickle'
pickle_paths = ['../pickle/06_active_learning/vstatlas_balanced_logistic_random.pickle',
'../pickle/06_active_learning/vstatlas_balanced_logistic_entropy.pickle',
'../pickle/06_active_learning/vstatlas_balanced_logistic_margin.pickle',
'../pickle/06_active_learning/vstatlas_balanced_logistic_qbb_margin.pickle',
'../pickle/06_active_learning/vstatlas_balanced_logistic_qbb_kl.pickle',
'../pickle/06_active_learning/vstatlas_balanced_logistic_pool_variance.pickle',
'../pickle/06_active_learning/vstatlas_balanced_logistic_pool_entropy.pickle']
if not results_exist(pickle_paths):
run_active_learning_expt(X_pool, y_pool, kfold, logistic, logistic_committee, heuristics, pickle_paths)
bl_lcs, bl_sels = zip(*load_results(pickle_paths))
bl_lcs = list(bl_lcs)
bl_lcs.append(load_results(thompson_path))
bl_df = order_learning_curves(bl_lcs, labels)
bl_df.columns
Out[6]:
In [7]:
fig = plt.figure(figsize=(4, 4))
ax = plot_learning_curve_df(xrange, bl_df, bl_df.columns[7:], colors, linestyles, upper=upper_l, ylim=(.83, 1))
fig.savefig(fig_dir+'5_active/vstatlas_bl_ind_lower.pdf', bbox_inches='tight')
In [8]:
fig = plt.figure(figsize=(4, 4))
ax = plot_learning_curve_df(xrange, bl_df, bl_df.columns[:8], colors, linestyles, upper=upper_l, ylim=(.83, 1))
fig.savefig(fig_dir+'5_active/vstatlas_bl_ind_upper.pdf', bbox_inches='tight')
In [9]:
fig = plt.figure(figsize=(10, 2))
ax = plot_final_accuracy(bl_lcs, labels, colors, ylim=(.9, 1))
fig.savefig(fig_dir+'5_active/vstatlas_bl_ind_violin.pdf', bbox_inches='tight')
In [10]:
if running_expt:
rbf = SVC(kernel='rbf', gamma=0.001, C=1000000, cache_size=2000, class_weight='auto', probability=True)
rbf_committee = BaggingClassifier(rbf, n_estimators=11, n_jobs=-1, max_samples=300)
X_pool, _, y_pool, _ = balanced_train_test_split(X, y, train_size=2360, test_size=0, random_state=11)
kfold = StratifiedShuffleSplit(y_pool, n_iter=10, test_size=0.3, train_size=0.7, random_state=19)
In [11]:
heuristics = [random_h, entropy_h, margin_h, qbb_margin_h, qbb_kl_h, pool_variance_h, pool_entropy_h]
thompson_path = '../pickle/07_thompson_sampling/vstatlas_balanced_rbf_thompson.pickle'
pickle_paths = ['../pickle/06_active_learning/vstatlas_balanced_rbf_random.pickle',
'../pickle/06_active_learning/vstatlas_balanced_rbf_entropy.pickle',
'../pickle/06_active_learning/vstatlas_balanced_rbf_margin.pickle',
'../pickle/06_active_learning/vstatlas_balanced_rbf_qbb_margin.pickle',
'../pickle/06_active_learning/vstatlas_balanced_rbf_qbb_kl.pickle',
'../pickle/06_active_learning/vstatlas_balanced_rbf_pool_variance.pickle',
'../pickle/06_active_learning/vstatlas_balanced_rbf_pool_entropy.pickle']
if not results_exist(pickle_paths):
run_active_learning_expt(X_pool, y_pool, kfold, rbf, rbf_committee, heuristics, pickle_paths)
br_lcs, br_sels = zip(*load_results(pickle_paths))
br_lcs = list(br_lcs)
br_lcs.append(load_results(thompson_path))
br_df = order_learning_curves(br_lcs, labels)
br_df.columns
Out[11]:
In [12]:
fig = plt.figure(figsize=(4, 4))
ax = plot_learning_curve_df(xrange, br_df, br_df.columns[4:], colors, linestyles, upper=upper_r, ylim=(.83, 1))
fig.savefig(fig_dir+'5_active/vstatlas_br_ind_lower.pdf', bbox_inches='tight')
In [13]:
fig = plt.figure(figsize=(4, 4))
ax = plot_learning_curve_df(xrange, br_df, br_df.columns[:5], colors, linestyles, upper=upper_r, ylim=(.83, 1))
fig.savefig(fig_dir+'5_active/vstatlas_br_ind_upper.pdf', bbox_inches='tight')
In [14]:
fig = plt.figure(figsize=(10, 2))
ax = plot_final_accuracy(br_lcs, labels, colors, ylim=(.9, 1))
fig.savefig(fig_dir+'5_active/vstatlas_br_ind_violin.pdf', bbox_inches='tight')
In [15]:
if running_expt:
logistic = LogisticRegression(multi_class='ovr', penalty='l1', C=100, random_state=2, class_weight='auto')
logistic_committee = BaggingClassifier(logistic, n_estimators=11, n_jobs=-1, max_samples=300)
kfold = StratifiedShuffleSplit(y, n_iter=10, test_size=0.3, train_size=0.7, random_state=19)
In [16]:
heuristics = [random_h, entropy_h, margin_h, qbb_margin_h, qbb_kl_h, pool_variance_h, pool_entropy_h]
thompson_path = '../pickle/07_thompson_sampling/vstatlas_unbalanced_logistic_thompson.pickle'
pickle_paths = ['../pickle/06_active_learning/vstatlas_unbalanced_logistic_random.pickle',
'../pickle/06_active_learning/vstatlas_unbalanced_logistic_entropy.pickle',
'../pickle/06_active_learning/vstatlas_unbalanced_logistic_margin.pickle',
'../pickle/06_active_learning/vstatlas_unbalanced_logistic_qbb_margin.pickle',
'../pickle/06_active_learning/vstatlas_unbalanced_logistic_qbb_kl.pickle',
'../pickle/06_active_learning/vstatlas_unbalanced_logistic_pool_variance.pickle',
'../pickle/06_active_learning/vstatlas_unbalanced_logistic_pool_entropy.pickle']
if not results_exist(pickle_paths):
run_active_learning_expt(X_poly, y, kfold, logistic, logistic_committee, heuristics, pickle_paths)
ul_lcs, ul_sels = zip(*load_results(pickle_paths))
ul_lcs = list(ul_lcs)
ul_lcs.append(load_results(thompson_path))
ul_df = order_learning_curves(ul_lcs, labels)
ul_df.columns
Out[16]:
In [17]:
fig = plt.figure(figsize=(4, 4))
ax = plot_learning_curve_df(xrange, ul_df, ul_df.columns[7:], colors, linestyles, upper=upper_l, ylim=(.68, 1))
fig.savefig(fig_dir+'5_active/vstatlas_ul_ind_lower.pdf', bbox_inches='tight')
In [18]:
fig = plt.figure(figsize=(4, 4))
ax = plot_learning_curve_df(xrange, ul_df, ul_df.columns[:8], colors, linestyles, upper=upper_l, ylim=(.68, 1))
fig.savefig(fig_dir+'5_active/vstatlas_ul_ind_upper.pdf', bbox_inches='tight')
In [19]:
fig = plt.figure(figsize=(10, 2))
ax = plot_final_accuracy(ul_lcs, labels, colors, ylim=(.8, 1))
fig.savefig(fig_dir+'5_active/vstatlas_ul_ind_violin.pdf', bbox_inches='tight')
In [20]:
if running_expt:
rbf = SVC(kernel='rbf', gamma=0.001, C=1000000, cache_size=2000, class_weight='auto', probability=True)
rbf_committee = BaggingClassifier(rbf, n_estimators=11, n_jobs=-1, max_samples=300)
kfold = StratifiedShuffleSplit(y, n_iter=10, test_size=0.3, train_size=0.7, random_state=19)
In [21]:
heuristics = [random_h, entropy_h, margin_h, qbb_margin_h, qbb_kl_h, pool_variance_h, pool_entropy_h]
thompson_path = '../pickle/07_thompson_sampling/vstatlas_unbalanced_rbf_thompson.pickle'
pickle_paths = ['../pickle/06_active_learning/vstatlas_unbalanced_rbf_random.pickle',
'../pickle/06_active_learning/vstatlas_unbalanced_rbf_entropy.pickle',
'../pickle/06_active_learning/vstatlas_unbalanced_rbf_margin.pickle',
'../pickle/06_active_learning/vstatlas_unbalanced_rbf_qbb_margin.pickle',
'../pickle/06_active_learning/vstatlas_unbalanced_rbf_qbb_kl.pickle',
'../pickle/06_active_learning/vstatlas_unbalanced_rbf_pool_variance.pickle',
'../pickle/06_active_learning/vstatlas_unbalanced_rbf_pool_entropy.pickle']
if not results_exist(pickle_paths):
run_active_learning_expt(X, y, kfold, rbf, rbf_committee, heuristics, pickle_paths)
ur_lcs, ur_sels = zip(*load_results(pickle_paths))
ur_lcs = list(ur_lcs)
ur_lcs.append(load_results(thompson_path))
ur_df = order_learning_curves(ur_lcs, labels)
ur_df.columns
Out[21]:
In [22]:
fig = plt.figure(figsize=(4, 4))
ax = plot_learning_curve_df(xrange, ur_df, ur_df.columns[4:], colors, linestyles, upper=upper_r, ylim=(.76, 1))
fig.savefig(fig_dir+'5_active/vstatlas_ur_ind_lower.pdf', bbox_inches='tight')
In [23]:
fig = plt.figure(figsize=(4, 4))
ax = plot_learning_curve_df(xrange, ur_df, ur_df.columns[:5], colors, linestyles, upper=upper_r, ylim=(.76, 1))
fig.savefig(fig_dir+'5_active/vstatlas_ur_ind_upper.pdf', bbox_inches='tight')
In [24]:
fig = plt.figure(figsize=(10, 2))
ax = plot_final_accuracy(ur_lcs, labels, colors, ylim=(.8, 1))
fig.savefig(fig_dir+'5_active/vstatlas_ur_ind_violin.pdf', bbox_inches='tight')