In [2]:
import torch
from dpp_nets.my_torch.simulator2 import SimulRegressor
from dpp_nets.helper.plotting import plot_floats, plot_dict
In [3]:
# Global Settings
input_set_size = 50
n_clusters = 20
dtype = torch.DoubleTensor
path = '/Users/Max/Desktop/master thesis/latex/figures/plots'
In [9]:
# Deterministic Network To see how difficult it is
torch.manual_seed(0)
deterministic = SimulRegressor(input_set_size, n_clusters, dtype)
deterministic.train_deterministic(1000, 10, lr=1e-3)
deterministic.train_deterministic(1000, 100, lr=1e-5)
deterministic.sample_deterministic()
In [4]:
#### For the purpose of demonstrating the regularization, we will regularise the models to
#### return sets of 20, 15, 10, 5, 20 should be able to become a perfect predictor
#### hence train it first to find sensible settings, then find
#### we evaluate all models on random set size, sets of size 20, sets of size 15, sets of
#### size 10 and sets of size 5
# Baseline20
torch.manual_seed(0)
baseline20 = SimulRegressor(input_set_size, n_clusters, dtype)
# be very radical with enforcing the regularization mean
baseline20.train(500, 25, 5, baseline=True, lr=1e-3, reg=100, reg_mean=20)
baseline20.train(500, 25, 5, baseline=True, lr=1e-4, reg=100, reg_mean=20)
baseline20.train(500, 50, 5, baseline=True, lr=1e-5, reg=100, reg_mean=20)
In [6]:
#baseline20.train(1000,25, 5, baseline=True, lr=1e-3, reg=100, reg_mean=20)
baseline20.evaluate(1000)
Out[6]:
In [4]:
# Baseline15
torch.manual_seed(0)
baseline15 = SimulRegressor(input_set_size, n_clusters, dtype)
# be very radical with enforcing the regularization mean
baseline15.train(500, 25, 5, baseline=True, lr=1e-3)
baseline15.train(500, 25, 5, baseline=True, lr=1e-4)
baseline15.train(500, 50, 5, baseline=True, lr=1e-5)
In [44]:
# Baseline10
torch.manual_seed(0)
baseline10 = SimulRegressor(input_set_size, n_clusters, dtype)
# be very radical with enforcing the regularization mean
baseline10.train(500, 25, 5, baseline=True, lr=1e-3, reg=100, reg_mean=10)
baseline10.train(500, 25, 5, baseline=True, lr=1e-4, reg=100, reg_mean=10)
baseline10.train(500, 50, 5, baseline=True, lr=1e-5, reg=100, reg_mean=10)
In [72]:
## Holds all the data
# We have 4 models, reg 5, 10, 15, 20
# each of the four models gets evaluated, on 5, 10, 15, 20, and all sets.
model_no = 3
eval_no = 5
super_loss = [[None for j in range(eval_no)] for i in range(model_no)]
super_prec = [[None for j in range(eval_no)] for i in range(model_no)]
super_rec = [[None for j in range(eval_no)] for i in range(model_no)]
super_size = [[None for j in range(eval_no)] for i in range(model_no)]
evals = [None, 5, 10, 15, 20]
In [73]:
# Evaluate baseline 10 model
i = 0
for ix, e in enumerate(evals):
loss, prec, rec, size = baseline10.evaluate_fixed(1000, e)
super_loss[i][ix] = loss
super_prec[i][ix] = prec
super_rec[i][ix] = rec
super_size[i][ix] = size
# Evaluate baseline15 model
i += 1
for ix, e in enumerate(evals):
loss, prec, rec, size = baseline15.evaluate_fixed(1000, e)
super_loss[i][ix] = loss
super_prec[i][ix] = prec
super_rec[i][ix] = rec
super_size[i][ix] = size
# Evaluate baseline20 model
i +=1
for ix, e in enumerate(evals):
loss, prec, rec, size = baseline20.evaluate_fixed(1000, e)
super_loss[i][ix] = loss
super_prec[i][ix] = prec
super_rec[i][ix] = rec
super_size[i][ix] = size
In [15]:
fname = None #
title = 'Loss over Time'
xlabel = 'Training Steps'
ylabel = 'MSE'
plot_floats(deterministic.loss_dict, 300, fname, title, xlabel, ylabel)
In [ ]:
### Now the Big Evaluation of the 4 models to see how it looks.
# Training Settings for Comparison Purposes
train_steps = 1000
batch_size = 100
lr = 1e-5
reg=0.1
reg_mean = 10
sample_iter = 5
# Pure Reinforce
torch.manual_seed(0)
pure_reinforce = SimulRegressor(input_set_size, n_clusters, dtype)
print(pure_reinforce.signal_clusters)
pure_reinforce.evaluate(1000)
pure_reinforce.train(train_steps, batch_size, 1, baseline=False, lr=lr, reg=reg, reg_mean=reg_mean)
pure_reinforce.evaluate(1000)
plot_floats(pure_reinforce.loss_dict, 100, fname, title, xlabel, ylabel)
In [ ]:
# Reinforce Cheat
torch.manual_seed(0)
reinforce = SimulRegressor(input_set_size, n_clusters, dtype)
pure_reinforce.evaluate(1000)
reinforce.train(train_steps, batch_size, 1, baseline=False, lr=lr*sample_iter, reg=reg, reg_mean=reg_mean)
reinforce.evaluate(1000)
plot_floats(reinforce.loss_dict, 100, fname, title, xlabel, ylabel)
In [ ]:
# Multi-Sample
torch.manual_seed(0)
multi_sample = SimulRegressor(input_set_size, n_clusters, dtype)
multi_sample.evaluate(1000)
multi_sample.train(train_steps, batch_size, sample_iter, baseline=False, lr=lr, reg=reg, reg_mean=reg_mean)
multi_sample.evaluate(1000)
plot_floats(multi_sample.loss_dict, 100, fname, title, xlabel, ylabel)
In [ ]:
# Multi-Sample + Baseline
lr = 1e-3
torch.manual_seed(0)
baseline = SimulRegressor(input_set_size, n_clusters, dtype)
baseline.evaluate(1000)
baseline.train(train_steps, batch_size, sample_iter, baseline=True, lr=lr, reg=reg, reg_mean=reg_mean)
baseline.evaluate(1000)
plot_floats(baseline.loss_dict, 100, fname, title, xlabel, ylabel)
In [ ]:
In [182]:
import matplotlib.pyplot as plt
import numpy as np
def autolabel(rects, ax):
for rect in rects:
height = rect.get_height()
ax.text(rect.get_x() + rect.get_width()/2., 1.01*height,
'%.2f' % float(height),
ha='center', va='bottom')
def curate_ax(idx, ax, my_data):
x_c = 1
x_m = 0.5
x_positioning = [x_c + x_m*i for i in range(len(my_data))]
width = 0.3 # width of bars
colors = ['deepskyblue', 'dodgerblue', 'royalblue']
labels = ['label0','label1','label2','label3','label4','label5']
rects = []
for i, data in enumerate(my_data):
r = ax.bar(x_positioning[i], data[idx], width, color=colors[i], label=labels[i])
rects.append(r)
# autolabel(r, ax)
return rects
In [174]:
f, (ax0, ax1, ax2, ax3, ax4) = plt.subplots(1, 5, sharey=True)
# Create subplots subplot
rects0 = curate_ax(0, ax0, super_loss)
curate_ax(1, ax1, super_loss)
curate_ax(2, ax2, super_loss)
curate_ax(3, ax3, super_loss)
curate_ax(4, ax4, super_loss)
# Axis and Title Settings
head = plt.suptitle('Learning To Count Clusters')
# y-axis
ax0.set_yticks([0,5,10,15,20,25])
ax0.set_ylim([0,25])
ax0.set_ylabel('Accuracy')
# x-axis
ax0.set_xticks([],[])
ax1.set_xticks([],[])
ax2.set_xticks([],[])
ax3.set_xticks([],[])
ax4.set_xticks([],[])
#ax5.set_xticks([],[])
ax0.set_xlabel('All Sets')
ax1.set_xlabel('y = 5')
ax2.set_xlabel('y = 10')
ax3.set_xlabel('y = 15')
ax4.set_xlabel('y = 20')
#ax0.set_title('y = 5')
# Legend
# Legend
lgd = plt.legend((rects0[0], rects0[1], rects0[2]),
(r'$\lambda = 10$', r'$\lambda = 15$', r'$\lambda = 20$'),
loc = 'best', bbox_to_anchor = (0,-0.1,1.1,1),
fontsize=9, numpoints=3, handlelength=1,
bbox_transform = plt.gcf().transFigure)
plt.savefig('odl.pdf', bbox_extra_artists=(lgd,head), bbox_inches='tight')
plt.show()
In [99]:
super_loss
Out[99]:
In [192]:
f, axarr = plt.subplots(4, 5, sharey='row', sharex='col')
ax0 = axarr[0,0]
ax1 = axarr[0,1]
ax2 = axarr[0,2]
ax3 = axarr[0,3]
ax4 = axarr[0,4]
ax5 = axarr[1,0]
ax6 = axarr[1,1]
ax7 = axarr[1,2]
ax8 = axarr[1,3]
ax9 = axarr[1,4]
ax10 = axarr[2,0]
ax11 = axarr[2,1]
ax12 = axarr[2,2]
ax13 = axarr[2,3]
ax14 = axarr[2,4]
ax15 = axarr[3,0]
ax16 = axarr[3,1]
ax17 = axarr[3,2]
ax18 = axarr[3,3]
ax19 = axarr[3,4]
# Create subplots subplot
rects0 = curate_ax(0, ax0, super_loss)
curate_ax(1, ax1, super_loss)
curate_ax(2, ax2, super_loss)
curate_ax(3, ax3, super_loss)
curate_ax(4, ax4, super_loss)
curate_ax(0, ax5, super_prec)
curate_ax(1, ax6, super_prec)
curate_ax(2, ax7, super_prec)
curate_ax(3, ax8, super_prec)
curate_ax(4, ax9, super_prec)
curate_ax(0, ax10, super_rec)
curate_ax(1, ax11, super_rec)
curate_ax(2, ax12, super_rec)
curate_ax(3, ax13, super_rec)
curate_ax(4, ax14, super_rec)
curate_ax(0, ax15, super_size)
curate_ax(1, ax16, super_size)
curate_ax(2, ax17, super_size)
curate_ax(3, ax18, super_size)
curate_ax(4, ax19, super_size)
# Axis and Title Settings
#head = plt.suptitle('Learning To Count Clusters')
# y-axis
ax0.set_yticks([0,5,10,15,20,25])
ax0.set_yticklabels(['0','','','','','25'],size=8)
ax0.set_ylim([0,25])
ax0.set_ylabel('MSE', size=9)
ax0.get_yaxis().set_label_coords(-0.35,0.5)
ax5.set_yticks([0,0.25,0.5,0.75,1])
ax5.set_yticklabels(['0','','','','1'],size=8)
ax5.set_ylim([0,1])
ax5.set_ylabel('Precision', size=9)
ax5.get_yaxis().set_label_coords(-0.35,0.5)
ax10.set_yticks([0,0.25,0.5,0.75,1])
ax10.set_yticklabels(['0','','','','1'],size=8)
ax10.set_ylim([0,1])
ax10.set_ylabel('Recall', size=9)
ax10.get_yaxis().set_label_coords(-0.35,0.5)
ax15.set_yticks([0,5,10,15,20])
ax15.set_yticklabels(['0','','','','20'],size=8)
ax15.set_ylim([0,20])
ax15.set_ylabel('Cardinality', size=9)
ax15.get_yaxis().set_label_coords(-0.35,0.5)
# x-axis
ax0.set_xticks([],[])
ax1.set_xticks([],[])
ax2.set_xticks([],[])
ax3.set_xticks([],[])
ax4.set_xticks([],[])
#ax5.set_xticks([],[])
ax15.set_xlabel('Any $y$',size=9)
ax15.get_xaxis().set_label_coords(0.5,-0.2)
ax16.set_xlabel(r'$y = 5$', size=9)
ax16.get_xaxis().set_label_coords(0.5,-0.2)
ax17.set_xlabel(r'$y = 10$', size=9)
ax17.get_xaxis().set_label_coords(0.5,-0.2)
ax18.set_xlabel(r'$y = 15$', size=9)
ax18.get_xaxis().set_label_coords(0.5,-0.2)
ax19.set_xlabel(r'$y = 20$', size=9)
ax19.get_xaxis().set_label_coords(0.5,-0.2)
# Legend
lgd = plt.legend((rects0[0], rects0[1], rects0[2]),
(r'$\lambda = 10$', r'$\lambda = 15$', r'$\lambda = 20$'),
loc = 'best', bbox_to_anchor = (0,-0.1,1.07,1),
fontsize=9,handlelength =1,
bbox_transform = plt.gcf().transFigure)
f.subplots_adjust(hspace=0.25)
plt.savefig('no_header.pdf', bbox_extra_artists=(lgd,), bbox_inches='tight')
plt.show()
In [185]:
super_loss
Out[185]:
In [186]:
import pickle
with open('super_loss.pkl', 'wb') as f:
pickle.dump(super_loss, f)
In [187]:
with open('super_prec.pkl', 'wb') as f:
pickle.dump(super_prec, f)
In [188]:
with open('super_rec.pkl', 'wb') as f:
pickle.dump(super_rec, f)
In [189]:
with open('super_size.pkl', 'wb') as f:
pickle.dump(super_size, f)
In [ ]: