In [2]:
# this part imports libs and load data from csv file
import csv
from dateutil import parser
from datetime import timedelta
from sklearn import svm
import numpy as np
import pandas as pd
import pdb
import pickle
from sklearn.cross_validation import train_test_split
from sklearn import preprocessing
import sklearn
import scipy.stats as ss
import cPickle
import gzip
import os
import sys
import time
import numpy
import theano
import theano.tensor as T
from theano.tensor.shared_randomstreams import RandomStreams
from DL_libs import *
In [2]:
f = gzip.open('mnist.pkl.gz', 'rb')
train_set, valid_set, test_set = cPickle.load(f)
X_train,y_train = train_set
X_valid,y_valid = valid_set
X_total=np.vstack((X_train, X_valid))
X_total = np.array(X_total, dtype= theano.config.floatX)
print(X_total.shape)
y_total = np.concatenate([y_train, y_valid])
print y_total.shape
array_A =[]
array_B =[]
for i in range(100000):
array_A.append(np.random.random_integers(0, 59999))
array_B.append(np.random.random_integers(0, 59999))
pos_index = []
neg_index = []
for index in xrange(100000):
if y_total[array_A[index]] == y_total[array_B[index]]:
pos_index.append(index)
else:
neg_index.append(index)
print len(pos_index)
selected_neg_index= neg_index[ : len(pos_index)]
print len(selected_neg_index)
In [26]:
type(X_train)
Out[26]:
In [3]:
import pandas as pd
array_A = np.array(array_A)
array_B = np.array(array_B)
index_for_positive_image_A = array_A[pos_index]
index_for_positive_image_B = array_B[pos_index]
index_for_neg_image_A = array_A[selected_neg_index]
index_for_neg_image_B = array_B[selected_neg_index]
X_pos_A = X_total[index_for_positive_image_A]
X_pos_B = X_total[index_for_positive_image_B]
X_pos_whole = np.hstack((X_pos_A,X_pos_B))
X_neg_A = X_total[index_for_neg_image_A]
X_neg_B = X_total[index_for_neg_image_B]
X_neg_whole = np.hstack((X_neg_A, X_neg_B))
print X_pos_A.shape, X_pos_B.shape, X_pos_whole.shape
print X_neg_A.shape, X_neg_B.shape, X_neg_whole.shape
X_whole = np.vstack((X_pos_whole, X_neg_whole))
print X_whole.shape
y_pos = np.ones(X_pos_whole.shape[0])
y_neg = np.zeros(X_neg_whole.shape[0])
y_whole = np.concatenate([y_pos,y_neg])
print y_whole
In [4]:
x_train_pre_validation_minmax, x_test_minmax, y_train_pre_validation_minmax, y_test_minmax = train_test_split(X_whole,y_whole,\
test_size=0.2, random_state=211)
x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax = train_test_split(x_train_pre_validation_minmax,
y_train_pre_validation_minmax,\
test_size=0.2, random_state=21)
print x_train_minmax.shape, y_train_minmax.shape, x_validation_minmax.shape, \
y_validation_minmax.shape, x_test_minmax.shape, y_test_minmax.shape
In [9]:
# this is a use case.
pretraining_epochs=1
pretrain_lr=0.001
batch_size=30
hidden_layers_sizes =[100, 100]
corruption_levels=[0, 0]
x =x_train_minmax
print "original shape", x.shape
a_MAE = train_a_MultipleAEs(x, pretraining_epochs=pretraining_epochs, pretrain_lr=pretrain_lr, batch_size=batch_size,
hidden_layers_sizes =hidden_layers_sizes, corruption_levels=corruption_levels)
print a_MAE.transform(x).shape
In [16]:
new_x_train_minmax = a_MAE.transform(x_train_minmax)
new_x_test_minmax = a_MAE.transform(x_test_minmax)
In [12]:
x_train_minmax_A = x_train_minmax[:, :x_train_minmax.shape[1]/2].shape
Out[12]:
In [13]:
# get the new representation for A set. first 784-D
pretraining_epochs=15
pretrain_lr=0.001
batch_size=30
hidden_layers_sizes =[100, 100]
corruption_levels=[0, 0]
x = x_train_minmax[:, :x_train_minmax.shape[1]/2]
print "original shape", x.shape
a_MAE_A = train_a_MultipleAEs(x, pretraining_epochs=pretraining_epochs, pretrain_lr=pretrain_lr, batch_size=batch_size,
hidden_layers_sizes =hidden_layers_sizes, corruption_levels=corruption_levels)
new_x_train_minmax_A = a_MAE_A.transform(x_train_minmax[:, :x_train_minmax.shape[1]/2])
In [ ]:
# get the new representation for A set. first 784-D
pretraining_epochs=15
pretrain_lr=0.001
batch_size=30
hidden_layers_sizes =[100, 100]
corruption_levels=[0, 0]
x = x_train_minmax[:, x_train_minmax.shape[1]/2:]
print "original shape", x.shape
a_MAE_B = train_a_MultipleAEs(x, pretraining_epochs=pretraining_epochs, pretrain_lr=pretrain_lr, batch_size=batch_size,
hidden_layers_sizes =hidden_layers_sizes, corruption_levels=corruption_levels)
new_x_train_minmax_B = a_MAE_B.transform(x_train_minmax[:, x_train_minmax.shape[1]/2:])
In [39]:
new_x_test_minmax_A = a_MAE_A.transform(x_test_minmax[:, :x_test_minmax.shape[1]/2])
new_x_test_minmax_B = a_MAE_B.transform(x_test_minmax[:, x_test_minmax.shape[1]/2:])
new_x_validation_minmax_A = a_MAE_A.transform(x_validation_minmax[:, :x_validation_minmax.shape[1]/2])
new_x_validation_minmax_B = a_MAE_B.transform(x_validation_minmax[:, x_validation_minmax.shape[1]/2:])
new_x_train_minmax_whole = np.hstack((new_x_train_minmax_A, new_x_train_minmax_B))
new_x_test_minmax_whole = np.hstack((new_x_test_minmax_A, new_x_test_minmax_B))
new_x_validationt_minmax_whole = np.hstack((new_x_validation_minmax_A, new_x_validation_minmax_B))
In [24]:
new_x_train_minmax_whole.shape
Out[24]:
In [27]:
#### L1-based SVM original dataset
from sklearn.feature_selection import SelectKBest, chi2
from sklearn.linear_model import RidgeClassifier
from sklearn.svm import LinearSVC
from sklearn.linear_model import SGDClassifier
from sklearn.linear_model import Perceptron
from sklearn.linear_model import PassiveAggressiveClassifier
from sklearn.naive_bayes import BernoulliNB, MultinomialNB
from sklearn.neighbors import KNeighborsClassifier
from sklearn.neighbors import NearestCentroid
from sklearn.utils.extmath import density
from sklearn import metrics
x_train_pre_validation_scaled = x_train_minmax
X_test_scaled = x_test_minmax
print 'SVM result on original dataset'
L1_SVC_Selector= LinearSVC(C=1, penalty="l1", dual=False)
L1_SVC_X = L1_SVC_Selector.fit_transform(x_train_pre_validation_scaled, y_train_minmax)
traning_accuracy = L1_SVC_Selector.score(x_train_pre_validation_scaled, y_train_minmax)
print 'training accuracy', traning_accuracy
testing_accuracy = L1_SVC_Selector.score(X_test_scaled, y_test_minmax)
print 'testing accuracy', testing_accuracy
print L1_SVC_X.shape
predicted = L1_SVC_Selector.predict(X_test_scaled)
print 'testing precision', sklearn.metrics.precision_score(y_test_minmax, predicted, pos_label=1)
print 'testing recall', sklearn.metrics.recall_score(y_test_minmax, predicted, pos_label=1)
print sklearn.metrics.recall_score(y_test_minmax, predicted, pos_label=1)
In [19]:
#### L1-based SVM on trasformed original dataset
from sklearn.feature_selection import SelectKBest, chi2
from sklearn.linear_model import RidgeClassifier
from sklearn.svm import LinearSVC
from sklearn.linear_model import SGDClassifier
from sklearn.linear_model import Perceptron
from sklearn.linear_model import PassiveAggressiveClassifier
from sklearn.naive_bayes import BernoulliNB, MultinomialNB
from sklearn.neighbors import KNeighborsClassifier
from sklearn.neighbors import NearestCentroid
from sklearn.utils.extmath import density
from sklearn import metrics
x_train_pre_validation_scaled = new_x_train_minmax
X_test_scaled = new_x_test_minmax
L1_SVC_Selector= LinearSVC(C=1, penalty="l1", dual=False)
L1_SVC_X = L1_SVC_Selector.fit_transform(x_train_pre_validation_scaled, y_train_minmax)
traning_accuracy = L1_SVC_Selector.score(x_train_pre_validation_scaled, y_train_minmax)
print 'training accuracy', traning_accuracy
testing_accuracy = L1_SVC_Selector.score(X_test_scaled, y_test_minmax)
print 'testing accuracy', testing_accuracy
print L1_SVC_X.shape
predicted = L1_SVC_Selector.predict(X_test_scaled)
print 'testing precision', sklearn.metrics.precision_score(y_test_minmax, predicted, pos_label=1)
print 'testing recall', sklearn.metrics.recall_score(y_test_minmax, predicted, pos_label=1)
In [23]:
#### L1-based SVM on seperately trasformed original dataset
from sklearn.feature_selection import SelectKBest, chi2
from sklearn.linear_model import RidgeClassifier
from sklearn.svm import LinearSVC
from sklearn.linear_model import SGDClassifier
from sklearn.linear_model import Perceptron
from sklearn.linear_model import PassiveAggressiveClassifier
from sklearn.naive_bayes import BernoulliNB, MultinomialNB
from sklearn.neighbors import KNeighborsClassifier
from sklearn.neighbors import NearestCentroid
from sklearn.utils.extmath import density
from sklearn import metrics
x_train_pre_validation_scaled = new_x_train_minmax_whole
X_test_scaled = new_x_test_minmax_whole
L1_SVC_Selector= LinearSVC(C=1, penalty="l1", dual=False)
L1_SVC_X = L1_SVC_Selector.fit_transform(x_train_pre_validation_scaled, y_train_minmax)
traning_accuracy = L1_SVC_Selector.score(x_train_pre_validation_scaled, y_train_minmax)
print 'training accuracy', traning_accuracy
testing_accuracy = L1_SVC_Selector.score(X_test_scaled, y_test_minmax)
print 'testing accuracy', testing_accuracy
print L1_SVC_X.shape
predicted = L1_SVC_Selector.predict(X_test_scaled)
print 'testing precision', sklearn.metrics.precision_score(y_test_minmax, predicted, pos_label=1)
print 'testing recall', sklearn.metrics.recall_score(y_test_minmax, predicted, pos_label=1)
In [25]:
# get the new representation for A set. first 784-D
pretraining_epochs=15
pretrain_lr=0.001
batch_size=30
hidden_layers_sizes =[100, 100]
corruption_levels=[0, 0]
x = new_x_train_minmax_whole
print "original shape", x.shape
a_MAE_new_set = train_a_MultipleAEs(x, pretraining_epochs=pretraining_epochs, pretrain_lr=pretrain_lr, batch_size=batch_size,
hidden_layers_sizes =hidden_layers_sizes, corruption_levels=corruption_levels)
new_x_train_minmax_new_set = a_MAE_new_set.transform(new_x_train_minmax_whole)
new_x_test_minmax_new_set = a_MAE_new_set.transform(new_x_test_minmax_whole)
In [26]:
#### L1-based SVM on seperately trasformed original dataset
from sklearn.feature_selection import SelectKBest, chi2
from sklearn.linear_model import RidgeClassifier
from sklearn.svm import LinearSVC
from sklearn.linear_model import SGDClassifier
from sklearn.linear_model import Perceptron
from sklearn.linear_model import PassiveAggressiveClassifier
from sklearn.naive_bayes import BernoulliNB, MultinomialNB
from sklearn.neighbors import KNeighborsClassifier
from sklearn.neighbors import NearestCentroid
from sklearn.utils.extmath import density
from sklearn import metrics
x_train_pre_validation_scaled = new_x_train_minmax_new_set
X_test_scaled = new_x_test_minmax_new_set
L1_SVC_Selector= LinearSVC(C=1, penalty="l1", dual=False)
L1_SVC_X = L1_SVC_Selector.fit_transform(x_train_pre_validation_scaled, y_train_minmax)
traning_accuracy = L1_SVC_Selector.score(x_train_pre_validation_scaled, y_train_minmax)
print 'training accuracy', traning_accuracy
testing_accuracy = L1_SVC_Selector.score(X_test_scaled, y_test_minmax)
print 'testing accuracy', testing_accuracy
print L1_SVC_X.shape
predicted = L1_SVC_Selector.predict(X_test_scaled)
print 'testing precision', sklearn.metrics.precision_score(y_test_minmax, predicted, pos_label=1)
print 'testing recall', sklearn.metrics.recall_score(y_test_minmax, predicted, pos_label=1)
In [49]:
# feature log selection based on L1
log_clf = sklearn.linear_model.LogisticRegression(C=1, penalty='l1' )
log_clf.fit(x_train_pre_validation_scaled, y_train_pre_validation)
predicted = log_clf.predict(X_test_scaled)
print log_clf.score(x_train_pre_validation_scaled, y_train_pre_validation), log_clf.score(X_test_scaled, y_test)
print 'precision', sklearn.metrics.precision_score(y_test, predicted, pos_label=1)
print 'recall', sklearn.metrics.recall_score(y_test, predicted, pos_label=1)
In [52]:
# feature log selection based on L2
log_clf_l2 = sklearn.linear_model.LogisticRegression(C=1, penalty='l2' )
log_clf_l2.fit(x_train_pre_validation_scaled, y_train_pre_validation)
predicted = log_clf_l2.predict(X_test_scaled)
print log_clf_l2.score(x_train_pre_validation_scaled, y_train_pre_validation)
print log_clf_l2.score(X_test_scaled, y_test)
print 'precision', sklearn.metrics.precision_score(y_test, predicted, pos_label=1)
print 'recall', sklearn.metrics.recall_score(y_test, predicted, pos_label=1)
In [ ]:
In [28]:
# feature log selection based on L2 on original dataset
X_test = x_test_minmax
y_test = y_test_minmax
X_train = x_train_minmax
y_train = y_train_minmax
print 'performance for logisitc regression'
log_clf = sklearn.linear_model.LogisticRegression(C=1, penalty='l2' )
log_clf.fit(X_train, y_train)
predicted = log_clf.predict(X_test)
print 'training accuracy', log_clf.score(X_train, y_train)
print 'testing accuracy', log_clf.score(X_test, y_test)
print 'precision', sklearn.metrics.precision_score(y_test, predicted, pos_label=1)
print 'recall', sklearn.metrics.recall_score(y_test, predicted, pos_label=1)
In [29]:
# feature log selection based on L2 on transformed original
X_test = new_x_test_minmax
y_test = y_test_minmax
X_train = new_x_train_minmax
y_train = y_train_minmax
print 'performance for logisitc regression'
log_clf = sklearn.linear_model.LogisticRegression(C=1, penalty='l2' )
log_clf.fit(X_train, y_train)
predicted = log_clf.predict(X_test)
print 'training accuracy', log_clf.score(X_train, y_train)
print 'testing accuracy', log_clf.score(X_test, y_test)
print 'precision', sklearn.metrics.precision_score(y_test, predicted, pos_label=1)
print 'recall', sklearn.metrics.recall_score(y_test, predicted, pos_label=1)
In [30]:
# feature log selection based on L2 on seperately transformed dataset
X_test = new_x_test_minmax_new_set
y_test = y_test_minmax
X_train = new_x_train_minmax_new_set
y_train = y_train_minmax
print 'performance for logisitc regression'
log_clf = sklearn.linear_model.LogisticRegression(C=1, penalty='l2' )
log_clf.fit(X_train, y_train)
predicted = log_clf.predict(X_test)
print 'training accuracy', log_clf.score(X_train, y_train)
print 'testing accuracy', log_clf.score(X_test, y_test)
print 'precision', sklearn.metrics.precision_score(y_test, predicted, pos_label=1)
print 'recall', sklearn.metrics.recall_score(y_test, predicted, pos_label=1)
In [59]:
# build a sda, pretraining plus fine tuning on original data.
# set up paramaters
#finetune_lr=0.1
finetune_lr = 0.1
pretraining_epochs = 30
#pretrain_lr=0.001
pretrain_lr = 0.001
training_epochs = 300
batch_size = 30
hidden_layers_sizes= [100, 100]
corruption_levels = [0, 0]
sda = trainSda(x_train_minmax, y_train_minmax,
x_validation_minmax, y_validation_minmax ,
x_test_minmax, y_test_minmax,
hidden_layers_sizes = hidden_layers_sizes, corruption_levels = corruption_levels, batch_size = batch_size , \
training_epochs = training_epochs, pretraining_epochs = pretraining_epochs,
pretrain_lr = pretrain_lr, finetune_lr=finetune_lr
)
In [60]:
prob = sda.predict_p(x_train_minmax)
y_p = prob[:, 1]
y_p
Out[60]:
In [61]:
print 'hidden_layers_sizes:', hidden_layers_sizes
print 'corruption_levels:', corruption_levels
training_predicted = sda.predict(x_train_minmax)
y_train = y_train_minmax
print 'train accuracy: ', '{percent:.1%}'.format(percent=sklearn.metrics.accuracy_score(y_train, training_predicted))
print 'precision: ', '{percent:.1%}'.format(percent=sklearn.metrics.precision_score(y_train, training_predicted, pos_label=1))
print 'recall: ', '{percent:.1%}'.format( percent= sklearn.metrics.recall_score(y_train, training_predicted, pos_label=1))
test_predicted = sda.predict(x_test_minmax)
y_test = y_test_minmax
print 'testing accuracy: ', '{percent:.1%}'.format(percent=sklearn.metrics.accuracy_score(y_test, test_predicted))
print 'precision: ', '{percent:.1%}'.format(percent=sklearn.metrics.precision_score(y_test, test_predicted, pos_label=1))
print 'recall: ', '{percent:.1%}'.format( percent= sklearn.metrics.recall_score(y_test, test_predicted, pos_label=1))
In [49]:
# build a sda, pretraining plus fine tuning on seperately transformed data
# set up paramaters
#finetune_lr=0.1
finetune_lr = 0.1
pretraining_epochs = 30
#pretrain_lr=0.001
pretrain_lr = 0.001
training_epochs = 300
batch_size = 30
hidden_layers_sizes= [100, 100]
corruption_levels = [0, 0]
sda_transformed = trainSda(new_x_train_minmax_whole, y_train_minmax,
new_x_validationt_minmax_whole, y_validation_minmax ,
new_x_test_minmax_whole, y_test_minmax,
hidden_layers_sizes = hidden_layers_sizes, corruption_levels = corruption_levels, batch_size = batch_size , \
training_epochs = training_epochs, pretraining_epochs = pretraining_epochs,
pretrain_lr = pretrain_lr, finetune_lr=finetune_lr
)
In [ ]:
sda_transformed.p(new_x_test_minmax_whole)
In [47]:
import pickle
with open('array_A.pickle', 'wb') as handle:
pickle.dump(array_A, handle)
with open('array_B.pickle', 'wb') as handle:
pickle.dump(array_B, handle)
with open('pos_index.pickle', 'wb') as handle:
pickle.dump(pos_index, handle)
with open('neg_index.pickle', 'wb') as handle:
pickle.dump(neg_index, handle)
In [58]:
import cPickle
import gzip
import os
import sys
import time
import numpy
import theano
import theano.tensor as T
from theano.tensor.shared_randomstreams import RandomStreams
from logistic_sgd import LogisticRegression, load_data
from mlp import HiddenLayer
from dA import dA
import warnings
warnings.filterwarnings("ignore")
def shared_dataset(data_xy, borrow=True):
""" Function that loads the dataset into shared variables
The reason we store our dataset in shared variables is to allow
Theano to copy it into the GPU memory (when code is run on GPU).
Since copying data into the GPU is slow, copying a minibatch everytime
is needed (the default behaviour if the data is not in a shared
variable) would lead to a large decrease in performance.
"""
data_x, data_y = data_xy
shared_x = theano.shared(numpy.asarray(data_x,
dtype=theano.config.floatX),
borrow=borrow)
shared_y = theano.shared(numpy.asarray(data_y,
dtype=theano.config.floatX),
borrow=borrow)
# When storing data on the GPU it has to be stored as floats
# therefore we will store the labels as ``floatX`` as well
# (``shared_y`` does exactly that). But during our computations
# we need them as ints (we use labels as index, and if they are
# floats it doesn't make sense) therefore instead of returning
# ``shared_y`` we will have to cast it to int. This little hack
# lets ous get around this issue
return shared_x, T.cast(shared_y, 'int32')
def shared_dataset_X(data_x, borrow=True):
shared_x = theano.shared(numpy.asarray(data_x,
dtype=theano.config.floatX),
borrow=borrow)
return shared_x
class MultipleAEs(object):
"""Stacked denoising auto-encoder class that extract hi level features.
get the last hidden layer activation
"""
def __init__(self, numpy_rng, theano_rng=None, n_ins=784,
hidden_layers_sizes=[500, 500],
corruption_levels=[0.1, 0.1]):
""" This class is made to support a variable number of layers.
:type numpy_rng: numpy.random.RandomState
:param numpy_rng: numpy random number generator used to draw initial
weights
:type theano_rng: theano.tensor.shared_randomstreams.RandomStreams
:param theano_rng: Theano random generator; if None is given one is
generated based on a seed drawn from `rng`
:type n_ins: int
:param n_ins: dimension of the input to the sdA
:type n_layers_sizes: list of ints
:param n_layers_sizes: intermediate layers size, must contain
:type corruption_levels: list of float
:param corruption_levels: amount of corruption to use for each
layer
"""
self.sigmoid_layers = []
self.dA_layers = []
self.params = []
self.n_layers = len(hidden_layers_sizes)
assert self.n_layers > 0
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
# allocate symbolic variables for the data
self.x = T.matrix('x') # the data is presented as rasterized images
self.y = T.ivector('y') # the labels are presented as 1D vector of
# [int] labels
# The SdA is an MLP, for which all weights of intermediate layers
# are shared with a different denoising autoencoders
# We will first construct the SdA as a deep multilayer perceptron,
# and when constructing each sigmoidal layer we also construct a
# denoising autoencoder that shares weights with that layer
# During pretraining we will train these autoencoders (which will
# lead to chainging the weights of the MLP as well)
# During finetunining we will finish training the SdA by doing
# stochastich gradient descent on the MLP
for i in xrange(self.n_layers):
# construct the sigmoidal layer
# the size of the input is either the number of hidden units of
# the layer below or the input size if we are on the first layer
if i == 0:
input_size = n_ins
else:
input_size = hidden_layers_sizes[i - 1]
# the input to this layer is either the activation of the hidden
# layer below or the input of the SdA if you are on the first
# layer
if i == 0:
layer_input = self.x
else:
layer_input = self.sigmoid_layers[-1].output
sigmoid_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=hidden_layers_sizes[i],
activation=T.nnet.sigmoid)
# add the layer to our list of layers
self.sigmoid_layers.append(sigmoid_layer)
# its arguably a philosophical question...
# but we are going to only declare that the parameters of the
# sigmoid_layers are parameters of the StackedDAA
# the visible biases in the dA are parameters of those
# dA, but not the SdA
self.params.extend(sigmoid_layer.params)
# Construct a denoising autoencoder that shared weights with this
# layer
dA_layer = dA(numpy_rng=numpy_rng,
theano_rng=theano_rng,
input=layer_input,
n_visible=input_size,
n_hidden=hidden_layers_sizes[i],
W=sigmoid_layer.W,
bhid=sigmoid_layer.b)
self.dA_layers.append(dA_layer)
'''
we don't need this layer since is AE
# We now need to add a logistic layer on top of the MLP
self.logLayer = LogisticRegression(
input=self.sigmoid_layers[-1].output,
n_in=hidden_layers_sizes[-1], n_out=n_outs)
self.params.extend(self.logLayer.params)
# construct a function that implements one step of finetunining
# compute the cost for second phase of training,
# defined as the negative log likelihood
self.finetune_cost = self.logLayer.negative_log_likelihood(self.y)
# compute the gradients with respect to the model parameters
# symbolic variable that points to the number of errors made on the
# minibatch given by self.x and self.y
self.errors = self.logLayer.errors(self.y)
'''
def transform(self,data_x): # get the last layaer activations to transform data.
last_layer_activations = self.sigmoid_layers[-1].output
theano_fn = theano.function(inputs=[],
outputs=last_layer_activations,
givens={self.x: data_x})
newFeatures=theano_fn()
return newFeatures
def pretraining_functions(self, train_set_x, batch_size):
''' Generates a list of functions, each of them implementing one
step in trainnig the dA corresponding to the layer with same index.
The function will require as input the minibatch index, and to train
a dA you just need to iterate, calling the corresponding function on
all minibatch indexes.
:type train_set_x: theano.tensor.TensorType
:param train_set_x: Shared variable that contains all datapoints used
for training the dA
:type batch_size: int
:param batch_size: size of a [mini]batch
:type learning_rate: float
:param learning_rate: learning rate used during training for any of
the dA layers
'''
# index to a [mini]batch
index = T.lscalar('index') # index to a minibatch
corruption_level = T.scalar('corruption') # % of corruption to use
learning_rate = T.scalar('lr') # learning rate to use
# number of batches
n_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
# begining of a batch, given `index`
batch_begin = index * batch_size
# ending of a batch given `index`
batch_end = batch_begin + batch_size
pretrain_fns = []
for dA in self.dA_layers:
# get the cost and the updates list
cost, updates = dA.get_cost_updates(corruption_level,
learning_rate)
# compile the theano function
fn = theano.function(inputs=[index,
theano.Param(corruption_level, default=0.2),
theano.Param(learning_rate, default=0.1)],
outputs=cost,
updates=updates,
givens={self.x: train_set_x[batch_begin:
batch_end]})
# append `fn` to the list of functions
pretrain_fns.append(fn)
return pretrain_fns
def build_finetune_functions(self, datasets, batch_size, learning_rate):
'''Generates a function `train` that implements one step of
finetuning, a function `validate` that computes the error on
a batch from the validation set, and a function `test` that
computes the error on a batch from the testing set
:type datasets: list of pairs of theano.tensor.TensorType
:param datasets: It is a list that contain all the datasets;
the has to contain three pairs, `train`,
`valid`, `test` in this order, where each pair
is formed of two Theano variables, one for the
datapoints, the other for the labels
:type batch_size: int
:param batch_size: size of a minibatch
:type learning_rate: float
:param learning_rate: learning rate used during finetune stage
'''
(train_set_x, train_set_y) = datasets[0]
(valid_set_x, valid_set_y) = datasets[1]
(test_set_x, test_set_y) = datasets[2]
# compute number of minibatches for training, validation and testing
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]
n_valid_batches /= batch_size
n_test_batches = test_set_x.get_value(borrow=True).shape[0]
n_test_batches /= batch_size
index = T.lscalar('index') # index to a [mini]batch
# compute the gradients with respect to the model parameters
gparams = T.grad(self.finetune_cost, self.params)
# compute list of fine-tuning updates
updates = []
for param, gparam in zip(self.params, gparams):
updates.append((param, param - gparam * learning_rate))
train_fn = theano.function(inputs=[index],
outputs=self.finetune_cost,
updates=updates,
givens={
self.x: train_set_x[index * batch_size:
(index + 1) * batch_size],
self.y: train_set_y[index * batch_size:
(index + 1) * batch_size]},
name='train')
test_score_i = theano.function([index], self.errors,
givens={
self.x: test_set_x[index * batch_size:
(index + 1) * batch_size],
self.y: test_set_y[index * batch_size:
(index + 1) * batch_size]},
name='test')
valid_score_i = theano.function([index], self.errors,
givens={
self.x: valid_set_x[index * batch_size:
(index + 1) * batch_size],
self.y: valid_set_y[index * batch_size:
(index + 1) * batch_size]},
name='valid')
# Create a function that scans the entire validation set
def valid_score():
return [valid_score_i(i) for i in xrange(n_valid_batches)]
# Create a function that scans the entire test set
def test_score():
return [test_score_i(i) for i in xrange(n_test_batches)]
return train_fn, valid_score, test_score
def train_a_MultipleAEs(X, pretraining_epochs=10, pretrain_lr=0.001, batch_size=30,
hidden_layers_sizes=[100, 100], corruption_levels=[0, 0]):
# get a shared copy of X
train_set_x = shared_dataset_X(X)
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0]
n_train_batches /= batch_size
# numpy random generator
numpy_rng = numpy.random.RandomState(89677)
print '... building the model'
# construct the stacked denoising autoencoder class
sda = MultipleAEs(numpy_rng=numpy_rng, n_ins=train_set_x.get_value(borrow=True).shape[1],
hidden_layers_sizes=hidden_layers_sizes, corruption_levels = corruption_levels)
#########################
# PRETRAINING THE MODEL #
#########################
print '... getting the pretraining functions'
pretraining_fns = sda.pretraining_functions(train_set_x=train_set_x,
batch_size=batch_size)
print '... pre-training the model'
start_time = time.clock()
## Pre-train layer-wise
corruption_levels = [.1, .2, .3]
for i in xrange(sda.n_layers):
# go through pretraining epochs
for epoch in xrange(pretraining_epochs):
# go through the training set
c = []
for batch_index in xrange(n_train_batches):
c.append(pretraining_fns[i](index=batch_index,
corruption=corruption_levels[i],
lr=pretrain_lr))
print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
print numpy.mean(c)
end_time = time.clock()
return sda
class SdA(object):
"""Stacked denoising auto-encoder class (SdA)
A stacked denoising autoencoder model is obtained by stacking several
dAs. The hidden layer of the dA at layer `i` becomes the input of
the dA at layer `i+1`. The first layer dA gets as input the input of
the SdA, and the hidden layer of the last dA represents the output.
Note that after pretraining, the SdA is dealt with as a normal MLP,
the dAs are only used to initialize the weights.
"""
def __init__(self, numpy_rng, theano_rng=None, n_ins=784,
hidden_layers_sizes=[500, 500], n_outs=10,
corruption_levels=[0.1, 0.1]):
""" This class is made to support a variable number of layers.
:type numpy_rng: numpy.random.RandomState
:param numpy_rng: numpy random number generator used to draw initial
weights
:type theano_rng: theano.tensor.shared_randomstreams.RandomStreams
:param theano_rng: Theano random generator; if None is given one is
generated based on a seed drawn from `rng`
:type n_ins: int
:param n_ins: dimension of the input to the sdA
:type n_layers_sizes: list of ints
:param n_layers_sizes: intermediate layers size, must contain
at least one value
:type n_outs: int
:param n_outs: dimension of the output of the network
:type corruption_levels: list of float
:param corruption_levels: amount of corruption to use for each
layer
"""
self.sigmoid_layers = []
self.dA_layers = []
self.params = []
self.n_layers = len(hidden_layers_sizes)
assert self.n_layers > 0
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
# allocate symbolic variables for the data
self.x = T.matrix('x') # the data is presented as rasterized images
self.y = T.ivector('y') # the labels are presented as 1D vector of
# [int] labels
# The SdA is an MLP, for which all weights of intermediate layers
# are shared with a different denoising autoencoders
# We will first construct the SdA as a deep multilayer perceptron,
# and when constructing each sigmoidal layer we also construct a
# denoising autoencoder that shares weights with that layer
# During pretraining we will train these autoencoders (which will
# lead to chainging the weights of the MLP as well)
# During finetunining we will finish training the SdA by doing
# stochastich gradient descent on the MLP
for i in xrange(self.n_layers):
# construct the sigmoidal layer
# the size of the input is either the number of hidden units of
# the layer below or the input size if we are on the first layer
if i == 0:
input_size = n_ins
else:
input_size = hidden_layers_sizes[i - 1]
# the input to this layer is either the activation of the hidden
# layer below or the input of the SdA if you are on the first
# layer
if i == 0:
layer_input = self.x
else:
layer_input = self.sigmoid_layers[-1].output
sigmoid_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=hidden_layers_sizes[i],
activation=T.nnet.sigmoid)
# add the layer to our list of layers
self.sigmoid_layers.append(sigmoid_layer)
# its arguably a philosophical question...
# but we are going to only declare that the parameters of the
# sigmoid_layers are parameters of the StackedDAA
# the visible biases in the dA are parameters of those
# dA, but not the SdA
self.params.extend(sigmoid_layer.params)
# Construct a denoising autoencoder that shared weights with this
# layer
dA_layer = dA(numpy_rng=numpy_rng,
theano_rng=theano_rng,
input=layer_input,
n_visible=input_size,
n_hidden=hidden_layers_sizes[i],
W=sigmoid_layer.W,
bhid=sigmoid_layer.b)
self.dA_layers.append(dA_layer)
# We now need to add a logistic layer on top of the MLP
self.logLayer = LogisticRegression(
input=self.sigmoid_layers[-1].output,
n_in=hidden_layers_sizes[-1], n_out=n_outs)
self.params.extend(self.logLayer.params)
# construct a function that implements one step of finetunining
# compute the cost for second phase of training,
# defined as the negative log likelihood
self.finetune_cost = self.logLayer.negative_log_likelihood(self.y)
# compute the gradients with respect to the model parameters
# symbolic variable that points to the number of errors made on the
# minibatch given by self.x and self.y
self.errors = self.logLayer.errors(self.y)
def pretraining_functions(self, train_set_x, batch_size):
''' Generates a list of functions, each of them implementing one
step in trainnig the dA corresponding to the layer with same index.
The function will require as input the minibatch index, and to train
a dA you just need to iterate, calling the corresponding function on
all minibatch indexes.
:type train_set_x: theano.tensor.TensorType
:param train_set_x: Shared variable that contains all datapoints used
for training the dA
:type batch_size: int
:param batch_size: size of a [mini]batch
:type learning_rate: float
:param learning_rate: learning rate used during training for any of
the dA layers
'''
# index to a [mini]batch
index = T.lscalar('index') # index to a minibatch
corruption_level = T.scalar('corruption') # % of corruption to use
learning_rate = T.scalar('lr') # learning rate to use
# number of batches
n_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
# begining of a batch, given `index`
batch_begin = index * batch_size
# ending of a batch given `index`
batch_end = batch_begin + batch_size
pretrain_fns = []
for dA in self.dA_layers:
# get the cost and the updates list
cost, updates = dA.get_cost_updates(corruption_level,
learning_rate)
# compile the theano function
fn = theano.function(inputs=[index,
theano.Param(corruption_level, default=0.2),
theano.Param(learning_rate, default=0.1)],
outputs=cost,
updates=updates,
givens={self.x: train_set_x[batch_begin:
batch_end]})
# append `fn` to the list of functions
pretrain_fns.append(fn)
return pretrain_fns
def predict(self, x_dataset):
predict_fn = theano.function([], sda.logLayer.y_pred,
givens={sda.x: x_dataset})
predicted = predict_fn()
return predicted
def predict_p(self, x_dataset):
predict_p_fn = theano.function([], sda.logLayer.p_y_given_x,
givens={sda.x: x_dataset})
predicted_p = predict_p_fn()
return predicted_p
def build_finetune_functions(self, datasets, batch_size, learning_rate):
'''Generates a function `train` that implements one step of
finetuning, a function `validate` that computes the error on
a batch from the validation set, and a function `test` that
computes the error on a batch from the testing set
:type datasets: list of pairs of theano.tensor.TensorType
:param datasets: It is a list that contain all the datasets;
the has to contain three pairs, `train`,
`valid`, `test` in this order, where each pair
is formed of two Theano variables, one for the
datapoints, the other for the labels
:type batch_size: int
:param batch_size: size of a minibatch
:type learning_rate: float
:param learning_rate: learning rate used during finetune stage
'''
(train_set_x, train_set_y) = datasets[0]
(valid_set_x, valid_set_y) = datasets[1]
(test_set_x, test_set_y) = datasets[2]
# compute number of minibatches for training, validation and testing
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]
n_valid_batches /= batch_size
n_test_batches = test_set_x.get_value(borrow=True).shape[0]
n_test_batches /= batch_size
index = T.lscalar('index') # index to a [mini]batch
# compute the gradients with respect to the model parameters
gparams = T.grad(self.finetune_cost, self.params)
# compute list of fine-tuning updates
updates = []
for param, gparam in zip(self.params, gparams):
updates.append((param, param - gparam * learning_rate))
train_fn = theano.function(inputs=[index],
outputs=self.finetune_cost,
updates=updates,
givens={
self.x: train_set_x[index * batch_size:
(index + 1) * batch_size],
self.y: train_set_y[index * batch_size:
(index + 1) * batch_size]},
name='train')
test_score_i = theano.function([index], self.errors,
givens={
self.x: test_set_x[index * batch_size:
(index + 1) * batch_size],
self.y: test_set_y[index * batch_size:
(index + 1) * batch_size]},
name='test')
valid_score_i = theano.function([index], self.errors,
givens={
self.x: valid_set_x[index * batch_size:
(index + 1) * batch_size],
self.y: valid_set_y[index * batch_size:
(index + 1) * batch_size]},
name='valid')
# Create a function that scans the entire validation set
def valid_score():
return [valid_score_i(i) for i in xrange(n_valid_batches)]
# Create a function that scans the entire test set
def test_score():
return [test_score_i(i) for i in xrange(n_test_batches)]
return train_fn, valid_score, test_score
##### create a function to train an Sda and return it.
def trainSda(X_train_minmax, y_train,
X_validation_minmax, y_validation ,
X_test_minmax, y_test,
hidden_layers_sizes = [100, 100, 100], corruption_levels = [0, 0, 0], batch_size = 30 , \
training_epochs = 100, pretraining_epochs = 100, pretrain_lr = 0.001, finetune_lr=0.1
):
n_visible = X_train_minmax.shape[1]
# compute number of minibatches for training, validation and testing
train_set_x, train_set_y = shared_dataset( (X_train_minmax, y_train), borrow=True)
valid_set_x, valid_set_y = shared_dataset( (X_validation_minmax, y_validation), borrow=True)
test_set_x, test_set_y = shared_dataset( (X_test_minmax, y_test), borrow=True)
n_train_batches = train_set_x.get_value(borrow=True).shape[0]
n_train_batches /= batch_size
# numpy random generator
numpy_rng = numpy.random.RandomState(89677)
print '... building the model'
# construct the stacked denoising autoencoder class
sda = SdA(numpy_rng=numpy_rng, n_ins=n_visible,
hidden_layers_sizes= hidden_layers_sizes,
n_outs=2)
#########################
# PRETRAINING THE MODEL #
#########################
print '... getting the pretraining functions'
pretraining_fns = sda.pretraining_functions(train_set_x=train_set_x,
batch_size=batch_size)
print '... pre-training the model'
start_time = time.clock()
## Pre-train layer-wise
for i in xrange(sda.n_layers):
# go through pretraining epochs
for epoch in xrange(pretraining_epochs):
# go through the training set
c = []
for batch_index in xrange(n_train_batches):
c.append(pretraining_fns[i](index=batch_index,
corruption=corruption_levels[i],
lr=pretrain_lr))
print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
print numpy.mean(c)
end_time = time.clock()
print >> sys.stderr, ('The pretraining code ran for %.2fm' % ((end_time - start_time) / 60.))
########################
# FINETUNING THE MODEL #
########################
# get the training, validation and testing function for the model
print '... getting the finetuning functions'
datasets = [(train_set_x, train_set_y) , (valid_set_x, valid_set_y), (test_set_x, test_set_y)]
train_fn, validate_model, test_model = sda.build_finetune_functions(
datasets=datasets, batch_size=batch_size,
learning_rate=finetune_lr)
print '... finetunning the model'
# early-stopping parameters
patience = 10 * n_train_batches # look as this many examples regardless
patience_increase = 2. # wait this much longer when a new best is
# found
improvement_threshold = 0.995 # a relative improvement of this much is
# considered significant
validation_frequency = min(n_train_batches, patience / 2)
# go through this many
# minibatche before checking the network
# on the validation set; in this case we
# check every epoch
best_params = None
best_validation_loss = numpy.inf
test_score = 0.
start_time = time.clock()
done_looping = False
epoch = 0
while (epoch < training_epochs) and (not done_looping):
epoch = epoch + 1
for minibatch_index in xrange(n_train_batches):
minibatch_avg_cost = train_fn(minibatch_index)
iter = (epoch - 1) * n_train_batches + minibatch_index
if (iter + 1) % validation_frequency == 0:
validation_losses = validate_model()
this_validation_loss = numpy.mean(validation_losses)
print('epoch %i, minibatch %i/%i, validation error %f %%' %
(epoch, minibatch_index + 1, n_train_batches,
this_validation_loss * 100.))
# if we got the best validation score until now
if this_validation_loss < best_validation_loss:
#improve patience if loss improvement is good enough
if (this_validation_loss < best_validation_loss *
improvement_threshold):
patience = max(patience, iter * patience_increase)
# save best validation score and iteration number
best_validation_loss = this_validation_loss
best_iter = iter
# test it on the test set
test_losses = test_model()
test_score = numpy.mean(test_losses)
print((' epoch %i, minibatch %i/%i, test error of '
'best model %f %%') %
(epoch, minibatch_index + 1, n_train_batches,
test_score * 100.))
if patience <= iter:
done_looping = True
break
return sda
# the following is a split stacked auto encoder
class Split_SdA(object):
"""Split Stacked denoising auto-encoder class (SdA)
"""
def __init__(self, numpy_rng, theano_rng=None, n_ins=784,
hidden_layers_sizes_A=[100, 100],
hidden_layers_sizes_B=[100, 100],
n_outs=2,
corruption_levels_A=[0, 0],
corruption_levels_B=[0, 0]):
""" This class is made to support a variable number of layers.
:type numpy_rng: numpy.random.RandomState
:param numpy_rng: numpy random number generator used to draw initial
weights
:type theano_rng: theano.tensor.shared_randomstreams.RandomStreams
:param theano_rng: Theano random generator; if None is given one is
generated based on a seed drawn from `rng`
:type n_ins: int
:param n_ins: dimension of the input to the sdA
:type n_layers_sizes: list of ints
:param n_layers_sizes: intermediate layers size, must contain
at least one value
:type n_outs: int
:param n_outs: dimension of the output of the network
:type corruption_levels: list of float
:param corruption_levels: amount of corruption to use for each
layer
"""
self.sigmoid_layers = []
self.dA_layers = []
self.params = []
self.n_layers = len(hidden_layers_sizes)
assert self.n_layers > 0
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
# allocate symbolic variables for the data
self.x = T.matrix('x') # the data is presented as rasterized images
self.y = T.ivector('y') # the labels are presented as 1D vector of
# [int] labels
# The SdA is an MLP, for which all weights of intermediate layers
# are shared with a different denoising autoencoders
# We will first construct the SdA as a deep multilayer perceptron,
# and when constructing each sigmoidal layer we also construct a
# denoising autoencoder that shares weights with that layer
# During pretraining we will train these autoencoders (which will
# lead to chainging the weights of the MLP as well)
# During finetunining we will finish training the SdA by doing
# stochastich gradient descent on the MLP
for i in xrange(self.n_layers):
# construct the sigmoidal layer
# the size of the input is either the number of hidden units of
# the layer below or the input size if we are on the first layer
if i == 0:
input_size = n_ins
else:
input_size = hidden_layers_sizes[i - 1]
# the input to this layer is either the activation of the hidden
# layer below or the input of the SdA if you are on the first
# layer
if i == 0:
layer_input = self.x
else:
layer_input = self.sigmoid_layers[-1].output
sigmoid_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=hidden_layers_sizes[i],
activation=T.nnet.sigmoid)
# add the layer to our list of layers
self.sigmoid_layers.append(sigmoid_layer)
# its arguably a philosophical question...
# but we are going to only declare that the parameters of the
# sigmoid_layers are parameters of the StackedDAA
# the visible biases in the dA are parameters of those
# dA, but not the SdA
self.params.extend(sigmoid_layer.params)
# Construct a denoising autoencoder that shared weights with this
# layer
dA_layer = dA(numpy_rng=numpy_rng,
theano_rng=theano_rng,
input=layer_input,
n_visible=input_size,
n_hidden=hidden_layers_sizes[i],
W=sigmoid_layer.W,
bhid=sigmoid_layer.b)
self.dA_layers.append(dA_layer)
# We now need to add a logistic layer on top of the MLP
self.logLayer = LogisticRegression(
input=self.sigmoid_layers[-1].output,
n_in=hidden_layers_sizes[-1], n_out=n_outs)
self.params.extend(self.logLayer.params)
# construct a function that implements one step of finetunining
# compute the cost for second phase of training,
# defined as the negative log likelihood
self.finetune_cost = self.logLayer.negative_log_likelihood(self.y)
# compute the gradients with respect to the model parameters
# symbolic variable that points to the number of errors made on the
# minibatch given by self.x and self.y
self.errors = self.logLayer.errors(self.y)