In [ ]:
import sys
sys.path.insert(0, '../')
In [ ]:
import pandas as pd
import numpy as np
np.set_printoptions(precision=3, linewidth=200, suppress=True)
from sklearn.model_selection import KFold
from sklearn.model_selection import train_test_split as train_test_split
from sklearn.preprocessing import StandardScaler, MinMaxScaler
import sklearn.metrics as skm
from sklearn import svm
from sklearn.externals import joblib
import os, time
import matplotlib.pyplot as plt
from scipy.misc import toimage
%matplotlib inline
In [ ]:
from sklearn.neural_network import MLPClassifier
In [ ]:
from library.datasets.cifar10 import CIFAR10
from library.utils import file_utils
from library.plot_tools import plot
In [ ]:
total_time = 0
In [ ]:
exp_no = 1
file_no = 3
one_hot = False
data_source = 'Website'
train_validate_split = 0.2
num_images_required = 1.0
scale_method = 'StandardScaler'
In [ ]:
mlp_hidden_size = (3072, 3072)
mlp_iterations = 2000
mlp_solver = 'adam'
mlp_alpha = 1e-5
early_stop=False
param_name = 'exp_' + str(exp_no).zfill(3) + '_solver_' + mlp_solver
In [ ]:
param_name = 'exp_' + str(exp_no).zfill(3) + '_solver_' + mlp_solver
output_directory = '../logs/cifar10/' + str(file_no).zfill(3) + '_mlp_raw/' + 'exp_no_' + str(exp_no).zfill(3) + '/'
In [ ]:
start = time.time()
cifar10 = CIFAR10(one_hot_encode=one_hot, num_images=num_images_required,
train_validate_split=None, endian='little')
cifar10.load_data(train=True, test=True, data_directory='./datasets/cifar10/')
end = time.time()
print('[ Step 0] Dataset loaded in %5.6f ms' %((end-start)*1000))
print('Dataset size: ' + str(cifar10.train.data.shape))
num_train_images = cifar10.train.data.shape[0]
total_time += (end-start)
In [ ]:
cifar10.plot_sample(plot_data=True, plot_test=True, fig_size=(7, 7))
In [ ]:
cifar10.plot_images(cifar10.train.data[:50, :], cifar10.train.class_names[:50],
nrows=5, ncols=10, fig_size=(20,50), fontsize=35, convert=True)
In [ ]:
start = time.time()
if scale_method == 'StandardScaler':
ss = StandardScaler()
elif scale_method == 'MinMaxScaler':
ss = MinMaxScaler()
else:
ss = StandardScaler()
data_images = ss.fit_transform(cifar10.train.data)
test_images = ss.fit_transform(cifar10.test.data)
end = time.time()
print('[ Step 2] Dataset transformations done in %.6f ms' %((end-start)*1000))
print('Training the classifier on %d images' % num_train_images)
print('Dataset size: ' + str(cifar10.train.data.shape))
total_time += (end-start)
In [ ]:
start = time.time()
X_train, X_test, y_train, y_test = train_test_split(data_images, cifar10.train.class_labels,
test_size=train_validate_split)
X_train = np.array(X_train)
X_test = np.array(X_test)
y_train = np.array(y_train)
y_test = np.array(y_test)
print('Train set shape : ' + str(X_train.shape))
print('Train labels shape : ' + str(y_train.shape))
print('Validation set shape : ' + str(X_test.shape))
print('Validation labels shape : ' + str(y_test.shape))
end = time.time()
print('[ Step 3] Train-test split done in %.6f ms' %((end-start)*1000))
total_time += (end-start)
In [ ]:
start = time.time()
clf = MLPClassifier(solver=mlp_solver, alpha=mlp_alpha, hidden_layer_sizes=mlp_hidden_size, random_state=1,
max_iter=mlp_iterations, verbose=True, early_stopping=early_stop)
print(clf)
end = time.time()
print('[ Step 4] Made the Multi layer perceptron classifier in %.6f ms' %((end-start)*1000))
total_time += (end-start)
In [ ]:
start = time.time()
print(X_train.shape)
print(y_train.shape)
clf.fit(X_train, y_train)
end = time.time()
print('[ Step 5] Made the model in %.6f ms' %((end-start)*1000))
total_time += (end-start)
In [ ]:
start = time.time()
file_utils.mkdir_p(output_directory)
model_output_path = output_directory + '03_' + param_name + '.pkl'
joblib.dump(clf, model_output_path)
end = time.time()
print('[ Step 6] Write obtained model to %s in %.6f ms' %(model_output_path, ((end-start)*1000)))
total_time += (end-start)
In [ ]:
y_predict = clf.predict(X_test)
labels = sorted(list(set(cifar10.train.class_labels)))
print('Labels: ' + str(labels))
plot.plot_confusion_matrix(y_test, y_predict, classes=cifar10.classes, fig_size=(8,6),
normalize=True, title='Confusion matrix for validation set with SVC')
end = time.time()
print('[ Step 7] Make prediction on validation dataset in %.6f ms' %((end-start)*1000))
total_time += (end-start)
In [ ]:
start = time.time()
print('Classification report:')
print(skm.classification_report(y_test, y_predict, target_names=cifar10.classes))
validate_accuracy = skm.accuracy_score(y_test, y_predict, normalize=True)
print('Validation accuracy score: ' + str(validate_accuracy))
end = time.time()
print('[ Step 8] Generating classification on validation dataset in %.6f ms' %((end-start)*1000))
total_time += (end-start)
In [ ]:
start = time.time()
prediction_numbers = clf.predict(test_images)
prediction_classes = []
num_test_images = test_images.shape[0]
for i in range(num_test_images):
prediction_classes.append(cifar10.classes[int(prediction_numbers[i])])
end = time.time()
print('[ Step 9] Make prediction on test dataset in %.6f ms' %((end-start)*1000))
total_time += (end-start)
In [ ]:
cifar10.plot_images(cifar10.test.data[:50], cifar10.test.class_names[:50], cls_pred=prediction_classes[:50],
nrows=5, ncols=10, fig_size=(20,50), fontsize=30, convert=True)
In [ ]:
start = time.time()
plot.plot_confusion_matrix(cifar10.test.class_labels, prediction_numbers, classes=cifar10.classes,
normalize=True, title='Confusion matrix for test set with MLP with raw pixels')
print(skm.classification_report(cifar10.test.class_labels, prediction_numbers, target_names=cifar10.classes))
test_accuracy = skm.accuracy_score(cifar10.test.class_labels, prediction_numbers, normalize=True)
print('Accuracy score on test data: ' + str(test_accuracy))
end = time.time()
total_time += (end-start)
In [ ]:
start = time.time()
print('Prediction done on %d images' %cifar10.test.data.shape[0])
print('Accuracy of the classifier: %.4f' %clf.score(test_images, cifar10.test.class_labels))
end = time.time()
total_time += (end-start)
In [ ]:
start = time.time()
indices = np.arange(1, cifar10.test.data.shape[0]+1)
predictions = np.column_stack((indices, prediction_classes))
file_utils.mkdir_p(output_directory)
output_csv_file = output_directory + '03_' + param_name + '.csv'
column_names = ['id', 'label']
predict_test_df = pd.DataFrame(data=predictions, columns=column_names)
predict_test_df.to_csv(output_csv_file, index=False)
end = time.time()
print('[ Step 11] Writing the test data to file: %s in %.6f ms' %(output_csv_file, (end-start)*1000))
total_time += (end-start)
In [ ]:
print('Code took %.6f s to run on training with %d examples' % (total_time,num_train_images))
In [ ]:
print('| %d | ' %exp_no, end='')
print('%d | '%num_train_images, end='')
print('Hidden layers size = %s, Max. Iterations: %d, Alpha: %.6f, Solver: %s, Random state: 1 | '
%(str(mlp_hidden_size), mlp_iterations, mlp_alpha, mlp_solver), end='')
print('[Link](%s) | ' %output_csv_file, end='')
print('%.4f | ' %validate_accuracy, end='')
print('%.4f | ' %test_accuracy, end='')
print('%s | ' %data_source, end='')
print('[Link](%s) |' %model_output_path)
In [ ]:
def output_HTML(read_file, output_file):
from nbconvert import HTMLExporter
import codecs
import nbformat
exporter = HTMLExporter()
output_notebook = nbformat.read(read_file, as_version=4)
print()
output, resources = exporter.from_notebook_node(output_notebook)
codecs.open(output_file, 'w', encoding='utf-8').write(output)
In [ ]:
%%javascript
var notebook = IPython.notebook
notebook.save_notebook()
In [ ]:
%%javascript
var kernel = IPython.notebook.kernel;
var thename = window.document.getElementById("notebook_name").innerHTML;
var command = "theNotebook = " + "'"+thename+"'";
kernel.execute(command);
In [ ]:
current_file = './' + theNotebook + '.ipynb'
output_file = output_directory + str(file_no).zfill(2) + '_exp_no_' + str(exp_no) + '_' + theNotebook + '.html'
print('Current file: ' + str(current_file))
print('Output file: ' + str(output_file))
file_utils.mkdir_p(output_directory)
output_HTML(current_file, output_file)
In [ ]:
print('Code took %.6f s to run on training with %d examples' % (total_time,num_train_images))
In [ ]: