In [5]:
import collections
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import sklearn
import seaborn as sns
from sklearn import svm
from my_rbm import Rbm
import six.moves.cPickle as pickle
import sys
from pandas import *
from sklearn.preprocessing import OneHotEncoder
from yadlt.models.rbm_models import dbn
%matplotlib inline
In [2]:
mnist = input_data.read_data_sets("MNIST_data/", one_hot=False)
train_images = mnist.train.images
test_images = mnist.test.images
In [3]:
num_hidden = 64
num_epochs=1
rbm = Rbm(num_hidden=num_hidden, num_classes=10, num_features=784, learning_rate=0.01)
rbm.init_rbm()
rbm.fit(mnist.train.images, mnist.test.images, num_epochs=num_epochs)
In [28]:
probs_index = dict()
probs = dict()
probs_avg = dict()
probs_min = dict()
probs_max = dict()
x = [i for i in range(num_hidden)]
for i in range(10):
probs_index[i] = [idx for idx, label in enumerate(mnist.train.labels) if i == mnist.train.labels[idx]]
probs[i] = [rbm.get_h_prob_out([mnist.train.images[idx]])[0] for idx in probs_index[i]]
probs_avg[i] = np.average(probs[i], axis=0)
probs_min[i] = np.min(probs[i], axis=0)
probs_max[i] = np.max(probs[i], axis=0)
In [30]:
num = 0
sorted_indexes = np.argsort(probs_avg[num])
for i in range(10):
probs_avg[i] = [probs_avg[i][j] for j in sorted_indexes]
col_num = 10
plt.figure()
plt.suptitle('Average of activation probabilities after %s epochs, sorted by number: %s' % (num_epochs, str(num)))
for cidx in range(col_num):
plt.subplot(col_num, 1, cidx + 1, axisbg='white')
plt.bar(x, probs_avg[cidx])
plt.xticks(())
plt.yticks(())
In [31]:
num = 0
sorted_indexes = np.argsort(probs_min[num])
for i in range(10):
probs_min[i] = [probs_min[i][j] for j in sorted_indexes]
col_num = 10
plt.figure()
plt.suptitle('Minimum of activation probabilities after %s epochs, sorted by number: %s' % (num_epochs, str(num)))
for cidx in range(col_num):
plt.subplot(col_num, 1, cidx + 1, axisbg='white')
plt.bar(x, probs_min[cidx])
plt.xticks(())
plt.yticks(())
In [32]:
num = 0
sorted_indexes = np.argsort(probs_max[num])
for i in range(10):
probs_max[i] = [probs_max[i][j] for j in sorted_indexes]
col_num = 10
plt.figure()
plt.suptitle('Maximum of activation probabilities after %s epochs, sorted by number: %s' % (num_epochs, str(num)))
for cidx in range(col_num):
plt.subplot(col_num, 1, cidx + 1, axisbg='white')
plt.bar(x, probs_max[cidx])
plt.xticks(())
plt.yticks(())
In [4]:
mnist_oh = input_data.read_data_sets("MNIST_data/", one_hot=True)
rbm.fit_predictor(train_data=mnist_oh.train.images, train_labels=mnist_oh.train.labels,
test_data=mnist_oh.test.images, test_labels=mnist_oh.test.labels, num_steps=30000)
In [35]:
probs_index = dict()
probs = dict()
probs_avg = dict()
probs_min = dict()
probs_max = dict()
x = [i for i in range(num_hidden)]
for i in range(10):
probs_index[i] = [idx for idx, label in enumerate(mnist.train.labels) if i == mnist.train.labels[idx]]
probs[i] = [rbm.get_h_prob_out([mnist.train.images[idx]])[0] for idx in probs_index[i]]
probs_avg[i] = np.average(probs[i], axis=0)
probs_min[i] = np.min(probs[i], axis=0)
probs_max[i] = np.max(probs[i], axis=0)
In [37]:
num = 0
sorted_indexes = np.argsort(probs_avg[num])
for i in range(10):
probs_avg[i] = [probs_avg[i][j] for j in sorted_indexes]
col_num = 10
plt.figure()
plt.suptitle('Average of activation probabilities after %s epochs, sorted by number: %s' % (num_epochs, str(num)))
for cidx in range(col_num):
plt.subplot(col_num, 1, cidx + 1, axisbg='white')
plt.bar(x, probs_avg[cidx])
plt.xticks(())
plt.yticks(())
In [36]:
num = 0
sorted_indexes = np.argsort(probs_min[num])
for i in range(10):
probs_min[i] = [probs_min[i][j] for j in sorted_indexes]
col_num = 10
plt.figure()
plt.suptitle('Minimum of activation probabilities after %s epochs, sorted by number: %s' % (num_epochs, str(num)))
for cidx in range(col_num):
plt.subplot(col_num, 1, cidx + 1, axisbg='white')
plt.bar(x, probs_min[cidx])
plt.xticks(())
plt.yticks(())
In [42]:
probs_avg_df = DataFrame(probs_avg)
In [69]:
probs_avg_df.transpose().corr()
Out[69]:
In [68]:
sns.heatmap(probs_avg_df.transpose().corr())
Out[68]:
In [43]:
row_num = 1
col_num = 10
plt.figure(figsize=(col_num * 2, row_num + 1))
for ridx in range(row_num):
for cidx in range(col_num):
plt.subplot(row_num, col_num, (ridx * col_num + cidx) + 1, axisbg='white')
plt.bar(x, probs[ridx][cidx])
plt.xticks(())
plt.yticks(())
In [117]:
row_num = 1
col_num = 10
plt.figure(figsize=(col_num * 2, row_num + 1))
for ridx in range(row_num):
for cidx in range(col_num):
plt.subplot(row_num, col_num, (ridx * col_num + cidx) + 1, axisbg='white')
plt.bar(x, probs[ridx + 1][cidx])
plt.xticks(())
plt.yticks(())
In [125]:
row_num = 1
col_num = 10
number_offset = 3
plt.figure(figsize=(col_num * 2, row_num + 1))
for ridx in range(row_num):
for cidx in range(col_num):
plt.subplot(row_num, col_num, (ridx * col_num + cidx) + 1, axisbg='white')
plt.bar(x, probs[ridx + number_offset][cidx])
plt.xticks(())
plt.yticks(())
In [126]:
row_num = 1
col_num = 10
offset = 3
plt.figure(figsize=(col_num * 2, row_num + 1))
for ridx in range(row_num):
for cidx in range(col_num):
plt.subplot(row_num, col_num, (ridx * col_num + cidx) + 1, axisbg='white')
plt.imshow(mnist.train.images[probs_index[ridx + offset][cidx]].reshape(28,28))
plt.xticks(())
plt.yticks(())
In [7]:
last_output = np.empty((0, rbm.get_h_prob_out([train_images[0]]).shape[1]))
last_output.shape
for i in range(mnist.train.images.shape[0]):
last_output = np.vstack((last_output, rbm.get_h_prob_out([mnist.train.images[i]])))
if i % 1000 == 0:
print('done with: %s' % i)
In [8]:
last_output_test = np.empty((0, rbm.get_h_prob_out([mnist.test.images[0]]).shape[1]))
last_output_test.shape
for i in range(mnist.test.images.shape[0]):
last_output_test = np.vstack((last_output_test, rbm.get_h_prob_out([mnist.test.images[i]])))
if i % 999 == 0:
print('done with: %s' % (i + 1))
In [18]:
sklearn.learn.datasets
last_output.shape
In [4]:
clf = svm.SVC(decision_function_shape='ovr', kernel='poly', degree=4, coef0=1.0)
clf.fit(mnist.train.images, mnist.train.labels)
clf.score(mnist.test.images, y_true=mnist.test.labels)
In [11]:
clf = svm.SVC(decision_function_shape='ovr', kernel='sigmoid', coef0=1.0)
clf.fit(last_output, mnist.train.labels)
y_pred = clf.predict(last_output_test)
sklearn.metrics.accuracy_score(y_pred=y_pred, y_true=mnist.test.labels)
Out[11]:
In [12]:
clf = svm.SVC(decision_function_shape='ovr', kernel='sigmoid', coef0=0.0)
clf.fit(last_output, mnist.train.labels)
y_pred = clf.predict(last_output_test)
sklearn.metrics.accuracy_score(y_pred=y_pred, y_true=mnist.test.labels)
Out[12]:
In [16]:
clf = svm.SVC(decision_function_shape='ovr', kernel='poly', degree=2, coef0=0.0)
clf.fit(last_output, mnist.train.labels)
y_pred = clf.predict(last_output_test)
sklearn.metrics.accuracy_score(y_pred=y_pred, y_true=mnist.test.labels)
Out[16]:
In [17]:
clf = svm.SVC(decision_function_shape='ovr', kernel='poly', degree=2, coef0=1.0)
clf.fit(last_output, mnist.train.labels)
y_pred = clf.predict(last_output_test)
sklearn.metrics.accuracy_score(y_pred=y_pred, y_true=mnist.test.labels)
Out[17]:
In [21]:
clf_raw = svm.SVC(decision_function_shape='ovr', kernel='linear')
clf_raw.fit(mnist.train.images, mnist.train.labels)
Out[21]:
In [23]:
y_pred_raw = clf_raw.predict(mnist.test.images)
sklearn.metrics.accuracy_score(y_pred=y_pred_raw, y_true=mnist.test.labels)
Out[23]:
In [45]:
a = np.asarray([[1,2,3], [4,5,6], [7,8,9]])
with open("foo.csv", 'ab') as f:
np.savetxt(f, dw, delimiter=',')
In [22]:
import sys
sys.getsizeof(dw_0) / 1024 / 1024 * mnist.test.images.shape[0]
Out[22]:
In [14]:
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
train_images = mnist.train.images
test_images = mnist.test.images
In [5]:
num_hidden = 64
rbm = Rbm(num_hidden=num_hidden, num_classes=10, num_features=784, learning_rate=0.01)
rbm.init_rbm()
In [12]:
rbm.fit(mnist.train.images, mnist.test.images, num_epochs=1)
rbm.fit_predictor(train_data=mnist.train.images,
train_labels=mnist.train.labels,
test_data=mnist.test.images,
test_labels=mnist.test.labels)
In [24]:
rbm = Rbm(num_hidden=128, num_classes=10, num_features=784, learning_rate=0.01)
rbm.init_rbm()
rbm.fit(mnist.train.images, mnist.test.images, num_epochs=10)
rbm.fit_predictor(train_data=mnist.train.images, train_labels=mnist.train.labels,
test_data=mnist.test.images, test_labels=mnist.test.labels, num_steps=30000)
In [2]:
def unpickle(file):
fo = open(file, 'rb')
dict = pickle.load(fo, encoding='latin-1')
fo.close()
return dict
def from_flat_to_3d(image):
# print(image.shape)
return np.dstack((image[0:1024].reshape(32,32),
image[1024:2048].reshape(32,32),
image[2048:3072].reshape(32,32)))
cifar_test = unpickle('cifar-10-batches-py/test_batch')
cifar_test['data'] = cifar_test['data'].astype(np.float32) / 255
cifar_test['data_3d'] = np.array([from_flat_to_3d(image) for image in cifar_test['data']])
cifar = unpickle('cifar-10-batches-py/data_batch_1')
for i in range(2, 6):
tmp = unpickle('cifar-10-batches-py/data_batch_' + str(i))
cifar['data'] = np.vstack((cifar['data'], tmp['data']))
cifar['labels'] = np.concatenate((cifar['labels'], tmp['labels']))
cifar['data'] = cifar['data'].astype(np.float32) / 255
cifar['data_3d'] = np.array([from_flat_to_3d(image) for image in cifar['data']])
# cifar['data_bw'] = (cifar['data'][:,0:1024] + cifar['data'][:,1024:2048] + cifar['data'][:, 2048:3072]) / 3
# cifar_test['data_bw'] = (cifar_test['data'][:,0:1024] + cifar_test['data'][:,1024:2048] + cifar_test['data'][:, 2048:3072]) / 3
enc = OneHotEncoder()
cifar['labels_oh'] = enc.fit_transform(cifar['labels'].reshape(-1, 1))
cifar['labels_oh'] = cifar['labels_oh'].toarray()
cifar_test['labels'] = np.array(cifar_test['labels'])
cifar_test['labels_oh'] = enc.fit_transform(cifar_test['labels'].reshape(-1, 1))
cifar_test['labels_oh'] = cifar_test['labels_oh'].toarray()
# pca = PCA(whiten=True)
# cifar['data_bw_whitened'] = pca.fit_transform(cifar['data_bw'])
# cifar_test['data_bw_whitened'] = pca.fit_transform(cifar_test['data_bw'])
In [8]:
print(cifar['data'].shape)
print(cifar['labels'].shape)
In [9]:
plt.imshow(cifar['data'][3][0:1024].reshape(32,32))
Out[9]:
In [4]:
num_hidden = 761
num_epochs=10
rbm = Rbm(num_hidden=num_hidden, num_classes=10, num_features=3072, learning_rate=0.001)
rbm.init_rbm()
rbm.fit(cifar['data'], cifar_test['data'], num_epochs=num_epochs)
In [8]:
last_output = np.empty((0, rbm.get_h_prob_out([cifar['data'][0]]).shape[1]))
last_output.shape
for i in range(cifar['data'].shape[0]):
last_output = np.vstack((last_output, rbm.get_h_prob_out([cifar['data'][i]])))
if i % 1000 == 0:
print('done with: %s' % i)
In [14]:
last_output_test = np.empty((0, rbm.get_h_prob_out([cifar_test['data'][0]]).shape[1]))
last_output_test.shape
for i in range(cifar_test['data'].shape[0]):
last_output_test = np.vstack((last_output_test, rbm.get_h_prob_out([cifar_test['data'][i]])))
if i % 999 == 0:
print('done with: %s' % (i + 1))
In [11]:
clf = svm.SVC(decision_function_shape='ovr', kernel='poly', degree=3, coef0=1.0)
clf.fit(last_output, cifar['labels'])
In [15]:
clf.score(last_output_test, cifar_test['labels'])
Out[15]:
In [13]:
cifar_test['labels'].shape
last_output_test.shape
Out[13]:
In [19]:
num_hidden = 256
num_epochs=10
from my_rbm import Rbm
rbm = Rbm(num_hidden=num_hidden, num_classes=10, num_features=3072, learning_rate=0.01)
rbm.init_rbm()
rbm.fit(cifar['data'], cifar['labels'], num_epochs=0)
rbm.fit_predictor(cifar['data'], cifar['labels'], cifar_test['data'], cifar_test['labels'], num_steps=30000)
In [12]:
rbm_layers = [761]
rbm_learning_rate = [0.001]
rbm_num_epochs = [10]
rbm_batch_size = [25]
rbm_gibbs_k = [1]
srbm = dbn.DeepBeliefNetwork(
do_pretrain=True,
rbm_layers=rbm_layers, dataset='cifar',
finetune_act_func='relu', rbm_learning_rate=rbm_learning_rate,
verbose=True, rbm_num_epochs=rbm_num_epochs, rbm_gibbs_k=rbm_gibbs_k,
rbm_gauss_visible=False, rbm_stddev=0.1,
momentum=0.5, rbm_batch_size=rbm_batch_size, finetune_learning_rate=0.01,
finetune_num_epochs=10, finetune_batch_size=25,
finetune_opt='gradient_descent', finetune_loss_func='softmax_cross_entropy',
finetune_dropout=1)
In [ ]:
srbm.pretrain(cifar['data'], cifar_test['data'])
In [ ]: