In [8]:
import numpy as np
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
In [9]:
mnist = input_data.read_data_sets('MNIST_data', one_hot = True)
In [70]:
def xavier_init(shape, uniform=True):
n_inputs = shape[0]
n_outputs = shape[1]
if uniform:
init_range = np.sqrt(6.0 / (n_inputs + n_outputs))
return tf.random_uniform(shape, minval = -init_range, maxval = init_range, dtype = tf.float32)
else:
stddev = tf.sqrt(3.0 / (n_inputs + n_outputs))
return tf.truncated_normal(shape, stddev=stddev)
class SimpleAutoencoder(object):
def __init__(self, n_input, n_hidden, transfer_function = tf.nn.relu,
optimizer = tf.train.AdamOptimizer(learning_rate=0.001), scale = 0.01):
self.n_input = n_input
self.n_hidden = n_hidden
self.transfer = transfer_function
self.scale = tf.constant(scale)
self.weights = self._initialize_weights()
self.x = tf.placeholder(tf.float32, [None, self.n_input])
noise_data = tf.add(self.x, scale * tf.random_normal((n_input,)))
self.hidden = self.transfer(tf.add(tf.matmul(noise_data, self.weights['w1']), self.weights['b1']))
#self.decoder = tf.matmul(self.hidden, self.weights['w2']) + self.weights['b2']
self.decoder = tf.nn.softmax(tf.matmul(self.hidden, self.weights['w2']) + self.weights['b2'])
self.cost = tf.reduce_sum(tf.pow(tf.subtract(self.decoder, self.x), 2.0))
#self.cost = tf.reduce_mean(-tf.reduce_sum(self.x * tf.log(self.decoder), reduction_indices = [1]))
#self.optimizer = optimizer.minimize(self.cost)
self.optimizer = tf.train.GradientDescentOptimizer(0.001).minimize(self.cost)
self.y = tf.placeholder(tf.float32, [None, 10])
self.pred_y = tf.nn.softmax(tf.matmul(self.hidden, self.weights['w3']) + self.weights['b3'])
#self.hidden_drop = tf.nn.dropout(self.hidden, 0.75)
#self.pred_y = tf.nn.softmax(tf.matmul(self.hidden_drop, self.weights['w3']) + self.weights['b3'])
self.pred_cost = tf.reduce_mean(-tf.reduce_sum(self.y * tf.log(self.pred_y), reduction_indices = [1]))
#self.pred_optimizer = optimizer.minimize(self.pred_cost)
self.pred_optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(self.pred_cost)
self.correct_pred = tf.equal(tf.argmax(self.pred_y, 1), tf.argmax(self.y, 1))
self.accuracy = tf.reduce_mean(tf.cast(self.correct_pred, tf.float32))
self.sess = tf.Session()
self.sess.run(tf.global_variables_initializer())
def _initialize_weights(self):
weights = dict()
weights['w1'] = tf.Variable(xavier_init([self.n_input, self.n_hidden], uniform=True))
weights['b1'] = tf.Variable(tf.zeros([self.n_hidden]))
weights['w2'] = tf.Variable(xavier_init([self.n_hidden, self.n_input], uniform=True))
weights['b2'] = tf.Variable(tf.zeros([self.n_input]))
weights['w3'] = tf.Variable(xavier_init([self.n_hidden, 10], uniform=True))
weights['b3'] = tf.Variable(tf.zeros([10]))
return weights
def train(self, X):
cost, _ = self.sess.run((self.cost, self.optimizer),
feed_dict = {self.x: X})
return cost
def pred_train(self, X, Y):
cost, _ = self.sess.run((self.pred_cost, self.pred_optimizer),
feed_dict = {self.x: X, self.y: Y})
return cost
def pred_test(self, X, Y):
return self.sess.run(self.accuracy, feed_dict = {self.x:mnist.test.images, self.y:mnist.test.labels})
def getWeights(self, s):
return self.sess.run(self.weights[s])
autoencoder = SimpleAutoencoder(n_input = 784, n_hidden = 300)
In [71]:
training_epochs = 10000
batch_size = 128
for epoch in range(training_epochs):
batch_xs, batch_ys = mnist.train.next_batch(batch_size)
pred_cost = autoencoder.pred_train(batch_xs, batch_ys)
acc = autoencoder.pred_test(mnist.test.images, mnist.test.labels)
cost = autoencoder.train(batch_xs)
if epoch % 200 == 0:
print(pred_cost, acc, cost)
# w3 = autoencoder.getWeights('w3')
# print(np.min(w3), np.max(w3))
In [ ]: