In [14]:
#import tf
import tensorflow as tf
import numpy as np
import warnings
warnings.filterwarnings('ignore')
In [15]:
import PIL.ImageGrab
from matplotlib.pyplot import imshow,subplots,plot
import seaborn as sns
%matplotlib inline
def print_img_from_clipboard():
img = PIL.ImageGrab.grabclipboard()
fig, ax = subplots(figsize=(90, 30))
imshow(img, interpolation='nearest')
In [3]:
#import mnist
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets('MNIST',one_hot=True)
In [4]:
#lets import iterate the batches to be used in train
def iterate_over_batches(input_array,batch_size=100):
#input_values - np.array which is needed to be iterated
#yeilds batched with size = batch_size
total_values = len(list(input_array))
#print(total_values)
for index in range(int(total_values/batch_size)):
start_index = index*batch_size
end_index = start_index+batch_size
yield input_array[start_index:end_index]
if total_values%batch_size!=0.0:
yield input_array[-int(total_values%batch_size):]
In [5]:
#lets create a fuction that creates deep fully connected network
def create_fully_connected_network_softmax(input_placeholder,output_placeholder,dimensions,non_linear=tf.nn.tanh,softmax=tf.nn.softmax):
#input
#dimensions - dimension list a deep layers
#
#input_placeholder = tf.placeholder that contains input placeholder
#output_placeholder = tf.placeholder that contains output placeholder
#output
#y_predicted tensor which is required to evaluate our loss function
current_input = input_placeholder
n_input = int(list(input_placeholder.get_shape())[1])
#lets glue input placeholder to deep part to the graph by creating each deep layer
for layer_index,n_output in enumerate(dimensions):
#creating layer with specific name
with tf.variable_scope('deep/layer%i' % layer_index):
#create weight matrix of the layer. it reduces dimension from n_input to n_output by using linear algebra
with tf.name_scope('W'):
W = tf.Variable(tf.random_normal(name='W',shape=[n_input,n_output],mean=0.0,stddev=0.02))
with tf.name_scope('b'):
b = tf.Variable(tf.zeros(name='b',shape=[n_output]))
with tf.name_scope('summator'):
h = tf.add(tf.matmul(current_input,W),b)
#and lets create a nonlinear transformation
with tf.name_scope('non_linear'):
non_linear_transformation = non_linear(h)
#and use this transformation and its column space as an input to the next layer
current_input,n_input = non_linear_transformation,n_output
#lets glue deep part to output placeholder
with tf.variable_scope('softmax'):
n_output= int(list(output_placeholder.get_shape())[1])
with tf.name_scope('W'):
W = tf.Variable(tf.random_normal(name='W',shape=[n_input,n_output],mean=0.0,stddev=0.02))
with tf.name_scope('b'):
b = tf.Variable(tf.zeros(name='b',shape=[n_output]))
with tf.name_scope('summator'):
h = tf.add(tf.matmul(current_input,W),b)
with tf.name_scope('softmax'):
y_pred = softmax(h)
return y_pred
In [9]:
#lets create a fuction that runs the fucking training
def train_fully_connected_network_softmax(X_train,y_train,X,y,loss,metric,optimizer=tf.train.AdamOptimizer,\
X_val = None,y_val=None,X_test=None,y_test=None,
learning_rate=1e-3,n_epochs = 10 ,\
standartize=True,save=False,\
log=False
):
#X_Train,y_train - arrays required for a train
#X_val,y_val - validation datasets
#X_test,y_test - test dataset
#X,y - input and output placeholders of the graph
#loss - loss which will be optimized
#metric - target metric to calculate
#learning rate
#n_epochs
#standartize - whether to substract mean
#save - filename - if specified saves model to filename
#log - filename - if specified will log the train to filename
#we will substract mean if required
if standartize:
mean_value = np.mean(X_train,axis=0)
else:
mean_value = np.zeros(X_train.shape[1])
#lets initialize values of the variables
with tf.name_scope('Optimizer'):
optimizer = optimizer(learning_rate=learning_rate).minimize(loss)
init_op = tf.global_variables_initializer()
#we will log loss and metric if requiored
loss_sum = tf.scalar_summary('train_loss',loss)
metric_sum = tf.scalar_summary('train_accuracy',metric)
summapy_op = tf.merge_summary([loss_sum,metric_sum])
if X_val!=None:
val_loss_sum = tf.scalar_summary('validation_loss',loss)
val_metric_sum = tf.scalar_summary('validation_accuracy',metric)
val_summapy_op = tf.merge_summary([val_loss_sum,val_metric_sum])
#we will save model if required
if save:
saver = tf.train.Saver()
#lets start the session
with tf.Session() as sess:
sess.run(init_op)
#lets create a log writer if required and counter
if log:
writer = tf.train.SummaryWriter(log, graph=tf.get_default_graph())
#lets start training
for epoch in range(n_epochs):
#over all the batches lets train
for batch_xs,batch_ys in zip(iterate_over_batches(X_train),iterate_over_batches(y_train)):
#lets just train if its too boring
sess.run(optimizer, feed_dict={X: (batch_xs-mean_value), y: batch_ys})
epoch_loss,train_metric,train_sum = sess.run([loss,metric,summapy_op], feed_dict={X:(X_train[:10000]-mean_value),y:y_train[:10000]})
if log:
writer.add_summary(train_sum,epoch)
if X_val !=None:
val_metric,val_sum = sess.run([metric,val_summapy_op], feed_dict={X:(X_val-mean_value),y:y_val})
if log:
writer.add_summary(val_sum,epoch)
print('epoch %i loss: %f train_metric %f validation_metric %f' % (epoch,epoch_loss,train_metric,val_metric))
else:
print('epoch %i loss: %f train_metric %f' % (epoch,epoch_loss,train_metric))
if save:
saver.save(sess,save)
if X_test!=None:
test_metric = sess.run(metric, feed_dict={X:(X_test-mean_value),y:y_test})
print('test_metric %f' % test_metric)
if log:
writer.close()
In [12]:
#parameters
learning_rate = 0.0001
batch_size = 100
display_step = 10
tf.reset_default_graph()
#lets do a fucking split by hand
X_train,X_val,X_test,y_train,y_val,y_test = mnist.train.images,mnist.validation.images,mnist.test.images,mnist.train.labels,mnist.validation.labels,mnist.test.labels
#lets specify input dimensios
input_dimension = X_train.shape[1]
output_dimension = y_train.shape[1]
#lets specify deep dimension
deep_dimensions = [512,256,128]
In [13]:
#lets create a deep model
with tf.name_scope('X'):
X = tf.placeholder(tf.float32,shape=[None,input_dimension])
with tf.name_scope('y'):
y = tf.placeholder(tf.float32,shape=[None,output_dimension])
y_pred = create_fully_connected_network_softmax(X,y,deep_dimensions,non_linear=tf.nn.tanh)
#lets define a loss functions with will be optimized
with tf.name_scope('cross_entropy'):
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(y_pred, y))
#lets specify second metric to check
with tf.name_scope('accuracy'):
correct_predictions = tf.equal(tf.arg_max(y_pred,1),tf.arg_max(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_predictions,tf.float32))
#lets create loss and accuracy summarizer which will draw our graphs
In [16]:
#lets train
train_fully_connected_network_softmax(X_train,y_train,X,y,cross_entropy,accuracy,X_val=X_val,y_val=y_val,n_epochs=10,X_test=X_test,y_test=y_test,learning_rate=1e-3,standartize=False,log='log')
In [25]:
#96.4% accuracy on a test set. Impressive for the simplest network with 10 epochs
In [18]:
!tensorboard --logdir=run1:log/ --port 6006
In [19]:
print_img_from_clipboard()
In [22]:
#lets check train loss
print_img_from_clipboard()
In [ ]:
In [23]:
#lets check training accuracy
print_img_from_clipboard()
In [24]:
#lets check validation accuracy
print_img_from_clipboard()
In [ ]: