In [1]:
# Header
# Basic libraries & options
from __future__ import print_function
#Basic libraries
import numpy as np
import tensorflow as tf
print('Tensorflow version: ', tf.__version__)
import time
# Select GPU
import os
os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"]="1"
#Show images
import matplotlib.pyplot as plt
%matplotlib inline
# plt configuration
plt.rcParams['figure.figsize'] = (10, 10) # size of images
plt.rcParams['image.interpolation'] = 'nearest' # show exact image
plt.rcParams['image.cmap'] = 'gray' # use grayscale
In [2]:
from tensorflow.contrib.keras import datasets, layers, models, optimizers
# Data
(X_train, y_train), (X_test, y_test) = datasets.mnist.load_data(path='mnist.npz')
X_train = X_train.astype('float32') / 255.
X_test = X_test.astype('float32') / 255.
In [3]:
# Base Keras Model
images = layers.Input(batch_shape=(None, 28, 28), dtype='float32', name='Images')
flat = layers.Flatten(name='Flat_image')(images)
dense = layers.Dense(500, activation='relu', name='Dense_layer')(flat)
output = layers.Dense(10, activation='softmax', name='Dense_output')(dense)
model_linear = models.Model(inputs=images, outputs=output)
# Train
sgd_optimizer = optimizers.SGD(lr=0.01)
model_linear.compile(loss='sparse_categorical_crossentropy',
optimizer=optimizers.SGD(lr=0.01), metrics=['accuracy'])
history_linear = model_linear.fit(X_train, y_train, batch_size=128, epochs=50,
verbose=1, validation_data=(X_test, y_test))
In [4]:
# Start interactive session
sess = tf.InteractiveSession()
In [5]:
# Define input placeholders
x = tf.placeholder(tf.float32, shape=[None, 28, 28])
y = tf.placeholder(tf.int64, shape=[None, ])
# Define model using Keras layers
flat = layers.Flatten(name='Flat_image')(x)
dense = layers.Dense(500, activation='relu', name='Dense_layer')(flat)
output = layers.Dense(10, activation='relu', name='Dense_output')(dense)
# Define the Tensorflow Loss
cross_entropy = tf.losses.sparse_softmax_cross_entropy(y, output)
# Define the Tensorflow Train function
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy)
# Create an Accuracy metric to evaluate in test
y_pred = tf.nn.softmax(output)
correct_prediction = tf.equal(y, tf.argmax(y_pred,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
In [6]:
# Define a generator to data. Same code to an HDF5 source
def batch_generator(X, y, batch_size=64):
data_size = X.shape[0]
# Randomize batches in each epoch
batch_randomized = np.random.permutation(range(0, data_size-batch_size, batch_size))
# Iterate over each batch
for batch in batch_randomized:
x_batch = X[batch : batch+batch_size]
y_batch = y[batch : batch+batch_size]
yield x_batch, y_batch
In [7]:
# Intialize vars
sess.run(tf.global_variables_initializer())
In [8]:
# Basic iterator over the epochs and the batches to execute the trainer.
batch_size = 128
num_epoch = 50
for epoch in range(num_epoch):
for x_batch, y_batch in batch_generator(X_train, y_train, batch_size=batch_size):
train_step.run(feed_dict={x: x_batch, y: y_batch})
print(epoch, accuracy.eval(feed_dict={x: X_test, y: y_test}))
In [9]:
# Intialize vars
sess.run(tf.global_variables_initializer())
# Early stopping code. Stop if the current accuray is < that the min of the last 5 epochs.
batch_size = 128
acc_test=[]
epoch=0
stop=True
while stop:
#Train
epoch += 1
for x_batch, y_batch in batch_generator(X_train, y_train, batch_size=batch_size):
train_step.run(feed_dict={x: x_batch, y: y_batch})
# Test
acc_test += [accuracy.eval(feed_dict={x: X_test, y: y_test})]
if epoch%10==0:
print('Epoch: ', epoch, 'Accuracy: ', acc_test[-1])
# Stopping criteria
if epoch>10 and acc_test[-1] < min(acc_test[-10:-1]):
stop=False
print('STOP. Accuracy: ', acc_test[-1])
In [10]:
#When finish, close the interactive session
sess.close()
# Reset the graph to the next experiments
tf.reset_default_graph()
In [11]:
# Start interactive session
sess = tf.InteractiveSession()
# Convolutional model
x = tf.placeholder(tf.float32, shape=[None, 28, 28])
y = tf.placeholder(tf.int64, shape=[None, ])
image = tf.reshape(x,[-1,28,28,1])
conv1 = layers.Conv2D(20, (5,5))(image)
pool1 = layers.MaxPooling2D(pool_size=(2, 2))(conv1)
conv2 = layers.Conv2D(20, (5,5))(pool1)
pool2 = layers.MaxPooling2D(pool_size=(2, 2))(conv2)
flat = layers.Flatten(name='Flat_image')(pool2)
print('Flat layer: ', flat)
# Tensorflow Dense layer
W_dense = tf.Variable(tf.truncated_normal([320, 500], stddev=0.1), name='W_dense')
b_dense = tf.Variable(tf.constant(0.1, shape=[500]), name='b_dense')
dense1 = tf.nn.relu(tf.matmul(flat, W_dense) + b_dense)
#dense1 = layers.Dense(500, activation='relu', name='Dense_1')(flat)
output = layers.Dense(10, activation='linear', name='Dense_output')(dense1)
# Define the Tensorflow Loss
cross_entropy = tf.losses.sparse_softmax_cross_entropy(y, output)
# Define the Tensorflow Train function
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy)
# Create an Accuracy metric to evaluate in test
y_pred = tf.nn.softmax(output)
correct_prediction = tf.equal(y, tf.argmax(y_pred,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
In [12]:
# Intialize vars
sess.run(tf.global_variables_initializer())
# Early stopping code. Stop if the current accuray is < that the min of the last 5 epochs.
batch_size = 128
acc_test=[]
epoch=0
stop=True
while stop:
#Train
epoch += 1
for x_batch, y_batch in batch_generator(X_train, y_train, batch_size=batch_size):
train_step.run(feed_dict={x: x_batch, y: y_batch})
# Test
acc_test += [accuracy.eval(feed_dict={x: X_test, y: y_test})]
if epoch%1==0:
print('Epoch: ', epoch, 'Accuracy: ', acc_test[-1])
# Stopping criteria
if epoch>10 and acc_test[-1] < min(acc_test[-10:-1]):
stop=False
print('STOP. Accuracy: ', acc_test[-1])
In [13]:
#When finish, close the interactive session
sess.close()
# Reset the graph to the next experiments
tf.reset_default_graph()
- The same previous convolutional model with the commands that need tensorboard
Based on https://www.tensorflow.org/how_tos/summaries_and_tensorboard/index.html
In [18]:
def variable_summaries(var, name):
"""Attach a lot of summaries to a Tensor."""
with tf.name_scope('summaries'):
mean = tf.reduce_mean(var)
tf.summary.scalar('mean/' + name, mean)
tf.summary.scalar('sttdev/' + name, tf.sqrt(tf.reduce_mean(tf.square(var - mean))))
tf.summary.scalar('max/' + name, tf.reduce_max(var))
tf.summary.scalar('min/' + name, tf.reduce_min(var))
tf.summary.histogram(name, var)
In [19]:
# Start interactive session
sess = tf.InteractiveSession()
# Convolutional model
x = tf.placeholder(tf.float32, shape=[None, 28, 28])
y = tf.placeholder(tf.int64, shape=[None, ])
image = tf.reshape(x,[-1,28,28,1])
with tf.name_scope('Conv1') as scope:
conv1 = layers.Conv2D(20, (5,5))(image)
pool1 = layers.MaxPooling2D(pool_size=(2, 2))(conv1)
variable_summaries(pool1, "pool1_summary")
with tf.name_scope('Conv2') as scope:
conv2 = layers.Conv2D(20, (5,5))(pool1)
pool2 = layers.MaxPooling2D(pool_size=(2, 2))(conv2)
variable_summaries(pool2, "pool2_summary")
with tf.name_scope('Dense') as scope:
flat = layers.Flatten(name='Flat_image')(pool2)
# Tensorflow Dense layer
W_dense = tf.Variable(tf.truncated_normal([320, 500], stddev=0.1), name='W_dense')
variable_summaries(W_dense, "W_dense_summary") # Summaries of the layer weigths
b_dense = tf.Variable(tf.constant(0.1, shape=[500]), name='b_dense')
variable_summaries(b_dense, "b_dense_summary") # Summaries of the layer weigths
dense1 = tf.nn.relu(tf.matmul(flat, W_dense) + b_dense)
variable_summaries(dense1, "dense1_summary") # Summaries of the layer output
output = layers.Dense(10, activation='linear', name='Dense_output')(dense1)
# Define the Tensorflow Loss
#with tf.name_scope('loss') as scope:
cross_entropy = tf.losses.sparse_softmax_cross_entropy(y, output)
loss_summary = tf.summary.scalar("loss", cross_entropy)
# Define the Tensorflow Train function
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy)
# Create an Accuracy metric to evaluate in test
#with tf.name_scope('acc') as scope:
y_pred = tf.nn.softmax(output)
correct_prediction = tf.equal(y, tf.argmax(y_pred,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
accuracy_summary = tf.summary.scalar("acc", accuracy)
# Add summaries to the graph
summaries_dir = '/tmp/tensorboard/tf'
with tf.name_scope('summaries') as scope:
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(summaries_dir + '/train', sess.graph)
test_writer = tf.summary.FileWriter(summaries_dir + '/test')
In [20]:
# Intialize vars
sess.run(tf.global_variables_initializer())
# Early stopping code. Stop if the current accuray is < that the min of the last 5 epochs.
batch_size = 128
acc_test=[]
epoch=0
stop=True
while stop:
#Train
epoch += 1
counter = 0
for x_batch, y_batch in batch_generator(X_train, y_train, batch_size=batch_size):
train_step.run(feed_dict={x: x_batch, y: y_batch})
counter += 1
if counter%10 == 0:
summary_str = merged.eval(feed_dict={x: x_batch, y: y_batch}) #TENSORBOARD
train_writer.add_summary(summary_str, epoch) #TENSORBOARD
# Test
acc_test += [accuracy.eval(feed_dict={x: X_test, y: y_test})]
summary_str = merged.eval(feed_dict={x: X_test, y: y_test}) #TENSORBOARD
test_writer.add_summary(summary_str, epoch) #TENSORBOARD
if epoch%1==0:
print('Epoch: ', epoch, 'Accuracy: ', acc_test[-1])
# Stopping criteria
if epoch>10 and acc_test[-1] < min(acc_test[-10:-1]):
stop=False
print('STOP. Accuracy: ', acc_test[-1])
In [21]:
sess.close()
tf.reset_default_graph()
At the end execute tensorboar with:
cd /tmp
tensorboard --logdir=./tensorboard
And accest to it in:
http://<My_server_IP>:6006
In [ ]: