In [2]:
import tensorflow as tf
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline
In [3]:
import math
In [6]:
TRAIN_PATH = '../dataset/iris/iris_training.csv'
TEST_PATH = '../dataset/iris/iris_test.csv'
train_dataset = pd.read_csv(TRAIN_PATH,names=['1','2','3','4','label'],dtype={'labels':np.int64})
test_dataset = pd.read_csv(TEST_PATH,names=['1','2','3','4','label'],dtype={'labels':np.int64})
In [4]:
train_dataset.drop(0,axis=0,inplace=True)
test_dataset.drop(0,axis=0,inplace=True)
# test_dataset.head()
In [5]:
class Dataset():
def __init__(self,dataset,batch_size,name):
self.batch_index = 0
self.dataset = dataset
# self.inputs = dataset
self.name = name
self.batch_size = batch_size
self.data_size = dataset.shape[0]
self.left = 1
def next_batch(self):
# no data sample left can be used
# if(self.left==0):
# return
# get the samples of batch_size amount
# the head index and end index in dataframe
head = self.batch_index*self.batch_size
# check head
if(head>=self.data_size):
# reset it to zero
self.batch_index = 0
head = 0
end = head + self.batch_size
# check end if the end is excess the bound of the dataset
if(end > self.data_size):
end = self.data_size
# self.left = 0
# self.batch_index = 0
# should convert dataframe to numpy-like array
batch_data = self.dataset[head:end].as_matrix()
self.batch_index+=1
# should return the tuple type that contains inputs and labels
# return type (inputs,labels)
return batch_data[:,0:-1].astype(np.float64),batch_data[:,-1].astype(np.int64)
def feed_all(self):
data = self.dataset.as_matrix()
return data[:,0:4].astype(np.float64),data[:,-1].astype(np.int64)
In [103]:
# test Dataset class using train_dataset
# trainDataset = Dataset(train_dataset,10,'train')
# inputs,labels = trainDataset.next_batch()
# inputs
testDataset = Dataset(test_dataset,12,'test')
# inputs ,labels = testDataset.next_batch()
In [107]:
# trainDataset.next_batch()[1]
# a.obj
# len(inputs)
testDataset.data_size
Out[107]:
In [101]:
# def fully_connected_layer(inputs_data,inputs_size,outputs_size,layer_name):
# inputs_data is a tensor that is like [batch_size,inputs_size] and if tensor is initial input inputs_size is
# the same as feature_number
def fully_connected_layer(inputs_data,inputs_size,outputs_size,layer_name):
with tf.name_scope(layer_name):
Weights = tf.Variable(tf.truncated_normal([inputs_size,outputs_size],
stddev=1.0/math.sqrt(float(inputs_size))),
name='Weights',
dtype=tf.float32)
# biases =[batch_size,outputs_size] (batch_size can be ignored)
biases = tf.Variable(tf.zeros(outputs_size),
name='biases',dtype=tf.float32)
outputs_data = tf.nn.relu(tf.matmul(inputs_data,Weights) + biases)
return outputs_data
def softmax_layer(inputs_data,inputs_size):
# to classify the classes
with tf.name_scope('softmax_layer'):
outputs_size = 3
Weights = tf.Variable(tf.truncated_normal([inputs_size,outputs_size],
stddev=1.0/math.sqrt(float(inputs_size))),
name='Weights',
dtype=tf.float32)
# biases =[batch_size,outputs_size] (batch_size can be ignored)
biases = tf.Variable(tf.zeros(outputs_size),
name='biases',
dtype=tf.float32)
logits = tf.matmul(inputs_data,Weights) + biases
return logits
def getloss(logits,labels):
# convert string to int
labels = tf.to_int64(labels)
# print(labels.get_shape())
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits,labels,
name='xentropy')
loss = tf.reduce_mean(cross_entropy,name='xentropy_mean')
return loss
def inference(inputs_data,hidden_units):
# inputs_size is the same as featrue_number
inputs_size = 4
for i in range(len(hidden_units)):
outputs_size = hidden_units[i]
outputs_data = fully_connected_layer(inputs_data,inputs_size,outputs_size,'layer_'+str(i))
inputs_data = outputs_data
inputs_size = outputs_size
logits = softmax_layer(outputs_data,inputs_size)
return logits
def evaluate(logits,labels):
# return number of correct entries
# correct_number is [batch_size] of bool type
# labels = tf.to_int(labels)
labels = tf.to_int64(labels)
# logits = tf.nn.softmax(logits)
correct_number = tf.nn.in_top_k(logits,labels,1)
return tf.reduce_sum(tf.cast(correct_number,tf.int32))
# def training(loss,
# learning_rate,
# batch_size):
# # track the step
# global_step = tf.Variable(0,name='global_step',trainable=False,dtype=tf.int64)
# # learning_rate_tensor = tf.Variable(learning_rate,name='learning_rate',trainable=False,dtype=tf.float32)
# learning_rate_tensor = tf.train.exponential_decay(learning_rate,
# global_step * batch_size,
# 1000,
# 0.95)
# # learning_rate_tensor = tf.cond(tf.equal(tf.mod(global_step+1,1000),0),lambda:,lambda:learning_rate_tensor)
# optimizer = tf.train.GradientDescentOptimizer(learning_rate_tensor)
# train_op = optimizer.minimize(loss,global_step=global_step)
# # train_op = tf.contrib.layers.optimize_loss(optimizer=optimizer,global_step=global_step,loss=loss,learning_rate_decay_fn=tf.train.exponential_decay(decay_rate=0.95,decay_steps=1000),learning_rate=learning_rate)
# return train_op,learning_rate_tensor
def learning_policy(learning_rate_base,
global_step,
decay_step,
decay_rate):
decay_size = (global_step+1) / decay_step
learning_rate_now = learning_rate_base *tf.pow(decay_rate,tf.to_float(decay_size))
return learning_rate_now
def training(loss,
learning_rate,
momentum=0.95,
decay_step=4000,
decay_rate=0.1):
# track steps
global_step = tf.Variable(0,
dtype=tf.int64,
name='global_step',
trainable=False)
# just store learing_rate ,it should not be modified
learning_rate_base = tf.Variable(learning_rate,
dtype=tf.float32,
name='learning_rate_base',
trainable=False)
# here we set high parameter in inner
learning_rate_now = learning_policy(learning_rate_base,
global_step,
decay_step,
decay_rate)
optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate_now,
momentum=momentum)
train_op = optimizer.minimize(loss,
global_step=global_step)
return train_op,learning_rate_now
def placeholder_inputs(batch_size):
# return the packed input samples and labels at batch_size
inputs_placeholder = tf.placeholder(tf.float32,shape=(None,4))
labels_placeholder = tf.placeholder(tf.int32,shape=(None))
return inputs_placeholder, labels_placeholder
# have bugs here
def full_fill_dict(dataset,inputs_placeholder,labels_placeholder):
# the arguments should be placehold type
inputs,labels = trainDataset.next_batch()
return {
inputs_placeholder:inputs,
labels_placeholder:labels,
}
def do_eval(sess,
dataset,
eval_correct,
inputs_placeholder,
labels_placeholder):
# dataset should be test dataset and eval_correct is just a Variable for testing
# here we test all samples in testdataset
# sum_correct = 0
sum_all = dataset.data_size
# for step in range(max_step):
data_all = dataset.feed_all()
feed_dict = {
inputs_placeholder:data_all[0],
labels_placeholder:data_all[1]
}
sum_correct =sess.run(eval_correct,feed_dict=feed_dict)
precision = float(sum_correct) / sum_all
# evaluation in test dataset here
print(' Num examples: %d Num correct: %d Precision @ 1: %0.04f' %
(sum_all, sum_correct, precision))
return precision
def run_training(batch_size,
learning_rate,
hidden_units,
max_steps):
# prepare to train
# build the graph of nn
inputs_placeholder , labels_placeholder = placeholder_inputs(batch_size)
logits = inference(inputs_placeholder,hidden_units)
# be used during training for updating weights and biases
loss = getloss(logits,labels_placeholder)
# be used during testing for evaluating performance of model
eval_correct = evaluate(logits,labels_placeholder)
# get the train operation
train_op,learning_rate_now = training(loss,learning_rate)
# create Dataset class for convience to use
trainDataset = Dataset(train_dataset,
batch_size,
'train')
testDataset = Dataset(test_dataset,
batch_size,
'test')
sess = tf.Session()
loss_all = []
accuracy_all = []
# must put at last
init = tf.initialize_all_variables()
sess.run(init)
for step in range(max_steps):
# train here
# get the samples from train dataset to feed
feed_dict = full_fill_dict(trainDataset,
inputs_placeholder,
labels_placeholder
)
# inputs,labels = trainDataset.next_batch()
# feed_dict = {
# inputs_placeholder:inputs,
# labels_placeholder:labels,
# }
_ , loss_value,learning_rate_tensor_value = sess.run([train_op,
loss,
learning_rate_now],
feed_dict=feed_dict)
loss_all.append(loss_value)
if (step %100==0):
# print status here
print('Step %d: loss = %.5f learning_rate = %.5f' % (step,
loss_value,
learning_rate_tensor_value))
if((step + 1) %200==0 or (step + 1) == max_steps):
print('Testing Data eval:')
precision = do_eval(sess,
testDataset,
eval_correct,
inputs_placeholder,
labels_placeholder)
accuracy_all.append(precision)
# plot graph for training and testing
# loss
plt.figure(1)
# plt.subplot(111)
plt.plot(range(len(loss_all)),loss_all)
plt.title('loss')
#
plt.figure(2)
# plt.subplot(211)
plt.plot(range(len(accuracy_all)),accuracy_all)
plt.title('accuracy')
plt.show()
sess.close()
In [102]:
tf.reset_default_graph()
run_training(2,0.001,[10,20,10],10000)
In [79]:
tf.train.MomentumOptimizer?