In [ ]:
samplePerType = nblPerTrnFile + nblPerTestFile
data0 = 0.8*np.random.uniform(size=(samplePerType,nchannel,nrod))
# data0 = 1.0*np.random.triangular(0.,0.5,1.0,size=(samplePerType,nrod
data1 = 1.0*np.random.triangular(0.,0.5,1.0,size=(samplePerType,nchannel,nrod))
set0 = np.asarray([[data0[i],np.asarray([1.,0.,0.,0.,0.,0.])] for i in range(samplePerType)])
set1 = np.asarray([[data1[i],np.asarray([0.,1.,0.,0.,0.,0.])] for i in range(samplePerType)])
trn_data = np.concatenate((set0[:nblPerTrnFile],set1[:nblPerTrnFile]))
test_data = np.concatenate((set0[nblPerTrnFile:],set1[nblPerTrnFile:]))
In [1]:
#%source bin/activate
import tensorflow as tf
import random
import time
import matplotlib.pyplot as plt
import numpy as np
from tensorflow.contrib import rnn
%matplotlib inline
In [2]:
rundir = "/home/walterms/project/walterms/mcmd/nn/data/"
trndir = rundir+"train/"
testdir = rundir+"test/"
unlbldir = rundir+"unlbl/"
bThetas = False
nrod = 400
nlabel = 6
nchannel = 3
if bThetas: nchannel = 1
nblPerTrnFile = 5000
nblPerTestFile = 300
# trnlist = ["tjam", "xjam", "ujam", "ljam", "edge15.00", "edge30.00"]
trnlist = ["edge15.00", "edge30.00"]
# testlist = ["tjam", "xjam", "ujam", "ljam", "edge15.00", "edge30.00"]
testlist = ["edge15.00", "edge30.00"]
unlbllist = ["xmelt"]
trnlist = [trndir+x for x in trnlist]
testlist = [testdir+x for x in testlist]
unlbllist = [unlbldir+x for x in unlbllist]
In [3]:
# Compile training set
trn_data = []
shufIdx = [p for p in range(nrod)]
random.shuffle(shufIdx)
for f in trnlist:
print "Processing " + f + " as training data"
thetas = []
xs = []
ys = []
fin = open(f, 'r')
nTrn = 0
for line in fin.readlines():
if line == "\n": continue
if line.startswith("label"):
# Done a block
lbls = np.zeros((nlabel))
lbl = int(float(line.split()[1]))
lbls[lbl] = 1.
if bThetas:
thdata = None
random.shuffle(thetas)
thdata = (-0.5) + np.asarray(thetas)
formatted_data = np.stack([thdata,thdata,thdata])
# thdata = thdata.reshape(1,nrod)
trn_data.append([formatted_data, lbls])
else:
# Insert data as triplets
channels = [xs,ys,thetas]
prep_data = []
for ch in channels:
prep_data.append((-0.5)+np.asarray(ch))
formatted_data = np.stack(prep_data)
# formatted_data = np.stack([np.asarray(xs),np.asarray(ys),np.asarray(thetas)])
np.random.shuffle(formatted_data)
trn_data.append([formatted_data, lbls])
thetas = []
xs = []
ys = []
nTrn+=1
if nTrn == nblPerTrnFile:
break
continue
spt = [float(x) for x in line.split()]
xs.append(spt[0])
ys.append(spt[1])
thetas.append(spt[2])
fin.close()
trn_data = np.asarray(trn_data)
print "Done compiling training set,", len(trn_data), "images"
In [4]:
test_data = []
for f in testlist:
print "Adding " + f + " to test set"
thetas = []
xs = []
ys = []
fin = open(f, 'r')
nbl = 0
for line in fin.readlines():
if line == "\n": continue
if line.startswith("label"):
# Done a block
lbls = np.zeros((nlabel))
lbl = int(float(line.split()[1]))
lbls[lbl] = 1.
if bThetas:
random.shuffle(thetas)
thdata = (-0.5) + np.asarray(thetas)
formatted_data = np.stack([thdata,thdata,thdata])
# thdata = thdata.reshape(1,nrod)
test_data.append([formatted_data, lbls])
else:
# Insert data as triplets
channels = [xs,ys,thetas]
prep_data = []
for ch in channels:
prep_data.append((-0.5)+np.asarray(ch))
formatted_data = np.stack(prep_data)
# formatted_data = np.stack([np.asarray(xs),np.asarray(ys),np.asarray(thetas)])
np.random.shuffle(formatted_data)
test_data.append([formatted_data, lbls])
thetas = []
xs = []
ys = []
nbl+=1
if nbl == nblPerTestFile:
break
continue
spt = [float(x) for x in line.split()]
xs.append(spt[0])
ys.append(spt[1])
thetas.append(spt[2])
fin.close()
test_data = np.asarray(test_data)
print "Done compiling test set,", len(test_data), "images"
In [5]:
print len(trn_data),len(test_data)
In [ ]:
plt.hist([trn_data[0][0][0],trn_data[-1][0][0]])
In [7]:
###################
# RNN #
###################
tf.reset_default_graph()
eta = 1e-4
batchsize = 500
nInput = nrod
nHidden = 256
nLayer = 1
sizedict = {"nchannel": nchannel,
"batchsize": batchsize,
"nHidden": nHidden,
"nLayer": nLayer,
"nlabel": nlabel}
X = tf.placeholder("float", [None, nchannel, nInput])
Y = tf.placeholder("float", [None, nlabel])
weights = {
'h1': tf.Variable(tf.random_normal([nHidden, nHidden],stddev=0.1, dtype=tf.float32), name='W1'),
'out': tf.Variable(tf.random_normal([nHidden, nlabel],stddev=0.1, dtype=tf.float32), name='W'),
}
biases = {
'b1': tf.Variable(tf.random_normal([nHidden],stddev=0.1, dtype=tf.float32), name='b1'),
'out': tf.Variable(tf.random_normal([nlabel],stddev=0.1, dtype=tf.float32), name='bias')
}
# Define an lstm cell with tensorflow
def lstm_cell(nUnits):
return rnn.BasicLSTMCell(nUnits)
def RNN(x, weights, biases):
# Build high lvl rnn for time series
x = tf.unstack(x, nchannel, 1)
cell = lstm_cell(nHidden)
stack = rnn.MultiRNNCell([lstm_cell(nHidden) for _ in range(nLayer)])
# Get lstm cell output
outputs, states = rnn.static_rnn(stack, x, dtype=tf.float32)
layer_trans = tf.add(tf.matmul(outputs[-1], weights['h1']), biases['b1'])
layer_trans = tf.nn.sigmoid(layer_trans)
# Linear activation, using rnn inner loop last output
return tf.matmul(layer_trans, weights['out']) + biases['out']
###############################
# Save structure that works
# def RNN(x, weights, biases):
# # Lower level RNN structure to learn classifications
# x = tf.unstack(x, nchannel, 1)
# cell = lstm_cell(nHidden)
# stack = rnn.MultiRNNCell([lstm_cell(nHidden) for _ in range(nLayer)])
# # Get lstm cell output
# outputs, states = rnn.static_rnn(stack, x, dtype=tf.float32)
# layer_trans = tf.add(tf.matmul(outputs[-1], weights['h1']), biases['b1'])
# layer_trans = tf.nn.sigmoid(layer_trans)
# # Linear activation, using rnn inner loop last output
# return tf.matmul(layer_trans, weights['out']) + biases['out']
###############################
logits = RNN(X,weights,biases)
prediction = tf.nn.softmax(logits)
# Define loss and optimizer
loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
logits=logits, labels=Y))
optimizer = tf.train.AdamOptimizer(learning_rate=eta).minimize(loss_op)
correct_prediction = tf.equal(tf.argmax(prediction, 1), tf.argmax(Y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
print "Done"
In [ ]:
nEpoch = 200
epochEval = 20
batchesPerEpoch = int(len(trn_data)/batchsize)
ntestbatches = int(len(test_data)/batchsize)
testdata_tmp = [t for t in test_data]
random.shuffle(testdata_tmp)
xtest = np.asarray([t[0] for t in testdata_tmp])
ytest = np.asarray([t[1] for t in testdata_tmp])
y_out_0 = []
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
testy = sess.run([teststate],feed_dict={X: xtest[0:batchsize], Y: ytest[0:batchsize]})
print testy
print np.asarray(testy).shape
In [9]:
class runrecord:
def __init__(self,
trnlist,
testlist,
bThetas,
sizedict,
batchsize,
eta,
nEpoch):
self.params = {
"trnlist":trnlist,
"testlist":testlist,
"bThetas":bThetas,
"batchsize":batchsize,
"eta":eta,
"nEpoch":nEpoch
}
self.sizedict=sizedict
self.testacc = np.zeros(shape=(nEpoch,))
self.trnacc = np.zeros(shape=(nEpoch,))
self.testlosses = np.zeros(shape=(nEpoch,))
self.trnlosses = np.zeros(shape=(nEpoch,))
records = []
In [17]:
# Remove most recent run from records
tmp = list(records)
records = []
for r in tmp[0:-1]:
records.append(r)
In [10]:
nEpoch = 100
epochEval = 5
batchesPerEpoch = int(len(trn_data)/batchsize)
ntestbatches = int(len(test_data)/batchsize)
testdata_tmp = [t for t in test_data]
random.shuffle(testdata_tmp)
xtest = np.asarray([t[0] for t in testdata_tmp])
ytest = np.asarray([t[1] for t in testdata_tmp])
y_out_0 = []
thisrecord = runrecord(trnlist,testlist,bThetas,sizedict,batchsize,eta,nEpoch)
records.append(thisrecord)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
with sess.as_default():
assert tf.get_default_session() is sess
# with tf.Session() as sess:
# sess.run(tf.global_variables_initializer())
for e in range(nEpoch):
avg_loss = 0.
batchIdx = [bb for bb in range(batchesPerEpoch)]
random.shuffle(batchIdx)
for b in batchIdx:
ib = b*batchsize
# Pepare a batch
yin = np.asarray([trn_data[ib+iib][1] for iib in xrange(batchsize)])
xin = np.asarray([trn_data[ib+iib][0] for iib in xrange(batchsize)])
_,l = sess.run([optimizer, loss_op], feed_dict={X: xin, Y: yin})
# Run optimization op (backprop)
avg_loss += l / batchesPerEpoch
if e % epochEval == 0:
# Eval on test set
avg_testloss = 0.
testacc = 0.
for tb in range(ntestbatches):
ib = tb*batchsize
xin = xtest[ib:ib+batchsize]
yin = ytest[ib:ib+batchsize]
acc, youts, testloss = sess.run([accuracy,prediction,loss_op],feed_dict={
X: xin, Y: yin})
testacc += acc/float(ntestbatches)
avg_testloss += testloss/float(ntestbatches)
print('epoch %d, test accuracy %g, testloss %g, avg_loss %g'
% (e,testacc,avg_testloss,avg_loss))
y_out_0.append(youts[0])
# thisrecord.testlosses[e] = testcost
# thisrecord.trnlosses[e] = trncost
# thisrecord.testacc[e] = test_accuracy
# thisrecord.trnacc[e] = trnacc
print "Done"
In [22]:
# Compile unlabeled data
unlbl_data = []
maxUnlbl = -1
for f in unlbllist:
print "Processing " + f + " as training data"
thetas = []
xs = []
ys = []
fin = open(f, 'r')
nbl = 0
for line in fin.readlines():
if line == "\n":
# Done a block
if bThetas:
thdata = None
random.shuffle(thetas)
thdata = (-0.5) + np.asarray(thetas)
formatted_data = np.stack([thdata,thdata,thdata])
# thdata = thdata.reshape(1,nrod)
trn_data.append([formatted_data, lbls])
else:
# Insert data as triplets
channels = [xs,ys,thetas]
prep_data = []
for ch in channels:
prep_data.append((-0.5)+np.asarray(ch))
formatted_data = np.stack(prep_data)
# formatted_data = np.stack([np.asarray(xs),np.asarray(ys),np.asarray(thetas)])
np.random.shuffle(formatted_data)
unlbl_data.append([formatted_data, lbls])
thetas = []
xs = []
ys = []
nbl+=1
if nbl == maxUnlbl:
break
continue
spt = [float(x) for x in line.split()]
xs.append(spt[0])
ys.append(spt[1])
thetas.append(spt[2])
fin.close()
unlbl_data = np.asarray(unlbl_data)
print "Done compiling unlabeled set,", len(unlbl_data), "images"
In [ ]:
with sess.as_default():
assert tf.get_default_session() is sess
# Pepare a batch
xin = np.asarray([trn_data[ib+iib][0] for iib in xrange(batchsize)])
_,l = sess.run([optimizer, loss_op], feed_dict={X: xin, Y: yin})
# Run optimization op (backprop)
avg_loss += l / batchesPerEpoch
if e % epochEval == 0:
# Eval on test set
avg_testloss = 0.
testacc = 0.
for tb in range(ntestbatches):
ib = tb*batchsize
xin = xtest[ib:ib+batchsize]
yin = ytest[ib:ib+batchsize]
acc, youts, testloss = sess.run([accuracy,prediction,loss_op],feed_dict={
X: xin, Y: yin})
testacc += acc/float(ntestbatches)
avg_testloss += testloss/float(ntestbatches)
print('epoch %d, test accuracy %g, testloss %g, avg_loss %g'
% (e,testacc,avg_testloss,avg_loss))
y_out_0.append(youts[0])
print "Done"
In [ ]:
# Loss plot
# Mask on zeros
f, ax = plt.subplots(1,2)
for r in records[-1:]:
x = np.argwhere(r.testlosses>0)
ax0, = ax[0].plot(x,r.trnlosses[x],'b-',label='train loss')
axtwin = ax[0].twinx()
ax0t, = axtwin.plot(x,r.testlosses[x],'g:',label='test loss')
l1 = plt.legend([ax0,ax0t],["train loss", "test loss"])
ax[1].plot(x,r.trnacc[x],'b-',label="train")
ax[1].plot(x,r.testacc[x],'g:',label="test")
ax[1].legend()
ax[0].set_yscale("log", nonposy='mask')
axtwin.set_yscale("log", nonposy='mask')
plt.gcf().set_size_inches(14,5)