In [1]:
import datetime
import Image
import gc
import numpy as np
import os
import random
from scipy import misc
import string
import time
# Set some Theano config before initializing
os.environ["THEANO_FLAGS"] = "mode=FAST_RUN,device=cpu,floatX=float32,allow_gc=False,openmp=True"
import theano
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot as plt
import emotion_model
import dwdii_transforms
random.seed(20275)
In [2]:
print "device:", theano.config.device
print "floatX:", theano.config.floatX
print "mode:", theano.config.mode
print "openmp:", theano.config.openmp
print "allow_gc:", theano.config.allow_gc
In [3]:
imagePath = "/root/facial_expressions/images"
dataPath = "/root/facial_expressions/data/legend.csv"
imgResize = (150, 150)
In [4]:
os.listdir('/root/facial_expressions/data')
Out[4]:
In [7]:
maxData = 1845
X_data, Y_data = dwdii_transforms.load_data(dataPath, imagePath, maxData = maxData, verboseFreq = 200, imgResize=imgResize)
print X_data.shape
print Y_data.shape
In [8]:
imgDataGenCount = 12
transformCount = 9 + imgDataGenCount
newImgs = np.zeros([X_data.shape[0] * transformCount, X_data.shape[1], X_data.shape[2]])
newYs = np.zeros([Y_data.shape[0] * transformCount, Y_data.shape[1]], dtype=np.int8)
print newImgs.shape
print newYs.shape
In [9]:
img = X_data[0]
img.shape
Out[9]:
In [10]:
ndx = 0
for i in range(X_data.shape[0]):
img = X_data[i]
img0 = dwdii_transforms.reflectY(img)
newImgs[ndx] = img0
newYs[ndx] = Y_data[i]
#misc.imsave("test0.png", img0)
ndx += 1
img1 = dwdii_transforms.cvDilate(img)
newImgs[ndx] = img1
newYs[ndx] = Y_data[i]
#misc.imsave("test1.png", img1)
ndx += 1
img2 = dwdii_transforms.cvErode(img)
newImgs[ndx] = img2
newYs[ndx] = Y_data[i]
#misc.imsave("test2.png", img2)
ndx += 1
img3 = dwdii_transforms.cvDilate2(img)
newImgs[ndx] = img3
newYs[ndx] = Y_data[i]
#misc.imsave("test3.png", img3)
ndx += 1
#img4 = dwdii_transforms.cvMedianBlur(img)
#newImgs[ndx] = img4
#newYs[ndx] = Y_data[i]
#misc.imsave("test4.png", img4)
#ndx += 1
img5 = dwdii_transforms.cvExcessiveSharpening(img)
newImgs[ndx] = img5
newYs[ndx] = Y_data[i]
#misc.imsave("test5.png", img5)
ndx += 1
img6 = dwdii_transforms.cvEdgeEnhancement(img)
newImgs[ndx] = img6
newYs[ndx] = Y_data[i]
#misc.imsave("test6.png", img6)
ndx += 1
img7 = dwdii_transforms.cvBlurMotion1(img)
newImgs[ndx] = img7
newYs[ndx] = Y_data[i]
#misc.imsave("test7.png", img7)
ndx += 1
img8 = dwdii_transforms.cvBlurMotion2(img)
newImgs[ndx] = img8
newYs[ndx] = Y_data[i]
#misc.imsave("test8.png", img8)
ndx += 1
img9 = dwdii_transforms.reflectY(img)
newImgs[ndx] = img9
#print img9.shape
newYs[ndx] = Y_data[i]
#misc.imsave("test9.png", img9)
ndx += 1
for n in range(imgDataGenCount):
imgX = emotion_model.imageDataGenTransform(img, Y_data[i])
#print imgX
#print imgX.shape
imgX = imgX.reshape(150, 150)
#print imgX.shape
newImgs[ndx] = imgX
newYs[ndx] = Y_data[i]
#misc.imsave("testX.png", imgX)
ndx += 1
#break
print("Done", str(datetime.datetime.now()))
In [11]:
import numpy
print numpy.version.version
print numpy.__version__
In [12]:
gc.collect()
Out[12]:
In [13]:
X_data2 = np.concatenate((X_data, newImgs))
Y_data2 = np.concatenate((Y_data, newYs))
print X_data2.shape
print Y_data2.shape
The following code segment splits the data into training and test data sets. Currently this is a standard 80/20 split for training and test respectively after performing a random shuffle using the unison_shuffled_copies help method.
In [14]:
skippedTransforms = False
if skippedTransforms:
X_data2 = X_data
Y_data2 = Y_data
In [15]:
gc.collect()
Out[15]:
In [16]:
def unison_shuffled_copies(a, b):
"""http://stackoverflow.com/a/4602224/2604144"""
assert len(a) == len(b)
p = np.random.permutation(len(a))
return a[p], b[p]
# First shuffle the data
X_data2, Y_data2 = unison_shuffled_copies(X_data2, Y_data2)
# Split the data into Training and Test sets
trainNdx = int(X_data2.shape[0] * .8)
print trainNdx
X_train, X_test = np.split(X_data2, [trainNdx])
Y_train, Y_test = np.split(Y_data2, [trainNdx])
print X_train.shape
print X_test.shape
print Y_train.shape
print Y_test.shape
In [17]:
# Map the emotions to integers for categorization later.
emotions = dwdii_transforms.emotionNumerics()
print emotions
print len(emotions)
Our model is a convolutional neural network with 4 hidden layers.
In [18]:
#model = emotion_model.emotion_model_v3_2(len(emotions), verbose=True)
model = emotion_model.emotion_model_jh_v5(len(emotions), verbose=True,
input_shape=(1,X_train.shape[1],X_train.shape[2]))
#print(model.summary())
The following code segment trains the model using the run_network helper function. Previously, I was hitting a memory issue (my interpretation), when I have batches above a certain threshold. Batches=10 work fine, but batches of 100 are too big. May need to allocate more RAM to the docker container. I have since moved to a Docker Cloud / Amazon Web Services instance with increased memory and this issue has been mitigated.
In [19]:
# Reshape to the appropriate shape for the CNN input
testX = X_test.reshape(X_test.shape[0], 1, X_train.shape[1],X_train.shape[2])
trainX = X_train.reshape(X_train.shape[0], 1, X_train.shape[1],X_train.shape[2])
In [20]:
loadWeights = False
if loadWeights:
model.load_weights("dwdii-emo-01vjh-1-Cloud.hdf5")
In [ ]:
print "Training start: " + str(datetime.datetime.now())
m, h = emotion_model.run_network([trainX, testX, Y_train, Y_test], model, batch=200, epochs=30, verbosity=1)
In [ ]:
model.save_weights("dwdii-emo-150-jhv5-21tf-30e-Cloud.hdf5", overwrite=True)
In [ ]:
predictOutput = model.predict(testX)
In [ ]:
predictOutput[0]
In [ ]:
import collections
prMetrics = {}
# For each emotion
for e in emotions.keys():
prMetrics[e] = collections.defaultdict(int)
print prMetrics
numEmo = dwdii_transforms.numericEmotions()
print numEmo
In [ ]:
# For each predicted image
for i in range(len(predictOutput)):
arPred = np.array(predictOutput[i])
predictionProb = arPred.max()
predictionNdx = arPred.argmax()
predictedEmo = numEmo[predictionNdx]
# True Positives
if predictionNdx == Y_test[i]:
prMetrics[predictedEmo]["TruePos"] += 1.0
# False Positives
else:
prMetrics[predictedEmo]["FalsePos"] += 1.0
# Look for false negatives
for i in range(len(Y_test)):
arPred = np.array(predictOutput[i])
predictionProb = arPred.max()
predictionNdx = arPred.argmax()
predictedEmo = numEmo[predictionNdx]
yEmo = numEmo[int(Y_test[i])]
if Y_test[i] == predictionNdx:
# Ok
pass
else:
prMetrics[yEmo]["FalseNeg"] += 1.0
In [ ]:
prMetrics
In [ ]:
emotionPrecision = {}
emotionRecall = {}
for p in prMetrics:
emotionPrecision[p] = prMetrics[p]["TruePos"] / ( prMetrics[p]["TruePos"] + prMetrics[p]["FalsePos"])
emotionRecall[p] = prMetrics[p]["TruePos"] /( prMetrics[p]["TruePos"] + prMetrics[p]["FalseNeg"])
print "Precision by Emotion"
print "--------------------"
for e in emotionPrecision:
print e, emotionPrecision[e]
print
print "Recall by Emotion"
print "--------------------"
for e in emotionRecall:
print e, emotionRecall[e]
In [ ]: