Compare the predictive performance of using MRD with various feature representations for the feature space and the observation space.
In this Ipython notebook, the predictive performance of 3 different feature representations of each observation space are evaluated. The metrics for evaluation are RMS error, normalized RMS error and pearson correlation.
In [ ]:
# import the modules
import GPy
import csv
import numpy as np
import pandas as pd
import seaborn as sns
import cPickle as pickle
import scipy.stats as stats
import sklearn.metrics as metrics
from matplotlib import pyplot as plt
%matplotlib notebook
In [ ]:
# sns properties
sns.set_style("white")
sns.set_context("paper", font_scale=2.5, rc={"lines.linewidth": 2.0})
In [ ]:
kinectExts = ['E','CO','C']
kinectDims = [640,2500,7500]
kinectKeys = ['ESF','Color','Cloud']
mocapDims = [21,36,8]
mocapExts = ['CP','M','T']
mocapKeys = ['CircleParam','Marker','TopCoord']
spaces = ['Kinect','Mocap']
dataTypes = ['train','test']
parameters = ['rmse','nrmse','corr']
In [ ]:
nFeat = 3
nTypes = 2
nParam = 3
nSpaces = 2
nTrials = 6
metricRes = np.zeros((nSpaces,nTypes,nFeat,nParam,nTrials))
spaceInd = 0
mocapExt = 'T'
for dT,dataType in enumerate(dataTypes):
for kE,kinectExt in enumerate(kinectExts):
for nT in range(nTrials):
res = pickle.load(open('../Results/Exp5/%s%sRes%d.p' % (kinectExt,mocapExt,nT+1),'rb'))
for p,parameter in zip(range(nParam),parameters):
metricRes[spaceInd,dT,kE,p,nT] = res[dataType][parameter].mean()
spaceInd = 1
kinectExt = 'C'
for dT,dataType in enumerate(dataTypes):
for mE,mocapExt in enumerate(mocapExts):
for nT in range(nTrials):
res = pickle.load(open('../Results/Exp5/%s%sRes%d.p' % (kinectExt,mocapExt,nT+1),'rb'))
for p,parameter in zip(range(nParam),parameters):
metricRes[spaceInd,dT,mE,p,nT] = res[dataType][parameter].mean()
pickle.dump(metricRes,open('Result/metricResults.p','wb'))
In [ ]:
W = {}
W[6] = {'max1':21, 'min1':0, 'p1':0.025, 'max2':19, 'min2':2, 'p2':0.05}
# function to compute wilcoxon metric for one-sided x > y, greater than test
def wilcoxon(x,y):
# convert to numpy arrays
x,y = map(np.asarray, (x,y))
# compute differences
d = x - y
# remove zero differences
d = np.compress(np.not_equal(d, 0), d, axis=-1)
# compute rank of differences
r = stats.rankdata(abs(d))
# compute sum of ranked differences
T = np.sum((d > 0)*r, axis=0)
# significant or not significant
n = d.shape[0]
# assign significance
if (T >= W[n]['max1'] or T <= W[n]['min1']):
result = W[n]['p1']
elif (T >= W[n]['max2'] or T <= W[n]['min2']):
result = W[n]['p2']
else:
result = None
# return statistic and result
return T, result
In [ ]:
# variables for analysis
kinectExts = ['E','CO','C']
kinectDims = [640,2500,7500]
kinectKeys = ['ESF','Color','Cloud']
mocapDims = [21,36,8]
mocapExts = ['CP','M','T']
mocapKeys = ['CircleParam','Marker','TopCoord']
spaces = ['Kinect','Mocap']
dataTypes = ['train','test']
parameters = ['rmse','nrmse','corr']
statRes = {'Kinect':{},'Mocap':{}}
pairs = {'Kinect':{'ECO':[0,1],'EC':[0,2],'COC':[1,2]},
'Mocap':{'CPM':[0,1],'CPT':[0,2],'MT':[1,2]}}
for sI,space in enumerate(spaces):
for dT,dataType in enumerate(dataTypes):
statRes[space][dataType] = {}
for key in pairs[space].keys():
indices = pairs[space][key]
statRes[space][dataType][key] = {}
for p,parameter in enumerate(parameters):
tVal, pval = wilcoxon(metricRes[sI,dT,indices[0],p,:], metricRes[sI,dT,indices[1],p,:])
statRes[space][dataType][key][parameter] = pval
print dataType,key,': ',statRes[space][dataType][key]
pickle.dump(statRes,open('Result/statResults.p','wb'))
In [ ]:
nTr = 6
spaces = ['Kinect','Mocap']
dataTypes = ['Train','Test']
parameters = ['rmse','nrmse','corr']
kinectKeys = ['ESF','Color','Cloud']
mocapKeys = ['Circle','Marker','TopCoord']
columns = ['Value','Space','Type','Feature','Param']
dataFrame = pd.DataFrame(columns=columns)
for s,space in enumerate(spaces):
for d,dataType in enumerate(dataTypes):
if space == 'Kinect':
features = kinectKeys
else:
features = mocapKeys
for f,feature in enumerate(features):
for p,parameter in enumerate(parameters):
dat = np.atleast_2d(metricRes[s,d,f,p,:])
dat = np.concatenate((dat,np.atleast_2d([space]*nTr)),axis=0)
dat = np.concatenate((dat,np.atleast_2d([dataType]*nTr)),axis=0)
dat = np.concatenate((dat,np.atleast_2d([feature]*nTr)),axis=0)
dat = np.concatenate((dat,np.atleast_2d([parameter]*nTr)),axis=0)
dF = pd.DataFrame(dat.transpose(),columns=columns)
dataFrame = dataFrame.append(dF)
dataFrame.index = range(dataFrame.shape[0])
dataFrame['Value'] = dataFrame['Value'].apply(pd.to_numeric)
In [ ]:
mocapFrame = dataFrame[dataFrame['Space'] == 'Mocap']
kinectFrame = dataFrame[dataFrame['Space'] == 'Kinect']
In [ ]:
props = {'connectionstyle':'bar,fraction=0.15', 'arrowstyle':'-', 'lw':2}
dF = kinectFrame[kinectFrame['Param'] == 'nrmse']
fig = plt.figure()
ax = sns.boxplot(x='Type', y='Value', hue='Feature', data=dF, whis=1.5, fliersize=5, notch=False)
ax.set_xlabel('')
ax.set_ylabel('NRMSE')
ax.set_ylim([0.0,0.2])
ax.legend_.remove()
ax.text(1.0, 0.185, '*', horizontalalignment='center', verticalalignment='center')
ax.text(0.875, 0.168, '*', horizontalalignment='center', verticalalignment='center')
ax.text(1.125, 0.12, 'n.s', horizontalalignment='center', verticalalignment='center')
ax.annotate("", xy=(0.75,0.17), xycoords='data', xytext=(1.25,0.17), textcoords='data', arrowprops=props)
ax.annotate("", xy=(0.75,0.16), xycoords='data', xytext=(1.0,0.16), textcoords='data', arrowprops=props)
ax.annotate("", xy=(1.0,0.105), xycoords='data', xytext=(1.25,0.105), textcoords='data', arrowprops=props)
plt.tight_layout()
plt.savefig('Result/kinectNRMSE.pdf', format='pdf')
In [ ]:
props = {'connectionstyle':'bar,fraction=0.15', 'arrowstyle':'-', 'lw':2}
dF = kinectFrame[kinectFrame['Param'] == 'corr']
fig = plt.figure()
ax = sns.boxplot(x='Type', y='Value', hue='Feature', data=dF, whis=1.5, fliersize=5, notch=False)
ax.set_xlabel('')
ax.set_ylabel('Pearson Correlation')
ax.set_ylim([0.85,1.03])
ax.legend_.set_title('')
ax.text(1.0, 1.005, '*', horizontalalignment='center', verticalalignment='center')
ax.text(0.875, 0.988, '*', horizontalalignment='center', verticalalignment='center')
ax.text(1.125, 0.992, 'n.s', horizontalalignment='center', verticalalignment='center')
ax.annotate("", xy=(0.75,0.99), xycoords='data', xytext=(1.25,0.99), textcoords='data', arrowprops=props)
ax.annotate("", xy=(0.75,0.98), xycoords='data', xytext=(1.0,0.98), textcoords='data', arrowprops=props)
ax.annotate("", xy=(1.0,0.98), xycoords='data', xytext=(1.25,0.98), textcoords='data', arrowprops=props)
plt.tight_layout()
plt.savefig('Result/kinectCorr.pdf', format='pdf')
In [ ]:
props = {'connectionstyle':'bar,fraction=0.15', 'arrowstyle':'-', 'lw':2}
dF = mocapFrame[mocapFrame['Param'] == 'nrmse']
fig = plt.figure()
ax = sns.boxplot(x='Type', y='Value', hue='Feature', data=dF, whis=1.5, fliersize=5, notch=False)
ax.set_xlabel('')
ax.set_ylabel('NRMSE')
ax.set_ylim([0.0,0.48])
ax.legend_.set_title('')
ax.text(1.0, 0.45, '*', horizontalalignment='center', verticalalignment='center')
ax.text(0.875, 0.42, '*', horizontalalignment='center', verticalalignment='center')
ax.text(1.125, 0.24, '*', horizontalalignment='center', verticalalignment='center')
ax.annotate("", xy=(0.75,0.42), xycoords='data', xytext=(1.25,0.42), textcoords='data', arrowprops=props)
ax.annotate("", xy=(0.75,0.40), xycoords='data', xytext=(1.0,0.40), textcoords='data', arrowprops=props)
ax.annotate("", xy=(1.0,0.22), xycoords='data', xytext=(1.25,0.22), textcoords='data', arrowprops=props)
plt.tight_layout()
plt.savefig('Result/mocapNRMSE.pdf', format='pdf')
In [ ]:
props = {'connectionstyle':'bar,fraction=0.15', 'arrowstyle':'-', 'lw':2}
dF = mocapFrame[mocapFrame['Param'] == 'corr']
fig = plt.figure()
ax = sns.boxplot(x='Type', y='Value', hue='Feature', data=dF, whis=1.5, fliersize=5, notch=False)
ax.set_xlabel('')
ax.set_ylabel('Pearson Correlation')
ax.set_ylim([0.45,1.08])
ax.legend_.remove()
ax.text(1.0, 1.04, '*', horizontalalignment='center', verticalalignment='center')
ax.text(0.875, 0.85, '*', horizontalalignment='center', verticalalignment='center')
ax.text(1.125, 1.0, '*', horizontalalignment='center', verticalalignment='center')
ax.annotate("", xy=(0.75,1.00), xycoords='data', xytext=(1.25,1.00), textcoords='data', arrowprops=props)
ax.annotate("", xy=(0.75,0.83), xycoords='data', xytext=(1.0,0.83), textcoords='data', arrowprops=props)
ax.annotate("", xy=(1.0,0.98), xycoords='data', xytext=(1.25,0.98), textcoords='data', arrowprops=props)
plt.tight_layout()
plt.savefig('Result/mocapCorr.pdf', format='pdf')
In [ ]: