Compare the predictive performance of using different inference techniques in MRD. The performance is evaluated over unseen T-shirts and postures that are not part of the training dataset.
The metrics for evaluation are RMS error, normalized RMS error and pearson correlation. Statistical significance is evaluated using wilcoxon signed rank sum test.
In this notebook, the various metrics are collected from all the pickle files along with evaluating statistical significance of differences between the inference techniques. Finally plots are made to be included in the paper.
In [ ]:
# import the modules
import GPy
import csv
import numpy as np
import pandas as pd
import seaborn as sns
import cPickle as pickle
from matplotlib import cm
import scipy.stats as stats
from scipy.stats import norm
import sklearn.metrics as metrics
from numpy import sqrt, abs, round
from matplotlib import pyplot as plt
from sklearn.neighbors import NearestNeighbors
%matplotlib notebook
In [ ]:
# sns properties
sns.set_style("white")
sns.set_context("paper", font_scale=2.0, rc={"lines.linewidth": 2.0})
In [ ]:
nPos = 6
nShr = 4
nFunc = 3
nParam = 4
nTypes = 2
dataTypes = ['Pos','Shr']
parameters = ['rmse','nrmse','corr','time']
functions = ['nnsearch','filter','optimize']
metricRes = {}
nTrials = {'Pos':1,'Shr':3}
for dT,dataType in zip(range(nTypes),dataTypes):
metricRes[dataType] = np.zeros((nFunc,nParam,nShr,nPos,nTrials[dataType]))
for nS in range(nShr):
for nP in range(nPos):
if dataType == 'Shr':
inds = [((nS+1)%4)*nPos+nP,((nS+1)%4)*nPos+(nP+1)%nPos,((nS+1)%4)*nPos+(nP+2)%nPos]
for nT in range(nTrials[dataType]):
if dataType == 'Pos':
res = pickle.load(open('../Results/Exp3/Res%s%d%d.p' % (dataType,nS+1,nP+1),'rb'))
else:
res = pickle.load(open('../Results/Exp3/Res%s%d%d%02d.p' % (dataType,nS+1,nP+1,inds[nT]+1),'rb'))
for f,function in zip(range(nFunc),functions):
for p,parameter in zip(range(nParam),parameters):
if parameter != 'time':
metricRes[dataType][f,p,nS,nP,nT] = res[function][parameter].mean()
else:
metricRes[dataType][f,p,nS,nP,nT] = res[function][parameter]
pickle.dump(metricRes,open('Result/metricResults.p','wb'))
In [ ]:
# variables for analysis
statRes1 = {}
pairs = {'fn':[0,1],'no':[0,2],'of':[1,2]}
statRes1 = {}
for key in pairs.keys():
indices = pairs[key]
statRes1[key] = {}
for p,parameter in zip(range(nParam),parameters):
data1 = np.empty(0)
data2 = np.empty(0)
for dT,dataType in zip(range(nTypes),dataTypes):
data1 = np.concatenate((data1,metricRes[dataType][indices[0],p,:,:,:].flatten()))
data2 = np.concatenate((data2,metricRes[dataType][indices[1],p,:,:,:].flatten()))
_, pval = stats.wilcoxon(data1, data2)
statRes1[key][parameter] = pval/2.0
print key,': ',statRes1[key]
pickle.dump(statRes1,open('Result/statResults1.p','wb'))
In [ ]:
# variables for analysis
statRes2 = {}
pairs = {'fn':[0,1],'no':[0,2],'of':[1,2]}
for dT,dataType in zip(range(nTypes),dataTypes):
statRes2[dataType] = {}
for key in pairs.keys():
indices = pairs[key]
statRes2[dataType][key] = {}
for p,parameter in zip(range(nParam),parameters):
_, pval = stats.wilcoxon(metricRes[dataType][indices[0],p,:,:,:].flatten(), metricRes[dataType][indices[1],p,:,:,:].flatten())
# divide by 2 to perform one-sided test
statRes2[dataType][key][parameter] = pval/2.0
print dataType,key,': ',statRes2[dataType][key]
pickle.dump(statRes2,open('Result/statResults2.p','wb'))
In [ ]:
dataTypes = ['Pos','Shr']
parameters = ['rmse','nrmse','corr','time']
functions = ['NNSearch','Filter','Optimize']
columns = ['Value','Type','Function','Param']
dataFrame = pd.DataFrame(columns=columns)
for d,dataType in zip(range(nTypes),dataTypes):
nTr = nShr*nPos*nTrials[dataType]
for f,function in zip(range(nFunc),functions):
for p,parameter in zip(range(nParam),parameters):
dat = np.atleast_2d(metricRes[dataType][f,p,:,:,:].flatten())
dat = np.concatenate((dat,np.atleast_2d([dataType]*nTr)),axis=0)
dat = np.concatenate((dat,np.atleast_2d([function]*nTr)),axis=0)
dat = np.concatenate((dat,np.atleast_2d([parameter]*nTr)),axis=0)
dF = pd.DataFrame(dat.transpose(),columns=columns)
dataFrame = dataFrame.append(dF)
dataFrame.index = range(dataFrame.shape[0])
dataFrame['Value'] = dataFrame['Value'].apply(pd.to_numeric)
In [ ]:
props = {'connectionstyle':'bar,fraction=0.15', 'arrowstyle':'-', 'lw':2}
dF = dataFrame[dataFrame['Param'] == 'rmse']
fig = plt.figure()
ax = sns.boxplot(x='Type', y='Value', hue='Function', data=dF, whis=1.0, fliersize=0, notch=True)
ax.set_xlabel('')
ax.set_ylabel('RMSE')
ax.set_ylim([0.2,2.3])
ax.legend_.set_title('')
# Posture Annotation
ax.text(0.0, 1.28, '***', horizontalalignment='center', verticalalignment='center')
ax.text(-0.125, 1.11, '***', horizontalalignment='center', verticalalignment='center')
ax.text(0.125, 1.05, '**', horizontalalignment='center', verticalalignment='center')
ax.annotate("", xy=(-0.25,1.15), xycoords='data', xytext=(0.25,1.15), textcoords='data', arrowprops=props)
ax.annotate("", xy=(-0.25,1.03), xycoords='data', xytext=(0.0,1.03), textcoords='data', arrowprops=props)
ax.annotate("", xy=(0.0,0.98), xycoords='data', xytext=(0.25,0.98), textcoords='data', arrowprops=props)
# Shirt Annotation
ax.text(1.0, 1.38, '***', horizontalalignment='center', verticalalignment='center')
ax.text(0.875, 1.24, '***', horizontalalignment='center', verticalalignment='center')
ax.text(1.125, 1.22, '***', horizontalalignment='center', verticalalignment='center')
ax.annotate("", xy=(0.75,1.25), xycoords='data', xytext=(1.25,1.25), textcoords='data', arrowprops=props)
ax.annotate("", xy=(0.75,1.16), xycoords='data', xytext=(1.0,1.16), textcoords='data', arrowprops=props)
ax.annotate("", xy=(1.0,1.14), xycoords='data', xytext=(1.25,1.14), textcoords='data', arrowprops=props)
plt.tight_layout()
plt.savefig('Result/inferRMSE.pdf', format='pdf')
In [ ]:
dF = dataFrame[dataFrame['Param'] == 'nrmse']
fig = plt.figure()
ax = sns.boxplot(x='Type', y='Value', hue='Function', data=dF, whis=1.0, fliersize=0, notch=True)
ax.set_xlabel('')
ax.legend_.remove()
ax.set_ylabel('NRMSE')
ax.set_ylim([0.05,0.27])
# Posture Annotation
ax.text(0.0, 0.22, '***', horizontalalignment='center', verticalalignment='center')
ax.text(-0.125, 0.205, '*', horizontalalignment='center', verticalalignment='center')
ax.text(0.125, 0.195, '**', horizontalalignment='center', verticalalignment='center')
ax.annotate("", xy=(-0.25,0.205), xycoords='data', xytext=(0.25,0.205), textcoords='data', arrowprops=props)
ax.annotate("", xy=(-0.25,0.19), xycoords='data', xytext=(0.0,0.19), textcoords='data', arrowprops=props)
ax.annotate("", xy=(0.0,0.185), xycoords='data', xytext=(0.25,0.185), textcoords='data', arrowprops=props)
# Shirt Annotation
ax.text(1.0, 0.255, '**', horizontalalignment='center', verticalalignment='center')
ax.text(0.875, 0.235, 'n.s', horizontalalignment='center', verticalalignment='center')
ax.text(1.125, 0.23, '*', horizontalalignment='center', verticalalignment='center')
ax.annotate("", xy=(0.75,0.24), xycoords='data', xytext=(1.25,0.24), textcoords='data', arrowprops=props)
ax.annotate("", xy=(0.75,0.22), xycoords='data', xytext=(1.0,0.22), textcoords='data', arrowprops=props)
ax.annotate("", xy=(1.0,0.22), xycoords='data', xytext=(1.25,0.22), textcoords='data', arrowprops=props)
plt.tight_layout()
plt.savefig('Result/inferNRMSE.pdf', format='pdf')
In [ ]:
dF = dataFrame[dataFrame['Param'] == 'corr']
fig = plt.figure()
ax = sns.boxplot(x='Type', y='Value', hue='Function', data=dF, whis=1.0, fliersize=0, notch=True)
ax.set_xlabel('')
ax.set_ylim([0.7,1.05])
ax.legend_.set_title('')
ax.set_ylabel('Pearson Correlation')
# Posture Annotation
ax.text(0.0, 1.027, '***', horizontalalignment='center', verticalalignment='center')
ax.text(-0.125, 0.99, '***', horizontalalignment='center', verticalalignment='center')
ax.text(0.125, 0.99, '**', horizontalalignment='center', verticalalignment='center')
ax.annotate("", xy=(-0.25,1.0), xycoords='data', xytext=(0.25,1.0), textcoords='data', arrowprops=props)
ax.annotate("", xy=(-0.25,0.975), xycoords='data', xytext=(0.0,0.975), textcoords='data', arrowprops=props)
ax.annotate("", xy=(0.0,0.98), xycoords='data', xytext=(0.25,0.98), textcoords='data', arrowprops=props)
# Shirt Annotation
ax.text(1.0, 1.03, '**', horizontalalignment='center', verticalalignment='center')
ax.text(0.875, 1.006, 'n.s', horizontalalignment='center', verticalalignment='center')
ax.text(1.125, 1.006, 'n.s', horizontalalignment='center', verticalalignment='center')
ax.annotate("", xy=(0.75,1.0), xycoords='data', xytext=(1.25,1.0), textcoords='data', arrowprops=props)
ax.annotate("", xy=(0.75,0.98), xycoords='data', xytext=(1.0,0.98), textcoords='data', arrowprops=props)
ax.annotate("", xy=(1.0,0.98), xycoords='data', xytext=(1.25,0.98), textcoords='data', arrowprops=props)
plt.tight_layout()
plt.savefig('Result/inferCorr.pdf', format='pdf')
In [ ]:
dF = dataFrame[dataFrame['Param'] == 'time']
fig = plt.figure()
ax = sns.boxplot(x='Type', y='Value', hue='Function', data=dF, whis=1.0, fliersize=6, notch=True)
ax.set_xlabel('')
ax.legend_.remove()
ax.set_ylabel('Frames per second')
plt.tight_layout()
plt.savefig('Result/inferTime.pdf', format='pdf')
In [ ]: