Compare the predictive performance of using MRD with respect to other standard regression frameworks such as nearest neighbor regression, linear regression, neural networks and gaussian process regression.
In this Ipython notebook, all the metrics are loaded and statistical significance tests are applied in comparison to MRD. Finally bar plots are made indicating the difference visually.
In [ ]:
# import the modules
import GPy
import csv
import sys
import numpy as np
import pandas as pd
import seaborn as sns
import cPickle as pickle
from matplotlib import cm
import scipy.stats as stats
from scipy.stats import norm
import sklearn.metrics as metrics
from numpy import sqrt, abs, round
from matplotlib import pyplot as plt
from sklearn.neighbors import NearestNeighbors
%matplotlib notebook
In [ ]:
# sns properties
sns.set_style("white")
sns.set_context("paper", font_scale=2.0, rc={"lines.linewidth": 2.0})
In [ ]:
kinectExt = 'C'
kinectDim = 7500
kinectKey = 'Cloud'
mocapDim = 8
mocapExt = 'T'
mocapKey = 'TopCoord'
dataTypes = ['train','test']
parameters = ['rmse','nrmse','corr']
models = ['mlp','lr','nn','gp','mrd']
In [ ]:
nPos = 6
nShr = 4
nMod = 5
nTypes = 2
nParam = 3
dims = [kinectDim,mocapDim]
keys = [kinectKey,mocapKey]
names = []
for nS in range(nShr):
for nP in range(nPos):
names.append('K1S%dP%dT1' % (nS+1,nP+1))
compRes = np.zeros((nTypes,nMod,nParam,nShr,nPos))
for nS in range(nShr):
for nP in range(nPos):
regRes = pickle.load(open('../Results/Exp2/RegRes%d%d.p' % (nS+1,nP+1),'rb'))
mrdRes = pickle.load(open('../Results/Exp2/MRDRes%d%d.p' % (nS+1,nP+1),'rb'))
for d,dataType in zip(range(nTypes),dataTypes):
for m,model in zip(range(nMod-1),models):
for p,parameter in zip(range(nParam),parameters):
compRes[d,m,p,nS,nP] = regRes[dataType][model][parameter].mean()
for d,dataType in zip(range(nTypes),dataTypes):
for p,parameter in zip(range(nParam),parameters):
compRes[d,-1,p,nS,nP] = mrdRes[dataType][parameter].mean()
pickle.dump(compRes,open('Result/metricResults.p','wb'))
In [ ]:
# variables for analysis
statRes = {}
for m,model in zip(range(len(models)-1),models):
statRes[model] = {}
for p,parameter in zip(range(len(parameters)),parameters):
_, pval = stats.wilcoxon(compRes[1,m,p,:,:].flatten(), compRes[1,-1,p,:,:].flatten())
statRes[model][parameter] = pval/2.0
print model,': ',statRes[model]
pickle.dump(statRes,open('Result/statResults.p','wb'))
In [ ]:
nTrials = nShr*nPos
dataTypes = ['Train','Test']
columns = ['Value','Type','Model','Param']
parameters = ['rmse','nrmse','corr']
models = ['MLP','Lin Reg','K-NN','GP Reg','MRD']
dataFrame = pd.DataFrame(columns=columns)
for d,dataType in zip(range(nTypes),dataTypes):
for m,model in zip(range(nMod),models):
for p,parameter in zip(range(nParam),parameters):
dat = np.atleast_2d(compRes[d,m,p,:,:].flatten())
dat = np.concatenate((dat,np.atleast_2d([dataType]*nTrials)),axis=0)
dat = np.concatenate((dat,np.atleast_2d([model]*nTrials)),axis=0)
dat = np.concatenate((dat,np.atleast_2d([parameter]*nTrials)),axis=0)
dF = pd.DataFrame(dat.transpose(),columns=columns)
dataFrame = dataFrame.append(dF)
dataFrame.index = range(nTypes*nMod*nParam*nTrials)
dataFrame['Value'] = dataFrame['Value'].apply(pd.to_numeric)
In [ ]:
props = {'connectionstyle':'bar,fraction=0.15', 'arrowstyle':'-', 'lw':2}
dF = dataFrame[dataFrame['Param'] == 'rmse']
fig = plt.figure()
ax = sns.boxplot(x='Type', y='Value', hue='Model', data=dF, notch=True)
ax.set_xlabel('')
ax.set_ylabel('RMSE')
ax.legend_.set_title('')
ax.set_ylim([0.0,1.2])
ax.text(1.24375, 1.03, '*', horizontalalignment='center', verticalalignment='center')
ax.annotate("", xy=(1.1625,1), xycoords='data', xytext=(1.325,1), textcoords='data', arrowprops=props)
ax.text(1.1625, 1.1, '***', horizontalalignment='center', verticalalignment='center')
ax.annotate("", xy=(1.0,1.05), xycoords='data', xytext=(1.325,1.05), textcoords='data', arrowprops=props)
plt.tight_layout()
plt.savefig('Result/modelRMSE.pdf', format='pdf')
In [ ]:
dF = dataFrame[dataFrame['Param'] == 'nrmse']
fig = plt.figure()
ax = sns.boxplot(x='Type', y='Value', hue='Model', data=dF, notch=True, fliersize=0)
ax.set_xlabel('')
ax.set_ylabel('NRMSE')
ax.set_ylim([0.0,0.35])
ax.legend_.set_title('')
ax.text(1.24375, 0.20, '***', horizontalalignment='center', verticalalignment='center')
ax.annotate("", xy=(1.1625,0.19), xycoords='data', xytext=(1.325,0.19), textcoords='data', arrowprops=props)
ax.text(1.1625, 0.225, '***', horizontalalignment='center', verticalalignment='center')
ax.annotate("", xy=(1.0,0.21), xycoords='data', xytext=(1.325,0.21), textcoords='data', arrowprops=props)
plt.tight_layout()
plt.savefig('Result/modelNRMSE.pdf', format='pdf')
In [ ]:
dF = dataFrame[dataFrame['Param'] == 'corr']
fig = plt.figure()
ax = sns.boxplot(x='Type', y='Value', hue='Model', data=dF, notch=True, fliersize=0)
ax.set_xlabel('')
ax.legend_.set_title('')
ax.set_ylim([0.7,1.04])
ax.set_ylabel('Pearson Correlation')
ax.text(1.24375, 0.99, '***', horizontalalignment='center', verticalalignment='center')
ax.annotate("", xy=(1.1625,0.98), xycoords='data', xytext=(1.325,0.98), textcoords='data', arrowprops=props)
ax.text(1.1625, 1.02, '***', horizontalalignment='center', verticalalignment='center')
ax.annotate("", xy=(1.0,1.0), xycoords='data', xytext=(1.325,1.0), textcoords='data', arrowprops=props)
plt.tight_layout()
plt.savefig('Result/modelCorr.pdf', format='pdf')