Evaluation



In [3]:
import site
site.addsitedir('..')
import menpo.io as mio
import menpodetect
import numpy as np
import facefit
import hickle
%matplotlib inline
from facefit import esr, ert, lbf
from facefit.test import fit_all
from facefit.test.end2end import test_model
from facefit import util

In [6]:
esr_model = hickle.load("../models/esr_lfpw_helen.hkl", safe=False)
ert_model = hickle.load("../models/ert_lfpw_helen.hkl", safe=False)
lbf_fast_model = hickle.load("../models/lbf-fast_lfpw+helen.hkl", safe=False)

In [5]:
lfpw_test = util.read_images("../datasets/lfpw/testset/", normalise=True)
helen_test = util.read_images("../datasets/helen_cropped/testset_cropped/", normalise=True)
test_images = np.concatenate([lfpw_test, helen_test])


- Loading 330 assets: [====================] 100%

In [48]:
lfpw_gt_shapes = util.get_gt_shapes(lfpw_test)
helen_gt_shapes = util.get_gt_shapes(helen_test)

In [53]:
esr_initerr, esr_finerr, _, esr_shapes = test_model(esr_model, lfpw_test, 5)


224/224

In [54]:
print "Mean initial error: {}".format(np.mean(esr_initerr))
print "Mean final error: {}".format(np.mean(esr_finerr))


Mean initial error: 0.102570194775
Mean final error: 0.0230739874073

In [55]:
from menpofit.visualize import plot_ced
plot_ced(esr_finerr)



In [18]:
ert_initerr, ert_finerr, _, ert_shapes = test_model(ert_model, lfpw_test, 5)


224/224

In [51]:
(0.0234185059366-0.0227215150235) / 0.0234185059366 * 100.0


Out[51]:
2.9762398804899677

In [19]:
print "Mean initial error: {}".format(np.mean(ert_initerr))
print "Mean final error: {}".format(np.mean(ert_finerr))


Mean initial error: 0.102570194775
Mean final error: 0.0227215150235

In [20]:
from menpofit.visualize import plot_ced
plot_ced(ert_finerr)



In [27]:
lbf_fast_initerr, lbf_fast_finerr, _, lbf_fast_shapes = test_model(lbf_fast_model, lfpw_test, 1)


224/224

In [28]:
print "Mean initial error: {}".format(np.mean(lbf_fast_initerr))
print "Mean final error: {}".format(np.mean(lbf_fast_finerr))


Mean initial error: 0.102570194775
Mean final error: 0.0234881140285

In [52]:
( 0.0234881140285 - 0.0227215150235) /  0.0234881140285*100


Out[52]:
3.2637741968972946

In [30]:
from menpofit.visualize import plot_ced
plot_ced(lbf_fast_finerr)



In [105]:
testset = helen_test
esr_initerr, esr_finerr, _, esr_shapes = test_model(esr_model, testset, 5)
ert_initerr, ert_finerr, _, ert_shapes = test_model(ert_model, testset, 5)
lbf_fast_initerr, lbf_fast_finerr, _, lbf_fast_shapes = test_model(lbf_fast_model, testset, 1)
print "ESR Mean final error: {}".format(np.mean(esr_finerr))
print "ERT Mean final error: {}".format(np.mean(ert_finerr))
print "LBF-fast Mean final error: {}".format(np.mean(lbf_fast_finerr))


330/330ESR Mean final error: 0.0264447843989
ERT Mean final error: 0.0259486128112
LBF-fast Mean final error: 0.0268462358782

In [120]:
plot_ced([esr_initerr, esr_finerr, ert_finerr, lbf_fast_finerr])



In [115]:
testset = lfpw_test
esr_initerr_lfpw, esr_finerr_lfpw, _, _ = test_model(esr_model, testset, 5)
ert_initerr_lfpw, ert_finerr_lfpw, _, _ = test_model(ert_model, testset, 5)
lbf_fast_initerr_lfpw, lbf_fast_finerr_lfpw, _, _ = test_model(lbf_fast_model, testset, 1)
print "ESR Mean final error: {}".format(np.mean(esr_finerr_lfpw))
print "ERT Mean final error: {}".format(np.mean(ert_finerr_lfpw))
print "LBF-fast Mean final error: {}".format(np.mean(lbf_fast_finerr_lfpw))


224/224ESR Mean final error: 0.0234134630791
ERT Mean final error: 0.0221381220762
LBF-fast Mean final error: 0.0234881140285

In [118]:
plot_ced([esr_initerr_lfpw, esr_finerr_lfpw, ert_finerr_lfpw, lbf_fast_finerr_lfpw])



In [130]:
np.array(esr_finerr)


Out[130]:
array([ 0.01779597,  0.04371473,  0.02716165,  0.0123992 ,  0.05062899,
        0.01987197,  0.01577982,  0.0204431 ,  0.02940782,  0.02081472,
        0.03246655,  0.02407153,  0.02568017,  0.01672935,  0.03725349,
        0.05696922,  0.02451197,  0.02564323,  0.02196374,  0.0157632 ,
        0.03988404,  0.02825826,  0.02274409,  0.02319358,  0.03434073,
        0.0197609 ,  0.02889168,  0.01818251,  0.0210034 ,  0.02019764,
        0.01191617,  0.01927971,  0.01464399,  0.02614835,  0.01467068,
        0.02135292,  0.02688524,  0.0202704 ,  0.02095779,  0.03210673,
        0.01606498,  0.0183054 ,  0.0278893 ,  0.0199867 ,  0.02775859,
        0.01073692,  0.02288831,  0.03101142,  0.02668166,  0.04191728,
        0.03348635,  0.01760935,  0.03658279,  0.0246106 ,  0.02679804,
        0.03105976,  0.01520805,  0.02341579,  0.0279519 ,  0.03530289,
        0.03923976,  0.01558873,  0.02428346,  0.02623934,  0.02422327,
        0.03327221,  0.01683121,  0.06493784,  0.02586821,  0.02026042,
        0.01730441,  0.01758331,  0.02153462,  0.01819317,  0.03793594,
        0.02082583,  0.01692215,  0.02028612,  0.02613928,  0.0178392 ,
        0.01952633,  0.02007572,  0.01320654,  0.01380998,  0.01468643,
        0.01358022,  0.02370339,  0.02487397,  0.02315626,  0.02551915,
        0.03002023,  0.04203556,  0.02767738,  0.01699166,  0.01631215,
        0.28199289,  0.02064471,  0.02989379,  0.0294292 ,  0.01784465,
        0.01706938,  0.02460906,  0.01710761,  0.01929152,  0.03062004,
        0.01905608,  0.01387988,  0.01970924,  0.02933223,  0.02237162,
        0.01621725,  0.02361776,  0.02004133,  0.02812967,  0.04094776,
        0.01160845,  0.01666673,  0.03067358,  0.02737887,  0.01691678,
        0.02020372,  0.02431306,  0.02580731,  0.02623255,  0.01765207,
        0.0116688 ,  0.02050793,  0.03386249,  0.02074273,  0.01371446,
        0.01535709,  0.03334659,  0.02002843,  0.02009818,  0.02695542,
        0.01572202,  0.01808202,  0.02181518,  0.02300567,  0.01826021,
        0.02056167,  0.01613766,  0.01697498,  0.02084925,  0.02107352,
        0.02759721,  0.03133763,  0.02834323,  0.03049238,  0.02437204,
        0.01905797,  0.01930127,  0.03287512,  0.01352232,  0.01977441,
        0.01815466,  0.01954054,  0.02617084,  0.02040459,  0.01597293,
        0.03451888,  0.15255941,  0.02094514,  0.01838191,  0.02003425,
        0.02232   ,  0.02171701,  0.01714005,  0.02112149,  0.02095917,
        0.03043313,  0.01847182,  0.02285009,  0.03994536,  0.01876635,
        0.09564612,  0.02280618,  0.01641378,  0.01160279,  0.15262251,
        0.01863853,  0.02212714,  0.01806883,  0.02082739,  0.02081966,
        0.01751303,  0.02353092,  0.02336321,  0.02223707,  0.02234771,
        0.02953031,  0.02231426,  0.0214087 ,  0.02480247,  0.03516586,
        0.02223397,  0.02211751,  0.01334486,  0.03223038,  0.02636273,
        0.02019934,  0.0138644 ,  0.01366991,  0.02327827,  0.02039058,
        0.02717247,  0.01776082,  0.05265343,  0.01780479,  0.01852386,
        0.01345043,  0.02622752,  0.04048921,  0.04436517,  0.0137158 ,
        0.02794939,  0.0209358 ,  0.02064592,  0.02973955,  0.02371268,
        0.02003545,  0.02291168,  0.03140389,  0.04687366,  0.01806297,
        0.02586105,  0.01315775,  0.0342016 ,  0.03191376,  0.02570796,
        0.01771187,  0.01981864,  0.09624695,  0.01443918,  0.02887785,
        0.03373199,  0.03362478,  0.03275142,  0.01740082,  0.01701988,
        0.02100471,  0.01715478,  0.02221948,  0.06316226,  0.0299156 ,
        0.01938156,  0.02754348,  0.02248566,  0.01637349,  0.01798405,
        0.01771508,  0.01675967,  0.02713172,  0.02140172,  0.01454379,
        0.01331761,  0.10518925,  0.03387408,  0.01592474,  0.03146106,
        0.02323456,  0.02547589,  0.02226322,  0.02418138,  0.02693392,
        0.0366134 ,  0.01903776,  0.01844359,  0.01776205,  0.02032051,
        0.01386124,  0.01786212,  0.06680016,  0.0163828 ,  0.02869619,
        0.01716759,  0.01295339,  0.02586589,  0.04786585,  0.02862175,
        0.02742429,  0.05098639,  0.02003176,  0.017882  ,  0.01648505,
        0.02309408,  0.0825034 ,  0.02442653,  0.01925808,  0.01267449,
        0.02195319,  0.01131514,  0.01605277,  0.02345729,  0.01525537,
        0.0186213 ,  0.01473478,  0.0213001 ,  0.02085909,  0.02236852,
        0.02117853,  0.02282801,  0.06145364,  0.02830262,  0.01931262,
        0.01696599,  0.01383581,  0.01379577,  0.01624432,  0.02042032,
        0.0267887 ,  0.05671969,  0.03975798,  0.02853648,  0.0203765 ,
        0.03393319,  0.02260683,  0.07770326,  0.02069515,  0.04348765,
        0.03129665,  0.04150416,  0.01671373,  0.02919645,  0.02608689,
        0.01488383,  0.01750979,  0.01344271,  0.0139397 ,  0.02069977])

In [129]:
np.array(ert_finerr)


Out[129]:
array([ 0.01664902,  0.03699565,  0.02609249,  0.01632842,  0.10791756,
        0.01320857,  0.01742769,  0.02144126,  0.02993564,  0.02176183,
        0.0271991 ,  0.02465961,  0.02687206,  0.01652712,  0.06744524,
        0.06113526,  0.02393369,  0.02725447,  0.01945353,  0.01362942,
        0.04319505,  0.02404395,  0.02162623,  0.03564239,  0.0317206 ,
        0.01879724,  0.02573303,  0.01869665,  0.01717236,  0.02033234,
        0.01739837,  0.01723125,  0.01460521,  0.02320301,  0.01533743,
        0.02118221,  0.0289204 ,  0.02090835,  0.0172693 ,  0.02129251,
        0.01722295,  0.01485927,  0.02432529,  0.01481526,  0.02527312,
        0.01191869,  0.02030142,  0.02884387,  0.02633476,  0.05686986,
        0.02579529,  0.02924407,  0.04247004,  0.03022364,  0.02852123,
        0.02923754,  0.0193126 ,  0.02291639,  0.02721781,  0.03428562,
        0.0306342 ,  0.01327716,  0.02600783,  0.02691875,  0.02213268,
        0.03347219,  0.01285892,  0.04410256,  0.0237566 ,  0.02537479,
        0.01846851,  0.01803455,  0.0218802 ,  0.01464057,  0.04036096,
        0.02061528,  0.01781212,  0.02429513,  0.0265856 ,  0.01635715,
        0.01837248,  0.02076134,  0.01314534,  0.01142584,  0.01508186,
        0.01172216,  0.0214989 ,  0.02637684,  0.02504105,  0.02181461,
        0.02705454,  0.05071095,  0.03002713,  0.0169872 ,  0.01676151,
        0.27439276,  0.01933487,  0.02420404,  0.02315446,  0.02074754,
        0.01521459,  0.02708872,  0.02005043,  0.01580667,  0.02094351,
        0.01447653,  0.01370524,  0.01945086,  0.02948793,  0.02646861,
        0.01834622,  0.0274571 ,  0.02203112,  0.02608926,  0.0866871 ,
        0.01224851,  0.01635591,  0.03407119,  0.02793216,  0.01627281,
        0.02132871,  0.02192041,  0.02809111,  0.02652931,  0.01841502,
        0.01001514,  0.02163901,  0.02550065,  0.02119386,  0.01590444,
        0.01396128,  0.03771669,  0.01408093,  0.0189125 ,  0.02803378,
        0.01441045,  0.01693288,  0.02086486,  0.0228582 ,  0.01534504,
        0.02223301,  0.01672156,  0.02000599,  0.01890823,  0.02228182,
        0.03213399,  0.02378303,  0.02314197,  0.0281445 ,  0.02297106,
        0.01621155,  0.02308533,  0.03294271,  0.01191255,  0.0191362 ,
        0.01406592,  0.02251899,  0.02384627,  0.02026789,  0.01518576,
        0.02938265,  0.03133982,  0.02156752,  0.01659252,  0.01926649,
        0.02537508,  0.01950531,  0.01566916,  0.02016375,  0.01867939,
        0.02390292,  0.01853905,  0.02967297,  0.04350567,  0.0202008 ,
        0.05228778,  0.01901243,  0.01781411,  0.01136287,  0.21334987,
        0.0171586 ,  0.02270961,  0.01979499,  0.01874549,  0.01783763,
        0.01798983,  0.02874117,  0.01713013,  0.02276692,  0.01769889,
        0.0288581 ,  0.02186118,  0.02380924,  0.02470948,  0.03021504,
        0.02358697,  0.02674784,  0.0152101 ,  0.03131107,  0.02768134,
        0.01786475,  0.01212956,  0.0136263 ,  0.02126134,  0.0195216 ,
        0.02862257,  0.01478135,  0.05158801,  0.01765938,  0.01672029,
        0.01096581,  0.02886358,  0.03301898,  0.02956976,  0.0154585 ,
        0.02818506,  0.02058723,  0.02068433,  0.02545693,  0.02283956,
        0.01812996,  0.01895562,  0.02443991,  0.05833592,  0.0196093 ,
        0.01901982,  0.01400105,  0.03065211,  0.0324091 ,  0.02504746,
        0.01682486,  0.02162878,  0.11628746,  0.0156913 ,  0.02614886,
        0.03322146,  0.02881837,  0.02880512,  0.02303297,  0.02026688,
        0.02881401,  0.01791633,  0.02127411,  0.04534531,  0.03182379,
        0.01817878,  0.02821923,  0.01807561,  0.01779135,  0.02753262,
        0.01645011,  0.0168746 ,  0.02444242,  0.02081376,  0.01357117,
        0.01492995,  0.19573215,  0.03420947,  0.01618649,  0.03329226,
        0.02398751,  0.019972  ,  0.0175665 ,  0.02903025,  0.02653689,
        0.03548788,  0.01795593,  0.01862927,  0.01441289,  0.01690356,
        0.01498162,  0.0170846 ,  0.0307343 ,  0.0163421 ,  0.02517483,
        0.01859825,  0.01443571,  0.02395352,  0.03391761,  0.02561188,
        0.02525562,  0.0222319 ,  0.01952136,  0.02260699,  0.02021351,
        0.02538363,  0.05966619,  0.02503309,  0.0143654 ,  0.0127445 ,
        0.02554175,  0.01338257,  0.01728129,  0.02200972,  0.01327612,
        0.01691045,  0.01519114,  0.01779847,  0.01981739,  0.02157211,
        0.02004002,  0.01775709,  0.05429099,  0.02769328,  0.01701776,
        0.01564188,  0.01116236,  0.01489537,  0.0181755 ,  0.02277631,
        0.02659953,  0.04288603,  0.05807566,  0.05502194,  0.01903676,
        0.01967358,  0.01796909,  0.05322427,  0.01864404,  0.04510372,
        0.02561184,  0.04187484,  0.01505149,  0.02586015,  0.02504214,
        0.01422796,  0.01335965,  0.01606174,  0.01440961,  0.02107635])

In [128]:
np.array(lbf_fast_finerr)


Out[128]:
array([ 0.01666727,  0.03942364,  0.02403543,  0.01474697,  0.12350088,
        0.01338946,  0.0165314 ,  0.01873095,  0.03388807,  0.02309576,
        0.02589334,  0.02468165,  0.02296541,  0.0150485 ,  0.06214587,
        0.10439298,  0.03408694,  0.0261279 ,  0.02029534,  0.01377824,
        0.03705636,  0.02471387,  0.0203823 ,  0.08161052,  0.03153079,
        0.01882238,  0.02954927,  0.02517823,  0.01950761,  0.01582148,
        0.01576291,  0.01737909,  0.01762653,  0.02526043,  0.01421521,
        0.02308985,  0.02732142,  0.02151748,  0.01479432,  0.03310975,
        0.01509317,  0.01675762,  0.03310932,  0.01750911,  0.02681348,
        0.01157069,  0.02161593,  0.02786069,  0.03059276,  0.05709749,
        0.02088069,  0.02567281,  0.0494823 ,  0.02967351,  0.02630272,
        0.02116305,  0.01699745,  0.02370633,  0.02209466,  0.0375928 ,
        0.03517598,  0.01486341,  0.02480253,  0.02321619,  0.02354561,
        0.03468997,  0.01653067,  0.06107015,  0.02251674,  0.02012598,
        0.02180284,  0.01854256,  0.02168864,  0.01697622,  0.05802979,
        0.02163544,  0.01736965,  0.0184879 ,  0.02505102,  0.01555323,
        0.03289572,  0.02637117,  0.01352133,  0.0120523 ,  0.01696606,
        0.01434921,  0.01698285,  0.02283318,  0.02175852,  0.02120348,
        0.02375314,  0.049863  ,  0.02974467,  0.01797173,  0.01737051,
        0.13090419,  0.01870159,  0.02178458,  0.02338954,  0.02010839,
        0.01533142,  0.02737239,  0.02168314,  0.01647328,  0.02171356,
        0.01749537,  0.01378447,  0.01709213,  0.0502734 ,  0.03728057,
        0.01612445,  0.02248998,  0.01648203,  0.02553755,  0.0821255 ,
        0.01344325,  0.01552048,  0.03505169,  0.02579573,  0.01772208,
        0.02225302,  0.02293226,  0.02282311,  0.03331885,  0.01823342,
        0.01195434,  0.02398103,  0.02629336,  0.03009118,  0.01157729,
        0.01433173,  0.06305784,  0.0159156 ,  0.01647972,  0.03047336,
        0.0142267 ,  0.02104906,  0.02394718,  0.0342628 ,  0.01652529,
        0.01900468,  0.01636124,  0.01849612,  0.02178242,  0.0193146 ,
        0.04363195,  0.03552292,  0.02303189,  0.03411633,  0.02394293,
        0.01433297,  0.01705858,  0.03915137,  0.01310739,  0.01982974,
        0.0133054 ,  0.01878838,  0.02324643,  0.02138338,  0.01634401,
        0.02698999,  0.06302962,  0.02192145,  0.01659699,  0.01802958,
        0.03013404,  0.042065  ,  0.01545824,  0.02318616,  0.01733622,
        0.01891036,  0.01855257,  0.03022939,  0.0577722 ,  0.02155199,
        0.09157985,  0.01603453,  0.01649598,  0.0137446 ,  0.214381  ,
        0.01522823,  0.02310481,  0.02532752,  0.01938365,  0.01630821,
        0.0165853 ,  0.02287357,  0.01749958,  0.02104572,  0.01920774,
        0.02870235,  0.02248628,  0.02123091,  0.02270318,  0.02925958,
        0.02497299,  0.02772489,  0.01336271,  0.03147996,  0.05653222,
        0.02099   ,  0.01614863,  0.01468028,  0.04200082,  0.02018229,
        0.03071444,  0.0172426 ,  0.05539409,  0.01546506,  0.01919494,
        0.01181642,  0.02605107,  0.02734984,  0.02742433,  0.01269414,
        0.0309975 ,  0.02049203,  0.04386141,  0.02091887,  0.02647486,
        0.01796933,  0.01708889,  0.0252859 ,  0.04180007,  0.02390122,
        0.02607526,  0.01515609,  0.03087882,  0.03408275,  0.03022806,
        0.01705601,  0.02420026,  0.12322895,  0.01451559,  0.02170634,
        0.0335677 ,  0.02557157,  0.03644261,  0.02983856,  0.02230817,
        0.03079762,  0.01734977,  0.01859303,  0.05581704,  0.03178872,
        0.01848361,  0.03213747,  0.02641752,  0.01841007,  0.02274081,
        0.01687416,  0.01678221,  0.02555189,  0.0199169 ,  0.01422269,
        0.0145302 ,  0.0824327 ,  0.041644  ,  0.01496629,  0.02863764,
        0.01971575,  0.01823156,  0.01966681,  0.02342617,  0.02498472,
        0.03579154,  0.01969989,  0.01583771,  0.01426108,  0.01763486,
        0.01418049,  0.01779596,  0.02204486,  0.01511269,  0.02581698,
        0.0176799 ,  0.0139617 ,  0.02184109,  0.03458151,  0.02766148,
        0.02145584,  0.01993973,  0.01904919,  0.02009994,  0.02284068,
        0.01938106,  0.10036449,  0.02759265,  0.01824198,  0.01374353,
        0.0214628 ,  0.01271769,  0.01801939,  0.02198052,  0.01190535,
        0.01959362,  0.02169705,  0.01867745,  0.02588723,  0.02541382,
        0.01915096,  0.01640836,  0.06396394,  0.02549226,  0.01790873,
        0.02490355,  0.01368331,  0.01342093,  0.01571607,  0.02364099,
        0.03983766,  0.04922049,  0.05254295,  0.05934184,  0.01800293,
        0.01990012,  0.01989881,  0.04501604,  0.01417039,  0.04782827,
        0.03213685,  0.04484826,  0.01529139,  0.03047023,  0.02376416,
        0.01643775,  0.01550967,  0.0203025 ,  0.01693774,  0.01767624])

In [138]:
reload(facefit.test.end2end)


Out[138]:
<module 'facefit.test.end2end' from '/Users/andrejm/Google Drive/Work/BEng project/beng/facefit/test/end2end.py'>

In [139]:
from facefit.test.end2end import test_model

In [140]:
testset = helen_test
esr_mean_stages_helen = test_model(esr_model, testset, 5)
ert_mean_stages_helen = test_model(ert_model, testset, 5)
lbf_mean_stages = test_model(lbf_fast_model, testset, 1)


Detecting face 329/330

In [141]:
testset = lfpw_test
esr_mean_stages_lfpw = test_model(esr_model, testset, 5)
ert_mean_stages_lfpw = test_model(ert_model, testset, 5)
lbf_mean_stages_lfpw = test_model(lbf_fast_model, testset, 1)


Detecting face 223/224

In [134]:
np.array(esr_mean_stages_helen).shape


Out[134]:
(4, 330)

In [142]:
img = testset[0]

In [144]:
import menpofit

In [145]:
face_detector = menpodetect.load_dlib_frontal_face_detector()

In [146]:
bbox = face_detector(img)

In [148]:
img.view()
bbox[0].view()


Out[148]:
<menpo.visualize.viewmatplotlib.MatplotlibPointGraphViewer2d at 0x1a44665d0>

In [156]:
%timeit esr_model.apply(img, (bbox, 1, None))


10 loops, best of 3: 38.4 ms per loop

In [158]:
%timeit ert_model.apply(img, (bbox, 1, None))


1 loops, best of 3: 110 ms per loop

In [163]:
%timeit lbf_fast_model.apply(img, (bbox, 1, None))


10 loops, best of 3: 73.5 ms per loop

In [164]:
im = mio.import_builtin_asset('lenna.png')

In [169]:
im.view()


Out[169]:
<menpo.visualize.viewmatplotlib.MatplotlibImageViewer2d at 0x1b65e6390>

In [181]:
img = mio.import_image("../../lfpw/trainset/image_0010.png")

In [182]:
img.crop_to_landmarks_proportion_inplace(0.5)


Out[182]:
<menpo.image.base.Image at 0x1acb6f450>

In [183]:
img = img.as_greyscale(mode='average')

In [195]:
shape.points += [-20]

In [196]:
shape.view()
img.view()


Out[196]:
<menpo.visualize.viewmatplotlib.MatplotlibImageViewer2d at 0x1ac5bba90>