In [3]:
%load_ext autoreload
%autoreload 2
import re
import os
import aphrodite.results
import pandas as pd
import vislab
import vislab.datasets
import vislab.results
The autoreload extension is already loaded. To reload it, use:
%reload_ext autoreload
In [4]:
c = vislab.util.get_mongodb_client()['predict']['ava_style_aesth_oct29']
if c.find({'features': 'noise'}).count() > 0:
c.remove({'features': 'noise'})
pd.DataFrame([x for x in c.find()])
Out[4]:
_id
data
features
num_test
num_train
num_val
quadratic
results_name
score_test
score_val
task
0
526f86549622958641aaee6f
ava_style_rating_std_bin
[lab_hist]
2809
8984
2286
False
data_ava_style_rating_std_bin_features_['lab_h...
0.535583
0.551982
clf
1
526f86689622958641aaee70
ava_style_rating_std_cn_bin
[lab_hist]
2809
9170
2100
False
data_ava_style_rating_std_cn_bin_features_['la...
0.543307
0.569578
clf
2
526f868b9622958641aaee71
ava_style_rating_mean_bin
[lab_hist]
2809
9060
2210
False
data_ava_style_rating_mean_bin_features_['lab_...
0.543319
0.583865
clf
3
526f873a9622958641aaee72
ava_style_rating_mean_cn_bin
[lab_hist]
2809
8998
2272
False
data_ava_style_rating_mean_cn_bin_features_['l...
0.549347
0.548444
clf
4
526f87f19622958641aaee73
ava_style_rating_std_bin
[gbvs_saliency]
2809
8984
2286
False
data_ava_style_rating_std_bin_features_['gbvs_...
0.511687
0.542105
clf
5
526f8d349622958641aaee75
ava_style_rating_std_cn_bin
[decaf_fc6]
2809
9170
2100
False
data_ava_style_rating_std_cn_bin_features_['de...
0.563672
0.601905
clf
6
526f8d499622958641aaee76
ava_style_rating_std_bin
[decaf_fc6]
2809
8984
2286
False
data_ava_style_rating_std_bin_features_['decaf...
0.554665
0.566054
clf
7
526f8df79622958641aaee77
ava_style_rating_mean_cn_bin
[decaf_fc6]
2809
8998
2272
False
data_ava_style_rating_mean_cn_bin_features_['d...
0.621670
0.626320
clf
8
526f92cb9622958641aaee78
ava_style_rating_std_cn_bin
[decaf_fc6_flatten]
2809
9170
2100
False
data_ava_style_rating_std_cn_bin_features_['de...
0.542578
0.671279
clf
9
526f93819622958641aaee79
ava_style_rating_mean_bin
[decaf_fc6_flatten]
2809
9060
2210
False
data_ava_style_rating_mean_bin_features_['deca...
0.586433
0.673303
clf
10
527014449622958641aaee7a
ava_style_rating_mean_bin
[gbvs_saliency]
2809
9060
2210
False
data_ava_style_rating_mean_bin_features_['gbvs...
0.508035
0.551410
clf
11
5270145a9622958641aaee7b
ava_style_rating_std_cn_bin
[gbvs_saliency]
2809
9170
2100
False
data_ava_style_rating_std_cn_bin_features_['gb...
0.526254
0.546367
clf
12
527014989622958641aaee7c
ava_style_rating_mean_cn_bin
[gbvs_saliency]
2809
8998
2272
False
data_ava_style_rating_mean_cn_bin_features_['g...
0.507942
0.522546
clf
13
52701bdb9622958641aaee7d
ava_style_rating_mean_bin
[decaf_fc6]
2809
9060
2210
False
data_ava_style_rating_mean_bin_features_['deca...
0.630831
0.612217
clf
14
527020209622958641aaee7e
ava_style_rating_std_bin
[decaf_fc6_flatten]
2809
8984
2286
False
data_ava_style_rating_std_bin_features_['decaf...
0.535350
0.587204
clf
15
527022309622958641aaee7f
ava_style_rating_mean_cn_bin
[decaf_fc6_flatten]
2809
8998
2272
False
data_ava_style_rating_mean_cn_bin_features_['d...
0.585735
0.593034
clf
16
52702b579622958641aaee80
ava_style_rating_std_bin
[gist_256]
2809
8984
2286
False
data_ava_style_rating_std_bin_features_['gist_...
0.526603
0.566054
clf
17
52702b5e9622958641aaee81
ava_style_rating_mean_bin
[gist_256]
2809
9060
2210
False
data_ava_style_rating_mean_bin_features_['gist...
0.523324
0.535294
clf
18
52702baa9622958641aaee82
ava_style_rating_mean_cn_bin
[gist_256]
2809
8998
2272
False
data_ava_style_rating_mean_cn_bin_features_['g...
0.534917
0.573504
clf
19
52702bb19622958641aaee83
ava_style_rating_std_cn_bin
[gist_256]
2809
9170
2100
False
data_ava_style_rating_std_cn_bin_features_['gi...
0.533203
0.536667
clf
20
52702cab9622958641aaee84
ava_style_rating_std_bin
[mc_bit]
2809
8984
2286
False
data_ava_style_rating_std_bin_features_['mc_bi...
0.530448
0.580318
clf
21
52702d2c9622958641aaee85
ava_style_rating_std_cn_bin
[mc_bit]
2809
9170
2100
False
data_ava_style_rating_std_cn_bin_features_['mc...
0.565200
0.570881
clf
22
52702e439622958641aaee86
ava_style_rating_mean_bin
[mc_bit]
2809
9060
2210
False
data_ava_style_rating_mean_bin_features_['mc_b...
0.627473
0.725733
clf
23
52702e479622958641aaee87
ava_style_rating_mean_cn_bin
[mc_bit]
2809
8998
2272
False
data_ava_style_rating_mean_cn_bin_features_['m...
0.620927
0.653404
clf
24
5272510b8dc9cfbe92ca2345
ava_style_rating_mean_bin
[fusion_ava_style_oct22]
2809
9060
2210
None
data_ava_style_rating_mean_bin_features_['fusi...
0.559131
0.574208
clf
25
527251378dc9cfbe92ca2346
ava_style_rating_std_bin
[fusion_ava_style_oct22]
2809
8984
2286
None
data_ava_style_rating_std_bin_features_['fusio...
0.545741
0.571741
clf
26
527250e49622958641af684d
ava_style_rating_std_bin
[fusion_ava_style_oct22]
2809
8984
2286
None
data_ava_style_rating_std_bin_features_['fusio...
0.562303
0.582677
clf
27
527319678dc9cfbe92ca2347
ava_style_rating_mean_bin
[fusion_ava_style_oct22, pascal_mc_for_fusion_...
2809
9060
2210
fp
data_ava_style_rating_mean_bin_features_['fusi...
0.558327
0.567873
clf
In [5]:
results_df, preds_panel = aphrodite.results.load_pred_results(
'ava_style_aesth_oct29', os.path.expanduser('~/work/aphrodite/data/results2'),
multiclass=False, force=True)
pred_prefix = 'ava_style'
print preds_panel.minor_axis
preds_panel
Results in collection ava_style_aesth_oct29: 28
Index([u'decaf_fc6 False vw', u'decaf_fc6_flatten False vw', u'fusion_ava_style_oct22 None vw', u'fusion_ava_style_oct22,pascal_mc_for_fusion_ava_style_oct22 fp vw', u'gbvs_saliency False vw', u'gist_256 False vw', u'lab_hist False vw', u'label', u'mc_bit False vw', u'split'], dtype='object')
Out[5]:
<class 'pandas.core.panel.Panel'>
Dimensions: 4 (items) x 14079 (major_axis) x 10 (minor_axis)
Items axis: clf ava_style_rating_mean_bin to clf ava_style_rating_std_cn_bin
Major_axis axis: 1187 to 97009
Minor_axis axis: decaf_fc6 False vw to split
In [6]:
label_df = vislab.datasets.ava.get_ratings_df()
label_df.iloc[:5]
Out[6]:
rating_mean
rating_std
rating_mean_bin
rating_std_bin
rating_mean_cn_bin
rating_std_cn_bin
image_id
953619
5.6371
1.4218
True
False
False
True
953958
4.6984
1.9851
False
True
False
True
954184
5.6746
1.1043
True
False
False
False
954113
5.7734
1.2822
True
False
True
False
953980
5.2093
1.1592
False
False
False
False
In [7]:
task_name = 'clf ava_style_rating_mean_bin'
task_names = preds_panel.items
task_metrics = {}
for task_name in task_names:
pred_df = preds_panel[task_name]
feature_names = [x for x in pred_df.columns if x not in ['label', 'split']]
feat_metrics = {}
for feature_name in feature_names:
pdf = pred_df[[feature_name, 'label']]
pdf.columns = ['pred', 'label']
feat_metrics[feature_name] = vislab.results.binary_metrics(
pdf, task_name + ' ' + feature_name, balanced=True,
with_plot=False, with_print=True)
task_metrics[task_name] = feat_metrics
acc_df = pd.DataFrame([
pd.Series(
[
task_metrics[task_name][key]['accuracy']
for key in sorted(task_metrics[task_name].keys())
],
sorted(task_metrics[task_name].keys()),
name=task_name
)
for task_name in task_names
])
------------------------------------------------------------
Classification metrics on clf ava_style_rating_mean_bin decaf_fc6 False vw balanced
ap_sklearn: 0.749066184304
mcc: 0.371219456864
ap: 0.749851428549
precision recall f1-score support
False 0.681406 0.697014 0.689121 6898
True 0.689911 0.674108 0.681918 6898
accuracy: 0.685561032183
------------------------------------------------------------
Classification metrics on clf ava_style_rating_mean_bin decaf_fc6_flatten False vw balanced
ap_sklearn: 0.869449111107
mcc: 0.558725093791
ap: 0.869543392057
precision recall f1-score support
False 0.777506 0.782691 0.780090 6898
True 0.781232 0.776022 0.778618 6898
accuracy: 0.77935633517
------------------------------------------------------------
Classification metrics on clf ava_style_rating_mean_bin fusion_ava_style_oct22 None vw balanced
ap_sklearn: 0.548160089853
mcc: 0.169300167358
ap: 0.616493919884
precision recall f1-score support
False 0.599896 0.501450 0.546273 6898
True 0.571731 0.665555 0.615086 6898
accuracy: 0.583502464482
------------------------------------------------------------
Classification metrics on clf ava_style_rating_mean_bin fusion_ava_style_oct22,pascal_mc_for_fusion_ava_style_oct22 fp vw balanced
ap_sklearn: 0.541793205005
mcc: 0.194151409817
ap: 0.645018790104
precision recall f1-score support
False 0.627494 0.460568 0.531226 6898
True 0.573915 0.726587 0.641290 6898
accuracy: 0.593577848652
------------------------------------------------------------
Classification metrics on clf ava_style_rating_mean_bin gbvs_saliency False vw balanced
ap_sklearn: 0.744156022126
mcc: 0.0776281722926
ap: 0.746245131622
precision recall f1-score support
False 0.542743 0.490577 0.515343 6898
True 0.535247 0.586692 0.559790 6898
accuracy: 0.538634386779
------------------------------------------------------------
Classification metrics on clf ava_style_rating_mean_bin gist_256 False vw balanced
ap_sklearn: 0.576194597093
mcc: 0.115442377921
ap: 0.57761317677
precision recall f1-score support
False 0.556104 0.571905 0.563894 6898
True 0.559385 0.543491 0.551324 6898
accuracy: 0.557697883444
------------------------------------------------------------
Classification metrics on clf ava_style_rating_mean_bin lab_hist False vw balanced
ap_sklearn: 0.665550326596
mcc: 0.147378632246
ap: 0.70478931719
precision recall f1-score support
False 0.577969 0.545375 0.561199 6898
True 0.569645 0.601769 0.585266 6898
accuracy: 0.57357204987
------------------------------------------------------------
Classification metrics on clf ava_style_rating_mean_bin mc_bit False vw balanced
ap_sklearn: 0.911798128465
mcc: 0.685629238165
ap: 0.928899991326
precision recall f1-score support
False 0.834371 0.855175 0.844645 6898
True 0.851472 0.830241 0.840722 6898
accuracy: 0.842708031313
------------------------------------------------------------
Classification metrics on clf ava_style_rating_mean_cn_bin decaf_fc6 False vw balanced
ap_sklearn: 0.738716445192
mcc: 0.37358109213
ap: 0.739409295433
precision recall f1-score support
False 0.683436 0.69582 0.689572 7009
True 0.690206 0.67770 0.683896 7009
accuracy: 0.686759880154
------------------------------------------------------------
Classification metrics on clf ava_style_rating_mean_cn_bin decaf_fc6_flatten False vw balanced
ap_sklearn: 0.805215518639
mcc: 0.576988960153
ap: 0.816463034094
precision recall f1-score support
False 0.786321 0.792267 0.789283 7009
True 0.790684 0.784705 0.787683 7009
accuracy: 0.788486231987
------------------------------------------------------------
Classification metrics on clf ava_style_rating_mean_cn_bin fusion_ava_style_oct22 None vw balanced
ap_sklearn: 0.500600375675
mcc: 0.0
ap: 0.504011380946
precision recall f1-score support
False 0.5 1 0.666667 7009
True 0.0 0 0.000000 7009
accuracy: 0.5
------------------------------------------------------------
Classification metrics on clf ava_style_rating_mean_cn_bin fusion_ava_style_oct22,pascal_mc_for_fusion_ava_style_oct22 fp vw balanced
ap_sklearn: 0.50159498011
mcc: 0.0
ap: 0.504924701141
precision recall f1-score support
False 0.5 1 0.666667 7009
True 0.0 0 0.000000 7009
accuracy: 0.5
------------------------------------------------------------
Classification metrics on clf ava_style_rating_mean_cn_bin gbvs_saliency False vw balanced
ap_sklearn: 0.541418562595
mcc: 0.0905304254399
ap: 0.601004300733
precision recall f1-score support
False 0.549499 0.500499 0.523856 7009
True 0.541394 0.589670 0.564502 7009
accuracy: 0.545084890855
------------------------------------------------------------
Classification metrics on clf ava_style_rating_mean_cn_bin gist_256 False vw balanced
ap_sklearn: 0.560323827475
mcc: 0.115371794222
ap: 0.562965225231
precision recall f1-score support
False 0.555434 0.577543 0.566273 7009
True 0.560030 0.537737 0.548657 7009
accuracy: 0.557640176915
------------------------------------------------------------
Classification metrics on clf ava_style_rating_mean_cn_bin lab_hist False vw balanced
ap_sklearn: 0.401516016035
mcc: 0.150538406449
ap: 0.513465852355
precision recall f1-score support
False 0.560112 0.683978 0.615879 7009
True 0.594248 0.462833 0.520372 7009
accuracy: 0.573405621344
------------------------------------------------------------
Classification metrics on clf ava_style_rating_mean_cn_bin mc_bit False vw balanced
ap_sklearn: 0.480236181123
mcc: 0.682088791949
ap: 0.632661665457
precision recall f1-score support
False 0.835015 0.849907 0.842396 7009
True 0.847182 0.832073 0.839559 7009
accuracy: 0.840990155514
------------------------------------------------------------
Classification metrics on clf ava_style_rating_std_bin decaf_fc6 False vw balanced
ap_sklearn: 0.686333285139
mcc: 0.300033392604
ap: 0.687235733168
precision recall f1-score support
False 0.650842 0.647271 0.649052 6926
True 0.649196 0.652758 0.650972 6926
accuracy: 0.650014438348
------------------------------------------------------------
Classification metrics on clf ava_style_rating_std_bin decaf_fc6_flatten False vw balanced
ap_sklearn: 0.52244284987
mcc: 0.543916532442
ap: 0.638878707758
precision recall f1-score support
False 0.774523 0.767254 0.770871 6926
True 0.769418 0.776639 0.773011 6926
accuracy: 0.771946289344
------------------------------------------------------------
Classification metrics on clf ava_style_rating_std_bin fusion_ava_style_oct22 None vw balanced
ap_sklearn: 0.400488534806
mcc: 0.167329024066
ap: 0.510982096323
precision recall f1-score support
False 0.560053 0.739243 0.637292 6926
True 0.616561 0.419290 0.499141 6926
accuracy: 0.579266531909
------------------------------------------------------------
Classification metrics on clf ava_style_rating_std_bin fusion_ava_style_oct22,pascal_mc_for_fusion_ava_style_oct22 fp vw balanced
ap_sklearn: 0.475335258902
mcc: 0.0
ap: 0.514164896125
precision recall f1-score support
False 0.5 1 0.666667 6926
True 0.0 0 0.000000 6926
accuracy: 0.5
------------------------------------------------------------
Classification metrics on clf ava_style_rating_std_bin gbvs_saliency False vw balanced
ap_sklearn: 0.384435488416
mcc: 0.0867147640917
ap: 0.507921037342
precision recall f1-score support
False 0.541482 0.565406 0.553185 6926
True 0.545317 0.521224 0.532999 6926
accuracy: 0.543315044759
------------------------------------------------------------
Classification metrics on clf ava_style_rating_std_bin gist_256 False vw balanced
ap_sklearn: 0.566175708313
mcc: 0.109157109023
ap: 0.567837817106
precision recall f1-score support
False 0.546335 0.634999 0.587340 6926
True 0.564288 0.472712 0.514456 6926
accuracy: 0.553855038984
------------------------------------------------------------
Classification metrics on clf ava_style_rating_std_bin lab_hist False vw balanced
ap_sklearn: 0.408496404846
mcc: 0.145113431153
ap: 0.506130411241
precision recall f1-score support
False 0.558729 0.675137 0.611442 6926
True 0.589641 0.466792 0.521073 6926
accuracy: 0.570964481663
------------------------------------------------------------
Classification metrics on clf ava_style_rating_std_bin mc_bit False vw balanced
ap_sklearn: 0.4565498512
mcc: 0.564808691604
ap: 0.54545255991
precision recall f1-score support
False 0.737166 0.864568 0.795800 6926
True 0.836272 0.691741 0.757171 6926
accuracy: 0.778154779093
------------------------------------------------------------
Classification metrics on clf ava_style_rating_std_cn_bin decaf_fc6 False vw balanced
ap_sklearn: 0.704879712741
mcc: 0.333746648688
ap: 0.705943661324
precision recall f1-score support
False 0.665956 0.669626 0.667786 6532
True 0.667796 0.664115 0.665950 6532
accuracy: 0.666870789957
------------------------------------------------------------
Classification metrics on clf ava_style_rating_std_cn_bin decaf_fc6_flatten False vw balanced
ap_sklearn: 0.8171375431
mcc: 0.553561939834
ap: 0.822665905195
precision recall f1-score support
False 0.782908 0.765769 0.774243 6532
True 0.770787 0.787661 0.779132 6532
accuracy: 0.77671463564
------------------------------------------------------------
Classification metrics on clf ava_style_rating_std_cn_bin fusion_ava_style_oct22 None vw balanced
ap_sklearn: 0.496307551719
mcc: 0.0
ap: 0.502507332852
precision recall f1-score support
False 0.5 1 0.666667 6532
True 0.0 0 0.000000 6532
accuracy: 0.5
------------------------------------------------------------
Classification metrics on clf ava_style_rating_std_cn_bin fusion_ava_style_oct22,pascal_mc_for_fusion_ava_style_oct22 fp vw balanced
ap_sklearn: 0.497322445025
mcc: 0.0
ap: 0.503390743785
precision recall f1-score support
False 0.5 1 0.666667 6532
True 0.0 0 0.000000 6532
accuracy: 0.5
------------------------------------------------------------
Classification metrics on clf ava_style_rating_std_cn_bin gbvs_saliency False vw balanced
ap_sklearn: 0.640363678587
mcc: 0.0955299865268
ap: 0.656249719747
precision recall f1-score support
False 0.547882 0.54654 0.547210 6532
True 0.547648 0.54899 0.548318 6532
accuracy: 0.547764849969
------------------------------------------------------------
Classification metrics on clf ava_style_rating_std_cn_bin gist_256 False vw balanced
ap_sklearn: 0.577441455089
mcc: 0.108475855516
ap: 0.578585733381
precision recall f1-score support
False 0.547830 0.616350 0.580073 6532
True 0.561505 0.491274 0.524047 6532
accuracy: 0.553812002449
------------------------------------------------------------
Classification metrics on clf ava_style_rating_std_cn_bin lab_hist False vw balanced
ap_sklearn: 0.713886018826
mcc: 0.159221119206
ap: 0.734740774005
precision recall f1-score support
False 0.569747 0.644672 0.604898 6532
True 0.590869 0.513166 0.549283 6532
accuracy: 0.578919167177
------------------------------------------------------------
Classification metrics on clf ava_style_rating_std_cn_bin mc_bit False vw balanced
ap_sklearn: 0.817488711759
mcc: 0.637019692825
ap: 0.863165007963
precision recall f1-score support
False 0.787311 0.868187 0.825774 6532
True 0.853097 0.765462 0.806907 6532
accuracy: 0.816824862217
/Users/sergeyk/work/vislab/vislab/results.py:72: UserWarning: The sum of true positives and false positives are equal to zero for some labels. Precision is ill defined for those labels [ True]. The precision and recall are equal to zero for some labels. fbeta_score is ill defined for those labels [ True].
pred_df['label'], pred_df['pred_bin'])
In [8]:
acc_df
Out[8]:
decaf_fc6 False vw
decaf_fc6_flatten False vw
fusion_ava_style_oct22 None vw
fusion_ava_style_oct22,pascal_mc_for_fusion_ava_style_oct22 fp vw
gbvs_saliency False vw
gist_256 False vw
lab_hist False vw
mc_bit False vw
clf ava_style_rating_mean_bin
0.685561
0.779356
0.583502
0.593578
0.538634
0.557698
0.573572
0.842708
clf ava_style_rating_mean_cn_bin
0.686760
0.788486
0.500000
0.500000
0.545085
0.557640
0.573406
0.840990
clf ava_style_rating_std_bin
0.650014
0.771946
0.579267
0.500000
0.543315
0.553855
0.570964
0.778155
clf ava_style_rating_std_cn_bin
0.666871
0.776715
0.500000
0.500000
0.547765
0.553812
0.578919
0.816825
In [9]:
fig = vislab.results_viz.plot_df_bar(acc_df)
# TODO: why are these all so similar?
In [19]:
nice_feat_names = {
'decaf_fc6 False vw': 'DeCAF_6',
'decaf_fc6_flatten False vw': 'DeCAF_5',
'decaf_tuned_fc6 False vw': 'Fine-tuned DeCAF_6',
'fusion_ava_style_oct22,pascal_mc_for_fusion_ava_style_oct22 fp vw': 'Late-fusion x Content',
'mc_bit False vw': 'MC-bit',
'gbvs_saliency False vw': 'Graph-based Saliency',
'gist_256 False vw': 'GIST',
'lab_hist False vw': 'L*a*b* Histogram'
}
df = acc_df.ix[['clf ava_style_rating_mean_bin']]
df.columns = [
nice_feat_names[x] if x in nice_feat_names else x
for x in df.columns
]
del df['fusion_ava_style_oct22 None vw']
df.index = ['AVA Rating']
df.to_csv('/Users/sergeyk/work/aphrodite-writeup/results/ava_style_aesthetics_acc_df.csv')
df
Out[19]:
DeCAF_6
DeCAF_5
Late-fusion x Content
Graph-based Saliency
GIST
L*a*b* Histogram
MC-bit
AVA Rating
0.685561
0.779356
0.593578
0.538634
0.557698
0.573572
0.842708
In [16]:
Content source: Jai-Chaudhary/vislab
Similar notebooks: