In [89]:

    

%load_ext autoreload
%autoreload 2
import os
import pandas as pd
%matplotlib inline

import vislab
import vislab.datasets
import vislab._results
import vislab.results


    

    




The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload

In [90]:

    

c = vislab.util.get_mongodb_client()['predict']['ava_style_jul20']
if c.find({'features': 'noise'}).count() > 0:
    c.remove({'features': 'noise'})
pd.DataFrame([x for x in c.find()])


    

    
Out[90]:






  

    

      

      
_id
      
data
      
features
      
num_test
      
num_train
      
num_val
      
quadratic
      
results_name
      
score_test
      
score_val
      
task
    
  
  

    

      
0
      
 53cc87339d99bd3578004526
      
 ava_style_style_ALL
      
 [caffe_fc6]
      
 2573
      
 8501
      
 2769
      
 None
      
 data_ava_style_style_ALL_features_['caffe_fc6'...
      
 0.433892
      
 0.481512
      
 clf
    
  

In [91]:

    

label_df = vislab.datasets.ava.get_style_df()


    

In [92]:

    

results_df, preds_panel = vislab._results.load_pred_results(
    'ava_style_jul20', os.path.expanduser('~/work/vislab-git/data/results'),
    multiclass=True, force=True)
pred_prefix = 'pred'
print preds_panel.minor_axis


    

    




Results in collection ava_style_jul20: 1
Index([u'caffe_fc6 None vw'], dtype='object')

In [93]:

    

feat_to_evaluate = [
    u'caffe_fc6 None vw',
]
nice_feat_names = {
    'caffe_fc6 None vw': 'DeCAF_6',
}
preds_panel = preds_panel.select(lambda x: x in feat_to_evaluate, 'minor')


    

In [94]:

    

balanced=False

mc_metrics = vislab.results.multiclass_metrics_feat_comparison(
    preds_panel, label_df, pred_prefix, features=preds_panel.minor_axis.tolist() + ['random'],
    balanced=balanced, with_plot=False, with_print=True, nice_feat_names=nice_feat_names)

ap_df = mc_metrics['ap_df'].copy()
ap_df = ap_df[sorted(ap_df.columns)]

ap_df['Murray-CVPR-2012'] = 0
ap_df['Murray-CVPR-2012'].iloc[:-1] = [
    .44, .51, .64, .74, .73, .43, .50, .40, .69, .30, .48, .72, .39, .57]
ap_df['Murray-CVPR-2012'].iloc[-1] = ap_df['Murray-CVPR-2012'].iloc[:-1].mean()
ap_df

column_order = ap_df.columns[(-ap_df.ix['_mean']).argsort().values]
ap_df.index = [x.replace('style_', '') for x in ap_df.index]
ap_df = ap_df.reindex_axis(column_order, axis=1)
# ap_df.to_csv('/Users/sergeyk/work/aphrodite-writeup/results/ava_style_ap_df.csv')
fig_imbalanced = vislab.results_viz.plot_df_bar(ap_df, fontsize=14)
# fig.savefig('/Users/sergeyk/work/aphrodite-writeup/figures/ava_style_ap_barplot.pdf', bbox_inches='tight')

ap_df_imbalanced = ap_df


    

    




Looks like the preds frame already has gt info.
Only taking 'test' split predictions.
********************caffe_fc6 None vw********************
------------------------------------------------------------
Classification metrics on {}
                            precision    recall  f1-score  support
style_Complementary_Colors   0.451777  0.549383  0.495822      162
style_Duotones               0.590244  0.338936  0.430605      357
style_HDR                    0.372549  0.527778  0.436782       72
style_Image_Grain            0.221698  0.456311  0.298413      103
style_Light_On_White         0.457778  0.927928  0.613095      111
style_Long_Exposure          0.492857  0.410714  0.448052      168
style_Macro                  0.479339  0.358025  0.409894      324
style_Motion_Blur            0.327586  0.365385  0.345455      104
style_Negative_Image         0.473171  0.617834  0.535912      157
style_Rule_of_Thirds         0.422819  0.263598  0.324742      239
style_Shallow_DOF            0.547718  0.316547  0.401216      417
style_Silhouettes            0.632353  0.701087  0.664948      184
style_Soft_Focus             0.259887  0.442308  0.327402      104
style_Vanishing_Point        0.354430  0.788732  0.489083       71
accuracy: 0.444617178391

Looks like the preds frame already has gt info.
Only taking 'test' split predictions.
********************random********************
------------------------------------------------------------
Classification metrics on {}
                            precision    recall  f1-score  support
style_Complementary_Colors   0.087629  0.109677  0.097421      155
style_Duotones               0.156250  0.081967  0.107527      366
style_HDR                    0.023392  0.057143  0.033195       70
style_Image_Grain            0.035714  0.064815  0.046053      108
style_Light_On_White         0.032967  0.050420  0.039867      119
style_Long_Exposure          0.120370  0.151163  0.134021      172
style_Macro                  0.150602  0.080645  0.105042      310
style_Motion_Blur            0.040000  0.073394  0.051780      109
style_Negative_Image         0.052941  0.056962  0.054878      158
style_Rule_of_Thirds         0.116959  0.086580  0.099502      231
style_Shallow_DOF            0.206250  0.078385  0.113597      421
style_Silhouettes            0.052083  0.056180  0.054054      178
style_Soft_Focus             0.052023  0.088235  0.065455      102
style_Vanishing_Point        0.036842  0.094595  0.053030       74
accuracy: 0.0820054411193







    













    













    

In [95]:

    

balanced=True

mc_metrics_balanced = vislab.results.multiclass_metrics_feat_comparison(
    preds_panel, label_df, pred_prefix, features=preds_panel.minor_axis.tolist() + ['random'],
    balanced=balanced, with_plot=False, with_print=True, nice_feat_names=nice_feat_names)

ap_df = mc_metrics_balanced['ap_df'].copy()
ap_df = ap_df[sorted(ap_df.columns)]

ap_df['Murray-CVPR-2012'] = 0
ap_df['Murray-CVPR-2012'].iloc[:-1] = [
    .44, .51, .64, .74, .73, .43, .50, .40, .69, .30, .48, .72, .39, .57]
ap_df['Murray-CVPR-2012'].iloc[-1] = ap_df['Murray-CVPR-2012'].iloc[:-1].mean()
ap_df

column_order = ap_df.columns[(-ap_df.ix['_mean']).argsort().values]
ap_df.index = [x.replace('style_', '') for x in ap_df.index]
ap_df = ap_df.reindex_axis(column_order, axis=1)
# ap_df.to_csv('/Users/sergeyk/work/aphrodite-writeup/results/ava_style_ap_df.csv')
fig = vislab.results_viz.plot_df_bar(ap_df, fontsize=14)
# fig.savefig('/Users/sergeyk/work/aphrodite-writeup/figures/ava_style_ap_barplot.pdf', bbox_inches='tight')


    

    




Looks like the preds frame already has gt info.
Only taking 'test' split predictions.
********************caffe_fc6 None vw********************
------------------------------------------------------------
Classification metrics on {} balanced
                            precision    recall  f1-score  support
style_Complementary_Colors   0.382716  0.462687  0.418919       67
style_Duotones               0.441176  0.243902  0.314136      123
style_HDR                    0.616667  0.506849  0.556391       73
style_Image_Grain            0.309524  0.490566  0.379562       53
style_Light_On_White         0.547826  0.887324  0.677419       71
style_Long_Exposure          0.517241  0.379747  0.437956       79
style_Macro                  0.394737  0.340909  0.365854       88
style_Motion_Blur            0.456140  0.406250  0.429752       64
style_Negative_Image         0.505618  0.642857  0.566038       70
style_Rule_of_Thirds         0.327273  0.233766  0.272727       77
style_Shallow_DOF            0.500000  0.273504  0.353591      117
style_Silhouettes            0.626506  0.693333  0.658228       75
style_Soft_Focus             0.317647  0.465517  0.377622       58
style_Vanishing_Point        0.464646  0.779661  0.582278       59
accuracy: 0.459031657356

Looks like the preds frame already has gt info.
Only taking 'test' split predictions.
********************random********************
------------------------------------------------------------
Classification metrics on {} balanced
                            precision    recall  f1-score  support
style_Complementary_Colors   0.038462  0.046875  0.042254       64
style_Duotones               0.094595  0.060345  0.073684      116
style_HDR                    0.087500  0.095890  0.091503       73
style_Image_Grain            0.089744  0.120690  0.102941       58
style_Light_On_White         0.080000  0.084507  0.082192       71
style_Long_Exposure          0.086022  0.109589  0.096386       73
style_Macro                  0.078947  0.075000  0.076923       80
style_Motion_Blur            0.074074  0.095238  0.083333       63
style_Negative_Image         0.039474  0.041096  0.040268       73
style_Rule_of_Thirds         0.016667  0.012500  0.014286       80
style_Shallow_DOF            0.191176  0.109244  0.139037      119
style_Silhouettes            0.014085  0.012987  0.013514       77
style_Soft_Focus             0.101266  0.125000  0.111888       64
style_Vanishing_Point        0.037037  0.050847  0.042857       59
accuracy: 0.0738317757009







    













    













    

In [97]:

    

ap_df['DeCAF_6_Imbalanced'] = ap_df_imbalanced['DeCAF_6']


    

In [98]:

    

ap_df


    

    
Out[98]:






  

    

      

      
DeCAF_6
      
Murray-CVPR-2012
      
random
      
DeCAF_6_Imbalanced
    
  
  

    

      
Complementary_Colors
      
 0.5927225
      
 0.440000
      
  0.1056653
      
 0.5716854
    
    

      
Duotones
      
   0.77096
      
 0.510000
      
  0.2230531
      
  0.791697
    
    

      
HDR
      
 0.6949143
      
 0.640000
      
 0.08720688
      
  0.481301
    
    

      
Image_Grain
      
 0.5084107
      
 0.740000
      
  0.1193099
      
 0.4738979
    
    

      
Light_On_White
      
 0.9195684
      
 0.730000
      
  0.1007541
      
 0.8606687
    
    

      
Long_Exposure
      
 0.4874254
      
 0.430000
      
  0.1065927
      
 0.4837493
    
    

      
Macro
      
  0.570729
      
 0.500000
      
  0.1334218
      
 0.5616162
    
    

      
Motion_Blur
      
  0.510705
      
 0.400000
      
  0.1141356
      
 0.3853454
    
    

      
Negative_Image
      
 0.6365191
      
 0.690000
      
  0.1010129
      
 0.5673742
    
    

      
Rule_of_Thirds
      
 0.3608834
      
 0.300000
      
  0.1577499
      
 0.3866816
    
    

      
Shallow_DOF
      
 0.6254785
      
 0.480000
      
   0.252538
      
  0.729756
    
    

      
Silhouettes
      
 0.7906954
      
 0.720000
      
 0.09788466
      
 0.8015866
    
    

      
Soft_Focus
      
 0.3899362
      
 0.390000
      
  0.1203519
      
  0.302386
    
    

      
Vanishing_Point
      
 0.7386618
      
 0.570000
      
 0.08259215
      
 0.6294349
    
    

      
_mean
      
  0.614115
      
 0.538571
      
  0.1287335
      
   0.57337