In [8]:

    

%load_ext autoreload
%autoreload 2
import os
import re
import pandas as pd

import vislab
import vislab.vw3
import vislab.datasets
import vislab.results
import vislab._results


    

    




The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload

In [11]:

    

collection_name = 'flickr_mar23'
results_df, preds_panel = vislab._results.load_pred_results(
        collection_name, os.path.expanduser('~/work/vislab-git/data/results'),
        multiclass=True, force=False)


    

    




Loaded from cache: 3 records

In [43]:

    

def write_preds_as_feat(dataset_name, collection_name):
    results_df, preds_panel = vislab._results.load_pred_results(
        collection_name, os.path.expanduser('~/work/vislab-git/data/results'),
        multiclass=True, force=False)
    pred_prefix = 'pred'
    
    good_items = [x for x in preds_panel.items if x.startswith(pred_prefix)]
    dfs = []
    all_preds_ever_df = pd.DataFrame()
    for item in good_items:
        df = preds_panel[item].copy()
        df.columns = ['{}_{}'.format(item, x) for x in df.columns]
        all_preds_ever_df = all_preds_ever_df.join(df, how='outer')
    all_preds_ever_df = all_preds_ever_df.astype(float)
    df = all_preds_ever_df.fillna(all_preds_ever_df.mean())
    # Standardize
    df = (df - df.values.mean(0)) / df.values.std(0)
    print df.info()
    
    output_filename = '{}/{}/fusion_{}_all.txt.gz'.format(
        vislab.config['paths']['feats'], dataset_name, collection_name)
    vislab.vw3.write_data_in_vw_format(
        df, 'fusion_{}'.format(collection_name), output_filename)
    
def write_preds_as_feat_single_feat(dataset_name, collection_name, feat_name):
    results_df, preds_panel = vislab._results.load_pred_results(
        collection_name, os.path.expanduser('~/work/vislab-git/data/results'),
        multiclass=True, force=False)
    preds_df = preds_panel.minor_xs(feat_name)
    pred_prefix = 'pred'
    good_items = [x for x in preds_df.columns if x.startswith(pred_prefix)]
    all_preds_ever_df = preds_df[good_items].astype(float)
    df = all_preds_ever_df.fillna(all_preds_ever_df.mean())
    # Standardize
    df = (df - df.values.mean(0)) / df.values.std(0)
    print df.info()
    
    output_filename = '{}/{}/fusion_{}.txt.gz'.format(
        vislab.config['paths']['feats'], dataset_name, collection_name)
    vislab.vw3.write_data_in_vw_format(
        df, 'fusion_{}'.format(collection_name), output_filename)


    

In [7]:

    

write_preds_as_feat('ava_style', 'ava_style_oct22')


    

    




Loaded from cache: 8 records
(13843, 112)

In [44]:

    

write_preds_as_feat('flickr', 'flickr_mar23')


    

    




Loaded from cache: 3 records
<class 'pandas.core.frame.DataFrame'>
Index: 80000 entries, 10000032526 to 9999123683
Data columns (total 60 columns):
pred_style_Bokeh_caffe_fc6 None vw                    80000 non-null float64
pred_style_Bokeh_caffe_fc7 None vw                    80000 non-null float64
pred_style_Bokeh_mc_bit None vw                       80000 non-null float64
pred_style_Bright_caffe_fc6 None vw                   80000 non-null float64
pred_style_Bright_caffe_fc7 None vw                   80000 non-null float64
pred_style_Bright_mc_bit None vw                      80000 non-null float64
pred_style_Depth_of_Field_caffe_fc6 None vw           80000 non-null float64
pred_style_Depth_of_Field_caffe_fc7 None vw           80000 non-null float64
pred_style_Depth_of_Field_mc_bit None vw              80000 non-null float64
pred_style_Detailed_caffe_fc6 None vw                 80000 non-null float64
pred_style_Detailed_caffe_fc7 None vw                 80000 non-null float64
pred_style_Detailed_mc_bit None vw                    80000 non-null float64
pred_style_Ethereal_caffe_fc6 None vw                 80000 non-null float64
pred_style_Ethereal_caffe_fc7 None vw                 80000 non-null float64
pred_style_Ethereal_mc_bit None vw                    80000 non-null float64
pred_style_Geometric_Composition_caffe_fc6 None vw    80000 non-null float64
pred_style_Geometric_Composition_caffe_fc7 None vw    80000 non-null float64
pred_style_Geometric_Composition_mc_bit None vw       80000 non-null float64
pred_style_HDR_caffe_fc6 None vw                      80000 non-null float64
pred_style_HDR_caffe_fc7 None vw                      80000 non-null float64
pred_style_HDR_mc_bit None vw                         80000 non-null float64
pred_style_Hazy_caffe_fc6 None vw                     80000 non-null float64
pred_style_Hazy_caffe_fc7 None vw                     80000 non-null float64
pred_style_Hazy_mc_bit None vw                        80000 non-null float64
pred_style_Horror_caffe_fc6 None vw                   80000 non-null float64
pred_style_Horror_caffe_fc7 None vw                   80000 non-null float64
pred_style_Horror_mc_bit None vw                      80000 non-null float64
pred_style_Long_Exposure_caffe_fc6 None vw            80000 non-null float64
pred_style_Long_Exposure_caffe_fc7 None vw            80000 non-null float64
pred_style_Long_Exposure_mc_bit None vw               80000 non-null float64
pred_style_Macro_caffe_fc6 None vw                    80000 non-null float64
pred_style_Macro_caffe_fc7 None vw                    80000 non-null float64
pred_style_Macro_mc_bit None vw                       80000 non-null float64
pred_style_Melancholy_caffe_fc6 None vw               80000 non-null float64
pred_style_Melancholy_caffe_fc7 None vw               80000 non-null float64
pred_style_Melancholy_mc_bit None vw                  80000 non-null float64
pred_style_Minimal_caffe_fc6 None vw                  80000 non-null float64
pred_style_Minimal_caffe_fc7 None vw                  80000 non-null float64
pred_style_Minimal_mc_bit None vw                     80000 non-null float64
pred_style_Noir_caffe_fc6 None vw                     80000 non-null float64
pred_style_Noir_caffe_fc7 None vw                     80000 non-null float64
pred_style_Noir_mc_bit None vw                        80000 non-null float64
pred_style_Pastel_caffe_fc6 None vw                   80000 non-null float64
pred_style_Pastel_caffe_fc7 None vw                   80000 non-null float64
pred_style_Pastel_mc_bit None vw                      80000 non-null float64
pred_style_Romantic_caffe_fc6 None vw                 80000 non-null float64
pred_style_Romantic_caffe_fc7 None vw                 80000 non-null float64
pred_style_Romantic_mc_bit None vw                    80000 non-null float64
pred_style_Serene_caffe_fc6 None vw                   80000 non-null float64
pred_style_Serene_caffe_fc7 None vw                   80000 non-null float64
pred_style_Serene_mc_bit None vw                      80000 non-null float64
pred_style_Sunny_caffe_fc6 None vw                    80000 non-null float64
pred_style_Sunny_caffe_fc7 None vw                    80000 non-null float64
pred_style_Sunny_mc_bit None vw                       80000 non-null float64
pred_style_Texture_caffe_fc6 None vw                  80000 non-null float64
pred_style_Texture_caffe_fc7 None vw                  80000 non-null float64
pred_style_Texture_mc_bit None vw                     80000 non-null float64
pred_style_Vintage_caffe_fc6 None vw                  80000 non-null float64
pred_style_Vintage_caffe_fc7 None vw                  80000 non-null float64
pred_style_Vintage_mc_bit None vw                     80000 non-null float64
dtypes: float64(60)None

In [41]:

    

write_preds_as_feat_single_feat('flickr', 'flickr_mar23', 'caffe_fc7 None vw')


    

    




Loaded from cache: 3 records
<class 'pandas.core.frame.DataFrame'>
Index: 80000 entries, 10000032526 to 9999123683
Data columns (total 20 columns):
pred_style_Bokeh                    80000 non-null float64
pred_style_Bright                   80000 non-null float64
pred_style_Depth_of_Field           80000 non-null float64
pred_style_Detailed                 80000 non-null float64
pred_style_Ethereal                 80000 non-null float64
pred_style_Geometric_Composition    80000 non-null float64
pred_style_HDR                      80000 non-null float64
pred_style_Hazy                     80000 non-null float64
pred_style_Horror                   80000 non-null float64
pred_style_Long_Exposure            80000 non-null float64
pred_style_Macro                    80000 non-null float64
pred_style_Melancholy               80000 non-null float64
pred_style_Minimal                  80000 non-null float64
pred_style_Noir                     80000 non-null float64
pred_style_Pastel                   80000 non-null float64
pred_style_Romantic                 80000 non-null float64
pred_style_Serene                   80000 non-null float64
pred_style_Sunny                    80000 non-null float64
pred_style_Texture                  80000 non-null float64
pred_style_Vintage                  80000 non-null float64
dtypes: float64(20)None

In [10]:

    

write_preds_as_feat('wikipaintings', 'wikipaintings_mar23')


    

    




Results in collection wikipaintings_mar23: 2
(82459, 50)