In [ ]:

    

import sys
sys.path.append("d:/Kaggle_ws/Bosch/src")


    

In [ ]:

    

from include.dataset_fnames import generate_station_data_fname
from include.feature_lists import numeric_features


    

In [96]:

    

import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline


    

In [ ]:

    

import pandas as pd
import numpy as np


    

In [ ]:

    

s01_fname = generate_station_data_fname(station_id="L0S01", sample_type='test', data_type='numeric')
print s01_fname


    

In [ ]:

    

s01_df = pd.read_csv(s01_fname, index_col=['Id'])


    

In [ ]:

    

t = s01_df.isnull().any()


    

In [ ]:

    

for f in t.index:
    if (t.loc[f]==True):
        print f


    

In [ ]:

    

for station_id in sorted(numeric_features):
    print "Station:", station_id
    s_fname = generate_station_data_fname(station_id=station_id, sample_type='train', data_type='numeric')
    s_df = pd.read_csv(s_fname, index_col=['Id'])
    null_state = s_df.isnull().any()
    for ft in null_state.index:
        if (null_state.loc[ft]==True):
            print ft
    del s_fname
    del s_df
    del null_state


    

In [105]:

    

def impute_feature(df, feature):
    new_df = df.copy()
    print new_df.corr()
    sns.distplot(new_df['L0_S1_F24'])


    

In [103]:

    

def find_missing_features(station_id):
    print "Station:", station_id
    s_fname = generate_station_data_fname(station_id=station_id, sample_type='train', data_type='numeric')
    s_df = pd.read_csv(s_fname, index_col=['Id'])
    null_state = s_df.isnull().any()
    for ft in null_state.index:
        if (null_state.loc[ft]==True):
            print ft
            impute_feature(s_df, ft)
    del s_fname
    del s_df
    del null_state


    

In [106]:

    

find_missing_features('L0S01')


    

    




Station: L0S01
L0_S1_F24
           L0_S1_F24  L0_S1_F28
L0_S1_F24   1.000000  -0.270102
L0_S1_F28  -0.270102   1.000000






    




---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-106-62c90b35f5db> in <module>()
----> 1 find_missing_features('L0S01')

<ipython-input-103-83e40007812c> in find_missing_features(station_id)
      7         if (null_state.loc[ft]==True):
      8             print ft
----> 9             impute_feature(s_df, ft)
     10     del s_fname
     11     del s_df

<ipython-input-105-98d76e4c7fe3> in impute_feature(df, feature)
      2     new_df = df.copy()
      3     print new_df.corr()
----> 4     sns.distplot(new_df['L0_S1_F24'])

d:\Anaconda\envs\deep\lib\site-packages\seaborn\distributions.pyc in distplot(a, bins, hist, kde, rug, fit, hist_kws, kde_kws, rug_kws, fit_kws, color, vertical, norm_hist, axlabel, label, ax)
    207     if hist:
    208         if bins is None:
--> 209             bins = min(_freedman_diaconis_bins(a), 50)
    210         hist_kws.setdefault("alpha", 0.4)
    211         hist_kws.setdefault("normed", norm_hist)

d:\Anaconda\envs\deep\lib\site-packages\seaborn\distributions.pyc in _freedman_diaconis_bins(a)
     33         return int(np.sqrt(a.size))
     34     else:
---> 35         return int(np.ceil((a.max() - a.min()) / h))
     36 
     37 

ValueError: cannot convert float NaN to integer





    

In [ ]:

    

s01_df.info()


    

In [ ]:

    

from include.feature_lists import *


    

In [ ]:

    

def p(l):
    print len(l), l
    return len(l)


    

In [ ]:

    

l = 0
for f in (L0S01_numeric, L0S09_numeric, L0S10_numeric, L0S21_numeric, L0S22_numeric, # L1S24_numeric, L1S25_numeric, \
          L3S29_numeric, L3S30_numeric, L3S33_numeric, L3S35_numeric, L3S36_numeric, L3S40_numeric, L3S43_numeric, \
          L3S48_numeric, L3S49_numeric):
    l +=p (f)
print l


    

In [ ]:

    

s48_df = pd.read_csv(generate_station_data_fname(station_id='L3S48', sample_type='train', data_type='numeric'), index_col=['Id'])


    

In [ ]:

    

null_state = s48_df.isnull().any()
for ft in null_state.index:
    if (null_state.loc[ft]==True):
        print ft


    

In [ ]: