In [1]:

    

#TO RE-RUN
%reset -f


    

In [2]:

    

from sklearn import preprocessing
from time import time
import numpy as np
import csv
from sklearn import metrics
from sklearn.preprocessing import scale
from sklearn.feature_selection import VarianceThreshold
from sklearn.cross_validation import StratifiedShuffleSplit, cross_val_score

from sklearn.svm import SVC
from sklearn.svm import LinearSVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.naive_bayes import BernoulliNB, MultinomialNB, GaussianNB

from sklearn.model_selection import GridSearchCV, ParameterGrid
from sklearn.preprocessing import StandardScaler

from imblearn.over_sampling import SMOTE,ADASYN, RandomOverSampler
from imblearn.pipeline import Pipeline
from imblearn.pipeline import make_pipeline

from IPython.display import display, HTML
from operator import truediv
from datetime import datetime
import pandas as pd
import sklearn
import time
import os

from pylab import *
import seaborn as sns
import matplotlib.pyplot as plt

np.set_printoptions(suppress=True)
pd.options.display.float_format = '{:,.4f}'.format
plt.style.use('classic')

%matplotlib inline

print('The scikit-learn version is {}.'.format(sklearn.__version__))


    

    




/home/ilmira/.conda/envs/readmision/lib/python2.7/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.
  "This module will be removed in 0.20.", DeprecationWarning)






    




The scikit-learn version is 0.19.1.

In [3]:

    

import sys
sys.path.insert(1, "../src/")
from TypeFeatImputer import TypeFeatImputer
from UnivCombineFilter import UnivCombineFilter
import MLpipeline as MLpipeline
import readmision_methods as rm


    

In [4]:

    

typeEncounter = "last" # ['first','last']
typeHypothesis = "early_readmission_vs_none" # ['all_readmisssion_vs_none','early_readmission_vs_none']
typeDataFeatures = "extended_extra_diag_3" # ["reduced","extended','extended_extra','extended_extra_diag_1','extended_extra_diag_3']
typeDataExperiment = "all" #["all", "disease"]


    

In [5]:

    

verbose = True
cv_thr = 0.3
cv_folds = 5

tr_thrs = 0.2 # [0.1,0.2,0.4,0.6,1.0]
ts_thr = 1 - tr_thrs

fs_methods = ["none"] #["none","combine_fs","lasso_fs","rfe_rf_fs"]
cls_methods = ["rf"] #["rf","svmRBF","logReg","knn","nn","gbt"]
lms = ["recall","roc_auc"] #["f1_weighted","average_precision","roc_auc","recall"]
sm_types = ["none"] #["none","after"]tr_thrs
sm_method = "sm_smote"


    

In [6]:

    

#Load data
df_all = rm.load_data(typeEncounter, typeDataFeatures)
print "\nSHAPE:"
print df_all.shape

#Filter data by class
df_all = rm.filter_data_by_class(df_all, typeHypothesis)
print "\nSHAPE FILTERED:"
print df_all.shape

print "\nRows by class type:"
print df_all.iloc[:,-1].sort_values().unique(), np.sum(df_all["readmitted"] == 0), np.sum(df_all["readmitted"] == 1)

#Train & Test
X_train, X_test, y_train, y_test = MLpipeline.train_test_partition(df_all,ts_thr)
df_train = pd.DataFrame(np.hstack((X_train, y_train.reshape(-1,1))), columns=df_all.columns)
df_test = pd.DataFrame(np.hstack((X_test, y_test.reshape(-1,1))), columns=df_all.columns)
columns = df_all.columns

print "\nTrain:", X_train.shape, "Test:",  X_test.shape

#Create filters
featFilters = rm.create_filters(df_all)
print [[f[0],np.sum(f[1])] for f in featFilters]


    

    




SHAPE:
(67182, 69)

SHAPE FILTERED:
(45779, 69)

Rows by class type:
[0 1] 39785 5994

Train: (9155, 68) Test: (36624, 68)
[['patient_filter', 5], ['admision_discharge_filter', 29], ['hospital_filter', 9], ['Visits_filter', 8], ['diagnosis_filter', 14], ['medicines_filter', 15], ['extra_filter', 68], ['none_filter', 17]]

In [7]:

    

def train_ensemble(columns, X_train, X_test, y_train, y_test, cv_folds, cv_thr, fs_methods, sm_method, sm_types, 
                                            cls_methods, lms, featFilters):
    #Train & test inner models
    models, train_preds, test_preds = train_inner_models(columns, X_train, X_test, y_train, y_test, cv_folds, 
                                    cv_thr, fs_methods, sm_method, sm_types, cls_methods, lms, featFilters)
    
    #Train stack model
    stacker = train_stacker(train_preds, y_train)
    
    #Test stack model
    evaluate_stacker(stacker, test_preds, y_test)
    
    return models, stacker, train_preds, test_preds

def train_inner_models(columns, X_train, X_test, y_train, y_test, cv_folds, cv_thr, fs_methods, 
                       sm_method, sm_types, cls_methods, lms, featFilters,verbose=True, save=False):
    
    models = []
    cv_preds = []
    ts_preds = []
    for n,f in featFilters[:]:
        
        #Get categoric/numeric
        f_cols = columns[:-1][f==1].values.tolist()
        f_cols.append("readmitted")       
        catCols, reducedCols = rm.compute_type_features(f_cols)

        #Get hyperparams
        hyperparams = np.load("../src/default_hyperparams.npy")

        #Create pipeline
        pipe = MLpipeline.create_pipelines(catCols, reducedCols, hyperparams, fs_methods, sm_method, sm_types, 
                                            cls_methods, lms, f)
                 
        #Prepare cv data
        y_train = y_train.astype(int)
        y_test = y_test.astype(int)        
        cv_inner = StratifiedShuffleSplit(y_train, n_iter=cv_folds, test_size=cv_thr, random_state=24)
        cv_outer = StratifiedShuffleSplit(y_train, n_iter=cv_folds, test_size=cv_thr, random_state=42)
        
        #Run pipeline        
        res = MLpipeline.run_pipeline(n, pipe, X_train, X_test, y_train, y_test,  
                                      cv_inner = cv_inner, 
                                      cv_outer = cv_outer,
                                      verbose = verbose, save = save)[0]
        
        #Get predictions
        tr_pred = res[-1].predict_proba(X_train).tolist()
        ts_pred = res[-1].predict_proba(X_test).tolist()
        
        models.append(res)
        cv_preds.append(tr_pred)
        ts_preds.append(ts_pred)
        
    return models, cv_preds, ts_preds
    
def train_stacker(train_preds, y_train):
        
    #Prepare data stacker
    train_preds = np.array(train_preds).astype(int).T
    y_train = y_train.astype(int)    
    
    #Train stacker
    cls = RandomForestClassifier(random_state=13,)
    params = {'n_estimators':[10,50,100,150],"class_weight":["balanced"],"max_depth":[3,4]}
    
    scorings = {
        'recall_1':'recall',        
        'roc_auc': 'roc_auc',
        'f1_weighted':'f1_weighted',
        'precision_1':'precision',
        'precision_0' : metrics.make_scorer(MLpipeline.precision_0),
        'recall_0' : metrics.make_scorer(MLpipeline.recall_0),
        'spec': metrics.make_scorer(MLpipeline.specificity)
       } 
    selected_metric = "recall_1"
    grid_pipeline = GridSearchCV(cls, param_grid=params, verbose=True, 
                                 n_jobs=-1, cv=5, scoring= scorings, 
                                 refit=selected_metric, error_score = 0) 
    grid_pipeline.fit(train_preds, y_train)           
    results = grid_pipeline.cv_results_
    
    print "\nSTACKER train-cv evaluation:"
    print "*****************************"
    print "Learning metric:", selected_metric
    print "Best parameters:", grid_pipeline.best_params_.values()
    print "Best score:", grid_pipeline.best_score_
    
    for scorer in scorings:       
        best_index = np.nonzero(results['rank_test_%s' % scorer] == 1)[0][0]
        best_score = results['mean_test_%s' % scorer][best_index]
        print "BEST {}: {}".format(scorer, best_score)
    
    return grid_pipeline 

def train_voting(train_preds, y_train):
    
    #Prepare data stacker
    train_preds = np.array(train_preds).astype(int).T
    y_train = y_train.astype(int)
    
    #Compute votes
    y_pred = np.sum(train_preds, axis=1) > train_preds.shape[1]/2.0
    
    #Print results
    evaluate(name, y_train, y_pred)
    
def evaluate_stacker(stacker, test_preds, y_test):
    
    #Prepare test data
    test_preds = np.array(test_preds).astype(int).T   
    y_test = y_test.astype(int)    

    #Predict test data    
    y_pred = stacker.best_estimator_.predict(test_preds)

    #Evaluate results
    evaluate("test", y_test, y_pred)
    
    return None
    
def evaluate(name, y_test, y_pred):

    test_f1_w = metrics.f1_score(y_test, y_pred, average='weighted', pos_label=None)
    test_p, test_r, test_f1, test_s = metrics.precision_recall_fscore_support(y_test, y_pred,labels=None,average=None, sample_weight=None)
    fpr, tpr, _ = metrics.roc_curve(y_test, y_pred)
    test_auc = metrics.auc(fpr, tpr)                
    cm_test = metrics.confusion_matrix(y_test, y_pred)
    tn = cm_test[0,0]
    fp = cm_test[0,1]
    fn = cm_test[1,0]
    tp = cm_test[1,1]
    test_sens = test_r[1]
    test_spec = tn / float(tn+fp)

    print "\nEvaluation:", name
    print "*******************"
    print          
    print "TEST AUC: %0.3f" % (test_auc)                
    print "TEST sensitivity:", test_sens
    print "TEST Specificity:", test_spec
    print
    print "TEST f1 (weighted): %0.3f" % (test_f1_w)
    print "TEST Precision [c=0,1]:", test_p
    print "TEST Recall [c=0,1]:", test_r  
    
    print "Confussion matrix:"
    print "         | PRED"
    print "REAL-->  v "
    print cm_test
    
def test_ensemble(X_test, models, stacker):
    
    test_preds = []
    for m in models:
        test_preds.append(m[-1].predict(X_test.tolist()))
    
    test_preds = np.array(test_preds).astype(int).T
    test_preds = test_preds.astype(int)
    print test_preds.shape
    
    y_pred = stacker.predict(test_preds)
    y_test = df_test.iloc[:,-1].values.astype(int)
    print type(y_pred), np.unique(y_pred)
    print type(y_test), np.unique(y_test)
    
    #Evaluate results
    evaluate("test", y_test, y_pred.astype(int))


    

In [8]:

    

#Train ensemble
models, train_preds, test_preds = train_inner_models(columns, X_train, X_test, y_train, y_test, 
                                    cv_folds, cv_thr, fs_methods, sm_method, sm_types, cls_methods, lms, 
                                                              featFilters)


    

    




DataSet:
**********
**********
SIZE: 1.0
NAME: patient_filter
ALL TRAIN: (9155, 68)
TRAIN: [0's: 7956 1's: 1199 ]
ALL TEST: (36624, 68)
TEST: [0's: 31829 1's: 4795 ]

Num experiment: 0 / 1
****************
FS: none
SM: none
CLS: rf
METRIC: recall
Fitting 5 folds for each of 50 candidates, totalling 250 fits






    




[Parallel(n_jobs=-1)]: Done  18 tasks      | elapsed:   22.0s
[Parallel(n_jobs=-1)]: Done 168 tasks      | elapsed:  1.9min
[Parallel(n_jobs=-1)]: Done 250 out of 250 | elapsed:  2.8min finished






    




TRAIN f1 (weighted): 0.530
TRAIN Precision [c=0,1]: [ 0.89699571  0.15017505]
TRAIN Recall [c=0,1]: [ 0.42031171  0.67973311]
TRAIN AUC: 0.550
TRAIN sensitivity: 0.679733110926
TRAIN Specificity:  0.420311714429

CV INNER metric: recall
CV INNER selected params ['gini', 2, 'balanced_subsample', 50]
CV INNER score: 0.653333333333

CV OUTER f1-weighted score: 0.533  (+/-0.042)
CV OUTER prec score [c=0,1]: 0.895 (+/- 0.008), 0.149  (+/- 0.002)
CV OUTER rec  score [c=0,1]: 0.428 (+/- 0.055), 0.663  (+/- 0.068)
CV OUTER AUC score: 0.566  (+/-0.012)
CV OUTER sensitivity score: 0.663  (+/-0.068)
CV OUTER Specificity score: 0.428  (+/-0.055)
Selected params (bests from CV) ['gini', 2, 'balanced_subsample', 50]

TEST f1 (weighted): 0.528
TEST Precision [c=0,1]: [ 0.89780235  0.15048187]
TEST Recall [c=0,1]: [ 0.41842345  0.68383733]
TEST AUC: 0.551
TEST sensitivity: 0.683837330553
TEST Specificity: 0.418423450313
Confussion matrix:
         | PRED
REAL-->  v 
[[13318 18511]
 [ 1516  3279]]

Total time: 184.61264205

Num experiment: 1 / 1
****************
FS: none
SM: none
CLS: rf
METRIC: roc_auc
Fitting 5 folds for each of 50 candidates, totalling 250 fits






    




[Parallel(n_jobs=-1)]: Done  18 tasks      | elapsed:   21.0s
[Parallel(n_jobs=-1)]: Done 168 tasks      | elapsed:  1.9min
[Parallel(n_jobs=-1)]: Done 250 out of 250 | elapsed:  2.7min finished






    




TRAIN f1 (weighted): 0.539
TRAIN Precision [c=0,1]: [ 0.89557292  0.15014111]
TRAIN Recall [c=0,1]: [ 0.43225239  0.66555463]
TRAIN AUC: 0.549
TRAIN sensitivity: 0.665554628857
TRAIN Specificity:  0.432252388135

CV INNER metric: roc_auc
CV INNER selected params ['entropy', 2, 'balanced_subsample', 500]
CV INNER score: 0.558709910162

CV OUTER f1-weighted score: 0.543  (+/-0.041)
CV OUTER prec score [c=0,1]: 0.893 (+/- 0.005), 0.149  (+/- 0.002)
CV OUTER rec  score [c=0,1]: 0.441 (+/- 0.054), 0.649  (+/- 0.061)
CV OUTER AUC score: 0.568  (+/-0.011)
CV OUTER sensitivity score: 0.649  (+/-0.061)
CV OUTER Specificity score: 0.441  (+/-0.054)
Selected params (bests from CV) ['entropy', 2, 'balanced_subsample', 500]

TEST f1 (weighted): 0.540
TEST Precision [c=0,1]: [ 0.89717925  0.15128543]
TEST Recall [c=0,1]: [ 0.43369254  0.67007299]
TEST AUC: 0.552
TEST sensitivity: 0.670072992701
TEST Specificity: 0.433692544535
Confussion matrix:
         | PRED
REAL-->  v 
[[13804 18025]
 [ 1582  3213]]

Total time: 192.325235128

DataSet:
**********
**********
SIZE: 1.0
NAME: admision_discharge_filter
ALL TRAIN: (9155, 68)
TRAIN: [0's: 7956 1's: 1199 ]
ALL TEST: (36624, 68)
TEST: [0's: 31829 1's: 4795 ]

Num experiment: 0 / 1
****************
FS: none
SM: none
CLS: rf
METRIC: recall
Fitting 5 folds for each of 50 candidates, totalling 250 fits






    




/home/ilmira/.conda/envs/readmision/lib/python2.7/site-packages/sklearn/utils/validation.py:475: DataConversionWarning: Data with input dtype object was converted to float64 by StandardScaler.
  warnings.warn(msg, DataConversionWarning)
/home/ilmira/.conda/envs/readmision/lib/python2.7/site-packages/sklearn/utils/validation.py:475: DataConversionWarning: Data with input dtype object was converted to float64 by StandardScaler.
  warnings.warn(msg, DataConversionWarning)
/home/ilmira/.conda/envs/readmision/lib/python2.7/site-packages/sklearn/utils/validation.py:475: DataConversionWarning: Data with input dtype object was converted to float64 by StandardScaler.
  warnings.warn(msg, DataConversionWarning)
/home/ilmira/.conda/envs/readmision/lib/python2.7/site-packages/sklearn/utils/validation.py:475: DataConversionWarning: Data with input dtype object was converted to float64 by StandardScaler.
  warnings.warn(msg, DataConversionWarning)
/home/ilmira/.conda/envs/readmision/lib/python2.7/site-packages/sklearn/utils/validation.py:475: DataConversionWarning: Data with input dtype object was converted to float64 by StandardScaler.
  warnings.warn(msg, DataConversionWarning)
/home/ilmira/.conda/envs/readmision/lib/python2.7/site-packages/sklearn/utils/validation.py:475: DataConversionWarning: Data with input dtype object was converted to float64 by StandardScaler.
  warnings.warn(msg, DataConversionWarning)
/home/ilmira/.conda/envs/readmision/lib/python2.7/site-packages/sklearn/utils/validation.py:475: DataConversionWarning: Data with input dtype object was converted to float64 by StandardScaler.
  warnings.warn(msg, DataConversionWarning)
/home/ilmira/.conda/envs/readmision/lib/python2.7/site-packages/sklearn/utils/validation.py:475: DataConversionWarning: Data with input dtype object was converted to float64 by StandardScaler.
  warnings.warn(msg, DataConversionWarning)
/home/ilmira/.conda/envs/readmision/lib/python2.7/site-packages/sklearn/utils/validation.py:475: DataConversionWarning: Data with input dtype object was converted to float64 by StandardScaler.
  warnings.warn(msg, DataConversionWarning)
/home/ilmira/.conda/envs/readmision/lib/python2.7/site-packages/sklearn/utils/validation.py:475: DataConversionWarning: Data with input dtype object was converted to float64 by StandardScaler.
  warnings.warn(msg, DataConversionWarning)
/home/ilmira/.conda/envs/readmision/lib/python2.7/site-packages/sklearn/utils/validation.py:475: DataConversionWarning: Data with input dtype object was converted to float64 by StandardScaler.
  warnings.warn(msg, DataConversionWarning)
/home/ilmira/.conda/envs/readmision/lib/python2.7/site-packages/sklearn/utils/validation.py:475: DataConversionWarning: Data with input dtype object was converted to float64 by StandardScaler.
  warnings.warn(msg, DataConversionWarning)
/home/ilmira/.conda/envs/readmision/lib/python2.7/site-packages/sklearn/utils/validation.py:475: DataConversionWarning: Data with input dtype object was converted to float64 by StandardScaler.
  warnings.warn(msg, DataConversionWarning)
/home/ilmira/.conda/envs/readmision/lib/python2.7/site-packages/sklearn/utils/validation.py:475: DataConversionWarning: Data with input dtype object was converted to float64 by StandardScaler.
  warnings.warn(msg, DataConversionWarning)
/home/ilmira/.conda/envs/readmision/lib/python2.7/site-packages/sklearn/utils/validation.py:475: DataConversionWarning: Data with input dtype object was converted to float64 by StandardScaler.
  warnings.warn(msg, DataConversionWarning)
/home/ilmira/.conda/envs/readmision/lib/python2.7/site-packages/sklearn/utils/validation.py:475: DataConversionWarning: Data with input dtype object was converted to float64 by StandardScaler.
  warnings.warn(msg, DataConversionWarning)
[Parallel(n_jobs=-1)]: Done  18 tasks      | elapsed:   38.9s
[Parallel(n_jobs=-1)]: Done 168 tasks      | elapsed:  3.0min
[Parallel(n_jobs=-1)]: Done 250 out of 250 | elapsed:  4.2min finished
/home/ilmira/.conda/envs/readmision/lib/python2.7/site-packages/sklearn/utils/validation.py:475: DataConversionWarning: Data with input dtype object was converted to float64 by StandardScaler.
  warnings.warn(msg, DataConversionWarning)






    




TRAIN f1 (weighted): 0.711
TRAIN Precision [c=0,1]: [ 0.89595943  0.18507561]
TRAIN Recall [c=0,1]: [ 0.68841126  0.46955796]
TRAIN AUC: 0.579
TRAIN sensitivity: 0.469557964971
TRAIN Specificity:  0.688411261941

CV INNER metric: recall
CV INNER selected params ['entropy', 2, 'balanced_subsample', 50]
CV INNER score: 0.513888888889

CV OUTER f1-weighted score: 0.677  (+/-0.031)
CV OUTER prec score [c=0,1]: 0.897 (+/- 0.003), 0.176  (+/- 0.009)
CV OUTER rec  score [c=0,1]: 0.630 (+/- 0.051), 0.522  (+/- 0.045)
CV OUTER AUC score: 0.602  (+/-0.011)
CV OUTER sensitivity score: 0.522  (+/-0.045)
CV OUTER Specificity score: 0.630  (+/-0.051)
Selected params (bests from CV) ['entropy', 2, 'balanced_subsample', 50]

TEST f1 (weighted): 0.714
TEST Precision [c=0,1]: [ 0.89960169  0.19150843]
TEST Recall [c=0,1]: [ 0.68830312  0.49009385]
TEST AUC: 0.589
TEST sensitivity: 0.490093847758
TEST Specificity: 0.688303119796
Confussion matrix:
         | PRED
REAL-->  v 
[[21908  9921]
 [ 2445  2350]]

Total time: 285.66348505

Num experiment: 1 / 1
****************
FS: none
SM: none
CLS: rf
METRIC: roc_auc
Fitting 5 folds for each of 50 candidates, totalling 250 fits






    




[Parallel(n_jobs=-1)]: Done  18 tasks      | elapsed:   38.9s
[Parallel(n_jobs=-1)]: Done 168 tasks      | elapsed:  3.0min






    




---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-8-b93b1c4e2cb7> in <module>()
      2 models, train_preds, test_preds = train_inner_models(columns, X_train, X_test, y_train, y_test, 
      3                                     cv_folds, cv_thr, fs_methods, sm_method, sm_types, cls_methods, lms,
----> 4                                                               featFilters)  

<ipython-input-7-208ba92a2d7e> in train_inner_models(columns, X_train, X_test, y_train, y_test, cv_folds, cv_thr, fs_methods, sm_method, sm_types, cls_methods, lms, featFilters, verbose, save)
     43                                       cv_inner = cv_inner,
     44                                       cv_outer = cv_outer,
---> 45                                       verbose = verbose, save = save)[0]
     46 
     47         #Get predictions

/home/ilmira/healthforecast/readmission/src/MLpipeline.py in run_pipeline(name, pipeline, X_train, X_test, y_train, y_test, tr_thr, ts_thr, cv_inner, cv_outer, verbose, save)
    235         grid_pipeline = GridSearchCV(pipeline_cls, param_grid=pipeline_params, verbose=verbose, 
    236                                      n_jobs=-1, cv=cv_inner, scoring= lm, error_score = 0) 
--> 237         grid_pipeline.fit(X_train, y_train)
    238 
    239 

/home/ilmira/.conda/envs/readmision/lib/python2.7/site-packages/sklearn/model_selection/_search.pyc in fit(self, X, y, groups, **fit_params)
    637                                   error_score=self.error_score)
    638           for parameters, (train, test) in product(candidate_params,
--> 639                                                    cv.split(X, y, groups)))
    640 
    641         # if one choose to see train score, "out" will contain train score info

/home/ilmira/.conda/envs/readmision/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.pyc in __call__(self, iterable)
    787                 # consumption.
    788                 self._iterating = False
--> 789             self.retrieve()
    790             # Make sure that we get a last message telling us we are done
    791             elapsed_time = time.time() - self._start_time

/home/ilmira/.conda/envs/readmision/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.pyc in retrieve(self)
    697             try:
    698                 if getattr(self._backend, 'supports_timeout', False):
--> 699                     self._output.extend(job.get(timeout=self.timeout))
    700                 else:
    701                     self._output.extend(job.get())

/home/ilmira/.conda/envs/readmision/lib/python2.7/multiprocessing/pool.pyc in get(self, timeout)
    559 
    560     def get(self, timeout=None):
--> 561         self.wait(timeout)
    562         if not self._ready:
    563             raise TimeoutError

/home/ilmira/.conda/envs/readmision/lib/python2.7/multiprocessing/pool.pyc in wait(self, timeout)
    554         try:
    555             if not self._ready:
--> 556                 self._cond.wait(timeout)
    557         finally:
    558             self._cond.release()

/home/ilmira/.conda/envs/readmision/lib/python2.7/threading.pyc in wait(self, timeout)
    338         try:    # restore state no matter what (e.g., KeyboardInterrupt)
    339             if timeout is None:
--> 340                 waiter.acquire()
    341                 if __debug__:
    342                     self._note("%s.wait(): got it", self)

KeyboardInterrupt: 

In [18]:

    

train_voting(train_preds, y_train)
train_voting(test_preds, y_test)


    

    




Evaluation: CV
*******************

TEST AUC: 0.910
TEST sensitivity: 0.850945494994
TEST Specificity: 0.969501466276

TEST f1 (weighted): 0.954
TEST Precision [c=0,1]: [ 0.97736295  0.80781415]
TEST Recall [c=0,1]: [ 0.96950147  0.85094549]
Confussion matrix:
         | PRED
REAL-->  v 
[[11571   364]
 [  268  1530]]

Evaluation: CV
*******************

TEST AUC: 0.523
TEST sensitivity: 0.0972354623451
TEST Specificity: 0.949228007181

TEST f1 (weighted): 0.809
TEST Precision [c=0,1]: [ 0.87466914  0.22392975]
TEST Recall [c=0,1]: [ 0.94922801  0.09723546]
Confussion matrix:
         | PRED
REAL-->  v 
[[26436  1414]
 [ 3788   408]]

In [44]:

    

#print len(train_preds[:6])
stacker = train_stacker(train_preds[:6], y_train)


    

    




Fitting 5 folds for each of 21 candidates, totalling 105 fits






    




[Parallel(n_jobs=-1)]: Done  18 tasks      | elapsed:    3.7s
[Parallel(n_jobs=-1)]: Done 105 out of 105 | elapsed:   26.1s finished






    




STACKER train-cv evaluation:
*****************************
Learning metric: recall_1
Best parameters: [150, 2, 'balanced']
Best score: 0.813688292549
BEST f1_weighted: 0.792214881633
BEST precision_1: 0.322098029974
BEST precision_0: 0.960803353265
BEST recall_0: 0.752743163799
BEST recall_1: 0.813688292549
BEST spec: 0.752743163799
BEST roc_auc: 0.828442603914

In [45]:

    

stacker.best_estimator_.feature_importances_


    

    
Out[45]:





array([ 0.01026769,  0.06199544,  0.276107  ,  0.05591211,  0.13716804,
        0.45854971])

In [47]:

    

evaluate_stacker(stacker, test_preds[:6], y_test)


    

    




Evaluation: test
*******************

TEST AUC: 0.563
TEST sensitivity: 0.482125834128
TEST Specificity: 0.644272890485

TEST f1 (weighted): 0.683
TEST Precision [c=0,1]: [ 0.89197654  0.16957251]
TEST Recall [c=0,1]: [ 0.64427289  0.48212583]
Confussion matrix:
         | PRED
REAL-->  v 
[[17943  9907]
 [ 2173  2023]]

In [39]:

    

res = pd.DataFrame(np.array(models).reshape(-1,35), columns=
                              ["exp", "name",
                               "size_tr","fs","sm","cls","metric","params",
                               "tr_sens","tr_spec","tr_auc",
                               "tr_prec","tr_rec","tr_f1",
                               "cv_sens_mean","cv_sens_std","cv_spec_mean","cv_spec_std","cv_auc_mean","cv_auc_std",
                               "cv_prec_mean","cv_prec_std","cv_rec_mean","cv_rec_std",
                               "cv_f1_mean","cv_f1_std",
                               "test_sens","test_spec","test_auc",
                               "test_rec","test_prec","test_f1",
                               "cm_test",
                               "time","pipeline"])
res[["name","size_tr","cls","metric","params","tr_sens","tr_spec","tr_auc",
    "tr_f1","cv_sens_mean","cv_spec_mean","cv_auc_mean","cv_f1_mean",
    "test_sens","test_spec","test_auc","test_f1"]].sort_values("cv_sens_mean", ascending=False)


    

    
Out[39]:







  

    

      

      
name
      
size_tr
      
cls
      
metric
      
params
      
tr_sens
      
tr_spec
      
tr_auc
      
tr_f1
      
cv_sens_mean
      
cv_spec_mean
      
cv_auc_mean
      
cv_f1_mean
      
test_sens
      
test_spec
      
test_auc
      
test_f1
    
  
  

    

      
0
      
patient_filter
      
1.00
      
rf
      
precision
      
[entropy, 4, 400]
      
0.71
      
0.41
      
0.56
      
0.52
      
0.64
      
0.46
      
0.56
      
0.55
      
0.70
      
0.40
      
0.55
      
0.51
    
    

      
4
      
diagnosis_filter
      
1.00
      
rf
      
precision
      
[gini, 8, 200]
      
0.68
      
0.60
      
0.64
      
0.67
      
0.53
      
0.61
      
0.60
      
0.66
      
0.53
      
0.59
      
0.56
      
0.65
    
    

      
1
      
admision_discharge_filter
      
1.00
      
rf
      
precision
      
[entropy, 8, 200]
      
0.63
      
0.63
      
0.63
      
0.69
      
0.50
      
0.66
      
0.62
      
0.70
      
0.54
      
0.63
      
0.58
      
0.68
    
    

      
3
      
Visits_filter
      
1.00
      
rf
      
precision
      
[entropy, 4, 200]
      
0.41
      
0.78
      
0.60
      
0.76
      
0.44
      
0.74
      
0.63
      
0.74
      
0.37
      
0.78
      
0.58
      
0.76
    
    

      
2
      
hospital_filter
      
1.00
      
rf
      
precision
      
[gini, 8, 400]
      
0.68
      
0.69
      
0.69
      
0.74
      
0.42
      
0.69
      
0.58
      
0.71
      
0.45
      
0.67
      
0.56
      
0.70
    
    

      
7
      
none_filter
      
1.00
      
rf
      
precision
      
[entropy, 4, 500]
      
0.47
      
0.75
      
0.61
      
0.75
      
0.42
      
0.76
      
0.64
      
0.75
      
0.42
      
0.74
      
0.58
      
0.74
    
    

      
5
      
medicines_filter
      
1.00
      
rf
      
precision
      
[gini, 12, 400]
      
0.76
      
0.74
      
0.75
      
0.78
      
0.37
      
0.74
      
0.58
      
0.73
      
0.38
      
0.71
      
0.55
      
0.72
    
    

      
6
      
extra_filter
      
1.00
      
rf
      
precision
      
[gini, None, 350]
      
1.00
      
1.00
      
1.00
      
1.00
      
0.01
      
1.00
      
0.66
      
0.81
      
0.01
      
1.00
      
0.50
      
0.81
    
  

In [ ]: