notebook.community

Edit and run 





In [1]:



    


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn import cross_validation
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import RandomForestClassifier
from sklearn.calibration import CalibratedClassifierCV
from sklearn.svm import SVC
from sklearn import grid_search
from sklearn.metrics import log_loss
from sklearn.cross_validation import train_test_split
import xgboost as xgb

from sklearn.metrics import roc_auc_score, log_loss
from sklearn import metrics



    


















    






/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.
  "This module will be removed in 0.20.", DeprecationWarning)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/grid_search.py:43: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.
  DeprecationWarning)

















In [2]:



    


train = pd.read_csv('train.csv')
#train.drop('activity_id', axis=1, inplace = True) 
#train.drop_duplicates(inplace = True)

test = pd.read_csv('test.csv')
example = pd.read_csv('BloodDonationSubmissionFormat.csv')



    











In [3]:



    


train.head()



    


















    
Out[3]:










  
    

      
      Unnamed: 0
      Months since Last Donation
      Number of Donations
      Total Volume Donated (c.c.)
      Months since First Donation
      Made Donation in March 2007
    

  
  
    

      0
      619
      2
      50
      12500
      98
      1
    

    

      1
      664
      0
      13
      3250
      28
      1
    

    

      2
      441
      1
      16
      4000
      35
      1
    

    

      3
      160
      2
      20
      5000
      45
      1
    

    

      4
      358
      1
      24
      6000
      77
      0
    

  




















In [4]:



    


test.head()



    


















    
Out[4]:










  
    

      
      Unnamed: 0
      Months since Last Donation
      Number of Donations
      Total Volume Donated (c.c.)
      Months since First Donation
    

  
  
    

      0
      659
      2
      12
      3000
      52
    

    

      1
      276
      21
      7
      1750
      38
    

    

      2
      263
      4
      1
      250
      4
    

    

      3
      303
      11
      11
      2750
      38
    

    

      4
      83
      4
      12
      3000
      34
    

  




















In [5]:



    


test.shape



    


















    
Out[5]:








(200, 5)

















In [6]:



    


len(train.iloc[:,0]) - len(pd.unique(train.iloc[:,0])) # все значения уникальные, дропаем



    


















    
Out[6]:








0

















In [7]:



    


train.drop(u'Unnamed: 0', axis=1, inplace = True)



    











In [8]:



    


train.columns



    


















    
Out[8]:








Index([u'Months since Last Donation', u'Number of Donations',
       u'Total Volume Donated (c.c.)', u'Months since First Donation',
       u'Made Donation in March 2007'],
      dtype='object')

















In [9]:



    


X = train.drop(u'Made Donation in March 2007', axis=1)
y = train.iloc[:,-1]

id_test = test.iloc[:,0]
df_test = test.drop(u'Unnamed: 0', axis=1, inplace = True)

X_fit, X_eval, y_fit, y_eval= train_test_split(
    X, y, test_size=0.15, random_state=1
)



    











In [10]:



    


rf = RandomForestClassifier(n_estimators=1000, n_jobs=4, max_depth =7)
rf.fit(X_fit, y_fit)
print 'RandomForestClassifier ', (log_loss(y_eval, rf.predict_proba(X_eval)))

submission = pd.DataFrame({"":id_test, "Made Donation in March 2007":rf.predict_proba(test)[:,1]})
submission.to_csv("submission_rf_5.csv", index=False)



    


















    






RandomForestClassifier  0.430018365833

















In [11]:



    


lr = LogisticRegression()

CVscores = cross_validation.cross_val_score(lr, X, y, scoring='log_loss', cv=5)

# search for the regularization parameter
Cs = 10**np.linspace(-4, 4, num=15)
grid = {'C': Cs}
gridsearch = grid_search.GridSearchCV(lr, grid, scoring='log_loss', cv=5)
gridsearch.fit(X, y)
gridscores = [-x.mean_validation_score for x in gridsearch.grid_scores_]

C = Cs[np.argmin(gridscores)]

# refit the model with the new regularization parameter
lrCV = LogisticRegression(C=C)
lrCV.fit(X_fit, y_fit)

# write log to file
logloss_train = log_loss(y_fit, lrCV.predict_proba(X_fit))
logloss_val = log_loss(y_eval, lrCV.predict_proba(X_eval))

print 'logloss_train: ', logloss_train
print 'logloss_val: ', logloss_val



    


















    






/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)










    






logloss_train:  0.498092790347
logloss_val:  0.462576233776










    






/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)
/home/analyst/anaconda2/lib/python2.7/site-packages/sklearn/metrics/scorer.py:127: DeprecationWarning: Scoring method log_loss was renamed to neg_log_loss in version 0.18 and will be removed in 0.20.
  sample_weight=sample_weight)

















In [12]:



    


clf = xgb.XGBClassifier(missing=np.nan, max_depth=3, 
                        n_estimators=1500, learning_rate=0.005, gamma =0.3, min_child_weight = 3,
                        subsample=0.9, colsample_bytree=0.8, seed=2100,objective= 'binary:logistic')
# 'colsample_bytree': 0.8,
#   'gamma': 0.3,
#   'max_depth': 4,
#   'min_child_weight': 3,
#   'subsample': 0.9


# fitting
clf.fit(X_fit, y_fit, early_stopping_rounds=150,  eval_metric="logloss", eval_set=[(X_eval, y_eval)])

# scores
from  sklearn.metrics import log_loss
log_train = log_loss(y_fit, clf.predict_proba(X_fit)[:,1])
log_valid = log_loss(y_eval, clf.predict_proba(X_eval)[:,1])


print('\n-----------------------')
print('  logloss train: %.5f'%log_train)
print('  logloss valid: %.5f'%log_valid)
print('-----------------------')

print('\nModel parameters...')
print(clf.get_params())

#print y_pred
# y_pred= clf.predict_proba(test)[:,1]

# submission = pd.DataFrame({"t_id":id_test, "probability":y_pred})
# submission.to_csv("submission_xgboost.csv", index=False)



    


















    






Will train until validation_0 error hasn't decreased in 150 rounds.
[0]	validation_0-logloss:0.691433
[1]	validation_0-logloss:0.689240
[2]	validation_0-logloss:0.687480
[3]	validation_0-logloss:0.685347
[4]	validation_0-logloss:0.683568
[5]	validation_0-logloss:0.681769
[6]	validation_0-logloss:0.679714
[7]	validation_0-logloss:0.677558
[8]	validation_0-logloss:0.675443
[9]	validation_0-logloss:0.673438
[10]	validation_0-logloss:0.671399
[11]	validation_0-logloss:0.669271
[12]	validation_0-logloss:0.667280
[13]	validation_0-logloss:0.665649
[14]	validation_0-logloss:0.663594
[15]	validation_0-logloss:0.661664
[16]	validation_0-logloss:0.659694
[17]	validation_0-logloss:0.658079
[18]	validation_0-logloss:0.656236
[19]	validation_0-logloss:0.654683
[20]	validation_0-logloss:0.652834
[21]	validation_0-logloss:0.650884
[22]	validation_0-logloss:0.649062
[23]	validation_0-logloss:0.647371
[24]	validation_0-logloss:0.645618
[25]	validation_0-logloss:0.643871
[26]	validation_0-logloss:0.642088
[27]	validation_0-logloss:0.640371
[28]	validation_0-logloss:0.638972
[29]	validation_0-logloss:0.637549
[30]	validation_0-logloss:0.635836
[31]	validation_0-logloss:0.634100
[32]	validation_0-logloss:0.632578
[33]	validation_0-logloss:0.630899
[34]	validation_0-logloss:0.629506
[35]	validation_0-logloss:0.628040
[36]	validation_0-logloss:0.626443
[37]	validation_0-logloss:0.625114
[38]	validation_0-logloss:0.623419
[39]	validation_0-logloss:0.621964
[40]	validation_0-logloss:0.620460
[41]	validation_0-logloss:0.618871
[42]	validation_0-logloss:0.617273
[43]	validation_0-logloss:0.615722
[44]	validation_0-logloss:0.614163
[45]	validation_0-logloss:0.612877
[46]	validation_0-logloss:0.611343
[47]	validation_0-logloss:0.609798
[48]	validation_0-logloss:0.608325
[49]	validation_0-logloss:0.606743
[50]	validation_0-logloss:0.605206
[51]	validation_0-logloss:0.603707
[52]	validation_0-logloss:0.602306
[53]	validation_0-logloss:0.601073
[54]	validation_0-logloss:0.599693
[55]	validation_0-logloss:0.598510
[56]	validation_0-logloss:0.597065
[57]	validation_0-logloss:0.595668
[58]	validation_0-logloss:0.594224
[59]	validation_0-logloss:0.592830
[60]	validation_0-logloss:0.591380
[61]	validation_0-logloss:0.590140
[62]	validation_0-logloss:0.588868
[63]	validation_0-logloss:0.587558
[64]	validation_0-logloss:0.586414
[65]	validation_0-logloss:0.585093
[66]	validation_0-logloss:0.583736
[67]	validation_0-logloss:0.582417
[68]	validation_0-logloss:0.581156
[69]	validation_0-logloss:0.579855
[70]	validation_0-logloss:0.578600
[71]	validation_0-logloss:0.577591
[72]	validation_0-logloss:0.576347
[73]	validation_0-logloss:0.575138
[74]	validation_0-logloss:0.574076
[75]	validation_0-logloss:0.572850
[76]	validation_0-logloss:0.571624
[77]	validation_0-logloss:0.570672
[78]	validation_0-logloss:0.569439
[79]	validation_0-logloss:0.568218
[80]	validation_0-logloss:0.567197
[81]	validation_0-logloss:0.566175
[82]	validation_0-logloss:0.565263
[83]	validation_0-logloss:0.564059
[84]	validation_0-logloss:0.563193
[85]	validation_0-logloss:0.562028
[86]	validation_0-logloss:0.560997
[87]	validation_0-logloss:0.560032
[88]	validation_0-logloss:0.558958
[89]	validation_0-logloss:0.558029
[90]	validation_0-logloss:0.556935
[91]	validation_0-logloss:0.555902
[92]	validation_0-logloss:0.554812
[93]	validation_0-logloss:0.553751
[94]	validation_0-logloss:0.552773
[95]	validation_0-logloss:0.552064
[96]	validation_0-logloss:0.550990
[97]	validation_0-logloss:0.549969
[98]	validation_0-logloss:0.548931
[99]	validation_0-logloss:0.547922
[100]	validation_0-logloss:0.546945
[101]	validation_0-logloss:0.546215
[102]	validation_0-logloss:0.545355
[103]	validation_0-logloss:0.544342
[104]	validation_0-logloss:0.543422
[105]	validation_0-logloss:0.542472
[106]	validation_0-logloss:0.541726
[107]	validation_0-logloss:0.540879
[108]	validation_0-logloss:0.539896
[109]	validation_0-logloss:0.539011
[110]	validation_0-logloss:0.538116
[111]	validation_0-logloss:0.537092
[112]	validation_0-logloss:0.536180
[113]	validation_0-logloss:0.535263
[114]	validation_0-logloss:0.534349
[115]	validation_0-logloss:0.533607
[116]	validation_0-logloss:0.532718
[117]	validation_0-logloss:0.531829
[118]	validation_0-logloss:0.531038
[119]	validation_0-logloss:0.530191
[120]	validation_0-logloss:0.529322
[121]	validation_0-logloss:0.528449
[122]	validation_0-logloss:0.527672
[123]	validation_0-logloss:0.526878
[124]	validation_0-logloss:0.525949
[125]	validation_0-logloss:0.525149
[126]	validation_0-logloss:0.524363
[127]	validation_0-logloss:0.523574
[128]	validation_0-logloss:0.522804
[129]	validation_0-logloss:0.521996
[130]	validation_0-logloss:0.521288
[131]	validation_0-logloss:0.520547
[132]	validation_0-logloss:0.519695
[133]	validation_0-logloss:0.518943
[134]	validation_0-logloss:0.518132
[135]	validation_0-logloss:0.517324
[136]	validation_0-logloss:0.516589
[137]	validation_0-logloss:0.515946
[138]	validation_0-logloss:0.515306
[139]	validation_0-logloss:0.514630
[140]	validation_0-logloss:0.513964
[141]	validation_0-logloss:0.513204
[142]	validation_0-logloss:0.512497
[143]	validation_0-logloss:0.511774
[144]	validation_0-logloss:0.511082
[145]	validation_0-logloss:0.510434
[146]	validation_0-logloss:0.509698
[147]	validation_0-logloss:0.509096
[148]	validation_0-logloss:0.508366
[149]	validation_0-logloss:0.507764
[150]	validation_0-logloss:0.507181
[151]	validation_0-logloss:0.506545
[152]	validation_0-logloss:0.505925
[153]	validation_0-logloss:0.505195
[154]	validation_0-logloss:0.504570
[155]	validation_0-logloss:0.503853
[156]	validation_0-logloss:0.503231
[157]	validation_0-logloss:0.502608
[158]	validation_0-logloss:0.501937
[159]	validation_0-logloss:0.501315
[160]	validation_0-logloss:0.500727
[161]	validation_0-logloss:0.500085
[162]	validation_0-logloss:0.499573
[163]	validation_0-logloss:0.499091
[164]	validation_0-logloss:0.498559
[165]	validation_0-logloss:0.498042
[166]	validation_0-logloss:0.497430
[167]	validation_0-logloss:0.496843
[168]	validation_0-logloss:0.496227
[169]	validation_0-logloss:0.495637
[170]	validation_0-logloss:0.495057
[171]	validation_0-logloss:0.494460
[172]	validation_0-logloss:0.493889
[173]	validation_0-logloss:0.493257
[174]	validation_0-logloss:0.492735
[175]	validation_0-logloss:0.492302
[176]	validation_0-logloss:0.491849
[177]	validation_0-logloss:0.491397
[178]	validation_0-logloss:0.490867
[179]	validation_0-logloss:0.490297
[180]	validation_0-logloss:0.489705
[181]	validation_0-logloss:0.489164
[182]	validation_0-logloss:0.488551
[183]	validation_0-logloss:0.488076
[184]	validation_0-logloss:0.487492
[185]	validation_0-logloss:0.486955
[186]	validation_0-logloss:0.486481
[187]	validation_0-logloss:0.486039
[188]	validation_0-logloss:0.485548
[189]	validation_0-logloss:0.484955
[190]	validation_0-logloss:0.484428
[191]	validation_0-logloss:0.483897
[192]	validation_0-logloss:0.483509
[193]	validation_0-logloss:0.483073
[194]	validation_0-logloss:0.482693
[195]	validation_0-logloss:0.482230
[196]	validation_0-logloss:0.481669
[197]	validation_0-logloss:0.481254
[198]	validation_0-logloss:0.480775
[199]	validation_0-logloss:0.480354
[200]	validation_0-logloss:0.479959
[201]	validation_0-logloss:0.479513
[202]	validation_0-logloss:0.479088
[203]	validation_0-logloss:0.478664
[204]	validation_0-logloss:0.478288
[205]	validation_0-logloss:0.477832
[206]	validation_0-logloss:0.477421
[207]	validation_0-logloss:0.476910
[208]	validation_0-logloss:0.476493
[209]	validation_0-logloss:0.475954
[210]	validation_0-logloss:0.475502
[211]	validation_0-logloss:0.474960
[212]	validation_0-logloss:0.474516
[213]	validation_0-logloss:0.474091
[214]	validation_0-logloss:0.473725
[215]	validation_0-logloss:0.473272
[216]	validation_0-logloss:0.472796
[217]	validation_0-logloss:0.472392
[218]	validation_0-logloss:0.472043
[219]	validation_0-logloss:0.471680
[220]	validation_0-logloss:0.471346
[221]	validation_0-logloss:0.471060
[222]	validation_0-logloss:0.470603
[223]	validation_0-logloss:0.470237
[224]	validation_0-logloss:0.469859
[225]	validation_0-logloss:0.469539
[226]	validation_0-logloss:0.469208
[227]	validation_0-logloss:0.468812
[228]	validation_0-logloss:0.468396
[229]	validation_0-logloss:0.468021
[230]	validation_0-logloss:0.467550
[231]	validation_0-logloss:0.467279
[232]	validation_0-logloss:0.466974
[233]	validation_0-logloss:0.466560
[234]	validation_0-logloss:0.466126
[235]	validation_0-logloss:0.465656
[236]	validation_0-logloss:0.465369
[237]	validation_0-logloss:0.465145
[238]	validation_0-logloss:0.464626
[239]	validation_0-logloss:0.464266
[240]	validation_0-logloss:0.463928
[241]	validation_0-logloss:0.463625
[242]	validation_0-logloss:0.463306
[243]	validation_0-logloss:0.462862
[244]	validation_0-logloss:0.462461
[245]	validation_0-logloss:0.462106
[246]	validation_0-logloss:0.461835
[247]	validation_0-logloss:0.461601
[248]	validation_0-logloss:0.461301
[249]	validation_0-logloss:0.460945
[250]	validation_0-logloss:0.460670
[251]	validation_0-logloss:0.460416
[252]	validation_0-logloss:0.460094
[253]	validation_0-logloss:0.459751
[254]	validation_0-logloss:0.459489
[255]	validation_0-logloss:0.459157
[256]	validation_0-logloss:0.458864
[257]	validation_0-logloss:0.458475
[258]	validation_0-logloss:0.458113
[259]	validation_0-logloss:0.457857
[260]	validation_0-logloss:0.457561
[261]	validation_0-logloss:0.457191
[262]	validation_0-logloss:0.456937
[263]	validation_0-logloss:0.456657
[264]	validation_0-logloss:0.456290
[265]	validation_0-logloss:0.455874
[266]	validation_0-logloss:0.455587
[267]	validation_0-logloss:0.455249
[268]	validation_0-logloss:0.454918
[269]	validation_0-logloss:0.454704
[270]	validation_0-logloss:0.454432
[271]	validation_0-logloss:0.454126
[272]	validation_0-logloss:0.453874
[273]	validation_0-logloss:0.453592
[274]	validation_0-logloss:0.453262
[275]	validation_0-logloss:0.453010
[276]	validation_0-logloss:0.452756
[277]	validation_0-logloss:0.452446
[278]	validation_0-logloss:0.452108
[279]	validation_0-logloss:0.451785
[280]	validation_0-logloss:0.451508
[281]	validation_0-logloss:0.451238
[282]	validation_0-logloss:0.450868
[283]	validation_0-logloss:0.450539
[284]	validation_0-logloss:0.450339
[285]	validation_0-logloss:0.450109
[286]	validation_0-logloss:0.449794
[287]	validation_0-logloss:0.449540
[288]	validation_0-logloss:0.449281
[289]	validation_0-logloss:0.449085
[290]	validation_0-logloss:0.448792
[291]	validation_0-logloss:0.448554
[292]	validation_0-logloss:0.448314
[293]	validation_0-logloss:0.448056
[294]	validation_0-logloss:0.447731
[295]	validation_0-logloss:0.447488
[296]	validation_0-logloss:0.447218
[297]	validation_0-logloss:0.446891
[298]	validation_0-logloss:0.446631
[299]	validation_0-logloss:0.446413
[300]	validation_0-logloss:0.446100
[301]	validation_0-logloss:0.445835
[302]	validation_0-logloss:0.445610
[303]	validation_0-logloss:0.445393
[304]	validation_0-logloss:0.445104
[305]	validation_0-logloss:0.444893
[306]	validation_0-logloss:0.444651
[307]	validation_0-logloss:0.444392
[308]	validation_0-logloss:0.444166
[309]	validation_0-logloss:0.443853
[310]	validation_0-logloss:0.443648
[311]	validation_0-logloss:0.443383
[312]	validation_0-logloss:0.443272
[313]	validation_0-logloss:0.443081
[314]	validation_0-logloss:0.442818
[315]	validation_0-logloss:0.442642
[316]	validation_0-logloss:0.442342
[317]	validation_0-logloss:0.442165
[318]	validation_0-logloss:0.441931
[319]	validation_0-logloss:0.441612
[320]	validation_0-logloss:0.441397
[321]	validation_0-logloss:0.441185
[322]	validation_0-logloss:0.440956
[323]	validation_0-logloss:0.440694
[324]	validation_0-logloss:0.440509
[325]	validation_0-logloss:0.440358
[326]	validation_0-logloss:0.440162
[327]	validation_0-logloss:0.439944
[328]	validation_0-logloss:0.439753
[329]	validation_0-logloss:0.439586
[330]	validation_0-logloss:0.439342
[331]	validation_0-logloss:0.439134
[332]	validation_0-logloss:0.439034
[333]	validation_0-logloss:0.438812
[334]	validation_0-logloss:0.438627
[335]	validation_0-logloss:0.438415
[336]	validation_0-logloss:0.438221
[337]	validation_0-logloss:0.437971
[338]	validation_0-logloss:0.437722
[339]	validation_0-logloss:0.437545
[340]	validation_0-logloss:0.437369
[341]	validation_0-logloss:0.437145
[342]	validation_0-logloss:0.436939
[343]	validation_0-logloss:0.436737
[344]	validation_0-logloss:0.436547
[345]	validation_0-logloss:0.436415
[346]	validation_0-logloss:0.436192
[347]	validation_0-logloss:0.435960
[348]	validation_0-logloss:0.435816
[349]	validation_0-logloss:0.435737
[350]	validation_0-logloss:0.435480
[351]	validation_0-logloss:0.435398
[352]	validation_0-logloss:0.435213
[353]	validation_0-logloss:0.435033
[354]	validation_0-logloss:0.434827
[355]	validation_0-logloss:0.434670
[356]	validation_0-logloss:0.434543
[357]	validation_0-logloss:0.434320
[358]	validation_0-logloss:0.434059
[359]	validation_0-logloss:0.433919
[360]	validation_0-logloss:0.433827
[361]	validation_0-logloss:0.433650
[362]	validation_0-logloss:0.433474
[363]	validation_0-logloss:0.433286
[364]	validation_0-logloss:0.433124
[365]	validation_0-logloss:0.433054
[366]	validation_0-logloss:0.432878
[367]	validation_0-logloss:0.432755
[368]	validation_0-logloss:0.432630
[369]	validation_0-logloss:0.432424
[370]	validation_0-logloss:0.432267
[371]	validation_0-logloss:0.432084
[372]	validation_0-logloss:0.431956
[373]	validation_0-logloss:0.431751
[374]	validation_0-logloss:0.431660
[375]	validation_0-logloss:0.431533
[376]	validation_0-logloss:0.431330
[377]	validation_0-logloss:0.431146
[378]	validation_0-logloss:0.430994
[379]	validation_0-logloss:0.430791
[380]	validation_0-logloss:0.430611
[381]	validation_0-logloss:0.430503
[382]	validation_0-logloss:0.430343
[383]	validation_0-logloss:0.430227
[384]	validation_0-logloss:0.430122
[385]	validation_0-logloss:0.429972
[386]	validation_0-logloss:0.429768
[387]	validation_0-logloss:0.429586
[388]	validation_0-logloss:0.429464
[389]	validation_0-logloss:0.429384
[390]	validation_0-logloss:0.429225
[391]	validation_0-logloss:0.429103
[392]	validation_0-logloss:0.428946
[393]	validation_0-logloss:0.428758
[394]	validation_0-logloss:0.428607
[395]	validation_0-logloss:0.428483
[396]	validation_0-logloss:0.428363
[397]	validation_0-logloss:0.428198
[398]	validation_0-logloss:0.427991
[399]	validation_0-logloss:0.427815
[400]	validation_0-logloss:0.427734
[401]	validation_0-logloss:0.427590
[402]	validation_0-logloss:0.427448
[403]	validation_0-logloss:0.427240
[404]	validation_0-logloss:0.427135
[405]	validation_0-logloss:0.426969
[406]	validation_0-logloss:0.426869
[407]	validation_0-logloss:0.426775
[408]	validation_0-logloss:0.426736
[409]	validation_0-logloss:0.426605
[410]	validation_0-logloss:0.426464
[411]	validation_0-logloss:0.426345
[412]	validation_0-logloss:0.426155
[413]	validation_0-logloss:0.425935
[414]	validation_0-logloss:0.425830
[415]	validation_0-logloss:0.425703
[416]	validation_0-logloss:0.425568
[417]	validation_0-logloss:0.425484
[418]	validation_0-logloss:0.425424
[419]	validation_0-logloss:0.425289
[420]	validation_0-logloss:0.425142
[421]	validation_0-logloss:0.424979
[422]	validation_0-logloss:0.424846
[423]	validation_0-logloss:0.424679
[424]	validation_0-logloss:0.424555
[425]	validation_0-logloss:0.424427
[426]	validation_0-logloss:0.424458
[427]	validation_0-logloss:0.424348
[428]	validation_0-logloss:0.424235
[429]	validation_0-logloss:0.424103
[430]	validation_0-logloss:0.423880
[431]	validation_0-logloss:0.423842
[432]	validation_0-logloss:0.423729
[433]	validation_0-logloss:0.423689
[434]	validation_0-logloss:0.423622
[435]	validation_0-logloss:0.423483
[436]	validation_0-logloss:0.423410
[437]	validation_0-logloss:0.423235
[438]	validation_0-logloss:0.423136
[439]	validation_0-logloss:0.423022
[440]	validation_0-logloss:0.422901
[441]	validation_0-logloss:0.422906
[442]	validation_0-logloss:0.422766
[443]	validation_0-logloss:0.422707
[444]	validation_0-logloss:0.422612
[445]	validation_0-logloss:0.422498
[446]	validation_0-logloss:0.422413
[447]	validation_0-logloss:0.422204
[448]	validation_0-logloss:0.422097
[449]	validation_0-logloss:0.422010
[450]	validation_0-logloss:0.421911
[451]	validation_0-logloss:0.421823
[452]	validation_0-logloss:0.421728
[453]	validation_0-logloss:0.421638
[454]	validation_0-logloss:0.421542
[455]	validation_0-logloss:0.421450
[456]	validation_0-logloss:0.421286
[457]	validation_0-logloss:0.421127
[458]	validation_0-logloss:0.421100
[459]	validation_0-logloss:0.421030
[460]	validation_0-logloss:0.420916
[461]	validation_0-logloss:0.420866
[462]	validation_0-logloss:0.420766
[463]	validation_0-logloss:0.420709
[464]	validation_0-logloss:0.420643
[465]	validation_0-logloss:0.420545
[466]	validation_0-logloss:0.420456
[467]	validation_0-logloss:0.420439
[468]	validation_0-logloss:0.420327
[469]	validation_0-logloss:0.420216
[470]	validation_0-logloss:0.420071
[471]	validation_0-logloss:0.420019
[472]	validation_0-logloss:0.419945
[473]	validation_0-logloss:0.420001
[474]	validation_0-logloss:0.419889
[475]	validation_0-logloss:0.419758
[476]	validation_0-logloss:0.419649
[477]	validation_0-logloss:0.419572
[478]	validation_0-logloss:0.419444
[479]	validation_0-logloss:0.419354
[480]	validation_0-logloss:0.419186
[481]	validation_0-logloss:0.419047
[482]	validation_0-logloss:0.419009
[483]	validation_0-logloss:0.419017
[484]	validation_0-logloss:0.418913
[485]	validation_0-logloss:0.418774
[486]	validation_0-logloss:0.418653
[487]	validation_0-logloss:0.418540
[488]	validation_0-logloss:0.418466
[489]	validation_0-logloss:0.418377
[490]	validation_0-logloss:0.418371
[491]	validation_0-logloss:0.418278
[492]	validation_0-logloss:0.418186
[493]	validation_0-logloss:0.418162
[494]	validation_0-logloss:0.418131
[495]	validation_0-logloss:0.417995
[496]	validation_0-logloss:0.417918
[497]	validation_0-logloss:0.417851
[498]	validation_0-logloss:0.417754
[499]	validation_0-logloss:0.417709
[500]	validation_0-logloss:0.417722
[501]	validation_0-logloss:0.417722
[502]	validation_0-logloss:0.417658
[503]	validation_0-logloss:0.417510
[504]	validation_0-logloss:0.417441
[505]	validation_0-logloss:0.417249
[506]	validation_0-logloss:0.417261
[507]	validation_0-logloss:0.417162
[508]	validation_0-logloss:0.417128
[509]	validation_0-logloss:0.417056
[510]	validation_0-logloss:0.416950
[511]	validation_0-logloss:0.416870
[512]	validation_0-logloss:0.416793
[513]	validation_0-logloss:0.416744
[514]	validation_0-logloss:0.416698
[515]	validation_0-logloss:0.416655
[516]	validation_0-logloss:0.416535
[517]	validation_0-logloss:0.416390
[518]	validation_0-logloss:0.416361
[519]	validation_0-logloss:0.416410
[520]	validation_0-logloss:0.416336
[521]	validation_0-logloss:0.416260
[522]	validation_0-logloss:0.416136
[523]	validation_0-logloss:0.416093
[524]	validation_0-logloss:0.416084
[525]	validation_0-logloss:0.416063
[526]	validation_0-logloss:0.415940
[527]	validation_0-logloss:0.415929
[528]	validation_0-logloss:0.415846
[529]	validation_0-logloss:0.415779
[530]	validation_0-logloss:0.415636
[531]	validation_0-logloss:0.415583
[532]	validation_0-logloss:0.415627
[533]	validation_0-logloss:0.415521
[534]	validation_0-logloss:0.415420
[535]	validation_0-logloss:0.415405
[536]	validation_0-logloss:0.415429
[537]	validation_0-logloss:0.415347
[538]	validation_0-logloss:0.415352
[539]	validation_0-logloss:0.415368
[540]	validation_0-logloss:0.415293
[541]	validation_0-logloss:0.415235
[542]	validation_0-logloss:0.415192
[543]	validation_0-logloss:0.415143
[544]	validation_0-logloss:0.415115
[545]	validation_0-logloss:0.415080
[546]	validation_0-logloss:0.415053
[547]	validation_0-logloss:0.414947
[548]	validation_0-logloss:0.414930
[549]	validation_0-logloss:0.414879
[550]	validation_0-logloss:0.414802
[551]	validation_0-logloss:0.414734
[552]	validation_0-logloss:0.414670
[553]	validation_0-logloss:0.414594
[554]	validation_0-logloss:0.414522
[555]	validation_0-logloss:0.414443
[556]	validation_0-logloss:0.414336
[557]	validation_0-logloss:0.414257
[558]	validation_0-logloss:0.414211
[559]	validation_0-logloss:0.414135
[560]	validation_0-logloss:0.414146
[561]	validation_0-logloss:0.414136
[562]	validation_0-logloss:0.414114
[563]	validation_0-logloss:0.414090
[564]	validation_0-logloss:0.414013
[565]	validation_0-logloss:0.414049
[566]	validation_0-logloss:0.413974
[567]	validation_0-logloss:0.413899
[568]	validation_0-logloss:0.413847
[569]	validation_0-logloss:0.413836
[570]	validation_0-logloss:0.413775
[571]	validation_0-logloss:0.413738
[572]	validation_0-logloss:0.413716
[573]	validation_0-logloss:0.413633
[574]	validation_0-logloss:0.413531
[575]	validation_0-logloss:0.413386
[576]	validation_0-logloss:0.413317
[577]	validation_0-logloss:0.413221
[578]	validation_0-logloss:0.413139
[579]	validation_0-logloss:0.413101
[580]	validation_0-logloss:0.413123
[581]	validation_0-logloss:0.413096
[582]	validation_0-logloss:0.412988
[583]	validation_0-logloss:0.412958
[584]	validation_0-logloss:0.412986
[585]	validation_0-logloss:0.412978
[586]	validation_0-logloss:0.413020
[587]	validation_0-logloss:0.413014
[588]	validation_0-logloss:0.412929
[589]	validation_0-logloss:0.412897
[590]	validation_0-logloss:0.412796
[591]	validation_0-logloss:0.412669
[592]	validation_0-logloss:0.412562
[593]	validation_0-logloss:0.412548
[594]	validation_0-logloss:0.412554
[595]	validation_0-logloss:0.412462
[596]	validation_0-logloss:0.412434
[597]	validation_0-logloss:0.412359
[598]	validation_0-logloss:0.412320
[599]	validation_0-logloss:0.412235
[600]	validation_0-logloss:0.412189
[601]	validation_0-logloss:0.412232
[602]	validation_0-logloss:0.412237
[603]	validation_0-logloss:0.412240
[604]	validation_0-logloss:0.412276
[605]	validation_0-logloss:0.412227
[606]	validation_0-logloss:0.412205
[607]	validation_0-logloss:0.412239
[608]	validation_0-logloss:0.412225
[609]	validation_0-logloss:0.412133
[610]	validation_0-logloss:0.412109
[611]	validation_0-logloss:0.412083
[612]	validation_0-logloss:0.411995
[613]	validation_0-logloss:0.411958
[614]	validation_0-logloss:0.411848
[615]	validation_0-logloss:0.411844
[616]	validation_0-logloss:0.411892
[617]	validation_0-logloss:0.411885
[618]	validation_0-logloss:0.411949
[619]	validation_0-logloss:0.411913
[620]	validation_0-logloss:0.411869
[621]	validation_0-logloss:0.411856
[622]	validation_0-logloss:0.411824
[623]	validation_0-logloss:0.411860
[624]	validation_0-logloss:0.411770
[625]	validation_0-logloss:0.411743
[626]	validation_0-logloss:0.411671
[627]	validation_0-logloss:0.411642
[628]	validation_0-logloss:0.411634
[629]	validation_0-logloss:0.411556
[630]	validation_0-logloss:0.411503
[631]	validation_0-logloss:0.411507
[632]	validation_0-logloss:0.411484
[633]	validation_0-logloss:0.411416
[634]	validation_0-logloss:0.411292
[635]	validation_0-logloss:0.411274
[636]	validation_0-logloss:0.411209
[637]	validation_0-logloss:0.411207
[638]	validation_0-logloss:0.411209
[639]	validation_0-logloss:0.411293
[640]	validation_0-logloss:0.411330
[641]	validation_0-logloss:0.411352
[642]	validation_0-logloss:0.411287
[643]	validation_0-logloss:0.411270
[644]	validation_0-logloss:0.411176
[645]	validation_0-logloss:0.411194
[646]	validation_0-logloss:0.411068
[647]	validation_0-logloss:0.410997
[648]	validation_0-logloss:0.411013
[649]	validation_0-logloss:0.410974
[650]	validation_0-logloss:0.411002
[651]	validation_0-logloss:0.411012
[652]	validation_0-logloss:0.410967
[653]	validation_0-logloss:0.410919
[654]	validation_0-logloss:0.410833
[655]	validation_0-logloss:0.410836
[656]	validation_0-logloss:0.410707
[657]	validation_0-logloss:0.410591
[658]	validation_0-logloss:0.410584
[659]	validation_0-logloss:0.410626
[660]	validation_0-logloss:0.410604
[661]	validation_0-logloss:0.410510
[662]	validation_0-logloss:0.410441
[663]	validation_0-logloss:0.410395
[664]	validation_0-logloss:0.410429
[665]	validation_0-logloss:0.410404
[666]	validation_0-logloss:0.410282
[667]	validation_0-logloss:0.410230
[668]	validation_0-logloss:0.410145
[669]	validation_0-logloss:0.410136
[670]	validation_0-logloss:0.410114
[671]	validation_0-logloss:0.410054
[672]	validation_0-logloss:0.410036
[673]	validation_0-logloss:0.410058
[674]	validation_0-logloss:0.409975
[675]	validation_0-logloss:0.409920
[676]	validation_0-logloss:0.409880
[677]	validation_0-logloss:0.409909
[678]	validation_0-logloss:0.409909
[679]	validation_0-logloss:0.409883
[680]	validation_0-logloss:0.409939
[681]	validation_0-logloss:0.409953
[682]	validation_0-logloss:0.409943
[683]	validation_0-logloss:0.409880
[684]	validation_0-logloss:0.409865
[685]	validation_0-logloss:0.409922
[686]	validation_0-logloss:0.409935
[687]	validation_0-logloss:0.409869
[688]	validation_0-logloss:0.409850
[689]	validation_0-logloss:0.409797
[690]	validation_0-logloss:0.409804
[691]	validation_0-logloss:0.409753
[692]	validation_0-logloss:0.409718
[693]	validation_0-logloss:0.409730
[694]	validation_0-logloss:0.409619
[695]	validation_0-logloss:0.409568
[696]	validation_0-logloss:0.409596
[697]	validation_0-logloss:0.409563
[698]	validation_0-logloss:0.409557
[699]	validation_0-logloss:0.409498
[700]	validation_0-logloss:0.409501
[701]	validation_0-logloss:0.409487
[702]	validation_0-logloss:0.409450
[703]	validation_0-logloss:0.409453
[704]	validation_0-logloss:0.409407
[705]	validation_0-logloss:0.409419
[706]	validation_0-logloss:0.409393
[707]	validation_0-logloss:0.409350
[708]	validation_0-logloss:0.409286
[709]	validation_0-logloss:0.409267
[710]	validation_0-logloss:0.409185
[711]	validation_0-logloss:0.409128
[712]	validation_0-logloss:0.409057
[713]	validation_0-logloss:0.408981
[714]	validation_0-logloss:0.409018
[715]	validation_0-logloss:0.408994
[716]	validation_0-logloss:0.408947
[717]	validation_0-logloss:0.408929
[718]	validation_0-logloss:0.408961
[719]	validation_0-logloss:0.408903
[720]	validation_0-logloss:0.408948
[721]	validation_0-logloss:0.408957
[722]	validation_0-logloss:0.408911
[723]	validation_0-logloss:0.408846
[724]	validation_0-logloss:0.408857
[725]	validation_0-logloss:0.408868
[726]	validation_0-logloss:0.408823
[727]	validation_0-logloss:0.408780
[728]	validation_0-logloss:0.408796
[729]	validation_0-logloss:0.408829
[730]	validation_0-logloss:0.408886
[731]	validation_0-logloss:0.408778
[732]	validation_0-logloss:0.408740
[733]	validation_0-logloss:0.408756
[734]	validation_0-logloss:0.408660
[735]	validation_0-logloss:0.408602
[736]	validation_0-logloss:0.408567
[737]	validation_0-logloss:0.408539
[738]	validation_0-logloss:0.408602
[739]	validation_0-logloss:0.408557
[740]	validation_0-logloss:0.408571
[741]	validation_0-logloss:0.408494
[742]	validation_0-logloss:0.408440
[743]	validation_0-logloss:0.408462
[744]	validation_0-logloss:0.408435
[745]	validation_0-logloss:0.408408
[746]	validation_0-logloss:0.408392
[747]	validation_0-logloss:0.408404
[748]	validation_0-logloss:0.408402
[749]	validation_0-logloss:0.408367
[750]	validation_0-logloss:0.408365
[751]	validation_0-logloss:0.408247










    






-----------------------
  logloss train: 0.43208
  logloss valid: 0.40631
-----------------------

Model parameters...
{'reg_alpha': 0, 'colsample_bytree': 0.8, 'silent': True, 'colsample_bylevel': 1, 'scale_pos_weight': 1, 'learning_rate': 0.005, 'missing': None, 'max_delta_step': 0, 'nthread': -1, 'base_score': 0.5, 'n_estimators': 1500, 'subsample': 0.9, 'reg_lambda': 1, 'seed': 2100, 'min_child_weight': 3, 'objective': 'binary:logistic', 'max_depth': 3, 'gamma': 0.3}










    






[752]	validation_0-logloss:0.408198
[753]	validation_0-logloss:0.408142
[754]	validation_0-logloss:0.408108
[755]	validation_0-logloss:0.408085
[756]	validation_0-logloss:0.408114
[757]	validation_0-logloss:0.408100
[758]	validation_0-logloss:0.408112
[759]	validation_0-logloss:0.408151
[760]	validation_0-logloss:0.408139
[761]	validation_0-logloss:0.408058
[762]	validation_0-logloss:0.408073
[763]	validation_0-logloss:0.408031
[764]	validation_0-logloss:0.407997
[765]	validation_0-logloss:0.407922
[766]	validation_0-logloss:0.407933
[767]	validation_0-logloss:0.407884
[768]	validation_0-logloss:0.407892
[769]	validation_0-logloss:0.407831
[770]	validation_0-logloss:0.407818
[771]	validation_0-logloss:0.407773
[772]	validation_0-logloss:0.407824
[773]	validation_0-logloss:0.407839
[774]	validation_0-logloss:0.407871
[775]	validation_0-logloss:0.407848
[776]	validation_0-logloss:0.407846
[777]	validation_0-logloss:0.407834
[778]	validation_0-logloss:0.407837
[779]	validation_0-logloss:0.407814
[780]	validation_0-logloss:0.407821
[781]	validation_0-logloss:0.407869
[782]	validation_0-logloss:0.407897
[783]	validation_0-logloss:0.407933
[784]	validation_0-logloss:0.407976
[785]	validation_0-logloss:0.407976
[786]	validation_0-logloss:0.407942
[787]	validation_0-logloss:0.407904
[788]	validation_0-logloss:0.407816
[789]	validation_0-logloss:0.407856
[790]	validation_0-logloss:0.407833
[791]	validation_0-logloss:0.407810
[792]	validation_0-logloss:0.407843
[793]	validation_0-logloss:0.407689
[794]	validation_0-logloss:0.407579
[795]	validation_0-logloss:0.407517
[796]	validation_0-logloss:0.407453
[797]	validation_0-logloss:0.407447
[798]	validation_0-logloss:0.407354
[799]	validation_0-logloss:0.407230
[800]	validation_0-logloss:0.407171
[801]	validation_0-logloss:0.407280
[802]	validation_0-logloss:0.407298
[803]	validation_0-logloss:0.407317
[804]	validation_0-logloss:0.407331
[805]	validation_0-logloss:0.407296
[806]	validation_0-logloss:0.407253
[807]	validation_0-logloss:0.407210
[808]	validation_0-logloss:0.407110
[809]	validation_0-logloss:0.407154
[810]	validation_0-logloss:0.407161
[811]	validation_0-logloss:0.407143
[812]	validation_0-logloss:0.407152
[813]	validation_0-logloss:0.407130
[814]	validation_0-logloss:0.407123
[815]	validation_0-logloss:0.407117
[816]	validation_0-logloss:0.407136
[817]	validation_0-logloss:0.407139
[818]	validation_0-logloss:0.407075
[819]	validation_0-logloss:0.407099
[820]	validation_0-logloss:0.407130
[821]	validation_0-logloss:0.407123
[822]	validation_0-logloss:0.407101
[823]	validation_0-logloss:0.407030
[824]	validation_0-logloss:0.407056
[825]	validation_0-logloss:0.407081
[826]	validation_0-logloss:0.407074
[827]	validation_0-logloss:0.407059
[828]	validation_0-logloss:0.407128
[829]	validation_0-logloss:0.407087
[830]	validation_0-logloss:0.407114
[831]	validation_0-logloss:0.407126
[832]	validation_0-logloss:0.407108
[833]	validation_0-logloss:0.407101
[834]	validation_0-logloss:0.407159
[835]	validation_0-logloss:0.407155
[836]	validation_0-logloss:0.407126
[837]	validation_0-logloss:0.407070
[838]	validation_0-logloss:0.407058
[839]	validation_0-logloss:0.407010
[840]	validation_0-logloss:0.407044
[841]	validation_0-logloss:0.407128
[842]	validation_0-logloss:0.407151
[843]	validation_0-logloss:0.407135
[844]	validation_0-logloss:0.407132
[845]	validation_0-logloss:0.407148
[846]	validation_0-logloss:0.407139
[847]	validation_0-logloss:0.407204
[848]	validation_0-logloss:0.407223
[849]	validation_0-logloss:0.407260
[850]	validation_0-logloss:0.407229
[851]	validation_0-logloss:0.407232
[852]	validation_0-logloss:0.407238
[853]	validation_0-logloss:0.407210
[854]	validation_0-logloss:0.407200
[855]	validation_0-logloss:0.407215
[856]	validation_0-logloss:0.407179
[857]	validation_0-logloss:0.407072
[858]	validation_0-logloss:0.407021
[859]	validation_0-logloss:0.407009
[860]	validation_0-logloss:0.407025
[861]	validation_0-logloss:0.407078
[862]	validation_0-logloss:0.407050
[863]	validation_0-logloss:0.406954
[864]	validation_0-logloss:0.406938
[865]	validation_0-logloss:0.406922
[866]	validation_0-logloss:0.406871
[867]	validation_0-logloss:0.406842
[868]	validation_0-logloss:0.406834
[869]	validation_0-logloss:0.406871
[870]	validation_0-logloss:0.406899
[871]	validation_0-logloss:0.406891
[872]	validation_0-logloss:0.406893
[873]	validation_0-logloss:0.406839
[874]	validation_0-logloss:0.406793
[875]	validation_0-logloss:0.406819
[876]	validation_0-logloss:0.406813
[877]	validation_0-logloss:0.406766
[878]	validation_0-logloss:0.406736
[879]	validation_0-logloss:0.406776
[880]	validation_0-logloss:0.406807
[881]	validation_0-logloss:0.406789
[882]	validation_0-logloss:0.406773
[883]	validation_0-logloss:0.406745
[884]	validation_0-logloss:0.406762
[885]	validation_0-logloss:0.406798
[886]	validation_0-logloss:0.406706
[887]	validation_0-logloss:0.406679
[888]	validation_0-logloss:0.406699
[889]	validation_0-logloss:0.406723
[890]	validation_0-logloss:0.406719
[891]	validation_0-logloss:0.406708
[892]	validation_0-logloss:0.406726
[893]	validation_0-logloss:0.406684
[894]	validation_0-logloss:0.406696
[895]	validation_0-logloss:0.406612
[896]	validation_0-logloss:0.406585
[897]	validation_0-logloss:0.406559
[898]	validation_0-logloss:0.406518
[899]	validation_0-logloss:0.406421
[900]	validation_0-logloss:0.406405
[901]	validation_0-logloss:0.406411
[902]	validation_0-logloss:0.406409
[903]	validation_0-logloss:0.406438
[904]	validation_0-logloss:0.406342
[905]	validation_0-logloss:0.406404
[906]	validation_0-logloss:0.406353
[907]	validation_0-logloss:0.406323
[908]	validation_0-logloss:0.406313
[909]	validation_0-logloss:0.406334
[910]	validation_0-logloss:0.406289
[911]	validation_0-logloss:0.406286
[912]	validation_0-logloss:0.406287
[913]	validation_0-logloss:0.406199
[914]	validation_0-logloss:0.406234
[915]	validation_0-logloss:0.406255
[916]	validation_0-logloss:0.406255
[917]	validation_0-logloss:0.406176
[918]	validation_0-logloss:0.406228
[919]	validation_0-logloss:0.406231
[920]	validation_0-logloss:0.406259
[921]	validation_0-logloss:0.406207
[922]	validation_0-logloss:0.406292
[923]	validation_0-logloss:0.406296
[924]	validation_0-logloss:0.406287
[925]	validation_0-logloss:0.406291
[926]	validation_0-logloss:0.406298
[927]	validation_0-logloss:0.406321
[928]	validation_0-logloss:0.406325
[929]	validation_0-logloss:0.406330
[930]	validation_0-logloss:0.406393
[931]	validation_0-logloss:0.406375
[932]	validation_0-logloss:0.406408
[933]	validation_0-logloss:0.406391
[934]	validation_0-logloss:0.406415
[935]	validation_0-logloss:0.406383
[936]	validation_0-logloss:0.406430
[937]	validation_0-logloss:0.406408
[938]	validation_0-logloss:0.406388
[939]	validation_0-logloss:0.406411
[940]	validation_0-logloss:0.406477
[941]	validation_0-logloss:0.406464
[942]	validation_0-logloss:0.406458
[943]	validation_0-logloss:0.406527
[944]	validation_0-logloss:0.406573
[945]	validation_0-logloss:0.406572
[946]	validation_0-logloss:0.406635
[947]	validation_0-logloss:0.406605
[948]	validation_0-logloss:0.406626
[949]	validation_0-logloss:0.406616
[950]	validation_0-logloss:0.406663
[951]	validation_0-logloss:0.406725
[952]	validation_0-logloss:0.406741
[953]	validation_0-logloss:0.406764
[954]	validation_0-logloss:0.406800
[955]	validation_0-logloss:0.406800
[956]	validation_0-logloss:0.406920
[957]	validation_0-logloss:0.406892
[958]	validation_0-logloss:0.406942
[959]	validation_0-logloss:0.406914
[960]	validation_0-logloss:0.406946
[961]	validation_0-logloss:0.406959
[962]	validation_0-logloss:0.406946
[963]	validation_0-logloss:0.406979
[964]	validation_0-logloss:0.406875
[965]	validation_0-logloss:0.406922
[966]	validation_0-logloss:0.406912
[967]	validation_0-logloss:0.406910
[968]	validation_0-logloss:0.406944
[969]	validation_0-logloss:0.406924
[970]	validation_0-logloss:0.406926
[971]	validation_0-logloss:0.406821
[972]	validation_0-logloss:0.406803
[973]	validation_0-logloss:0.406793
[974]	validation_0-logloss:0.406768
[975]	validation_0-logloss:0.406738
[976]	validation_0-logloss:0.406762
[977]	validation_0-logloss:0.406800
[978]	validation_0-logloss:0.406873
[979]	validation_0-logloss:0.406954
[980]	validation_0-logloss:0.406944
[981]	validation_0-logloss:0.406920
[982]	validation_0-logloss:0.406977
[983]	validation_0-logloss:0.406994
[984]	validation_0-logloss:0.407003
[985]	validation_0-logloss:0.407013
[986]	validation_0-logloss:0.407001
[987]	validation_0-logloss:0.406978
[988]	validation_0-logloss:0.406983
[989]	validation_0-logloss:0.406959
[990]	validation_0-logloss:0.406947
[991]	validation_0-logloss:0.406965
[992]	validation_0-logloss:0.406982
[993]	validation_0-logloss:0.406981
[994]	validation_0-logloss:0.406938
[995]	validation_0-logloss:0.406884
[996]	validation_0-logloss:0.406841
[997]	validation_0-logloss:0.406831
[998]	validation_0-logloss:0.406817
[999]	validation_0-logloss:0.406739
[1000]	validation_0-logloss:0.406708
[1001]	validation_0-logloss:0.406659
[1002]	validation_0-logloss:0.406603
[1003]	validation_0-logloss:0.406558
[1004]	validation_0-logloss:0.406580
[1005]	validation_0-logloss:0.406578
[1006]	validation_0-logloss:0.406517
[1007]	validation_0-logloss:0.406542
[1008]	validation_0-logloss:0.406567
[1009]	validation_0-logloss:0.406603
[1010]	validation_0-logloss:0.406597
[1011]	validation_0-logloss:0.406570
[1012]	validation_0-logloss:0.406512
[1013]	validation_0-logloss:0.406477
[1014]	validation_0-logloss:0.406465
[1015]	validation_0-logloss:0.406522
[1016]	validation_0-logloss:0.406567
[1017]	validation_0-logloss:0.406567
[1018]	validation_0-logloss:0.406596
[1019]	validation_0-logloss:0.406600
[1020]	validation_0-logloss:0.406612
[1021]	validation_0-logloss:0.406603
[1022]	validation_0-logloss:0.406597
[1023]	validation_0-logloss:0.406618
[1024]	validation_0-logloss:0.406651
[1025]	validation_0-logloss:0.406694
[1026]	validation_0-logloss:0.406644
[1027]	validation_0-logloss:0.406660
[1028]	validation_0-logloss:0.406636
[1029]	validation_0-logloss:0.406627
[1030]	validation_0-logloss:0.406582
[1031]	validation_0-logloss:0.406573
[1032]	validation_0-logloss:0.406581
[1033]	validation_0-logloss:0.406543
[1034]	validation_0-logloss:0.406538
[1035]	validation_0-logloss:0.406512
[1036]	validation_0-logloss:0.406533
[1037]	validation_0-logloss:0.406551
[1038]	validation_0-logloss:0.406490
[1039]	validation_0-logloss:0.406545
[1040]	validation_0-logloss:0.406507
[1041]	validation_0-logloss:0.406520
[1042]	validation_0-logloss:0.406486
[1043]	validation_0-logloss:0.406479
[1044]	validation_0-logloss:0.406483
[1045]	validation_0-logloss:0.406484
[1046]	validation_0-logloss:0.406479
[1047]	validation_0-logloss:0.406368
[1048]	validation_0-logloss:0.406389
[1049]	validation_0-logloss:0.406446
[1050]	validation_0-logloss:0.406439
[1051]	validation_0-logloss:0.406522
[1052]	validation_0-logloss:0.406422
[1053]	validation_0-logloss:0.406437
[1054]	validation_0-logloss:0.406409
[1055]	validation_0-logloss:0.406379
[1056]	validation_0-logloss:0.406395
[1057]	validation_0-logloss:0.406424
[1058]	validation_0-logloss:0.406407
[1059]	validation_0-logloss:0.406393
[1060]	validation_0-logloss:0.406446
[1061]	validation_0-logloss:0.406411
[1062]	validation_0-logloss:0.406309
[1063]	validation_0-logloss:0.406318
[1064]	validation_0-logloss:0.406289
[1065]	validation_0-logloss:0.406302
[1066]	validation_0-logloss:0.406268
[1067]	validation_0-logloss:0.406307
Stopping. Best iteration:
[917]	validation_0-logloss:0.406176


















In [13]:



    


#print y_pred
y_pred= clf.predict_proba(test)[:,1]

submission = pd.DataFrame({"":id_test, "Made Donation in March 2007":y_pred})
submission.to_csv("submission_xgboost.csv", index=False)



    











In [14]:



    


from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation
from keras.optimizers import Adam
from keras.utils.np_utils import to_categorical


nn = Sequential()
nn.add(Dense(175, input_dim=4, init='uniform'))
nn.add(Dropout(0.5))
nn.add(Activation('sigmoid'))
nn.add(Dense(2, init='uniform'))
nn.add(Activation('softmax'))

opt = Adam(lr=1e-4)
nn.compile(loss='sparse_categorical_crossentropy', optimizer=opt)
#X_fit, y_fit, early_stopping_rounds=150,  eval_metric="logloss", eval_set=[(X_eval, y_eval)
hist = nn.fit(X_fit.values, y_fit.values, nb_epoch=63, verbose=0,validation_data=(X_eval.values, y_eval.values))



    


















    






Using Theano backend.

















In [15]:



    


y_pred= nn.predict_proba(test.values, verbose=0)[:,1]

submission = pd.DataFrame({"":id_test, "Made Donation in March 2007":y_pred})
submission.to_csv("submission_NN.csv", index=False)



    











In [16]:



    


%matplotlib inline

plt.plot(hist.history['loss'][0:])
plt.plot(hist.history['val_loss'][0:])
plt.legend(['Train loss', 'Validation loss'])



    


















    
Out[16]:








<matplotlib.legend.Legend at 0x7f65465491d0>










    
























In [17]:



    


clf1_p = rf.predict_proba(X_eval)[:,1]

#Logreg
clf7_p = lrCV.predict_proba(X_eval)[:,1]

#xgb
clf8_p = clf.predict_proba(X_eval)[:,1]

# #NN
clf3_p = nn.predict_proba(X_eval.values, verbose = 0)[:,1]



    











In [18]:



    


classifiers = [clf1_p, 
               clf8_p, 
               clf3_p]

predictions = pd.DataFrame(classifiers[0:2]).T

for classif in classifiers:
    print(log_loss(y_eval, classif))



    


















    






0.430018365833
0.40630695956
0.420360229567

















In [19]:



    


print(np.corrcoef(classifiers))



    


















    






[[ 1.          0.9373986   0.72588806]
 [ 0.9373986   1.          0.7829662 ]
 [ 0.72588806  0.7829662   1.        ]]

















In [20]:



    


Ps = np.array(classifiers).T
def ensemble_logloss(w):
    return log_loss(y_eval, (Ps*w).sum(1).T/np.sum(w))



    











In [21]:



    


ensemble_logloss(np.ones(len(classifiers)))



    


















    
Out[21]:








0.40818553776718286

















In [22]:



    


def ensemble_roc_auc(w):
    return roc_auc_score(y_eval, (Ps*w).sum(1).T/np.sum(w))



    











In [23]:



    


ensemble_roc_auc(np.ones(len(classifiers)))



    


















    
Out[23]:








0.78655462184873948

















In [24]:



    


from scipy.optimize import minimize

opt = minimize(ensemble_logloss, x0=np.ones(len(classifiers)))
print("Function result:", opt.fun)
print("Array:", opt.x)



    


















    






('Function result:', 0.4058665515428579)
('Array:', array([ 0.05570982,  2.50483474,  0.43416609]))

















In [25]:



    


w = opt.x/opt.x.sum()
w



    


















    
Out[25]:








array([ 0.01860274,  0.83641962,  0.14497764])

















In [26]:



    


ensebmle_prob_validation = np.zeros(len(clf1_p))
for i in range(len(classifiers)):
    ensebmle_prob_validation += np.array(classifiers[i]) * w[i]



    











In [27]:



    


print log_loss(y_eval, ensebmle_prob_validation), roc_auc_score(y_eval, ensebmle_prob_validation)



    


















    






0.405866552072 0.787394957983

















In [28]:



    


def optimal(values):
    optimal_classifiers = []
    for i in range(len(values)):
        if values[i] > 0:
            print("Classifier", i+1, "is good")
            optimal_classifiers.append(classifiers[i])
    return optimal_classifiers



    











In [29]:



    


def full_negative_elimination(weights):
    global classifiers, Ps
    while sum(weights<0) > 0:
        opt = minimize(ensemble_logloss, x0=np.ones(len(classifiers)))
        print("Function result:", opt.fun)
        classifiers = optimal(opt.x)
        Ps = np.array(classifiers).T
        weights = opt.x/opt.x.sum()
        print("Current weights:", weights)
        print('=======')
    return weights



    











In [30]:



    


w = full_negative_elimination(w)



    











In [31]:



    


dtest = xgb.DMatrix(test)
p1 = rf.predict_proba(test)[:,1]

#Logreg
p7 = lrCV.predict_proba(test)[:,1]

#xgb
p8 = clf.predict_proba(test)[:,1]

p3 = nn.predict_proba(test.values, verbose = 0)[:,1]

test_predictions = [p1,  p8, p3]



    











In [32]:



    


ensebmle_prob_test = np.zeros(len(p1))
for i in range(len(test_predictions)):
    ensebmle_prob_test += np.array(test_predictions[i]) * w[i]



    











In [33]:



    


pd.DataFrame(ensebmle_prob_validation).hist()



    


















    
Out[33]:








array([[<matplotlib.axes._subplots.AxesSubplot object at 0x7f64d34d4350>]], dtype=object)










    
























In [34]:



    


pd.DataFrame(y).hist()



    


















    
Out[34]:








array([[<matplotlib.axes._subplots.AxesSubplot object at 0x7f64d4c88c50>]], dtype=object)










    
























In [35]:



    


submission = pd.DataFrame({"":id_test, "Made Donation in March 2007":ensebmle_prob_test})
submission.to_csv("submission_ansemble.csv", index=False)



    











In [ ]:

Content source: Diyago/Machine-Learning-scripts

Similar notebooks:

notebook.community | gallery | about