In [1]:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn import cross_validation
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import RandomForestClassifier
from sklearn.calibration import CalibratedClassifierCV
from sklearn.svm import SVC
from sklearn import grid_search
from sklearn.metrics import log_loss
from sklearn.cross_validation import train_test_split
import xgboost as xgb
from sklearn.metrics import roc_auc_score, log_loss
from sklearn import metrics
In [2]:
train = pd.read_csv('numerai_training_data.csv')
test = pd.read_csv('numerai_tournament_data.csv')
example = pd.read_csv('example_predictions.csv')
#train = train.iloc[10000:80000]
In [3]:
X = train.drop('target', axis=1)
y = train.target
id_test = test.t_id
df_test = test.drop('t_id', axis=1, inplace = True)
X_fit, X_eval, y_fit, y_eval= train_test_split(
X, y, test_size=0.10, random_state=1
)
In [23]:
rf = RandomForestClassifier(n_estimators=1000, n_jobs=4, max_depth =7)
rf.fit(X_fit, y_fit)
print 'RandomForestClassifier ', (log_loss(y_eval, rf.predict_proba(X_eval)))
submission = pd.DataFrame({"t_id":id_test, "probability":rf.predict_proba(test)[:,1]})
submission.to_csv("submission_rf_5.csv", index=False)
RandomForestClassifier 0.691367878221
In [24]:
lr = LogisticRegression()
CVscores = cross_validation.cross_val_score(lr, X, y, scoring='log_loss', cv=5)
# search for the regularization parameter
Cs = 10**np.linspace(-4, 4, num=15)
grid = {'C': Cs}
gridsearch = grid_search.GridSearchCV(lr, grid, scoring='log_loss', cv=5)
gridsearch.fit(X, y)
gridscores = [-x.mean_validation_score for x in gridsearch.grid_scores_]
C = Cs[np.argmin(gridscores)]
# refit the model with the new regularization parameter
lrCV = LogisticRegression(C=C)
lrCV.fit(X_fit, y_fit)
# write log to file
logloss_train = log_loss(y_fit, lrCV.predict_proba(X_fit))
logloss_val = log_loss(y_eval, lrCV.predict_proba(X_eval))
print 'logloss_train: ', logloss_train
print 'logloss_val: ', logloss_val
logloss_train: 0.691439680306
logloss_val: 0.691449256838
In [25]:
#n_estimators and early_stopping_rounds should be increased
clf = xgb.XGBClassifier(missing=np.nan, max_depth=3,
n_estimators=1500, learning_rate=0.005, gamma =0.3, min_child_weight = 3,
subsample=0.9, colsample_bytree=0.8, seed=2100,objective= 'binary:logistic')
# 'colsample_bytree': 0.8,
# 'gamma': 0.3,
# 'max_depth': 4,
# 'min_child_weight': 3,
# 'subsample': 0.9
# fitting
clf.fit(X_fit, y_fit, early_stopping_rounds=150, eval_metric="logloss", eval_set=[(X_eval, y_eval)])
# scores
from sklearn.metrics import log_loss
log_train = log_loss(y_fit, clf.predict_proba(X_fit)[:,1])
log_valid = log_loss(y_eval, clf.predict_proba(X_eval)[:,1])
print('\n-----------------------')
print(' logloss train: %.5f'%log_train)
print(' logloss valid: %.5f'%log_valid)
print('-----------------------')
print('\nModel parameters...')
print(clf.get_params())
#print y_pred
# y_pred= clf.predict_proba(test)[:,1]
# submission = pd.DataFrame({"t_id":id_test, "probability":y_pred})
# submission.to_csv("submission_xgboost.csv", index=False)
Will train until validation_0 error hasn't decreased in 150 rounds.
[0] validation_0-logloss:0.693143
[1] validation_0-logloss:0.693119
[2] validation_0-logloss:0.693123
[3] validation_0-logloss:0.693103
[4] validation_0-logloss:0.693102
[5] validation_0-logloss:0.693087
[6] validation_0-logloss:0.693077
[7] validation_0-logloss:0.693072
[8] validation_0-logloss:0.693058
[9] validation_0-logloss:0.693051
[10] validation_0-logloss:0.693039
[11] validation_0-logloss:0.693033
[12] validation_0-logloss:0.693023
[13] validation_0-logloss:0.693014
[14] validation_0-logloss:0.693004
[15] validation_0-logloss:0.692996
[16] validation_0-logloss:0.692993
[17] validation_0-logloss:0.692982
[18] validation_0-logloss:0.692973
[19] validation_0-logloss:0.692961
[20] validation_0-logloss:0.692954
[21] validation_0-logloss:0.692946
[22] validation_0-logloss:0.692939
[23] validation_0-logloss:0.692928
[24] validation_0-logloss:0.692921
[25] validation_0-logloss:0.692911
[26] validation_0-logloss:0.692902
[27] validation_0-logloss:0.692892
[28] validation_0-logloss:0.692882
[29] validation_0-logloss:0.692874
[30] validation_0-logloss:0.692868
[31] validation_0-logloss:0.692861
[32] validation_0-logloss:0.692852
[33] validation_0-logloss:0.692844
[34] validation_0-logloss:0.692837
[35] validation_0-logloss:0.692832
[36] validation_0-logloss:0.692824
[37] validation_0-logloss:0.692818
[38] validation_0-logloss:0.692808
[39] validation_0-logloss:0.692803
[40] validation_0-logloss:0.692796
[41] validation_0-logloss:0.692789
[42] validation_0-logloss:0.692781
[43] validation_0-logloss:0.692771
[44] validation_0-logloss:0.692761
[45] validation_0-logloss:0.692751
[46] validation_0-logloss:0.692743
[47] validation_0-logloss:0.692739
[48] validation_0-logloss:0.692732
[49] validation_0-logloss:0.692724
[50] validation_0-logloss:0.692717
[51] validation_0-logloss:0.692711
[52] validation_0-logloss:0.692709
[53] validation_0-logloss:0.692704
[54] validation_0-logloss:0.692698
[55] validation_0-logloss:0.692695
[56] validation_0-logloss:0.692686
[57] validation_0-logloss:0.692683
[58] validation_0-logloss:0.692678
[59] validation_0-logloss:0.692672
[60] validation_0-logloss:0.692665
[61] validation_0-logloss:0.692657
[62] validation_0-logloss:0.692651
[63] validation_0-logloss:0.692648
[64] validation_0-logloss:0.692643
[65] validation_0-logloss:0.692640
[66] validation_0-logloss:0.692629
[67] validation_0-logloss:0.692625
[68] validation_0-logloss:0.692618
[69] validation_0-logloss:0.692613
[70] validation_0-logloss:0.692607
[71] validation_0-logloss:0.692601
[72] validation_0-logloss:0.692595
[73] validation_0-logloss:0.692588
[74] validation_0-logloss:0.692583
[75] validation_0-logloss:0.692577
[76] validation_0-logloss:0.692572
[77] validation_0-logloss:0.692570
[78] validation_0-logloss:0.692566
[79] validation_0-logloss:0.692561
[80] validation_0-logloss:0.692556
[81] validation_0-logloss:0.692550
[82] validation_0-logloss:0.692547
[83] validation_0-logloss:0.692541
[84] validation_0-logloss:0.692539
[85] validation_0-logloss:0.692532
[86] validation_0-logloss:0.692523
[87] validation_0-logloss:0.692520
[88] validation_0-logloss:0.692515
[89] validation_0-logloss:0.692510
[90] validation_0-logloss:0.692506
[91] validation_0-logloss:0.692499
[92] validation_0-logloss:0.692494
[93] validation_0-logloss:0.692489
[94] validation_0-logloss:0.692485
[95] validation_0-logloss:0.692482
[96] validation_0-logloss:0.692477
[97] validation_0-logloss:0.692473
[98] validation_0-logloss:0.692466
[99] validation_0-logloss:0.692460
[100] validation_0-logloss:0.692455
[101] validation_0-logloss:0.692452
[102] validation_0-logloss:0.692448
[103] validation_0-logloss:0.692444
[104] validation_0-logloss:0.692437
[105] validation_0-logloss:0.692434
[106] validation_0-logloss:0.692425
[107] validation_0-logloss:0.692418
[108] validation_0-logloss:0.692414
[109] validation_0-logloss:0.692411
[110] validation_0-logloss:0.692403
[111] validation_0-logloss:0.692397
[112] validation_0-logloss:0.692393
[113] validation_0-logloss:0.692387
[114] validation_0-logloss:0.692383
[115] validation_0-logloss:0.692379
[116] validation_0-logloss:0.692374
[117] validation_0-logloss:0.692369
[118] validation_0-logloss:0.692365
[119] validation_0-logloss:0.692361
[120] validation_0-logloss:0.692356
[121] validation_0-logloss:0.692353
[122] validation_0-logloss:0.692348
[123] validation_0-logloss:0.692340
[124] validation_0-logloss:0.692333
[125] validation_0-logloss:0.692329
[126] validation_0-logloss:0.692325
[127] validation_0-logloss:0.692322
[128] validation_0-logloss:0.692321
[129] validation_0-logloss:0.692318
[130] validation_0-logloss:0.692313
[131] validation_0-logloss:0.692310
[132] validation_0-logloss:0.692304
[133] validation_0-logloss:0.692299
[134] validation_0-logloss:0.692296
[135] validation_0-logloss:0.692289
[136] validation_0-logloss:0.692286
[137] validation_0-logloss:0.692282
[138] validation_0-logloss:0.692279
[139] validation_0-logloss:0.692274
[140] validation_0-logloss:0.692268
[141] validation_0-logloss:0.692265
[142] validation_0-logloss:0.692259
[143] validation_0-logloss:0.692255
[144] validation_0-logloss:0.692248
[145] validation_0-logloss:0.692247
[146] validation_0-logloss:0.692244
[147] validation_0-logloss:0.692240
[148] validation_0-logloss:0.692237
[149] validation_0-logloss:0.692235
[150] validation_0-logloss:0.692230
[151] validation_0-logloss:0.692227
[152] validation_0-logloss:0.692224
[153] validation_0-logloss:0.692222
[154] validation_0-logloss:0.692219
[155] validation_0-logloss:0.692218
[156] validation_0-logloss:0.692213
[157] validation_0-logloss:0.692210
[158] validation_0-logloss:0.692205
[159] validation_0-logloss:0.692200
[160] validation_0-logloss:0.692198
[161] validation_0-logloss:0.692196
[162] validation_0-logloss:0.692191
[163] validation_0-logloss:0.692190
[164] validation_0-logloss:0.692187
[165] validation_0-logloss:0.692183
[166] validation_0-logloss:0.692180
[167] validation_0-logloss:0.692175
[168] validation_0-logloss:0.692172
[169] validation_0-logloss:0.692171
[170] validation_0-logloss:0.692170
[171] validation_0-logloss:0.692170
[172] validation_0-logloss:0.692166
[173] validation_0-logloss:0.692163
[174] validation_0-logloss:0.692161
[175] validation_0-logloss:0.692155
[176] validation_0-logloss:0.692152
[177] validation_0-logloss:0.692147
[178] validation_0-logloss:0.692144
[179] validation_0-logloss:0.692140
[180] validation_0-logloss:0.692136
[181] validation_0-logloss:0.692133
[182] validation_0-logloss:0.692132
[183] validation_0-logloss:0.692131
[184] validation_0-logloss:0.692132
[185] validation_0-logloss:0.692128
[186] validation_0-logloss:0.692125
[187] validation_0-logloss:0.692118
[188] validation_0-logloss:0.692115
[189] validation_0-logloss:0.692112
[190] validation_0-logloss:0.692108
[191] validation_0-logloss:0.692105
[192] validation_0-logloss:0.692104
[193] validation_0-logloss:0.692097
[194] validation_0-logloss:0.692095
[195] validation_0-logloss:0.692093
[196] validation_0-logloss:0.692091
[197] validation_0-logloss:0.692087
[198] validation_0-logloss:0.692085
[199] validation_0-logloss:0.692082
[200] validation_0-logloss:0.692079
[201] validation_0-logloss:0.692075
[202] validation_0-logloss:0.692070
[203] validation_0-logloss:0.692068
[204] validation_0-logloss:0.692063
[205] validation_0-logloss:0.692058
[206] validation_0-logloss:0.692055
[207] validation_0-logloss:0.692050
[208] validation_0-logloss:0.692049
[209] validation_0-logloss:0.692043
[210] validation_0-logloss:0.692040
[211] validation_0-logloss:0.692035
[212] validation_0-logloss:0.692033
[213] validation_0-logloss:0.692029
[214] validation_0-logloss:0.692027
[215] validation_0-logloss:0.692023
[216] validation_0-logloss:0.692020
[217] validation_0-logloss:0.692015
[218] validation_0-logloss:0.692015
[219] validation_0-logloss:0.692014
[220] validation_0-logloss:0.692010
[221] validation_0-logloss:0.692008
[222] validation_0-logloss:0.692005
[223] validation_0-logloss:0.692003
[224] validation_0-logloss:0.692000
[225] validation_0-logloss:0.691999
[226] validation_0-logloss:0.691994
[227] validation_0-logloss:0.691992
[228] validation_0-logloss:0.691987
[229] validation_0-logloss:0.691987
[230] validation_0-logloss:0.691985
[231] validation_0-logloss:0.691983
[232] validation_0-logloss:0.691980
[233] validation_0-logloss:0.691979
[234] validation_0-logloss:0.691978
[235] validation_0-logloss:0.691974
[236] validation_0-logloss:0.691973
[237] validation_0-logloss:0.691969
[238] validation_0-logloss:0.691967
[239] validation_0-logloss:0.691964
[240] validation_0-logloss:0.691960
[241] validation_0-logloss:0.691961
[242] validation_0-logloss:0.691958
[243] validation_0-logloss:0.691956
[244] validation_0-logloss:0.691956
[245] validation_0-logloss:0.691953
[246] validation_0-logloss:0.691953
[247] validation_0-logloss:0.691952
[248] validation_0-logloss:0.691950
[249] validation_0-logloss:0.691947
[250] validation_0-logloss:0.691947
[251] validation_0-logloss:0.691944
[252] validation_0-logloss:0.691942
[253] validation_0-logloss:0.691940
[254] validation_0-logloss:0.691939
[255] validation_0-logloss:0.691936
[256] validation_0-logloss:0.691936
[257] validation_0-logloss:0.691937
[258] validation_0-logloss:0.691934
[259] validation_0-logloss:0.691929
[260] validation_0-logloss:0.691929
[261] validation_0-logloss:0.691927
[262] validation_0-logloss:0.691924
[263] validation_0-logloss:0.691922
[264] validation_0-logloss:0.691919
[265] validation_0-logloss:0.691917
[266] validation_0-logloss:0.691914
[267] validation_0-logloss:0.691914
[268] validation_0-logloss:0.691911
[269] validation_0-logloss:0.691908
[270] validation_0-logloss:0.691907
[271] validation_0-logloss:0.691906
[272] validation_0-logloss:0.691903
[273] validation_0-logloss:0.691902
[274] validation_0-logloss:0.691898
[275] validation_0-logloss:0.691895
[276] validation_0-logloss:0.691895
[277] validation_0-logloss:0.691893
[278] validation_0-logloss:0.691889
[279] validation_0-logloss:0.691887
[280] validation_0-logloss:0.691884
[281] validation_0-logloss:0.691883
[282] validation_0-logloss:0.691879
[283] validation_0-logloss:0.691876
[284] validation_0-logloss:0.691875
[285] validation_0-logloss:0.691872
[286] validation_0-logloss:0.691872
[287] validation_0-logloss:0.691872
[288] validation_0-logloss:0.691868
[289] validation_0-logloss:0.691868
[290] validation_0-logloss:0.691865
[291] validation_0-logloss:0.691863
[292] validation_0-logloss:0.691862
[293] validation_0-logloss:0.691859
[294] validation_0-logloss:0.691857
[295] validation_0-logloss:0.691853
[296] validation_0-logloss:0.691850
[297] validation_0-logloss:0.691845
[298] validation_0-logloss:0.691843
[299] validation_0-logloss:0.691838
[300] validation_0-logloss:0.691837
[301] validation_0-logloss:0.691832
[302] validation_0-logloss:0.691832
[303] validation_0-logloss:0.691832
[304] validation_0-logloss:0.691829
[305] validation_0-logloss:0.691827
[306] validation_0-logloss:0.691828
[307] validation_0-logloss:0.691824
[308] validation_0-logloss:0.691822
[309] validation_0-logloss:0.691821
[310] validation_0-logloss:0.691817
[311] validation_0-logloss:0.691815
[312] validation_0-logloss:0.691813
[313] validation_0-logloss:0.691810
[314] validation_0-logloss:0.691807
[315] validation_0-logloss:0.691808
[316] validation_0-logloss:0.691808
[317] validation_0-logloss:0.691808
[318] validation_0-logloss:0.691809
[319] validation_0-logloss:0.691806
[320] validation_0-logloss:0.691805
[321] validation_0-logloss:0.691805
[322] validation_0-logloss:0.691801
[323] validation_0-logloss:0.691799
[324] validation_0-logloss:0.691802
[325] validation_0-logloss:0.691800
[326] validation_0-logloss:0.691799
[327] validation_0-logloss:0.691797
[328] validation_0-logloss:0.691795
[329] validation_0-logloss:0.691792
[330] validation_0-logloss:0.691791
[331] validation_0-logloss:0.691789
[332] validation_0-logloss:0.691789
[333] validation_0-logloss:0.691788
[334] validation_0-logloss:0.691786
[335] validation_0-logloss:0.691783
[336] validation_0-logloss:0.691784
[337] validation_0-logloss:0.691782
[338] validation_0-logloss:0.691779
[339] validation_0-logloss:0.691775
[340] validation_0-logloss:0.691774
[341] validation_0-logloss:0.691773
[342] validation_0-logloss:0.691772
[343] validation_0-logloss:0.691770
[344] validation_0-logloss:0.691769
[345] validation_0-logloss:0.691769
[346] validation_0-logloss:0.691768
[347] validation_0-logloss:0.691767
[348] validation_0-logloss:0.691763
[349] validation_0-logloss:0.691761
[350] validation_0-logloss:0.691759
[351] validation_0-logloss:0.691756
[352] validation_0-logloss:0.691755
[353] validation_0-logloss:0.691756
[354] validation_0-logloss:0.691753
[355] validation_0-logloss:0.691751
[356] validation_0-logloss:0.691749
[357] validation_0-logloss:0.691749
[358] validation_0-logloss:0.691746
[359] validation_0-logloss:0.691743
[360] validation_0-logloss:0.691744
[361] validation_0-logloss:0.691742
[362] validation_0-logloss:0.691740
[363] validation_0-logloss:0.691738
[364] validation_0-logloss:0.691738
[365] validation_0-logloss:0.691734
[366] validation_0-logloss:0.691733
[367] validation_0-logloss:0.691734
[368] validation_0-logloss:0.691734
[369] validation_0-logloss:0.691731
[370] validation_0-logloss:0.691727
[371] validation_0-logloss:0.691725
[372] validation_0-logloss:0.691723
[373] validation_0-logloss:0.691720
[374] validation_0-logloss:0.691719
[375] validation_0-logloss:0.691720
[376] validation_0-logloss:0.691719
[377] validation_0-logloss:0.691719
[378] validation_0-logloss:0.691718
[379] validation_0-logloss:0.691716
[380] validation_0-logloss:0.691717
[381] validation_0-logloss:0.691714
[382] validation_0-logloss:0.691713
[383] validation_0-logloss:0.691712
[384] validation_0-logloss:0.691711
[385] validation_0-logloss:0.691709
[386] validation_0-logloss:0.691705
[387] validation_0-logloss:0.691705
[388] validation_0-logloss:0.691704
[389] validation_0-logloss:0.691702
[390] validation_0-logloss:0.691702
[391] validation_0-logloss:0.691701
[392] validation_0-logloss:0.691698
[393] validation_0-logloss:0.691698
[394] validation_0-logloss:0.691694
[395] validation_0-logloss:0.691692
[396] validation_0-logloss:0.691689
[397] validation_0-logloss:0.691689
[398] validation_0-logloss:0.691687
[399] validation_0-logloss:0.691685
[400] validation_0-logloss:0.691681
[401] validation_0-logloss:0.691681
[402] validation_0-logloss:0.691679
[403] validation_0-logloss:0.691678
[404] validation_0-logloss:0.691674
[405] validation_0-logloss:0.691675
[406] validation_0-logloss:0.691674
[407] validation_0-logloss:0.691674
[408] validation_0-logloss:0.691670
[409] validation_0-logloss:0.691669
[410] validation_0-logloss:0.691669
[411] validation_0-logloss:0.691668
[412] validation_0-logloss:0.691666
[413] validation_0-logloss:0.691664
[414] validation_0-logloss:0.691662
[415] validation_0-logloss:0.691661
[416] validation_0-logloss:0.691660
[417] validation_0-logloss:0.691661
[418] validation_0-logloss:0.691659
[419] validation_0-logloss:0.691658
[420] validation_0-logloss:0.691653
[421] validation_0-logloss:0.691655
[422] validation_0-logloss:0.691654
[423] validation_0-logloss:0.691652
[424] validation_0-logloss:0.691653
[425] validation_0-logloss:0.691652
[426] validation_0-logloss:0.691650
[427] validation_0-logloss:0.691649
[428] validation_0-logloss:0.691650
[429] validation_0-logloss:0.691646
[430] validation_0-logloss:0.691647
[431] validation_0-logloss:0.691648
[432] validation_0-logloss:0.691647
[433] validation_0-logloss:0.691645
[434] validation_0-logloss:0.691646
[435] validation_0-logloss:0.691647
[436] validation_0-logloss:0.691645
[437] validation_0-logloss:0.691644
[438] validation_0-logloss:0.691640
[439] validation_0-logloss:0.691638
[440] validation_0-logloss:0.691638
[441] validation_0-logloss:0.691636
[442] validation_0-logloss:0.691633
[443] validation_0-logloss:0.691632
[444] validation_0-logloss:0.691632
[445] validation_0-logloss:0.691631
[446] validation_0-logloss:0.691627
[447] validation_0-logloss:0.691623
[448] validation_0-logloss:0.691621
[449] validation_0-logloss:0.691620
[450] validation_0-logloss:0.691620
[451] validation_0-logloss:0.691618
[452] validation_0-logloss:0.691617
[453] validation_0-logloss:0.691615
[454] validation_0-logloss:0.691614
[455] validation_0-logloss:0.691611
[456] validation_0-logloss:0.691611
[457] validation_0-logloss:0.691609
[458] validation_0-logloss:0.691607
[459] validation_0-logloss:0.691606
[460] validation_0-logloss:0.691603
[461] validation_0-logloss:0.691601
[462] validation_0-logloss:0.691601
[463] validation_0-logloss:0.691600
[464] validation_0-logloss:0.691596
[465] validation_0-logloss:0.691598
[466] validation_0-logloss:0.691596
[467] validation_0-logloss:0.691597
[468] validation_0-logloss:0.691594
[469] validation_0-logloss:0.691593
[470] validation_0-logloss:0.691593
[471] validation_0-logloss:0.691592
[472] validation_0-logloss:0.691590
[473] validation_0-logloss:0.691590
[474] validation_0-logloss:0.691591
[475] validation_0-logloss:0.691587
[476] validation_0-logloss:0.691588
[477] validation_0-logloss:0.691589
[478] validation_0-logloss:0.691589
[479] validation_0-logloss:0.691587
[480] validation_0-logloss:0.691589
[481] validation_0-logloss:0.691586
[482] validation_0-logloss:0.691587
[483] validation_0-logloss:0.691583
[484] validation_0-logloss:0.691580
[485] validation_0-logloss:0.691579
[486] validation_0-logloss:0.691577
[487] validation_0-logloss:0.691578
[488] validation_0-logloss:0.691576
[489] validation_0-logloss:0.691575
[490] validation_0-logloss:0.691575
[491] validation_0-logloss:0.691571
[492] validation_0-logloss:0.691571
[493] validation_0-logloss:0.691573
[494] validation_0-logloss:0.691574
[495] validation_0-logloss:0.691572
[496] validation_0-logloss:0.691573
[497] validation_0-logloss:0.691570
[498] validation_0-logloss:0.691571
[499] validation_0-logloss:0.691567
[500] validation_0-logloss:0.691564
[501] validation_0-logloss:0.691564
[502] validation_0-logloss:0.691562
[503] validation_0-logloss:0.691562
[504] validation_0-logloss:0.691560
[505] validation_0-logloss:0.691560
[506] validation_0-logloss:0.691561
[507] validation_0-logloss:0.691562
[508] validation_0-logloss:0.691561
[509] validation_0-logloss:0.691561
[510] validation_0-logloss:0.691557
[511] validation_0-logloss:0.691558
[512] validation_0-logloss:0.691556
[513] validation_0-logloss:0.691554
[514] validation_0-logloss:0.691550
[515] validation_0-logloss:0.691550
[516] validation_0-logloss:0.691549
[517] validation_0-logloss:0.691548
[518] validation_0-logloss:0.691547
[519] validation_0-logloss:0.691544
[520] validation_0-logloss:0.691542
[521] validation_0-logloss:0.691539
[522] validation_0-logloss:0.691539
[523] validation_0-logloss:0.691536
[524] validation_0-logloss:0.691536
[525] validation_0-logloss:0.691534
[526] validation_0-logloss:0.691534
[527] validation_0-logloss:0.691534
[528] validation_0-logloss:0.691534
[529] validation_0-logloss:0.691533
[530] validation_0-logloss:0.691532
[531] validation_0-logloss:0.691531
[532] validation_0-logloss:0.691531
[533] validation_0-logloss:0.691530
[534] validation_0-logloss:0.691529
[535] validation_0-logloss:0.691531
[536] validation_0-logloss:0.691528
[537] validation_0-logloss:0.691526
[538] validation_0-logloss:0.691525
[539] validation_0-logloss:0.691525
[540] validation_0-logloss:0.691524
[541] validation_0-logloss:0.691524
[542] validation_0-logloss:0.691521
[543] validation_0-logloss:0.691521
[544] validation_0-logloss:0.691520
[545] validation_0-logloss:0.691520
[546] validation_0-logloss:0.691518
[547] validation_0-logloss:0.691519
[548] validation_0-logloss:0.691518
[549] validation_0-logloss:0.691518
[550] validation_0-logloss:0.691516
[551] validation_0-logloss:0.691517
[552] validation_0-logloss:0.691517
[553] validation_0-logloss:0.691515
[554] validation_0-logloss:0.691515
[555] validation_0-logloss:0.691515
[556] validation_0-logloss:0.691513
[557] validation_0-logloss:0.691513
[558] validation_0-logloss:0.691514
[559] validation_0-logloss:0.691511
[560] validation_0-logloss:0.691512
[561] validation_0-logloss:0.691512
[562] validation_0-logloss:0.691512
[563] validation_0-logloss:0.691511
[564] validation_0-logloss:0.691510
[565] validation_0-logloss:0.691512
[566] validation_0-logloss:0.691513
[567] validation_0-logloss:0.691512
[568] validation_0-logloss:0.691511
[569] validation_0-logloss:0.691509
[570] validation_0-logloss:0.691508
[571] validation_0-logloss:0.691508
[572] validation_0-logloss:0.691505
[573] validation_0-logloss:0.691501
[574] validation_0-logloss:0.691503
[575] validation_0-logloss:0.691501
[576] validation_0-logloss:0.691501
[577] validation_0-logloss:0.691502
[578] validation_0-logloss:0.691500
[579] validation_0-logloss:0.691496
[580] validation_0-logloss:0.691495
[581] validation_0-logloss:0.691495
[582] validation_0-logloss:0.691494
[583] validation_0-logloss:0.691492
[584] validation_0-logloss:0.691494
[585] validation_0-logloss:0.691494
[586] validation_0-logloss:0.691492
[587] validation_0-logloss:0.691495
[588] validation_0-logloss:0.691495
[589] validation_0-logloss:0.691493
[590] validation_0-logloss:0.691493
[591] validation_0-logloss:0.691495
[592] validation_0-logloss:0.691495
[593] validation_0-logloss:0.691494
[594] validation_0-logloss:0.691492
[595] validation_0-logloss:0.691491
[596] validation_0-logloss:0.691489
[597] validation_0-logloss:0.691489
[598] validation_0-logloss:0.691487
[599] validation_0-logloss:0.691486
[600] validation_0-logloss:0.691485
[601] validation_0-logloss:0.691483
[602] validation_0-logloss:0.691482
[603] validation_0-logloss:0.691478
[604] validation_0-logloss:0.691478
[605] validation_0-logloss:0.691477
[606] validation_0-logloss:0.691478
[607] validation_0-logloss:0.691478
[608] validation_0-logloss:0.691475
[609] validation_0-logloss:0.691473
[610] validation_0-logloss:0.691472
[611] validation_0-logloss:0.691470
[612] validation_0-logloss:0.691471
[613] validation_0-logloss:0.691471
[614] validation_0-logloss:0.691470
[615] validation_0-logloss:0.691470
[616] validation_0-logloss:0.691469
[617] validation_0-logloss:0.691471
[618] validation_0-logloss:0.691473
[619] validation_0-logloss:0.691472
[620] validation_0-logloss:0.691472
[621] validation_0-logloss:0.691473
[622] validation_0-logloss:0.691472
[623] validation_0-logloss:0.691471
[624] validation_0-logloss:0.691468
[625] validation_0-logloss:0.691467
[626] validation_0-logloss:0.691467
[627] validation_0-logloss:0.691466
[628] validation_0-logloss:0.691463
[629] validation_0-logloss:0.691463
[630] validation_0-logloss:0.691463
[631] validation_0-logloss:0.691461
[632] validation_0-logloss:0.691459
[633] validation_0-logloss:0.691460
[634] validation_0-logloss:0.691458
[635] validation_0-logloss:0.691459
[636] validation_0-logloss:0.691459
[637] validation_0-logloss:0.691455
[638] validation_0-logloss:0.691456
[639] validation_0-logloss:0.691457
[640] validation_0-logloss:0.691454
[641] validation_0-logloss:0.691453
[642] validation_0-logloss:0.691453
[643] validation_0-logloss:0.691452
[644] validation_0-logloss:0.691451
[645] validation_0-logloss:0.691452
[646] validation_0-logloss:0.691451
[647] validation_0-logloss:0.691452
[648] validation_0-logloss:0.691453
[649] validation_0-logloss:0.691454
[650] validation_0-logloss:0.691455
[651] validation_0-logloss:0.691453
[652] validation_0-logloss:0.691453
[653] validation_0-logloss:0.691452
[654] validation_0-logloss:0.691453
[655] validation_0-logloss:0.691452
[656] validation_0-logloss:0.691452
[657] validation_0-logloss:0.691450
[658] validation_0-logloss:0.691450
[659] validation_0-logloss:0.691449
[660] validation_0-logloss:0.691451
[661] validation_0-logloss:0.691450
[662] validation_0-logloss:0.691446
[663] validation_0-logloss:0.691446
[664] validation_0-logloss:0.691446
[665] validation_0-logloss:0.691446
[666] validation_0-logloss:0.691445
[667] validation_0-logloss:0.691445
[668] validation_0-logloss:0.691443
[669] validation_0-logloss:0.691444
[670] validation_0-logloss:0.691443
[671] validation_0-logloss:0.691442
[672] validation_0-logloss:0.691441
[673] validation_0-logloss:0.691442
[674] validation_0-logloss:0.691439
[675] validation_0-logloss:0.691440
[676] validation_0-logloss:0.691438
[677] validation_0-logloss:0.691437
[678] validation_0-logloss:0.691435
[679] validation_0-logloss:0.691436
[680] validation_0-logloss:0.691435
[681] validation_0-logloss:0.691433
[682] validation_0-logloss:0.691434
[683] validation_0-logloss:0.691433
[684] validation_0-logloss:0.691432
[685] validation_0-logloss:0.691433
[686] validation_0-logloss:0.691430
[687] validation_0-logloss:0.691428
[688] validation_0-logloss:0.691428
[689] validation_0-logloss:0.691426
[690] validation_0-logloss:0.691427
[691] validation_0-logloss:0.691428
[692] validation_0-logloss:0.691427
[693] validation_0-logloss:0.691427
[694] validation_0-logloss:0.691426
[695] validation_0-logloss:0.691422
[696] validation_0-logloss:0.691422
[697] validation_0-logloss:0.691420
[698] validation_0-logloss:0.691420
[699] validation_0-logloss:0.691419
[700] validation_0-logloss:0.691418
[701] validation_0-logloss:0.691418
[702] validation_0-logloss:0.691416
[703] validation_0-logloss:0.691416
[704] validation_0-logloss:0.691415
[705] validation_0-logloss:0.691416
[706] validation_0-logloss:0.691413
[707] validation_0-logloss:0.691414
[708] validation_0-logloss:0.691412
[709] validation_0-logloss:0.691411
[710] validation_0-logloss:0.691411
[711] validation_0-logloss:0.691411
[712] validation_0-logloss:0.691410
[713] validation_0-logloss:0.691410
[714] validation_0-logloss:0.691412
[715] validation_0-logloss:0.691413
[716] validation_0-logloss:0.691414
[717] validation_0-logloss:0.691412
[718] validation_0-logloss:0.691413
[719] validation_0-logloss:0.691413
[720] validation_0-logloss:0.691411
[721] validation_0-logloss:0.691411
[722] validation_0-logloss:0.691411
[723] validation_0-logloss:0.691411
[724] validation_0-logloss:0.691410
[725] validation_0-logloss:0.691412
[726] validation_0-logloss:0.691410
[727] validation_0-logloss:0.691410
[728] validation_0-logloss:0.691407
[729] validation_0-logloss:0.691408
[730] validation_0-logloss:0.691407
[731] validation_0-logloss:0.691406
[732] validation_0-logloss:0.691405
[733] validation_0-logloss:0.691404
[734] validation_0-logloss:0.691402
[735] validation_0-logloss:0.691400
[736] validation_0-logloss:0.691400
[737] validation_0-logloss:0.691397
[738] validation_0-logloss:0.691397
[739] validation_0-logloss:0.691396
[740] validation_0-logloss:0.691398
[741] validation_0-logloss:0.691397
[742] validation_0-logloss:0.691396
[743] validation_0-logloss:0.691397
[744] validation_0-logloss:0.691396
[745] validation_0-logloss:0.691398
[746] validation_0-logloss:0.691397
[747] validation_0-logloss:0.691396
[748] validation_0-logloss:0.691393
[749] validation_0-logloss:0.691395
[750] validation_0-logloss:0.691392
[751] validation_0-logloss:0.691391
[752] validation_0-logloss:0.691390
[753] validation_0-logloss:0.691389
[754] validation_0-logloss:0.691391
[755] validation_0-logloss:0.691391
[756] validation_0-logloss:0.691388
[757] validation_0-logloss:0.691387
[758] validation_0-logloss:0.691386
[759] validation_0-logloss:0.691387
[760] validation_0-logloss:0.691385
[761] validation_0-logloss:0.691385
[762] validation_0-logloss:0.691384
[763] validation_0-logloss:0.691381
[764] validation_0-logloss:0.691382
[765] validation_0-logloss:0.691383
[766] validation_0-logloss:0.691383
[767] validation_0-logloss:0.691383
[768] validation_0-logloss:0.691383
[769] validation_0-logloss:0.691382
[770] validation_0-logloss:0.691381
[771] validation_0-logloss:0.691382
[772] validation_0-logloss:0.691381
[773] validation_0-logloss:0.691380
[774] validation_0-logloss:0.691379
[775] validation_0-logloss:0.691378
[776] validation_0-logloss:0.691377
[777] validation_0-logloss:0.691378
[778] validation_0-logloss:0.691379
[779] validation_0-logloss:0.691379
[780] validation_0-logloss:0.691378
[781] validation_0-logloss:0.691377
[782] validation_0-logloss:0.691377
[783] validation_0-logloss:0.691377
[784] validation_0-logloss:0.691375
[785] validation_0-logloss:0.691373
[786] validation_0-logloss:0.691371
[787] validation_0-logloss:0.691370
[788] validation_0-logloss:0.691368
[789] validation_0-logloss:0.691368
[790] validation_0-logloss:0.691365
[791] validation_0-logloss:0.691363
[792] validation_0-logloss:0.691361
[793] validation_0-logloss:0.691360
[794] validation_0-logloss:0.691359
[795] validation_0-logloss:0.691357
[796] validation_0-logloss:0.691357
[797] validation_0-logloss:0.691357
[798] validation_0-logloss:0.691355
[799] validation_0-logloss:0.691355
[800] validation_0-logloss:0.691357
[801] validation_0-logloss:0.691357
[802] validation_0-logloss:0.691357
[803] validation_0-logloss:0.691357
[804] validation_0-logloss:0.691357
[805] validation_0-logloss:0.691359
[806] validation_0-logloss:0.691358
[807] validation_0-logloss:0.691359
[808] validation_0-logloss:0.691357
[809] validation_0-logloss:0.691358
[810] validation_0-logloss:0.691356
[811] validation_0-logloss:0.691357
[812] validation_0-logloss:0.691357
[813] validation_0-logloss:0.691359
[814] validation_0-logloss:0.691356
[815] validation_0-logloss:0.691356
[816] validation_0-logloss:0.691358
[817] validation_0-logloss:0.691356
[818] validation_0-logloss:0.691357
[819] validation_0-logloss:0.691357
[820] validation_0-logloss:0.691357
[821] validation_0-logloss:0.691356
[822] validation_0-logloss:0.691354
[823] validation_0-logloss:0.691355
[824] validation_0-logloss:0.691354
[825] validation_0-logloss:0.691354
[826] validation_0-logloss:0.691352
[827] validation_0-logloss:0.691350
[828] validation_0-logloss:0.691350
[829] validation_0-logloss:0.691348
[830] validation_0-logloss:0.691345
[831] validation_0-logloss:0.691344
[832] validation_0-logloss:0.691344
[833] validation_0-logloss:0.691345
[834] validation_0-logloss:0.691348
[835] validation_0-logloss:0.691343
[836] validation_0-logloss:0.691343
[837] validation_0-logloss:0.691343
[838] validation_0-logloss:0.691342
[839] validation_0-logloss:0.691341
[840] validation_0-logloss:0.691341
[841] validation_0-logloss:0.691339
[842] validation_0-logloss:0.691340
[843] validation_0-logloss:0.691340
[844] validation_0-logloss:0.691337
[845] validation_0-logloss:0.691337
[846] validation_0-logloss:0.691337
[847] validation_0-logloss:0.691335
[848] validation_0-logloss:0.691338
[849] validation_0-logloss:0.691338
[850] validation_0-logloss:0.691338
[851] validation_0-logloss:0.691338
[852] validation_0-logloss:0.691336
[853] validation_0-logloss:0.691336
[854] validation_0-logloss:0.691337
[855] validation_0-logloss:0.691335
[856] validation_0-logloss:0.691335
[857] validation_0-logloss:0.691334
[858] validation_0-logloss:0.691332
[859] validation_0-logloss:0.691331
[860] validation_0-logloss:0.691330
[861] validation_0-logloss:0.691329
[862] validation_0-logloss:0.691328
[863] validation_0-logloss:0.691325
[864] validation_0-logloss:0.691325
[865] validation_0-logloss:0.691325
[866] validation_0-logloss:0.691324
[867] validation_0-logloss:0.691322
[868] validation_0-logloss:0.691323
[869] validation_0-logloss:0.691324
[870] validation_0-logloss:0.691325
[871] validation_0-logloss:0.691326
[872] validation_0-logloss:0.691326
[873] validation_0-logloss:0.691323
[874] validation_0-logloss:0.691323
[875] validation_0-logloss:0.691323
[876] validation_0-logloss:0.691324
[877] validation_0-logloss:0.691324
[878] validation_0-logloss:0.691325
[879] validation_0-logloss:0.691325
[880] validation_0-logloss:0.691324
[881] validation_0-logloss:0.691326
[882] validation_0-logloss:0.691323
[883] validation_0-logloss:0.691319
[884] validation_0-logloss:0.691319
[885] validation_0-logloss:0.691320
[886] validation_0-logloss:0.691320
[887] validation_0-logloss:0.691317
[888] validation_0-logloss:0.691317
[889] validation_0-logloss:0.691315
[890] validation_0-logloss:0.691314
[891] validation_0-logloss:0.691315
[892] validation_0-logloss:0.691315
[893] validation_0-logloss:0.691315
[894] validation_0-logloss:0.691313
[895] validation_0-logloss:0.691312
[896] validation_0-logloss:0.691311
[897] validation_0-logloss:0.691313
[898] validation_0-logloss:0.691312
[899] validation_0-logloss:0.691312
[900] validation_0-logloss:0.691314
[901] validation_0-logloss:0.691315
[902] validation_0-logloss:0.691316
[903] validation_0-logloss:0.691315
[904] validation_0-logloss:0.691316
[905] validation_0-logloss:0.691315
[906] validation_0-logloss:0.691312
[907] validation_0-logloss:0.691312
[908] validation_0-logloss:0.691313
[909] validation_0-logloss:0.691312
[910] validation_0-logloss:0.691312
[911] validation_0-logloss:0.691311
[912] validation_0-logloss:0.691310
[913] validation_0-logloss:0.691309
[914] validation_0-logloss:0.691310
[915] validation_0-logloss:0.691309
[916] validation_0-logloss:0.691309
[917] validation_0-logloss:0.691307
[918] validation_0-logloss:0.691308
[919] validation_0-logloss:0.691308
[920] validation_0-logloss:0.691309
[921] validation_0-logloss:0.691308
[922] validation_0-logloss:0.691308
[923] validation_0-logloss:0.691306
[924] validation_0-logloss:0.691304
[925] validation_0-logloss:0.691302
[926] validation_0-logloss:0.691301
[927] validation_0-logloss:0.691301
[928] validation_0-logloss:0.691300
[929] validation_0-logloss:0.691301
[930] validation_0-logloss:0.691299
[931] validation_0-logloss:0.691297
[932] validation_0-logloss:0.691298
[933] validation_0-logloss:0.691298
[934] validation_0-logloss:0.691296
[935] validation_0-logloss:0.691295
[936] validation_0-logloss:0.691295
[937] validation_0-logloss:0.691297
[938] validation_0-logloss:0.691296
[939] validation_0-logloss:0.691297
[940] validation_0-logloss:0.691298
[941] validation_0-logloss:0.691294
[942] validation_0-logloss:0.691294
[943] validation_0-logloss:0.691295
[944] validation_0-logloss:0.691294
[945] validation_0-logloss:0.691292
[946] validation_0-logloss:0.691292
[947] validation_0-logloss:0.691292
[948] validation_0-logloss:0.691292
[949] validation_0-logloss:0.691291
[950] validation_0-logloss:0.691293
[951] validation_0-logloss:0.691294
[952] validation_0-logloss:0.691293
[953] validation_0-logloss:0.691289
[954] validation_0-logloss:0.691285
[955] validation_0-logloss:0.691285
[956] validation_0-logloss:0.691283
[957] validation_0-logloss:0.691280
[958] validation_0-logloss:0.691280
[959] validation_0-logloss:0.691278
[960] validation_0-logloss:0.691278
[961] validation_0-logloss:0.691276
[962] validation_0-logloss:0.691277
[963] validation_0-logloss:0.691276
[964] validation_0-logloss:0.691276
[965] validation_0-logloss:0.691275
[966] validation_0-logloss:0.691274
[967] validation_0-logloss:0.691274
[968] validation_0-logloss:0.691274
[969] validation_0-logloss:0.691275
[970] validation_0-logloss:0.691272
[971] validation_0-logloss:0.691270
[972] validation_0-logloss:0.691269
[973] validation_0-logloss:0.691270
[974] validation_0-logloss:0.691272
[975] validation_0-logloss:0.691273
[976] validation_0-logloss:0.691271
[977] validation_0-logloss:0.691272
[978] validation_0-logloss:0.691272
[979] validation_0-logloss:0.691268
[980] validation_0-logloss:0.691269
[981] validation_0-logloss:0.691269
[982] validation_0-logloss:0.691265
[983] validation_0-logloss:0.691265
[984] validation_0-logloss:0.691263
[985] validation_0-logloss:0.691265
[986] validation_0-logloss:0.691265
[987] validation_0-logloss:0.691260
[988] validation_0-logloss:0.691260
[989] validation_0-logloss:0.691260
[990] validation_0-logloss:0.691257
[991] validation_0-logloss:0.691258
[992] validation_0-logloss:0.691259
[993] validation_0-logloss:0.691256
[994] validation_0-logloss:0.691255
[995] validation_0-logloss:0.691255
[996] validation_0-logloss:0.691254
[997] validation_0-logloss:0.691255
[998] validation_0-logloss:0.691255
[999] validation_0-logloss:0.691255
[1000] validation_0-logloss:0.691255
[1001] validation_0-logloss:0.691253
[1002] validation_0-logloss:0.691251
[1003] validation_0-logloss:0.691250
[1004] validation_0-logloss:0.691248
[1005] validation_0-logloss:0.691249
[1006] validation_0-logloss:0.691248
[1007] validation_0-logloss:0.691247
[1008] validation_0-logloss:0.691248
[1009] validation_0-logloss:0.691249
[1010] validation_0-logloss:0.691247
[1011] validation_0-logloss:0.691245
[1012] validation_0-logloss:0.691246
[1013] validation_0-logloss:0.691246
[1014] validation_0-logloss:0.691245
[1015] validation_0-logloss:0.691246
[1016] validation_0-logloss:0.691247
[1017] validation_0-logloss:0.691246
[1018] validation_0-logloss:0.691244
[1019] validation_0-logloss:0.691243
[1020] validation_0-logloss:0.691244
[1021] validation_0-logloss:0.691243
[1022] validation_0-logloss:0.691241
[1023] validation_0-logloss:0.691239
[1024] validation_0-logloss:0.691239
[1025] validation_0-logloss:0.691237
[1026] validation_0-logloss:0.691238
[1027] validation_0-logloss:0.691237
[1028] validation_0-logloss:0.691241
[1029] validation_0-logloss:0.691241
[1030] validation_0-logloss:0.691241
[1031] validation_0-logloss:0.691240
[1032] validation_0-logloss:0.691237
[1033] validation_0-logloss:0.691238
[1034] validation_0-logloss:0.691238
[1035] validation_0-logloss:0.691239
[1036] validation_0-logloss:0.691240
[1037] validation_0-logloss:0.691242
[1038] validation_0-logloss:0.691243
[1039] validation_0-logloss:0.691241
[1040] validation_0-logloss:0.691238
[1041] validation_0-logloss:0.691239
[1042] validation_0-logloss:0.691237
[1043] validation_0-logloss:0.691236
[1044] validation_0-logloss:0.691236
[1045] validation_0-logloss:0.691234
[1046] validation_0-logloss:0.691236
[1047] validation_0-logloss:0.691236
[1048] validation_0-logloss:0.691237
[1049] validation_0-logloss:0.691236
[1050] validation_0-logloss:0.691234
[1051] validation_0-logloss:0.691234
[1052] validation_0-logloss:0.691234
[1053] validation_0-logloss:0.691236
[1054] validation_0-logloss:0.691237
[1055] validation_0-logloss:0.691236
[1056] validation_0-logloss:0.691237
[1057] validation_0-logloss:0.691238
[1058] validation_0-logloss:0.691237
[1059] validation_0-logloss:0.691237
[1060] validation_0-logloss:0.691234
[1061] validation_0-logloss:0.691233
[1062] validation_0-logloss:0.691233
[1063] validation_0-logloss:0.691234
[1064] validation_0-logloss:0.691233
[1065] validation_0-logloss:0.691233
[1066] validation_0-logloss:0.691234
[1067] validation_0-logloss:0.691236
[1068] validation_0-logloss:0.691236
[1069] validation_0-logloss:0.691234
[1070] validation_0-logloss:0.691233
[1071] validation_0-logloss:0.691233
[1072] validation_0-logloss:0.691232
[1073] validation_0-logloss:0.691231
[1074] validation_0-logloss:0.691229
[1075] validation_0-logloss:0.691229
[1076] validation_0-logloss:0.691229
[1077] validation_0-logloss:0.691230
[1078] validation_0-logloss:0.691230
[1079] validation_0-logloss:0.691229
[1080] validation_0-logloss:0.691231
[1081] validation_0-logloss:0.691231
[1082] validation_0-logloss:0.691230
[1083] validation_0-logloss:0.691232
[1084] validation_0-logloss:0.691232
[1085] validation_0-logloss:0.691232
[1086] validation_0-logloss:0.691234
[1087] validation_0-logloss:0.691234
[1088] validation_0-logloss:0.691232
[1089] validation_0-logloss:0.691232
[1090] validation_0-logloss:0.691234
[1091] validation_0-logloss:0.691234
[1092] validation_0-logloss:0.691236
[1093] validation_0-logloss:0.691237
[1094] validation_0-logloss:0.691235
[1095] validation_0-logloss:0.691235
[1096] validation_0-logloss:0.691236
[1097] validation_0-logloss:0.691235
[1098] validation_0-logloss:0.691235
[1099] validation_0-logloss:0.691234
[1100] validation_0-logloss:0.691233
[1101] validation_0-logloss:0.691235
[1102] validation_0-logloss:0.691234
[1103] validation_0-logloss:0.691234
[1104] validation_0-logloss:0.691233
[1105] validation_0-logloss:0.691234
[1106] validation_0-logloss:0.691233
[1107] validation_0-logloss:0.691234
[1108] validation_0-logloss:0.691234
[1109] validation_0-logloss:0.691235
[1110] validation_0-logloss:0.691236
[1111] validation_0-logloss:0.691235
[1112] validation_0-logloss:0.691234
[1113] validation_0-logloss:0.691234
[1114] validation_0-logloss:0.691232
[1115] validation_0-logloss:0.691233
[1116] validation_0-logloss:0.691233
[1117] validation_0-logloss:0.691237
[1118] validation_0-logloss:0.691237
[1119] validation_0-logloss:0.691236
[1120] validation_0-logloss:0.691237
[1121] validation_0-logloss:0.691238
[1122] validation_0-logloss:0.691239
[1123] validation_0-logloss:0.691239
[1124] validation_0-logloss:0.691241
[1125] validation_0-logloss:0.691244
[1126] validation_0-logloss:0.691245
[1127] validation_0-logloss:0.691243
[1128] validation_0-logloss:0.691245
[1129] validation_0-logloss:0.691247
[1130] validation_0-logloss:0.691245
[1131] validation_0-logloss:0.691246
[1132] validation_0-logloss:0.691244
[1133] validation_0-logloss:0.691247
[1134] validation_0-logloss:0.691247
[1135] validation_0-logloss:0.691248
[1136] validation_0-logloss:0.691248
[1137] validation_0-logloss:0.691248
[1138] validation_0-logloss:0.691249
[1139] validation_0-logloss:0.691248
[1140] validation_0-logloss:0.691248
[1141] validation_0-logloss:0.691247
[1142] validation_0-logloss:0.691248
[1143] validation_0-logloss:0.691249
[1144] validation_0-logloss:0.691247
[1145] validation_0-logloss:0.691249
[1146] validation_0-logloss:0.691250
[1147] validation_0-logloss:0.691248
[1148] validation_0-logloss:0.691249
[1149] validation_0-logloss:0.691249
[1150] validation_0-logloss:0.691250
[1151] validation_0-logloss:0.691249
[1152] validation_0-logloss:0.691246
[1153] validation_0-logloss:0.691243
[1154] validation_0-logloss:0.691244
[1155] validation_0-logloss:0.691245
[1156] validation_0-logloss:0.691245
[1157] validation_0-logloss:0.691245
[1158] validation_0-logloss:0.691243
[1159] validation_0-logloss:0.691242
[1160] validation_0-logloss:0.691243
[1161] validation_0-logloss:0.691242
[1162] validation_0-logloss:0.691241
[1163] validation_0-logloss:0.691239
[1164] validation_0-logloss:0.691238
[1165] validation_0-logloss:0.691239
[1166] validation_0-logloss:0.691242
[1167] validation_0-logloss:0.691241
[1168] validation_0-logloss:0.691241
[1169] validation_0-logloss:0.691243
[1170] validation_0-logloss:0.691241
[1171] validation_0-logloss:0.691241
[1172] validation_0-logloss:0.691240
[1173] validation_0-logloss:0.691238
[1174] validation_0-logloss:0.691238
[1175] validation_0-logloss:0.691237
[1176] validation_0-logloss:0.691236
[1177] validation_0-logloss:0.691235
[1178] validation_0-logloss:0.691234
[1179] validation_0-logloss:0.691234
[1180] validation_0-logloss:0.691235
[1181] validation_0-logloss:0.691234
[1182] validation_0-logloss:0.691233
[1183] validation_0-logloss:0.691234
[1184] validation_0-logloss:0.691233
[1185] validation_0-logloss:0.691231
[1186] validation_0-logloss:0.691232
[1187] validation_0-logloss:0.691231
[1188] validation_0-logloss:0.691233
[1189] validation_0-logloss:0.691233
[1190] validation_0-logloss:0.691235
[1191] validation_0-logloss:0.691234
[1192] validation_0-logloss:0.691235
[1193] validation_0-logloss:0.691233
[1194] validation_0-logloss:0.691231
[1195] validation_0-logloss:0.691232
[1196] validation_0-logloss:0.691232
[1197] validation_0-logloss:0.691232
[1198] validation_0-logloss:0.691234
[1199] validation_0-logloss:0.691233
[1200] validation_0-logloss:0.691233
[1201] validation_0-logloss:0.691233
[1202] validation_0-logloss:0.691232
[1203] validation_0-logloss:0.691232
[1204] validation_0-logloss:0.691233
[1205] validation_0-logloss:0.691233
[1206] validation_0-logloss:0.691234
[1207] validation_0-logloss:0.691231
[1208] validation_0-logloss:0.691230
[1209] validation_0-logloss:0.691230
[1210] validation_0-logloss:0.691228
[1211] validation_0-logloss:0.691229
[1212] validation_0-logloss:0.691230
[1213] validation_0-logloss:0.691230
[1214] validation_0-logloss:0.691229
[1215] validation_0-logloss:0.691229
[1216] validation_0-logloss:0.691229
[1217] validation_0-logloss:0.691229
[1218] validation_0-logloss:0.691226
[1219] validation_0-logloss:0.691223
[1220] validation_0-logloss:0.691225
[1221] validation_0-logloss:0.691224
[1222] validation_0-logloss:0.691224
[1223] validation_0-logloss:0.691221
[1224] validation_0-logloss:0.691222
[1225] validation_0-logloss:0.691222
[1226] validation_0-logloss:0.691221
[1227] validation_0-logloss:0.691221
[1228] validation_0-logloss:0.691220
[1229] validation_0-logloss:0.691220
[1230] validation_0-logloss:0.691218
[1231] validation_0-logloss:0.691216
[1232] validation_0-logloss:0.691211
[1233] validation_0-logloss:0.691213
[1234] validation_0-logloss:0.691212
[1235] validation_0-logloss:0.691211
[1236] validation_0-logloss:0.691210
[1237] validation_0-logloss:0.691210
[1238] validation_0-logloss:0.691206
[1239] validation_0-logloss:0.691205
[1240] validation_0-logloss:0.691206
[1241] validation_0-logloss:0.691205
[1242] validation_0-logloss:0.691205
[1243] validation_0-logloss:0.691203
[1244] validation_0-logloss:0.691204
[1245] validation_0-logloss:0.691203
[1246] validation_0-logloss:0.691202
[1247] validation_0-logloss:0.691204
[1248] validation_0-logloss:0.691203
[1249] validation_0-logloss:0.691204
[1250] validation_0-logloss:0.691205
[1251] validation_0-logloss:0.691205
[1252] validation_0-logloss:0.691206
[1253] validation_0-logloss:0.691207
[1254] validation_0-logloss:0.691208
[1255] validation_0-logloss:0.691207
[1256] validation_0-logloss:0.691207
[1257] validation_0-logloss:0.691205
[1258] validation_0-logloss:0.691206
[1259] validation_0-logloss:0.691206
[1260] validation_0-logloss:0.691205
[1261] validation_0-logloss:0.691205
[1262] validation_0-logloss:0.691204
[1263] validation_0-logloss:0.691208
[1264] validation_0-logloss:0.691208
[1265] validation_0-logloss:0.691209
[1266] validation_0-logloss:0.691207
[1267] validation_0-logloss:0.691208
[1268] validation_0-logloss:0.691205
[1269] validation_0-logloss:0.691204
[1270] validation_0-logloss:0.691204
[1271] validation_0-logloss:0.691204
[1272] validation_0-logloss:0.691204
[1273] validation_0-logloss:0.691204
[1274] validation_0-logloss:0.691202
[1275] validation_0-logloss:0.691202
[1276] validation_0-logloss:0.691201
[1277] validation_0-logloss:0.691201
[1278] validation_0-logloss:0.691203
[1279] validation_0-logloss:0.691201
[1280] validation_0-logloss:0.691199
[1281] validation_0-logloss:0.691198
[1282] validation_0-logloss:0.691197
[1283] validation_0-logloss:0.691197
[1284] validation_0-logloss:0.691200
[1285] validation_0-logloss:0.691200
[1286] validation_0-logloss:0.691200
[1287] validation_0-logloss:0.691200
[1288] validation_0-logloss:0.691200
[1289] validation_0-logloss:0.691202
[1290] validation_0-logloss:0.691201
[1291] validation_0-logloss:0.691200
[1292] validation_0-logloss:0.691197
[1293] validation_0-logloss:0.691198
[1294] validation_0-logloss:0.691200
[1295] validation_0-logloss:0.691201
[1296] validation_0-logloss:0.691204
[1297] validation_0-logloss:0.691202
[1298] validation_0-logloss:0.691204
[1299] validation_0-logloss:0.691202
[1300] validation_0-logloss:0.691203
[1301] validation_0-logloss:0.691203
[1302] validation_0-logloss:0.691203
[1303] validation_0-logloss:0.691204
[1304] validation_0-logloss:0.691202
[1305] validation_0-logloss:0.691202
[1306] validation_0-logloss:0.691202
[1307] validation_0-logloss:0.691204
[1308] validation_0-logloss:0.691205
[1309] validation_0-logloss:0.691206
[1310] validation_0-logloss:0.691207
[1311] validation_0-logloss:0.691210
[1312] validation_0-logloss:0.691208
[1313] validation_0-logloss:0.691210
[1314] validation_0-logloss:0.691211
[1315] validation_0-logloss:0.691209
[1316] validation_0-logloss:0.691209
[1317] validation_0-logloss:0.691208
[1318] validation_0-logloss:0.691209
[1319] validation_0-logloss:0.691209
[1320] validation_0-logloss:0.691208
[1321] validation_0-logloss:0.691206
[1322] validation_0-logloss:0.691208
[1323] validation_0-logloss:0.691208
[1324] validation_0-logloss:0.691210
[1325] validation_0-logloss:0.691213
[1326] validation_0-logloss:0.691214
[1327] validation_0-logloss:0.691213
[1328] validation_0-logloss:0.691213
[1329] validation_0-logloss:0.691214
[1330] validation_0-logloss:0.691211
[1331] validation_0-logloss:0.691212
[1332] validation_0-logloss:0.691212
[1333] validation_0-logloss:0.691210
[1334] validation_0-logloss:0.691211
[1335] validation_0-logloss:0.691213
[1336] validation_0-logloss:0.691213
[1337] validation_0-logloss:0.691215
[1338] validation_0-logloss:0.691215
[1339] validation_0-logloss:0.691215
[1340] validation_0-logloss:0.691215
[1341] validation_0-logloss:0.691216
[1342] validation_0-logloss:0.691214
[1343] validation_0-logloss:0.691211
[1344] validation_0-logloss:0.691211
[1345] validation_0-logloss:0.691209
[1346] validation_0-logloss:0.691209
[1347] validation_0-logloss:0.691210
[1348] validation_0-logloss:0.691209
[1349] validation_0-logloss:0.691210
[1350] validation_0-logloss:0.691209
[1351] validation_0-logloss:0.691209
[1352] validation_0-logloss:0.691210
[1353] validation_0-logloss:0.691209
[1354] validation_0-logloss:0.691208
[1355] validation_0-logloss:0.691208
[1356] validation_0-logloss:0.691207
[1357] validation_0-logloss:0.691209
[1358] validation_0-logloss:0.691208
[1359] validation_0-logloss:0.691208
[1360] validation_0-logloss:0.691209
[1361] validation_0-logloss:0.691209
[1362] validation_0-logloss:0.691210
[1363] validation_0-logloss:0.691212
[1364] validation_0-logloss:0.691213
[1365] validation_0-logloss:0.691212
[1366] validation_0-logloss:0.691212
[1367] validation_0-logloss:0.691212
[1368] validation_0-logloss:0.691212
[1369] validation_0-logloss:0.691211
[1370] validation_0-logloss:0.691215
[1371] validation_0-logloss:0.691214
[1372] validation_0-logloss:0.691215
[1373] validation_0-logloss:0.691214
[1374] validation_0-logloss:0.691214
[1375] validation_0-logloss:0.691214
[1376] validation_0-logloss:0.691214
[1377] validation_0-logloss:0.691214
[1378] validation_0-logloss:0.691216
[1379] validation_0-logloss:0.691215
[1380] validation_0-logloss:0.691214
[1381] validation_0-logloss:0.691214
[1382] validation_0-logloss:0.691214
[1383] validation_0-logloss:0.691216
[1384] validation_0-logloss:0.691216
[1385] validation_0-logloss:0.691214
[1386] validation_0-logloss:0.691213
[1387] validation_0-logloss:0.691211
[1388] validation_0-logloss:0.691210
[1389] validation_0-logloss:0.691210
[1390] validation_0-logloss:0.691212
[1391] validation_0-logloss:0.691212
[1392] validation_0-logloss:0.691212
[1393] validation_0-logloss:0.691213
[1394] validation_0-logloss:0.691213
[1395] validation_0-logloss:0.691215
[1396] validation_0-logloss:0.691216
[1397] validation_0-logloss:0.691213
[1398] validation_0-logloss:0.691211
[1399] validation_0-logloss:0.691211
[1400] validation_0-logloss:0.691212
[1401] validation_0-logloss:0.691213
[1402] validation_0-logloss:0.691213
[1403] validation_0-logloss:0.691212
[1404] validation_0-logloss:0.691212
[1405] validation_0-logloss:0.691210
[1406] validation_0-logloss:0.691209
[1407] validation_0-logloss:0.691209
[1408] validation_0-logloss:0.691212
[1409] validation_0-logloss:0.691212
[1410] validation_0-logloss:0.691213
[1411] validation_0-logloss:0.691212
[1412] validation_0-logloss:0.691212
[1413] validation_0-logloss:0.691211
[1414] validation_0-logloss:0.691210
[1415] validation_0-logloss:0.691211
[1416] validation_0-logloss:0.691209
[1417] validation_0-logloss:0.691210
[1418] validation_0-logloss:0.691209
[1419] validation_0-logloss:0.691209
[1420] validation_0-logloss:0.691209
[1421] validation_0-logloss:0.691208
[1422] validation_0-logloss:0.691210
[1423] validation_0-logloss:0.691209
[1424] validation_0-logloss:0.691208
[1425] validation_0-logloss:0.691208
[1426] validation_0-logloss:0.691209
[1427] validation_0-logloss:0.691210
[1428] validation_0-logloss:0.691210
[1429] validation_0-logloss:0.691212
[1430] validation_0-logloss:0.691210
[1431] validation_0-logloss:0.691209
[1432] validation_0-logloss:0.691205
Stopping. Best iteration:
[1282] validation_0-logloss:0.691197
-----------------------
logloss train: 0.68744
logloss valid: 0.69121
-----------------------
Model parameters...
{'reg_alpha': 0, 'colsample_bytree': 0.8, 'silent': True, 'colsample_bylevel': 1, 'scale_pos_weight': 1, 'learning_rate': 0.005, 'missing': None, 'max_delta_step': 0, 'nthread': -1, 'base_score': 0.5, 'n_estimators': 1500, 'subsample': 0.9, 'reg_lambda': 1, 'seed': 2100, 'min_child_weight': 3, 'objective': 'binary:logistic', 'max_depth': 3, 'gamma': 0.3}
In [5]:
y_pred= clf.predict_proba(test)[:,1]
submission = pd.DataFrame({"t_id":id_test, "probability":y_pred})
submission.to_csv("submission_xgboost.csv", index=False)
In [8]:
print X_fit.values.shape , y_fit.shape, X_eval.shape, y_eval.shape
(86688, 21) (86688,) (9632, 21) (9632,)
In [19]:
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation
from keras.optimizers import Adam
from keras.utils.np_utils import to_categorical
nn = Sequential()
nn.add(Dense(250, input_dim=21, init='uniform'))
nn.add(Dropout(0.4))
nn.add(Activation('sigmoid'))
nn.add(Dense(2, init='uniform'))
nn.add(Activation('softmax'))
opt = Adam(lr=1e-3)
nn.compile(loss='sparse_categorical_crossentropy', optimizer=opt)
#X_fit, y_fit, early_stopping_rounds=150, eval_metric="logloss", eval_set=[(X_eval, y_eval)
hist = nn.fit(X_fit.values, y_fit.values, nb_epoch=125, verbose=0,validation_data=(X_eval.values, y_eval.values))
In [20]:
y_pred= nn.predict_proba(test.values, verbose=0)[:,1]
submission = pd.DataFrame({"t_id":id_test, "probability":y_pred})
submission.to_csv("submission_NN.csv", index=False)
In [21]:
%matplotlib inline
plt.plot(hist.history['loss'][0:])
plt.plot(hist.history['val_loss'][0:])
plt.legend(['Train loss', 'Validation loss'])
Out[21]:
<matplotlib.legend.Legend at 0x7f47c1c8d910>
In [50]:
clf1_p = rf.predict_proba(X_eval)[:,1]
#Logreg
clf7_p = lrCV.predict_proba(X_eval)[:,1]
#xgb
clf8_p = clf.predict_proba(X_eval)[:,1]
# #NN
clf3_p = nn.predict_proba(X_eval.values, verbose = 0)[:,1]
In [51]:
classifiers = [clf1_p,
clf8_p,
clf3_p,
clf7_p]
predictions = pd.DataFrame(classifiers[0:2]).T
for classif in classifiers:
print(log_loss(y_eval, classif))
0.691367878221
0.691205202806
0.691689995766
0.691449256838
In [52]:
print(np.corrcoef(classifiers))
[[ 1. 0.93818692 0.88003664 0.85373216]
[ 0.93818692 1. 0.85944264 0.84256706]
[ 0.88003664 0.85944264 1. 0.98117145]
[ 0.85373216 0.84256706 0.98117145 1. ]]
In [53]:
Ps = np.array(classifiers).T
def ensemble_logloss(w):
return log_loss(y_eval, (Ps*w).sum(1).T/np.sum(w))
In [54]:
ensemble_logloss(np.ones(len(classifiers)))
Out[54]:
0.69127890242990808
In [55]:
def ensemble_roc_auc(w):
return roc_auc_score(y_eval, (Ps*w).sum(1).T/np.sum(w))
In [56]:
ensemble_roc_auc(np.ones(len(classifiers)))
Out[56]:
0.5348664624138677
In [57]:
from scipy.optimize import minimize
opt = minimize(ensemble_logloss, x0=np.ones(len(classifiers)))
print("Function result:", opt.fun)
print("Array:", opt.x)
('Function result:', 0.6911392619336544)
('Array:', array([-0.58555103, 3.6020983 , -1.62525096, 2.6093859 ]))
In [58]:
w = opt.x/opt.x.sum()
w
Out[58]:
array([-0.1463628 , 0.90037102, -0.40624346, 0.65223524])
In [59]:
ensebmle_prob_validation = np.zeros(len(clf1_p))
for i in range(len(classifiers)):
ensebmle_prob_validation += np.array(classifiers[i]) * w[i]
In [60]:
print log_loss(y_eval, ensebmle_prob_validation), roc_auc_score(y_eval, ensebmle_prob_validation)
0.691139261991 0.53506998657
In [38]:
def optimal(values):
optimal_classifiers = []
for i in range(len(values)):
if values[i] > 0:
print("Classifier", i+1, "is good")
optimal_classifiers.append(classifiers[i])
return optimal_classifiers
In [39]:
def full_negative_elimination(weights):
global classifiers, Ps
while sum(weights<0) > 0:
opt = minimize(ensemble_logloss, x0=np.ones(len(classifiers)))
print("Function result:", opt.fun)
classifiers = optimal(opt.x)
Ps = np.array(classifiers).T
weights = opt.x/opt.x.sum()
print("Current weights:", weights)
print('=======')
return weights
In [40]:
w = full_negative_elimination(w)
('Function result:', 0.6911392619336544)
('Classifier', 2, 'is good')
('Classifier', 4, 'is good')
('Current weights:', array([-0.1463628 , 0.90037102, -0.40624346, 0.65223524]))
=======
('Function result:', 0.6911713540305157)
('Classifier', 1, 'is good')
('Classifier', 2, 'is good')
('Current weights:', array([ 0.74141642, 0.25858358]))
=======
In [61]:
dtest = xgb.DMatrix(test)
p1 = rf.predict_proba(test)[:,1]
#Logreg
p7 = lrCV.predict_proba(test)[:,1]
#xgb
p8 = clf.predict_proba(test)[:,1]
p3 = nn.predict_proba(test.values, verbose = 0)[:,1]
test_predictions = [p1, p7, p8, p3]
In [62]:
ensebmle_prob_test = np.zeros(len(p1))
for i in range(len(test_predictions)):
ensebmle_prob_test += np.array(test_predictions[i]) * w[i]
In [63]:
pd.DataFrame(ensebmle_prob_validation).hist()
Out[63]:
array([[<matplotlib.axes._subplots.AxesSubplot object at 0x7f47b80631d0>]], dtype=object)
In [64]:
pd.DataFrame(ensebmle_prob_test).hist()
Out[64]:
array([[<matplotlib.axes._subplots.AxesSubplot object at 0x7f47be26a450>]], dtype=object)
In [65]:
submission = pd.DataFrame({"t_id":id_test, "probability":ensebmle_prob_test})
submission.to_csv("submission_ansemble.csv", index=False)
In [ ]:
Content source: Diyago/Machine-Learning-scripts
Similar notebooks: