Imports


In [1]:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
plt.figure(figsize=(12,12))
%matplotlib inline

In [2]:
PATH = "data\\bulldozers\\"
train = pd.read_csv(f'{PATH}train_jDb5RBj.csv', low_memory=False)
ids, y = train['ID'], train['Purchase']

In [3]:
test = pd.read_csv(PATH + 'test_dan2xFI.csv')

In [4]:
train_ = train.drop(['ID','Purchase'],axis = 1)
test_ = test.copy()
test_ = test_.drop('ID',axis = 1)
for i in train_.columns:
    train_[i] = train_[i].apply(str)
    test_[i] = test_[i].apply(str)

In [5]:
categorical_features_indices = np.where(train_.dtypes != np.float)[0]

In [6]:
from sklearn.model_selection import train_test_split
X_train, X_validation, y_train, y_validation = train_test_split(train_, train['Purchase'], train_size=0.8, random_state=1234)


C:\ProgramData\Anaconda3\lib\site-packages\sklearn\model_selection\_split.py:2026: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.
  FutureWarning)

In [7]:
#importing library and building model
from catboost import CatBoostClassifier
model=CatBoostClassifier(iterations=100, depth=10, learning_rate=0.01, loss_function='Logloss',class_weights=[1,3])
model.fit(X_train, y_train,cat_features=categorical_features_indices,eval_set=(X_validation, y_validation))


0:	learn: 0.6846107	test: 0.6851212	best: 0.6851212 (0)	total: 1.34s	remaining: 2m 12s
1:	learn: 0.6774166	test: 0.6781991	best: 0.6781991 (1)	total: 1.46s	remaining: 1m 11s
2:	learn: 0.6703312	test: 0.6709523	best: 0.6709523 (2)	total: 1.52s	remaining: 49.1s
3:	learn: 0.6633268	test: 0.6641514	best: 0.6641514 (3)	total: 2.8s	remaining: 1m 7s
4:	learn: 0.6568817	test: 0.6575711	best: 0.6575711 (4)	total: 2.99s	remaining: 56.7s
5:	learn: 0.6498401	test: 0.6508585	best: 0.6508585 (5)	total: 3.32s	remaining: 52s
6:	learn: 0.6428361	test: 0.6441284	best: 0.6441284 (6)	total: 4.56s	remaining: 1m
7:	learn: 0.6342452	test: 0.6366085	best: 0.6366085 (7)	total: 5.94s	remaining: 1m 8s
8:	learn: 0.6290908	test: 0.6316594	best: 0.6316594 (8)	total: 5.99s	remaining: 1m
9:	learn: 0.6237369	test: 0.6260270	best: 0.6260270 (9)	total: 6s	remaining: 54s
10:	learn: 0.6176003	test: 0.6203778	best: 0.6203778 (10)	total: 6.11s	remaining: 49.5s
11:	learn: 0.6117612	test: 0.6147960	best: 0.6147960 (11)	total: 6.23s	remaining: 45.7s
12:	learn: 0.6053775	test: 0.6088854	best: 0.6088854 (12)	total: 6.4s	remaining: 42.8s
13:	learn: 0.5987142	test: 0.6038027	best: 0.6038027 (13)	total: 7.63s	remaining: 46.9s
14:	learn: 0.5929690	test: 0.5991334	best: 0.5991334 (14)	total: 7.95s	remaining: 45s
15:	learn: 0.5879208	test: 0.5945857	best: 0.5945857 (15)	total: 8.05s	remaining: 42.3s
16:	learn: 0.5823904	test: 0.5898558	best: 0.5898558 (16)	total: 8.37s	remaining: 40.9s
17:	learn: 0.5766412	test: 0.5847569	best: 0.5847569 (17)	total: 9.54s	remaining: 43.5s
18:	learn: 0.5702046	test: 0.5789695	best: 0.5789695 (18)	total: 10.7s	remaining: 45.7s
19:	learn: 0.5653197	test: 0.5743271	best: 0.5743271 (19)	total: 10.9s	remaining: 43.6s
20:	learn: 0.5601283	test: 0.5694374	best: 0.5694374 (20)	total: 11.1s	remaining: 41.8s
21:	learn: 0.5540715	test: 0.5648978	best: 0.5648978 (21)	total: 12.4s	remaining: 43.8s
22:	learn: 0.5490823	test: 0.5611365	best: 0.5611365 (22)	total: 13.5s	remaining: 45.3s
23:	learn: 0.5440077	test: 0.5569110	best: 0.5569110 (23)	total: 14.7s	remaining: 46.5s
24:	learn: 0.5383787	test: 0.5518770	best: 0.5518770 (24)	total: 15.5s	remaining: 46.5s
25:	learn: 0.5339498	test: 0.5483642	best: 0.5483642 (25)	total: 16.7s	remaining: 47.5s
26:	learn: 0.5285625	test: 0.5438955	best: 0.5438955 (26)	total: 18s	remaining: 48.6s
27:	learn: 0.5236876	test: 0.5400300	best: 0.5400300 (27)	total: 19.2s	remaining: 49.4s
28:	learn: 0.5180306	test: 0.5354992	best: 0.5354992 (28)	total: 20.5s	remaining: 50.1s
29:	learn: 0.5133375	test: 0.5315355	best: 0.5315355 (29)	total: 21.7s	remaining: 50.6s
30:	learn: 0.5093522	test: 0.5278998	best: 0.5278998 (30)	total: 21.8s	remaining: 48.6s
31:	learn: 0.5052116	test: 0.5243207	best: 0.5243207 (31)	total: 22.1s	remaining: 47s
32:	learn: 0.5005616	test: 0.5212322	best: 0.5212322 (32)	total: 23.4s	remaining: 47.4s
33:	learn: 0.4953107	test: 0.5171557	best: 0.5171557 (33)	total: 24.6s	remaining: 47.7s
34:	learn: 0.4908019	test: 0.5141842	best: 0.5141842 (34)	total: 25.7s	remaining: 47.7s
35:	learn: 0.4864196	test: 0.5105994	best: 0.5105994 (35)	total: 26.9s	remaining: 47.9s
36:	learn: 0.4832281	test: 0.5080687	best: 0.5080687 (36)	total: 28.1s	remaining: 47.9s
37:	learn: 0.4786549	test: 0.5049929	best: 0.5049929 (37)	total: 29.3s	remaining: 47.9s
38:	learn: 0.4737738	test: 0.5012561	best: 0.5012561 (38)	total: 30.6s	remaining: 47.9s
39:	learn: 0.4700828	test: 0.4984139	best: 0.4984139 (39)	total: 31.8s	remaining: 47.7s
40:	learn: 0.4658365	test: 0.4953485	best: 0.4953485 (40)	total: 32.9s	remaining: 47.3s
41:	learn: 0.4619283	test: 0.4922813	best: 0.4922813 (41)	total: 34.2s	remaining: 47.2s
42:	learn: 0.4579420	test: 0.4896317	best: 0.4896317 (42)	total: 35.4s	remaining: 47s
43:	learn: 0.4541355	test: 0.4870450	best: 0.4870450 (43)	total: 36.7s	remaining: 46.7s
44:	learn: 0.4500458	test: 0.4844845	best: 0.4844845 (44)	total: 38s	remaining: 46.5s
45:	learn: 0.4466108	test: 0.4819691	best: 0.4819691 (45)	total: 39.3s	remaining: 46.1s
46:	learn: 0.4429026	test: 0.4794433	best: 0.4794433 (46)	total: 40.5s	remaining: 45.7s
47:	learn: 0.4400498	test: 0.4773387	best: 0.4773387 (47)	total: 41.8s	remaining: 45.3s
48:	learn: 0.4364705	test: 0.4748048	best: 0.4748048 (48)	total: 43s	remaining: 44.7s
49:	learn: 0.4348272	test: 0.4733271	best: 0.4733271 (49)	total: 43s	remaining: 43s
50:	learn: 0.4300294	test: 0.4709129	best: 0.4709129 (50)	total: 44.5s	remaining: 42.7s
51:	learn: 0.4266212	test: 0.4685052	best: 0.4685052 (51)	total: 45.8s	remaining: 42.3s
52:	learn: 0.4236955	test: 0.4664086	best: 0.4664086 (52)	total: 47.1s	remaining: 41.8s
53:	learn: 0.4209936	test: 0.4645258	best: 0.4645258 (53)	total: 48.6s	remaining: 41.4s
54:	learn: 0.4188116	test: 0.4632887	best: 0.4632887 (54)	total: 48.8s	remaining: 39.9s
55:	learn: 0.4163085	test: 0.4615756	best: 0.4615756 (55)	total: 50.1s	remaining: 39.3s
56:	learn: 0.4133502	test: 0.4599033	best: 0.4599033 (56)	total: 51.2s	remaining: 38.7s
57:	learn: 0.4108964	test: 0.4581467	best: 0.4581467 (57)	total: 52.2s	remaining: 37.8s
58:	learn: 0.4077358	test: 0.4559715	best: 0.4559715 (58)	total: 53.5s	remaining: 37.2s
59:	learn: 0.4052004	test: 0.4541004	best: 0.4541004 (59)	total: 54.9s	remaining: 36.6s
60:	learn: 0.4015643	test: 0.4523911	best: 0.4523911 (60)	total: 56.3s	remaining: 36s
61:	learn: 0.3988386	test: 0.4506580	best: 0.4506580 (61)	total: 57.6s	remaining: 35.3s
62:	learn: 0.3962874	test: 0.4490255	best: 0.4490255 (62)	total: 59s	remaining: 34.6s
63:	learn: 0.3932912	test: 0.4474110	best: 0.4474110 (63)	total: 1m	remaining: 33.9s
64:	learn: 0.3906673	test: 0.4458125	best: 0.4458125 (64)	total: 1m 1s	remaining: 33.2s
65:	learn: 0.3884126	test: 0.4444111	best: 0.4444111 (65)	total: 1m 2s	remaining: 32.4s
66:	learn: 0.3856602	test: 0.4428466	best: 0.4428466 (66)	total: 1m 4s	remaining: 31.6s
67:	learn: 0.3838804	test: 0.4416219	best: 0.4416219 (67)	total: 1m 5s	remaining: 30.8s
68:	learn: 0.3818333	test: 0.4405034	best: 0.4405034 (68)	total: 1m 6s	remaining: 30s
69:	learn: 0.3793150	test: 0.4387252	best: 0.4387252 (69)	total: 1m 8s	remaining: 29.2s
70:	learn: 0.3769366	test: 0.4368836	best: 0.4368836 (70)	total: 1m 9s	remaining: 28.4s
71:	learn: 0.3749031	test: 0.4360504	best: 0.4360504 (71)	total: 1m 10s	remaining: 27.5s
72:	learn: 0.3729186	test: 0.4348334	best: 0.4348334 (72)	total: 1m 11s	remaining: 26.6s
73:	learn: 0.3712127	test: 0.4340776	best: 0.4340776 (73)	total: 1m 13s	remaining: 25.7s
74:	learn: 0.3690301	test: 0.4324599	best: 0.4324599 (74)	total: 1m 14s	remaining: 24.9s
75:	learn: 0.3675519	test: 0.4317664	best: 0.4317664 (75)	total: 1m 15s	remaining: 23.9s
76:	learn: 0.3654604	test: 0.4302705	best: 0.4302705 (76)	total: 1m 17s	remaining: 23.1s
77:	learn: 0.3628167	test: 0.4292806	best: 0.4292806 (77)	total: 1m 18s	remaining: 22.3s
78:	learn: 0.3605580	test: 0.4283900	best: 0.4283900 (78)	total: 1m 20s	remaining: 21.4s
79:	learn: 0.3588577	test: 0.4271313	best: 0.4271313 (79)	total: 1m 21s	remaining: 20.4s
80:	learn: 0.3563797	test: 0.4258146	best: 0.4258146 (80)	total: 1m 23s	remaining: 19.5s
81:	learn: 0.3546985	test: 0.4246836	best: 0.4246836 (81)	total: 1m 24s	remaining: 18.6s
82:	learn: 0.3531101	test: 0.4241985	best: 0.4241985 (82)	total: 1m 26s	remaining: 17.7s
83:	learn: 0.3516658	test: 0.4231903	best: 0.4231903 (83)	total: 1m 27s	remaining: 16.7s
84:	learn: 0.3486089	test: 0.4221170	best: 0.4221170 (84)	total: 1m 28s	remaining: 15.7s
85:	learn: 0.3466808	test: 0.4217272	best: 0.4217272 (85)	total: 1m 30s	remaining: 14.7s
86:	learn: 0.3451212	test: 0.4210886	best: 0.4210886 (86)	total: 1m 31s	remaining: 13.6s
87:	learn: 0.3436808	test: 0.4203547	best: 0.4203547 (87)	total: 1m 32s	remaining: 12.6s
88:	learn: 0.3415225	test: 0.4187994	best: 0.4187994 (88)	total: 1m 34s	remaining: 11.6s
89:	learn: 0.3390746	test: 0.4177020	best: 0.4177020 (89)	total: 1m 35s	remaining: 10.6s
90:	learn: 0.3372971	test: 0.4172210	best: 0.4172210 (90)	total: 1m 37s	remaining: 9.6s
91:	learn: 0.3349815	test: 0.4164435	best: 0.4164435 (91)	total: 1m 38s	remaining: 8.57s
92:	learn: 0.3328779	test: 0.4154042	best: 0.4154042 (92)	total: 1m 39s	remaining: 7.52s
93:	learn: 0.3315426	test: 0.4144781	best: 0.4144781 (93)	total: 1m 41s	remaining: 6.46s
94:	learn: 0.3301840	test: 0.4141345	best: 0.4141345 (94)	total: 1m 42s	remaining: 5.39s
95:	learn: 0.3285664	test: 0.4131713	best: 0.4131713 (95)	total: 1m 43s	remaining: 4.33s
96:	learn: 0.3267167	test: 0.4124298	best: 0.4124298 (96)	total: 1m 45s	remaining: 3.26s
97:	learn: 0.3251636	test: 0.4120925	best: 0.4120925 (97)	total: 1m 46s	remaining: 2.18s
98:	learn: 0.3234726	test: 0.4112408	best: 0.4112408 (98)	total: 1m 48s	remaining: 1.09s
99:	learn: 0.3211949	test: 0.4103270	best: 0.4103270 (99)	total: 1m 49s	remaining: 0us

bestTest = 0.4103269702
bestIteration = 99

Out[7]:
<catboost.core.CatBoostClassifier at 0x234e69846d8>

In [77]:
model=CatBoostClassifier(iterations=300, depth=10, learning_rate=0.02, loss_function='Logloss')
model.fit(X_train, y_train,cat_features=categorical_features_indices,eval_set=(X_validation, y_validation))


0:	learn: 0.6699248	test: 0.6698186	best: 0.6698186 (0)	total: 49.7ms	remaining: 14.9s
1:	learn: 0.6476912	test: 0.6478399	best: 0.6478399 (1)	total: 208ms	remaining: 31s
2:	learn: 0.6267564	test: 0.6267739	best: 0.6267739 (2)	total: 223ms	remaining: 22.1s
3:	learn: 0.6069451	test: 0.6068209	best: 0.6068209 (3)	total: 337ms	remaining: 25s
4:	learn: 0.5880965	test: 0.5878693	best: 0.5878693 (4)	total: 343ms	remaining: 20.3s
5:	learn: 0.5701798	test: 0.5701452	best: 0.5701452 (5)	total: 373ms	remaining: 18.3s
6:	learn: 0.5532184	test: 0.5531126	best: 0.5531126 (6)	total: 387ms	remaining: 16.2s
7:	learn: 0.5369603	test: 0.5369145	best: 0.5369145 (7)	total: 416ms	remaining: 15.2s
8:	learn: 0.5217177	test: 0.5215659	best: 0.5215659 (8)	total: 422ms	remaining: 13.6s
9:	learn: 0.5072663	test: 0.5070108	best: 0.5070108 (9)	total: 429ms	remaining: 12.4s
10:	learn: 0.4935067	test: 0.4930766	best: 0.4930766 (10)	total: 443ms	remaining: 11.6s
11:	learn: 0.4805356	test: 0.4800046	best: 0.4800046 (11)	total: 456ms	remaining: 11s
12:	learn: 0.4679805	test: 0.4669682	best: 0.4669682 (12)	total: 500ms	remaining: 11s
13:	learn: 0.4563390	test: 0.4552751	best: 0.4552751 (13)	total: 549ms	remaining: 11.2s
14:	learn: 0.4452851	test: 0.4441467	best: 0.4441467 (14)	total: 555ms	remaining: 10.6s
15:	learn: 0.4348114	test: 0.4336158	best: 0.4336158 (15)	total: 569ms	remaining: 10.1s
16:	learn: 0.4240683	test: 0.4225771	best: 0.4225771 (16)	total: 597ms	remaining: 9.94s
17:	learn: 0.4146895	test: 0.4131061	best: 0.4131061 (17)	total: 622ms	remaining: 9.74s
18:	learn: 0.4057878	test: 0.4041497	best: 0.4041497 (18)	total: 628ms	remaining: 9.28s
19:	learn: 0.3972199	test: 0.3955213	best: 0.3955213 (19)	total: 671ms	remaining: 9.4s
20:	learn: 0.3891578	test: 0.3874087	best: 0.3874087 (20)	total: 1.8s	remaining: 23.9s
21:	learn: 0.3815362	test: 0.3797194	best: 0.3797194 (21)	total: 1.85s	remaining: 23.4s
22:	learn: 0.3741071	test: 0.3720933	best: 0.3720933 (22)	total: 1.96s	remaining: 23.6s
23:	learn: 0.3673073	test: 0.3653080	best: 0.3653080 (23)	total: 2.01s	remaining: 23.1s
24:	learn: 0.3608499	test: 0.3588011	best: 0.3588011 (24)	total: 2.04s	remaining: 22.5s
25:	learn: 0.3537078	test: 0.3513342	best: 0.3513342 (25)	total: 2.23s	remaining: 23.5s
26:	learn: 0.3479241	test: 0.3455111	best: 0.3455111 (26)	total: 2.24s	remaining: 22.7s
27:	learn: 0.3424080	test: 0.3399733	best: 0.3399733 (27)	total: 2.32s	remaining: 22.5s
28:	learn: 0.3371983	test: 0.3347304	best: 0.3347304 (28)	total: 2.33s	remaining: 21.8s
29:	learn: 0.3321343	test: 0.3295918	best: 0.3295918 (29)	total: 2.51s	remaining: 22.6s
30:	learn: 0.3274274	test: 0.3248548	best: 0.3248548 (30)	total: 2.54s	remaining: 22s
31:	learn: 0.3228180	test: 0.3202891	best: 0.3202891 (31)	total: 2.61s	remaining: 21.9s
32:	learn: 0.3185756	test: 0.3160298	best: 0.3160298 (32)	total: 2.64s	remaining: 21.4s
33:	learn: 0.3145545	test: 0.3119763	best: 0.3119763 (33)	total: 2.65s	remaining: 20.7s
34:	learn: 0.3105990	test: 0.3079465	best: 0.3079465 (34)	total: 2.67s	remaining: 20.2s
35:	learn: 0.3069430	test: 0.3042702	best: 0.3042702 (35)	total: 2.69s	remaining: 19.7s
36:	learn: 0.3034988	test: 0.3007944	best: 0.3007944 (36)	total: 2.69s	remaining: 19.1s
37:	learn: 0.3000976	test: 0.2974785	best: 0.2974785 (37)	total: 2.73s	remaining: 18.8s
38:	learn: 0.2969688	test: 0.2943242	best: 0.2943242 (38)	total: 2.74s	remaining: 18.4s
39:	learn: 0.2936732	test: 0.2910222	best: 0.2910222 (39)	total: 2.86s	remaining: 18.6s
40:	learn: 0.2908743	test: 0.2881981	best: 0.2881981 (40)	total: 2.87s	remaining: 18.2s
41:	learn: 0.2882085	test: 0.2855062	best: 0.2855062 (41)	total: 2.88s	remaining: 17.7s
42:	learn: 0.2856753	test: 0.2829461	best: 0.2829461 (42)	total: 2.9s	remaining: 17.3s
43:	learn: 0.2832632	test: 0.2805085	best: 0.2805085 (43)	total: 2.9s	remaining: 16.9s
44:	learn: 0.2809683	test: 0.2781883	best: 0.2781883 (44)	total: 2.91s	remaining: 16.5s
45:	learn: 0.2781508	test: 0.2750774	best: 0.2750774 (45)	total: 2.93s	remaining: 16.2s
46:	learn: 0.2760860	test: 0.2729907	best: 0.2729907 (46)	total: 2.93s	remaining: 15.8s
47:	learn: 0.2730478	test: 0.2697246	best: 0.2697246 (47)	total: 2.96s	remaining: 15.6s
48:	learn: 0.2711982	test: 0.2678563	best: 0.2678563 (48)	total: 2.97s	remaining: 15.2s
49:	learn: 0.2692829	test: 0.2659138	best: 0.2659138 (49)	total: 3s	remaining: 15s
50:	learn: 0.2674820	test: 0.2641628	best: 0.2641628 (50)	total: 3.07s	remaining: 15s
51:	learn: 0.2658132	test: 0.2624157	best: 0.2624157 (51)	total: 3.08s	remaining: 14.7s
52:	learn: 0.2642961	test: 0.2608812	best: 0.2608812 (52)	total: 3.09s	remaining: 14.4s
53:	learn: 0.2628507	test: 0.2594184	best: 0.2594184 (53)	total: 3.1s	remaining: 14.1s
54:	learn: 0.2612322	test: 0.2578579	best: 0.2578579 (54)	total: 3.21s	remaining: 14.3s
55:	learn: 0.2578508	test: 0.2552524	best: 0.2552524 (55)	total: 4.51s	remaining: 19.6s
56:	learn: 0.2550612	test: 0.2527427	best: 0.2527427 (56)	total: 4.69s	remaining: 20s
57:	learn: 0.2539190	test: 0.2515928	best: 0.2515928 (57)	total: 4.7s	remaining: 19.6s
58:	learn: 0.2524797	test: 0.2501967	best: 0.2501967 (58)	total: 4.73s	remaining: 19.3s
59:	learn: 0.2500328	test: 0.2478789	best: 0.2478789 (59)	total: 4.79s	remaining: 19.2s
60:	learn: 0.2490693	test: 0.2469134	best: 0.2469134 (60)	total: 4.8s	remaining: 18.8s
61:	learn: 0.2480771	test: 0.2459185	best: 0.2459185 (61)	total: 4.82s	remaining: 18.5s
62:	learn: 0.2466865	test: 0.2448099	best: 0.2448099 (62)	total: 4.94s	remaining: 18.6s
63:	learn: 0.2450999	test: 0.2430377	best: 0.2430377 (63)	total: 4.96s	remaining: 18.3s
64:	learn: 0.2443158	test: 0.2422470	best: 0.2422470 (64)	total: 4.97s	remaining: 18s
65:	learn: 0.2435698	test: 0.2415018	best: 0.2415018 (65)	total: 4.99s	remaining: 17.7s
66:	learn: 0.2428543	test: 0.2407866	best: 0.2407866 (66)	total: 5.02s	remaining: 17.4s
67:	learn: 0.2421741	test: 0.2401037	best: 0.2401037 (67)	total: 5.02s	remaining: 17.1s
68:	learn: 0.2415245	test: 0.2394513	best: 0.2394513 (68)	total: 5.03s	remaining: 16.8s
69:	learn: 0.2395700	test: 0.2375326	best: 0.2375326 (69)	total: 5.22s	remaining: 17.1s
70:	learn: 0.2377648	test: 0.2357762	best: 0.2357762 (70)	total: 6.3s	remaining: 20.3s
71:	learn: 0.2362351	test: 0.2344290	best: 0.2344290 (71)	total: 6.39s	remaining: 20.2s
72:	learn: 0.2357313	test: 0.2339276	best: 0.2339276 (72)	total: 6.41s	remaining: 19.9s
73:	learn: 0.2338019	test: 0.2324064	best: 0.2324064 (73)	total: 6.64s	remaining: 20.3s
74:	learn: 0.2323257	test: 0.2311581	best: 0.2311581 (74)	total: 6.74s	remaining: 20.2s
75:	learn: 0.2308908	test: 0.2305792	best: 0.2305792 (75)	total: 7.8s	remaining: 23s
76:	learn: 0.2298852	test: 0.2294796	best: 0.2294796 (76)	total: 7.87s	remaining: 22.8s
77:	learn: 0.2291614	test: 0.2291436	best: 0.2291436 (77)	total: 8.03s	remaining: 22.9s
78:	learn: 0.2283219	test: 0.2285090	best: 0.2285090 (78)	total: 8.13s	remaining: 22.8s
79:	learn: 0.2279903	test: 0.2281836	best: 0.2281836 (79)	total: 8.14s	remaining: 22.4s
80:	learn: 0.2260114	test: 0.2271351	best: 0.2271351 (80)	total: 9.2s	remaining: 24.9s
81:	learn: 0.2257179	test: 0.2268487	best: 0.2268487 (81)	total: 9.21s	remaining: 24.5s
82:	learn: 0.2252395	test: 0.2264539	best: 0.2264539 (82)	total: 9.22s	remaining: 24.1s
83:	learn: 0.2243148	test: 0.2256937	best: 0.2256937 (83)	total: 9.27s	remaining: 23.8s
84:	learn: 0.2233506	test: 0.2247065	best: 0.2247065 (84)	total: 9.34s	remaining: 23.6s
85:	learn: 0.2227842	test: 0.2241265	best: 0.2241265 (85)	total: 9.45s	remaining: 23.5s
86:	learn: 0.2214429	test: 0.2236277	best: 0.2236277 (86)	total: 10.7s	remaining: 26.1s
87:	learn: 0.2212347	test: 0.2234251	best: 0.2234251 (87)	total: 10.7s	remaining: 25.7s
88:	learn: 0.2196218	test: 0.2227375	best: 0.2227375 (88)	total: 11.3s	remaining: 26.7s
89:	learn: 0.2182690	test: 0.2221267	best: 0.2221267 (89)	total: 12.4s	remaining: 29s
90:	learn: 0.2177043	test: 0.2218750	best: 0.2218750 (90)	total: 12.6s	remaining: 28.9s
91:	learn: 0.2164782	test: 0.2211767	best: 0.2211767 (91)	total: 13.7s	remaining: 31.1s
92:	learn: 0.2151982	test: 0.2207988	best: 0.2207988 (92)	total: 14.9s	remaining: 33.2s
93:	learn: 0.2150588	test: 0.2206674	best: 0.2206674 (93)	total: 14.9s	remaining: 32.7s
94:	learn: 0.2149202	test: 0.2205574	best: 0.2205574 (94)	total: 14.9s	remaining: 32.2s
95:	learn: 0.2129441	test: 0.2197810	best: 0.2197810 (95)	total: 16s	remaining: 34.1s
96:	learn: 0.2124909	test: 0.2192588	best: 0.2192588 (96)	total: 16s	remaining: 33.6s
97:	learn: 0.2115224	test: 0.2189902	best: 0.2189902 (97)	total: 16.4s	remaining: 33.8s
98:	learn: 0.2101401	test: 0.2184358	best: 0.2184358 (98)	total: 17.6s	remaining: 35.7s
99:	learn: 0.2089639	test: 0.2180690	best: 0.2180690 (99)	total: 18.9s	remaining: 37.7s
100:	learn: 0.2080176	test: 0.2180141	best: 0.2180141 (100)	total: 19.4s	remaining: 38.3s
101:	learn: 0.2074456	test: 0.2174465	best: 0.2174465 (101)	total: 19.5s	remaining: 37.8s
102:	learn: 0.2070024	test: 0.2169332	best: 0.2169332 (102)	total: 19.5s	remaining: 37.3s
103:	learn: 0.2057159	test: 0.2167689	best: 0.2167689 (103)	total: 20.7s	remaining: 39s
104:	learn: 0.2046113	test: 0.2166293	best: 0.2166293 (104)	total: 22s	remaining: 40.8s
105:	learn: 0.2032887	test: 0.2165436	best: 0.2165436 (105)	total: 23.2s	remaining: 42.4s
106:	learn: 0.2024536	test: 0.2163619	best: 0.2163619 (106)	total: 24.8s	remaining: 44.7s
107:	learn: 0.2012789	test: 0.2163007	best: 0.2163007 (107)	total: 26.1s	remaining: 46.5s
108:	learn: 0.2006345	test: 0.2161704	best: 0.2161704 (108)	total: 26.8s	remaining: 46.9s
109:	learn: 0.1997828	test: 0.2159044	best: 0.2159044 (109)	total: 27.1s	remaining: 46.8s
110:	learn: 0.1989807	test: 0.2156203	best: 0.2156203 (110)	total: 27.2s	remaining: 46.3s
111:	learn: 0.1980651	test: 0.2154132	best: 0.2154132 (111)	total: 28.4s	remaining: 47.7s
112:	learn: 0.1978297	test: 0.2151250	best: 0.2151250 (112)	total: 28.5s	remaining: 47.1s
113:	learn: 0.1970880	test: 0.2150038	best: 0.2150038 (113)	total: 29.7s	remaining: 48.5s
114:	learn: 0.1968154	test: 0.2150377	best: 0.2150038 (113)	total: 29.8s	remaining: 47.9s
115:	learn: 0.1960675	test: 0.2146209	best: 0.2146209 (115)	total: 30.5s	remaining: 48.3s
116:	learn: 0.1953365	test: 0.2142172	best: 0.2142172 (116)	total: 31.9s	remaining: 49.9s
117:	learn: 0.1947166	test: 0.2140275	best: 0.2140275 (117)	total: 33.3s	remaining: 51.4s
118:	learn: 0.1941316	test: 0.2141365	best: 0.2140275 (117)	total: 34s	remaining: 51.7s
119:	learn: 0.1936138	test: 0.2139612	best: 0.2139612 (119)	total: 34.7s	remaining: 52s
120:	learn: 0.1932606	test: 0.2138885	best: 0.2138885 (120)	total: 34.8s	remaining: 51.5s
121:	learn: 0.1925428	test: 0.2136448	best: 0.2136448 (121)	total: 35s	remaining: 51.1s
122:	learn: 0.1915764	test: 0.2137175	best: 0.2136448 (121)	total: 36.3s	remaining: 52.2s
123:	learn: 0.1913611	test: 0.2134407	best: 0.2134407 (123)	total: 36.3s	remaining: 51.6s
124:	learn: 0.1913500	test: 0.2134324	best: 0.2134324 (124)	total: 36.3s	remaining: 50.9s
125:	learn: 0.1909685	test: 0.2134454	best: 0.2134324 (124)	total: 36.7s	remaining: 50.7s
126:	learn: 0.1907279	test: 0.2133736	best: 0.2133736 (126)	total: 36.8s	remaining: 50.2s
127:	learn: 0.1901277	test: 0.2132110	best: 0.2132110 (127)	total: 37.4s	remaining: 50.3s
128:	learn: 0.1892769	test: 0.2133296	best: 0.2132110 (127)	total: 38.6s	remaining: 51.2s
129:	learn: 0.1892695	test: 0.2133241	best: 0.2132110 (127)	total: 38.6s	remaining: 50.5s
130:	learn: 0.1885227	test: 0.2134298	best: 0.2132110 (127)	total: 39.9s	remaining: 51.5s
131:	learn: 0.1883382	test: 0.2131721	best: 0.2131721 (131)	total: 39.9s	remaining: 50.8s
132:	learn: 0.1875077	test: 0.2132107	best: 0.2131721 (131)	total: 41.3s	remaining: 51.9s
133:	learn: 0.1865804	test: 0.2131056	best: 0.2131056 (133)	total: 42.6s	remaining: 52.8s
134:	learn: 0.1858250	test: 0.2130744	best: 0.2130744 (134)	total: 44.1s	remaining: 53.9s
135:	learn: 0.1846827	test: 0.2126581	best: 0.2126581 (135)	total: 45.3s	remaining: 54.6s
136:	learn: 0.1840702	test: 0.2127420	best: 0.2126581 (135)	total: 46.6s	remaining: 55.5s
137:	learn: 0.1837537	test: 0.2123713	best: 0.2123713 (137)	total: 46.8s	remaining: 55s
138:	learn: 0.1836295	test: 0.2122033	best: 0.2122033 (138)	total: 46.9s	remaining: 54.3s
139:	learn: 0.1836269	test: 0.2122020	best: 0.2122020 (139)	total: 46.9s	remaining: 53.6s
140:	learn: 0.1824344	test: 0.2118646	best: 0.2118646 (140)	total: 48.1s	remaining: 54.3s
141:	learn: 0.1821004	test: 0.2116973	best: 0.2116973 (141)	total: 48.2s	remaining: 53.6s
142:	learn: 0.1813607	test: 0.2115113	best: 0.2115113 (142)	total: 49.6s	remaining: 54.4s
143:	learn: 0.1812895	test: 0.2115144	best: 0.2115113 (142)	total: 49.7s	remaining: 53.8s
144:	learn: 0.1806882	test: 0.2114765	best: 0.2114765 (144)	total: 50.9s	remaining: 54.4s
145:	learn: 0.1803960	test: 0.2115350	best: 0.2114765 (144)	total: 52.2s	remaining: 55s
146:	learn: 0.1795559	test: 0.2113556	best: 0.2113556 (146)	total: 53.5s	remaining: 55.7s
147:	learn: 0.1791134	test: 0.2114517	best: 0.2113556 (146)	total: 53.7s	remaining: 55.2s
148:	learn: 0.1779051	test: 0.2113807	best: 0.2113556 (146)	total: 55s	remaining: 55.8s
149:	learn: 0.1777051	test: 0.2112082	best: 0.2112082 (149)	total: 55.4s	remaining: 55.4s
150:	learn: 0.1773778	test: 0.2114617	best: 0.2112082 (149)	total: 55.7s	remaining: 55s
151:	learn: 0.1766426	test: 0.2114923	best: 0.2112082 (149)	total: 57.2s	remaining: 55.7s
152:	learn: 0.1765374	test: 0.2115734	best: 0.2112082 (149)	total: 57.3s	remaining: 55s
153:	learn: 0.1763600	test: 0.2113889	best: 0.2112082 (149)	total: 57.3s	remaining: 54.4s
154:	learn: 0.1760661	test: 0.2115619	best: 0.2112082 (149)	total: 57.4s	remaining: 53.7s
155:	learn: 0.1760068	test: 0.2115477	best: 0.2112082 (149)	total: 57.5s	remaining: 53s
156:	learn: 0.1755044	test: 0.2114911	best: 0.2112082 (149)	total: 58.9s	remaining: 53.6s
157:	learn: 0.1747890	test: 0.2112543	best: 0.2112082 (149)	total: 1m	remaining: 54.1s
158:	learn: 0.1742286	test: 0.2111359	best: 0.2111359 (158)	total: 1m	remaining: 53.9s
159:	learn: 0.1733770	test: 0.2107491	best: 0.2107491 (159)	total: 1m 2s	remaining: 54.5s
160:	learn: 0.1733501	test: 0.2107512	best: 0.2107491 (159)	total: 1m 2s	remaining: 53.8s
161:	learn: 0.1728581	test: 0.2108665	best: 0.2107491 (159)	total: 1m 3s	remaining: 54s
162:	learn: 0.1720217	test: 0.2109352	best: 0.2107491 (159)	total: 1m 4s	remaining: 54.4s
163:	learn: 0.1718354	test: 0.2108385	best: 0.2107491 (159)	total: 1m 4s	remaining: 53.7s
164:	learn: 0.1715802	test: 0.2109880	best: 0.2107491 (159)	total: 1m 5s	remaining: 53.3s
165:	learn: 0.1713541	test: 0.2109640	best: 0.2107491 (159)	total: 1m 5s	remaining: 52.7s
166:	learn: 0.1704103	test: 0.2110402	best: 0.2107491 (159)	total: 1m 6s	remaining: 52.9s
167:	learn: 0.1698856	test: 0.2112169	best: 0.2107491 (159)	total: 1m 7s	remaining: 53.1s
168:	learn: 0.1694085	test: 0.2109766	best: 0.2107491 (159)	total: 1m 7s	remaining: 52.6s
169:	learn: 0.1689011	test: 0.2110475	best: 0.2107491 (159)	total: 1m 9s	remaining: 52.9s
170:	learn: 0.1682299	test: 0.2108179	best: 0.2107491 (159)	total: 1m 10s	remaining: 53.1s
171:	learn: 0.1681043	test: 0.2108379	best: 0.2107491 (159)	total: 1m 10s	remaining: 52.4s
172:	learn: 0.1676697	test: 0.2107879	best: 0.2107491 (159)	total: 1m 11s	remaining: 52.6s
173:	learn: 0.1675876	test: 0.2107808	best: 0.2107491 (159)	total: 1m 11s	remaining: 51.9s
174:	learn: 0.1671980	test: 0.2107777	best: 0.2107491 (159)	total: 1m 12s	remaining: 52s
175:	learn: 0.1668005	test: 0.2107094	best: 0.2107094 (175)	total: 1m 13s	remaining: 51.5s
176:	learn: 0.1667357	test: 0.2107346	best: 0.2107094 (175)	total: 1m 13s	remaining: 50.9s
177:	learn: 0.1662050	test: 0.2107503	best: 0.2107094 (175)	total: 1m 13s	remaining: 50.4s
178:	learn: 0.1661550	test: 0.2108078	best: 0.2107094 (175)	total: 1m 13s	remaining: 49.7s
179:	learn: 0.1657059	test: 0.2107762	best: 0.2107094 (175)	total: 1m 14s	remaining: 49.8s
180:	learn: 0.1656982	test: 0.2107756	best: 0.2107094 (175)	total: 1m 14s	remaining: 49.1s
181:	learn: 0.1651692	test: 0.2107160	best: 0.2107094 (175)	total: 1m 15s	remaining: 49.2s
182:	learn: 0.1650216	test: 0.2107158	best: 0.2107094 (175)	total: 1m 15s	remaining: 48.6s
183:	learn: 0.1643315	test: 0.2106444	best: 0.2106444 (183)	total: 1m 17s	remaining: 48.7s
184:	learn: 0.1641247	test: 0.2105629	best: 0.2105629 (184)	total: 1m 17s	remaining: 48.1s
185:	learn: 0.1635606	test: 0.2106516	best: 0.2105629 (184)	total: 1m 18s	remaining: 48.3s
186:	learn: 0.1630418	test: 0.2107466	best: 0.2105629 (184)	total: 1m 20s	remaining: 48.4s
187:	learn: 0.1622979	test: 0.2107067	best: 0.2105629 (184)	total: 1m 21s	remaining: 48.4s
188:	learn: 0.1619498	test: 0.2108159	best: 0.2105629 (184)	total: 1m 22s	remaining: 48.4s
189:	learn: 0.1612040	test: 0.2108903	best: 0.2105629 (184)	total: 1m 23s	remaining: 48.5s
190:	learn: 0.1606559	test: 0.2108716	best: 0.2105629 (184)	total: 1m 24s	remaining: 48.5s
191:	learn: 0.1603144	test: 0.2111124	best: 0.2105629 (184)	total: 1m 25s	remaining: 47.9s
192:	learn: 0.1599591	test: 0.2112714	best: 0.2105629 (184)	total: 1m 25s	remaining: 47.4s
193:	learn: 0.1591892	test: 0.2109697	best: 0.2105629 (184)	total: 1m 26s	remaining: 47.3s
194:	learn: 0.1591304	test: 0.2109918	best: 0.2105629 (184)	total: 1m 26s	remaining: 46.7s
195:	learn: 0.1582843	test: 0.2111627	best: 0.2105629 (184)	total: 1m 28s	remaining: 46.8s
196:	learn: 0.1575778	test: 0.2112481	best: 0.2105629 (184)	total: 1m 29s	remaining: 46.8s
197:	learn: 0.1575356	test: 0.2111671	best: 0.2105629 (184)	total: 1m 29s	remaining: 46.1s
198:	learn: 0.1572521	test: 0.2111305	best: 0.2105629 (184)	total: 1m 30s	remaining: 46.1s
199:	learn: 0.1570667	test: 0.2112099	best: 0.2105629 (184)	total: 1m 31s	remaining: 45.5s
200:	learn: 0.1565235	test: 0.2110642	best: 0.2105629 (184)	total: 1m 32s	remaining: 45.4s
201:	learn: 0.1560769	test: 0.2110470	best: 0.2105629 (184)	total: 1m 33s	remaining: 45.3s
202:	learn: 0.1554796	test: 0.2112327	best: 0.2105629 (184)	total: 1m 34s	remaining: 45.3s
203:	learn: 0.1548500	test: 0.2113793	best: 0.2105629 (184)	total: 1m 35s	remaining: 45.1s
204:	learn: 0.1538404	test: 0.2115721	best: 0.2105629 (184)	total: 1m 37s	remaining: 45.1s
205:	learn: 0.1534344	test: 0.2116212	best: 0.2105629 (184)	total: 1m 38s	remaining: 45s
206:	learn: 0.1533537	test: 0.2115857	best: 0.2105629 (184)	total: 1m 38s	remaining: 44.3s
207:	learn: 0.1529354	test: 0.2114844	best: 0.2105629 (184)	total: 1m 40s	remaining: 44.2s
208:	learn: 0.1526224	test: 0.2115806	best: 0.2105629 (184)	total: 1m 41s	remaining: 44.1s
209:	learn: 0.1521269	test: 0.2114180	best: 0.2105629 (184)	total: 1m 42s	remaining: 43.9s
210:	learn: 0.1515228	test: 0.2117044	best: 0.2105629 (184)	total: 1m 43s	remaining: 43.7s
211:	learn: 0.1509387	test: 0.2116367	best: 0.2105629 (184)	total: 1m 45s	remaining: 43.6s
212:	learn: 0.1509101	test: 0.2115945	best: 0.2105629 (184)	total: 1m 45s	remaining: 42.9s
213:	learn: 0.1504259	test: 0.2116494	best: 0.2105629 (184)	total: 1m 46s	remaining: 42.7s
214:	learn: 0.1503878	test: 0.2116174	best: 0.2105629 (184)	total: 1m 46s	remaining: 42s
215:	learn: 0.1497679	test: 0.2117874	best: 0.2105629 (184)	total: 1m 47s	remaining: 41.9s
216:	learn: 0.1493553	test: 0.2119217	best: 0.2105629 (184)	total: 1m 49s	remaining: 41.7s
217:	learn: 0.1491507	test: 0.2119381	best: 0.2105629 (184)	total: 1m 50s	remaining: 41.5s
218:	learn: 0.1487211	test: 0.2119195	best: 0.2105629 (184)	total: 1m 51s	remaining: 41.3s
219:	learn: 0.1483408	test: 0.2120121	best: 0.2105629 (184)	total: 1m 53s	remaining: 41.1s
220:	learn: 0.1482607	test: 0.2120417	best: 0.2105629 (184)	total: 1m 53s	remaining: 40.4s
221:	learn: 0.1475548	test: 0.2121579	best: 0.2105629 (184)	total: 1m 53s	remaining: 40s
222:	learn: 0.1471031	test: 0.2121958	best: 0.2105629 (184)	total: 1m 55s	remaining: 39.7s
223:	learn: 0.1466523	test: 0.2123258	best: 0.2105629 (184)	total: 1m 56s	remaining: 39.4s
224:	learn: 0.1465652	test: 0.2123194	best: 0.2105629 (184)	total: 1m 56s	remaining: 38.7s
225:	learn: 0.1465503	test: 0.2123147	best: 0.2105629 (184)	total: 1m 56s	remaining: 38.1s
226:	learn: 0.1460098	test: 0.2123344	best: 0.2105629 (184)	total: 1m 57s	remaining: 37.8s
227:	learn: 0.1457447	test: 0.2121581	best: 0.2105629 (184)	total: 1m 58s	remaining: 37.3s
228:	learn: 0.1457087	test: 0.2121473	best: 0.2105629 (184)	total: 1m 58s	remaining: 36.7s
229:	learn: 0.1447036	test: 0.2121301	best: 0.2105629 (184)	total: 1m 59s	remaining: 36.4s
230:	learn: 0.1445263	test: 0.2121849	best: 0.2105629 (184)	total: 1m 59s	remaining: 35.8s
231:	learn: 0.1444137	test: 0.2121423	best: 0.2105629 (184)	total: 2m	remaining: 35.2s
232:	learn: 0.1438470	test: 0.2121112	best: 0.2105629 (184)	total: 2m 1s	remaining: 34.9s
233:	learn: 0.1437699	test: 0.2121108	best: 0.2105629 (184)	total: 2m 1s	remaining: 34.3s
234:	learn: 0.1436833	test: 0.2120332	best: 0.2105629 (184)	total: 2m 1s	remaining: 33.6s
235:	learn: 0.1432256	test: 0.2120161	best: 0.2105629 (184)	total: 2m 2s	remaining: 33.3s
236:	learn: 0.1430572	test: 0.2118206	best: 0.2105629 (184)	total: 2m 2s	remaining: 32.7s
237:	learn: 0.1429754	test: 0.2118755	best: 0.2105629 (184)	total: 2m 2s	remaining: 32s
238:	learn: 0.1429587	test: 0.2118899	best: 0.2105629 (184)	total: 2m 3s	remaining: 31.4s
239:	learn: 0.1427858	test: 0.2118411	best: 0.2105629 (184)	total: 2m 3s	remaining: 30.9s
240:	learn: 0.1422623	test: 0.2120257	best: 0.2105629 (184)	total: 2m 4s	remaining: 30.6s
241:	learn: 0.1418849	test: 0.2119379	best: 0.2105629 (184)	total: 2m 6s	remaining: 30.2s
242:	learn: 0.1414238	test: 0.2118761	best: 0.2105629 (184)	total: 2m 7s	remaining: 29.8s
243:	learn: 0.1409429	test: 0.2119048	best: 0.2105629 (184)	total: 2m 8s	remaining: 29.5s
244:	learn: 0.1402331	test: 0.2118422	best: 0.2105629 (184)	total: 2m 9s	remaining: 29.1s
245:	learn: 0.1396998	test: 0.2119976	best: 0.2105629 (184)	total: 2m 10s	remaining: 28.7s
246:	learn: 0.1394136	test: 0.2118322	best: 0.2105629 (184)	total: 2m 11s	remaining: 28.3s
247:	learn: 0.1393844	test: 0.2118017	best: 0.2105629 (184)	total: 2m 12s	remaining: 27.7s
248:	learn: 0.1390569	test: 0.2117003	best: 0.2105629 (184)	total: 2m 12s	remaining: 27.1s
249:	learn: 0.1387523	test: 0.2116376	best: 0.2105629 (184)	total: 2m 13s	remaining: 26.7s
250:	learn: 0.1383560	test: 0.2114555	best: 0.2105629 (184)	total: 2m 14s	remaining: 26.3s
251:	learn: 0.1383151	test: 0.2114565	best: 0.2105629 (184)	total: 2m 14s	remaining: 25.7s
252:	learn: 0.1379961	test: 0.2112767	best: 0.2105629 (184)	total: 2m 14s	remaining: 25.1s
253:	learn: 0.1375304	test: 0.2114441	best: 0.2105629 (184)	total: 2m 16s	remaining: 24.6s
254:	learn: 0.1374991	test: 0.2113986	best: 0.2105629 (184)	total: 2m 16s	remaining: 24s
255:	learn: 0.1374766	test: 0.2113675	best: 0.2105629 (184)	total: 2m 16s	remaining: 23.4s
256:	learn: 0.1368452	test: 0.2115120	best: 0.2105629 (184)	total: 2m 17s	remaining: 23s
257:	learn: 0.1368239	test: 0.2114825	best: 0.2105629 (184)	total: 2m 17s	remaining: 22.4s
258:	learn: 0.1364529	test: 0.2112619	best: 0.2105629 (184)	total: 2m 17s	remaining: 21.8s
259:	learn: 0.1359126	test: 0.2114734	best: 0.2105629 (184)	total: 2m 18s	remaining: 21.4s
260:	learn: 0.1353688	test: 0.2115095	best: 0.2105629 (184)	total: 2m 19s	remaining: 20.9s
261:	learn: 0.1350476	test: 0.2114847	best: 0.2105629 (184)	total: 2m 20s	remaining: 20.4s
262:	learn: 0.1345884	test: 0.2116627	best: 0.2105629 (184)	total: 2m 21s	remaining: 19.9s
263:	learn: 0.1344486	test: 0.2117483	best: 0.2105629 (184)	total: 2m 21s	remaining: 19.3s
264:	learn: 0.1341644	test: 0.2117374	best: 0.2105629 (184)	total: 2m 22s	remaining: 18.8s
265:	learn: 0.1335466	test: 0.2118263	best: 0.2105629 (184)	total: 2m 23s	remaining: 18.3s
266:	learn: 0.1328196	test: 0.2121324	best: 0.2105629 (184)	total: 2m 24s	remaining: 17.9s
267:	learn: 0.1323330	test: 0.2123350	best: 0.2105629 (184)	total: 2m 25s	remaining: 17.4s
268:	learn: 0.1323115	test: 0.2123507	best: 0.2105629 (184)	total: 2m 25s	remaining: 16.8s
269:	learn: 0.1320012	test: 0.2123244	best: 0.2105629 (184)	total: 2m 27s	remaining: 16.4s
270:	learn: 0.1316152	test: 0.2123628	best: 0.2105629 (184)	total: 2m 28s	remaining: 15.9s
271:	learn: 0.1310850	test: 0.2125313	best: 0.2105629 (184)	total: 2m 29s	remaining: 15.4s
272:	learn: 0.1305229	test: 0.2123729	best: 0.2105629 (184)	total: 2m 31s	remaining: 14.9s
273:	learn: 0.1303218	test: 0.2122395	best: 0.2105629 (184)	total: 2m 31s	remaining: 14.3s
274:	learn: 0.1301488	test: 0.2122505	best: 0.2105629 (184)	total: 2m 31s	remaining: 13.8s
275:	learn: 0.1297599	test: 0.2122969	best: 0.2105629 (184)	total: 2m 32s	remaining: 13.3s
276:	learn: 0.1295283	test: 0.2121953	best: 0.2105629 (184)	total: 2m 33s	remaining: 12.8s
277:	learn: 0.1292509	test: 0.2122108	best: 0.2105629 (184)	total: 2m 35s	remaining: 12.3s
278:	learn: 0.1290380	test: 0.2121328	best: 0.2105629 (184)	total: 2m 36s	remaining: 11.8s
279:	learn: 0.1284968	test: 0.2121849	best: 0.2105629 (184)	total: 2m 37s	remaining: 11.3s
280:	learn: 0.1284793	test: 0.2121486	best: 0.2105629 (184)	total: 2m 37s	remaining: 10.7s
281:	learn: 0.1281923	test: 0.2120986	best: 0.2105629 (184)	total: 2m 38s	remaining: 10.1s
282:	learn: 0.1281894	test: 0.2120914	best: 0.2105629 (184)	total: 2m 38s	remaining: 9.55s
283:	learn: 0.1274435	test: 0.2123112	best: 0.2105629 (184)	total: 2m 40s	remaining: 9.03s
284:	learn: 0.1270979	test: 0.2123783	best: 0.2105629 (184)	total: 2m 41s	remaining: 8.51s
285:	learn: 0.1266098	test: 0.2125951	best: 0.2105629 (184)	total: 2m 43s	remaining: 7.98s
286:	learn: 0.1259064	test: 0.2124510	best: 0.2105629 (184)	total: 2m 44s	remaining: 7.44s
287:	learn: 0.1258898	test: 0.2124156	best: 0.2105629 (184)	total: 2m 44s	remaining: 6.84s
288:	learn: 0.1257976	test: 0.2124057	best: 0.2105629 (184)	total: 2m 44s	remaining: 6.25s
289:	learn: 0.1250494	test: 0.2123619	best: 0.2105629 (184)	total: 2m 45s	remaining: 5.7s
290:	learn: 0.1243749	test: 0.2125701	best: 0.2105629 (184)	total: 2m 46s	remaining: 5.15s
291:	learn: 0.1239163	test: 0.2125791	best: 0.2105629 (184)	total: 2m 47s	remaining: 4.6s
292:	learn: 0.1234270	test: 0.2128082	best: 0.2105629 (184)	total: 2m 49s	remaining: 4.04s
293:	learn: 0.1234113	test: 0.2127735	best: 0.2105629 (184)	total: 2m 49s	remaining: 3.45s
294:	learn: 0.1229887	test: 0.2128833	best: 0.2105629 (184)	total: 2m 50s	remaining: 2.89s
295:	learn: 0.1229378	test: 0.2129169	best: 0.2105629 (184)	total: 2m 50s	remaining: 2.3s
296:	learn: 0.1228867	test: 0.2128850	best: 0.2105629 (184)	total: 2m 50s	remaining: 1.72s
297:	learn: 0.1225454	test: 0.2128257	best: 0.2105629 (184)	total: 2m 51s	remaining: 1.15s
298:	learn: 0.1225267	test: 0.2127913	best: 0.2105629 (184)	total: 2m 51s	remaining: 573ms
299:	learn: 0.1224219	test: 0.2127564	best: 0.2105629 (184)	total: 2m 52s	remaining: 0us

bestTest = 0.2105628955
bestIteration = 184

Out[77]:
<catboost.core.CatBoostClassifier at 0x234ea4a2160>

In [78]:
prediction_proba = model.predict_proba(test_)

In [79]:
def make_submission(probs):
    sample = pd.read_csv(PATH + 'sample.csv')
    submit = sample.copy()
    submit['Purchase'] = probs
    return submit

In [80]:
submit = make_submission(prediction_proba[:,1])

In [81]:
submit.to_csv(PATH + 'cat_300_.02.csv')

Grid Cv


In [26]:
from sklearn.ensemble import GradientBoostingClassifier  #GBM algorithm
from sklearn import cross_validation, metrics   #Additional scklearn functions
from sklearn.grid_search import GridSearchCV   #Perforing grid search


C:\ProgramData\Anaconda3\lib\site-packages\sklearn\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.
  "This module will be removed in 0.20.", DeprecationWarning)
C:\ProgramData\Anaconda3\lib\site-packages\sklearn\grid_search.py:42: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.
  DeprecationWarning)

In [60]:
def modelfit(alg, dtrain, predictors, performCV=True, printFeatureImportance=True, cv_folds=5):
    #Fit the algorithm on the data
    alg.fit(dtrain[predictors], y)
        
    #Predict training set:
    dtrain_predictions = alg.predict(dtrain[predictors])
    dtrain_predprob = alg.predict_proba(dtrain[predictors])[:,1]
    
    #Perform cross-validation:
    if performCV:
        cv_score = cross_validation.cross_val_score(alg, dtrain[predictors], y, cv=cv_folds, scoring='roc_auc')
    
    #Print model report:
    print ("\nModel Report")
    print ("Accuracy : %.4g" % metrics.accuracy_score(y , dtrain_predictions))
    print ("AUC Score (Train): %f" % metrics.roc_auc_score(y , dtrain_predprob))
    
    if performCV:
        print ("CV Score : Mean - %.7g | Std - %.7g | Min - %.7g | Max - %.7g" % (np.mean(cv_score),np.std(cv_score),np.min(cv_score),np.max(cv_score)))
        
    #Print Feature Importance:
    if printFeatureImportance:
        feat_imp = pd.Series(alg.feature_importances_, predictors).sort_values(ascending=False)
        plt.figure(figsize=(20,20))
        feat_imp.plot(kind='bar', title='Feature Importances')
        plt.ylabel('Feature Importance Score')

In [40]:
#Choose all predictors except target & IDcols
predictors = train_.columns
gbm0 = GradientBoostingClassifier(random_state=10)
modelfit(gbm0, train, predictors)


Model Report
Accuracy : 0.9507
AUC Score (Train): 0.919207
CV Score : Mean - 0.746342 | Std - 0.03290251 | Min - 0.6999938 | Max - 0.7866071

In [45]:
param_test1 = {'n_estimators':[20, 30, 40, 50, 60, 70, 80, 90]}
gsearch1 = GridSearchCV(estimator = GradientBoostingClassifier(learning_rate=0.05, min_samples_split=500,
                        min_samples_leaf=50,max_depth=8,max_features='sqrt',subsample=0.8,random_state=10), 
                        param_grid = param_test1, scoring='roc_auc',n_jobs=4,iid=False, cv=5)
gsearch1.fit(train[predictors], y)


Out[45]:
GridSearchCV(cv=5, error_score='raise',
       estimator=GradientBoostingClassifier(criterion='friedman_mse', init=None,
              learning_rate=0.05, loss='deviance', max_depth=8,
              max_features='sqrt', max_leaf_nodes=None,
              min_impurity_decrease=0.0, min_impurity_split=None,
              min_samples_leaf=50, min_samples_split=500,
              min_weight_fraction_leaf=0.0, n_estimators=100,
              presort='auto', random_state=10, subsample=0.8, verbose=0,
              warm_start=False),
       fit_params={}, iid=False, n_jobs=4,
       param_grid={'n_estimators': [20, 30, 40, 50, 60, 70, 80, 90]},
       pre_dispatch='2*n_jobs', refit=True, scoring='roc_auc', verbose=0)

In [46]:
gsearch1.grid_scores_, gsearch1.best_params_, gsearch1.best_score_


Out[46]:
([mean: 0.75786, std: 0.04541, params: {'n_estimators': 20},
  mean: 0.75956, std: 0.04903, params: {'n_estimators': 30},
  mean: 0.75750, std: 0.05016, params: {'n_estimators': 40},
  mean: 0.75898, std: 0.04599, params: {'n_estimators': 50},
  mean: 0.75835, std: 0.04518, params: {'n_estimators': 60},
  mean: 0.76000, std: 0.04347, params: {'n_estimators': 70},
  mean: 0.75769, std: 0.04377, params: {'n_estimators': 80},
  mean: 0.75599, std: 0.04094, params: {'n_estimators': 90}],
 {'n_estimators': 70},
 0.760000019680155)

In [55]:
## Test 2
param_test2 = {'max_depth':[5, 7, 9, 11, 13, 15] ,'min_samples_split': [200, 400, 600, 800, 1000]}
gsearch2 = GridSearchCV(estimator = GradientBoostingClassifier(learning_rate=0.05, n_estimators=70, max_features='sqrt', subsample=0.8, random_state=10), 
param_grid = param_test2, scoring='roc_auc',n_jobs=4,iid=False, cv=5)

In [56]:
gsearch2.fit(train[predictors], y)
gsearch2.grid_scores_, gsearch2.best_params_, gsearch2.best_score_


Out[56]:
([mean: 0.75963, std: 0.03429, params: {'max_depth': 5, 'min_samples_split': 200},
  mean: 0.75616, std: 0.03440, params: {'max_depth': 5, 'min_samples_split': 400},
  mean: 0.76514, std: 0.03326, params: {'max_depth': 5, 'min_samples_split': 600},
  mean: 0.77028, std: 0.02879, params: {'max_depth': 5, 'min_samples_split': 800},
  mean: 0.76813, std: 0.02803, params: {'max_depth': 5, 'min_samples_split': 1000},
  mean: 0.75295, std: 0.03801, params: {'max_depth': 7, 'min_samples_split': 200},
  mean: 0.75495, std: 0.03283, params: {'max_depth': 7, 'min_samples_split': 400},
  mean: 0.75903, std: 0.03121, params: {'max_depth': 7, 'min_samples_split': 600},
  mean: 0.76978, std: 0.02988, params: {'max_depth': 7, 'min_samples_split': 800},
  mean: 0.76512, std: 0.03029, params: {'max_depth': 7, 'min_samples_split': 1000},
  mean: 0.74006, std: 0.03903, params: {'max_depth': 9, 'min_samples_split': 200},
  mean: 0.75114, std: 0.03818, params: {'max_depth': 9, 'min_samples_split': 400},
  mean: 0.75961, std: 0.03249, params: {'max_depth': 9, 'min_samples_split': 600},
  mean: 0.76883, std: 0.02767, params: {'max_depth': 9, 'min_samples_split': 800},
  mean: 0.76445, std: 0.02889, params: {'max_depth': 9, 'min_samples_split': 1000},
  mean: 0.73599, std: 0.03411, params: {'max_depth': 11, 'min_samples_split': 200},
  mean: 0.75126, std: 0.03745, params: {'max_depth': 11, 'min_samples_split': 400},
  mean: 0.76109, std: 0.03591, params: {'max_depth': 11, 'min_samples_split': 600},
  mean: 0.76694, std: 0.02944, params: {'max_depth': 11, 'min_samples_split': 800},
  mean: 0.76525, std: 0.02925, params: {'max_depth': 11, 'min_samples_split': 1000},
  mean: 0.74638, std: 0.03677, params: {'max_depth': 13, 'min_samples_split': 200},
  mean: 0.75265, std: 0.03290, params: {'max_depth': 13, 'min_samples_split': 400},
  mean: 0.76035, std: 0.03306, params: {'max_depth': 13, 'min_samples_split': 600},
  mean: 0.76739, std: 0.03040, params: {'max_depth': 13, 'min_samples_split': 800},
  mean: 0.76588, std: 0.02869, params: {'max_depth': 13, 'min_samples_split': 1000},
  mean: 0.74147, std: 0.03625, params: {'max_depth': 15, 'min_samples_split': 200},
  mean: 0.75212, std: 0.03323, params: {'max_depth': 15, 'min_samples_split': 400},
  mean: 0.76047, std: 0.03150, params: {'max_depth': 15, 'min_samples_split': 600},
  mean: 0.76365, std: 0.02971, params: {'max_depth': 15, 'min_samples_split': 800},
  mean: 0.76499, std: 0.02722, params: {'max_depth': 15, 'min_samples_split': 1000}],
 {'max_depth': 5, 'min_samples_split': 800},
 0.7702836854978986)

In [57]:
#test 3
param_test3 = {'min_samples_split': [800, 1000, 1200, 1400, 1600] , 'min_samples_leaf': [30, 40, 50, 60, 70]}
gsearch3 = GridSearchCV(estimator = GradientBoostingClassifier(learning_rate=0.05, n_estimators=70,max_depth=5,max_features='sqrt', subsample=0.8, random_state=10), 
param_grid = param_test3, scoring='roc_auc',n_jobs=4,iid=False, cv=5)

In [58]:
gsearch3.fit(train[predictors], y)
gsearch3.grid_scores_, gsearch2.best_params_, gsearch2.best_score_


Out[58]:
([mean: 0.75961, std: 0.04145, params: {'min_samples_leaf': 30, 'min_samples_split': 800},
  mean: 0.75970, std: 0.04062, params: {'min_samples_leaf': 30, 'min_samples_split': 1000},
  mean: 0.76251, std: 0.04009, params: {'min_samples_leaf': 30, 'min_samples_split': 1200},
  mean: 0.76317, std: 0.03833, params: {'min_samples_leaf': 30, 'min_samples_split': 1400},
  mean: 0.76284, std: 0.04049, params: {'min_samples_leaf': 30, 'min_samples_split': 1600},
  mean: 0.76191, std: 0.04055, params: {'min_samples_leaf': 40, 'min_samples_split': 800},
  mean: 0.75857, std: 0.03985, params: {'min_samples_leaf': 40, 'min_samples_split': 1000},
  mean: 0.76197, std: 0.03988, params: {'min_samples_leaf': 40, 'min_samples_split': 1200},
  mean: 0.76334, std: 0.04063, params: {'min_samples_leaf': 40, 'min_samples_split': 1400},
  mean: 0.76216, std: 0.04121, params: {'min_samples_leaf': 40, 'min_samples_split': 1600},
  mean: 0.75848, std: 0.04235, params: {'min_samples_leaf': 50, 'min_samples_split': 800},
  mean: 0.75965, std: 0.04054, params: {'min_samples_leaf': 50, 'min_samples_split': 1000},
  mean: 0.76182, std: 0.04044, params: {'min_samples_leaf': 50, 'min_samples_split': 1200},
  mean: 0.76305, std: 0.04027, params: {'min_samples_leaf': 50, 'min_samples_split': 1400},
  mean: 0.76191, std: 0.04065, params: {'min_samples_leaf': 50, 'min_samples_split': 1600},
  mean: 0.76113, std: 0.04293, params: {'min_samples_leaf': 60, 'min_samples_split': 800},
  mean: 0.76112, std: 0.04080, params: {'min_samples_leaf': 60, 'min_samples_split': 1000},
  mean: 0.76215, std: 0.04059, params: {'min_samples_leaf': 60, 'min_samples_split': 1200},
  mean: 0.76313, std: 0.04013, params: {'min_samples_leaf': 60, 'min_samples_split': 1400},
  mean: 0.76129, std: 0.04145, params: {'min_samples_leaf': 60, 'min_samples_split': 1600},
  mean: 0.75981, std: 0.04243, params: {'min_samples_leaf': 70, 'min_samples_split': 800},
  mean: 0.76051, std: 0.04138, params: {'min_samples_leaf': 70, 'min_samples_split': 1000},
  mean: 0.76061, std: 0.04171, params: {'min_samples_leaf': 70, 'min_samples_split': 1200},
  mean: 0.76201, std: 0.04118, params: {'min_samples_leaf': 70, 'min_samples_split': 1400},
  mean: 0.76108, std: 0.04164, params: {'min_samples_leaf': 70, 'min_samples_split': 1600}],
 {'max_depth': 5, 'min_samples_split': 800},
 0.7702836854978986)

In [61]:
modelfit(gsearch3.best_estimator_, train, predictors)


Model Report
Accuracy : 0.9374
AUC Score (Train): 0.797835
CV Score : Mean - 0.7633389 | Std - 0.04062501 | Min - 0.7205147 | Max - 0.8170092

In [69]:
#test 4
param_test4 = {'max_features': [7, 9, 11, 13, 15, 17, 19, 21]}
gsearch4 = GridSearchCV(estimator = GradientBoostingClassifier(learning_rate=0.05, min_samples_split = 800, n_estimators=70,max_depth=5,max_features='sqrt', subsample=0.8, random_state=10), 
param_grid = param_test4, scoring='roc_auc',n_jobs=4,iid=False, cv=5)

In [70]:
gsearch4.fit(train[predictors], y )
gsearch4.grid_scores_, gsearch4.best_params_, gsearch4.best_score_


Out[70]:
([mean: 0.76454, std: 0.02451, params: {'max_features': 7},
  mean: 0.77028, std: 0.02879, params: {'max_features': 9},
  mean: 0.76460, std: 0.02670, params: {'max_features': 11},
  mean: 0.76892, std: 0.03054, params: {'max_features': 13},
  mean: 0.76609, std: 0.02405, params: {'max_features': 15},
  mean: 0.76901, std: 0.02707, params: {'max_features': 17},
  mean: 0.76856, std: 0.02800, params: {'max_features': 19},
  mean: 0.76722, std: 0.02624, params: {'max_features': 21}],
 {'max_features': 9},
 0.7702836854978986)

In [71]:
#test 5
param_test5 = {'subsample':[0.6,0.7,0.75,0.8,0.85,0.9]}
gsearch5 = GridSearchCV(estimator = GradientBoostingClassifier(learning_rate=0.05, min_samples_split = 800, 
                                                               n_estimators=70,max_depth=5,max_features= 9 , 
                                                               subsample=0.8, random_state=10), 
param_grid = param_test5, scoring='roc_auc',n_jobs=4,iid=False, cv=5)

In [73]:
gsearch5.fit(train[predictors], y )
gsearch5.grid_scores_, gsearch5.best_params_, gsearch5.best_score_


Out[73]:
([mean: 0.76525, std: 0.02880, params: {'subsample': 0.6},
  mean: 0.76375, std: 0.02847, params: {'subsample': 0.7},
  mean: 0.76945, std: 0.03092, params: {'subsample': 0.75},
  mean: 0.77028, std: 0.02879, params: {'subsample': 0.8},
  mean: 0.77283, std: 0.03167, params: {'subsample': 0.85},
  mean: 0.77211, std: 0.03195, params: {'subsample': 0.9}],
 {'subsample': 0.85},
 0.7728346744178137)

Fun


In [74]:
gbm_tuned_1 = GradientBoostingClassifier(learning_rate=0.025, n_estimators=140,max_depth=5,
                                         min_samples_split=800,min_samples_leaf=60, 
                                         subsample=0.85, random_state=10, max_features=9)
modelfit(gbm_tuned_1, train, predictors)


Model Report
Accuracy : 0.9374
AUC Score (Train): 0.835988
CV Score : Mean - 0.7618293 | Std - 0.04131622 | Min - 0.7182929 | Max - 0.8192927

In [75]:
gbm_tuned_2 = GradientBoostingClassifier(learning_rate=0.05, n_estimators=900,max_depth=5, 
                                         min_samples_split=800,min_samples_leaf=60, 
                                         subsample=0.85, random_state=10, max_features=9)
modelfit(gbm_tuned_2, train, predictors)


Model Report
Accuracy : 0.9374
AUC Score (Train): 0.841179
CV Score : Mean - 0.7612796 | Std - 0.04203202 | Min - 0.7179843 | Max - 0.8191076

In [76]:
prediction_proba_2 = gbm_tuned_2.predict_proba(test_)

In [77]:
submit = make_submission(prediction_proba_2[:,1])

In [78]:
submit.to_csv(PATH + 'gbm_tuned_2.csv')

Ensemble


In [97]:
def ensemble():
    stacked_1 = pd.read_csv('ensemble_.csv')#.842
    stacked_2 = pd.read_csv('315_37929_us_ensemble.csv')#.842
    stacked_3 = pd.read_csv('315_67174_us_asd.csv')#.838
    stacked_4 = pd.read_csv('315_67174_us_cat_1.csv')#.835
    stacked_5 = pd.read_csv('ensemble_.csv')#835
    stacked_6 = pd.read_csv('ensemble_.csv')#.841
    
    sub = pd.DataFrame()
    sub['ID'] = stacked_1['ID']
    sub['Purchase'] = (np.mean(
        [
            stacked_1['Purchase'].apply(lambda x: np.abs(x)), \
            stacked_2['Purchase'].apply(lambda x: np.abs(x)), \
            stacked_3['Purchase'].apply(lambda x: np.abs(x)), \
            stacked_4['Purchase'].apply(lambda x: np.abs(x)), \
            stacked_5['Purchase'].apply(lambda x: np.abs(x)), \
            stacked_6['Purchase'].apply(lambda x: np.abs(x)), \
            ], axis=0))
    sub.to_csv('ensemble_3_last.csv', index=False, float_format='%.6f')

In [98]:
ensemble()