

In [1]:

    
import pandas as pd
import numpy as np
import xgboost as xgb
from sklearn.cross_validation import KFold
from sklearn.metrics import mean_absolute_error
from random import randint
from hyperopt import hp
from hyperopt import fmin, tpe, hp, STATUS_OK, Trials, rand

train = pd.read_csv('./data/allstate/train.csv')
test = pd.read_csv('./data/allstate/test.csv')

test['loss'] = np.nan
joined = pd.concat([train, test])

def logregobj(preds, dtrain):
    labels = dtrain.get_label()
    con = 2
    x =preds-labels
    grad =con*x / (np.abs(x)+con)
    hess =con**2 / (np.abs(x)+con)**2
    return grad, hess 

def evalerror(preds, dtrain):
    labels = dtrain.get_label()
    return 'mae', mean_absolute_error(np.exp(preds), np.exp(labels))


cat_feature = [n for n in joined.columns if n.startswith('cat')]    
cont_feature = [n for n in joined.columns if n.startswith('cont')] 

for column in cat_feature:
        joined[column] = pd.factorize(joined[column].values, sort=True)[0]

    
train = joined[joined['loss'].notnull()]
test = joined[joined['loss'].isnull()]
ids = test['id']
X = train.drop(['loss', 'id'], 1)
X_test = test.drop(['loss', 'id'], 1)


def score(params):           
    print("Training with params : ")
    print(params)

    final_fold_prediction= []
    final_fold_real = []

    prediction = np.zeros(X.shape[0])
    
    n_folds  = params["n_folds"]
    del(params["n_folds"])
    
    shift = params["shift"]
    del(params["shift"])
    y = np.log(train['loss'] + shift)
    xgfull = xgb.DMatrix(X, label=y)
    
    kf = KFold(X.shape[0], n_folds=n_folds)
    
    for i, (train_index, test_index) in enumerate(kf):        
        X_train, X_val = X.iloc[train_index], X.iloc[test_index]
        y_train, y_val = y.iloc[train_index], y.iloc[test_index]    
          
        xgtrain = xgb.DMatrix(X_train, label=y_train)
        xgtrain_2 = xgb.DMatrix(X_val, label=y_val)
        
        watchlist = [(xgtrain, 'train'), (xgtrain_2, 'eval')]                    

        model = xgb.train(params, xgtrain, 100000, watchlist, obj=logregobj, feval=evalerror, 
        early_stopping_rounds=300, verbose_eval=200)        

        X_val = xgb.DMatrix(X_val) 
        temp_serises = pd.Series(np.exp(model.predict(X_val))-shift)
        final_fold_prediction.append( temp_serises )
        temp_serises = np.exp(y_val) -shift
        final_fold_real.append(temp_serises )
        
        temp_cv_score = mean_absolute_error(np.exp(model.predict(X_val))-shift, np.exp(y_val) -shift)
        print("Fold {} score: {}".format(i + 1, temp_cv_score))
        
        prediction += np.exp(model.predict(xgfull)) - shift

    prediction = prediction/n_folds
    score =  mean_absolute_error(y, prediction)
        
    print("\tMAE {0}\n\n".format(score))    
    return {'loss': score, 'status': STATUS_OK}

def optimize(trials):
    RANDOM_STATE = 2016
    space = {
             'min_child_weight' : hp.quniform('min_child_weight', 0.01, 1.0, 0.01),
             'eta' : hp.quniform('eta', 0.001, 1.0, 0.01),
             'colsample_bytree' : hp.quniform('colsample_bytree', 0.01, 1.0, 0.01),
             'max_depth' : hp.choice('batch_size', np.arange(1, 20, dtype=int)),   
             'subsample' : hp.quniform('subsample', 0.6, 1.0, 0.01),
             'alpha' : hp.quniform('alpha', 0.01, 1.0, 0.01),
             'gamma' : hp.quniform('gamma', 0.01, 1.0, 0.01),                        
             'seed' : RANDOM_STATE,
             'n_folds' : hp.choice('n_folds', np.arange(2, 10, dtype=int)), 
             'shift' : hp.choice('shift', np.arange(150, 5000, dtype=int)),
             }

    best = fmin(score, space, algo=tpe.suggest, trials=trials, max_evals=250)

    print("Best params predicted by hyperopt are:")    
    print(best)

#Trials object where the history of search will be stored
trials = Trials()

optimize(trials)









    



/home/arvc/anaconda3/envs/tensorflow/lib/python3.5/site-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.
  "This module will be removed in 0.20.", DeprecationWarning)






    



Training with params : 
{'n_folds': 4, 'colsample_bytree': 0.9400000000000001, 'subsample': 0.9500000000000001, 'seed': 2016, 'min_child_weight': 0.05, 'gamma': 0.55, 'alpha': 0.14, 'eta': 0.48, 'max_depth': 16, 'shift': 426}
[0]	train-mae:1.04112e+08	eval-mae:1.04864e+08
Multiple eval metrics have been passed: 'eval-mae' will be used for early stopping.

Will train until eval-mae hasn't improved in 300 rounds.






    



/home/arvc/anaconda3/envs/tensorflow/lib/python3.5/site-packages/ipykernel/__main__.py:26: RuntimeWarning: overflow encountered in exp






    



---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-1-2317c896fa54> in <module>()
    112 trials = Trials()
    113 
--> 114 optimize(trials)

<ipython-input-1-2317c896fa54> in optimize(trials)
    104              }
    105 
--> 106     best = fmin(score, space, algo=tpe.suggest, trials=trials, max_evals=250)
    107 
    108     print("Best params predicted by hyperopt are:")

/home/arvc/anaconda3/envs/tensorflow/lib/python3.5/site-packages/hyperopt/fmin.py in fmin(fn, space, algo, max_evals, trials, rstate, allow_trials_fmin, pass_expr_memo_ctrl, catch_eval_exceptions, verbose, return_argmin)
    304             verbose=verbose,
    305             catch_eval_exceptions=catch_eval_exceptions,
--> 306             return_argmin=return_argmin,
    307             )
    308 

/home/arvc/anaconda3/envs/tensorflow/lib/python3.5/site-packages/hyperopt/base.py in fmin(self, fn, space, algo, max_evals, rstate, verbose, pass_expr_memo_ctrl, catch_eval_exceptions, return_argmin)
    631             pass_expr_memo_ctrl=pass_expr_memo_ctrl,
    632             catch_eval_exceptions=catch_eval_exceptions,
--> 633             return_argmin=return_argmin)
    634 
    635 

/home/arvc/anaconda3/envs/tensorflow/lib/python3.5/site-packages/hyperopt/fmin.py in fmin(fn, space, algo, max_evals, trials, rstate, allow_trials_fmin, pass_expr_memo_ctrl, catch_eval_exceptions, verbose, return_argmin)
    317                     verbose=verbose)
    318     rval.catch_eval_exceptions = catch_eval_exceptions
--> 319     rval.exhaust()
    320     if return_argmin:
    321         return trials.argmin

/home/arvc/anaconda3/envs/tensorflow/lib/python3.5/site-packages/hyperopt/fmin.py in exhaust(self)
    196     def exhaust(self):
    197         n_done = len(self.trials)
--> 198         self.run(self.max_evals - n_done, block_until_done=self.async)
    199         self.trials.refresh()
    200         return self

/home/arvc/anaconda3/envs/tensorflow/lib/python3.5/site-packages/hyperopt/fmin.py in run(self, N, block_until_done)
    170             else:
    171                 # -- loop over trials and do the jobs directly
--> 172                 self.serial_evaluate()
    173 
    174             if stopped:

/home/arvc/anaconda3/envs/tensorflow/lib/python3.5/site-packages/hyperopt/fmin.py in serial_evaluate(self, N)
     87                 ctrl = base.Ctrl(self.trials, current_trial=trial)
     88                 try:
---> 89                     result = self.domain.evaluate(spec, ctrl)
     90                 except Exception as e:
     91                     logger.info('job exception: %s' % str(e))

/home/arvc/anaconda3/envs/tensorflow/lib/python3.5/site-packages/hyperopt/base.py in evaluate(self, config, ctrl, attach_attachments)
    836                 memo=memo,
    837                 print_node_on_error=self.rec_eval_print_node_on_error)
--> 838             rval = self.fn(pyll_rval)
    839 
    840         if isinstance(rval, (float, int, np.number)):

<ipython-input-1-2317c896fa54> in score(params)
     70 
     71         model = xgb.train(params, xgtrain, 100000, watchlist, obj=logregobj, feval=evalerror, 
---> 72         early_stopping_rounds=300, verbose_eval=200)        
     73 
     74         X_val = xgb.DMatrix(X_val)

/home/arvc/xgboost/python-package/xgboost/training.py in train(params, dtrain, num_boost_round, evals, obj, feval, maximize, early_stopping_rounds, evals_result, verbose_eval, learning_rates, xgb_model, callbacks)
    201                            evals=evals,
    202                            obj=obj, feval=feval,
--> 203                            xgb_model=xgb_model, callbacks=callbacks)
    204 
    205 

/home/arvc/xgboost/python-package/xgboost/training.py in _train_internal(params, dtrain, num_boost_round, evals, obj, feval, xgb_model, callbacks)
     82         # check evaluation result.
     83         if len(evals) != 0:
---> 84             bst_eval_set = bst.eval_set(evals, i, feval)
     85             if isinstance(bst_eval_set, STRING_TYPES):
     86                 msg = bst_eval_set

/home/arvc/xgboost/python-package/xgboost/core.py in eval_set(self, evals, iteration, feval)
    883             res = '[%d]' % iteration
    884             for dmat, evname in evals:
--> 885                 feval_ret = feval(self.predict(dmat), dmat)
    886                 if isinstance(feval_ret, list):
    887                     for name, val in feval_ret:

<ipython-input-1-2317c896fa54> in evalerror(preds, dtrain)
     24 def evalerror(preds, dtrain):
     25     labels = dtrain.get_label()
---> 26     return 'mae', mean_absolute_error(np.exp(preds), np.exp(labels))
     27 
     28 

/home/arvc/anaconda3/envs/tensorflow/lib/python3.5/site-packages/sklearn/metrics/regression.py in mean_absolute_error(y_true, y_pred, sample_weight, multioutput)
    161     """
    162     y_type, y_true, y_pred, multioutput = _check_reg_targets(
--> 163         y_true, y_pred, multioutput)
    164     output_errors = np.average(np.abs(y_pred - y_true),
    165                                weights=sample_weight, axis=0)

/home/arvc/anaconda3/envs/tensorflow/lib/python3.5/site-packages/sklearn/metrics/regression.py in _check_reg_targets(y_true, y_pred, multioutput)
     73     """
     74     check_consistent_length(y_true, y_pred)
---> 75     y_true = check_array(y_true, ensure_2d=False)
     76     y_pred = check_array(y_pred, ensure_2d=False)
     77 

/home/arvc/anaconda3/envs/tensorflow/lib/python3.5/site-packages/sklearn/utils/validation.py in check_array(array, accept_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, warn_on_dtype, estimator)
    405                              % (array.ndim, estimator_name))
    406         if force_all_finite:
--> 407             _assert_all_finite(array)
    408 
    409     shape_repr = _shape_repr(array.shape)

/home/arvc/anaconda3/envs/tensorflow/lib/python3.5/site-packages/sklearn/utils/validation.py in _assert_all_finite(X)
     56             and not np.isfinite(X).all()):
     57         raise ValueError("Input contains NaN, infinity"
---> 58                          " or a value too large for %r." % X.dtype)
     59 
     60 

ValueError: Input contains NaN, infinity or a value too large for dtype('float32').