In [2]:

    

from sklearn.cross_validation import train_test_split
from sklearn.metrics import log_loss
from sklearn.metrics import explained_variance_score,r2_score,mean_absolute_error
from sklearn import preprocessing
from random import randint
import numpy as np
import pandas as pd
from hyperopt import hp
from hyperopt import fmin, tpe, hp, STATUS_OK, Trials, rand
import sys
import os
from utils import get_allstate_train_valid_test_testids, to_xy
import tensorflow as tf
from tensorflow.contrib import learn
import tensorflow.contrib.learn as skflow
from sklearn.cross_validation import KFold

df=pd.read_csv('./data/allstate/train.csv')
cat_feature = [n for n in df.columns if n.startswith('cat')]    
cont_feature = [n for n in df.columns if n.startswith('cont')]

for column in cat_feature:
        df[column] = pd.factorize(df[column].values, sort=True)[0]

y = df['loss']
x = df.drop(['loss', 'id'], 1)

n_folds  = 6
kf = KFold(x.shape[0], n_folds=n_folds)


    

    




/home/arvc/anaconda3/envs/tensorflow/lib/python3.5/site-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.
  "This module will be removed in 0.20.", DeprecationWarning)

In [ ]:

    

def score(params):           
    print("Training with params : ")
    print(params) 
    
    prediction = np.zeros(x.shape[0])
    
    for i, (train_index, test_index) in enumerate(kf):
        print('\n Fold %d' % (i + 1))
        x_train, x_valid = x.iloc[train_index], x.iloc[test_index]
        y_train, y_valid = y.iloc[train_index], y.iloc[test_index]    
   
        feature_columns = learn.infer_real_valued_columns_from_input(x_train) 
        model = skflow.DNNRegressor(hidden_units=[130, 400, 200, 1], 
                                feature_columns=feature_columns,
                                dropout=params["dropout"],                              
                                optimizer=tf.train.ProximalAdagradOptimizer(
                                  learning_rate=params["learning_rate"],
                                  l1_regularization_strength=params["l1_regularization_strength"]
                                  )
                                )

        model.fit(x_train, y_train, steps = params["steps"],batch_size = params["batch_size"],)
   
        prediction += model.predict(x)

    prediction = prediction/n_folds
  
    score =  mean_absolute_error(y, prediction)  
    print("\tMAE {0}\n\n".format(score))
    return {'loss': score, 'status': STATUS_OK}

def optimize(trials):
    space = {
             'steps' : hp.choice('steps', np.arange(10, 10000, dtype=int)),
             'batch_size' : hp.choice('batch_size', np.arange(1, 500, dtype=int)),   
             'l1_regularization_strength' : hp.quniform('l1_regularization_strength', 0.001, 0.2, 0.001),
             'learning_rate' : hp.quniform('learning_rate', 0.01, 1.0, 0.01),
             'dropout' : hp.quniform('dropout', 0.01, 0.40, 0.01)             
             }

    best = fmin(score, space, algo=tpe.suggest, trials=trials, max_evals=1000)

    print("Best params are:")
    print(best)

#Trials object where the history of search will be stored
trials = Trials()

optimize(trials)


    

    




Training with params : 
{'steps': 1161, 'batch_size': 203, 'l1_regularization_strength': 0.035, 'dropout': 0.21, 'learning_rate': 0.28}

 Fold 1






    




WARNING:tensorflow:float64 is not supported by many models, consider casting to float32.
WARNING:tensorflow:Change warning: default value of `enable_centered_bias` will change after 2016-10-09. It will be disabled by default.Instructions for keeping existing behaviour:
Explicitly set `enable_centered_bias` to 'True' if you want to keep existing behaviour.
WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmpi0yapaha
WARNING:tensorflow:Using default config.
WARNING:tensorflow:float64 is not supported by many models, consider casting to float32.
WARNING:tensorflow:float64 is not supported by many models, consider casting to float32.
WARNING:tensorflow:Calling BaseEstimator.predict (from tensorflow.contrib.learn.python.learn.estimators.estimator) with as_iterable=False is deprecated and will be removed after 2016-09-15.
Instructions for updating:
The default behavior of predict() is changing. The default value for
as_iterable will change to True, and then the flag will be removed
altogether. The behavior of this flag is described below.
WARNING:tensorflow:float64 is not supported by many models, consider casting to float32.






    




 Fold 2






    




WARNING:tensorflow:float64 is not supported by many models, consider casting to float32.
WARNING:tensorflow:Change warning: default value of `enable_centered_bias` will change after 2016-10-09. It will be disabled by default.Instructions for keeping existing behaviour:
Explicitly set `enable_centered_bias` to 'True' if you want to keep existing behaviour.
WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmp6yfl4f2x
WARNING:tensorflow:Using default config.
WARNING:tensorflow:float64 is not supported by many models, consider casting to float32.
WARNING:tensorflow:float64 is not supported by many models, consider casting to float32.
WARNING:tensorflow:Calling BaseEstimator.predict (from tensorflow.contrib.learn.python.learn.estimators.estimator) with as_iterable=False is deprecated and will be removed after 2016-09-15.
Instructions for updating:
The default behavior of predict() is changing. The default value for
as_iterable will change to True, and then the flag will be removed
altogether. The behavior of this flag is described below.
WARNING:tensorflow:float64 is not supported by many models, consider casting to float32.






    




 Fold 3






    




WARNING:tensorflow:float64 is not supported by many models, consider casting to float32.
WARNING:tensorflow:Change warning: default value of `enable_centered_bias` will change after 2016-10-09. It will be disabled by default.Instructions for keeping existing behaviour:
Explicitly set `enable_centered_bias` to 'True' if you want to keep existing behaviour.
WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmpe2gbbiyb
WARNING:tensorflow:Using default config.
WARNING:tensorflow:float64 is not supported by many models, consider casting to float32.
WARNING:tensorflow:float64 is not supported by many models, consider casting to float32.
WARNING:tensorflow:Calling BaseEstimator.predict (from tensorflow.contrib.learn.python.learn.estimators.estimator) with as_iterable=False is deprecated and will be removed after 2016-09-15.
Instructions for updating:
The default behavior of predict() is changing. The default value for
as_iterable will change to True, and then the flag will be removed
altogether. The behavior of this flag is described below.
WARNING:tensorflow:float64 is not supported by many models, consider casting to float32.






    




 Fold 4






    




WARNING:tensorflow:float64 is not supported by many models, consider casting to float32.
WARNING:tensorflow:Change warning: default value of `enable_centered_bias` will change after 2016-10-09. It will be disabled by default.Instructions for keeping existing behaviour:
Explicitly set `enable_centered_bias` to 'True' if you want to keep existing behaviour.
WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmpm8s4mtj1
WARNING:tensorflow:Using default config.
WARNING:tensorflow:float64 is not supported by many models, consider casting to float32.
WARNING:tensorflow:float64 is not supported by many models, consider casting to float32.
WARNING:tensorflow:Calling BaseEstimator.predict (from tensorflow.contrib.learn.python.learn.estimators.estimator) with as_iterable=False is deprecated and will be removed after 2016-09-15.
Instructions for updating:
The default behavior of predict() is changing. The default value for
as_iterable will change to True, and then the flag will be removed
altogether. The behavior of this flag is described below.
WARNING:tensorflow:float64 is not supported by many models, consider casting to float32.






    




 Fold 5






    




WARNING:tensorflow:float64 is not supported by many models, consider casting to float32.
WARNING:tensorflow:Change warning: default value of `enable_centered_bias` will change after 2016-10-09. It will be disabled by default.Instructions for keeping existing behaviour:
Explicitly set `enable_centered_bias` to 'True' if you want to keep existing behaviour.
WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmpnq3qgzuw
WARNING:tensorflow:Using default config.
WARNING:tensorflow:float64 is not supported by many models, consider casting to float32.
WARNING:tensorflow:float64 is not supported by many models, consider casting to float32.
WARNING:tensorflow:Calling BaseEstimator.predict (from tensorflow.contrib.learn.python.learn.estimators.estimator) with as_iterable=False is deprecated and will be removed after 2016-09-15.
Instructions for updating:
The default behavior of predict() is changing. The default value for
as_iterable will change to True, and then the flag will be removed
altogether. The behavior of this flag is described below.
WARNING:tensorflow:float64 is not supported by many models, consider casting to float32.






    




 Fold 6






    




WARNING:tensorflow:float64 is not supported by many models, consider casting to float32.
WARNING:tensorflow:Change warning: default value of `enable_centered_bias` will change after 2016-10-09. It will be disabled by default.Instructions for keeping existing behaviour:
Explicitly set `enable_centered_bias` to 'True' if you want to keep existing behaviour.
WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmp5wbv5k15
WARNING:tensorflow:Using default config.
WARNING:tensorflow:float64 is not supported by many models, consider casting to float32.
WARNING:tensorflow:float64 is not supported by many models, consider casting to float32.
WARNING:tensorflow:Calling BaseEstimator.predict (from tensorflow.contrib.learn.python.learn.estimators.estimator) with as_iterable=False is deprecated and will be removed after 2016-09-15.
Instructions for updating:
The default behavior of predict() is changing. The default value for
as_iterable will change to True, and then the flag will be removed
altogether. The behavior of this flag is described below.
WARNING:tensorflow:float64 is not supported by many models, consider casting to float32.






    




	MAE 10099.76423602289


Training with params : 
{'steps': 2560, 'batch_size': 218, 'l1_regularization_strength': 0.10400000000000001, 'dropout': 0.04, 'learning_rate': 0.04}

 Fold 1






    




WARNING:tensorflow:float64 is not supported by many models, consider casting to float32.
WARNING:tensorflow:Change warning: default value of `enable_centered_bias` will change after 2016-10-09. It will be disabled by default.Instructions for keeping existing behaviour:
Explicitly set `enable_centered_bias` to 'True' if you want to keep existing behaviour.
WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmpzau18i6p
WARNING:tensorflow:Using default config.
WARNING:tensorflow:float64 is not supported by many models, consider casting to float32.
WARNING:tensorflow:float64 is not supported by many models, consider casting to float32.

In [ ]:

    




    

In [ ]: