In [1]:

    

import ensembles as en
import pandas as pd
import numpy as np
import xgboost as xgb
import category_encoders as ce
from sklearn import datasets, linear_model, preprocessing, grid_search
from sklearn.preprocessing import Imputer, PolynomialFeatures, StandardScaler
from sklearn.tree import DecisionTreeClassifier
from sklearn.cross_validation import KFold
from sklearn.ensemble import RandomForestClassifier
from sklearn.cross_validation import StratifiedKFold, KFold
from sklearn.preprocessing import OneHotEncoder
from sklearn.externals import joblib
from keras.layers import Dense, Activation, Dropout
from keras.models import Sequential
from keras.regularizers import l2, activity_l2
from sklearn.metrics import roc_auc_score, average_precision_score, f1_score, log_loss, accuracy_score, \
mean_absolute_error, mean_squared_error, r2_score
from sklearn.cross_validation import train_test_split
from joblib import Parallel, delayed
from sklearn.pipeline import Pipeline
from hyperopt import hp, fmin, tpe, STATUS_OK, Trials 
from hyperas import optim
from hyperas.distributions import choice, uniform, conditional
from functools import partial
np.random.seed(1338)


    

    




/home/prajwal/anaconda3/lib/python3.5/site-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.
  "This module will be removed in 0.20.", DeprecationWarning)
/home/prajwal/anaconda3/lib/python3.5/site-packages/sklearn/grid_search.py:43: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.
  DeprecationWarning)
Using Theano backend.

In [2]:

    

#Setting the parameters for the Multi Layer Perceptron Model (Number Of Layers = 3)
en.set_no_of_layers(3)


    

In [3]:

    

#Default Values
param_mlp = en.parameter_set_multi_layer_perceptron(hyper_parameter_optimisation = False)
print(param_mlp)


    

    




{'b_regularizer0': None, 'init_layer2': 'glorot_uniform', 'b_constraint2': None, 'dim_layer0': 32, 'W_regularizer0': None, 'dropout1': 0, 'b_regularizer2': None, 'weights0': None, 'optimizer': 'rmsprop', 'init_layer0': 'glorot_uniform', 'init_layer1': 'glorot_uniform', 'W_regularizer1': None, 'dim_layer2': 1, 'weights1': None, 'activity_regularizer0': None, 'b_constraint0': None, 'bias1': True, 'activation_layer0': 'sigmoid', 'b_regularizer1': None, 'activity_regularizer2': None, 'weights2': None, 'dim_layer1': 32, 'b_constraint1': None, 'W_constraint2': None, 'activity_regularizer1': None, 'bias0': True, 'W_constraint1': None, 'dropout2': 0, 'W_regularizer2': None, 'activation_layer1': 'sigmoid', 'dropout0': 0, 'W_constraint0': None, 'activation_layer2': 'sigmoid', 'bias2': True, 'hyper_parameter_optimisation': False}

In [4]:

    

#Setting dim_layer, activation, rest are deafault values
param_mlp = en.parameter_set_multi_layer_perceptron(hyper_parameter_optimisation = False, \
                                                    dim_layer = [[32], [64], [1]], \
                                                   activation = [['sigmoid'], ['sigmoid'], ['relu']])
print(param_mlp)


    

    




{'b_regularizer0': None, 'init_layer2': 'glorot_uniform', 'b_constraint2': None, 'dim_layer0': 32, 'W_regularizer0': None, 'dropout1': 0, 'b_regularizer2': None, 'weights0': None, 'optimizer': 'rmsprop', 'init_layer0': 'glorot_uniform', 'init_layer1': 'glorot_uniform', 'W_regularizer1': None, 'dim_layer2': 1, 'weights1': None, 'activity_regularizer0': None, 'b_constraint0': None, 'bias1': True, 'activation_layer0': 'sigmoid', 'b_regularizer1': None, 'activity_regularizer2': None, 'weights2': None, 'dim_layer1': 64, 'b_constraint1': None, 'W_constraint2': None, 'activity_regularizer1': None, 'bias0': True, 'W_constraint1': None, 'dropout2': 0, 'W_regularizer2': None, 'activation_layer1': 'sigmoid', 'dropout0': 0, 'W_constraint0': None, 'activation_layer2': 'relu', 'bias2': True, 'hyper_parameter_optimisation': False}

In [5]:

    

#Setting dim_layer, activation, rest are default values
#Hyper parameter optimisation : dim_layer - Layer1 and Layer 2
#Hyper parameter optimisation : activation - Layer1 
param_mlp = en.parameter_set_multi_layer_perceptron(hyper_parameter_optimisation = True, \
                                                    dim_layer = [[32,64,128], [32,64], [1]], \
                                                   activation = [['sigmoid','relu'], ['sigmoid'], ['relu']], \
                                                   optimizer = 'sgd')
print(param_mlp)


    

    




{'b_regularizer0': [None], 'init_layer2': ['glorot_uniform'], 'b_constraint2': [None], 'dim_layer0': [32, 64, 128], 'W_regularizer0': [None], 'dropout1': [0], 'b_regularizer2': [None], 'weights0': [None], 'optimizer': 'sgd', 'init_layer0': ['glorot_uniform'], 'init_layer1': ['glorot_uniform'], 'W_regularizer1': [None], 'dim_layer2': [1], 'weights1': [None], 'activity_regularizer0': [None], 'b_constraint0': [None], 'bias1': [True], 'activation_layer0': ['sigmoid', 'relu'], 'b_regularizer1': [None], 'activity_regularizer2': [None], 'weights2': [None], 'dim_layer1': [32, 64], 'b_constraint1': [None], 'W_constraint2': [None], 'activity_regularizer1': [None], 'bias0': [True], 'W_constraint1': [None], 'dropout2': [0], 'W_regularizer2': [None], 'activation_layer1': ['sigmoid'], 'dropout0': [0], 'W_constraint0': [None], 'activation_layer2': ['relu'], 'bias2': [True], 'hyper_parameter_optimisation': True}

In [6]:

    

#Setting dim_layer, activation, init_layer, rest are default values
#Hyper parameter optimisation : dim_layer - Layer1 
#Hyper parameter optimisation : activation - Layer1 
param_mlp = en.parameter_set_multi_layer_perceptron(hyper_parameter_optimisation = True, \
                                                    dim_layer = [[32,64,128], [64], [1]], \
                                                   activation = [['sigmoid','relu'], ['sigmoid'], ['relu']],\
                                                   init_layer = [['glorot_uniform'],['normal'],['glorot_uniform']])
print(param_mlp)


    

    




{'b_regularizer0': [None], 'init_layer2': ['glorot_uniform'], 'b_constraint2': [None], 'dim_layer0': [32, 64, 128], 'W_regularizer0': [None], 'dropout1': [0], 'b_regularizer2': [None], 'weights0': [None], 'optimizer': ['rmsprop'], 'init_layer0': ['glorot_uniform'], 'init_layer1': ['normal'], 'W_regularizer1': [None], 'dim_layer2': [1], 'weights1': [None], 'activity_regularizer0': [None], 'b_constraint0': [None], 'bias1': [True], 'activation_layer0': ['sigmoid', 'relu'], 'b_regularizer1': [None], 'activity_regularizer2': [None], 'weights2': [None], 'dim_layer1': [64], 'b_constraint1': [None], 'W_constraint2': [None], 'activity_regularizer1': [None], 'bias0': [True], 'W_constraint1': [None], 'dropout2': [0], 'W_regularizer2': [None], 'activation_layer1': ['sigmoid'], 'dropout0': [0], 'W_constraint0': [None], 'activation_layer2': ['relu'], 'bias2': [True], 'hyper_parameter_optimisation': True}

In [7]:

    

#Setting dim_layer, activation, init_layer, rest are default values
#Hyper parameter optimisation : dim_layer - Layer1 
#Hyper parameter optimisation : activation - Layer1 
#Hyper parameter optimisation : init_layer - Layer1 and Layer3
param_mlp = en.parameter_set_multi_layer_perceptron(hyper_parameter_optimisation = True, \
                                                    dim_layer = [[32,64,128], [64], [1]], \
                                                   activation = [['sigmoid','relu'], ['sigmoid'], ['relu']],\
                                                   init_layer = [['glorot_uniform','uniform'],\
                                                                 ['normal'], ['glorot_uniform','glorot_uniform']])
print(param_mlp)


    

    




{'b_regularizer0': [None], 'init_layer2': ['glorot_uniform', 'glorot_uniform'], 'b_constraint2': [None], 'dim_layer0': [32, 64, 128], 'W_regularizer0': [None], 'dropout1': [0], 'b_regularizer2': [None], 'weights0': [None], 'optimizer': ['rmsprop'], 'init_layer0': ['glorot_uniform', 'uniform'], 'init_layer1': ['normal'], 'W_regularizer1': [None], 'dim_layer2': [1], 'weights1': [None], 'activity_regularizer0': [None], 'b_constraint0': [None], 'bias1': [True], 'activation_layer0': ['sigmoid', 'relu'], 'b_regularizer1': [None], 'activity_regularizer2': [None], 'weights2': [None], 'dim_layer1': [64], 'b_constraint1': [None], 'W_constraint2': [None], 'activity_regularizer1': [None], 'bias0': [True], 'W_constraint1': [None], 'dropout2': [0], 'W_regularizer2': [None], 'activation_layer1': ['sigmoid'], 'dropout0': [0], 'W_constraint0': [None], 'activation_layer2': ['relu'], 'bias2': [True], 'hyper_parameter_optimisation': True}