1. Initialise <br> 2. Generate Features <br> 3. Read Data <br> 4. Filter Features <br> 5. Set Samples & Target Features <br> 6. Recategorise & Transform <br> 7. Rank & Select Features <br> 8. Model <br>
This Jupyter IPython Notebook applies the Temporal-Comorbidity Adjusted Risk of Emergency Readmission (TCARER).
This Jupyter IPython Notebook extract aggregated features from the MySQL database, & then pre-process, configure & apply several modelling approaches.
The pre-processing framework & modelling algorithms in this Notebook are developed as part of the Integrated Care project at the Health & Social Care Modelling Group (HSCMG), The University of Westminster.
Note that some of the scripts are optional or subject to some pre-configurations. Please refer to the comments & the project documentations for further details.
<hr> Copyright 2017 The Project Authors. All Rights Reserved.
It is licensed under the Apache License, Version 2.0. you may not use this file except in compliance with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.</font>
<hr>
Reload modules
In [ ]:
# Reload modules
# It is an optional step. It is useful to run when external Python modules are being modified
# It is reloading all modules (except those excluded by %aimport) every time before executing the Python code typed.
# Note: It may conflict with serialisation, when external modules are being modified
# %load_ext autoreload
# %autoreload 2
Import libraries
In [ ]:
# Import Python libraries
import logging
import os
import sys
import gc
import pandas as pd
from IPython.display import display, HTML
from collections import OrderedDict
import numpy as np
import statistics
from scipy.stats import stats
In [ ]:
# Import local Python modules
from Configs.CONSTANTS import CONSTANTS
from Configs.Logger import Logger
from Features.Variables import Variables
from ReadersWriters.ReadersWriters import ReadersWriters
from Stats.PreProcess import PreProcess
from Stats.FeatureSelection import FeatureSelection
from Stats.TrainingMethod import TrainingMethod
from Stats.Plots import Plots
In [ ]:
# Check the interpreter
print("\nMake sure the correct Python interpreter is used!")
print(sys.version)
print("\nMake sure sys.path of the Python interpreter is correct!")
print(os.getcwd())
Main configuration Settings:
External Configration Files:
In [ ]:
config_path = os.path.abspath("ConfigInputs/CONFIGURATIONS.ini")
io_path = os.path.abspath("../../tmp/TCARER/Basic_prototype")
app_name = "T-CARER"
submodel_name = "hesIp"
submodel_input_name = "tcarer_model_features_ip"
print("\n The full path of the configuration file: \n\t", config_path,
"\n The full path of the output folder: \n\t", io_path,
"\n The application name (the suffix of the outputs file name): \n\t", app_name,
"\n The sub-model name, to locate the related feature configuration: \n\t", submodel_name,
"\n The the sub-model's the file name of the input: \n\t", submodel_input_name)
Initialise logs
In [ ]:
if not os.path.exists(io_path):
os.makedirs(io_path, exist_ok=True)
logger = Logger(path=io_path, app_name=app_name, ext="log")
logger = logging.getLogger(app_name)
Initialise constants and some of classes
In [ ]:
# Initialise constants
CONSTANTS.set(io_path, app_name)
In [ ]:
# Initialise other classes
readers_writers = ReadersWriters()
preprocess = PreProcess(io_path)
feature_selection = FeatureSelection()
plts = Plots()
In [ ]:
# Set print settings
pd.set_option('display.width', 1600, 'display.max_colwidth', 800)
Read the input features' confugration file & store the features metadata
In [ ]:
# variables settings
features_metadata = dict()
features_metadata_all = readers_writers.load_csv(path=CONSTANTS.io_path, title=CONSTANTS.config_features_path, dataframing=True)
features_metadata = features_metadata_all.loc[(features_metadata_all["Selected"] == 1) &
(features_metadata_all["Table_Reference_Name"] == submodel_name)]
features_metadata.reset_index()
# print
display(features_metadata)
Set input features' metadata dictionaries
In [ ]:
# Dictionary of features types, dtypes, & max-states
features_types = dict()
features_dtypes = dict()
features_states_values = dict()
features_names_group = dict()
for _, row in features_metadata.iterrows():
if not pd.isnull(row["Variable_Max_States"]):
states_values = str(row["Variable_Max_States"]).split(',')
states_values = list(map(int, states_values))
else:
states_values = None
if not pd.isnull(row["Variable_Aggregation"]):
postfixes = row["Variable_Aggregation"].replace(' ', '').split(',')
f_types = row["Variable_Type"].replace(' ', '').split(',')
f_dtypes = row["Variable_dType"].replace(' ', '').split(',')
for p in range(len(postfixes)):
features_types[row["Variable_Name"] + "_" + postfixes[p]] = f_types[p]
features_dtypes[row["Variable_Name"] + "_" + postfixes[p]] = pd.Series(dtype=f_dtypes[p])
features_states_values[row["Variable_Name"] + "_" + postfixes[p]] = states_values
features_names_group[row["Variable_Name"] + "_" + postfixes[p]] = row["Variable_Name"] + "_" + postfixes[p]
else:
features_types[row["Variable_Name"]] = row["Variable_Type"]
features_dtypes[row["Variable_Name"]] = row["Variable_dType"]
features_states_values[row["Variable_Name"]] = states_values
features_names_group[row["Variable_Name"]] = row["Variable_Name"]
if states_values is not None:
for postfix in states_values:
features_names_group[row["Variable_Name"] + "_" + str(postfix)] = row["Variable_Name"]
features_dtypes = pd.DataFrame(features_dtypes).dtypes
In [ ]:
# Dictionary of features groups
features_types_group = OrderedDict()
f_types = set([f_type for f_type in features_types.values()])
features_types_group = OrderedDict(zip(list(f_types), [set() for _ in range(len(f_types))]))
for f_name, f_type in features_types.items():
features_types_group[f_type].add(f_name)
print("Available features types: " + ','.join(f_types))
Notes:
In [ ]:
skip = True
# settings
csv_schema = ["my_db_schema"]
csv_input_tables = ["tcarer_features"]
csv_history_tables = ["hesIp"]
csv_column_index = "localID"
csv_output_table = "tcarer_model_features_ip"
csv_query_batch_size = 100000
In [ ]:
if skip is False:
# generate the csv file
variables = Variables(submodel_name,
CONSTANTS.io_path,
CONSTANTS.io_path,
CONSTANTS.config_features_path,
csv_output_table)
variables.set(csv_schema, csv_input_tables, csv_history_tables, csv_column_index, csv_query_batch_size)
Read the input features from the CSV input file
In [ ]:
features_input = readers_writers.load_csv(path=CONSTANTS.io_path, title=submodel_input_name, dataframing=True)
features_input.astype(dtype=features_dtypes)
print("Number of columns: ", len(features_input.columns), "; Total records: ", len(features_input.index))
Verify features visually
In [ ]:
display(features_input.head())
Produce a descriptive stat report of 'Categorical', 'Continuous', & 'TARGET' features
In [ ]:
file_name = "Step_04_Data_ColumnNames"
readers_writers.save_csv(path=CONSTANTS.io_path, title=file_name, data=list(features_input.columns.values), append=False)
file_name = "Step_04_Stats_Categorical"
o_stats = preprocess.stats_discrete_df(df=features_input, includes=features_types_group["CATEGORICAL"],
file_name=file_name)
file_name = "Step_04_Stats_Continuous"
o_stats = preprocess.stats_continuous_df(df=features_input, includes=features_types_group["CONTINUOUS"],
file_name=file_name)
file_name = "Step_04_Stats_Target"
o_stats = preprocess.stats_discrete_df(df=features_input, includes=features_types_group["TARGET"],
file_name=file_name)
Nothing to do!
Notes:
In [ ]:
# Exclusion of unused features
# excluded = [name for name in features_input.columns if name not in features_names_group.keys()]
# features_input = features_input.drop(excluded, axis=1)
# print("Number of columns: ", len(features_input.columns), "; Total records: ", len(features_input.index))
Set the samples
In [ ]:
frac_train = 0.50
replace = False
random_state = 100
nrows = len(features_input.index)
features = {"train": dict(), "test": dict()}
features["train"] = features_input.sample(frac=frac_train, replace=False, random_state=100)
features["test"] = features_input.drop(features["train"].index)
features["train"] = features["train"].reset_index(drop=True)
features["test"] = features["test"].reset_index(drop=True)
Verify features visually
In [ ]:
display(features_input.head())
Clean-Up
In [ ]:
features_input = None
gc.collect()
Set independent, target & ID features
In [ ]:
target_labels = list(features_types_group["TARGET"])
target_id = ["patientID"]
In [ ]:
features["train_indep"] = dict()
features["train_target"] = dict()
features["train_id"] = dict()
features["test_indep"] = dict()
features["test_target"] = dict()
features["test_id"] = dict()
# Independent and target features
def set_features_indep_target(df):
df_targets = pd.DataFrame(dict(zip(target_labels, [[]] * len(target_labels))))
for i in range(len(target_labels)):
df_targets[target_labels[i]] = df[target_labels[i]]
df_indep = df.drop(target_labels + target_id, axis=1)
df_id = pd.DataFrame({target_id[0]: df[target_id[0]]})
return df_indep, df_targets, df_id
In [ ]:
# train & test sets
features["train_indep"], features["train_target"], features["train_id"] = set_features_indep_target(features["train"])
features["test_indep"], features["test_target"], features["test_id"] = set_features_indep_target(features["test"])
# print
print("Number of columns: ", len(features["train_indep"].columns))
print("features: {train: ", len(features["train_indep"]), ", test: ", len(features["test_indep"]), "}")
Verify features visually
In [ ]:
display(pd.concat([features["train_id"].head(), features["train_target"].head(), features["train_indep"].head()], axis=1))
display(pd.concat([features["test_id"].head(), features["test_target"].head(), features["test_indep"].head()], axis=1))
Clean-Up
In [ ]:
del features["train"]
del features["test"]
gc.collect()
Serialise & save the samples before any feature transformation.
This snapshot of the samples may be used for the population profiling
In [ ]:
file_name = "Step_05_Features"
readers_writers.save_serialised_compressed(path=CONSTANTS.io_path, title=file_name, objects=features)
# print
print("Number of columns: ", len(features["train_indep"].columns),
"features: {train: ", len(features["train_indep"]), ", test: ", len(features["test_indep"]), "}")
In order to reduce sparseness and invalid features, highly stationary ones were withdrawn. The features that had constant counts less than or equal a threshold were ltered out, to exclude highly constants and near-zero variances.
The near zero variance rules are presented in below:
Configure: the function
In [ ]:
thresh_unique_cut = 100
thresh_freq_cut = 1000
excludes = []
file_name = "Step_05_Preprocess_NZV_config"
features["train_indep"], o_summaries = preprocess.near_zero_var_df(df=features["train_indep"],
excludes=excludes,
file_name=file_name,
thresh_unique_cut=thresh_unique_cut,
thresh_freq_cut=thresh_freq_cut,
to_search=True)
file_name = "Step_05_Preprocess_NZV"
readers_writers.save_text(path=CONSTANTS.io_path, title=file_name, data=o_summaries, append=False, ext="log")
file_name = "Step_05_Preprocess_NZV_config"
features["test_indep"], o_summaries = preprocess.near_zero_var_df(df=features["test_indep"],
excludes=excludes,
file_name=file_name,
thresh_unique_cut=thresh_unique_cut,
thresh_freq_cut=thresh_freq_cut,
to_search=False)
# print
print("Number of columns: ", len(features["train_indep"].columns))
print("features: {train: ", len(features["train_indep"]), ", test: ", len(features["test_indep"]), "}")
In [ ]:
thresh_corr_cut = 0.95
excludes = list(features_types_group["CATEGORICAL"])
file_name = "Step_05_Preprocess_Corr_config"
features["train_indep"], o_summaries = preprocess.high_linear_correlation_df(df=features["train_indep"],
excludes=excludes,
file_name=file_name,
thresh_corr_cut=thresh_corr_cut,
to_search=True)
file_name = "Step_05_Preprocess_Corr"
readers_writers.save_text(path=CONSTANTS.io_path, title=file_name, data=o_summaries, append=False, ext="log")
file_name = "Step_05_Preprocess_Corr_config"
features["test_indep"], o_summaries = preprocess.high_linear_correlation_df(df=features["test_indep"],
excludes=excludes,
file_name=file_name,
thresh_corr_cut=thresh_corr_cut,
to_search=False)
# print
print("Number of columns: ", len(features["train_indep"].columns))
print("features: {train: ", len(features["train_indep"]), ", test: ", len(features["test_indep"]), "}")
Produce a descriptive stat report of 'Categorical', 'Continuous', & 'TARGET' features
In [ ]:
# columns
file_name = "Step_05_Data_ColumnNames_Train"
readers_writers.save_csv(path=CONSTANTS.io_path, title=file_name,
data=list(features["train_indep"].columns.values), append=False)
# Sample - Train
file_name = "Step_05_Stats_Categorical_Train"
o_stats = preprocess.stats_discrete_df(df=features["train_indep"], includes=features_types_group["CATEGORICAL"],
file_name=file_name)
file_name = "Step_05_Stats_Continuous_Train"
o_stats = preprocess.stats_continuous_df(df=features["train_indep"], includes=features_types_group["CONTINUOUS"],
file_name=file_name)
# Sample - Test
file_name = "Step_05_Stats_Categorical_Test"
o_stats = preprocess.stats_discrete_df(df=features["test_indep"], includes=features_types_group["CATEGORICAL"],
file_name=file_name)
file_name = "Step_05_Stats_Continuous_Test"
o_stats = preprocess.stats_continuous_df(df=features["test_indep"], includes=features_types_group["CONTINUOUS"],
file_name=file_name)
Verify features visually
In [ ]:
display(pd.concat([features["train_id"].head(), features["train_target"].head(), features["train_indep"].head()], axis=1))
display(pd.concat([features["test_id"].head(), features["test_target"].head(), features["test_indep"].head()], axis=1))
Define the factorisation function to generate dummy features for the categorical features.
In [ ]:
def factorise_settings(max_categories_frac, min_categories_num, exclude_zero):
categories_dic = dict()
labels_dic = dict()
dtypes_dic = dict()
dummies = []
for f_name in features_types_group["CATEGORICAL"]:
if f_name in features["train_indep"]:
# find top & valid states
summaries = stats.itemfreq(features["train_indep"][f_name])
summaries = pd.DataFrame({"value": summaries[:, 0], "freq": summaries[:, 1]})
summaries["value"] = list(map(int, summaries["value"]))
summaries = summaries.sort_values("freq", ascending=False)
summaries = list(summaries["value"])
# exclude zero state
if exclude_zero is True and len(summaries) > 1:
summaries = [s for s in summaries if s != 0]
# if included in the states
summaries = [v for v in summaries if v in set(features_states_values[f_name])]
# limit number of states
max_cnt = max(int(len(summaries) * max_categories_frac), min_categories_num)
# set states
categories_dic[f_name] = summaries[0:max_cnt]
labels_dic[f_name] = [f_name + "_" + str(c) for c in categories_dic[f_name]]
dtypes_dic = {**dtypes_dic,
**dict(zip(labels_dic[f_name], [pd.Series(dtype='i') for _ in range(len(categories_dic[f_name]))]))}
dummies += labels_dic[f_name]
dtypes_dic = pd.DataFrame(dtypes_dic).dtypes
# print
print("Total Categorical Variables : ", len(categories_dic.keys()),
"; Total Number of Dummy Variables: ", sum([len(categories_dic[f_name]) for f_name in categories_dic.keys()]))
return categories_dic, labels_dic, dtypes_dic, features_types
Select categories: by order of freq., max_categories_frac, & max_categories_num
Configure: The input arguments are:
In [ ]:
max_categories_frac = 0.90
min_categories_num = 1
exclude_zero = False # if possible remove state zero
categories_dic, labels_dic, dtypes_dic, features_types_group["DUMMIES"] = \
factorise_settings(max_categories_frac, min_categories_num, exclude_zero)
Manually add dummy variables to the dataframe & remove the original Categorical variables
In [ ]:
features["train_indep_temp"] = preprocess.factoring_feature_wise(features["train_indep"], categories_dic, labels_dic, dtypes_dic, threaded=False)
features["test_indep_temp"] = preprocess.factoring_feature_wise(features["test_indep"], categories_dic, labels_dic, dtypes_dic, threaded=False)
# print
print("Number of columns: ", len(features["train_indep"].columns))
print("features: {train: ", len(features["train_indep"]), ", test: ", len(features["test_indep"]), "}")
Verify features visually
In [ ]:
display(pd.concat([features["train_id"].head(), features["train_target"].head(), features["train_indep_temp"].head()], axis=1))
display(pd.concat([features["test_id"].head(), features["test_target"].head(), features["test_indep_temp"].head()], axis=1))
Set
In [ ]:
features["train_indep"] = features["train_indep_temp"].copy(True)
features["test_indep"] = features["test_indep_temp"].copy(True)
Clean-Up
In [ ]:
del features["train_indep_temp"]
del features["test_indep_temp"]
gc.collect()
Optional: Remove more features with near zero variance, after the factorisation step. Configure: the function
In [ ]:
# the cutoff for the percentage of distinct values out of the number of total samples (upper limit). e.g. 10 * 100 / 100
thresh_unique_cut = 100
# the cutoff for the ratio of the most common value to the second most common value (lower limit). eg. 95/5
thresh_freq_cut = 1000
excludes = []
file_name = "Step_06_Preprocess_NZV_config"
features["train_indep"], o_summaries = preprocess.near_zero_var_df(df=features["train_indep"],
excludes=excludes,
file_name=file_name,
thresh_unique_cut=thresh_unique_cut,
thresh_freq_cut=thresh_freq_cut,
to_search=True)
file_name = "Step_06_Preprocess_NZV"
readers_writers.save_text(path=CONSTANTS.io_path, title=file_name, data=o_summaries, append=False, ext="log")
file_name = "Step_06_Preprocess_NZV_config"
features["test_indep"], o_summaries = preprocess.near_zero_var_df(df=features["test_indep"],
excludes=excludes,
file_name=file_name,
thresh_unique_cut=thresh_unique_cut,
thresh_freq_cut=thresh_freq_cut,
to_search=False)
# print
print("Number of columns: ", len(features["train_indep"].columns))
print("features: {train: ", len(features["train_indep"]), ", test: ", len(features["test_indep"]), "}")
Optional: Remove more features with highly linearly correlated, after the factorisation step. Configure: the function
In [ ]:
# A numeric value for the pair-wise absolute correlation cutoff. e.g. 0.95
thresh_corr_cut = 0.95
excludes = []
file_name = "Step_06_Preprocess_Corr_config"
features["train_indep"], o_summaries = preprocess.high_linear_correlation_df(df=features["train_indep"],
excludes=excludes,
file_name=file_name,
thresh_corr_cut=thresh_corr_cut,
to_search=True)
file_name = "Step_06_Preprocess_Corr"
readers_writers.save_text(path=CONSTANTS.io_path, title=file_name, data=o_summaries, append=False, ext="log")
file_name = "Step_06_Preprocess_Corr_config"
features["test_indep"], o_summaries = preprocess.high_linear_correlation_df(df=features["test_indep"],
excludes=excludes,
file_name=file_name,
thresh_corr_cut=thresh_corr_cut,
to_search=False)
# print
print("Number of columns: ", len(features["train_indep"].columns))
print("features: {train: ", len(features["train_indep"]), ", test: ", len(features["test_indep"]), "}")
Produce a descriptive stat report of 'Categorical', 'Continuous', & 'TARGET' features
In [ ]:
# columns
file_name = "Step_06_4_Data_ColumnNames_Train"
readers_writers.save_csv(path=CONSTANTS.io_path, title=file_name,
data=list(features["train_indep"].columns.values), append=False)
# Sample - Train
file_name = "Step_06_4_Stats_Categorical_Train"
o_stats = preprocess.stats_discrete_df(df=features["train_indep"], includes=features_types_group["CATEGORICAL"],
file_name=file_name)
file_name = "Step_06_4_Stats_Continuous_Train"
o_stats = preprocess.stats_continuous_df(df=features["train_indep"], includes=features_types_group["CONTINUOUS"],
file_name=file_name)
# Sample - Test
file_name = "Step_06_4_Stats_Categorical_Test"
o_stats = preprocess.stats_discrete_df(df=features["test_indep"], includes=features_types_group["CATEGORICAL"],
file_name=file_name)
file_name = "Step_06_4_Stats_Continuous_Test"
o_stats = preprocess.stats_continuous_df(df=features["test_indep"], includes=features_types_group["CONTINUOUS"],
file_name=file_name)
Verify features visually
In [ ]:
display(pd.concat([features["train_id"].head(), features["train_target"].head(), features["train_indep"].head()], axis=1))
display(pd.concat([features["test_id"].head(), features["test_target"].head(), features["test_indep"].head()], axis=1))
Tranformation: scale Note:: It is highly resource intensive
In [ ]:
transform_type = "scale"
kwargs = {"with_mean": True}
method_args = dict()
excludes = list(features_types_group["CATEGORICAL"]) + list(features_types_group["DUMMIES"])
features["train_indep"], method_args = preprocess.transform_df(df=features["train_indep"], excludes=excludes,
transform_type=transform_type, threaded=False,
method_args=method_args, **kwargs)
features["test_indep"], _ = preprocess.transform_df(df=features["test_indep"], excludes=excludes,
transform_type=transform_type, threaded=False,
method_args=method_args, **kwargs)
# print("Metod arguments:", method_args)
Tranformation: Yeo-Johnson Note:: It is highly resource intensive
In [ ]:
transform_type = "yeo_johnson"
kwargs = {"lmbda": -0.5, "derivative": 0, "epsilon": np.finfo(np.float).eps, "inverse": False}
method_args = dict()
excludes = list(features_types_group["CATEGORICAL"]) + list(features_types_group["DUMMIES"])
features["train_indep"], method_args = preprocess.transform_df(df=features["train_indep"], excludes=excludes,
transform_type=transform_type, threaded=False,
method_args=method_args, **kwargs)
features["test_indep"], _ = preprocess.transform_df(df=features["test_indep"], excludes=excludes,
transform_type=transform_type, threaded=False,
method_args=method_args, **kwargs)
# print("Metod arguments:", method_args)
Visual verification
In [ ]:
display(pd.concat([features["train_id"].head(), features["train_target"].head(), features["train_indep"].head()], axis=1))
display(pd.concat([features["test_id"].head(), features["test_target"].head(), features["test_indep"].head()], axis=1))
Produce a descriptive stat report of 'Categorical', 'Continuous', & 'TARGET' features
In [ ]:
# Statsistics report for 'Categorical', 'Continuous', & 'TARGET' variables
# columns
file_name = "Step_06_6_Data_ColumnNames_Train"
readers_writers.save_csv(path=CONSTANTS.io_path, title=file_name,
data=list(features["train_indep"].columns.values), append=False)
# Sample - Train
file_name = "Step_06_6_Stats_Categorical_Train"
o_stats = preprocess.stats_discrete_df(df=features["train_indep"], includes=features_types_group["CATEGORICAL"],
file_name=file_name)
file_name = "Step_06_6_Stats_Continuous_Train"
o_stats = preprocess.stats_continuous_df(df=features["train_indep"], includes=features_types_group["CONTINUOUS"],
file_name=file_name)
# Sample - Test
file_name = "Step_06_6_Stats_Categorical_Test"
o_stats = preprocess.stats_discrete_df(df=features["test_indep"], includes=features_types_group["CATEGORICAL"],
file_name=file_name)
file_name = "Step_06_6_Stats_Continuous_Test"
o_stats = preprocess.stats_continuous_df(df=features["test_indep"], includes=features_types_group["CONTINUOUS"],
file_name=file_name)
Configure: the general settings
In [ ]:
# select the target variable
target_feature = "label365" # "label30", "label365"
# number of trials
num_trials = 1
model_rank = dict()
o_summaries_df = dict()
Ranking Method: Random forest classifier (Brieman)
Define a set of classifiers with different settings, to be used in feature ranking trials.
In [ ]:
def rank_random_forest_brieman(features_indep_arg, features_target_arg, num_trials):
num_settings = 3
o_summaries_df = [pd.DataFrame({'Name': list(features_indep_arg.columns.values)}) for _ in range(num_trials * num_settings)]
model_rank = [None] * (num_trials * num_settings)
# trials
for i in range(num_trials):
print("Trial: " + str(i))
# setting-1
s_i = i
model_rank[s_i] = feature_selection.rank_random_forest_breiman(
features_indep_arg.values, features_target_arg.values,
**{"n_estimators": 10, "criterion": 'gini', "max_depth": None, "min_samples_split": 2, "min_samples_leaf": 1,
"min_weight_fraction_leaf": 0.0, "max_features": 'auto', "max_leaf_nodes": None, "bootstrap": True,
"oob_score": False, "n_jobs": -1, "random_state": None, "verbose": 0, "warm_start": False, "class_weight": None})
# setting-2
s_i = num_trials + i
model_rank[s_i] = feature_selection.rank_random_forest_breiman(
features_indep_arg.values, features_target_arg.values,
**{"n_estimators": 10, "criterion": 'gini', "max_depth": None, "min_samples_split": 50, "min_samples_leaf": 25,
"min_weight_fraction_leaf": 0.0, "max_features": 'auto', "max_leaf_nodes": None, "bootstrap": True,
"oob_score": False, "n_jobs": -1, "random_state": None, "verbose": 0, "warm_start": False, "class_weight": None})
# setting-3
s_i = (num_trials * 2) + i
model_rank[s_i] = feature_selection.rank_random_forest_breiman(
features_indep_arg.values, features_target_arg.values,
**{"n_estimators": 10, "criterion": 'gini', "max_depth": None, "min_samples_split": 40, "min_samples_leaf": 20,
"min_weight_fraction_leaf": 0.0, "max_features": 'auto', "max_leaf_nodes": None, "bootstrap": True,
"oob_score": False, "n_jobs": -1, "random_state": None, "verbose": 0, "warm_start": True, "class_weight": None})
for i in range((num_trials * num_settings)):
o_summaries_df[i]['Importance'] = list(model_rank[i].feature_importances_)
o_summaries_df[i] = o_summaries_df[i].sort_values(['Importance'], ascending = [0])
o_summaries_df[i] = o_summaries_df[i].reset_index(drop = True)
o_summaries_df[i]['Order'] = range(1, len(o_summaries_df[i]['Importance']) + 1)
return model_rank, o_summaries_df
Ranking Method: Gradient Boosted Regression Trees (GBRT)
Define a set of classifiers with different settings, to be used in feature ranking trials.
In [ ]:
def rank_gbrt(features_indep_arg, features_target_arg, num_trials):
num_settings = 3
o_summaries_df = [pd.DataFrame({'Name': list(features_indep_arg.columns.values)}) for _ in range(num_trials * num_settings)]
model_rank = [None] * (num_trials * num_settings)
# trials
for i in range(num_trials):
print("Trial: " + str(i))
# setting-1
s_i = i
model_rank[s_i] = feature_selection.rank_tree_gbrt(
features_indep_arg.values, features_target_arg.values,
**{"loss": 'ls', "learning_rate": 0.1, "n_estimators": 100, "subsample": 1.0, "min_samples_split": 2, "min_samples_leaf": 1,
"min_weight_fraction_leaf": 0.0, "max_depth": 10, "init": None, "random_state": None, "max_features": None, "alpha": 0.9,
"verbose": 0, "max_leaf_nodes": None, "warm_start": False, "presort": True})
# setting-2
s_i = num_trials + i
model_rank[s_i] = feature_selection.rank_tree_gbrt(
features_indep_arg.values, features_target_arg.values,
**{"loss": 'ls', "learning_rate": 0.1, "n_estimators": 100, "subsample": 1.0, "min_samples_split": 2, "min_samples_leaf": 1,
"min_weight_fraction_leaf": 0.0, "max_depth": 5, "init": None, "random_state": None, "max_features": None, "alpha": 0.9,
"verbose": 0, "max_leaf_nodes": None, "warm_start": False, "presort": True})
# setting-3
s_i = (num_trials * 2) + i
model_rank[s_i] = feature_selection.rank_tree_gbrt(
features_indep_arg.values, features_target_arg.values,
**{"loss": 'ls', "learning_rate": 0.1, "n_estimators": 100, "subsample": 1.0, "min_samples_split": 2, "min_samples_leaf": 1,
"min_weight_fraction_leaf": 0.0, "max_depth": 3, "init": None, "random_state": None, "max_features": None, "alpha": 0.9,
"verbose": 0, "max_leaf_nodes": None, "warm_start": False, "presort": True})
for i in range((num_trials * num_settings)):
o_summaries_df[i]['Importance'] = list(model_rank[i].feature_importances_)
o_summaries_df[i] = o_summaries_df[i].sort_values(['Importance'], ascending = [0])
o_summaries_df[i] = o_summaries_df[i].reset_index(drop = True)
o_summaries_df[i]['Order'] = range(1, len(o_summaries_df[i]['Importance']) + 1)
return model_rank, o_summaries_df
Ranking Method: Randomized Logistic Regression
Define a set of classifiers with different settings, to be used in feature ranking trials.
In [ ]:
def rank_randLogit(features_indep_arg, features_target_arg, num_trials):
num_settings = 3
o_summaries_df = [pd.DataFrame({'Name': list(features_indep_arg.columns.values)}) for _ in range(num_trials * num_settings)]
model_rank = [None] * (num_trials * num_settings)
# trials
for i in range(num_trials):
print("Trial: " + str(i))
# setting-1
s_i = i
model_rank[s_i] = feature_selection.rank_random_logistic_regression(
features_indep_arg.values, features_target_arg.values,
**{"C": 1, "scaling": 0.5, "sample_fraction": 0.75, "n_resampling": 200, "selection_threshold": 0.25, "tol": 0.001,
"fit_intercept": True, "verbose": False, "normalize": True, "random_state": None, "n_jobs": 1, "pre_dispatch": '3*n_jobs'})
# setting-2
s_i = num_trials + i
model_rank[s_i] = feature_selection.rank_random_logistic_regression(
features_indep_arg.values, features_target_arg.values,
**{"C": 1, "scaling": 0.5, "sample_fraction": 0.50, "n_resampling": 200, "selection_threshold": 0.25, "tol": 0.001,
"fit_intercept": True, "verbose": False, "normalize": True, "random_state": None, "n_jobs": 1, "pre_dispatch": '3*n_jobs'})
# setting-3
s_i = (num_trials * 2) + i
model_rank[s_i] = feature_selection.rank_random_logistic_regression(
features_indep_arg.values, features_target_arg.values,
**{"C": 1, "scaling": 0.5, "sample_fraction": 0.90, "n_resampling": 200, "selection_threshold": 0.25, "tol": 0.001,
"fit_intercept": True, "verbose": False, "normalize": True, "random_state": None, "n_jobs": 1, "pre_dispatch": '3*n_jobs'})
for i in range((num_trials * num_settings)):
o_summaries_df[i]['Importance'] = list(model_rank[i].scores_)
o_summaries_df[i] = o_summaries_df[i].sort_values(['Importance'], ascending = [0])
o_summaries_df[i] = o_summaries_df[i].reset_index(drop = True)
o_summaries_df[i]['Order'] = range(1, len(o_summaries_df[i]['Importance']) + 1)
return model_rank, o_summaries_df
Run one or more feature ranking methods and trials
Ranking Method: Random forest classifier (Brieman) Note:: It is moderately resource intensive
In [ ]:
rank_model = "rfc"
model_rank[rank_model] = dict()
o_summaries_df[rank_model] = dict()
model_rank[rank_model], o_summaries_df[rank_model] = rank_random_forest_brieman(
features["train_indep"], features["train_target"][target_feature], num_trials)
Ranking Method: Gradient Boosted Regression Trees (GBRT) Note:: It is moderately resource intensive
In [ ]:
rank_model = "gbrt"
model_rank[rank_model] = dict()
o_summaries_df[rank_model] = dict()
model_rank[rank_model], o_summaries_df[rank_model] = rank_gbrt(
features["train_indep"], features["train_target"][target_feature], num_trials)
Ranking Method: Randomized Logistic Regression Note:: It is moderately resource intensive
In [ ]:
rank_model = "randLogit"
model_rank[rank_model] = dict()
o_summaries_df[rank_model] = dict()
model_rank[rank_model], o_summaries_df[rank_model] = rank_randLogit(
features["train_indep"], features["train_target"][target_feature], num_trials)
In [ ]:
# combine scores
def rank_summarise (features_arg, o_summaries_df_arg):
summaries_temp = {'Order_avg': [], 'Order_max': [], 'Order_min': [], 'Importance_avg': []}
summary_order = []
summary_importance = []
for f_name in list(features_arg.columns.values):
for i in range(len(o_summaries_df_arg)):
summary_order.append(o_summaries_df_arg[i][o_summaries_df_arg[i]['Name'] == f_name]['Order'].values)
summary_importance.append(o_summaries_df_arg[i][o_summaries_df_arg[i]['Name'] == f_name]['Importance'].values)
summaries_temp['Order_avg'].append(statistics.mean(np.concatenate(summary_order)))
summaries_temp['Order_max'].append(max(np.concatenate(summary_order)))
summaries_temp['Order_min'].append(min(np.concatenate(summary_order)))
summaries_temp['Importance_avg'].append(statistics.mean(np.concatenate(summary_importance)))
summaries_df = pd.DataFrame({'Name': list(features_arg.columns.values)})
summaries_df['Order_avg'] = summaries_temp['Order_avg']
summaries_df['Order_max'] = summaries_temp['Order_max']
summaries_df['Order_min'] = summaries_temp['Order_min']
summaries_df['Importance_avg'] = summaries_temp['Importance_avg']
summaries_df = summaries_df.sort_values(['Order_avg'], ascending = [1])
return summaries_df
In [ ]:
# combine scores
summaries_df = dict()
for rank_model in o_summaries_df.keys():
summaries_df[rank_model] = dict()
summaries_df[rank_model] = rank_summarise(features["train_indep"], o_summaries_df[rank_model])
Save
In [ ]:
for rank_model in model_rank.keys():
file_name = "Step_07_Model_Train_model_rank_" + rank_model
readers_writers.save_serialised_compressed(path=CONSTANTS.io_path, title=file_name, objects=model_rank[rank_model])
file_name = "Step_07_Model_Train_model_rank_summaries_" + rank_model
readers_writers.save_serialised_compressed(path=CONSTANTS.io_path, title=file_name, objects=o_summaries_df[rank_model])
Configure: the selection method
In [ ]:
rank_model = "rfc"
file_name = "Step_07_Top_Features_" + rank_model
rank_top_features_max = 400
rank_top_features_score_min = 0.1 * (10 ^ -20)
# sort features
features_names_selected = summaries_df[rank_model]['Name'][summaries_df[rank_model]['Order_avg'] >= rank_top_features_score_min]
features_names_selected = (features_names_selected[0:rank_top_features_max]).tolist()
Save
In [ ]:
# save to CSV
readers_writers.save_csv(path=CONSTANTS.io_path, title=file_name, data=features_names_selected, append=False, header=False)
# print
print("Number of columns: ", len(features["train_indep"].columns))
print("features: {train: ", len(features["train_indep"]), ", test: ", len(features["test_indep"]), "}")
print("List of sorted features, which can be modified:\n " + CONSTANTS.io_path + file_name + "csv")
Configure: the selected feature manually if it isnecessary!
In [ ]:
file_name = "Step_07_Top_Features_rfc_adhoc"
features_names_selected = readers_writers.load_csv(path=CONSTANTS.io_path, title=file_name, dataframing=False)[0]
features_names_selected = [f.replace("\n", "") for f in features_names_selected]
display(pd.DataFrame(features_names_selected))
Verify the top features visually
In [ ]:
# print
print("Number of columns: ", len(features["train_indep"].columns),
";\nNumber of top columns: ", len(features["train_indep"][features_names_selected].columns))
print("features: {train: ", len(features["train_indep"][features_names_selected]), ", test: ", len(features["test_indep"][features_names_selected]), "}")
Produce a descriptive stat report of 'Categorical', 'Continuous', & 'TARGET' features
In [ ]:
# columns
file_name = "Step_07_Data_ColumnNames_Train"
readers_writers.save_csv(path=CONSTANTS.io_path, title=file_name,
data=list(features["train_indep"][features_names_selected].columns.values), append=False)
# Sample - Train
file_name = "Step_07_Stats_Categorical_Train"
o_stats = preprocess.stats_discrete_df(df=features["train_indep"][features_names_selected], includes=features_types_group["CATEGORICAL"],
file_name=file_name)
file_name = "Step_07_Stats_Continuous_Train"
o_stats = preprocess.stats_continuous_df(df=features["train_indep"][features_names_selected], includes=features_types_group["CONTINUOUS"],
file_name=file_name)
# Sample - Test
file_name = "Step_07_Stats_Categorical_Test"
o_stats = preprocess.stats_discrete_df(df=features["test_indep"][features_names_selected], includes=features_types_group["CATEGORICAL"],
file_name=file_name)
file_name = "Step_07_Stats_Continuous_Test"
o_stats = preprocess.stats_continuous_df(df=features["test_indep"][features_names_selected], includes=features_types_group["CONTINUOUS"],
file_name=file_name)
In [ ]:
file_name = "Step_07_Features"
readers_writers.save_serialised_compressed(path=CONSTANTS.io_path, title=file_name, objects=features)
# print
print("File size: ", os.stat(os.path.join(CONSTANTS.io_path, file_name + ".bz2")).st_size)
print("Number of columns: ", len(features["train_indep"].columns))
print("features: {train: ", len(features["train_indep"]), ", test: ", len(features["test_indep"]), "}")
Load a Saved Samples and Features Ranking:
It is an optional step. The step loads the serialised & compressed outputs of Step-7.
In [ ]:
# open fetures
file_name = "Step_07_Features"
features = readers_writers.load_serialised_compressed(path=CONSTANTS.io_path, title=file_name)
# print
print("File size: ", os.stat(os.path.join(CONSTANTS.io_path, file_name + ".bz2")).st_size)
print("Number of columns: ", len(features["train_indep"].columns))
print("features: {train: ", len(features["train_indep"]), ", test: ", len(features["test_indep"]), "}")
In [ ]:
# open scoring model files
rank_models = ["rfc", "gbrt", "randLogit"]
model_rank = dict()
o_summaries_df = dict()
for rank_model in rank_models:
file_name = "Step_07_Model_Train_model_rank_" + rank_model
if not readers_writers.exists_serialised(path=CONSTANTS.io_path, title=file_name, ext="bz2"):
continue
file_name = "Step_07_Model_Train_model_rank_" + rank_model
model_rank[rank_model] = readers_writers.load_serialised_compressed(path=CONSTANTS.io_path, title=file_name)
file_name = "Step_07_Model_Train_model_rank_summaries_" + rank_model
o_summaries_df[rank_model] = readers_writers.load_serialised_compressed(path=CONSTANTS.io_path, title=file_name)
Verify features visually
In [ ]:
display(pd.concat([features["train_id"].head(), features["train_target"].head(), features["train_indep"].head()], axis=1))
display(pd.concat([features["test_id"].head(), features["test_target"].head(), features["test_indep"].head()], axis=1))
Configure: the trianing algorithm
Algorithm 1: Random Forest
In [ ]:
method_name = "rfc"
kwargs = {"n_estimators": 20, "criterion": 'gini', "max_depth": None, "min_samples_split": 100,
"min_samples_leaf": 50, "min_weight_fraction_leaf": 0.0, "max_features": 'auto',
"max_leaf_nodes": None, "bootstrap": True, "oob_score": False, "n_jobs": -1, "random_state": None,
"verbose": 0, "warm_start": False, "class_weight": "balanced_subsample"}
Algorithm 2: Logistic Regression
In [ ]:
method_name = "lr"
kwargs = {"penalty": 'l1', "dual": False, "tol": 0.0001, "C": 1, "fit_intercept": True, "intercept_scaling": 1,
"class_weight": None, "random_state": None, "solver": 'liblinear', "max_iter": 100, "multi_class": 'ovr',
"verbose": 0, "warm_start": False, "n_jobs": -1}
Algorithm 3: Logistic Cross-Validation
In [ ]:
method_name = "lr_cv"
kwargs = {"Cs": 10, "fit_intercept": True, "cv": None, "dual": False, "penalty": 'l2', "scoring": None,
"solver": 'lbfgs', "tol": 0.0001, "max_iter": 10, "class_weight": None, "n_jobs": -1, "verbose": 0,
"refit": True, "intercept_scaling": 1.0, "multi_class": "ovr", "random_state": None}
Algorithm 4: Neural Network
In [ ]:
method_name = "nn"
kwargs = {"solver": 'lbfgs', "alpha": 1e-5, "hidden_layer_sizes": (5, 2), "random_state": 1}
Algorithm 5: k-Nearest Neighbourhood
In [ ]:
method_name = "knc"
kwargs = {"n_neighbors": 5, "weights": 'distance', "algorithm": 'auto', "leaf_size": 30,
"p": 2, "metric": 'minkowski', "metric_params": None, "n_jobs": -1}
Algorithm 6: Decision Tree
In [ ]:
method_name = "dtc"
kwargs = {"criterion": 'gini', "splitter": 'best', "max_depth": None, "min_samples_split": 30,
"min_samples_leaf": 30, "min_weight_fraction_leaf": 0.0, "max_features": None,
"random_state": None, "max_leaf_nodes": None, "class_weight": None, "presort": False}
Algorithm 7: Gradient Boosting Classifier
In [ ]:
method_name = "gbc"
kwargs = {"loss": 'deviance', "learning_rate": 0.1, "n_estimators": 100, "subsample": 1.0, "min_samples_split": 30,
"min_samples_leaf": 30, "min_weight_fraction_leaf": 0.0, "max_depth": 3, "init": None, "random_state": None,
"max_features": None, "verbose": 0, "max_leaf_nodes": None, "warm_start": False, "presort": 'auto'}
Algorithm 8: Naive Bayes
Note: features must be positive
In [ ]:
method_name = "nb"
training_method = TrainingMethod(method_name)
kwargs = {"alpha": 1.0, "fit_prior": True, "class_prior": None}
Configure: other modelling settings
In [ ]:
# select the target variable
target_feature = "label365" # "label30" , "label365"
# file name
file_name = "Step_09_Model_" + method_name + "_" + target_feature
# initialise
training_method = TrainingMethod(method_name)
In [ ]:
sample_train = features["train_indep"][features_names_selected] # features["train_indep"][features_names_selected], features["train_indep"]
sample_train_target = features["train_target"][target_feature] # features["train_target"][target_feature]
sample_test = features["test_indep"][features_names_selected] # features["test_indep"][features_names_selected], features["test_indep"]
sample_test_target = features["test_target"][target_feature] # features["test_target"][target_feature]
Fit the model, using the train sample
In [ ]:
o_summaries = dict()
# Fit
model = training_method.train(sample_train, sample_train_target, **kwargs)
training_method.save_model(path=CONSTANTS.io_path, title=file_name)
In [ ]:
# load model
# training_method.load(path=CONSTANTS.io_path, title=file_name)
In [ ]:
# short summary
o_summaries = training_method.train_summaries()
Predict & report performance, using the train sample
In [ ]:
o_summaries = dict()
# predict
model = training_method.predict(sample_train, "train")
In [ ]:
# short summary
o_summaries = training_method.predict_summaries(pd.Series(sample_train_target), "train")
# Print the main performance statistics
for k in o_summaries.keys():
print(k, o_summaries[k])
# Print the by risk-bands of a selection of statistics
o_summaries = training_method.predict_summaries_risk_bands(pd.Series(sample_train_target), "train", np.arange(0, 1.05, 0.05))
display(o_summaries)
Predict & report performance, using the test sample
In [ ]:
o_summaries = dict()
# predict
model = training_method.predict(sample_test, "test")
In [ ]:
# short summary
o_summaries = training_method.predict_summaries(pd.Series(sample_test_target), "test")
# Print the main performance statistics
for k in o_summaries.keys():
print(k, o_summaries[k])
# Print the by risk-bands of a selection of statistics
o_summaries = training_method.predict_summaries_risk_bands(pd.Series(sample_test_target), "test", np.arange(0, 1.05, 0.05))
display(o_summaries)
Perform k-fold cross-validation
In [ ]:
o_summaries = dict()
score = training_method.cross_validate(sample_test, sample_test_target, scoring="neg_mean_squared_error", cv=10)
In [ ]:
# short summary
o_summaries = training_method.cross_validate_summaries()
print("Scores: ", o_summaries)
Save the training model.
In [ ]:
training_method.save_model(path=CONSTANTS.io_path, title=file_name)
Fin!