Initialise


In [20]:
# Reload modules 
# It is an optional step. It is useful to run when external Python modules are being modified
# It is reloading all modules (except those excluded by %aimport) every time before executing the Python code typed.
# Note: It may conflict with serialisation, when external modules are being modified

%load_ext autoreload 
%autoreload 2


The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload

In [21]:
# Import Python libraries
import logging
import os
import sys
import gc
import pandas as pd
from IPython.display import display, HTML
import numpy as np
import matplotlib.pyplot as plt

In [22]:
# Import local Python modules
from Configs.CONSTANTS import CONSTANTS
from Configs.Logger import Logger
from ReadersWriters.ReadersWriters import ReadersWriters
from Stats.Plots import Plots

In [23]:
# Check the interpreter
print("\nMake sure the correct Python interpreter is used!")
print(sys.version)
print("\nMake sure sys.path of the Python interpreter is correct!")
print(os.getcwd())


Make sure the correct Python interpreter is used!
3.5.3 (v3.5.3:1880cb95a742, Jan 16 2017, 16:02:32) [MSC v.1900 64 bit (AMD64)]

Make sure sys.path of the Python interpreter is correct!
C:\Users\eagle\Documents\GitHub\Analytics_UoW\TCARER

Set constants


In [24]:
config_path = os.path.abspath("ConfigInputs/CONFIGURATIONS.ini")
io_path = os.path.abspath("../../tmp/TCARER/scores_adhoc")
app_name = 'tcarerPlots'

print("\n The full path of the configuration file: \n\t", config_path,
      "\n The full path of the output folder: \n\t", io_path,
      "\n The application name (the suffix of the outputs file name): \n\t", app_name)


 The full path of the configuration file: 
	 C:\Users\eagle\Documents\GitHub\Analytics_UoW\TCARER\ConfigInputs\CONFIGURATIONS.ini 
 The full path of the output folder: 
	 C:\Users\eagle\Documents\GitHub\tmp\TCARER\scores_adhoc 
 The application name (the suffix of the outputs file name): 
	 tcarerPlots



In [25]:
if not os.path.exists(io_path):
    os.makedirs(io_path, exist_ok=True)

logger = Logger(path=io_path, app_name=app_name, ext="log")
logger = logging.getLogger(app_name)


2017-10-29 12:29:33,032 - tcarerPlots - INFO - Creating 'C:\Users\eagle\Documents\GitHub\tmp\TCARER\scores_adhoc\tcarerPlots.log' File.
2017-10-29 12:29:33,032 - tcarerPlots - INFO - Creating 'C:\Users\eagle\Documents\GitHub\tmp\TCARER\scores_adhoc\tcarerPlots.log' File.
2017-10-29 12:29:33,032 - tcarerPlots - INFO - Creating 'C:\Users\eagle\Documents\GitHub\tmp\TCARER\scores_adhoc\tcarerPlots.log' File.

In [26]:
# Initialise constants        
CONSTANTS.set(io_path, app_name)

In [27]:
# Initialise other classes
readers_writers = ReadersWriters()
plots = Plots()

In [35]:
samples = ["Basic_prototype"]
targets = ["365"]
submodels = ["rfc", "wdnn"]
paths = [os.path.abspath("../../tmp/TCARER/Basic_prototype")]
file_names = [["Step_09_Model_rfc_label365", "Step_09_Model_wdnn_label365"]]



Load Predictions


In [37]:
models = dict()
for i in range(len(samples)):
    models[samples[i]] = dict()
    for j in range(len(targets)):
        models[samples[i]][targets[j]] = dict()
        for k in range(len(submodels)):
            models[samples[i]][targets[j]][submodels[k]] = \
                readers_writers.load_serialised(path=paths[i], title=file_names[j][k])

hack - correction of dictionaries


In [40]:
for i in range(len(samples)):
    for j in range(len(targets)):
        models[samples[i]][targets[j]]["wdnn"] = {'model_predict': models[samples[i]][targets[j]]["wdnn"]}


---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
<ipython-input-40-a1f912f3713b> in <module>()
      1 for i in range(len(samples)):
      2     for j in range(len(targets)):
----> 3         models[samples[i]][targets[j]]["wdnn"] = {'model_predict': models[samples[i]][targets[j]]["wdnn"]}

KeyError: 'wdnn'

In [39]:
models["sample-1"]["30"]["wdnn"]['model_predict']['test']['score'] = np.array([i[1] for i in models["sample-1"]["30"]["wdnn"]['model_predict']['test']['predict_proba']])
models["sample-1"]["365"]["wdnn"]['model_predict']['test']['score'] = np.array([i[1] for i in models["sample-1"]["365"]["wdnn"]['model_predict']['test']['predict_proba']])
models["sample-2"]["30"]["wdnn"]['model_predict']['test']['score'] = np.array([i[1] for i in models["sample-2"]["30"]["wdnn"]['model_predict']['test']['predict_proba']])
models["sample-2"]["365"]["wdnn"]['model_predict']['test']['score'] = np.array([i[1] for i in models["sample-2"]["365"]["wdnn"]['model_predict']['test']['predict_proba']])


---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
<ipython-input-39-0533bcf87ed4> in <module>()
----> 1 models["sample-1"]["30"]["wdnn"]['model_predict']['test']['score'] = np.array([i[1] for i in models["sample-1"]["30"]["wdnn"]['model_predict']['test']['predict_proba']])
      2 models["sample-1"]["365"]["wdnn"]['model_predict']['test']['score'] = np.array([i[1] for i in models["sample-1"]["365"]["wdnn"]['model_predict']['test']['predict_proba']])
      3 models["sample-2"]["30"]["wdnn"]['model_predict']['test']['score'] = np.array([i[1] for i in models["sample-2"]["30"]["wdnn"]['model_predict']['test']['predict_proba']])
      4 models["sample-2"]["365"]["wdnn"]['model_predict']['test']['score'] = np.array([i[1] for i in models["sample-2"]["365"]["wdnn"]['model_predict']['test']['predict_proba']])

KeyError: 'sample-1'



temporary solution - Load samples


In [ ]:
file_names = ["Step_07_Features", "Step_07_Features"]

In [ ]:
target_labels = dict()
for i in range(len(samples)):
    target_labels[samples[i]] = readers_writers.load_serialised(path=paths[i], title=file_names[i])["test_target"]
    
for i in range(len(samples)):
    target_labels[samples[i]] = {"30": target_labels[samples[i]]["label30"], 
                                 "365": target_labels[samples[i]]["label365"]}
    target_labels["label30"] = None
    target_labels["label365"] = None

gc.collect()

temporary solution - correction of wdnn labels


In [ ]:
for sample in samples:
    for target in targets:
        print(len(target_labels[sample][target]), len(models[sample][target]["wdnn"]['model_predict']['test']['score']))
        target_labels[sample][target] = np.append(target_labels[sample][target], target_labels[sample][target])
        target_labels[sample][target] = target_labels[sample][target][0:len(models[sample][target]["wdnn"]['model_predict']['test']['score'])]




Save


In [ ]:
file_name = "saved_models"
readers_writers.save_serialised(path=io_path, title=file_name, objects=models)

file_name = "saved_target_labels"
readers_writers.save_serialised(path=io_path, title=file_name, objects=target_labels)

Load


In [ ]:
file_name = "saved_models"
models = readers_writers.load_serialised(path=io_path, title=file_name)

file_name = "saved_target_labels"
target_labels = readers_writers.load_serialised(path=io_path, title=file_name)



Plot


In [ ]:
file_name = "Combined_plots"

#%pylab inline
#pylab.rcParams['figure.figsize'] = (10, 10)

Combine


In [ ]:
markers = {'.': 'point', ',': 'pixel', 'o': 'circle', 'v': 'triangle_down', '^': 'triangle_up', '<': 'triangle_left', '>': 'triangle_right', '1': 'tri_down', '2': 'tri_up', '3': 'tri_left', '4': 'tri_right', '8': 'octagon', 's': 'square', 'p': 'pentagon', '*': 'star', 'h': 'hexagon1', 'H': 'hexagon2', '+': 'plus', 'x': 'x', 'D': 'diamond', 'd': 'thin_diamond', '|': 'vline', '_': 'hline', 'P': 'plus_filled', 'X': 'x_filled', 0: 'tickleft', 1: 'tickright', 2: 'tickup', 3: 'tickdown', 4: 'caretleft', 5: 'caretright', 6: 'caretup', 7: 'caretdown', 8: 'caretleftbase', 9: 'caretrightbase', 10: 'caretupbase', 11: 'caretdownbase', 'None': 'nothing', None: 'nothing', ' ': 'nothing', '': 'nothing'}
scores_list = []
target_labels_list = []
label_list = []
marker_list = ['.', 'o', 'v', '2', 'p', 's', '*', '+', '|'] #['.', 'o', 'v', '2', 'p', 's', '*', '+', '|']
linestyle_list = [':', ':', ':', ':', ':', ':', ':', ':', ':'] #[':', '-.', '--', '-', ':', '-.', '--', '-', ':'] 
color_list = ['grey', 'firebrick', 'coral', 'darkgreen', 'coral', 'mediumblue', 'lightseagreen', 'blueviolet', 'darkgray']

for sample in samples:
    for target in targets:
        for submodel in submodels:
            scores_list.append(models[sample][target][submodel]['model_predict']['test']['score'])
            target_labels_list.append(np.array(target_labels[sample][target][
                0:len(models[sample][target][submodel]['model_predict']['test']['score'])]))
            label_list.append(sample + ", " + submodel + "-" + target)

ROC Plot


In [ ]:
plt.clf()
fig, o_summaries = plots.roc_multiple(scores_list, target_labels_list, 
                                      label_list=label_list,  marker_list=marker_list, 
                                      linestyle_list=linestyle_list, color_list=color_list, 
                                      lw=2, markersize=8, legend_prop=15, legend_markerscale=1)

In [ ]:
display(fig)

In [ ]:
# save
fig.savefig(os.path.join(io_path, file_name + '_roc.png'), 
            dpi=1500, facecolor='w', edgecolor='w', orientation='portrait', papertype=None, format='png',
            transparent=False, bbox_inches=None, pad_inches=0.1, frameon=None)
plt.clf()

Precision-Recall Plot


In [ ]:
plt.clf()
fig, o_summaries = plots.precision_recall_multiple(scores_list, target_labels_list, 
                                                   label_list=label_list,  marker_list=marker_list, 
                                                   linestyle_list=linestyle_list, color_list=color_list,
                                                   lw=2, markersize=8, legend_prop=10, legend_markerscale=1)

In [ ]:
display(fig)

In [ ]:
# save
fig.savefig(os.path.join(io_path, file_name + '_precision_recall.png'), 
            dpi=1500, facecolor='w', edgecolor='w', orientation='portrait', papertype=None, format='png',
            transparent=False, bbox_inches=None, pad_inches=0.1, frameon=None)
plt.clf()

Fin!