Read Data Sample


In [1]:
import pandas as pd
import numpy as np
import os
from collections import namedtuple
pd.set_option("display.max_rows",100)
%matplotlib inline

%%bash rm dataset/scores/tf_dense_only_nsl_kdd_scores_all.pkl


In [2]:
class dataset:
    kdd_train_2labels = pd.read_pickle("dataset/kdd_train_2labels.pkl")
    kdd_test_2labels = pd.read_pickle("dataset/kdd_test_2labels.pkl")
    kdd_test__2labels = pd.read_pickle("dataset/kdd_test__2labels.pkl")

    kdd_train_5labels = pd.read_pickle("dataset/kdd_train_5labels.pkl")
    kdd_test_5labels = pd.read_pickle("dataset/kdd_test_5labels.pkl")

In [3]:
dataset.kdd_train_2labels.shape


Out[3]:
(125973, 124)

In [4]:
dataset.kdd_test_2labels.shape


Out[4]:
(22544, 124)

In [5]:
from sklearn import model_selection as ms
from sklearn import preprocessing as pp

class preprocess:
    
    output_columns_2labels = ['is_Normal','is_Attack']
    
    x_input = dataset.kdd_train_2labels.drop(output_columns_2labels, axis = 1)
    y_output = dataset.kdd_train_2labels.loc[:,output_columns_2labels]

    x_test_input = dataset.kdd_test_2labels.drop(output_columns_2labels, axis = 1)
    y_test = dataset.kdd_test_2labels.loc[:,output_columns_2labels]
    
    x_test__input = dataset.kdd_test__2labels.drop(output_columns_2labels, axis = 1)
    y_test_ = dataset.kdd_test__2labels.loc[:,output_columns_2labels]

    ss = pp.StandardScaler()

    x_train = ss.fit_transform(x_input)
    x_test = ss.transform(x_test_input)
    x_test_ = ss.transform(x_test__input)

    y_train = y_output.values
    y_test = y_test.values
    y_test_ = y_test_.values

preprocess.x_train.shape


Out[5]:
(125973, 122)

In [6]:
import tensorflow as tf

In [7]:
class network(object):
    
    input_dim = 122
    classes = 2
    hidden_encoder_dim = 122
    hidden_layers = 1
    latent_dim = 18

    def __init__(self, classes, hidden_layers, num_of_features):
        self.classes = classes
        self.hidden_layers = hidden_layers
        self.latent_dim = num_of_features
            
    def build_layers(self):
        tf.reset_default_graph()
        #learning_rate = tf.Variable(initial_value=0.001)

        input_dim = self.input_dim
        classes = self.classes
        hidden_encoder_dim = self.hidden_encoder_dim
        hidden_layers = self.hidden_layers
        latent_dim = self.latent_dim
        
        with tf.variable_scope("Input"):
            self.x = tf.placeholder("float", shape=[None, input_dim])
            self.y_ = tf.placeholder("float", shape=[None, classes])
            self.keep_prob = tf.placeholder("float")
            self.lr = tf.placeholder("float")
        
        with tf.variable_scope("Layer_Encoder"):

            hidden_encoder = tf.layers.dense(self.x, hidden_encoder_dim, activation = tf.nn.relu, kernel_regularizer=tf.nn.l2_loss)
            hidden_encoder = tf.nn.dropout(hidden_encoder, self.keep_prob)
            for h in range(hidden_layers - 1):
                hidden_encoder = tf.layers.dense(hidden_encoder, latent_dim, activation = tf.nn.relu, kernel_regularizer=tf.nn.l2_loss)
                hidden_encoder = tf.nn.dropout(hidden_encoder, self.keep_prob)
            
            #hidden_encoder = tf.layers.dense(self.x, latent_dim, activation = tf.nn.relu, kernel_regularizer=tf.nn.l2_loss)
            #hidden_encoder = tf.nn.dropout(hidden_encoder, self.keep_prob)
            
        with tf.variable_scope("Layer_Dense_Softmax"):
            self.y = tf.layers.dense(hidden_encoder, classes, activation=tf.nn.softmax)
            
        with tf.variable_scope("Loss"):
            
            loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels = self.y_, logits = self.y))

            #loss = tf.clip_by_value(loss, -1e-1, 1e-1)
            #loss = tf.where(tf.is_nan(loss), 1e-1, loss)
            #loss = tf.where(tf.equal(loss, -1e-1), tf.random_normal(loss.shape), loss)
            #loss = tf.where(tf.equal(loss, 1e-1), tf.random_normal(loss.shape), loss)
            
            self.regularized_loss = loss
            correct_prediction = tf.equal(tf.argmax(self.y_, 1), tf.argmax(self.y, 1))
            self.tf_accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name = "Accuracy")

        with tf.variable_scope("Optimizer"):
            learning_rate=self.lr
            optimizer = tf.train.AdamOptimizer(learning_rate)
            gradients, variables = zip(*optimizer.compute_gradients(self.regularized_loss))
            gradients = [
                None if gradient is None else tf.clip_by_value(gradient, -1, 1)
                for gradient in gradients]
            self.train_op = optimizer.apply_gradients(zip(gradients, variables))
            #self.train_op = optimizer.minimize(self.regularized_loss)
            
        # add op for merging summary
        #self.summary_op = tf.summary.merge_all()
        self.pred = tf.argmax(self.y, axis = 1)
        self.actual = tf.argmax(self.y_, axis = 1)

        # add Saver ops
        self.saver = tf.train.Saver()

In [8]:
import collections
import time
import sklearn.metrics as me 

class Train:    
    
    result = namedtuple("score", ['epoch', 'no_of_features','hidden_layers','train_score', 'test_score', 'f1_score', 'test_score_20', 'f1_score_20', 'time_taken'])

    predictions = {}
    predictions_ = {}

    results = []
    
    best_acc = 0
    best_acc_global = 0
    
    def train(epochs, net, h,f, lrs):
        batch_iterations = 200
        train_loss = None
        Train.best_acc = 0
        os.makedirs("dataset/tf_dense_only_nsl_kdd/hidden layers_{}_features count_{}".format(epochs,h,f),
                    exist_ok = True)
        with tf.Session() as sess:
            #summary_writer_train = tf.summary.FileWriter('./logs/kdd/VAE/training', graph=sess.graph)
            #summary_writer_valid = tf.summary.FileWriter('./logs/kdd/VAE/validation')

            sess.run(tf.global_variables_initializer())
            start_time = time.perf_counter()
            for c, lr in enumerate(lrs):
                for epoch in range(1, (epochs+1)):
                    x_train, x_valid, y_train, y_valid, = ms.train_test_split(preprocess.x_train, 
                                                                              preprocess.y_train, 
                                                                              test_size=0.1)
                    batch_indices = np.array_split(np.arange(x_train.shape[0]), 
                                               batch_iterations)

                    for i in batch_indices:

                        def train_batch():
                            nonlocal train_loss
                            _, train_loss = sess.run([net.train_op, 
                                                               net.regularized_loss, 
                                                               ], #net.summary_op
                                                              feed_dict={net.x: x_train[i,:], 
                                                                         net.y_: y_train[i,:], 
                                                                         net.keep_prob:0.5, net.lr:lr})

                        train_batch()
                        #summary_writer_train.add_summary(summary_str, epoch)
                        while((train_loss > 1e4 or np.isnan(train_loss)) and epoch > 1):
                            print("Step {} | Training Loss: {:.6f}".format(epoch, train_loss))
                            net.saver.restore(sess, 
                                              tf.train.latest_checkpoint('dataset/tf_dense_only_nsl_kdd/hidden_layers_{}_features_count_{}'
                                                                         .format(epochs,h,f)))
                            train_batch()


                    valid_accuracy = sess.run(net.tf_accuracy, #net.summary_op 
                                                          feed_dict={net.x: x_valid, 
                                                                     net.y_: y_valid, 
                                                                     net.keep_prob:1, net.lr:lr})
                    #summary_writer_valid.add_summary(summary_str, epoch)


                    accuracy, pred_value, actual_value, y_pred = sess.run([net.tf_accuracy, 
                                                                   net.pred, 
                                                                   net.actual, net.y], 
                                                                  feed_dict={net.x: preprocess.x_test, 
                                                                             net.y_: preprocess.y_test, 
                                                                             net.keep_prob:1, net.lr:lr})
                    f1_score = me.f1_score(actual_value, pred_value)
                    accuracy_, pred_value_, actual_value_, y_pred_ = sess.run([net.tf_accuracy, 
                                                                   net.pred, 
                                                                   net.actual, net.y], 
                                                                  feed_dict={net.x: preprocess.x_test_, 
                                                                             net.y_: preprocess.y_test_, 
                                                                             net.keep_prob:1, net.lr:lr})
                    f1_score_ = me.f1_score(actual_value_, pred_value_)
                    
                    print("Step {} | Training Loss: {:.6f} | Validation Accuracy: {:.6f}".format(epoch, train_loss, valid_accuracy))
                    print("Accuracy on Test data: {}, {}".format(accuracy, accuracy_))

                    if accuracy > Train.best_acc_global:
                        Train.best_acc_global = accuracy
                        Train.pred_value = pred_value
                        Train.actual_value = actual_value
                        Train.pred_value_ = pred_value_
                        Train.actual_value_ = actual_value_
                        Train.best_parameters = "Hidden Layers:{}, Features Count:{}".format(h, f)

                    if accuracy > Train.best_acc:
                        Train.best_acc = accuracy

                        if not (np.isnan(train_loss)):
                            net.saver.save(sess, 
                                       "dataset/tf_dense_only_nsl_kdd/hidden_layers_{}_features_count_{}".format(h,f),
                                        global_step = epochs)
                        curr_pred = pd.DataFrame({"Attack_prob":y_pred[:,-2], "Normal_prob":y_pred[:, -1], "Prediction":pred_value, "Actual":actual_value})
                        curr_pred_ = pd.DataFrame({"Attack_prob":y_pred_[:,-2], "Normal_prob":y_pred_[:, -1], "Prediction":pred_value_, "Actual": actual_value_})
                        
                        Train.predictions.update({"{}_{}_{}".format((epoch+1)*(c+1),f,h):(curr_pred, 
                                                   Train.result((epoch+1)*(c+1), f, h, valid_accuracy, accuracy, f1_score, accuracy_, f1_score_, time.perf_counter() - start_time))})
                        Train.predictions_.update({"{}_{}_{}".format((epoch+1)*(c+1),f,h):(curr_pred_, 
                                                   Train.result((epoch+1)*(c+1), f, h, valid_accuracy, accuracy, f1_score, accuracy_, f1_score_, time.perf_counter() - start_time))})

                        #Train.results.append(Train.result(epochs, f, h,valid_accuracy, accuracy))

In [9]:
import itertools

df_results = []
past_scores = []

class Hyperparameters:
#    features_arr = [2, 4, 8, 16, 32, 64, 128, 256]
#    hidden_layers_arr = [2, 4, 6, 10]

    def start_training():
        print("********************************** Training ******************************")

        global df_results
        global past_scores
        Train.predictions = {}
        Train.predictions_ = {}

        Train.results = []
    
        
        features_arr = [1, 12, 24, 48, 122]
        hidden_layers_arr = [1, 3]

        epochs = [5]
        lrs = [1e-5, 1e-6]
        print("********************************** Entering Loop ******************************")

        for e, h, f in itertools.product(epochs, hidden_layers_arr, features_arr):
            print("Current Layer Attributes - epochs:{} hidden layers:{} features count:{}".format(e,h,f))
            n = network(2,h,f)
            n.build_layers()
            Train.train(e, n, h,f, lrs)
            
        dict1 = {}
        dict1_ = {}
        dict2 = []

        for k, (v1, v2) in Train.predictions.items():
            dict1.update({k: v1})
            dict2.append(v2)

        for k, (v1_, v2) in Train.predictions_.items():
            dict1_.update({k: v1_})

        Train.predictions = dict1
        Train.predictions_ = dict1_
        
        Train.results = dict2
        df_results = pd.DataFrame(Train.results)

        #temp = df_results.set_index(['no_of_features', 'hidden_layers'])

        if not os.path.isfile('dataset/scores/tf_dense_only_nsl_kdd_scores_all.pkl'):
            past_scores = df_results
        else:
            past_scores = pd.read_pickle("dataset/scores/tf_dense_only_nsl_kdd_scores_all.pkl")
            past_scores = past_scores.append(df_results, ignore_index=True)
        past_scores.to_pickle("dataset/scores/tf_dense_only_nsl_kdd_scores_all.pkl")

In [10]:
#%%timeit -r 10
#capture
Hyperparameters.start_training()


********************************** Training ******************************
********************************** Entering Loop ******************************
Current Layer Attributes - epochs:5 hidden layers:1 features count:1
Step 1 | Training Loss: 0.704092 | Validation Accuracy: 0.645896
Accuracy on Test data: 0.6112934947013855, 0.33789029717445374
Step 2 | Training Loss: 0.640290 | Validation Accuracy: 0.774488
Accuracy on Test data: 0.690693736076355, 0.4293670952320099
Step 3 | Training Loss: 0.607380 | Validation Accuracy: 0.851961
Accuracy on Test data: 0.7224538922309875, 0.48219409584999084
Step 4 | Training Loss: 0.557025 | Validation Accuracy: 0.880060
Accuracy on Test data: 0.7379790544509888, 0.5088607668876648
Step 5 | Training Loss: 0.513937 | Validation Accuracy: 0.913081
Accuracy on Test data: 0.7531493902206421, 0.5367088317871094
Step 1 | Training Loss: 0.517118 | Validation Accuracy: 0.920384
Accuracy on Test data: 0.7543914318084717, 0.5390717387199402
Step 2 | Training Loss: 0.510755 | Validation Accuracy: 0.925067
Accuracy on Test data: 0.7553229331970215, 0.5401687622070312
Step 3 | Training Loss: 0.513694 | Validation Accuracy: 0.929513
Accuracy on Test data: 0.756609320640564, 0.5422784686088562
Step 4 | Training Loss: 0.510859 | Validation Accuracy: 0.927846
Accuracy on Test data: 0.7574964761734009, 0.5438818335533142
Step 5 | Training Loss: 0.502903 | Validation Accuracy: 0.933561
Accuracy on Test data: 0.7586497664451599, 0.5459915399551392
Current Layer Attributes - epochs:5 hidden layers:1 features count:12
Step 1 | Training Loss: 0.714985 | Validation Accuracy: 0.520003
Accuracy on Test data: 0.5994943380355835, 0.645232081413269
Step 2 | Training Loss: 0.660893 | Validation Accuracy: 0.698365
Accuracy on Test data: 0.7581618428230286, 0.65139240026474
Step 3 | Training Loss: 0.607649 | Validation Accuracy: 0.789729
Accuracy on Test data: 0.8019428849220276, 0.6635442972183228
Step 4 | Training Loss: 0.579113 | Validation Accuracy: 0.862121
Accuracy on Test data: 0.819109320640564, 0.6725738644599915
Step 5 | Training Loss: 0.532859 | Validation Accuracy: 0.881965
Accuracy on Test data: 0.8294003009796143, 0.6893671154975891
Step 1 | Training Loss: 0.537820 | Validation Accuracy: 0.885537
Accuracy on Test data: 0.8301987051963806, 0.6908860802650452
Step 2 | Training Loss: 0.527341 | Validation Accuracy: 0.886411
Accuracy on Test data: 0.8308640718460083, 0.6921519041061401
Step 3 | Training Loss: 0.515487 | Validation Accuracy: 0.888236
Accuracy on Test data: 0.8317512273788452, 0.6937552690505981
Step 4 | Training Loss: 0.518110 | Validation Accuracy: 0.896174
Accuracy on Test data: 0.8324165940284729, 0.6949366927146912
Step 5 | Training Loss: 0.511986 | Validation Accuracy: 0.897841
Accuracy on Test data: 0.8333480954170227, 0.6966244578361511
Current Layer Attributes - epochs:5 hidden layers:1 features count:24
Step 1 | Training Loss: 0.691997 | Validation Accuracy: 0.625655
Accuracy on Test data: 0.7745741605758667, 0.706244707107544
Step 2 | Training Loss: 0.638065 | Validation Accuracy: 0.740753
Accuracy on Test data: 0.8202182650566101, 0.7161181569099426
Step 3 | Training Loss: 0.590750 | Validation Accuracy: 0.856565
Accuracy on Test data: 0.8501153588294983, 0.7293670773506165
Step 4 | Training Loss: 0.555348 | Validation Accuracy: 0.892046
Accuracy on Test data: 0.85801100730896, 0.7402531504631042
Step 5 | Training Loss: 0.521922 | Validation Accuracy: 0.901810
Accuracy on Test data: 0.8670599460601807, 0.7554430365562439
Step 1 | Training Loss: 0.521976 | Validation Accuracy: 0.907049
Accuracy on Test data: 0.867237389087677, 0.755696177482605
Step 2 | Training Loss: 0.534170 | Validation Accuracy: 0.912208
Accuracy on Test data: 0.866084098815918, 0.7535021305084229
Step 3 | Training Loss: 0.515956 | Validation Accuracy: 0.914590
Accuracy on Test data: 0.8597853183746338, 0.7413502335548401
Step 4 | Training Loss: 0.529092 | Validation Accuracy: 0.915860
Accuracy on Test data: 0.8555269837379456, 0.7332489490509033
Step 5 | Training Loss: 0.512805 | Validation Accuracy: 0.919511
Accuracy on Test data: 0.851091206073761, 0.7248101234436035
Current Layer Attributes - epochs:5 hidden layers:1 features count:48
Step 1 | Training Loss: 0.629006 | Validation Accuracy: 0.754961
Accuracy on Test data: 0.7967973947525024, 0.7071729898452759
Step 2 | Training Loss: 0.592226 | Validation Accuracy: 0.848389
Accuracy on Test data: 0.8427075743675232, 0.7191561460494995
Step 3 | Training Loss: 0.548871 | Validation Accuracy: 0.890776
Accuracy on Test data: 0.8444375395774841, 0.7143459916114807
Step 4 | Training Loss: 0.507608 | Validation Accuracy: 0.909192
Accuracy on Test data: 0.8459900617599487, 0.7146835327148438
Step 5 | Training Loss: 0.494673 | Validation Accuracy: 0.913796
Accuracy on Test data: 0.8472321033477783, 0.7156118154525757
Step 1 | Training Loss: 0.471714 | Validation Accuracy: 0.915701
Accuracy on Test data: 0.8462561964988708, 0.7137552499771118
Step 2 | Training Loss: 0.485304 | Validation Accuracy: 0.915860
Accuracy on Test data: 0.8435060381889343, 0.7085232138633728
Step 3 | Training Loss: 0.469505 | Validation Accuracy: 0.920702
Accuracy on Test data: 0.843018114566803, 0.7075949311256409
Step 4 | Training Loss: 0.469571 | Validation Accuracy: 0.917288
Accuracy on Test data: 0.8431511521339417, 0.7077637314796448
Step 5 | Training Loss: 0.470555 | Validation Accuracy: 0.924909
Accuracy on Test data: 0.8429293632507324, 0.7069198489189148
Current Layer Attributes - epochs:5 hidden layers:1 features count:122
Step 1 | Training Loss: 0.675309 | Validation Accuracy: 0.629147
Accuracy on Test data: 0.6239354014396667, 0.6817721724510193
Step 2 | Training Loss: 0.612087 | Validation Accuracy: 0.753929
Accuracy on Test data: 0.8024308085441589, 0.7309704422950745
Step 3 | Training Loss: 0.593459 | Validation Accuracy: 0.829179
Accuracy on Test data: 0.8449254631996155, 0.7784810066223145
Step 4 | Training Loss: 0.545930 | Validation Accuracy: 0.901572
Accuracy on Test data: 0.8462561964988708, 0.7356961965560913
Step 5 | Training Loss: 0.515321 | Validation Accuracy: 0.919114
Accuracy on Test data: 0.8518452644348145, 0.7337552905082703
Step 1 | Training Loss: 0.499849 | Validation Accuracy: 0.916574
Accuracy on Test data: 0.8523775935173035, 0.7337552905082703
Step 2 | Training Loss: 0.503184 | Validation Accuracy: 0.914193
Accuracy on Test data: 0.8522888422012329, 0.7330801486968994
Step 3 | Training Loss: 0.490915 | Validation Accuracy: 0.925067
Accuracy on Test data: 0.8513573408126831, 0.7312236428260803
Step 4 | Training Loss: 0.481220 | Validation Accuracy: 0.925067
Accuracy on Test data: 0.8463449478149414, 0.7214345932006836
Step 5 | Training Loss: 0.497714 | Validation Accuracy: 0.921972
Accuracy on Test data: 0.8456795811653137, 0.7198312282562256
Current Layer Attributes - epochs:5 hidden layers:3 features count:1
/home/ritesh_malaiya/anaconda3/envs/p3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1113: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no predicted samples.
  'precision', 'predicted', average, warn_for)
Step 1 | Training Loss: 0.693097 | Validation Accuracy: 0.529925
Accuracy on Test data: 0.43075764179229736, 0.18160337209701538
Step 2 | Training Loss: 0.693101 | Validation Accuracy: 0.533021
Accuracy on Test data: 0.43075764179229736, 0.18160337209701538
Step 3 | Training Loss: 0.693125 | Validation Accuracy: 0.537625
Accuracy on Test data: 0.43075764179229736, 0.18160337209701538
Step 4 | Training Loss: 0.692802 | Validation Accuracy: 0.536276
Accuracy on Test data: 0.43075764179229736, 0.18160337209701538
Step 5 | Training Loss: 0.692893 | Validation Accuracy: 0.531513
Accuracy on Test data: 0.43075764179229736, 0.18160337209701538
Step 1 | Training Loss: 0.692826 | Validation Accuracy: 0.539609
Accuracy on Test data: 0.43075764179229736, 0.18160337209701538
Step 2 | Training Loss: 0.692912 | Validation Accuracy: 0.530402
Accuracy on Test data: 0.43075764179229736, 0.18160337209701538
Step 3 | Training Loss: 0.692909 | Validation Accuracy: 0.535482
Accuracy on Test data: 0.43075764179229736, 0.18160337209701538
Step 4 | Training Loss: 0.692509 | Validation Accuracy: 0.532783
Accuracy on Test data: 0.43075764179229736, 0.18160337209701538
Step 5 | Training Loss: 0.692771 | Validation Accuracy: 0.538260
Accuracy on Test data: 0.43075764179229736, 0.18160337209701538
Current Layer Attributes - epochs:5 hidden layers:3 features count:12
Step 1 | Training Loss: 0.747871 | Validation Accuracy: 0.302270
Accuracy on Test data: 0.49574166536331177, 0.753333330154419
Step 2 | Training Loss: 0.705273 | Validation Accuracy: 0.346563
Accuracy on Test data: 0.5076295137405396, 0.7574683427810669
Step 3 | Training Loss: 0.705840 | Validation Accuracy: 0.590252
Accuracy on Test data: 0.6894517540931702, 0.7579746842384338
Step 4 | Training Loss: 0.704902 | Validation Accuracy: 0.745753
Accuracy on Test data: 0.8082416653633118, 0.7280168533325195
Step 5 | Training Loss: 0.665448 | Validation Accuracy: 0.796237
Accuracy on Test data: 0.828025221824646, 0.7273417711257935
Step 1 | Training Loss: 0.662826 | Validation Accuracy: 0.794491
Accuracy on Test data: 0.8286905884742737, 0.7275949120521545
Step 2 | Training Loss: 0.678499 | Validation Accuracy: 0.797746
Accuracy on Test data: 0.8290454149246216, 0.7281012535095215
Step 3 | Training Loss: 0.667243 | Validation Accuracy: 0.801000
Accuracy on Test data: 0.8294003009796143, 0.7286075949668884
Step 4 | Training Loss: 0.652857 | Validation Accuracy: 0.797508
Accuracy on Test data: 0.8300212621688843, 0.7297890186309814
Step 5 | Training Loss: 0.681853 | Validation Accuracy: 0.801873
Accuracy on Test data: 0.8305092453956604, 0.7302109599113464
Current Layer Attributes - epochs:5 hidden layers:3 features count:24
Step 1 | Training Loss: 0.688986 | Validation Accuracy: 0.804969
Accuracy on Test data: 0.7739975452423096, 0.6429535746574402
Step 2 | Training Loss: 0.641489 | Validation Accuracy: 0.868471
Accuracy on Test data: 0.8045599460601807, 0.6561181545257568
Step 3 | Training Loss: 0.619636 | Validation Accuracy: 0.890776
Accuracy on Test data: 0.8147178888320923, 0.6610126495361328
Step 4 | Training Loss: 0.607743 | Validation Accuracy: 0.907287
Accuracy on Test data: 0.8205287456512451, 0.6669198274612427
Step 5 | Training Loss: 0.615180 | Validation Accuracy: 0.915066
Accuracy on Test data: 0.8207948803901672, 0.6650632619857788
Step 1 | Training Loss: 0.602186 | Validation Accuracy: 0.910462
Accuracy on Test data: 0.8219038248062134, 0.6671729683876038
Step 2 | Training Loss: 0.597917 | Validation Accuracy: 0.915860
Accuracy on Test data: 0.8218594789505005, 0.6670886278152466
Step 3 | Training Loss: 0.589698 | Validation Accuracy: 0.917050
Accuracy on Test data: 0.8219038248062134, 0.6670886278152466
Step 4 | Training Loss: 0.604934 | Validation Accuracy: 0.918955
Accuracy on Test data: 0.821726381778717, 0.6667510271072388
Step 5 | Training Loss: 0.590326 | Validation Accuracy: 0.922527
Accuracy on Test data: 0.8216376900672913, 0.6664978861808777
Current Layer Attributes - epochs:5 hidden layers:3 features count:48
Step 1 | Training Loss: 0.671034 | Validation Accuracy: 0.815288
Accuracy on Test data: 0.6449165940284729, 0.4879325032234192
Step 2 | Training Loss: 0.650770 | Validation Accuracy: 0.899428
Accuracy on Test data: 0.799237072467804, 0.6342616081237793
Step 3 | Training Loss: 0.627489 | Validation Accuracy: 0.926179
Accuracy on Test data: 0.8395581841468811, 0.7070042490959167
Step 4 | Training Loss: 0.571884 | Validation Accuracy: 0.925226
Accuracy on Test data: 0.8531316518783569, 0.7314767837524414
Step 5 | Training Loss: 0.572623 | Validation Accuracy: 0.922289
Accuracy on Test data: 0.8635557293891907, 0.751139223575592
Step 1 | Training Loss: 0.591467 | Validation Accuracy: 0.926973
Accuracy on Test data: 0.8635113835334778, 0.7510548233985901
Step 2 | Training Loss: 0.535877 | Validation Accuracy: 0.926258
Accuracy on Test data: 0.8661284446716309, 0.7560337781906128
Step 3 | Training Loss: 0.565682 | Validation Accuracy: 0.926258
Accuracy on Test data: 0.8664389848709106, 0.756540060043335
Step 4 | Training Loss: 0.545241 | Validation Accuracy: 0.923797
Accuracy on Test data: 0.8666163682937622, 0.7568776607513428
Step 5 | Training Loss: 0.536672 | Validation Accuracy: 0.924750
Accuracy on Test data: 0.8669712543487549, 0.7575527429580688
Current Layer Attributes - epochs:5 hidden layers:3 features count:122
Step 1 | Training Loss: 0.674893 | Validation Accuracy: 0.746150
Accuracy on Test data: 0.8286018371582031, 0.7761181592941284
Step 2 | Training Loss: 0.617569 | Validation Accuracy: 0.860137
Accuracy on Test data: 0.8515791296958923, 0.7477636933326721
Step 3 | Training Loss: 0.579240 | Validation Accuracy: 0.899032
Accuracy on Test data: 0.8380944132804871, 0.7081012725830078
Step 4 | Training Loss: 0.567790 | Validation Accuracy: 0.907366
Accuracy on Test data: 0.8240330219268799, 0.6759493947029114
Step 5 | Training Loss: 0.521153 | Validation Accuracy: 0.915304
Accuracy on Test data: 0.8080198764801025, 0.6442193984985352
Step 1 | Training Loss: 0.494850 | Validation Accuracy: 0.913796
Accuracy on Test data: 0.8050479292869568, 0.6383122205734253
Step 2 | Training Loss: 0.513884 | Validation Accuracy: 0.914193
Accuracy on Test data: 0.8018985390663147, 0.6323207020759583
Step 3 | Training Loss: 0.502671 | Validation Accuracy: 0.911732
Accuracy on Test data: 0.7997693419456482, 0.6282700300216675
Step 4 | Training Loss: 0.508571 | Validation Accuracy: 0.918797
Accuracy on Test data: 0.7975514531135559, 0.6240506172180176
Step 5 | Training Loss: 0.499664 | Validation Accuracy: 0.910541
Accuracy on Test data: 0.7956440448760986, 0.6202531456947327

In [ ]:


In [11]:
#g = df_results.groupby(by=['no_of_features'])
#idx = g['test_score'].transform(max) == df_results['test_score']
#df_results[idx].sort_values(by = 'test_score', ascending = False)

In [12]:
#g = df_results.groupby(by=['no_of_features'])
#idx = g['test_score_20'].transform(max) == df_results['test_score_20']
#df_results[idx].sort_values(by = 'test_score_20', ascending = False)

In [13]:
#df_results.sort_values(by = 'test_score', ascending = False)

In [14]:
#Train.predictions_

In [15]:
pd.Panel(Train.predictions).to_pickle("dataset/tf_dense_only_nsl_kdd_predictions.pkl")
pd.Panel(Train.predictions_).to_pickle("dataset/tf_dense_only_nsl_kdd_predictions__.pkl")

df_results.to_pickle("dataset/tf_dense_only_nsl_kdd_scores.pkl")

In [ ]:


In [16]:
import numpy as np
import matplotlib.pyplot as plt
import itertools

def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    np.set_printoptions(precision=4)

    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        #print("Normalized confusion matrix")
    else:
        #print('Confusion matrix, without normalization')
        pass
    
    #print(cm)

    label = [["\n True Negative", "\n False Positive \n Type II Error"],
             ["\n False Negative \n Type I Error", "\n True Positive"]
            ]
    
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        
        plt.text(j, i, "{} {}".format(cm[i, j].round(4), label[i][j]),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

def plot(actual_value, pred_value):
    from sklearn.metrics import confusion_matrix

    cm_2labels = confusion_matrix(y_pred = pred_value, y_true = actual_value)
    plt.figure(figsize=[6,6])
    plot_confusion_matrix(cm_2labels, ['Normal', 'Attack'], normalize = False)

In [17]:
#plot(actual_value = Train.actual_value, pred_value = Train.pred_value)

In [18]:
#plot(actual_value = Train.actual_value_, pred_value = Train.pred_value_)

In [19]:
past_scores = pd.read_pickle("dataset/scores/tf_dense_only_nsl_kdd_scores_all.pkl")

In [20]:
past_scores.sort_values(by='f1_score',ascending=False)


Out[20]:
epoch f1_score f1_score_20 hidden_layers no_of_features test_score test_score_20 time_taken train_score
1110 12 0.873902 0.834989 3.0 48.0 0.866971 0.757553 18.979939 0.924750
1109 10 0.873501 0.834416 3.0 48.0 0.866616 0.756878 17.183680 0.923797
1108 8 0.873269 0.834071 3.0 48.0 0.866439 0.756540 15.182953 0.926258
1107 6 0.872948 0.833669 3.0 48.0 0.866128 0.756034 13.206024 0.926258
1076 4 0.872752 0.831559 1.0 24.0 0.867237 0.755696 6.615212 0.907049
1078 6 0.872539 0.831315 1.0 24.0 0.867060 0.755443 5.527226 0.901810
1112 3 0.868402 0.841002 3.0 122.0 0.851579 0.747764 4.354387 0.860137
1086 4 0.862763 0.823989 1.0 122.0 0.852378 0.733755 7.787664 0.916574
1077 5 0.862754 0.818941 1.0 24.0 0.858011 0.740253 4.420664 0.892046
1088 6 0.862359 0.824165 1.0 122.0 0.851845 0.733755 6.520284 0.919114
1111 2 0.861654 0.870263 3.0 122.0 0.828602 0.776118 2.195020 0.746150
1087 5 0.858542 0.827095 1.0 122.0 0.846256 0.735696 5.203199 0.901572
1106 5 0.858122 0.812448 3.0 48.0 0.853132 0.731477 7.784258 0.925226
1097 12 0.857121 0.839242 3.0 12.0 0.830509 0.730211 16.995091 0.801873
1096 10 0.856716 0.838982 3.0 12.0 0.830021 0.729789 15.335974 0.797508
1095 8 0.856138 0.838197 3.0 12.0 0.829400 0.728608 13.734848 0.801000
1094 6 0.855882 0.837944 3.0 12.0 0.829045 0.728101 12.027643 0.797746
1092 4 0.855637 0.837724 3.0 12.0 0.828691 0.727595 10.303682 0.794491
1083 6 0.854733 0.806544 1.0 48.0 0.847232 0.715612 5.875551 0.913796
1082 5 0.853477 0.805835 1.0 48.0 0.845990 0.714684 4.594229 0.909192
1080 3 0.852312 0.811743 1.0 48.0 0.842708 0.719156 2.295216 0.848389
1081 4 0.852032 0.805783 1.0 48.0 0.844438 0.714346 3.419154 0.890776
1105 4 0.842595 0.791095 3.0 48.0 0.839558 0.707004 5.737464 0.926179
1093 5 0.841468 0.838503 3.0 12.0 0.808242 0.728017 6.960655 0.745753
1073 12 0.837183 0.785078 1.0 12.0 0.833348 0.696624 10.911904 0.897841
1072 10 0.836167 0.783675 1.0 12.0 0.832417 0.694937 9.859136 0.896174
1071 8 0.835452 0.782708 1.0 12.0 0.831751 0.693755 8.743644 0.888236
1085 3 0.835378 0.838909 1.0 122.0 0.802431 0.730970 2.609813 0.753929
1070 6 0.834512 0.781426 1.0 12.0 0.830864 0.692152 7.700856 0.886411
1068 4 0.833782 0.780383 1.0 12.0 0.830199 0.690886 6.587520 0.885537
1100 4 0.826648 0.765712 3.0 24.0 0.821904 0.667173 10.274888 0.910462
1102 6 0.825365 0.763848 3.0 24.0 0.820795 0.665063 8.518722 0.915066
1101 5 0.825287 0.765437 3.0 24.0 0.820529 0.666920 6.885301 0.907287
1069 5 0.822696 0.766715 1.0 12.0 0.819109 0.672574 4.404348 0.862121
1075 3 0.822057 0.800522 1.0 24.0 0.820218 0.716118 2.220499 0.740753
1079 2 0.820515 0.810775 1.0 48.0 0.796797 0.707173 1.140302 0.754961
1099 3 0.811693 0.758604 3.0 24.0 0.804560 0.656118 3.421427 0.868471
1104 3 0.793352 0.722357 3.0 48.0 0.799237 0.634262 3.780687 0.899428
1098 2 0.788633 0.751920 3.0 24.0 0.773998 0.642954 1.715472 0.804969
1074 2 0.781137 0.799885 1.0 24.0 0.774574 0.706245 1.152475 0.625655
1067 3 0.755472 0.750769 1.0 12.0 0.758162 0.651392 2.215889 0.698365
1065 12 0.749205 0.650831 1.0 1.0 0.758650 0.545992 10.894768 0.933561
1064 10 0.747704 0.648638 1.0 1.0 0.757496 0.543882 9.778554 0.927846
1063 8 0.746547 0.646967 1.0 1.0 0.756609 0.542278 8.735577 0.929513
1062 6 0.744889 0.644808 1.0 1.0 0.755323 0.540169 7.690903 0.925067
1060 4 0.743693 0.643752 1.0 1.0 0.754391 0.539072 6.562054 0.920384
1061 5 0.721670 0.611326 1.0 1.0 0.737979 0.508861 4.381407 0.880060
1084 2 0.680172 0.806922 1.0 122.0 0.623935 0.681772 1.334918 0.629147
1091 3 0.670975 0.861293 3.0 12.0 0.507630 0.757468 3.625248 0.346563
1059 3 0.664485 0.535194 1.0 1.0 0.690694 0.429367 2.229519 0.774488
... ... ... ... ... ... ... ... ... ...
1008 2 NaN NaN NaN NaN 0.752528 0.571561 1.565016 0.789332
1009 3 NaN NaN NaN NaN 0.786329 0.604641 3.070189 0.823940
1010 4 NaN NaN NaN NaN 0.797773 0.623797 4.573471 0.854342
1011 5 NaN NaN NaN NaN 0.811036 0.647426 6.084401 0.872678
1012 6 NaN NaN NaN NaN 0.818045 0.658903 7.585594 0.891729
1013 7 NaN NaN NaN NaN 0.822968 0.668017 9.087819 0.897127
1014 2 NaN NaN NaN NaN 0.784732 0.657468 2.059196 0.781711
1015 3 NaN NaN NaN NaN 0.798527 0.645232 4.067227 0.853469
1016 4 NaN NaN NaN NaN 0.809617 0.656962 6.087417 0.905461
1017 5 NaN NaN NaN NaN 0.812012 0.654852 8.131453 0.922369
1018 6 NaN NaN NaN NaN 0.812766 0.652996 10.171448 0.939832
1019 7 NaN NaN NaN NaN 0.814452 0.654008 12.186608 0.944436
1020 2 NaN NaN NaN NaN 0.430758 0.181603 1.472629 0.539054
1021 2 NaN NaN NaN NaN 0.430758 0.181603 1.650742 0.533736
1022 9 NaN NaN NaN NaN 0.661817 0.377722 32.087380 0.857358
1023 10 NaN NaN NaN NaN 0.495963 0.224135 19.576402 0.709001
1024 11 NaN NaN NaN NaN 0.471079 0.183966 13.169223 0.641372
1025 4 NaN NaN NaN NaN 0.476269 0.192152 14.774089 0.655818
1026 6 NaN NaN NaN NaN 0.659466 0.373502 30.513274 0.856088
1027 8 NaN NaN NaN NaN 0.490463 0.215105 17.982180 0.693443
1028 12 NaN NaN NaN NaN 0.665410 0.383713 33.647811 0.860057
1029 14 NaN NaN NaN NaN 0.587473 0.242110 22.724931 0.752104
1030 16 NaN NaN NaN NaN 0.596744 0.258312 24.286118 0.783299
1031 18 NaN NaN NaN NaN 0.668027 0.387511 36.816779 0.864423
1032 20 NaN NaN NaN NaN 0.629791 0.318481 27.448016 0.832910
1033 22 NaN NaN NaN NaN 0.656361 0.367679 28.963678 0.859422
1034 15 NaN NaN NaN NaN 0.666918 0.385907 35.223553 0.865534
1035 21 NaN NaN NaN NaN 0.669003 0.389114 38.440099 0.866010
1036 24 NaN NaN NaN NaN 0.669846 0.390464 40.048072 0.866090
1037 27 NaN NaN NaN NaN 0.670866 0.392152 41.646417 0.866884
1038 30 NaN NaN NaN NaN 0.671753 0.393840 43.278384 0.871249
1039 33 NaN NaN NaN NaN 0.673039 0.396034 44.906827 0.865296
1040 2 NaN NaN NaN NaN 0.513529 0.532152 1.949590 0.510398
1041 3 NaN NaN NaN NaN 0.645626 0.551392 3.848985 0.652246
1042 4 NaN NaN NaN NaN 0.868879 0.761688 20.928807 0.909589
1043 5 NaN NaN NaN NaN 0.786018 0.629958 7.647516 0.830846
1044 6 NaN NaN NaN NaN 0.869721 0.763122 22.827152 0.920464
1045 7 NaN NaN NaN NaN 0.819198 0.671983 11.425825 0.876806
1046 8 NaN NaN NaN NaN 0.827227 0.682025 13.333037 0.878949
1047 9 NaN NaN NaN NaN 0.836719 0.699578 15.241036 0.888792
1048 10 NaN NaN NaN NaN 0.869899 0.763207 26.090022 0.922289
1049 11 NaN NaN NaN NaN 0.867770 0.759241 19.046752 0.902524
1050 2 NaN NaN NaN NaN 0.610273 0.458819 3.025803 0.795126
1051 3 NaN NaN NaN NaN 0.801544 0.634768 5.939004 0.913875
1052 4 NaN NaN NaN NaN 0.818754 0.663460 8.846303 0.930862
1053 5 NaN NaN NaN NaN 0.826872 0.676456 11.801530 0.935069
1054 6 NaN NaN NaN NaN 0.833171 0.688101 14.741884 0.941737
1055 7 NaN NaN NaN NaN 0.847276 0.714262 17.673385 0.942531
1056 8 NaN NaN NaN NaN 0.853220 0.725148 20.603261 0.945388
1057 9 NaN NaN NaN NaN 0.858410 0.734599 23.537852 0.946896

1113 rows × 9 columns


In [21]:
psg = past_scores.sort_values(by='test_score', ascending=False).groupby(by=['no_of_features', 'hidden_layers'])
psg.first().sort_values(by='test_score', ascending=False)


Out[21]:
epoch f1_score f1_score_20 test_score test_score_20 time_taken train_score
no_of_features hidden_layers
24.0 1.0 4 0.872752 0.831559 0.867237 0.755696 6.615212 0.907049
48.0 3.0 12 0.873902 0.834989 0.866971 0.757553 18.979939 0.924750
122.0 1.0 4 0.862763 0.823989 0.852378 0.733755 7.787664 0.916574
3.0 3 0.868402 0.841002 0.851579 0.747764 4.354387 0.860137
48.0 1.0 6 0.854733 0.806544 0.847232 0.715612 5.875551 0.913796
12.0 1.0 12 0.837183 0.785078 0.833348 0.696624 10.911904 0.897841
3.0 12 0.857121 0.839242 0.830509 0.730211 16.995091 0.801873
24.0 3.0 4 0.826648 0.765712 0.821904 0.667173 10.274888 0.910462
1.0 1.0 12 0.749205 0.650831 0.758650 0.545992 10.894768 0.933561
3.0 2 0.000000 0.000000 0.430758 0.181603 1.590742 0.529925

In [22]:
psg.mean().sort_values(by='test_score', ascending=False)


Out[22]:
epoch f1_score f1_score_20 test_score test_score_20 time_taken train_score
no_of_features hidden_layers
122.0 3.0 2.50 0.865028 0.855632 0.840090 0.761941 3.274703 0.803143
24.0 1.0 4.00 0.842248 0.816444 0.837420 0.734751 3.987215 0.813462
48.0 1.0 4.00 0.846614 0.808136 0.835433 0.714194 3.464890 0.863423
3.0 6.25 0.818815 0.777829 0.825375 0.698460 10.472351 0.908398
24.0 3.0 4.00 0.815525 0.761104 0.808357 0.659646 6.163162 0.881251
122.0 1.0 4.00 0.819843 0.824216 0.795369 0.723190 4.691176 0.824067
12.0 1.0 6.25 0.800505 0.772256 0.791918 0.679694 6.445614 0.816836
3.0 6.25 0.806903 0.843851 0.744910 0.735390 10.099620 0.673401
1.0 1.0 6.25 0.709750 0.603261 0.727805 0.498439 6.430936 0.867102
3.0 2.00 0.000000 0.000000 0.430758 0.181603 1.590742 0.529925

In [23]:
Train.predictions = pd.read_pickle("dataset/tf_dense_only_nsl_kdd_predictions.pkl")
Train.predictions_ = pd.read_pickle("dataset/tf_dense_only_nsl_kdd_predictions__.pkl")

In [24]:
#epoch_nof_hidden
Train.predictions["12_12_3"]


Out[24]:
Actual Attack_prob Normal_prob Prediction
0 1.0 0.290404 0.709596 1.0
1 1.0 0.290311 0.709689 1.0
2 0.0 0.441519 0.558481 1.0
3 1.0 0.204695 0.795305 1.0
4 1.0 0.359508 0.640492 1.0
5 0.0 0.666080 0.333920 0.0
6 0.0 0.522911 0.477089 0.0
7 1.0 0.135548 0.864452 1.0
8 0.0 0.662445 0.337555 0.0
9 1.0 0.586753 0.413247 0.0
10 1.0 0.398439 0.601561 1.0
11 0.0 0.518839 0.481161 0.0
12 1.0 0.285653 0.714347 1.0
13 1.0 0.389742 0.610258 1.0
14 0.0 0.397625 0.602374 1.0
15 0.0 0.678045 0.321955 0.0
16 0.0 0.658028 0.341972 0.0
17 0.0 0.675036 0.324964 0.0
18 0.0 0.334857 0.665143 1.0
19 1.0 0.288760 0.711240 1.0
20 1.0 0.257760 0.742240 1.0
21 1.0 0.324181 0.675819 1.0
22 0.0 0.655833 0.344167 0.0
23 0.0 0.670421 0.329579 0.0
24 1.0 0.138075 0.861925 1.0
25 1.0 0.318148 0.681852 1.0
26 0.0 0.662715 0.337285 0.0
27 0.0 0.652813 0.347187 0.0
28 1.0 0.466128 0.533872 1.0
29 0.0 0.345170 0.654830 1.0
30 1.0 0.219251 0.780749 1.0
31 0.0 0.543380 0.456620 0.0
32 0.0 0.679098 0.320902 0.0
33 0.0 0.425952 0.574048 1.0
34 1.0 0.229939 0.770061 1.0
35 1.0 0.087901 0.912099 1.0
36 0.0 0.654126 0.345874 0.0
37 1.0 0.311646 0.688354 1.0
38 0.0 0.430014 0.569986 1.0
39 0.0 0.462587 0.537413 1.0
40 1.0 0.384066 0.615934 1.0
41 0.0 0.676055 0.323945 0.0
42 0.0 0.666674 0.333326 0.0
43 0.0 0.657220 0.342780 0.0
44 1.0 0.236042 0.763958 1.0
45 0.0 0.652876 0.347124 0.0
46 1.0 0.176722 0.823278 1.0
47 1.0 0.639299 0.360701 0.0
48 1.0 0.413357 0.586643 1.0
49 0.0 0.353809 0.646191 1.0
... ... ... ... ...
22494 1.0 0.012420 0.987580 1.0
22495 0.0 0.665835 0.334165 0.0
22496 1.0 0.271595 0.728405 1.0
22497 1.0 0.289419 0.710581 1.0
22498 1.0 0.287932 0.712068 1.0
22499 0.0 0.640072 0.359928 0.0
22500 1.0 0.288896 0.711104 1.0
22501 1.0 0.422658 0.577342 1.0
22502 1.0 0.222248 0.777752 1.0
22503 1.0 0.348491 0.651509 1.0
22504 1.0 0.230068 0.769932 1.0
22505 1.0 0.636328 0.363673 0.0
22506 0.0 0.607841 0.392159 0.0
22507 0.0 0.410168 0.589832 1.0
22508 0.0 0.648628 0.351372 0.0
22509 1.0 0.364853 0.635147 1.0
22510 1.0 0.356348 0.643652 1.0
22511 0.0 0.629731 0.370269 0.0
22512 1.0 0.486851 0.513149 1.0
22513 1.0 0.128828 0.871172 1.0
22514 0.0 0.441461 0.558539 1.0
22515 1.0 0.494665 0.505335 1.0
22516 0.0 0.667365 0.332635 0.0
22517 1.0 0.394626 0.605374 1.0
22518 0.0 0.612999 0.387001 0.0
22519 1.0 0.390297 0.609703 1.0
22520 1.0 0.245386 0.754614 1.0
22521 1.0 0.406724 0.593276 1.0
22522 1.0 0.636279 0.363721 0.0
22523 0.0 0.666132 0.333868 0.0
22524 1.0 0.508207 0.491793 0.0
22525 1.0 0.198062 0.801938 1.0
22526 0.0 0.663925 0.336075 0.0
22527 0.0 0.643164 0.356836 0.0
22528 1.0 0.324625 0.675375 1.0
22529 0.0 0.547383 0.452617 0.0
22530 1.0 0.352351 0.647649 1.0
22531 1.0 0.413894 0.586106 1.0
22532 0.0 0.650537 0.349463 0.0
22533 0.0 0.676151 0.323849 0.0
22534 1.0 0.289614 0.710386 1.0
22535 0.0 0.655622 0.344379 0.0
22536 1.0 0.431428 0.568572 1.0
22537 1.0 0.590046 0.409954 0.0
22538 1.0 0.331959 0.668041 1.0
22539 0.0 0.601730 0.398269 0.0
22540 0.0 0.656616 0.343384 0.0
22541 1.0 0.630674 0.369326 0.0
22542 0.0 0.425283 0.574717 1.0
22543 1.0 0.331764 0.668236 1.0

22544 rows × 4 columns


In [25]:
Train.predictions_["12_12_3"]


Out[25]:
Actual Attack_prob Normal_prob Prediction
0 1.0 0.403012 0.596988 1.0
1 1.0 0.396761 0.603239 1.0
2 1.0 0.011543 0.988457 1.0
3 0.0 0.342929 0.657071 1.0
4 1.0 0.100866 0.899134 1.0
5 1.0 0.570735 0.429265 0.0
6 1.0 0.104104 0.895896 1.0
7 1.0 0.395437 0.604562 1.0
8 0.0 0.329435 0.670565 1.0
9 0.0 0.482691 0.517309 1.0
10 1.0 0.347291 0.652709 1.0
11 1.0 0.347256 0.652744 1.0
12 1.0 0.376822 0.623178 1.0
13 1.0 0.370854 0.629146 1.0
14 1.0 0.012452 0.987548 1.0
15 1.0 0.102070 0.897930 1.0
16 1.0 0.516925 0.483075 0.0
17 1.0 0.142840 0.857160 1.0
18 1.0 0.012216 0.987784 1.0
19 1.0 0.402125 0.597875 1.0
20 1.0 0.012140 0.987860 1.0
21 1.0 0.397413 0.602587 1.0
22 1.0 0.512977 0.487023 0.0
23 1.0 0.405979 0.594021 1.0
24 1.0 0.501969 0.498031 0.0
25 1.0 0.370833 0.629167 1.0
26 1.0 0.637455 0.362545 0.0
27 1.0 0.525610 0.474390 0.0
28 0.0 0.423820 0.576180 1.0
29 1.0 0.378979 0.621020 1.0
30 1.0 0.143435 0.856565 1.0
31 1.0 0.371802 0.628198 1.0
32 0.0 0.435273 0.564727 1.0
33 0.0 0.368529 0.631471 1.0
34 1.0 0.397406 0.602594 1.0
35 0.0 0.496896 0.503104 1.0
36 1.0 0.453324 0.546676 1.0
37 1.0 0.379357 0.620644 1.0
38 1.0 0.117689 0.882311 1.0
39 1.0 0.068886 0.931114 1.0
40 0.0 0.605587 0.394413 0.0
41 1.0 0.614922 0.385078 0.0
42 1.0 0.401664 0.598336 1.0
43 0.0 0.453789 0.546211 1.0
44 1.0 0.254491 0.745509 1.0
45 1.0 0.329483 0.670517 1.0
46 1.0 0.103922 0.896078 1.0
47 1.0 0.370503 0.629497 1.0
48 1.0 0.245634 0.754366 1.0
49 1.0 0.347161 0.652839 1.0
... ... ... ... ...
11800 1.0 0.349229 0.650771 1.0
11801 1.0 0.221894 0.778106 1.0
11802 0.0 0.337832 0.662167 1.0
11803 1.0 0.404219 0.595781 1.0
11804 1.0 0.421030 0.578970 1.0
11805 1.0 0.381074 0.618926 1.0
11806 0.0 0.437900 0.562100 1.0
11807 1.0 0.393244 0.606756 1.0
11808 1.0 0.101455 0.898545 1.0
11809 1.0 0.294265 0.705735 1.0
11810 1.0 0.470030 0.529970 1.0
11811 1.0 0.430750 0.569250 1.0
11812 0.0 0.326741 0.673259 1.0
11813 1.0 0.397465 0.602535 1.0
11814 1.0 0.017277 0.982723 1.0
11815 1.0 0.394181 0.605819 1.0
11816 1.0 0.407551 0.592449 1.0
11817 0.0 0.438897 0.561103 1.0
11818 0.0 0.601524 0.398476 0.0
11819 1.0 0.052833 0.947167 1.0
11820 1.0 0.369175 0.630825 1.0
11821 1.0 0.128646 0.871354 1.0
11822 1.0 0.438307 0.561693 1.0
11823 1.0 0.334591 0.665409 1.0
11824 1.0 0.555246 0.444754 0.0
11825 0.0 0.614823 0.385177 0.0
11826 1.0 0.289127 0.710873 1.0
11827 1.0 0.094855 0.905145 1.0
11828 1.0 0.344284 0.655716 1.0
11829 1.0 0.436262 0.563738 1.0
11830 1.0 0.570394 0.429607 0.0
11831 1.0 0.204457 0.795543 1.0
11832 1.0 0.248856 0.751144 1.0
11833 0.0 0.469030 0.530970 1.0
11834 1.0 0.201534 0.798466 1.0
11835 0.0 0.436682 0.563318 1.0
11836 1.0 0.122989 0.877011 1.0
11837 1.0 0.425958 0.574042 1.0
11838 1.0 0.094290 0.905710 1.0
11839 1.0 0.162568 0.837432 1.0
11840 0.0 0.567906 0.432094 0.0
11841 0.0 0.439834 0.560166 1.0
11842 1.0 0.206409 0.793591 1.0
11843 1.0 0.349581 0.650419 1.0
11844 1.0 0.379782 0.620218 1.0
11845 0.0 0.439294 0.560706 1.0
11846 0.0 0.636029 0.363971 0.0
11847 1.0 0.322738 0.677262 1.0
11848 1.0 0.329713 0.670287 1.0
11849 1.0 0.369175 0.630825 1.0

11850 rows × 4 columns


In [26]:
df = Train.predictions["12_12_3"].dropna()
df_ = Train.predictions_["12_12_3"].dropna()

In [27]:
from sklearn import metrics as me
def get_score(y_true, y_pred):
    f1 = me.f1_score(y_true, y_pred)
    pre = me.precision_score(y_true, y_pred)
    rec = me.recall_score(y_true, y_pred)
    acc = me.accuracy_score(y_true, y_pred)
    return {"F1 Score":f1, "Precision":pre, "Recall":rec, "Accuracy":acc}

In [28]:
from sklearn import metrics as me

scores = get_score(df.loc[:,'Actual'].values.astype(int),
                df.loc[:,'Prediction'].values.astype(int))
scores.update({"Scenario":"Train+/Test+"})
score_df = pd.DataFrame(scores, index=[0])

scores = get_score(df_.loc[:,'Actual'].values.astype(int),
                df_.loc[:,'Prediction'].values.astype(int))
scores.update({"Scenario":"Train+/Test-"})

score_df = score_df.append(pd.DataFrame(scores, index=[1]))

score_df


Out[28]:
Accuracy F1 Score Precision Recall Scenario
0 0.830509 0.857121 0.823940 0.893088 Train+/Test+
1 0.730211 0.839242 0.819021 0.860487 Train+/Test-

In [29]:
df.groupby(by="Actual").Actual.count()


Out[29]:
Actual
0.0     9711
1.0    12833
Name: Actual, dtype: int64

In [30]:
plot(actual_value = df.loc[:,'Actual'].values.astype(int),
     pred_value = df.loc[:,'Prediction'].values.astype(int))



In [31]:
df_.groupby(by="Actual").Actual.count()


Out[31]:
Actual
0.0    2152
1.0    9698
Name: Actual, dtype: int64

In [32]:
plot(actual_value = df_.loc[:,'Actual'].values.astype(int),
     pred_value = df_.loc[:,'Prediction'].values.astype(int))



In [33]:
from scipy import stats

def fn(x):
    #print(x)
    return stats.norm.interval(0.95, loc=x.f1_score.mean(), scale=x.f1_score.std())
psg.apply(fn)


Out[33]:
no_of_features  hidden_layers
1.0             1.0              (0.578452539765, 0.841046710607)
                3.0                                    (nan, nan)
12.0            1.0              (0.668723301504, 0.932285711705)
                3.0              (0.636233676898, 0.977571551948)
24.0            1.0                (0.7637835282, 0.920711831434)
                3.0               (0.783714231971, 0.84733658272)
48.0            1.0              (0.817941844822, 0.875285309714)
                3.0                (0.60901468163, 1.02861547867)
122.0           1.0              (0.665214663577, 0.974471096522)
                3.0              (0.855675801742, 0.874380756313)
dtype: object

In [ ]: