Read Data Sample

In [1]:

    

import pandas as pd
import numpy as np
import os
import time
from collections import namedtuple
pd.set_option("display.max_rows",35)
%matplotlib inline


    

In [2]:

    

%%bash
rm dataset/scores/tf_dense_only_nsl_kdd_scores_all-.pkl


    

In [3]:

    

class dataset:
    kdd_train_2labels = pd.read_pickle("dataset/kdd_train__2labels.pkl")
    kdd_test_2labels = pd.read_pickle("dataset/kdd_test_2labels.pkl")
    kdd_test__2labels = pd.read_pickle("dataset/kdd_test__2labels.pkl")
    
    kdd_train_5labels = pd.read_pickle("dataset/kdd_train_5labels.pkl")
    kdd_test_5labels = pd.read_pickle("dataset/kdd_test_5labels.pkl")


    

In [4]:

    

dataset.kdd_train_2labels.shape


    

    
Out[4]:





(25192, 124)

In [5]:

    

dataset.kdd_test_2labels.shape


    

    
Out[5]:





(22544, 124)

In [6]:

    

from sklearn import model_selection as ms
from sklearn import preprocessing as pp

class preprocess:
    
    output_columns_2labels = ['is_Normal','is_Attack']
    
    x_input = dataset.kdd_train_2labels.drop(output_columns_2labels, axis = 1)
    y_output = dataset.kdd_train_2labels.loc[:,output_columns_2labels]

    x_test_input = dataset.kdd_test_2labels.drop(output_columns_2labels, axis = 1)
    y_test = dataset.kdd_test_2labels.loc[:,output_columns_2labels]
    
    x_test__input = dataset.kdd_test__2labels.drop(output_columns_2labels, axis = 1)
    y_test_ = dataset.kdd_test__2labels.loc[:,output_columns_2labels]

    ss = pp.StandardScaler()

    x_train = ss.fit_transform(x_input)
    x_test = ss.transform(x_test_input)
    x_test_ = ss.transform(x_test__input)

    y_train = y_output.values
    y_test = y_test.values
    y_test_ = y_test_.values

    
preprocess.x_train.std()


    

    
Out[6]:





0.97509982675167528

In [7]:

    

import tensorflow as tf
from tensorflow.contrib.legacy_seq2seq.python.ops.seq2seq import basic_rnn_seq2seq
from tensorflow.contrib.rnn import RNNCell, LSTMCell, MultiRNNCell


    

In [8]:

    

class network(object):
    
    input_dim = 122
    classes = 2
    hidden_encoder_dim = 122
    hidden_layers = 1
    latent_dim = 10

    hidden_decoder_dim = 122
    lam = 0.01
    
    def __init__(self, classes, hidden_layers, num_of_features):
        self.classes = classes
        self.hidden_layers = hidden_layers
        self.latent_dim = num_of_features
            
    def build_layers(self):
        tf.reset_default_graph()
        #learning_rate = tf.Variable(initial_value=0.001)

        input_dim = self.input_dim
        classes = self.classes
        hidden_encoder_dim = self.hidden_encoder_dim
        hidden_layers = self.hidden_layers
        latent_dim = self.latent_dim
        hidden_decoder_dim = self.hidden_decoder_dim
        lam = self.lam
        
        with tf.variable_scope("Input"):
            self.x_input = tf.placeholder("float", shape=[None, 1, input_dim])
            self.y_input_ = tf.placeholder("float", shape=[None, 1, classes])
            self.keep_prob = tf.placeholder("float")
            self.lr = tf.placeholder("float")
            self.x_list = tf.unstack(self.x_input, axis= 1)
            self.y_list_ = tf.unstack(self.y_input_, axis = 1)
            self.y_ = self.y_list_[0]
            
            #GO = tf.fill((tf.shape(self.x)[0], 1), 0.5)
            
            #y_with_GO = tf.stack([self.y_, GO])
            
        with tf.variable_scope("lstm"):
            multi_cell = MultiRNNCell([LSTMCell(input_dim) for i in range(hidden_layers)] )
            
            self.y, states = basic_rnn_seq2seq(self.x_list, self.y_list_, multi_cell)
            #self.y = tf.slice(self.y, [0, 0], [-1,2])
            
            #self.out = tf.squeeze(self.y)
            
            #self.y = tf.layers.dense(self.y[0], classes, activation = None)
            
            self.y = tf.slice(self.y[0], [0, 0], [-1,2])
            
        with tf.variable_scope("Loss"):
            
            self.regularized_loss = tf.losses.mean_squared_error(self.y_, self.y)
            correct_prediction = tf.equal(tf.argmax(self.y_, 1), tf.argmax(self.y, 1))
            self.tf_accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name = "Accuracy")

        with tf.variable_scope("Optimizer"):
            learning_rate=self.lr
            optimizer = tf.train.AdamOptimizer(learning_rate)
            gradients, variables = zip(*optimizer.compute_gradients(self.regularized_loss))
            gradients = [
                None if gradient is None else tf.clip_by_value(gradient, -1, 1)
                for gradient in gradients]
            self.train_op = optimizer.apply_gradients(zip(gradients, variables))
            #self.train_op = optimizer.minimize(self.regularized_loss)
            
        # add op for merging summary
        #self.summary_op = tf.summary.merge_all()
        self.pred = tf.argmax(self.y, axis = 1)
        self.actual = tf.argmax(self.y_, axis = 1)

        # add Saver ops
        self.saver = tf.train.Saver()


    

In [9]:

    

import collections
import sklearn.metrics as me

class Train:    
    
    result = namedtuple("score", ['epoch', 'no_of_features','hidden_layers','train_score', 'test_score', 'f1_score', 'test_score_20', 'f1_score_20', 'time_taken'])

    predictions = {}
    predictions_ = {}

    results = []
    best_acc = 0
    best_acc_global = 0

    def train(epochs, net, h,f, lrs):
        batch_iterations = 200
        train_loss = None
        Train.best_acc = 0
        os.makedirs("dataset/tf_lstm_nsl_kdd-orig-/hidden layers_{}_features count_{}".format(h,f),
                    exist_ok = True)
        with tf.Session() as sess:
            #summary_writer_train = tf.summary.FileWriter('./logs/kdd/VAE/training', graph=sess.graph)
            #summary_writer_valid = tf.summary.FileWriter('./logs/kdd/VAE/validation')

            sess.run(tf.global_variables_initializer())
            start_time = time.perf_counter()
            
            accuracy, pred_value, actual_value, y_pred = sess.run([net.tf_accuracy, 
                                                                   net.pred, 
                                                                   net.actual, net.y], 
                                                                  feed_dict={net.x_input: preprocess.x_test[:,np.newaxis,:], 
                                                                             net.y_input_: preprocess.y_test[:,np.newaxis,:], 
                                                                             net.keep_prob:1})
            
            print("Initial Accuracy, before training: {}".format(accuracy))
            
            for c, lr in enumerate(lrs):
                for epoch in range(1, (epochs+1)):
                    x_train, x_valid, y_train, y_valid, = ms.train_test_split(preprocess.x_train, 
                                                                              preprocess.y_train, 
                                                                              test_size=0.1)
                    batch_indices = np.array_split(np.arange(x_train.shape[0]), 
                                               batch_iterations)

                    for i in batch_indices:

                        _, train_loss = sess.run([net.train_op, net.regularized_loss], #net.summary_op
                                                              feed_dict={net.x_input: x_train[i,np.newaxis,:], 
                                                                         net.y_input_: y_train[i,np.newaxis,:], 
                                                                         net.keep_prob:1, net.lr:lr})
                        #summary_writer_train.add_summary(summary_str, epoch)
                        if(train_loss > 1e9):
                            print("Step {} | Training Loss: {:.6f}".format(epoch, train_loss))


                    valid_accuracy,valid_loss = sess.run([net.tf_accuracy, net.regularized_loss], #net.summary_op 
                                                          feed_dict={net.x_input: x_valid[:,np.newaxis,:], 
                                                                     net.y_input_: y_valid[:,np.newaxis,:], 
                                                                     net.keep_prob:1, net.lr:lr})
                    #summary_writer_valid.add_summary(summary_str, epoch)



                    accuracy, pred_value, actual_value, y_pred = sess.run([net.tf_accuracy, 
                                                                   net.pred, 
                                                                   net.actual, net.y], 
                                                                  feed_dict={net.x_input: preprocess.x_test[:,np.newaxis,:], 
                                                                             net.y_input_: preprocess.y_test[:,np.newaxis,:], 
                                                                             net.keep_prob:1, net.lr:lr})
                    f1_score = me.f1_score(actual_value, pred_value)
                    accuracy_, pred_value_, actual_value_, y_pred_ = sess.run([net.tf_accuracy, 
                                                                   net.pred, 
                                                                   net.actual, net.y], 
                                                                  feed_dict={net.x_input: preprocess.x_test_[:,np.newaxis,:], 
                                                                             net.y_input_: preprocess.y_test_[:,np.newaxis,:], 
                                                                             net.keep_prob:1, net.lr:lr})
                    f1_score_ = me.f1_score(actual_value_, pred_value_)
                    print("Step {} | Training Loss: {:.6f} | Train Accuracy: {:.6f} | Test Accuracy: {:.6f}, {:.6f}".format(epoch, train_loss, valid_accuracy, accuracy, accuracy_))

                    if accuracy > Train.best_acc_global:
                                Train.best_acc_global = accuracy
                                Train.pred_value = pred_value
                                Train.actual_value = actual_value
                                Train.pred_value_ = pred_value_
                                Train.actual_value_ = actual_value_
                                Train.best_parameters = "Hidden Layers:{}, Features Count:{}".format(h, f)

                    if accuracy > Train.best_acc:

                        #net.saver.save(sess, "dataset/tf_vae_only_nsl_kdd_hidden layers_{}_features count_{}".format(epochs,h,f))
                        #Train.results.append(Train.result(epochs, f, h,valid_accuracy, accuracy))
                        #curr_pred = pd.DataFrame({"Attack_prob":y_pred[:,-2], "Normal_prob":y_pred[:, -1]})
                        #Train.predictions.update({"{}_{}_{}".format(epochs,f,h):curr_pred})

                        Train.best_acc = accuracy
                        if not (np.isnan(train_loss)):
                            net.saver.save(sess, 
                                       "dataset/tf_lstm_nsl_kdd-orig-/hidden layers_{}_features count_{}/model"
                                       .format(h,f), 
                                       global_step = epoch, 
                                       write_meta_graph=False)

                        curr_pred = pd.DataFrame({"Attack_prob":y_pred[:,-2], "Normal_prob":y_pred[:, -1], "Prediction":pred_value, "Actual":actual_value})
                        curr_pred_ = pd.DataFrame({"Attack_prob":y_pred_[:,-2], "Normal_prob":y_pred_[:, -1], "Prediction":pred_value_, "Actual": actual_value_})
                        Train.predictions.update({"{}_{}_{}".format((epochs+1)* (c+1),f,h):
                                                  (curr_pred,
                                                   Train.result((epochs+1)*(c+1), f, h,valid_accuracy, accuracy, f1_score, accuracy_, f1_score_, time.perf_counter() - start_time))})
                        Train.predictions_.update({"{}_{}_{}".format((epochs+1)* (c+1),f,h):
                                                  (curr_pred_,
                                                   Train.result((epochs+1)*(c+1), f, h,valid_accuracy, accuracy, f1_score, accuracy_, f1_score_, time.perf_counter() - start_time))})


    

In [10]:

    

import itertools

df_results = []
past_scores = []

class Hyperparameters:
#    features_arr = [2, 4, 8, 16, 32, 64, 128, 256]
#    hidden_layers_arr = [2, 4, 6, 10]

    def start_training():

        global df_results
        global past_scores
        
        Train.predictions = {}
        Train.results = []
        
        features_arr = [1] #[4, 8, 16, 32]
        hidden_layers_arr = [1, 3]

        epochs = [10]
        lrs = [1e-2, 1e-3]

        for e, h, f in itertools.product(epochs, hidden_layers_arr, features_arr):
            print("Current Layer Attributes - epochs:{} hidden layers:{} features count:{}".format(e,h,f))
            n = network(2,h,f)
            n.build_layers()
            Train.train(e, n, h,f, lrs)
            
        dict1 = {}
        dict1_ = {}
        dict2 = []
        for k, (v1, v2) in Train.predictions.items():
            dict1.update({k: v1})
            dict2.append(v2)

        for k, (v1_, v2) in Train.predictions.items():
            dict1_.update({k: v1_})

            
        Train.predictions = dict1
        Train.predictions_ = dict1_

        Train.results = dict2
        df_results = pd.DataFrame(Train.results)
        temp = df_results.set_index(['no_of_features', 'hidden_layers'])

        if not os.path.isfile('dataset/scores/tf_lstm_nsl_kdd-orig_all-.pkl'):
            past_scores = temp
        else:
            past_scores = pd.read_pickle("dataset/scores/tf_lstm_nsl_kdd-orig_all-.pkl")

        past_scores.append(temp).to_pickle("dataset/scores/tf_lstm_nsl_kdd-orig_all-.pkl")


    

In [11]:

    

#%%timeit -r 10

Hyperparameters.start_training()


    

    




Current Layer Attributes - epochs:10 hidden layers:1 features count:1
Initial Accuracy, before training: 0.631076991558075
Step 1 | Training Loss: 0.017782 | Train Accuracy: 0.984921 | Test Accuracy: 0.778300, 0.579325
Step 2 | Training Loss: 0.002807 | Train Accuracy: 0.998413 | Test Accuracy: 0.811746, 0.641857
Step 3 | Training Loss: 0.002106 | Train Accuracy: 0.998016 | Test Accuracy: 0.834634, 0.685401
Step 4 | Training Loss: 0.002415 | Train Accuracy: 0.999206 | Test Accuracy: 0.877972, 0.767848
Step 5 | Training Loss: 0.001275 | Train Accuracy: 0.998810 | Test Accuracy: 0.888751, 0.788354
Step 6 | Training Loss: 0.001042 | Train Accuracy: 0.999603 | Test Accuracy: 0.881964, 0.775443
Step 7 | Training Loss: 0.001309 | Train Accuracy: 0.998413 | Test Accuracy: 0.897933, 0.805823
Step 8 | Training Loss: 0.001469 | Train Accuracy: 0.999206 | Test Accuracy: 0.893630, 0.797637
Step 9 | Training Loss: 0.001851 | Train Accuracy: 0.999603 | Test Accuracy: 0.897933, 0.805823
Step 10 | Training Loss: 0.001222 | Train Accuracy: 0.999603 | Test Accuracy: 0.897889, 0.805738
Step 1 | Training Loss: 0.002581 | Train Accuracy: 0.999603 | Test Accuracy: 0.896957, 0.803966
Step 2 | Training Loss: 0.002195 | Train Accuracy: 0.999603 | Test Accuracy: 0.896469, 0.803038
Step 3 | Training Loss: 0.001027 | Train Accuracy: 0.999603 | Test Accuracy: 0.895538, 0.801266
Step 4 | Training Loss: 0.001704 | Train Accuracy: 0.999603 | Test Accuracy: 0.895316, 0.800844
Step 5 | Training Loss: 0.001718 | Train Accuracy: 1.000000 | Test Accuracy: 0.894251, 0.798819
Step 6 | Training Loss: 0.001270 | Train Accuracy: 0.999603 | Test Accuracy: 0.894340, 0.798987
Step 7 | Training Loss: 0.002221 | Train Accuracy: 0.999603 | Test Accuracy: 0.893941, 0.798228
Step 8 | Training Loss: 0.001197 | Train Accuracy: 0.999603 | Test Accuracy: 0.893852, 0.798059
Step 9 | Training Loss: 0.010017 | Train Accuracy: 0.999603 | Test Accuracy: 0.893763, 0.797890
Step 10 | Training Loss: 0.006140 | Train Accuracy: 0.999603 | Test Accuracy: 0.893896, 0.798143
Current Layer Attributes - epochs:10 hidden layers:3 features count:1
Initial Accuracy, before training: 0.41221609711647034
Step 1 | Training Loss: 0.000778 | Train Accuracy: 0.998810 | Test Accuracy: 0.973607, 0.949789
Step 2 | Training Loss: 0.000664 | Train Accuracy: 0.998810 | Test Accuracy: 0.977511, 0.957215
Step 3 | Training Loss: 0.000653 | Train Accuracy: 1.000000 | Test Accuracy: 0.978708, 0.959494
Step 4 | Training Loss: 0.000654 | Train Accuracy: 0.999206 | Test Accuracy: 0.969837, 0.942616
Step 5 | Training Loss: 0.000650 | Train Accuracy: 1.000000 | Test Accuracy: 0.998226, 0.996624
Step 6 | Training Loss: 0.000651 | Train Accuracy: 0.999603 | Test Accuracy: 0.998669, 0.997468
Step 7 | Training Loss: 0.000649 | Train Accuracy: 1.000000 | Test Accuracy: 0.983410, 0.968439
Step 8 | Training Loss: 0.000649 | Train Accuracy: 1.000000 | Test Accuracy: 0.983499, 0.968608
Step 9 | Training Loss: 0.000651 | Train Accuracy: 1.000000 | Test Accuracy: 0.999157, 0.998397
Step 10 | Training Loss: 0.000648 | Train Accuracy: 1.000000 | Test Accuracy: 0.999113, 0.998312
Step 1 | Training Loss: 0.000648 | Train Accuracy: 1.000000 | Test Accuracy: 0.999113, 0.998312
Step 2 | Training Loss: 0.000648 | Train Accuracy: 1.000000 | Test Accuracy: 0.999113, 0.998312
Step 3 | Training Loss: 0.000648 | Train Accuracy: 1.000000 | Test Accuracy: 0.999113, 0.998312
Step 4 | Training Loss: 0.000648 | Train Accuracy: 1.000000 | Test Accuracy: 0.999113, 0.998312
Step 5 | Training Loss: 0.000648 | Train Accuracy: 1.000000 | Test Accuracy: 0.999113, 0.998312
Step 6 | Training Loss: 0.000648 | Train Accuracy: 1.000000 | Test Accuracy: 0.999113, 0.998312
Step 7 | Training Loss: 0.000648 | Train Accuracy: 1.000000 | Test Accuracy: 0.999113, 0.998312
Step 8 | Training Loss: 0.000648 | Train Accuracy: 1.000000 | Test Accuracy: 0.999113, 0.998312
Step 9 | Training Loss: 0.000647 | Train Accuracy: 1.000000 | Test Accuracy: 0.999113, 0.998312
Step 10 | Training Loss: 0.000648 | Train Accuracy: 1.000000 | Test Accuracy: 0.999113, 0.998312

In [12]:

    

pd.Panel(Train.predictions).to_pickle("dataset/tf_lstm_nsl_kdd_predictions-.pkl")
pd.Panel(Train.predictions_).to_pickle("dataset/tf_lstm_nsl_kdd_predictions-__.pkl")

#df_results.to_pickle("dataset/tf_lstm_nsl_kdd_scores-.pkl")


    

In [13]:

    

import numpy as np
import matplotlib.pyplot as plt
import itertools

def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    np.set_printoptions(precision=4)

    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        #print("Normalized confusion matrix")
    else:
        #print('Confusion matrix, without normalization')
        pass
    
    #print(cm)

    label = [["\n True Negative", "\n False Positive \n Type II Error"],
             ["\n False Negative \n Type I Error", "\n True Positive"]
            ]
    
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        
        plt.text(j, i, "{} {}".format(cm[i, j].round(4), label[i][j]),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

def plot(actual_value, pred_value):
    from sklearn.metrics import confusion_matrix

    cm_2labels = confusion_matrix(y_pred = pred_value, y_true = actual_value)
    plt.figure(figsize=[6,6])
    plot_confusion_matrix(cm_2labels, ['Normal', 'Attack'], normalize = False)


    

In [14]:

    

past_scores = pd.read_pickle("dataset/scores/tf_lstm_nsl_kdd-orig_all-.pkl")


    

In [15]:

    

past_scores.sort_values(by='f1_score',ascending=False)


    

    
Out[15]:







  

    

      

      

      
epoch
      
train_score
      
test_score
      
f1_score
      
test_score_20
      
f1_score_20
      
time_taken
    
    

      
no_of_features
      
hidden_layers
      

      

      

      

      

      

      

    
  
  

    

      
1
      
3
      
11
      
1.000000
      
1.000000
      
1.000000
      
1.000000
      
1.000000
      
15.237297
    
    

      
3
      
11
      
1.000000
      
0.999734
      
0.999766
      
0.999494
      
0.999691
      
6.295392
    
    

      
3
      
11
      
1.000000
      
0.999734
      
0.999766
      
0.999494
      
0.999691
      
12.354382
    
    

      
3
      
11
      
0.998016
      
0.999379
      
0.999455
      
0.999156
      
0.999485
      
11.837164
    
    

      
3
      
11
      
1.000000
      
0.999290
      
0.999377
      
0.998650
      
0.999175
      
11.787666
    
    

      
3
      
11
      
1.000000
      
0.999157
      
0.999260
      
0.998397
      
0.999021
      
27.536535
    
    

      
3
      
11
      
1.000000
      
0.997072
      
0.997422
      
0.994430
      
0.996586
      
11.624922
    
    

      
3
      
22
      
1.000000
      
0.994633
      
0.995302
      
0.989789
      
0.993791
      
38.527330
    
    

      
3
      
11
      
1.000000
      
0.993701
      
0.994492
      
0.988017
      
0.992722
      
27.259746
    
    

      
3
      
11
      
0.998810
      
0.991084
      
0.992140
      
0.983122
      
0.989639
      
8.618293
    
    

      
3
      
11
      
1.000000
      
0.986914
      
0.988427
      
0.975105
      
0.984652
      
9.510270
    
    

      
3
      
11
      
0.999206
      
0.980704
      
0.983254
      
0.963291
      
0.977927
      
8.956038
    
    

      
3
      
11
      
0.999206
      
0.980704
      
0.983254
      
0.963291
      
0.977927
      
8.956038
    
    

      
3
      
11
      
1.000000
      
0.976579
      
0.979063
      
0.955443
      
0.972134
      
6.629817
    
    

      
1
      
11
      
1.000000
      
0.918559
      
0.924326
      
0.845063
      
0.897955
      
11.249477
    
    

      
1
      
11
      
1.000000
      
0.906450
      
0.914591
      
0.822025
      
0.885524
      
7.625335
    
    

      
1
      
11
      
1.000000
      
0.903699
      
0.912080
      
0.816793
      
0.882158
      
13.398739
    
    

      
1
      
11
      
1.000000
      
0.899707
      
0.908250
      
0.809198
      
0.876939
      
6.418205
    
    

      
1
      
11
      
1.000000
      
0.899574
      
0.908087
      
0.808945
      
0.876702
      
9.361751
    
    

      
1
      
11
      
1.000000
      
0.899175
      
0.907590
      
0.808186
      
0.875975
      
8.658957
    
    

      
1
      
11
      
1.000000
      
0.899175
      
0.907590
      
0.808186
      
0.875975
      
8.658957
    
    

      
1
      
11
      
1.000000
      
0.898776
      
0.907130
      
0.807426
      
0.875314
      
11.615757
    
    

      
1
      
11
      
0.998413
      
0.897933
      
0.906300
      
0.805823
      
0.874173
      
7.838056
    
    

      
1
      
11
      
1.000000
      
0.897844
      
0.906195
      
0.805654
      
0.874022
      
9.943235
    
    

      
1
      
11
      
1.000000
      
0.897667
      
0.905940
      
0.805316
      
0.873638
      
10.647751
    
    

      
1
      
11
      
1.000000
      
0.893586
      
0.902110
      
0.797553
      
0.868454
      
8.407638
    
    

      
1
      
11
      
1.000000
      
0.891501
      
0.899655
      
0.793586
      
0.864907
      
11.260245
    
  

In [16]:

    

psg = past_scores.sort_values(by='test_score', ascending=False).groupby(by=['no_of_features', 'hidden_layers'])
psg.first().sort_values(by='test_score', ascending=False)


    

    
Out[16]:







  

    

      

      

      
epoch
      
train_score
      
test_score
      
f1_score
      
test_score_20
      
f1_score_20
      
time_taken
    
    

      
no_of_features
      
hidden_layers
      

      

      

      

      

      

      

    
  
  

    

      
1
      
3
      
11
      
1.0
      
1.000000
      
1.000000
      
1.000000
      
1.000000
      
15.237297
    
    

      
1
      
11
      
1.0
      
0.918559
      
0.924326
      
0.845063
      
0.897955
      
11.249477
    
  

In [17]:

    

psg.mean().sort_values(by='test_score', ascending=False)


    

    
Out[17]:







  

    

      

      

      
epoch
      
train_score
      
test_score
      
f1_score
      
test_score_20
      
f1_score_20
      
time_taken
    
    

      
no_of_features
      
hidden_layers
      

      

      

      

      

      

      

    
  
  

    

      
1
      
3
      
11.785714
      
0.999660
      
0.992763
      
0.993641
      
0.986263
      
0.991603
      
14.652206
    
    

      
1
      
11.000000
      
0.999878
      
0.900280
      
0.908450
      
0.810289
      
0.877057
      
9.621854
    
  

In [18]:

    

Train.predictions = pd.read_pickle("dataset/tf_lstm_nsl_kdd_predictions-.pkl")
Train.predictions_ = pd.read_pickle("dataset/tf_lstm_nsl_kdd_predictions-__.pkl")


    

In [20]:

    

#epoch_nof_hidden
Train.predictions["11_1_3"].sample()


    

    
Out[20]:







  

    

      

      
Actual
      
Attack_prob
      
Normal_prob
      
Prediction
    
  
  

    

      
12621
      
0.0
      
0.964016
      
0.00015
      
0.0
    
  

In [21]:

    

Train.predictions_["11_1_3"].sample()


    

    
Out[21]:







  

    

      

      
Actual
      
Attack_prob
      
Normal_prob
      
Prediction
    
  
  

    

      
4349
      
0.0
      
0.964014
      
0.000171
      
0.0
    
  

In [23]:

    

df = Train.predictions["11_1_3"].dropna()
df_ = Train.predictions_["11_1_3"].dropna()


    

In [24]:

    

from sklearn import metrics as me
def get_score(y_true, y_pred):
    f1 = me.f1_score(y_true, y_pred)
    pre = me.precision_score(y_true, y_pred)
    rec = me.recall_score(y_true, y_pred)
    acc = me.accuracy_score(y_true, y_pred)
    return {"F1 Score":f1, "Precision":pre, "Recall":rec, "Accuracy":acc}


    

In [25]:

    

from sklearn import metrics as me

scores = get_score(df.loc[:,'Actual'].values.astype(int),
                df.loc[:,'Prediction'].values.astype(int))
scores.update({"Scenario":"Train+/Test+"})
score_df = pd.DataFrame(scores, index=[0])

scores = get_score(df_.loc[:,'Actual'].values.astype(int),
                df_.loc[:,'Prediction'].values.astype(int))
scores.update({"Scenario":"Train+/Test-"})

score_df = score_df.append(pd.DataFrame(scores, index=[1]))

score_df


    

    
Out[25]:







  

    

      

      
Accuracy
      
F1 Score
      
Precision
      
Recall
      
Scenario
    
  
  

    

      
0
      
0.999157
      
0.99926
      
0.998522
      
1.0
      
Train+/Test+
    
    

      
1
      
0.999157
      
0.99926
      
0.998522
      
1.0
      
Train+/Test-
    
  

In [26]:

    

df.groupby(by="Actual").Actual.count()


    

    
Out[26]:





Actual
0.0     9711
1.0    12833
Name: Actual, dtype: int64

In [27]:

    

plot(actual_value = df.loc[:,'Actual'].values.astype(int),
     pred_value = df.loc[:,'Prediction'].values.astype(int))


    

In [28]:

    

df_.groupby(by="Actual").Actual.count()


    

    
Out[28]:





Actual
0.0     9711
1.0    12833
Name: Actual, dtype: int64

In [29]:

    

plot(actual_value = df_.loc[:,'Actual'].values.astype(int),
     pred_value = df_.loc[:,'Prediction'].values.astype(int))


    

In [30]:

    

from scipy import stats

def fn(x):
    #print(x)
    return stats.norm.interval(0.95, loc=x.f1_score.mean(), scale=x.f1_score.std())
psg.apply(fn)


    

    
Out[30]:





no_of_features  hidden_layers
1               1                (0.896582997095, 0.920316318303)
                3                 (0.979369982016, 1.00791284291)
dtype: object

In [ ]: