Deep Learning

In [1]:

    

from keras.layers import Input, Dense, Lambda, Layer
from keras.models import Model
from keras import regularizers
import keras
import pandas as pd
import numpy as np
from keras import backend as K
from keras import metrics
from collections import namedtuple
pd.set_option("display.max_rows",35)
%matplotlib inline


    

    




Using TensorFlow backend.

In [2]:

    

kdd_train_2labels = pd.read_pickle("dataset/kdd_train_2labels.pkl")
kdd_test_2labels = pd.read_pickle("dataset/kdd_test_2labels.pkl")

#y_train_labels = pd.read_pickle("dataset/kdd_train_2labels_y.pkl")
#y_train_labels = pd.read_pickle("dataset/kdd_train_2labels.pkl")
#y_test_labels = pd.read_pickle("dataset/kdd_test_2labels_y.pkl")

output_columns_2labels = ['is_Attack','is_Normal']

from sklearn import model_selection as ms
from sklearn import preprocessing as pp

x_input = kdd_train_2labels.drop(output_columns_2labels, axis = 1)
y_output = kdd_train_2labels.loc[:,output_columns_2labels]

ss = pp.StandardScaler()
x_input = ss.fit_transform(x_input)

#le = pp.LabelEncoder()
#y_train = le.fit_transform(y_train_labels).reshape(-1, 1)
#y_test = le.transform(y_test_labels).reshape(-1, 1)

y_train = kdd_train_2labels.loc[:,output_columns_2labels]

x_train, x_valid, y_train, y_valid = ms.train_test_split(x_input, 
                              y_train, 
                              test_size=0.1)
#x_valid, x_test, y_valid, y_test = ms.train_test_split(x_valid, y_valid, test_size = 0.4)

x_test = kdd_test_2labels.drop(output_columns_2labels, axis = 1)
y_test = kdd_test_2labels.loc[:,output_columns_2labels]

x_test = ss.transform(x_test)

x_train = np.hstack((x_train, y_train))
x_valid = np.hstack((x_valid, y_valid))

x_test = np.hstack((x_test, np.random.normal(loc = 0, scale = 0.01, size = y_test.shape)))


    

In [3]:

    

input_dim = 124
intermediate_dim = 124
latent_dim = 32
batch_size = 1409
hidden_layers = 8

class Train:
    def train():
        Train.x = Input(shape=(input_dim,))
        
        hidden_encoder = Train.x
        for i in range(hidden_layers):
            hidden_encoder = Dense(intermediate_dim, activation='relu')(hidden_encoder)

        Train.mean_encoder = Dense(latent_dim, activation=None)(hidden_encoder)
        Train.logvar_encoder = Dense(latent_dim, activation=None)(hidden_encoder)

        def get_distrib(args):

            m_e, l_e = args

            # Sample epsilon
            epsilon = np.random.normal(loc=0.0, scale=0.05, size = (batch_size, latent_dim))

            # Sample latent variable
            z = m_e + K.exp(l_e / 2) * epsilon
            return z

        z = Lambda(get_distrib)([Train.mean_encoder, Train.logvar_encoder])

        hidden_decoder = z
        for i in range(hidden_layers):
            hidden_decoder = Dense(intermediate_dim, activation="relu")(hidden_decoder)

        Train.x_ = Dense(input_dim, activation=None)(hidden_decoder)
            

def label_accuracy(y_true, y_pred):
    return K.mean(K.equal(K.argmax(y_true[:,-2:], axis = 1), K.argmax(y_pred[:,-2:], axis = 1)))

def vae_loss(x, x_decoded_mean):
    xent_loss = input_dim * metrics.binary_crossentropy(x, x_decoded_mean)
    kl_loss = - 0.5 * K.sum(1 + Train.logvar_encoder - K.square(Train.mean_encoder) - K.exp(Train.logvar_encoder), axis=-1)
    return K.mean(xent_loss + kl_loss)


    

In [4]:

    

import itertools
#features_arr = [4, 16, 32, 256, 1024]
#hidden_layers_arr = [2, 6, 10, 100]

features_arr = [4, 16, 32]
hidden_layers_arr = [2, 6, 10]

epoch_arr = [5]

score = namedtuple("score", ['epoch', 'no_of_features','hidden_layers','train_score', 'test_score'])
scores = []
predictions = {}

for e, h, f in itertools.product(epoch_arr, hidden_layers_arr, features_arr):
    
    print(" \n Current Layer Attributes - epochs:{} hidden layers:{} features count:{}".format(e,h,f))
    latent_dim = f
    epochs = e
    hidden_layers = h

    Train.train()

    vae_model = Model(inputs = Train.x, outputs = Train.x_ )
    
    optimizer = keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-04, decay=0.1)
    #vae_model.compile(optimizer = optimizer, loss = keras.losses.binary_crossentropy , metrics = [label_accuracy] )
    
    vae_model.compile(optimizer = optimizer, loss = vae_loss , metrics = [label_accuracy] )
    
    #vae_model.compile(optimizer = "adam", loss = Lambda(get_loss)([Train.x, Train.x_]), metrics = ['accuracy', label_accuracy] )

    train_size = x_train.shape[0] - x_train.shape[0]%batch_size
    valid_size = x_valid.shape[0] - x_valid.shape[0]%batch_size

    vae_model.fit(x = x_train[:train_size,:], y = x_train[:train_size,:], 
                  shuffle=True, epochs=epochs, 
                  batch_size = batch_size, 
                  #validation_data = (x_valid[:valid_size,:], x_valid[:valid_size,:]),
                  validation_data = (x_test, x_test),
                  verbose = 1)
    
    score_train = vae_model.evaluate(x_valid[:valid_size,:], y = x_valid[:valid_size,:],
                               batch_size = batch_size,
                               verbose = 1)
    
    score_test = vae_model.evaluate(x_test, y = x_test,
                           batch_size = batch_size,
                           verbose = 1)
    y_test_pred = vae_model.predict(x_test, batch_size=batch_size)
    
    y_pred = np.argmax(y_test_pred[:,-2:], axis = 1)
    y_test_1d = np.argmax(y_test.values, axis = 1)
    
    #y_pred[y_pred >= y_test_pred[:,-1].mean()] = 1
    #y_pred[y_pred < y_test_pred[:,-1].mean()] = 0
    
    label_acc = np.mean(np.equal(y_test_1d, y_pred))
    
    scores.append(score(e,f,h,score_train[-1], label_acc)) #score_test[-1]))
    
    curr_pred = pd.DataFrame({"Attack_prob":y_test_pred[:,-2], "Normal_prob":y_test_pred[:,-1]})
    predictions.update({"{}_{}_{}".format(e,f,h):curr_pred})
    
    print("\n Train Acc: {}, Test Acc: {}, Label Acc: {}".format(score_train[-1], 
                                                                 score_test[-1], 
                                                                 label_acc)  )
    
scores = pd.DataFrame(scores)


    

    




 
 Current Layer Attributes - epochs:5 hidden layers:2 features count:4
Train on 112720 samples, validate on 22544 samples
Epoch 1/5
112720/112720 [==============================] - 1s - loss: 549251.6937 - label_accuracy: 0.6286 - val_loss: 37.0582 - val_label_accuracy: 0.4958
Epoch 2/5
112720/112720 [==============================] - 1s - loss: -77.7834 - label_accuracy: 0.6974 - val_loss: 24.3822 - val_label_accuracy: 0.4983
Epoch 3/5
112720/112720 [==============================] - 1s - loss: -85.1396 - label_accuracy: 0.6921 - val_loss: 13.1072 - val_label_accuracy: 0.4982
Epoch 4/5
112720/112720 [==============================] - 1s - loss: -91.8577 - label_accuracy: 0.6993 - val_loss: 6.4570 - val_label_accuracy: 0.4961
Epoch 5/5
112720/112720 [==============================] - 1s - loss: -95.1755 - label_accuracy: 0.7090 - val_loss: 1.2055 - val_label_accuracy: 0.4977
12681/22544 [===============>..............] - ETA: 0s
 Train Acc: 0.7069730386137962, Test Acc: 0.4977377559989691, Label Acc: 0.5900017743080199
 
 Current Layer Attributes - epochs:5 hidden layers:2 features count:16
Train on 112720 samples, validate on 22544 samples
Epoch 1/5
112720/112720 [==============================] - 2s - loss: 320295.0895 - label_accuracy: 0.3245 - val_loss: 6732.7284 - val_label_accuracy: 0.4996
Epoch 2/5
112720/112720 [==============================] - 1s - loss: 1355.1774 - label_accuracy: 0.2335 - val_loss: 2758.3816 - val_label_accuracy: 0.5001
Epoch 3/5
112720/112720 [==============================] - 1s - loss: 164.4455 - label_accuracy: 0.2634 - val_loss: 1710.1167 - val_label_accuracy: 0.5003
Epoch 4/5
112720/112720 [==============================] - 1s - loss: 108.5112 - label_accuracy: 0.2927 - val_loss: 1508.4472 - val_label_accuracy: 0.4993
Epoch 5/5
112720/112720 [==============================] - 1s - loss: 76.1236 - label_accuracy: 0.3117 - val_loss: 1342.7041 - val_label_accuracy: 0.5017
11272/11272 [==============================] - 0s     
12681/22544 [===============>..............] - ETA: 0s
 Train Acc: 0.3236337788403034, Test Acc: 0.5016855839639902, Label Acc: 0.2650816181689141
 
 Current Layer Attributes - epochs:5 hidden layers:2 features count:32
Train on 112720 samples, validate on 22544 samples
Epoch 1/5
112720/112720 [==============================] - 2s - loss: 27590435428.4190 - label_accuracy: 0.4702 - val_loss: 2826661695.0514 - val_label_accuracy: 0.5080
Epoch 2/5
112720/112720 [==============================] - 1s - loss: 8401.0486 - label_accuracy: 0.5042 - val_loss: 2604507675.1679 - val_label_accuracy: 0.5082
Epoch 3/5
112720/112720 [==============================] - 1s - loss: 6998.3304 - label_accuracy: 0.5245 - val_loss: 2606410809.8167 - val_label_accuracy: 0.5086
Epoch 4/5
112720/112720 [==============================] - 1s - loss: 6693.6128 - label_accuracy: 0.5257 - val_loss: 2419253656.5563 - val_label_accuracy: 0.5086
Epoch 5/5
112720/112720 [==============================] - 1s - loss: 5793.2562 - label_accuracy: 0.5280 - val_loss: 2092639519.3498 - val_label_accuracy: 0.5084
11272/11272 [==============================] - 0s     
21135/22544 [===========================>..] - ETA: 0s
 Train Acc: 0.534332849085331, Test Acc: 0.5083836056292057, Label Acc: 0.600026614620298
 
 Current Layer Attributes - epochs:5 hidden layers:6 features count:4
Train on 112720 samples, validate on 22544 samples
Epoch 1/5
112720/112720 [==============================] - 3s - loss: -52.2158 - label_accuracy: 0.5341 - val_loss: -15.0253 - val_label_accuracy: 0.4924
Epoch 2/5
112720/112720 [==============================] - 3s - loss: -75.6186 - label_accuracy: 0.5362 - val_loss: -26.1819 - val_label_accuracy: 0.4941
Epoch 3/5
112720/112720 [==============================] - 2s - loss: -81.5942 - label_accuracy: 0.5550 - val_loss: -36.6427 - val_label_accuracy: 0.4953
Epoch 4/5
112720/112720 [==============================] - 2s - loss: -84.3114 - label_accuracy: 0.5543 - val_loss: -40.9316 - val_label_accuracy: 0.4931
Epoch 5/5
112720/112720 [==============================] - 3s - loss: -85.7987 - label_accuracy: 0.5649 - val_loss: -43.1714 - val_label_accuracy: 0.4945
18317/22544 [=======================>......] - ETA: 0s
 Train Acc: 0.5699964463710785, Test Acc: 0.4944996479898691, Label Acc: 0.5455997161107168
 
 Current Layer Attributes - epochs:5 hidden layers:6 features count:16
Train on 112720 samples, validate on 22544 samples
Epoch 1/5
112720/112720 [==============================] - 4s - loss: -54.5823 - label_accuracy: 0.5333 - val_loss: -44.9367 - val_label_accuracy: 0.4924
Epoch 2/5
112720/112720 [==============================] - 3s - loss: -82.4535 - label_accuracy: 0.5348 - val_loss: -53.1474 - val_label_accuracy: 0.4924
Epoch 3/5
112720/112720 [==============================] - 3s - loss: -90.8288 - label_accuracy: 0.5348 - val_loss: -60.0877 - val_label_accuracy: 0.4924
Epoch 4/5
112720/112720 [==============================] - 3s - loss: -96.6670 - label_accuracy: 0.5348 - val_loss: -63.4357 - val_label_accuracy: 0.4925
Epoch 5/5
112720/112720 [==============================] - 3s - loss: -101.5171 - label_accuracy: 0.5377 - val_loss: -65.2250 - val_label_accuracy: 0.4926
18317/22544 [=======================>......] - ETA: 0s
 Train Acc: 0.5368168950080872, Test Acc: 0.492636626586318, Label Acc: 0.43275372604684176
 
 Current Layer Attributes - epochs:5 hidden layers:6 features count:32
Train on 112720 samples, validate on 22544 samples
Epoch 1/5
112720/112720 [==============================] - 4s - loss: -16.5453 - label_accuracy: 0.5348 - val_loss: -27.5802 - val_label_accuracy: 0.4924
Epoch 2/5
112720/112720 [==============================] - 3s - loss: -59.1021 - label_accuracy: 0.5348 - val_loss: -46.0821 - val_label_accuracy: 0.4924
Epoch 3/5
112720/112720 [==============================] - 3s - loss: -72.8910 - label_accuracy: 0.5348 - val_loss: -55.2629 - val_label_accuracy: 0.4924
Epoch 4/5
112720/112720 [==============================] - 3s - loss: -81.0036 - label_accuracy: 0.5348 - val_loss: -63.3003 - val_label_accuracy: 0.4924
Epoch 5/5
112720/112720 [==============================] - 3s - loss: -83.8671 - label_accuracy: 0.5348 - val_loss: -64.1391 - val_label_accuracy: 0.4924
18317/22544 [=======================>......] - ETA: 0s
 Train Acc: 0.5323811247944832, Test Acc: 0.4924148377031088, Label Acc: 0.43075762952448543
 
 Current Layer Attributes - epochs:5 hidden layers:10 features count:4
Train on 112720 samples, validate on 22544 samples
Epoch 1/5
112720/112720 [==============================] - 6s - loss: -11.5011 - label_accuracy: 0.4670 - val_loss: 8.4963 - val_label_accuracy: 0.5076
Epoch 2/5
112720/112720 [==============================] - 4s - loss: -40.0283 - label_accuracy: 0.4652 - val_loss: -4.6428 - val_label_accuracy: 0.5076
Epoch 3/5
112720/112720 [==============================] - 4s - loss: -50.5349 - label_accuracy: 0.4652 - val_loss: -12.7535 - val_label_accuracy: 0.5076
Epoch 4/5
112720/112720 [==============================] - 4s - loss: -60.1923 - label_accuracy: 0.4652 - val_loss: -15.2684 - val_label_accuracy: 0.5076
Epoch 5/5
112720/112720 [==============================] - 4s - loss: -62.5530 - label_accuracy: 0.5556 - val_loss: -18.6698 - val_label_accuracy: 0.5053
22544/22544 [==============================] - 0s     

 Train Acc: 0.7260468378663063, Test Acc: 0.5053229220211506, Label Acc: 0.7853087295954577
 
 Current Layer Attributes - epochs:5 hidden layers:10 features count:16
Train on 112720 samples, validate on 22544 samples
Epoch 1/5
112720/112720 [==============================] - 6s - loss: -4.3471 - label_accuracy: 0.4652 - val_loss: -9.4818 - val_label_accuracy: 0.5076
Epoch 2/5
112720/112720 [==============================] - 4s - loss: -34.6864 - label_accuracy: 0.4652 - val_loss: -29.9101 - val_label_accuracy: 0.5076
Epoch 3/5
112720/112720 [==============================] - 4s - loss: -54.6766 - label_accuracy: 0.4652 - val_loss: -42.2328 - val_label_accuracy: 0.5076
Epoch 4/5
112720/112720 [==============================] - 4s - loss: -62.4728 - label_accuracy: 0.4652 - val_loss: -51.7289 - val_label_accuracy: 0.5076
Epoch 5/5
112720/112720 [==============================] - 4s - loss: -65.5733 - label_accuracy: 0.4652 - val_loss: -52.1082 - val_label_accuracy: 0.5076
22544/22544 [==============================] - 0s     

 Train Acc: 0.4676188789308071, Test Acc: 0.5075851697474718, Label Acc: 0.5692423704755145
 
 Current Layer Attributes - epochs:5 hidden layers:10 features count:32
Train on 112720 samples, validate on 22544 samples
Epoch 1/5
112720/112720 [==============================] - 6s - loss: -22.7504 - label_accuracy: 0.4652 - val_loss: -26.8817 - val_label_accuracy: 0.5076
Epoch 2/5
112720/112720 [==============================] - 4s - loss: -38.2685 - label_accuracy: 0.4652 - val_loss: -27.0299 - val_label_accuracy: 0.5076
Epoch 3/5
112720/112720 [==============================] - 4s - loss: -41.7494 - label_accuracy: 0.4652 - val_loss: -28.4693 - val_label_accuracy: 0.5076
Epoch 4/5
112720/112720 [==============================] - 4s - loss: -43.8023 - label_accuracy: 0.4652 - val_loss: -26.2163 - val_label_accuracy: 0.5076
Epoch 5/5
112720/112720 [==============================] - 4s - loss: -46.0454 - label_accuracy: 0.4652 - val_loss: -30.1479 - val_label_accuracy: 0.5076
22544/22544 [==============================] - 0s     

 Train Acc: 0.4676188789308071, Test Acc: 0.5075851697474718, Label Acc: 0.5692423704755145

In [5]:

    

scores.sort_values("test_score", ascending=False)


    

    
Out[5]:






  

    

      

      
epoch
      
no_of_features
      
hidden_layers
      
train_score
      
test_score
    
  
  

    

      
6
      
5
      
4
      
10
      
0.726047
      
0.785309
    
    

      
2
      
5
      
32
      
2
      
0.534333
      
0.600027
    
    

      
0
      
5
      
4
      
2
      
0.706973
      
0.590002
    
    

      
7
      
5
      
16
      
10
      
0.467619
      
0.569242
    
    

      
8
      
5
      
32
      
10
      
0.467619
      
0.569242
    
    

      
3
      
5
      
4
      
6
      
0.569996
      
0.545600
    
    

      
4
      
5
      
16
      
6
      
0.536817
      
0.432754
    
    

      
5
      
5
      
32
      
6
      
0.532381
      
0.430758
    
    

      
1
      
5
      
16
      
2
      
0.323634
      
0.265082
    
  

In [6]:

    

pd.Panel(predictions).to_pickle("dataset/keras_vae_with_label_nsl_kdd_predictions.pkl")
scores.to_pickle("dataset/keras_vae_with_label_nsl_kdd_scores.pkl")