Read Data Sample

In [1]:

    

import pandas as pd
import numpy as np
import os
import time
from collections import namedtuple
pd.set_option("display.max_rows",35)
%matplotlib inline


    

In [2]:

    

class dataset:
    kdd_train_2labels = pd.read_pickle("dataset/kdd_train_2labels.pkl")
    kdd_test_2labels = pd.read_pickle("dataset/kdd_test_2labels.pkl")
    kdd_test__2labels = pd.read_pickle("dataset/kdd_test__2labels.pkl")
    
    kdd_train_5labels = pd.read_pickle("dataset/kdd_train_5labels.pkl")
    kdd_test_5labels = pd.read_pickle("dataset/kdd_test_5labels.pkl")


    

In [3]:

    

dataset.kdd_train_2labels.shape


    

    
Out[3]:





(125973, 124)

In [4]:

    

dataset.kdd_test_2labels.shape


    

    
Out[4]:





(22544, 124)

In [5]:

    

from sklearn import model_selection as ms
from sklearn import preprocessing as pp

class preprocess:
    
    output_columns_2labels = ['is_Normal','is_Attack']
    
    x_input = dataset.kdd_train_2labels.drop(output_columns_2labels, axis = 1)
    y_output = dataset.kdd_train_2labels.loc[:,output_columns_2labels]

    x_test_input = dataset.kdd_test_2labels.drop(output_columns_2labels, axis = 1)
    y_test = dataset.kdd_test_2labels.loc[:,output_columns_2labels]
    
    x_test__input = dataset.kdd_test__2labels.drop(output_columns_2labels, axis = 1)
    y_test_ = dataset.kdd_test__2labels.loc[:,output_columns_2labels]

    ss = pp.StandardScaler()

    x_train = ss.fit_transform(x_input)
    x_test = ss.transform(x_test_input)
    x_test_ = ss.transform(x_test__input)

    y_train = y_output.values
    y_test = y_test.values
    y_test_ = y_test_.values

    
preprocess.x_train.std()


    

    
Out[5]:





0.99589320646770185

In [6]:

    

import tensorflow as tf
from tensorflow.contrib.legacy_seq2seq.python.ops.seq2seq import basic_rnn_seq2seq
from tensorflow.contrib.rnn import RNNCell, GRUCell, MultiRNNCell


    

In [7]:

    

class network(object):
    
    input_dim = 122
    classes = 2
    hidden_encoder_dim = 122
    hidden_layers = 1
    latent_dim = 10

    hidden_decoder_dim = 122
    lam = 0.01
    
    def __init__(self, classes, hidden_layers, num_of_features):
        self.classes = classes
        self.hidden_layers = hidden_layers
        self.latent_dim = num_of_features
            
    def build_layers(self):
        tf.reset_default_graph()
        #learning_rate = tf.Variable(initial_value=0.001)

        input_dim = self.input_dim
        classes = self.classes
        hidden_encoder_dim = self.hidden_encoder_dim
        hidden_layers = self.hidden_layers
        latent_dim = self.latent_dim
        hidden_decoder_dim = self.hidden_decoder_dim
        lam = self.lam
        
        with tf.variable_scope("Input"):
            self.x_input = tf.placeholder("float", shape=[None, 1, input_dim])
            self.y_input_ = tf.placeholder("float", shape=[None, 1, classes])
            self.keep_prob = tf.placeholder("float")
            self.lr = tf.placeholder("float")
            self.x_list = tf.unstack(self.x_input, axis= 1)
            self.y_list_ = tf.unstack(self.y_input_, axis = 1)
            self.y_ = self.y_list_[0]
            
            #GO = tf.fill((tf.shape(self.x)[0], 1), 0.5)
            
            #y_with_GO = tf.stack([self.y_, GO])
            
        with tf.variable_scope("lstm"):
            multi_cell = MultiRNNCell([GRUCell(input_dim) for i in range(hidden_layers)] )
            
            self.y, states = basic_rnn_seq2seq(self.x_list, self.y_list_, multi_cell)
            #self.y = tf.slice(self.y, [0, 0], [-1,2])
            
            #self.out = tf.squeeze(self.y)
            
            #self.y = tf.layers.dense(self.y[0], classes, activation = None)
            
            self.y = tf.slice(self.y[0], [0, 0], [-1,2])
            
        with tf.variable_scope("Loss"):
            
            self.regularized_loss = tf.losses.mean_squared_error(self.y_, self.y)
            correct_prediction = tf.equal(tf.argmax(self.y_, 1), tf.argmax(self.y, 1))
            self.tf_accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name = "Accuracy")

        with tf.variable_scope("Optimizer"):
            learning_rate=self.lr
            optimizer = tf.train.AdamOptimizer(learning_rate)
            gradients, variables = zip(*optimizer.compute_gradients(self.regularized_loss))
            gradients = [
                None if gradient is None else tf.clip_by_value(gradient, -1, 1)
                for gradient in gradients]
            self.train_op = optimizer.apply_gradients(zip(gradients, variables))
            #self.train_op = optimizer.minimize(self.regularized_loss)
            
        # add op for merging summary
        #self.summary_op = tf.summary.merge_all()
        self.pred = tf.argmax(self.y, axis = 1)
        self.actual = tf.argmax(self.y_, axis = 1)

        # add Saver ops
        self.saver = tf.train.Saver()


    

In [8]:

    

import collections

class Train:    
    
    result = namedtuple("score", ['epoch', 'no_of_features','hidden_layers','train_score', 'test_score', 'test_score_20', 'time_taken'])

    predictions = {}

    results = []
    best_acc = 0
    best_acc_global = 0

    def train(epochs, net, h,f, lrs):
        batch_iterations = 200
        train_loss = None
        Train.best_acc = 0
        os.makedirs("dataset/tf_GRU_nsl_kdd-orig/hidden layers_{}_features count_{}".format(h,f),
                    exist_ok = True)
        with tf.Session() as sess:
            #summary_writer_train = tf.summary.FileWriter('./logs/kdd/VAE/training', graph=sess.graph)
            #summary_writer_valid = tf.summary.FileWriter('./logs/kdd/VAE/validation')

            sess.run(tf.global_variables_initializer())
            start_time = time.perf_counter()
            
            accuracy, pred_value, actual_value, y_pred = sess.run([net.tf_accuracy, 
                                                                   net.pred, 
                                                                   net.actual, net.y], 
                                                                  feed_dict={net.x_input: preprocess.x_test[:,np.newaxis,:], 
                                                                             net.y_input_: preprocess.y_test[:,np.newaxis,:], 
                                                                             net.keep_prob:1})
            
            print("Initial Accuracy, before training: {}".format(accuracy))
            
            for c, lr in enumerate(lrs):
                for epoch in range(1, (epochs+1)):
                    x_train, x_valid, y_train, y_valid, = ms.train_test_split(preprocess.x_train, 
                                                                              preprocess.y_train, 
                                                                              test_size=0.1)
                    batch_indices = np.array_split(np.arange(x_train.shape[0]), 
                                               batch_iterations)

                    for i in batch_indices:

                        _, train_loss = sess.run([net.train_op, net.regularized_loss], #net.summary_op
                                                              feed_dict={net.x_input: x_train[i,np.newaxis,:], 
                                                                         net.y_input_: y_train[i,np.newaxis,:], 
                                                                         net.keep_prob:1, net.lr:lr})
                        #summary_writer_train.add_summary(summary_str, epoch)
                        if(train_loss > 1e9):
                            print("Step {} | Training Loss: {:.6f}".format(epoch, train_loss))


                    valid_accuracy,valid_loss = sess.run([net.tf_accuracy, net.regularized_loss], #net.summary_op 
                                                          feed_dict={net.x_input: x_valid[:,np.newaxis,:], 
                                                                     net.y_input_: y_valid[:,np.newaxis,:], 
                                                                     net.keep_prob:1, net.lr:lr})
                    #summary_writer_valid.add_summary(summary_str, epoch)



                    accuracy, pred_value, actual_value, y_pred = sess.run([net.tf_accuracy, 
                                                                   net.pred, 
                                                                   net.actual, net.y], 
                                                                  feed_dict={net.x_input: preprocess.x_test[:,np.newaxis,:], 
                                                                             net.y_input_: preprocess.y_test[:,np.newaxis,:], 
                                                                             net.keep_prob:1, net.lr:lr})
                    
                    accuracy_, pred_value_, actual_value_, y_pred_ = sess.run([net.tf_accuracy, 
                                                                   net.pred, 
                                                                   net.actual, net.y], 
                                                                  feed_dict={net.x_input: preprocess.x_test_[:,np.newaxis,:], 
                                                                             net.y_input_: preprocess.y_test_[:,np.newaxis,:], 
                                                                             net.keep_prob:1, net.lr:lr})

                    print("Step {} | Training Loss: {:.6f} | Train Accuracy: {:.6f} | Test Accuracy: {:.6f}, {:.6f}".format(epoch, train_loss, valid_accuracy, accuracy, accuracy_))

                    if accuracy > Train.best_acc_global:
                                Train.best_acc_global = accuracy
                                Train.pred_value = pred_value
                                Train.actual_value = actual_value
                                Train.pred_value_ = pred_value_
                                Train.actual_value_ = actual_value_
                                Train.best_parameters = "Hidden Layers:{}, Features Count:{}".format(h, f)

                    if accuracy > Train.best_acc:

                        #net.saver.save(sess, "dataset/tf_vae_only_nsl_kdd_hidden layers_{}_features count_{}".format(epochs,h,f))
                        #Train.results.append(Train.result(epochs, f, h,valid_accuracy, accuracy))
                        #curr_pred = pd.DataFrame({"Attack_prob":y_pred[:,-2], "Normal_prob":y_pred[:, -1]})
                        #Train.predictions.update({"{}_{}_{}".format(epochs,f,h):curr_pred})

                        Train.best_acc = accuracy
                        if not (np.isnan(train_loss)):
                            net.saver.save(sess, 
                                       "dataset/tf_GRU_nsl_kdd-orig/hidden layers_{}_features count_{}/model"
                                       .format(h,f), 
                                       global_step = epoch, 
                                       write_meta_graph=False)

                        curr_pred = pd.DataFrame({"Attack_prob":y_pred[:,-2], "Normal_prob":y_pred[:, -1], "Prediction":pred_value})
                        Train.predictions.update({"{}_{}_{}".format((epochs+1)* (c+1),f,h):
                                                  (curr_pred, 
                                                   Train.result((epochs+1)*(c+1), f, h,valid_accuracy, accuracy, accuracy_, time.perf_counter() - start_time))})


    

In [9]:

    

import itertools
class Hyperparameters:
#    features_arr = [2, 4, 8, 16, 32, 64, 128, 256]
#    hidden_layers_arr = [2, 4, 6, 10]

    def start_training():

        global df_results
        global past_scores
        
        Train.predictions = {}
        Train.results = []
        
        features_arr = [1] #[4, 8, 16, 32]
        hidden_layers_arr = [1, 3, 5]

        epochs = [5]
        lrs = [1e-2, 1e-2/2, 1e-2/4]

        for e, h, f in itertools.product(epochs, hidden_layers_arr, features_arr):
            print("Current Layer Attributes - epochs:{} hidden layers:{} features count:{}".format(e,h,f))
            n = network(2,h,f)
            n.build_layers()
            Train.train(e, n, h,f, lrs)
            
        dict1 = {}
        dict2 = []
        for k, (v1, v2) in Train.predictions.items():
            dict1.update({k: v1})
            dict2.append(v2)
            
        Train.predictions = dict1
        Train.results = dict2
        df_results = pd.DataFrame(Train.results)
        temp = df_results.set_index(['no_of_features', 'hidden_layers'])

        if not os.path.isfile('dataset/tf_GRU_nsl_kdd-orig_all.pkl'):
            past_scores = temp
        else:
            past_scores = pd.read_pickle("dataset/tf_GRU_nsl_kdd-orig_all.pkl")

        past_scores.append(temp).to_pickle("dataset/tf_GRU_nsl_kdd-orig_all.pkl")


    

In [10]:

    

%%timeit -r 10

Hyperparameters.start_training()


    

    




Current Layer Attributes - epochs:5 hidden layers:1 features count:1
Initial Accuracy, before training: 0.7715134620666504
Step 1 | Training Loss: 0.002247 | Train Accuracy: 0.997857 | Test Accuracy: 0.800124, 0.619747
Step 2 | Training Loss: 0.001798 | Train Accuracy: 0.999524 | Test Accuracy: 0.878815, 0.769451
Step 3 | Training Loss: 0.000320 | Train Accuracy: 0.999921 | Test Accuracy: 0.913591, 0.835612
Step 4 | Training Loss: 0.000357 | Train Accuracy: 1.000000 | Test Accuracy: 0.932798, 0.872152
Step 5 | Training Loss: 0.000136 | Train Accuracy: 1.000000 | Test Accuracy: 0.937278, 0.880675
Step 1 | Training Loss: 0.000078 | Train Accuracy: 1.000000 | Test Accuracy: 0.955465, 0.915274
Step 2 | Training Loss: 0.000383 | Train Accuracy: 1.000000 | Test Accuracy: 0.957417, 0.918987
Step 3 | Training Loss: 0.000069 | Train Accuracy: 1.000000 | Test Accuracy: 0.957505, 0.919156
Step 4 | Training Loss: 0.000093 | Train Accuracy: 1.000000 | Test Accuracy: 0.957727, 0.919578
Step 5 | Training Loss: 0.000049 | Train Accuracy: 1.000000 | Test Accuracy: 0.957638, 0.919409
Step 1 | Training Loss: 0.000055 | Train Accuracy: 1.000000 | Test Accuracy: 0.957816, 0.919747
Step 2 | Training Loss: 0.000026 | Train Accuracy: 1.000000 | Test Accuracy: 0.957993, 0.920084
Step 3 | Training Loss: 0.000047 | Train Accuracy: 1.000000 | Test Accuracy: 0.958038, 0.920169
Step 4 | Training Loss: 0.000016 | Train Accuracy: 1.000000 | Test Accuracy: 0.958171, 0.920422
Step 5 | Training Loss: 0.000026 | Train Accuracy: 1.000000 | Test Accuracy: 0.958215, 0.920506
Current Layer Attributes - epochs:5 hidden layers:3 features count:1
Initial Accuracy, before training: 0.3616483211517334
Step 1 | Training Loss: 0.000003 | Train Accuracy: 0.999841 | Test Accuracy: 0.998181, 0.996540
Step 2 | Training Loss: 0.000182 | Train Accuracy: 0.999762 | Test Accuracy: 0.977954, 0.958059
Step 3 | Training Loss: 0.000003 | Train Accuracy: 0.999921 | Test Accuracy: 0.999423, 0.998903
Step 4 | Training Loss: 0.000001 | Train Accuracy: 1.000000 | Test Accuracy: 0.999823, 0.999662
Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999867, 0.999747
Step 1 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999911, 0.999831
Step 2 | Training Loss: 0.000001 | Train Accuracy: 1.000000 | Test Accuracy: 0.999911, 0.999831
Step 3 | Training Loss: 0.000004 | Train Accuracy: 1.000000 | Test Accuracy: 0.999911, 0.999831
Step 4 | Training Loss: 0.000001 | Train Accuracy: 1.000000 | Test Accuracy: 0.999911, 0.999831
Step 5 | Training Loss: 0.000004 | Train Accuracy: 1.000000 | Test Accuracy: 0.999556, 0.999156
Step 1 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999823, 0.999662
Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999823, 0.999662
Step 3 | Training Loss: 0.000800 | Train Accuracy: 1.000000 | Test Accuracy: 0.999823, 0.999662
Step 4 | Training Loss: 0.000001 | Train Accuracy: 1.000000 | Test Accuracy: 0.999823, 0.999662
Step 5 | Training Loss: 0.000001 | Train Accuracy: 1.000000 | Test Accuracy: 0.999823, 0.999662
Current Layer Attributes - epochs:5 hidden layers:5 features count:1
Initial Accuracy, before training: 0.5675567984580994
Step 1 | Training Loss: 0.020722 | Train Accuracy: 0.972218 | Test Accuracy: 0.760335, 0.546751
Step 2 | Training Loss: 0.020305 | Train Accuracy: 0.978489 | Test Accuracy: 0.772489, 0.568354
Step 3 | Training Loss: 0.013774 | Train Accuracy: 0.977457 | Test Accuracy: 0.750444, 0.525992
Step 4 | Training Loss: 0.026087 | Train Accuracy: 0.978171 | Test Accuracy: 0.776304, 0.574430
Step 5 | Training Loss: 0.005209 | Train Accuracy: 0.997460 | Test Accuracy: 0.894029, 0.798397
Step 1 | Training Loss: 0.001756 | Train Accuracy: 0.997460 | Test Accuracy: 0.904143, 0.817637
Step 2 | Training Loss: 0.003468 | Train Accuracy: 0.999047 | Test Accuracy: 0.899885, 0.809536
Step 3 | Training Loss: 0.002058 | Train Accuracy: 0.999127 | Test Accuracy: 0.908268, 0.825485
Step 4 | Training Loss: 0.000006 | Train Accuracy: 0.999841 | Test Accuracy: 0.959945, 0.923797
Step 5 | Training Loss: 0.000029 | Train Accuracy: 0.999762 | Test Accuracy: 0.962473, 0.928608
Step 1 | Training Loss: 0.001767 | Train Accuracy: 0.999682 | Test Accuracy: 0.962340, 0.928354
Step 2 | Training Loss: 0.000057 | Train Accuracy: 0.999762 | Test Accuracy: 0.961852, 0.927426
Step 3 | Training Loss: 0.000251 | Train Accuracy: 0.999841 | Test Accuracy: 0.961985, 0.927679
Step 4 | Training Loss: 0.000000 | Train Accuracy: 0.999762 | Test Accuracy: 0.962074, 0.927848
Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.972498, 0.947679
Current Layer Attributes - epochs:5 hidden layers:1 features count:1
Initial Accuracy, before training: 0.6001153588294983
Step 1 | Training Loss: 0.006357 | Train Accuracy: 0.990713 | Test Accuracy: 0.804161, 0.627426
Step 2 | Training Loss: 0.000876 | Train Accuracy: 0.999762 | Test Accuracy: 0.900373, 0.810464
Step 3 | Training Loss: 0.000928 | Train Accuracy: 0.999841 | Test Accuracy: 0.925080, 0.857468
Step 4 | Training Loss: 0.000142 | Train Accuracy: 0.999921 | Test Accuracy: 0.934972, 0.876287
Step 5 | Training Loss: 0.000247 | Train Accuracy: 0.999841 | Test Accuracy: 0.937278, 0.880675
Step 1 | Training Loss: 0.000080 | Train Accuracy: 0.999841 | Test Accuracy: 0.938520, 0.883038
Step 2 | Training Loss: 0.000068 | Train Accuracy: 1.000000 | Test Accuracy: 0.939141, 0.884219
Step 3 | Training Loss: 0.000042 | Train Accuracy: 1.000000 | Test Accuracy: 0.939319, 0.884557
Step 4 | Training Loss: 0.000091 | Train Accuracy: 0.999921 | Test Accuracy: 0.940206, 0.886245
Step 5 | Training Loss: 0.000976 | Train Accuracy: 0.999841 | Test Accuracy: 0.940339, 0.886498
Step 1 | Training Loss: 0.000063 | Train Accuracy: 0.999682 | Test Accuracy: 0.941004, 0.887764
Step 2 | Training Loss: 0.000052 | Train Accuracy: 0.999841 | Test Accuracy: 0.958747, 0.921519
Step 3 | Training Loss: 0.000053 | Train Accuracy: 1.000000 | Test Accuracy: 0.958747, 0.921519
Step 4 | Training Loss: 0.000011 | Train Accuracy: 0.999921 | Test Accuracy: 0.958481, 0.921013
Step 5 | Training Loss: 0.000906 | Train Accuracy: 0.999921 | Test Accuracy: 0.958614, 0.921266
Current Layer Attributes - epochs:5 hidden layers:3 features count:1
Initial Accuracy, before training: 0.6293026804924011
Step 1 | Training Loss: 0.000011 | Train Accuracy: 0.999921 | Test Accuracy: 0.999690, 0.999409
Step 2 | Training Loss: 0.000003 | Train Accuracy: 0.999921 | Test Accuracy: 0.999512, 0.999072
Step 3 | Training Loss: 0.000001 | Train Accuracy: 0.999841 | Test Accuracy: 0.999512, 0.999072
Step 4 | Training Loss: 0.000001 | Train Accuracy: 0.999921 | Test Accuracy: 0.999690, 0.999409
Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999444 | Test Accuracy: 0.998802, 0.998987
Step 1 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999734, 0.999494
Step 2 | Training Loss: 0.000001 | Train Accuracy: 0.999841 | Test Accuracy: 0.999202, 0.999409
Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999778, 0.999578
Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999734, 0.999494
Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999157, 0.998397
Step 1 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999157, 0.998397
Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999246, 0.998565
Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999335, 0.998734
Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999379, 0.998819
Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999246, 0.998565
Current Layer Attributes - epochs:5 hidden layers:5 features count:1
Initial Accuracy, before training: 0.6438519954681396
Step 1 | Training Loss: 0.011850 | Train Accuracy: 0.985791 | Test Accuracy: 0.814008, 0.647257
Step 2 | Training Loss: 0.007320 | Train Accuracy: 0.986506 | Test Accuracy: 0.823146, 0.665232
Step 3 | Training Loss: 0.016808 | Train Accuracy: 0.984998 | Test Accuracy: 0.822303, 0.662954
Step 4 | Training Loss: 0.005629 | Train Accuracy: 0.994205 | Test Accuracy: 0.910220, 0.829198
Step 5 | Training Loss: 0.000003 | Train Accuracy: 0.999682 | Test Accuracy: 0.973962, 0.950464
Step 1 | Training Loss: 0.000012 | Train Accuracy: 0.999682 | Test Accuracy: 0.984120, 0.969789
Step 2 | Training Loss: 0.000000 | Train Accuracy: 0.999841 | Test Accuracy: 0.974273, 0.951055
Step 3 | Training Loss: 0.000001 | Train Accuracy: 0.999762 | Test Accuracy: 0.975515, 0.953502
Step 4 | Training Loss: 0.000001 | Train Accuracy: 0.999762 | Test Accuracy: 0.974450, 0.951392
Step 5 | Training Loss: 0.000011 | Train Accuracy: 0.999682 | Test Accuracy: 0.974583, 0.951646
Step 1 | Training Loss: 0.001774 | Train Accuracy: 0.999603 | Test Accuracy: 0.984652, 0.970802
Step 2 | Training Loss: 0.000001 | Train Accuracy: 1.000000 | Test Accuracy: 0.984741, 0.971055
Step 3 | Training Loss: 0.000012 | Train Accuracy: 1.000000 | Test Accuracy: 0.985096, 0.971730
Step 4 | Training Loss: 0.000061 | Train Accuracy: 1.000000 | Test Accuracy: 0.984608, 0.970802
Step 5 | Training Loss: 0.000001 | Train Accuracy: 0.999682 | Test Accuracy: 0.984697, 0.970970
Current Layer Attributes - epochs:5 hidden layers:1 features count:1
Initial Accuracy, before training: 0.6022888422012329
Step 1 | Training Loss: 0.004444 | Train Accuracy: 0.997381 | Test Accuracy: 0.797418, 0.614599
Step 2 | Training Loss: 0.000639 | Train Accuracy: 0.999682 | Test Accuracy: 0.838627, 0.692996
Step 3 | Training Loss: 0.000394 | Train Accuracy: 1.000000 | Test Accuracy: 0.895759, 0.801688
Step 4 | Training Loss: 0.000356 | Train Accuracy: 1.000000 | Test Accuracy: 0.927475, 0.862025
Step 5 | Training Loss: 0.000170 | Train Accuracy: 1.000000 | Test Accuracy: 0.931778, 0.870211
Step 1 | Training Loss: 0.000120 | Train Accuracy: 1.000000 | Test Accuracy: 0.932887, 0.872321
Step 2 | Training Loss: 0.000238 | Train Accuracy: 1.000000 | Test Accuracy: 0.933597, 0.873671
Step 3 | Training Loss: 0.000072 | Train Accuracy: 1.000000 | Test Accuracy: 0.933863, 0.874177
Step 4 | Training Loss: 0.000063 | Train Accuracy: 1.000000 | Test Accuracy: 0.933641, 0.873755
Step 5 | Training Loss: 0.000041 | Train Accuracy: 1.000000 | Test Accuracy: 0.934839, 0.876034
Step 1 | Training Loss: 0.000130 | Train Accuracy: 0.999921 | Test Accuracy: 0.934661, 0.875696
Step 2 | Training Loss: 0.000033 | Train Accuracy: 1.000000 | Test Accuracy: 0.935593, 0.877468
Step 3 | Training Loss: 0.000050 | Train Accuracy: 1.000000 | Test Accuracy: 0.938698, 0.883376
Step 4 | Training Loss: 0.000055 | Train Accuracy: 1.000000 | Test Accuracy: 0.940738, 0.887257
Step 5 | Training Loss: 0.000029 | Train Accuracy: 0.999921 | Test Accuracy: 0.942246, 0.890127
Current Layer Attributes - epochs:5 hidden layers:3 features count:1
Initial Accuracy, before training: 0.6971256136894226
Step 1 | Training Loss: 0.001259 | Train Accuracy: 0.999841 | Test Accuracy: 0.979729, 0.961435
Step 2 | Training Loss: 0.000001 | Train Accuracy: 0.999682 | Test Accuracy: 0.981725, 0.965232
Step 3 | Training Loss: 0.000853 | Train Accuracy: 0.999921 | Test Accuracy: 0.983100, 0.967848
Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.983410, 0.968439
Step 5 | Training Loss: 0.000030 | Train Accuracy: 0.999921 | Test Accuracy: 0.999867, 0.999747
Step 1 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999956, 0.999916
Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999956, 0.999916
Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999956, 0.999916
Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999956, 0.999916
Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999956, 0.999916
Step 1 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999956, 0.999916
Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999956, 0.999916
Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999956, 0.999916
Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999956, 0.999916
Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999956, 0.999916
Current Layer Attributes - epochs:5 hidden layers:5 features count:1
Initial Accuracy, before training: 0.5984740853309631
Step 1 | Training Loss: 0.014215 | Train Accuracy: 0.977933 | Test Accuracy: 0.800080, 0.621097
Step 2 | Training Loss: 0.016396 | Train Accuracy: 0.984601 | Test Accuracy: 0.783046, 0.587257
Step 3 | Training Loss: 0.000149 | Train Accuracy: 0.999127 | Test Accuracy: 0.900772, 0.811224
Step 4 | Training Loss: 0.001780 | Train Accuracy: 0.998571 | Test Accuracy: 0.975603, 0.953587
Step 5 | Training Loss: 0.000003 | Train Accuracy: 0.999444 | Test Accuracy: 0.975958, 0.954262
Step 1 | Training Loss: 0.000002 | Train Accuracy: 0.999444 | Test Accuracy: 0.977866, 0.957890
Step 2 | Training Loss: 0.000001 | Train Accuracy: 0.999682 | Test Accuracy: 0.978220, 0.958565
Step 3 | Training Loss: 0.003521 | Train Accuracy: 0.999682 | Test Accuracy: 0.978220, 0.958565
Step 4 | Training Loss: 0.000001 | Train Accuracy: 0.999603 | Test Accuracy: 0.978531, 0.959156
Step 5 | Training Loss: 0.000001 | Train Accuracy: 0.999682 | Test Accuracy: 0.979773, 0.961519
Step 1 | Training Loss: 0.000001 | Train Accuracy: 0.999365 | Test Accuracy: 0.981769, 0.965316
Step 2 | Training Loss: 0.001761 | Train Accuracy: 0.999682 | Test Accuracy: 0.981902, 0.965570
Step 3 | Training Loss: 0.001767 | Train Accuracy: 0.999444 | Test Accuracy: 0.981902, 0.965570
Step 4 | Training Loss: 0.000001 | Train Accuracy: 0.999921 | Test Accuracy: 0.981946, 0.965654
Step 5 | Training Loss: 0.001730 | Train Accuracy: 0.999841 | Test Accuracy: 0.981946, 0.965654
Current Layer Attributes - epochs:5 hidden layers:1 features count:1
Initial Accuracy, before training: 0.6804915070533752
Step 1 | Training Loss: 0.001360 | Train Accuracy: 0.998809 | Test Accuracy: 0.831973, 0.680338
Step 2 | Training Loss: 0.000259 | Train Accuracy: 0.999841 | Test Accuracy: 0.907115, 0.823291
Step 3 | Training Loss: 0.000170 | Train Accuracy: 1.000000 | Test Accuracy: 0.930403, 0.867595
Step 4 | Training Loss: 0.000098 | Train Accuracy: 1.000000 | Test Accuracy: 0.938343, 0.882700
Step 5 | Training Loss: 0.000093 | Train Accuracy: 1.000000 | Test Accuracy: 0.934084, 0.874599
Step 1 | Training Loss: 0.000034 | Train Accuracy: 1.000000 | Test Accuracy: 0.935992, 0.878228
Step 2 | Training Loss: 0.000050 | Train Accuracy: 1.000000 | Test Accuracy: 0.937988, 0.882025
Step 3 | Training Loss: 0.000038 | Train Accuracy: 1.000000 | Test Accuracy: 0.937234, 0.880591
Step 4 | Training Loss: 0.000041 | Train Accuracy: 1.000000 | Test Accuracy: 0.955110, 0.914599
Step 5 | Training Loss: 0.000040 | Train Accuracy: 1.000000 | Test Accuracy: 0.955420, 0.915190
Step 1 | Training Loss: 0.000020 | Train Accuracy: 1.000000 | Test Accuracy: 0.955509, 0.915359
Step 2 | Training Loss: 0.000019 | Train Accuracy: 1.000000 | Test Accuracy: 0.956130, 0.916540
Step 3 | Training Loss: 0.000041 | Train Accuracy: 1.000000 | Test Accuracy: 0.956086, 0.916456
Step 4 | Training Loss: 0.000011 | Train Accuracy: 1.000000 | Test Accuracy: 0.956396, 0.917046
Step 5 | Training Loss: 0.000019 | Train Accuracy: 1.000000 | Test Accuracy: 0.956796, 0.917806
Current Layer Attributes - epochs:5 hidden layers:3 features count:1
Initial Accuracy, before training: 0.6314762234687805
Step 1 | Training Loss: 0.000002 | Train Accuracy: 0.999524 | Test Accuracy: 0.997605, 0.995443
Step 2 | Training Loss: 0.000001 | Train Accuracy: 0.999762 | Test Accuracy: 0.997560, 0.995359
Step 3 | Training Loss: 0.000001 | Train Accuracy: 0.999127 | Test Accuracy: 0.982967, 0.967595
Step 4 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.982346, 0.966413
Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.982434, 0.966582
Step 1 | Training Loss: 0.000000 | Train Accuracy: 0.999841 | Test Accuracy: 0.982567, 0.966835
Step 2 | Training Loss: 0.000000 | Train Accuracy: 0.999762 | Test Accuracy: 0.982567, 0.966835
Step 3 | Training Loss: 0.000894 | Train Accuracy: 0.999762 | Test Accuracy: 0.982612, 0.966920
Step 4 | Training Loss: 0.000007 | Train Accuracy: 1.000000 | Test Accuracy: 0.982745, 0.967173
Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.982745, 0.967173
Step 1 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.982745, 0.967173
Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.982745, 0.967173
Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.982745, 0.967173
Step 4 | Training Loss: 0.000003 | Train Accuracy: 1.000000 | Test Accuracy: 0.982745, 0.967173
Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.982745, 0.967173
Current Layer Attributes - epochs:5 hidden layers:5 features count:1
Initial Accuracy, before training: 0.7603353261947632
Step 1 | Training Loss: 0.004781 | Train Accuracy: 0.998492 | Test Accuracy: 0.973208, 0.949367
Step 2 | Training Loss: 0.001764 | Train Accuracy: 0.999762 | Test Accuracy: 0.985672, 0.972743
Step 3 | Training Loss: 0.001769 | Train Accuracy: 0.999444 | Test Accuracy: 0.998270, 0.996709
Step 4 | Training Loss: 0.003466 | Train Accuracy: 0.998889 | Test Accuracy: 0.997738, 0.995696
Step 5 | Training Loss: 0.000002 | Train Accuracy: 0.999841 | Test Accuracy: 0.996939, 0.994177
Step 1 | Training Loss: 0.000001 | Train Accuracy: 0.999841 | Test Accuracy: 0.997560, 0.995359
Step 2 | Training Loss: 0.000001 | Train Accuracy: 0.999762 | Test Accuracy: 0.997383, 0.995021
Step 3 | Training Loss: 0.000001 | Train Accuracy: 0.999921 | Test Accuracy: 0.997826, 0.995865
Step 4 | Training Loss: 0.000000 | Train Accuracy: 0.999603 | Test Accuracy: 0.997826, 0.995865
Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.997826, 0.995865
Step 1 | Training Loss: 0.003532 | Train Accuracy: 0.999762 | Test Accuracy: 0.997826, 0.995865
Step 2 | Training Loss: 0.000000 | Train Accuracy: 0.999762 | Test Accuracy: 0.997826, 0.995865
Step 3 | Training Loss: 0.000000 | Train Accuracy: 0.999841 | Test Accuracy: 0.997826, 0.995865
Step 4 | Training Loss: 0.000000 | Train Accuracy: 0.999762 | Test Accuracy: 0.997826, 0.995865
Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999762 | Test Accuracy: 0.997826, 0.995865
Current Layer Attributes - epochs:5 hidden layers:1 features count:1
Initial Accuracy, before training: 0.26969480514526367
Step 1 | Training Loss: 0.006713 | Train Accuracy: 0.997777 | Test Accuracy: 0.803983, 0.627173
Step 2 | Training Loss: 0.001863 | Train Accuracy: 0.999444 | Test Accuracy: 0.862136, 0.737722
Step 3 | Training Loss: 0.000161 | Train Accuracy: 0.999682 | Test Accuracy: 0.899308, 0.808439
Step 4 | Training Loss: 0.000312 | Train Accuracy: 0.999921 | Test Accuracy: 0.919003, 0.845907
Step 5 | Training Loss: 0.000057 | Train Accuracy: 0.999921 | Test Accuracy: 0.939186, 0.884304
Step 1 | Training Loss: 0.000145 | Train Accuracy: 1.000000 | Test Accuracy: 0.940871, 0.887511
Step 2 | Training Loss: 0.000109 | Train Accuracy: 0.999841 | Test Accuracy: 0.941270, 0.888270
Step 3 | Training Loss: 0.000028 | Train Accuracy: 0.999841 | Test Accuracy: 0.940916, 0.887595
Step 4 | Training Loss: 0.000022 | Train Accuracy: 0.999921 | Test Accuracy: 0.942291, 0.890211
Step 5 | Training Loss: 0.000036 | Train Accuracy: 1.000000 | Test Accuracy: 0.942202, 0.890042
Step 1 | Training Loss: 0.000061 | Train Accuracy: 1.000000 | Test Accuracy: 0.942335, 0.890295
Step 2 | Training Loss: 0.000039 | Train Accuracy: 0.999921 | Test Accuracy: 0.942557, 0.890717
Step 3 | Training Loss: 0.000931 | Train Accuracy: 1.000000 | Test Accuracy: 0.942867, 0.891308
Step 4 | Training Loss: 0.000026 | Train Accuracy: 0.999921 | Test Accuracy: 0.943488, 0.892489
Step 5 | Training Loss: 0.000035 | Train Accuracy: 0.999921 | Test Accuracy: 0.943843, 0.893165
Current Layer Attributes - epochs:5 hidden layers:3 features count:1
Initial Accuracy, before training: 0.5897799730300903
Step 1 | Training Loss: 0.001744 | Train Accuracy: 0.999286 | Test Accuracy: 0.982656, 0.967004
Step 2 | Training Loss: 0.000240 | Train Accuracy: 0.999365 | Test Accuracy: 0.995298, 0.991055
Step 3 | Training Loss: 0.000002 | Train Accuracy: 0.999762 | Test Accuracy: 0.983321, 0.968270
Step 4 | Training Loss: 0.000000 | Train Accuracy: 0.999365 | Test Accuracy: 0.998802, 0.997721
Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999682 | Test Accuracy: 0.998669, 0.997468
Step 1 | Training Loss: 0.003535 | Train Accuracy: 0.999524 | Test Accuracy: 0.998669, 0.997468
Step 2 | Training Loss: 0.000000 | Train Accuracy: 0.999286 | Test Accuracy: 0.998669, 0.997468
Step 3 | Training Loss: 0.000000 | Train Accuracy: 0.999762 | Test Accuracy: 0.998714, 0.997553
Step 4 | Training Loss: 0.000000 | Train Accuracy: 0.999762 | Test Accuracy: 0.998714, 0.997553
Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999603 | Test Accuracy: 0.998758, 0.997637
Step 1 | Training Loss: 0.001767 | Train Accuracy: 0.999603 | Test Accuracy: 0.998758, 0.997637
Step 2 | Training Loss: 0.000000 | Train Accuracy: 0.999603 | Test Accuracy: 0.998758, 0.997637
Step 3 | Training Loss: 0.000000 | Train Accuracy: 0.999603 | Test Accuracy: 0.998758, 0.997637
Step 4 | Training Loss: 0.000000 | Train Accuracy: 0.999682 | Test Accuracy: 0.998758, 0.997637
Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999603 | Test Accuracy: 0.998847, 0.997806
Current Layer Attributes - epochs:5 hidden layers:5 features count:1
Initial Accuracy, before training: 0.5375709533691406
Step 1 | Training Loss: 0.028795 | Train Accuracy: 0.975869 | Test Accuracy: 0.764017, 0.552236
Step 2 | Training Loss: 0.017046 | Train Accuracy: 0.985236 | Test Accuracy: 0.757940, 0.540675
Step 3 | Training Loss: 0.007034 | Train Accuracy: 0.991983 | Test Accuracy: 0.771159, 0.566160
Step 4 | Training Loss: 0.004413 | Train Accuracy: 0.990475 | Test Accuracy: 0.770848, 0.564219
Step 5 | Training Loss: 0.004568 | Train Accuracy: 0.990951 | Test Accuracy: 0.773909, 0.570127
Step 1 | Training Loss: 0.004503 | Train Accuracy: 0.993491 | Test Accuracy: 0.778167, 0.578059
Step 2 | Training Loss: 0.000443 | Train Accuracy: 0.994285 | Test Accuracy: 0.772800, 0.568186
Step 3 | Training Loss: 0.004171 | Train Accuracy: 0.994285 | Test Accuracy: 0.769384, 0.561350
Step 4 | Training Loss: 0.003424 | Train Accuracy: 0.994761 | Test Accuracy: 0.773953, 0.570127
Step 5 | Training Loss: 0.000211 | Train Accuracy: 0.992697 | Test Accuracy: 0.781627, 0.585232
Step 1 | Training Loss: 0.002419 | Train Accuracy: 0.996349 | Test Accuracy: 0.778389, 0.578903
Step 2 | Training Loss: 0.001653 | Train Accuracy: 0.996110 | Test Accuracy: 0.779498, 0.580675
Step 3 | Training Loss: 0.002940 | Train Accuracy: 0.996349 | Test Accuracy: 0.777768, 0.577637
Step 4 | Training Loss: 0.004739 | Train Accuracy: 0.996110 | Test Accuracy: 0.776659, 0.575612
Step 5 | Training Loss: 0.000518 | Train Accuracy: 0.995793 | Test Accuracy: 0.780429, 0.582700
Current Layer Attributes - epochs:5 hidden layers:1 features count:1
Initial Accuracy, before training: 0.3840489685535431
Step 1 | Training Loss: 0.003893 | Train Accuracy: 0.998333 | Test Accuracy: 0.828779, 0.674262
Step 2 | Training Loss: 0.001749 | Train Accuracy: 0.999524 | Test Accuracy: 0.879524, 0.770802
Step 3 | Training Loss: 0.000540 | Train Accuracy: 0.999762 | Test Accuracy: 0.906095, 0.821350
Step 4 | Training Loss: 0.000145 | Train Accuracy: 1.000000 | Test Accuracy: 0.928584, 0.864135
Step 5 | Training Loss: 0.000133 | Train Accuracy: 0.999841 | Test Accuracy: 0.933330, 0.873165
Step 1 | Training Loss: 0.000101 | Train Accuracy: 1.000000 | Test Accuracy: 0.936879, 0.879916
Step 2 | Training Loss: 0.000066 | Train Accuracy: 1.000000 | Test Accuracy: 0.937810, 0.881688
Step 3 | Training Loss: 0.000233 | Train Accuracy: 1.000000 | Test Accuracy: 0.938653, 0.883291
Step 4 | Training Loss: 0.000074 | Train Accuracy: 1.000000 | Test Accuracy: 0.939407, 0.884726
Step 5 | Training Loss: 0.000091 | Train Accuracy: 1.000000 | Test Accuracy: 0.939984, 0.885823
Step 1 | Training Loss: 0.000059 | Train Accuracy: 1.000000 | Test Accuracy: 0.940339, 0.886498
Step 2 | Training Loss: 0.000039 | Train Accuracy: 1.000000 | Test Accuracy: 0.940605, 0.887004
Step 3 | Training Loss: 0.000053 | Train Accuracy: 1.000000 | Test Accuracy: 0.940605, 0.887004
Step 4 | Training Loss: 0.000035 | Train Accuracy: 1.000000 | Test Accuracy: 0.940960, 0.887679
Step 5 | Training Loss: 0.000049 | Train Accuracy: 1.000000 | Test Accuracy: 0.941670, 0.889030
Current Layer Attributes - epochs:5 hidden layers:3 features count:1
Initial Accuracy, before training: 0.5593506097793579
Step 1 | Training Loss: 0.000022 | Train Accuracy: 0.999603 | Test Accuracy: 0.997560, 0.995359
Step 2 | Training Loss: 0.000004 | Train Accuracy: 0.999603 | Test Accuracy: 0.996584, 0.993502
Step 3 | Training Loss: 0.001769 | Train Accuracy: 0.999762 | Test Accuracy: 0.997826, 0.995865
Step 4 | Training Loss: 0.000001 | Train Accuracy: 0.999762 | Test Accuracy: 0.997826, 0.995865
Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.997782, 0.995781
Step 1 | Training Loss: 0.000000 | Train Accuracy: 0.999762 | Test Accuracy: 0.997826, 0.995865
Step 2 | Training Loss: 0.000002 | Train Accuracy: 0.999762 | Test Accuracy: 0.997826, 0.995865
Step 3 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.997826, 0.995865
Step 4 | Training Loss: 0.000000 | Train Accuracy: 0.999682 | Test Accuracy: 0.997826, 0.995865
Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.997826, 0.995865
Step 1 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.997826, 0.995865
Step 2 | Training Loss: 0.000000 | Train Accuracy: 0.999762 | Test Accuracy: 0.997826, 0.995865
Step 3 | Training Loss: 0.000000 | Train Accuracy: 0.999524 | Test Accuracy: 0.997826, 0.995865
Step 4 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.997871, 0.995949
Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.997560, 0.995359
Current Layer Attributes - epochs:5 hidden layers:5 features count:1
Initial Accuracy, before training: 0.6384847164154053
Step 1 | Training Loss: 0.029366 | Train Accuracy: 0.973885 | Test Accuracy: 0.745387, 0.517131
Step 2 | Training Loss: 0.029334 | Train Accuracy: 0.971742 | Test Accuracy: 0.754968, 0.536540
Step 3 | Training Loss: 0.011454 | Train Accuracy: 0.981743 | Test Accuracy: 0.753549, 0.533418
Step 4 | Training Loss: 0.012727 | Train Accuracy: 0.984918 | Test Accuracy: 0.747250, 0.519325
Step 5 | Training Loss: 0.006105 | Train Accuracy: 0.994364 | Test Accuracy: 0.784422, 0.589873
Step 1 | Training Loss: 0.006410 | Train Accuracy: 0.994682 | Test Accuracy: 0.761045, 0.545485
Step 2 | Training Loss: 0.000950 | Train Accuracy: 0.997777 | Test Accuracy: 0.797152, 0.614093
Step 3 | Training Loss: 0.001512 | Train Accuracy: 0.998968 | Test Accuracy: 0.800035, 0.619578
Step 4 | Training Loss: 0.003324 | Train Accuracy: 0.999286 | Test Accuracy: 0.833703, 0.683629
Step 5 | Training Loss: 0.001100 | Train Accuracy: 0.999127 | Test Accuracy: 0.856946, 0.727848
Step 1 | Training Loss: 0.000005 | Train Accuracy: 0.999682 | Test Accuracy: 0.938121, 0.882279
Step 2 | Training Loss: 0.000017 | Train Accuracy: 0.999444 | Test Accuracy: 0.969349, 0.941688
Step 3 | Training Loss: 0.000013 | Train Accuracy: 0.999603 | Test Accuracy: 0.968772, 0.940591
Step 4 | Training Loss: 0.000007 | Train Accuracy: 0.999841 | Test Accuracy: 0.937456, 0.881013
Step 5 | Training Loss: 0.000008 | Train Accuracy: 0.999682 | Test Accuracy: 0.968905, 0.940844
Current Layer Attributes - epochs:5 hidden layers:1 features count:1
Initial Accuracy, before training: 0.588449239730835
Step 1 | Training Loss: 0.006069 | Train Accuracy: 0.990872 | Test Accuracy: 0.798838, 0.617975
Step 2 | Training Loss: 0.001144 | Train Accuracy: 0.999841 | Test Accuracy: 0.878238, 0.768354
Step 3 | Training Loss: 0.000193 | Train Accuracy: 0.999762 | Test Accuracy: 0.937367, 0.880844
Step 4 | Training Loss: 0.000919 | Train Accuracy: 1.000000 | Test Accuracy: 0.982168, 0.966076
Step 5 | Training Loss: 0.000042 | Train Accuracy: 1.000000 | Test Accuracy: 0.970059, 0.943038
Step 1 | Training Loss: 0.000035 | Train Accuracy: 1.000000 | Test Accuracy: 0.979995, 0.961941
Step 2 | Training Loss: 0.000042 | Train Accuracy: 1.000000 | Test Accuracy: 0.967175, 0.937553
Step 3 | Training Loss: 0.000023 | Train Accuracy: 1.000000 | Test Accuracy: 0.962074, 0.927848
Step 4 | Training Loss: 0.000019 | Train Accuracy: 1.000000 | Test Accuracy: 0.960477, 0.924810
Step 5 | Training Loss: 0.000026 | Train Accuracy: 1.000000 | Test Accuracy: 0.961364, 0.926498
Step 1 | Training Loss: 0.000022 | Train Accuracy: 1.000000 | Test Accuracy: 0.960743, 0.925316
Step 2 | Training Loss: 0.000023 | Train Accuracy: 1.000000 | Test Accuracy: 0.959945, 0.923797
Step 3 | Training Loss: 0.000010 | Train Accuracy: 1.000000 | Test Accuracy: 0.959147, 0.922278
Step 4 | Training Loss: 0.000034 | Train Accuracy: 1.000000 | Test Accuracy: 0.959324, 0.922616
Step 5 | Training Loss: 0.000027 | Train Accuracy: 1.000000 | Test Accuracy: 0.959280, 0.922532
Current Layer Attributes - epochs:5 hidden layers:3 features count:1
Initial Accuracy, before training: 0.372826486825943
Step 1 | Training Loss: 0.000003 | Train Accuracy: 0.999047 | Test Accuracy: 0.999335, 0.998734
Step 2 | Training Loss: 0.000001 | Train Accuracy: 0.999762 | Test Accuracy: 0.997738, 0.995696
Step 3 | Training Loss: 0.000002 | Train Accuracy: 0.999762 | Test Accuracy: 0.997250, 0.994768
Step 4 | Training Loss: 0.000003 | Train Accuracy: 1.000000 | Test Accuracy: 0.997738, 0.995696
Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.997294, 0.994852
Step 1 | Training Loss: 0.000003 | Train Accuracy: 0.999921 | Test Accuracy: 0.997294, 0.994852
Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.997294, 0.994852
Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.997294, 0.994852
Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.998004, 0.996203
Step 5 | Training Loss: 0.000001 | Train Accuracy: 1.000000 | Test Accuracy: 0.997960, 0.996118
Step 1 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.997960, 0.996118
Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.997915, 0.996034
Step 3 | Training Loss: 0.000001 | Train Accuracy: 1.000000 | Test Accuracy: 0.997915, 0.996034
Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.997915, 0.996034
Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.997915, 0.996034
Current Layer Attributes - epochs:5 hidden layers:5 features count:1
Initial Accuracy, before training: 0.4764017164707184
Step 1 | Training Loss: 0.023943 | Train Accuracy: 0.965947 | Test Accuracy: 0.785841, 0.594515
Step 2 | Training Loss: 0.006488 | Train Accuracy: 0.986585 | Test Accuracy: 0.852156, 0.720338
Step 3 | Training Loss: 0.002552 | Train Accuracy: 0.987935 | Test Accuracy: 0.821771, 0.662110
Step 4 | Training Loss: 0.000031 | Train Accuracy: 0.999841 | Test Accuracy: 0.969349, 0.941688
Step 5 | Training Loss: 0.000004 | Train Accuracy: 0.999365 | Test Accuracy: 0.981148, 0.964135
Step 1 | Training Loss: 0.000009 | Train Accuracy: 0.999841 | Test Accuracy: 0.987314, 0.975865
Step 2 | Training Loss: 0.000005 | Train Accuracy: 0.999841 | Test Accuracy: 0.989132, 0.979325
Step 3 | Training Loss: 0.000004 | Train Accuracy: 0.999921 | Test Accuracy: 0.989132, 0.979325
Step 4 | Training Loss: 0.000002 | Train Accuracy: 1.000000 | Test Accuracy: 0.987358, 0.975949
Step 5 | Training Loss: 0.000001 | Train Accuracy: 0.999841 | Test Accuracy: 0.986648, 0.974599
Step 1 | Training Loss: 0.000000 | Train Accuracy: 0.999841 | Test Accuracy: 0.986737, 0.974768
Step 2 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.986870, 0.975021
Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.987181, 0.975612
Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.987491, 0.976203
Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.987713, 0.976624
Current Layer Attributes - epochs:5 hidden layers:1 features count:1
Initial Accuracy, before training: 0.6875443458557129
Step 1 | Training Loss: 0.002989 | Train Accuracy: 0.999286 | Test Accuracy: 0.836764, 0.689451
Step 2 | Training Loss: 0.000408 | Train Accuracy: 1.000000 | Test Accuracy: 0.917938, 0.843882
Step 3 | Training Loss: 0.000255 | Train Accuracy: 1.000000 | Test Accuracy: 0.929870, 0.866582
Step 4 | Training Loss: 0.001016 | Train Accuracy: 1.000000 | Test Accuracy: 0.931112, 0.868945
Step 5 | Training Loss: 0.000094 | Train Accuracy: 1.000000 | Test Accuracy: 0.956175, 0.916624
Step 1 | Training Loss: 0.000048 | Train Accuracy: 1.000000 | Test Accuracy: 0.938831, 0.883629
Step 2 | Training Loss: 0.000095 | Train Accuracy: 0.999921 | Test Accuracy: 0.939762, 0.885401
Step 3 | Training Loss: 0.000069 | Train Accuracy: 1.000000 | Test Accuracy: 0.939629, 0.885148
Step 4 | Training Loss: 0.000024 | Train Accuracy: 0.999841 | Test Accuracy: 0.940428, 0.886667
Step 5 | Training Loss: 0.000047 | Train Accuracy: 0.999921 | Test Accuracy: 0.940827, 0.887426
Step 1 | Training Loss: 0.000023 | Train Accuracy: 0.999841 | Test Accuracy: 0.941492, 0.888692
Step 2 | Training Loss: 0.000029 | Train Accuracy: 1.000000 | Test Accuracy: 0.941847, 0.889367
Step 3 | Training Loss: 0.000040 | Train Accuracy: 1.000000 | Test Accuracy: 0.941492, 0.888692
Step 4 | Training Loss: 0.000017 | Train Accuracy: 1.000000 | Test Accuracy: 0.941448, 0.888608
Step 5 | Training Loss: 0.000057 | Train Accuracy: 0.999921 | Test Accuracy: 0.941891, 0.889452
Current Layer Attributes - epochs:5 hidden layers:3 features count:1
Initial Accuracy, before training: 0.33223918080329895
Step 1 | Training Loss: 0.000001 | Train Accuracy: 0.999444 | Test Accuracy: 0.998847, 0.997806
Step 2 | Training Loss: 0.000000 | Train Accuracy: 0.999841 | Test Accuracy: 0.999113, 0.998312
Step 3 | Training Loss: 0.001419 | Train Accuracy: 1.000000 | Test Accuracy: 0.999468, 0.998987
Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999423, 0.998903
Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999423, 0.998903
Step 1 | Training Loss: 0.000000 | Train Accuracy: 0.999841 | Test Accuracy: 0.999423, 0.998903
Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999423, 0.998903
Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999423, 0.998903
Step 4 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.999468, 0.998987
Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999468, 0.998987
Step 1 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999468, 0.998987
Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999468, 0.998987
Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999468, 0.998987
Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999468, 0.998987
Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.999468, 0.998987
Current Layer Attributes - epochs:5 hidden layers:5 features count:1
Initial Accuracy, before training: 0.6654542088508606
Step 1 | Training Loss: 0.012044 | Train Accuracy: 0.973170 | Test Accuracy: 0.737136, 0.501266
Step 2 | Training Loss: 0.019624 | Train Accuracy: 0.979759 | Test Accuracy: 0.807399, 0.635105
Step 3 | Training Loss: 0.020175 | Train Accuracy: 0.983093 | Test Accuracy: 0.786551, 0.595443
Step 4 | Training Loss: 0.004278 | Train Accuracy: 0.987617 | Test Accuracy: 0.794047, 0.609705
Step 5 | Training Loss: 0.007113 | Train Accuracy: 0.991665 | Test Accuracy: 0.775195, 0.574177
Step 1 | Training Loss: 0.006551 | Train Accuracy: 0.990475 | Test Accuracy: 0.774663, 0.573165
Step 2 | Training Loss: 0.004136 | Train Accuracy: 0.991983 | Test Accuracy: 0.775594, 0.574599
Step 3 | Training Loss: 0.007239 | Train Accuracy: 0.993094 | Test Accuracy: 0.794047, 0.609705
Step 4 | Training Loss: 0.004331 | Train Accuracy: 0.995396 | Test Accuracy: 0.792229, 0.606245
Step 5 | Training Loss: 0.004040 | Train Accuracy: 0.997936 | Test Accuracy: 0.907825, 0.826160
Step 1 | Training Loss: 0.000007 | Train Accuracy: 0.999286 | Test Accuracy: 0.940871, 0.887595
Step 2 | Training Loss: 0.000025 | Train Accuracy: 0.999841 | Test Accuracy: 0.961276, 0.926329
Step 3 | Training Loss: 0.002310 | Train Accuracy: 0.999444 | Test Accuracy: 0.962961, 0.929620
Step 4 | Training Loss: 0.000041 | Train Accuracy: 0.999682 | Test Accuracy: 0.964824, 0.933165
Step 5 | Training Loss: 0.000004 | Train Accuracy: 0.999762 | Test Accuracy: 0.965401, 0.934262
Current Layer Attributes - epochs:5 hidden layers:1 features count:1
Initial Accuracy, before training: 0.45475515723228455
Step 1 | Training Loss: 0.000396 | Train Accuracy: 0.999524 | Test Accuracy: 0.896913, 0.803882
Step 2 | Training Loss: 0.000057 | Train Accuracy: 0.999921 | Test Accuracy: 0.947303, 0.899747
Step 3 | Training Loss: 0.000069 | Train Accuracy: 1.000000 | Test Accuracy: 0.944331, 0.894093
Step 4 | Training Loss: 0.000060 | Train Accuracy: 0.999841 | Test Accuracy: 0.944642, 0.894684
Step 5 | Training Loss: 0.000015 | Train Accuracy: 0.999841 | Test Accuracy: 0.944819, 0.895021
Step 1 | Training Loss: 0.000014 | Train Accuracy: 0.999921 | Test Accuracy: 0.944863, 0.895105
Step 2 | Training Loss: 0.000011 | Train Accuracy: 1.000000 | Test Accuracy: 0.945928, 0.897131
Step 3 | Training Loss: 0.000008 | Train Accuracy: 1.000000 | Test Accuracy: 0.945972, 0.897215
Step 4 | Training Loss: 0.000021 | Train Accuracy: 1.000000 | Test Accuracy: 0.946904, 0.898987
Step 5 | Training Loss: 0.000907 | Train Accuracy: 0.999841 | Test Accuracy: 0.947081, 0.899325
Step 1 | Training Loss: 0.000004 | Train Accuracy: 1.000000 | Test Accuracy: 0.948101, 0.901266
Step 2 | Training Loss: 0.000005 | Train Accuracy: 0.999841 | Test Accuracy: 0.948057, 0.901181
Step 3 | Training Loss: 0.000005 | Train Accuracy: 0.999841 | Test Accuracy: 0.949388, 0.903713
Step 4 | Training Loss: 0.000005 | Train Accuracy: 1.000000 | Test Accuracy: 0.949122, 0.903207
Step 5 | Training Loss: 0.000006 | Train Accuracy: 1.000000 | Test Accuracy: 0.950231, 0.905316
Current Layer Attributes - epochs:5 hidden layers:3 features count:1
Initial Accuracy, before training: 0.7089691162109375
Step 1 | Training Loss: 0.000002 | Train Accuracy: 0.999603 | Test Accuracy: 0.981503, 0.964810
Step 2 | Training Loss: 0.000002 | Train Accuracy: 0.999682 | Test Accuracy: 0.981458, 0.964726
Step 3 | Training Loss: 0.001645 | Train Accuracy: 0.999682 | Test Accuracy: 0.982612, 0.966920
Step 4 | Training Loss: 0.000008 | Train Accuracy: 0.999603 | Test Accuracy: 0.986027, 0.973418
Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999682 | Test Accuracy: 0.982789, 0.967257
Step 1 | Training Loss: 0.000000 | Train Accuracy: 0.999841 | Test Accuracy: 0.985185, 0.971814
Step 2 | Training Loss: 0.000000 | Train Accuracy: 0.999762 | Test Accuracy: 0.985495, 0.972405
Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.994633, 0.989789
Step 4 | Training Loss: 0.000001 | Train Accuracy: 1.000000 | Test Accuracy: 0.997782, 0.995781
Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999841 | Test Accuracy: 0.996451, 0.993249
Step 1 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.997782, 0.995781
Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.997782, 0.995781
Step 3 | Training Loss: 0.000000 | Train Accuracy: 0.999841 | Test Accuracy: 0.997782, 0.995781
Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.997782, 0.995781
Step 5 | Training Loss: 0.000002 | Train Accuracy: 0.999841 | Test Accuracy: 0.985140, 0.971730
Current Layer Attributes - epochs:5 hidden layers:5 features count:1
Initial Accuracy, before training: 0.4045865833759308
Step 1 | Training Loss: 0.025833 | Train Accuracy: 0.950865 | Test Accuracy: 0.765703, 0.557553
Step 2 | Training Loss: 0.014012 | Train Accuracy: 0.986506 | Test Accuracy: 0.744189, 0.514515
Step 3 | Training Loss: 0.007475 | Train Accuracy: 0.991268 | Test Accuracy: 0.779897, 0.582363
Step 4 | Training Loss: 0.007330 | Train Accuracy: 0.991507 | Test Accuracy: 0.774885, 0.572405
Step 5 | Training Loss: 0.005704 | Train Accuracy: 0.993332 | Test Accuracy: 0.786684, 0.594262
Step 1 | Training Loss: 0.004059 | Train Accuracy: 0.994840 | Test Accuracy: 0.796576, 0.613165
Step 2 | Training Loss: 0.007358 | Train Accuracy: 0.994761 | Test Accuracy: 0.787216, 0.595443
Step 3 | Training Loss: 0.005621 | Train Accuracy: 0.994920 | Test Accuracy: 0.833925, 0.684473
Step 4 | Training Loss: 0.006635 | Train Accuracy: 0.995555 | Test Accuracy: 0.784865, 0.590886
Step 5 | Training Loss: 0.005466 | Train Accuracy: 0.994761 | Test Accuracy: 0.777457, 0.577384
Step 1 | Training Loss: 0.003807 | Train Accuracy: 0.996031 | Test Accuracy: 0.786773, 0.594684
Step 2 | Training Loss: 0.001771 | Train Accuracy: 0.996825 | Test Accuracy: 0.788946, 0.598565
Step 3 | Training Loss: 0.002613 | Train Accuracy: 0.997301 | Test Accuracy: 0.810060, 0.639072
Step 4 | Training Loss: 0.001716 | Train Accuracy: 0.996746 | Test Accuracy: 0.847188, 0.709451
Step 5 | Training Loss: 0.001673 | Train Accuracy: 0.998571 | Test Accuracy: 0.921265, 0.850464
Current Layer Attributes - epochs:5 hidden layers:1 features count:1
Initial Accuracy, before training: 0.825141966342926
Step 1 | Training Loss: 0.005486 | Train Accuracy: 0.997539 | Test Accuracy: 0.808863, 0.636371
Step 2 | Training Loss: 0.001203 | Train Accuracy: 0.999762 | Test Accuracy: 0.881875, 0.775274
Step 3 | Training Loss: 0.000215 | Train Accuracy: 0.999603 | Test Accuracy: 0.918825, 0.845570
Step 4 | Training Loss: 0.000995 | Train Accuracy: 0.999682 | Test Accuracy: 0.956086, 0.916456
Step 5 | Training Loss: 0.000124 | Train Accuracy: 0.999206 | Test Accuracy: 0.958259, 0.920591
Step 1 | Training Loss: 0.000038 | Train Accuracy: 0.999921 | Test Accuracy: 0.958171, 0.920422
Step 2 | Training Loss: 0.000058 | Train Accuracy: 0.999682 | Test Accuracy: 0.954888, 0.914177
Step 3 | Training Loss: 0.000781 | Train Accuracy: 0.999841 | Test Accuracy: 0.957195, 0.918565
Step 4 | Training Loss: 0.000049 | Train Accuracy: 0.999682 | Test Accuracy: 0.955687, 0.915696
Step 5 | Training Loss: 0.000042 | Train Accuracy: 0.999365 | Test Accuracy: 0.957195, 0.918565
Step 1 | Training Loss: 0.000898 | Train Accuracy: 0.999921 | Test Accuracy: 0.957372, 0.918903
Step 2 | Training Loss: 0.000079 | Train Accuracy: 0.999921 | Test Accuracy: 0.957284, 0.918734
Step 3 | Training Loss: 0.000057 | Train Accuracy: 0.999921 | Test Accuracy: 0.955066, 0.914515
Step 4 | Training Loss: 0.000020 | Train Accuracy: 0.999921 | Test Accuracy: 0.956973, 0.918143
Step 5 | Training Loss: 0.000017 | Train Accuracy: 0.999921 | Test Accuracy: 0.954356, 0.913165
Current Layer Attributes - epochs:5 hidden layers:3 features count:1
Initial Accuracy, before training: 0.35122427344322205
Step 1 | Training Loss: 0.000004 | Train Accuracy: 1.000000 | Test Accuracy: 0.965224, 0.933840
Step 2 | Training Loss: 0.000001 | Train Accuracy: 0.999762 | Test Accuracy: 0.998714, 0.997553
Step 3 | Training Loss: 0.000001 | Train Accuracy: 1.000000 | Test Accuracy: 0.999290, 0.998650
Step 4 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.999290, 0.998650
Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999290, 0.998650
Step 1 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.999290, 0.998650
Step 2 | Training Loss: 0.000001 | Train Accuracy: 0.999921 | Test Accuracy: 0.999290, 0.998650
Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999290, 0.998650
Step 4 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.999290, 0.998650
Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999290, 0.998650
Step 1 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999290, 0.998650
Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999290, 0.998650
Step 3 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.999290, 0.998650
Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999290, 0.998650
Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.999290, 0.998650
Current Layer Attributes - epochs:5 hidden layers:5 features count:1
Initial Accuracy, before training: 0.6894517540931702
Step 1 | Training Loss: 0.024689 | Train Accuracy: 0.974361 | Test Accuracy: 0.780829, 0.584895
Step 2 | Training Loss: 0.022241 | Train Accuracy: 0.976425 | Test Accuracy: 0.738778, 0.504219
Step 3 | Training Loss: 0.014032 | Train Accuracy: 0.985950 | Test Accuracy: 0.764949, 0.554008
Step 4 | Training Loss: 0.007663 | Train Accuracy: 0.990872 | Test Accuracy: 0.754258, 0.533671
Step 5 | Training Loss: 0.005187 | Train Accuracy: 0.989046 | Test Accuracy: 0.770981, 0.566076
Step 1 | Training Loss: 0.003864 | Train Accuracy: 0.993015 | Test Accuracy: 0.782514, 0.587342
Step 2 | Training Loss: 0.003295 | Train Accuracy: 0.995317 | Test Accuracy: 0.794402, 0.608861
Step 3 | Training Loss: 0.005155 | Train Accuracy: 0.993015 | Test Accuracy: 0.788325, 0.597300
Step 4 | Training Loss: 0.010982 | Train Accuracy: 0.994602 | Test Accuracy: 0.782780, 0.586835
Step 5 | Training Loss: 0.003601 | Train Accuracy: 0.995634 | Test Accuracy: 0.797241, 0.614346
Step 1 | Training Loss: 0.005937 | Train Accuracy: 0.997460 | Test Accuracy: 0.803229, 0.625738
Step 2 | Training Loss: 0.000136 | Train Accuracy: 0.998651 | Test Accuracy: 0.951783, 0.908270
Step 3 | Training Loss: 0.000866 | Train Accuracy: 0.999047 | Test Accuracy: 0.953203, 0.910970
Step 4 | Training Loss: 0.001847 | Train Accuracy: 0.999921 | Test Accuracy: 0.954267, 0.912996
Step 5 | Training Loss: 0.000011 | Train Accuracy: 0.999762 | Test Accuracy: 0.966288, 0.935865
Current Layer Attributes - epochs:5 hidden layers:1 features count:1
Initial Accuracy, before training: 0.25385913252830505
Step 1 | Training Loss: 0.010233 | Train Accuracy: 0.989284 | Test Accuracy: 0.781317, 0.583966
Step 2 | Training Loss: 0.002050 | Train Accuracy: 0.999206 | Test Accuracy: 0.840756, 0.697131
Step 3 | Training Loss: 0.000738 | Train Accuracy: 0.999682 | Test Accuracy: 0.901437, 0.812489
Step 4 | Training Loss: 0.000229 | Train Accuracy: 0.999762 | Test Accuracy: 0.926677, 0.860506
Step 5 | Training Loss: 0.000237 | Train Accuracy: 1.000000 | Test Accuracy: 0.928007, 0.863038
Step 1 | Training Loss: 0.000169 | Train Accuracy: 1.000000 | Test Accuracy: 0.927298, 0.861688
Step 2 | Training Loss: 0.000070 | Train Accuracy: 0.999921 | Test Accuracy: 0.933242, 0.872996
Step 3 | Training Loss: 0.000071 | Train Accuracy: 0.999841 | Test Accuracy: 0.931290, 0.869283
Step 4 | Training Loss: 0.000154 | Train Accuracy: 1.000000 | Test Accuracy: 0.930004, 0.866835
Step 5 | Training Loss: 0.000058 | Train Accuracy: 0.999841 | Test Accuracy: 0.933907, 0.874262
Step 1 | Training Loss: 0.000043 | Train Accuracy: 0.999921 | Test Accuracy: 0.931157, 0.869030
Step 2 | Training Loss: 0.000037 | Train Accuracy: 1.000000 | Test Accuracy: 0.931778, 0.870211
Step 3 | Training Loss: 0.000092 | Train Accuracy: 0.999921 | Test Accuracy: 0.932887, 0.872321
Step 4 | Training Loss: 0.000197 | Train Accuracy: 1.000000 | Test Accuracy: 0.930758, 0.868270
Step 5 | Training Loss: 0.000107 | Train Accuracy: 0.999921 | Test Accuracy: 0.933375, 0.873249
Current Layer Attributes - epochs:5 hidden layers:3 features count:1
Initial Accuracy, before training: 0.7918736934661865
Step 1 | Training Loss: 0.000009 | Train Accuracy: 0.999682 | Test Accuracy: 0.981148, 0.964135
Step 2 | Training Loss: 0.000003 | Train Accuracy: 0.999841 | Test Accuracy: 0.998802, 0.997721
Step 3 | Training Loss: 0.000001 | Train Accuracy: 1.000000 | Test Accuracy: 0.997782, 0.995781
Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999024, 0.998143
Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999468, 0.998987
Step 1 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.999468, 0.998987
Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999468, 0.998987
Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999468, 0.998987
Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999468, 0.998987
Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999468, 0.998987
Step 1 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999512, 0.999072
Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999512, 0.999072
Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999512, 0.999072
Step 4 | Training Loss: 0.000001 | Train Accuracy: 1.000000 | Test Accuracy: 0.999512, 0.999072
Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999512, 0.999072
Current Layer Attributes - epochs:5 hidden layers:5 features count:1
Initial Accuracy, before training: 0.7087029814720154
Step 1 | Training Loss: 0.029560 | Train Accuracy: 0.982537 | Test Accuracy: 0.783801, 0.589789
Step 2 | Training Loss: 0.000902 | Train Accuracy: 0.998968 | Test Accuracy: 0.979817, 0.962869
Step 3 | Training Loss: 0.000316 | Train Accuracy: 0.999047 | Test Accuracy: 0.994455, 0.990717
Step 4 | Training Loss: 0.000001 | Train Accuracy: 0.999444 | Test Accuracy: 0.987048, 0.976624
Step 5 | Training Loss: 0.001754 | Train Accuracy: 0.999286 | Test Accuracy: 0.969304, 0.942869
Step 1 | Training Loss: 0.000001 | Train Accuracy: 0.999047 | Test Accuracy: 0.969260, 0.942785
Step 2 | Training Loss: 0.000001 | Train Accuracy: 0.999762 | Test Accuracy: 0.969482, 0.943207
Step 3 | Training Loss: 0.002556 | Train Accuracy: 0.999365 | Test Accuracy: 0.969482, 0.943207
Step 4 | Training Loss: 0.000009 | Train Accuracy: 0.999365 | Test Accuracy: 0.969482, 0.943207
Step 5 | Training Loss: 0.000005 | Train Accuracy: 0.999603 | Test Accuracy: 0.969482, 0.943207
Step 1 | Training Loss: 0.000884 | Train Accuracy: 0.999603 | Test Accuracy: 0.969482, 0.943207
Step 2 | Training Loss: 0.000000 | Train Accuracy: 0.999444 | Test Accuracy: 0.969438, 0.943122
Step 3 | Training Loss: 0.000000 | Train Accuracy: 0.999365 | Test Accuracy: 0.969482, 0.943207
Step 4 | Training Loss: 0.000006 | Train Accuracy: 0.999444 | Test Accuracy: 0.969526, 0.943291
Step 5 | Training Loss: 0.000003 | Train Accuracy: 0.999603 | Test Accuracy: 0.969615, 0.943460
7min 13s ± 56 s per loop (mean ± std. dev. of 10 runs, 1 loop each)

In [ ]:

    




    

In [ ]:

    




    

In [11]:

    

g = df_results.groupby(by=['no_of_features'])
idx = g['test_score'].transform(max) == df_results['test_score']
df_results[idx].sort_values(by = 'test_score', ascending = False)


    

    
Out[11]:







  

    

      

      
epoch
      
no_of_features
      
hidden_layers
      
train_score
      
test_score
      
test_score_20
      
time_taken
    
  
  

    

      
3
      
18
      
1
      
3
      
1.0
      
0.999512
      
0.999072
      
117.723715
    
  

In [12]:

    

df_results.sort_values(by = 'test_score', ascending = False)


    

    
Out[12]:







  

    

      

      
epoch
      
no_of_features
      
hidden_layers
      
train_score
      
test_score
      
test_score_20
      
time_taken
    
  
  

    

      
3
      
18
      
1
      
3
      
1.000000
      
0.999512
      
0.999072
      
117.723715
    
    

      
2
      
6
      
1
      
3
      
1.000000
      
0.999468
      
0.998987
      
54.473063
    
    

      
4
      
6
      
1
      
5
      
0.999047
      
0.994455
      
0.990717
      
52.805204
    
    

      
1
      
12
      
1
      
1
      
0.999841
      
0.933907
      
0.874262
      
38.998433
    
    

      
0
      
6
      
1
      
1
      
1.000000
      
0.928007
      
0.863038
      
19.724738
    
  

In [13]:

    

pd.Panel(Train.predictions).to_pickle("dataset/tf_GRU_nsl_kdd_predictions.pkl")
df_results.to_pickle("dataset/tf_GRU_nsl_kdd_scores.pkl")


    

In [14]:

    

import numpy as np
import matplotlib.pyplot as plt
import itertools

def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    np.set_printoptions(precision=4)

    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, cm[i, j].round(4),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

def plot(actual_value, pred_value):
    from sklearn.metrics import confusion_matrix
    cm_2labels = confusion_matrix(y_pred = pred_value, y_true = actual_value)
    plt.figure(figsize=[6,6])
    plot_confusion_matrix(cm_2labels, preprocess.output_columns_2labels, normalize = False,
                         title = Train.best_parameters)


    

In [15]:

    

plot(actual_value = Train.actual_value, pred_value = Train.pred_value)


    

    




Confusion matrix, without normalization
[[ 9710     1]
 [    0 12833]]






    

In [16]:

    

plot(actual_value = Train.actual_value_, pred_value = Train.pred_value_)


    

    




Confusion matrix, without normalization
[[2151    1]
 [   0 9698]]






    

In [17]:

    

#4.5 GB
pd.Series(Train.pred_value).to_csv('GRU_prediction_values.csv')


    

In [18]:

    

past_scores


    

    
Out[18]:







  

    

      

      

      
epoch
      
train_score
      
test_score
      
test_score_20
      
time_taken
    
    

      
no_of_features
      
hidden_layers
      

      

      

      

      

    
  
  

    

      
1
      
1
      
6
      
1.000000
      
0.937278
      
0.880675
      
14.150092
    
    

      
1
      
12
      
1.000000
      
0.957727
      
0.919578
      
25.316301
    
    

      
1
      
18
      
1.000000
      
0.958215
      
0.920506
      
42.115427
    
    

      
3
      
6
      
1.000000
      
0.999867
      
0.999747
      
39.416124
    
    

      
3
      
12
      
1.000000
      
0.999911
      
0.999831
      
47.097980
    
    

      
5
      
6
      
0.997460
      
0.894029
      
0.798397
      
62.728601
    
    

      
5
      
12
      
0.999762
      
0.962473
      
0.928608
      
124.716464
    
    

      
5
      
18
      
0.999921
      
0.972498
      
0.947679
      
186.515223
    
    

      
1
      
6
      
1.000000
      
0.937278
      
0.880675
      
14.150092
    
    

      
1
      
12
      
1.000000
      
0.957727
      
0.919578
      
25.316301
    
    

      
1
      
18
      
1.000000
      
0.958215
      
0.920506
      
42.115427
    
    

      
3
      
6
      
1.000000
      
0.999867
      
0.999747
      
39.416124
    
    

      
3
      
12
      
1.000000
      
0.999911
      
0.999831
      
47.097980
    
    

      
5
      
6
      
0.997460
      
0.894029
      
0.798397
      
62.728601
    
    

      
5
      
12
      
0.999762
      
0.962473
      
0.928608
      
124.716464
    
    

      
5
      
18
      
0.999921
      
0.972498
      
0.947679
      
186.515223
    
    

      
1
      
6
      
0.999841
      
0.937278
      
0.880675
      
14.001841
    
    

      
...
      
...
      
...
      
...
      
...
      
...
    
    

      
1
      
6
      
1.000000
      
0.956175
      
0.916624
      
19.982509
    
    

      
3
      
6
      
1.000000
      
0.999468
      
0.998987
      
33.803133
    
    

      
5
      
6
      
0.979759
      
0.807399
      
0.635105
      
35.490781
    
    

      
5
      
12
      
0.997936
      
0.907825
      
0.826160
      
171.106554
    
    

      
5
      
18
      
0.999762
      
0.965401
      
0.934262
      
255.321470
    
    

      
1
      
6
      
0.999921
      
0.947303
      
0.899747
      
7.824117
    
    

      
1
      
18
      
1.000000
      
0.950231
      
0.905316
      
56.515639
    
    

      
3
      
6
      
0.999603
      
0.986027
      
0.973418
      
42.663989
    
    

      
3
      
12
      
1.000000
      
0.997782
      
0.995781
      
94.612861
    
    

      
5
      
6
      
0.993332
      
0.786684
      
0.594262
      
85.770908
    
    

      
5
      
12
      
0.994920
      
0.833925
      
0.684473
      
136.350251
    
    

      
5
      
18
      
0.998571
      
0.921265
      
0.850464
      
256.658391
    
    

      
1
      
6
      
0.999206
      
0.958259
      
0.920591
      
19.687884
    
    

      
3
      
6
      
1.000000
      
0.999290
      
0.998650
      
33.079605
    
    

      
5
      
6
      
0.974361
      
0.780829
      
0.584895
      
17.424911
    
    

      
5
      
12
      
0.995634
      
0.797241
      
0.614346
      
173.522122
    
    

      
5
      
18
      
0.999762
      
0.966288
      
0.935865
      
260.093316
    
  

73 rows × 5 columns

In [19]:

    

pgb = past_scores.groupby(by=['no_of_features', 'hidden_layers'])
pgb.mean()


    

    
Out[19]:







  

    

      

      

      
epoch
      
train_score
      
test_score
      
test_score_20
      
time_taken
    
    

      
no_of_features
      
hidden_layers
      

      

      

      

      

    
  
  

    

      
1
      
1
      
11.307692
      
0.999930
      
0.947564
      
0.900243
      
27.491425
    
    

      
3
      
9.000000
      
0.999815
      
0.998428
      
0.997009
      
48.438291
    
    

      
5
      
11.379310
      
0.996620
      
0.911742
      
0.832329
      
129.095418
    
  

In [20]:

    

pgb.std()


    

    
Out[20]:







  

    

      

      

      
epoch
      
train_score
      
test_score
      
test_score_20
      
time_taken
    
    

      
no_of_features
      
hidden_layers
      

      

      

      

      

    
  
  

    

      
1
      
1
      
5.182812
      
0.000159
      
0.011661
      
0.022184
      
14.036460
    
    

      
3
      
4.242641
      
0.000278
      
0.003207
      
0.006102
      
35.543510
    
    

      
5
      
4.901995
      
0.006017
      
0.081695
      
0.155172
      
67.656089