In [1]:
import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer, HashingVectorizer
from sklearn.feature_selection import SelectKBest, chi2
from sklearn import metrics
In [45]:
from sklearn.svm import LinearSVC
from sklearn.linear_model import SGDClassifier
from sklearn.linear_model import Perceptron
from sklearn.linear_model import PassiveAggressiveClassifier
from sklearn.naive_bayes import BernoulliNB, MultinomialNB
from sklearn.neighbors import KNeighborsClassifier
from sklearn.neighbors import NearestCentroid
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.utils import shuffle
In [151]:
df = pd.read_csv('output_http_csic_2010_weka_with_duplications_utf8_escd_v02_full.csv')
df.head()
Out[151]:
In [152]:
# Remove columns that contain the same value
df = df.drop(['userAgent', 'pragma', 'cacheControl', 'acceptEncoding', 'acceptCharset', 'acceptLanguage'], 1)
df = df.drop(['connection', 'cookie', 'accept', 'protocol'], 1)
# Since everything is localhost let's keep only the port and remove the host part
df['port'] = df['host'].str.split(':', expand=True)[1]
df = df.drop(['host'], 1)
df.head()
Out[152]:
In [153]:
# Split the dataset in two to avoid mixed indices
df_anom = df[df['label']=='anom']
df_norm = df[df['label']=='norm']
print df_anom.describe()
print df_norm.describe()
In [154]:
df2_anom = df_anom[['index', 'payload', 'label']]
df2_anom = df2_anom.dropna()
print(df2_anom.head())
df2_norm = df_norm[['index', 'payload', 'label']]
df2_norm = df2_norm.dropna()
print(df2_norm.head())
In [155]:
#df3 = df2[['payload','label']].groupby(df2['index']).sum()
#df3 = df2[['payload','label']].groupby(df2['index']).agg(lambda x: ' '.join(set(x)))
df3_anom = df2_anom[['payload','label']].groupby(df2_anom['index']).agg(lambda x: ' '.join(set(x)))
df3_anom["payload"] = df3_anom['payload'].apply(lambda x: x.replace("=", " "))
print(df3_anom.head())
df3_anom['label'] = 1
print(df3_anom.head())
In [156]:
df3_norm = df2_norm[['payload','label']].groupby(df2_norm['index']).agg(lambda x: ' '.join(set(x)))
df3_norm["payload"] = df3_norm['payload'].apply(lambda x: x.replace("=", " "))
print(df3_norm.head())
df3_norm['label'] = 0
print(df3_norm.head())
In [157]:
df4 = pd.concat([df3_norm, df3_anom])
print(df4.head())
print(df4.describe())
print(df4.label.value_counts())
In [272]:
# Vectorize the payload by creating character n-grams
vec = TfidfVectorizer(analyzer='word',ngram_range=(3,3))
#vec = HashingVectorizer(analyzer='char',ngram_range=(6,6))
y = df4['label']
X = vec.fit_transform(df4['payload'].dropna())
In [273]:
print(X.shape, y.shape)
In [274]:
# Use a chi-squared test to extract features
ch2 = SelectKBest(chi2, k=600)
X_train = ch2.fit_transform(X, y)
print(X_train.shape)
In [275]:
X1, y1 = shuffle(X_train, y)
offset = int(X1.shape[0] * 0.8)
In [178]:
from keras.models import Sequential
from keras.layers import Dense, Activation, Dropout
In [295]:
model = Sequential()
model.add(Dense(input_dim=X1.shape[1], output_dim=600))
model.add(Activation('relu'))
model.add(Dense(600))
model.add(Activation('relu'))
#model.add(Dropout(0.5))
model.add(Dense(300))
model.add(Activation('relu'))
#model.add(Dropout(0.5))
#model.add(Dense(100))
#model.add(Activation('relu'))
model.add(Dense(10))
model.add(Activation('relu'))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
In [296]:
model.fit(X1[:offset].toarray(), y1[:offset], batch_size=100, nb_epoch=25, verbose=1, callbacks=[],
validation_data=(X1[offset:].toarray(), y1[offset:]),
shuffle=False, class_weight=None, sample_weight=None)
Out[296]:
In [297]:
pred = model.predict_classes(X1[offset:].toarray())
accuracy = metrics.accuracy_score(y1[offset:], pred)
f1_score = metrics.f1_score(y1[offset:], pred)
conf_matrix = metrics.confusion_matrix(y1[offset:], pred)
print("\n")
print(accuracy, f1_score)
print(conf_matrix)
In [ ]: