In [ ]:
import struct, socket
import csv, json
import os
import datetime
import operator
import itertools
import md5
from collections import defaultdict
try:
import ipywidgets as widgets # For jupyter/ipython >= 1.4
except ImportError:
from IPython.html import widgets
from IPython.display import display, HTML, clear_output, Javascript
path = os.getcwd().split("/")
date = path[len(path)-1]
dpath = '/'.join(['data' if var == 'ipynb' else var for var in path]) + '/'
anchor = ''
anchor_hash = ''
clientips = defaultdict(int)
reqmethods = defaultdict(int)
rescontype = defaultdict(int)
referers = defaultdict(int)
refered = defaultdict(int)
requests = []
top_results = 20
details_limit = 1000
proxy_comments = {}
In [ ]:
# Widget styles and initialization
topBox = widgets.Box()
bottomBox = widgets.Box()
mainBoxes_css = (
(None, 'width', '90%'),
(None, 'margin', '0 auto'),
)
topBox._css = mainBoxes_css
bottomBox._css = mainBoxes_css
threatBox = widgets.HBox(width='100%', height='auto')
threat_title = widgets.HTML(height='25px', width='100%')
threat_list_container = widgets.Box(width='80%', height='100%')
threat_button_container = widgets.Box(width='20%', height='100%')
susp_select = widgets.Select(height='100%', width='99%')
search_btn = widgets.Button(description='Search',height='100%', width='65px')
search_btn.button_style = 'primary'
susp_select._css = (
(None, 'height', '90%'),
(None, 'width', '95%'),
('select', 'overflow-x', 'auto'),
('select', 'margin', 0)
)
resultSummaryBox = widgets.Box()
result_title = widgets.HTML(width='100%')
result_summary_box = widgets.HBox(width='100%')
result_summary_container = widgets.Box(width='80%')
result_button_container = widgets.Box(width='20%')
result_summary_box.children = [result_title, result_summary_container, result_button_container]
resultTableBox = widgets.Box()
result_html_title = widgets.HTML(height='25px', width='100%')
result_html_box = widgets.Box() #this one has the scroll
result_html = widgets.HTML(width='100%')
result_box_css = (
(None, 'overflow', 'hidden'),
(None, 'width', '100%'),
)
resultSummaryBox._css = result_box_css
resultTableBox._css = result_box_css
result_html_box._css = (
(None, 'overflow','auto'),
(None, 'max-height', '300px'),
)
threat_button_container._css = (
(None, 'padding-top', '30px'),
)
topBox.children = [threatBox]
bottomBox.children = [resultSummaryBox,resultTableBox]
threat_list_container.children = [threat_title,susp_select]
threat_button_container.children = [search_btn]
threatBox.children = [threat_list_container, threat_button_container]
Interface
In [ ]:
yy = date[0:4]
mm = date[4:6]
dd = date[6:8]
def fill_list(list_control,source):
susp_select.options = list_control
susp_select.selected_label = list_control[0]
def data_loader():
clear_output()
c_uri = []
uri_sev=[]
global proxy_comments
response = GraphQLClient.request(
query="""query($date:SpotDateType!) {
proxy{
threats{
list(date:$date) {
score
uri
datetime
}
}
}
}""",
variables={
'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d')
}
)
proxy_comments = GraphQLClient.request(
query="""query($date:SpotDateType!) {
proxy{
threats{
comments(date:$date) {
uri
title
text
}
}
}
}""",
variables={
'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d')
}
)
proxy_comments = proxy_comments['data']['proxy']['threats']['comments']
if not 'errors' in response:
for row in response['data']['proxy']['threats']['list']:
if row['uri'] not in uri_sev and row['score'] == 1:
uri_sev.append(row['uri'])
else:
print "An error ocurred: " + response["errors"][0]["message"]
threat_title.value ="<h4>Suspicious URI</h4>"
if len(uri_sev) == 0:
display(Javascript("$('.widget-area > .widget-subarea > *').remove();"))
display(widgets.HTML(value="There are not high risk results."),)
else:
sorted_dict = sorted(uri_sev, key=operator.itemgetter(0))
fill_list(sorted_dict,susp_select)
def start_investigation():
display(Javascript("$('.widget-area > .widget-subarea > *').remove();"))
data_loader()
if susp_select.options:
display_controls()
def display_controls():
display(topBox)
def search_ip(b):
global anchor
global expanded_results
anchor=''
anchor = susp_select.value
removeWidget(3)
removeWidget(2)
removeWidget(1)
height=80
expanded_results = GraphQLClient.request(
query="""query($date:SpotDateType!,$uri:String!){
proxy{
threat{
details(date:$date,uri:$uri) {
username
webCategory
responseContentType
datetime
referer
clientToServerBytes
duration
userAgent
uri
serverIp
requestMethod
responseCode
uriPort
clientIp
serverToClientBytes
}
}
}
}
""",
variables={
'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d'),
'uri': anchor
}
)
if not 'errors' in expanded_results:
i = 0
table = "<table><th>TIME</th><th>CLIENT IP</th><th>USERNAME</th><th>DURATION</th> \
<th>FULL URI</th><th>WEB CATEGORY</th><th>RESPONSE CODE</th><th>REQUEST METHOD</th><th>USER AGENT</th> \
<th>MIME TYPE</th><th>REFERER</th><th>URI PORT</th><th>PROXY IP</th><th>SERVER BYTES</th><th>CLIENT BYTES</th>"
for row in expanded_results['data']['proxy']['threat']['details']:
if i < top_results:
table += "<tr><td>"+ str(row['datetime'])+"</td><td>"+str(row['clientIp'])+"</td>\
<td><div class='spot-text-wrapper' data-toggle='tooltip'>"+str(row['username'])+"\
</div></td><td>"+str(row['duration'])+"</td>\
<td><div class='spot-text-wrapper' data-toggle='tooltip'>"+str(row['uri'])+"</div>\
</td><td>"+str(row['webCategory'])+"</td>\
<td>"+str(row['responseCode'])+"</td><td>"+str(row['requestMethod'])+"</td>\
<td><div class='spot-text-wrapper' data-toggle='tooltip'>"+str(row['userAgent'])+"</div></td>\
<td><div class='spot-text-wrapper' data-toggle='tooltip'>"+str(row['responseContentType'])+"</div></td>\
<td><div class='spot-text-wrapper' data-toggle='tooltip'>"+str(row['referer'])+"</div></td>\
<td>"+str(row['uriPort'])+"</td><td>"+str(row['serverIp'])+"</td><td>\
"+str(row['serverToClientBytes'])+"</td><td>"+str(row['clientToServerBytes'])+"</td></tr>"
height += 20
i+=1
table += "</table>"
result_html_title.value='<h4>Displaying top {0} search results</h4>'.format(top_results)
else:
table = "<table></table>"
result_html_title.value='<h4>No results were found.</h4>'
result_html.value=table
result_html_box.children = [result_html]
display_threat_box(anchor)
resultTableBox.children = [result_html_title, result_html_box]
display(bottomBox)
search_btn.on_click(search_ip)
def display_threat_box(ip):
global expanded_results
global proxy_comments
title =""
comments = ""
title = next((item['title'] for item in proxy_comments if item.get("uri") == ip), "")
comments = next((item['text'] for item in proxy_comments if item.get("uri") == ip), "")
result_title.value="<h4 class='spot-text-wrapper spot-text-xlg' data-toggle='tooltip'>Threat summary for " + anchor +"</h4>"
tc_txt_title = widgets.Text(value=title, placeholder='Threat Title', width='100%')
tc_txa_summary = widgets.Textarea(value=comments, height=100, width='95%')
tc_btn_save = widgets.Button(description='Save', width='65px', layout='width:100%')
tc_btn_save.button_style = 'primary'
tc_txt_title._css = (
(None, 'width', '95%'),
)
result_summary_container.children = [tc_txt_title, tc_txa_summary]
result_button_container.children=[tc_btn_save]
result_summary_box.children = [result_summary_container, result_button_container]
resultSummaryBox.children = [result_title,result_summary_box]
def save_threat_summary(b):
result_msg = ""
threat_title = tc_txt_title.value
threat_comment = tc_txa_summary.value
if anchor != '':
mutation="""mutation(
$date: SpotDateType,
$uri: String!,
$text: String!,
$title: String!,
$threatDetails: [ProxyThreatDetailsInputType!]!,
$first:Int)
{
proxy {
createStoryboard(input:{
threatDetails: $threatDetails,
date: $date,
uri: $uri,
title: $title,
text: $text,
first:$first})
{success}
}
}"""
variables={
'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d'),
'uri': anchor,
'title': threat_title,
'text': threat_comment,
'threatDetails': expanded_results['data']['proxy']['threat']['details'],
'first':top_results
}
response = GraphQLClient.request(mutation,variables)
if not 'errors' in response:
start_investigation()
result_msg = "Story board successfully created"
else:
result_msg = response['errors'][0]['message']
else:
result_msg = "No data selected"
susp_select.selected_label = susp_select.options[0]
display(widgets.Box((widgets.HTML(value=result_msg, width='100%'),)))
tc_btn_save.on_click(save_threat_summary)
def removeWidget(index):
js_command = "$('.widget-area > .widget-subarea > .widget-box:eq({0})').remove();".format(index)
display(Javascript(js_command))
In [ ]:
start_investigation()