In [1]:

    

import json
from datetime import datetime, timedelta
import matplotlib.pylab as plot
import numpy as np


    

In [2]:

    

# Read data from http bro logs
with open("http.log",'r') as infile:
    file_data = infile.read()
    
# Split file by newlines
file_data = file_data.split('\n')

# Remove comment lines
http_data = []
for line in file_data:
    if line[0] is not None and line[0] != "#":
        http_data.append(line)


    

In [3]:

    

# Let's stack uris
uris = {}
for line in http_data:
    if len(line.split('\t')) > 9:
        uri = line.split('\t')[9].split('?')[0].split('&')[0]
        if uri not in uris.keys():
            uris[uri] = 1
        else:
            uris[uri] += 1

print(json.dumps(uris,indent=2))


    

    




{
  "/ftv2lastnode.gif": 2, 
  "/ftv2mnode.gif": 2, 
  "/pics/play_button_27x27px.gif": 4, 
  "/led.asp": 2, 
  "/pics/gray_corner_rt_5x50px.gif": 4, 
  "/img/device.gif": 4, 
  "/ RTSP/1.": 5, 
  "/pics/gray_corner_lt_5x50px.gif": 4, 
  "/webserverconfig.asp": 1, 
  "/auth/logo2_516.gif": 5, 
  "/index.htm": 1, 
  "/syslogserverconfig.asp": 2, 
  "/logo2_516.gif": 1, 
  "/neighbor_cache_table.asp": 2, 
  "/generalinst.htm": 1, 
  "/view/temp.shtml": 2, 
  "/img/checkbox_nchk.gif": 1, 
  "/jscript/sysstatus.js": 1, 
  "/SetModSerial.html": 1, 
  "/logo3.gif": 2, 
  "/status.jsp": 1, 
  "/port_setting.asp": 1, 
  "/syslog_message.asp": 1, 
  "/logo2_EDS-508A.gif": 1, 
  "/port_setting_show.asp": 1, 
  "/jscript/statistics.js": 3, 
  "/images/off.gif": 3, 
  "/pics/line_corner_rb_5x5px.gif": 4, 
  "/sysstatus.asp": 1, 
  "/overview.asp": 4, 
  "/jscript/powerconfig.js": 1, 
  "/jscript/login.js": 4, 
  "/mac_address_table_setting.asp": 4, 
  "/.git/HEAD": 11, 
  "/setid.html": 1, 
  "/network_setting_ipv6.asp": 1, 
  "/activate_button.gif": 10, 
  "/goform/svLogin": 3, 
  "/ftv2plastnode.gif": 1, 
  "/ftv2folderopen.gif": 2, 
  "/tasktracker.jsp": 1, 
  "/spconfig.asp": 4, 
  "/pics/line_corner_lt_5x5px.gif": 4, 
  "/pdmonitor.htm": 1, 
  "/settable.html": 1, 
  "/spconnect.asp": 2, 
  "/setdesc.html": 1, 
  "/jscript/ipconfig.js": 3, 
  "/syslogging.asp": 1, 
  "/images/connect.gif": 2, 
  "/jobtracker.jsp": 1, 
  "/ftv2pnode.gif": 1, 
  "/eip_setting.asp": 1, 
  "/ftv2mlastnode.gif": 2, 
  "/garp_timer_setting.asp": 1, 
  "/auth/md5.js": 13, 
  "/incl/activeX.js": 4, 
  "/pics/line_corner_lb_5x5px.gif": 4, 
  "/css/win_ns.css": 6, 
  "/browseDirectory.jsp": 1, 
  "/jscript/spconnect.js": 2, 
  "/modbus_setting.asp": 1, 
  "/master.jsp": 1, 
  "/hwinstall.htm": 1, 
  "/md5.js": 3, 
  "/snmpconfig.asp": 3, 
  "/bg.gif": 2, 
  "/url/ups1.scc": 1, 
  "/": 187, 
  "/rs-status": 1, 
  "/home.asp": 10, 
  "/bus_configuration.htm": 1, 
  "/pics/line_t_100x5px.gif": 4, 
  "/jscript/nfsserverconfig.js": 1, 
  "/setip.html": 1, 
  "/img/pxclogo.gif": 20, 
  "/robots.txt": 11, 
  "/port_setting726.asp": 2, 
  "/name.asp": 2, 
  "/dip_switch_setting.asp": 1, 
  "/jscript/powerunitmanage.js": 1, 
  "/jscript/syslogserverconfig.js": 2, 
  "/local_diagnostics.htm": 1, 
  "/jscript/slidemenu.js": 6, 
  "/powermanage.asp": 1, 
  "/ipconfig.asp": 3, 
  "/jscript/util.js": 4, 
  "/deviceinfo.htm": 2, 
  "/auth/led_auth.asp": 13, 
  "/images/ws_button3.gif": 4, 
  "/flumemaster.jsp": 1, 
  "/goform/EventLogList": 2, 
  "/settimeouts.html": 1, 
  "/tagbase_vlan_setting_show.asp": 1, 
  "12.1.2": 2, 
  "/img/device_s.gif": 20, 
  "/ftv2folderclosed.gif": 2, 
  "/favicon.ico": 81, 
  "/showstatus.html": 1, 
  "/techdata.htm": 2, 
  "/pics/blank.gif": 4, 
  "/dfshealth.jsp": 1, 
  "/images/block.gif": 3, 
  "/css/common.css": 6, 
  "/ftv2vertline.gif": 2, 
  "/stserial.asp": 80, 
  "/nice ports,/Trinity.txt.bak": 8, 
  "/port_setting_show726.asp": 2, 
  "/userloggedonlist.asp": 1, 
  "/reset_button.gif": 2, 
  "/login.asp": 5, 
  "/monitor_statistic_cnt_show.asp": 2, 
  "/getstatus.html": 4737, 
  "/ups1.scc": 1, 
  "/auth/topplan_auth.asp": 15, 
  "/pics/logo_70x29px.gif": 4, 
  "/view": 1, 
  "/ws_button3.gif": 2, 
  "sip:nm SIP/2.": 4, 
  "/pics/space.gif": 4, 
  "/jscript/rhostaccessctrl.js": 2, 
  "/powerconfig.asp": 1, 
  "/tagbase_vlan_setting.asp": 1, 
  "/ftv2node.gif": 2, 
  "/remote_diagnostics.htm": 1, 
  "/images/on.gif": 2, 
  "/jscript/webserverconfig.js": 1, 
  "/auth/loginin.gif": 13, 
  "/left_down_logo.asp": 2, 
  "/auth/accountpassword.asp": 13, 
  "/ftv2blank.gif": 2, 
  "/logo1.gif": 2, 
  "/images/logo.gif": 4, 
  "/rhostaccessctrl.asp": 2, 
  "/ipconfig.htm": 2, 
  "/auth/logo1.gif": 13, 
  "/view/index.shtml": 7, 
  "/ddnsconfig.asp": 2, 
  "/tcpserviceconfig.asp": 1, 
  "/auth/logo2_EDS-508A.gif": 8, 
  "/auth/name_auth.asp": 13, 
  "/monitor_port.asp": 2, 
  "/css/digistyle.css": 4, 
  "/pics/stop_button_27x27px.gif": 4, 
  "/pcp_configuration.htm": 1, 
  "/pics/line_b_100x5px.gif": 4, 
  "-": 45, 
  "/img/checkbox_chk.gif": 1, 
  "/view/view.shtml": 4, 
  "/img/hw_installation.gif": 1, 
  "/jscript/spconfig.js": 3, 
  "/jscript/snmpconfig.js": 3, 
  "/view/": 9, 
  "/vlan_set.asp": 1, 
  "/mjpg/video.mjpg": 7, 
  "/log_setting.asp": 2, 
  "/smtpconfig.asp": 1, 
  "/jscript/validation.js": 4, 
  "/clear_button.gif": 2, 
  "/phoenix_fl.js": 20, 
  "/jscript/smtpconfig.js": 1, 
  "/services.htm": 3, 
  "/pics/line_corner_rt_5x5px.gif": 4, 
  "/phoenix_fl.css": 20, 
  "/nfsserverconfig.asp": 1, 
  "/jscript/syslogging.js": 1, 
  "/auth/logo3.gif": 13, 
  "/stnetwork.asp": 1, 
  "/pics/gray_t_5x50px.gif": 4, 
  "/auth/auth.asp": 23, 
  "/jscript/default.js": 4, 
  "/d4-43.js": 2, 
  "/left.asp": 2, 
  "/jscript/ddnsconfig.js": 2, 
  "/img/sel.gif": 16, 
  "/ethernetconfig.asp": 1
}

In [4]:

    

# Let's stack user agents
user_agents = {}
for line in http_data:
    if len(line.split('\t')) > 12:
        user_agent = line.split('\t')[11]
        if user_agent not in user_agents.keys():
            user_agents[user_agent] = 1
        else:
            user_agents[user_agent] += 1

print(json.dumps(user_agents,indent=2))


    

    




{
  "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:39.0) Gecko/20100101 Firefox/39.0": 327, 
  "Mozilla/5.0 (compatible; Nmap Scripting Engine; http://nmap.org/book/nse.html)": 171, 
  "-": 103, 
  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.64 Safari/537.36": 5045, 
  "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:28.0) Gecko/20100101 Firefox/28.0": 12, 
  "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:41.0) Gecko/20100101 Firefox/41.0": 99, 
  "Wget/1.16.1 (linux-gnu)": 1
}

In [5]:

    

# Let's search for the nmap user agent
suspicious_user_agents = ['Mozilla/5.0 (compatible; Nmap Scripting Engine; http://nmap.org/book/nse.html)']
nmap_scanned_hosts = {}
for line in http_data:
    if len(line.split('\t')) > 12:
        timestamp = line.split('\t')[0]
        client = line.split('\t')[2]
        server = line.split('\t')[4]
        user_agent = line.split('\t')[11]
        if user_agent in suspicious_user_agents:
            if client not in nmap_scanned_hosts.keys():
                nmap_scanned_hosts[client] = {server:{timestamp:1}}
            elif server not in nmap_scanned_hosts[client].keys():
                nmap_scanned_hosts[client][server] = {timestamp: 1}
            elif timestamp not in nmap_scanned_hosts[client][server].keys():
                nmap_scanned_hosts[client][server][timestamp] = 1
            else:
                nmap_scanned_hosts[client][server][timestamp] += 1

print(json.dumps(nmap_scanned_hosts,indent=2))


    

    




{
  "192.168.2.42": {
    "192.168.88.115": {
      "1445425464.684730": 1, 
      "1445425489.066291": 1, 
      "1445425456.492019": 1, 
      "1445425472.897110": 1, 
      "1445425505.330748": 1, 
      "1445425497.221008": 1, 
      "1445425472.798104": 1, 
      "1445425464.734434": 1, 
      "1445425489.264708": 1, 
      "1445425481.058994": 1, 
      "1445425456.491738": 1, 
      "1445425456.492152": 1, 
      "1445425464.684854": 1, 
      "1445425521.550031": 1, 
      "1445425456.491596": 1, 
      "1445425456.492557": 1, 
      "1445425513.438493": 1, 
      "1445425480.908743": 1
    }
  }, 
  "192.168.2.64": {
    "192.168.88.25": {
      "1445422296.875484": 1, 
      "1445422290.967679": 1, 
      "1445422289.381463": 1, 
      "1445422289.591706": 1, 
      "1445422290.459930": 1, 
      "1445422323.002866": 1, 
      "1445422289.808332": 1, 
      "1445422291.185004": 1, 
      "1445422290.239258": 1, 
      "1445422296.668006": 1, 
      "1445422290.239120": 1, 
      "1445422292.854650": 1, 
      "1445422290.678547": 1, 
      "1445422290.020238": 1, 
      "1445422314.053171": 1, 
      "1445422313.799369": 1, 
      "1445422291.184861": 1, 
      "1445422300.715145": 1
    }, 
    "192.168.88.115": {
      "1445422321.290313": 1, 
      "1445422300.766784": 1, 
      "1445422320.650723": 1, 
      "1445422321.503861": 1, 
      "1445422300.184951": 1, 
      "1445422321.928420": 1, 
      "1445422320.867814": 1, 
      "1445422291.938518": 1, 
      "1445422322.355297": 1, 
      "1445422292.693354": 1, 
      "1445422321.713691": 1, 
      "1445422316.046787": 1, 
      "1445422322.142027": 1, 
      "1445422321.077807": 1, 
      "1445422291.454377": 1
    }, 
    "192.168.88.20": {
      "1445422298.992223": 1, 
      "1445422291.885333": 1, 
      "1445422302.855427": 1, 
      "1445422300.497165": 1, 
      "1445422299.414991": 1, 
      "1445422315.698055": 1, 
      "1445422300.287326": 1, 
      "1445422290.968135": 1, 
      "1445422299.207919": 1, 
      "1445422299.839276": 1, 
      "1445422298.777344": 1, 
      "1445422300.078390": 1, 
      "1445422313.532961": 1, 
      "1445422299.628075": 1
    }, 
    "192.168.88.100": {
      "1445422308.102295": 1, 
      "1445422289.380025": 1, 
      "1445422290.915620": 1, 
      "1445422297.138751": 1, 
      "1445422290.513640": 1
    }, 
    "192.168.88.51": {
      "1445422295.870961": 1, 
      "1445422300.023159": 1, 
      "1445422320.920019": 1, 
      "1445422303.707740": 1, 
      "1445422296.667868": 1, 
      "1445422289.754808": 1, 
      "1445422299.364282": 1, 
      "1445422297.667609": 1, 
      "1445422292.639583": 1, 
      "1445422298.789861": 1, 
      "1445422289.381938": 1, 
      "1445422290.520664": 1, 
      "1445422296.027733": 1, 
      "1445422300.212852": 1, 
      "1445422292.587508": 1, 
      "1445422300.341810": 1, 
      "1445422295.554722": 1, 
      "1445422299.694729": 1, 
      "1445422295.714594": 1, 
      "1445422300.498336": 1, 
      "1445422293.066879": 1, 
      "1445422292.476080": 1, 
      "1445422299.696478": 1, 
      "1445422289.592098": 1, 
      "1445422303.873797": 1, 
      "1445422300.660455": 1, 
      "1445422290.349694": 1, 
      "1445422299.260279": 1, 
      "1445422299.840329": 1, 
      "1445422289.385586": 1, 
      "1445422296.188602": 1, 
      "1445422299.518622": 1, 
      "1445422298.727806": 1, 
      "1445422320.466621": 1, 
      "1445422296.506938": 1, 
      "1445422296.349914": 1, 
      "1445422323.263679": 1, 
      "1445422296.824060": 1, 
      "1445422303.927905": 1
    }, 
    "192.168.88.49": {
      "1445422302.534936": 1, 
      "1445422292.047762": 1, 
      "1445422289.380561": 1, 
      "1445422302.965697": 1, 
      "1445422302.746772": 1, 
      "1445422291.619375": 1, 
      "1445422303.183484": 1, 
      "1445422307.565998": 1, 
      "1445422301.635377": 1, 
      "1445422313.849169": 1, 
      "1445422302.111056": 1, 
      "1445422303.397388": 1, 
      "1445422302.325429": 1, 
      "1445422301.899644": 1
    }, 
    "192.168.88.60": {
      "1445422289.865632": 1, 
      "1445422289.591967": 1, 
      "1445422291.235170": 1, 
      "1445422291.885204": 1, 
      "1445422289.381938": 1, 
      "1445422291.018922": 1, 
      "1445422306.307627": 1, 
      "1445422290.565864": 1, 
      "1445422292.319808": 1, 
      "1445422299.890418": 1, 
      "1445422292.100843": 1, 
      "1445422289.381132": 1, 
      "1445422298.992366": 1, 
      "1445422291.454248": 1, 
      "1445422289.379891": 1, 
      "1445422289.865921": 1, 
      "1445422298.777468": 1
    }, 
    "192.168.88.61": {
      "1445422300.131605": 1, 
      "1445422289.591833": 1, 
      "1445422300.988103": 1, 
      "1445422292.798306": 1, 
      "1445422289.866199": 1, 
      "1445422290.915767": 1, 
      "1445422299.679622": 1, 
      "1445422297.244478": 1, 
      "1445422300.766659": 1, 
      "1445422301.201058": 1, 
      "1445422299.466633": 1, 
      "1445422293.119720": 1, 
      "1445422300.548608": 1, 
      "1445422299.890145": 1, 
      "1445422300.339324": 1
    }, 
    "192.168.88.95": {
      "1445422289.380290": 1, 
      "1445422344.783066": 1, 
      "1445422352.905377": 1, 
      "1445422317.744378": 1, 
      "1445422321.022581": 1, 
      "1445422320.387407": 1, 
      "1445422295.370559": 1, 
      "1445422309.529967": 1, 
      "1445422336.568033": 1, 
      "1445422320.490386": 1, 
      "1445422301.580724": 1, 
      "1445422337.822249": 1, 
      "1445422305.513430": 1, 
      "1445422348.751232": 1, 
      "1445422290.347162": 1, 
      "1445422289.380169": 1
    }
  }
}

In [6]:

    

# Add up the number of requests the client made to the server
print("client ip,server ip,num requests")
suspicious_hosts = {}
for client in sorted(nmap_scanned_hosts.keys()):
    for server in sorted(nmap_scanned_hosts[client].keys()):
        print(client + "," + server + "," + str(len(nmap_scanned_hosts[client][server])))
        if client not in suspicious_hosts.keys():
            suspicious_hosts[client] = [server]
        else:
            suspicious_hosts[client].append(server)


    

    




client ip,server ip,num requests
192.168.2.42,192.168.88.115,18
192.168.2.64,192.168.88.100,5
192.168.2.64,192.168.88.115,15
192.168.2.64,192.168.88.20,14
192.168.2.64,192.168.88.25,18
192.168.2.64,192.168.88.49,14
192.168.2.64,192.168.88.51,39
192.168.2.64,192.168.88.60,17
192.168.2.64,192.168.88.61,15
192.168.2.64,192.168.88.95,16

In [7]:

    

# Write CSV file out for display/distribution in excel
with open('suspicious_http_records.csv','w') as outfile:
    outfile.write("ts,uid,id.orig_h,id.orig_p,id.resp_h,id.resp_p,trans_depth,method,host,uri,referrer,user_agent,request_body_len,response_body_len,status_code,status_msg,info_code,info_msg,filename,tags,username,password,proxied,orig_fuids,orig_mime_types,resp_fuids,resp_mime_types\n")
    for line in http_data:
        if len(line.split('\t')) > 12:
            timestamp = line.split('\t')[0]
            client = line.split('\t')[2]
            server = line.split('\t')[4]
            user_agent = line.split('\t')[11]
            uri = line.split('\t')[9]
            if client in suspicious_hosts.keys():
                if server in suspicious_hosts[client]:
                    outfile.write("\"" + line.replace("\t","\",\"") + "\"\n")