In [1]:

    

import urllib2
import json
import os
import csv

# getting date from the parent path. 
path = os.getcwd().split("/") 
date = path[len(path)-1]   
dsource = path[len(path)-2]  
dpath = '/'.join(['data' if var == 'ipynb' else var for var in path]) + '/'

with open('/etc/duxbay.conf') as conf:
    for line in conf.readlines():
        if "LUSER" in line: LUSER = line.split("=")[1].strip('\n').replace("'","");  break

sconnect = dpath + 'proxy_scores.csv'
sconnectbu = dpath + 'proxy_scores_bu.csv'
score_tmp = dpath + 'proxy_tmp.csv'


    

In [2]:

    

try:
    import ipywidgets as widgets # For jupyter/ipython >= 1.4
except ImportError:
    from IPython.html import widgets
from IPython.display import display, HTML, clear_output, Javascript 

def fill_list(list_control,source):
    options_list = ['--Select--'] 
    options_list.extend([s for s in source])
    list_control.options = options_list

# URI panel
uri_header = widgets.HTML(value="URI", height='10%', width='100%')
uri_select = widgets.Select(height='100%', width='99%')

uri_select._css = (
    (None, 'height', '90%'),
    (None, 'width', '95%'),
    ('select', 'overflow-x', 'auto'),
    ('select', 'width', '100%'),
    ('select', 'margin', 0)
)

uri_box = widgets.Box(width='70%', height='100%')
uri_box.children = [uri_header, uri_select]

# Actions Panel
actions_header = widgets.HTML(value=" ", width='100%', height='10%')
quick_text = widgets.Text(value='', width='100%', placeholder='Quick scoring')
quick_text._css = (
    (None, 'width', '100%'),
)
rating_btn = widgets.RadioButtons(description='Rating:', options=['1', '2', '3'], width='100%')
assign_btn = widgets.Button(description='Score', width='45%')
assign_btn.button_style = 'primary'
save_btn = widgets.Button(description='Save', width='45%')
save_btn.button_style = 'primary'
save_btn._css = (
    (None, 'margin-left', '10%'),
)

actions_box = widgets.Box(width='30%', height='100%')
actions_box.children = [actions_header,quick_text,rating_btn, assign_btn,save_btn]

scoring_form = widgets.HBox(width='90%', height=250)
scoring_form.children = [uri_box, actions_box]


def data_loader(): 
    us_uris = []

    with open(sconnect, 'r') as f:
        reader = csv.DictReader(f, delimiter=',')
        
        for row in reader: 
            if row['fulluri'] not in us_uris and row['uri_sev'] == '0': 
                us_uris.append(row['fulluri'])

    fill_list(uri_select,us_uris)
    uri_select.value = "--Select--"   


display(Javascript("$('.widget-area > .widget-subarea > *').remove();"))
data_loader()
display(scoring_form)


    

    




:0: FutureWarning: IPython widgets are experimental and may change in the future.






    

In [3]:

    

import csv
import datetime
import subprocess 


def assign_score(b):
    score_values = []
    
    clear_output()

    uri = quick_text.value or uri_select.value
    uri_sev = int(rating_btn.selected_label) if not "--Select--" in uri_select.value else ""

    with open(sconnect, 'r') as f:
        reader = csv.DictReader(f, delimiter=',')
        rowct = 0
        with open(score_tmp, 'w') as score:
            wr = csv.writer(score, delimiter=',', quoting=csv.QUOTE_NONE)
            wr = csv.DictWriter(score, delimiter=',', quoting=csv.QUOTE_NONE, fieldnames=reader.fieldnames)
            
            wr.writeheader()

            for row in reader:
                if row['fulluri'] == uri:
                    row['uri_sev'] = uri_sev
                    rowct += 1

                try:
                    wr.writerow(row)
                except:
                    print str(row)

    clear_output()
    print "{0} matching requests scored".format(rowct)
    !mv $score_tmp $sconnect

    data_loader()

    uri_select.value = "--Select--"
    quick_text.value = ""


def save(b):   
    clear_output()
    display(Javascript("$('.widget-area > .widget-subarea > *').remove();"))
    data_loader()
    display(scoring_form)
    display(Javascript('window.parent.iframeReloadHook && window.parent.iframeReloadHook();'))
    ml_feedback()
    print "Suspicious requests successfully updated"


assign_btn.on_click(assign_score)
save_btn.on_click(save)
        

def ml_feedback():
    str="DSOURCE=$dsource &&\
        FDATE={0} &&\
        source /etc/duxbay.conf &&\
        usr=$(echo $LUSER | cut -f3 -d'/') &&\
        mlnode=$MLNODE &&\
        lpath=$LPATH &&\
        scp {1} $usr@$mlnode:$lpath/".format(date,sconnect)  

    subprocess.call(str, shell=True)


    

In [4]:

    

#!cp $sconnectbu $sconnect


    

In [ ]: