In [ ]:

    

import urllib2
import json
import os
import csv 

# getting date from the parent path. 
path = os.getcwd().split("/") 
date = path[len(path)-1]   
dsource = path[len(path)-2]  
dpath = '/'.join(['data' if var == 'ipynb' else var for var in path]) + '/'

sconnect = dpath + 'proxy_scores.tsv'
sconnectbu = dpath + 'proxy_scores_bu.tsv'
score_tmp = dpath + 'proxy_tmp.tsv'  
score_fbk = dpath + 'proxy_scores_fb.csv'


    

In [ ]:

    

try:
    import ipywidgets as widgets # For jupyter/ipython >= 1.4
except ImportError:
    from IPython.html import widgets
from IPython.display import display, HTML, clear_output, Javascript 

def fill_list(list_control,source):
    options_list = ['--Select--'] 
    options_list.extend([s for s in source])
    list_control.options = options_list

# URI panel
uri_header = widgets.HTML(value="URI", height='10%', width='100%')
uri_select = widgets.Select(height='100%', width='99%')

uri_select._css = (
    (None, 'height', '90%'),
    (None, 'width', '95%'),
    ('select', 'overflow-x', 'auto'),
    ('select', 'width', '100%'),
    ('select', 'margin', 0)
)

uri_box = widgets.Box(width='70%', height='100%')
uri_box.children = [uri_header, uri_select]

# Actions Panel
actions_header = widgets.HTML(value=" ", width='100%', height='10%')
quick_text = widgets.Text(value='', width='100%', placeholder='Quick scoring')
quick_text._css = (
    (None, 'width', '100%'),
)
rating_btn = widgets.RadioButtons(description='Rating:', options=['1', '2', '3'], width='100%')
assign_btn = widgets.Button(description='Score', width='45%')
assign_btn.button_style = 'primary'
save_btn = widgets.Button(description='Save', width='45%')
save_btn.button_style = 'primary'
save_btn._css = (
    (None, 'margin-left', '10%'),
)

actions_box = widgets.Box(width='30%', height='100%')
actions_box.children = [actions_header,quick_text,rating_btn, assign_btn,save_btn]

scoring_form = widgets.HBox(width='90%', height=250)
scoring_form.children = [uri_box, actions_box]


def data_loader(): 
    us_uris = []

    with open(sconnect, 'r') as f:
        reader = csv.DictReader(f, delimiter='\t')
        
        for row in reader: 
            if row['fulluri'] not in us_uris and row['uri_sev'] == '0': 
                us_uris.append(row['fulluri'])

    fill_list(uri_select,us_uris)
    uri_select.value = "--Select--"   

display(Javascript("$('.widget-area > .widget-subarea > *').remove();"))
data_loader()
display(scoring_form)


    

In [ ]:

    

import csv
import datetime
import subprocess 

def assign_score(b):
    scored_threats = []
    clear_output()
    uri = quick_text.value or uri_select.value
    uri_sev = int(rating_btn.selected_label) if not "--Select--" in uri_select.value else ""

    with open(sconnect, 'r') as f:
        reader = csv.DictReader(f, delimiter='\t')
        rowct = 0
        with open(score_tmp, 'w') as score:
            wr = csv.DictWriter(score, delimiter='\t', quoting=csv.QUOTE_NONE, fieldnames=reader.fieldnames)            
            wr.writeheader()
                
            for row in reader:
                if row['fulluri'] == uri:
                    row['uri_sev'] = uri_sev
                    scored_threats.append(row)                    
                    rowct += 1
                try:
                    wr.writerow(row)
                except:
                    print str(row)
                    
        #works on the feedback tab-separated file
        if not os.path.exists(score_fbk):  
            with open(score_fbk, 'w') as feedback:
                wr = csv.DictWriter(feedback, delimiter='\t', quoting=csv.QUOTE_NONE, fieldnames=reader.fieldnames)            
                wr.writeheader()
            
        with open(score_fbk, 'a') as feedback:
            for row in scored_threats:
                wr = csv.DictWriter(feedback, delimiter='\t', quoting=csv.QUOTE_NONE, fieldnames=reader.fieldnames)            
                wr.writerow(row)
  
    clear_output()
    print "{0} matching requests scored".format(rowct)
    !mv $score_tmp $sconnect
    data_loader()
    uri_select.value = "--Select--"
    quick_text.value = ""


def save(b):   
    clear_output()
    display(Javascript("$('.widget-area > .widget-subarea > *').remove();"))
    data_loader()
    display(scoring_form)
    display(Javascript('reloadParentData();')) 
    ml_feedback()
    print "Suspicious requests successfully updated"


assign_btn.on_click(assign_score)
save_btn.on_click(save)
        

def ml_feedback():
    dst_name = os.path.basename(sconnect)
    str_fb="DSOURCE={0} &&\
        FDATE={1} &&\
        source /etc/duxbay.conf &&\
        usr=$(echo $LUSER | cut -f3 -d'/') &&\
        mlnode=$MLNODE &&\
        lpath=$LPATH &&\
        scp {2} $usr@$mlnode:$lpath/{3}".format(dsource,date,score_fbk,dst_name)
    subprocess.call(str_fb, shell=True)


    

In [ ]:

    

# !cp $sconnectbu $sconnect