Detailed Evaluation of Hand-Tuned algorithm

In [1]:

    

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import cross_val_score
from sklearn.metrics import matthews_corrcoef
from sklearn.metrics import precision_score
from sklearn.metrics import f1_score
from IPython.display import Image  
import pandas
import seaborn
import matplotlib.pyplot as plt
%matplotlib inline


    

In [4]:

    

def get_pd_files(folder):
    sibf = folder + "hosts.csvcapture.pcap.ts.siblingresult.csv"
    nonsibf = folder + "hosts.csv__nonsiblings_seed1_n*capture.pcap.ts.siblingresult.csv"
    import glob
    for filename in glob.glob(nonsibf):
        nonsibf = filename
    import os.path
    if os.path.isfile(sibf) and os.path.isfile(nonsibf):
        print("Loading from filenames {} and {}".format(sibf, nonsibf))
    else:
        print("Files not found {} and {}".format(sibf, nonsibf))
        
    sib = pd.read_csv(sibf, index_col=0)
    sib['label'] = 1
    nonsib = pd.read_csv(nonsibf, index_col=0)
    nonsib['label'] = 0
    print("Read {} siblings and {} non-siblings from files.".format(len(sib), len(nonsib)))
    return sib, nonsib
       
def dec2prd_ours(df):
    df.loc[df["decision"].str.contains("^sibling"), "dec_prd"] =  1
    df.loc[df["decision"].str.contains("^non-sibling"), "dec_prd"] =  0
    return  # df is changed in place so no returning necessary

def dec2prd_bev(df):
    df.loc[df["dec_bev"].str.contains("^sibling"), "dec_bev_prd"] =  1
    df.loc[df["dec_bev"].str.contains("^non-sibling"), "dec_bev_prd"] =  0
    return  # df is changed in place so no returning necessary

def mix_sib_nonsib(sib, nonsib, mode, rs=42):
    if mode == "equal":
        nonsibint = nonsib.sample(n=len(sib), replace=True, weights=None, random_state=rs)
    else:
        nonsibint = nonsib
    datain = pd.concat([sib,nonsibint])
    #print("merged shape: {}".format(datain.shape))
    #print("columns: {}".format(list(datain.columns.values)))
    return datain


def get_ouralgo_stats(sib, nonsib):
    #print("Our algo stats:")
    df = mix_sib_nonsib(sib, nonsib, "full", 42)
    df_ours = df[["label", "decision"]].copy()
    #print(df_ours[df_ours.isnull().any(axis=1)])
    dec2prd_ours(df_ours)
    undec = len(df_ours[df_ours.isnull().any(axis=1)])
    print("Our algo: Not deciding on {} pairs for unknown/error reasons.".format(undec))
    df_ours = df_ours.dropna()
    weights = get_sample_weight_one_input(df_ours)
    mcc = matthews_corrcoef(df_ours["label"], df_ours["dec_prd"], sample_weight=None)
    f1 = f1_score(df_ours["label"], df_ours["dec_prd"], sample_weight=None)
    print("Our algo stats: ({}) undecided, mcc: {}, f1: {}".format(undec, mcc, f1))
    statsv = list(stats(df_ours["label"], df_ours["dec_prd"]))
    #prec = precision_score(df_ours["label"], df ,_ours["dec_prd"], sample_weight=None)
    statsv.append(mcc)
    return statsv

def get_bev_stats(sib, nonsib):
    #print("Beverly algo stats:")
    df = mix_sib_nonsib(sib, nonsib, "full", 42)
    #print(df[df.isnull().any(axis=1)])
    df_tmp = df[["label", "dec_bev"]].copy()
    dec_nan = len(df_tmp[df_tmp["dec_bev"].isnull() == True])
    df_tmp = df_tmp[df_tmp["dec_bev"].isnull() == False]
    dec2prd_bev(df_tmp)
    undec = len(df_tmp[df_tmp.isnull().any(axis=1)])
    df_tmp = df_tmp.dropna()
    weights = get_sample_weight_one_input(df_tmp)
    mcc = matthews_corrcoef(df_tmp["label"], df_tmp["dec_bev_prd"], sample_weight=None)
    f1 = f1_score(df_tmp["label"], df_tmp["dec_bev_prd"], sample_weight=None)
    print("Beverly algo: Not deciding on {} pairs for NaN and {} pairs for unknown/error reasons.".format(dec_nan, undec))
    print("Beverly algo stats: ({}) undecided, mcc: {}, f1: {}".format(undec, mcc, f1))
    statsv =  list(stats(df_tmp["label"], df_tmp["dec_bev_prd"]))
    statsv.append(mcc)
    return statsv
    

def match_nonsibs_slow(sib, nonsib, rs=42):
    ctr = 0 
    for i, ii in sib.iterrows():
        for j, jj in sib.iterrows():
            if ii[1] != jj[1]:
                nscand = ii[1] + "_+_" +  jj[1]
                #print(nscand)
                ctr += 1 
                #if not (nonsib["domain"] == nscand).any():
                #   print("fail for {}".format(nscand))
                #  return
        #print(ctr)
    return
                
def match_nonsibs(sib, nonsib, rs=42):
    ctr = 0 
    a = []
    sd = dict() # siblings dict
    nsd = dict()  # non siblings dict
    for i in sib.itertuples():
        sd[i[0]] = 0
    for i in nonsib.itertuples():
        nsd[i[0]] = 0
    nscand = dict()
    #nstmp = pandas.DataFrame()
    for i in sd.keys():
        for j in sd.keys():
            if i != j:
                nscandstr = i + "_+_" +  j
                nscand[nscandstr] = 1
    print("Generated {} non-sibling candidates from {} siblings.".format(len(nscand), len(sib)))
    fails = []
    for k1 in nsd.keys():
        if k1 not in nscand.keys():
            fails.append(k1)
            #print("fail! {} ".format(i))    
    nsfiltered = nonsib.copy()
    nsfiltered.drop(fails, inplace=True)
    return nsfiltered

def assign_groups_old(datain):
    datain["group"] = "servers"
    datain.loc[datain["domain"].str.contains("nlnog.net"), "group"] = "nlnog"
    datain.loc[datain["domain"].str.contains("RA_"), "group"] = "RA"
    datain.loc[datain["domain"].str.extract("RA_([0-9]{4})") < 6019, "group"] = "RAv1"
    datain.loc[datain["domain"].str.extract("RA_([0-9]{4})") > 6018, "group"] = "RAv2"
    return
    #groups = datain["group"].as_matrix()
    #del datain["group"]
    #return groups

def assign_groups(datain):
    datain["group"] = "servers"
    #sib.loc[sib.index.str.contains("nlnog.net"), "group"] = "nlnog"
    datain.loc[datain.index.str.contains("nlnog.net"), "group"] = "nlnog"
    datain.loc[datain.index.str.contains("RA_"), "group"] = "RA"
    datain["ra_id"] = datain.index.str.extract("RA_([0-9]{4})", expand=False).astype(float).fillna(0).astype(int) 
    #datain.index.str.extract("RA_([0-9]{4})", expand=False).astype(float).fillna(0).astype(int) > 6018
    datain.loc[(datain.ra_id > 5999) & (datain.ra_id < 6019), "group"] = "RAv1"
    datain.loc[datain.ra_id > 6018, "group"] = "RAv2"    
    #datain.loc[datain.index.str.extract("RA_([0-9]{4})", expand=False) > 6018, "group"] = "RAv2"
    groups = datain["group"].as_matrix()
    return groups
    
    
def prune_datain(datain):
    errorc = len(datain[datain["decision"].str.contains("ERROR|error") == True])
    print("Removing {} errors values from datain.".format(errorc))
    datain = datain[datain["decision"].str.contains("ERROR|error") == False]

    hzdiffc = len(datain[datain["hzdiff"] != 0])
    print("Deciding {} hzdiff hosts as non-sib, stats:".format(hzdiffc))
    lbl = datain[datain["hzdiff"] != 0]["label"]
    prd = lbl.copy()
    prd[:] = 0
    stats(lbl,prd)
    dataout = datain[datain["hzdiff"] == 0]
    #  datain = datain[datain["domain"].str.contains("nlnog.net") == True]
    return dataout, lbl, prd

def prune_data_for_ml(datain):
    # just kick hzdiff out
    # problem: NaNs might be in non-feature columns such as RA_ID
    erridx = datain[datain.decision.str.contains("ERROR|error") == True].index
    labels, features = make_labels_features(datain)
    naidx = datain[features.isnull().any(axis=1) == True].index
    bothidx = erridx | naidx
    dataout = datain.drop(bothidx)
    #dataout = dataout[dataout.decision.str.contains("ERROR|error") == False]
    # TODO: should also calculcate stats on this!
    lbl = datain.loc[bothidx, "label"]
    prd = lbl.copy()
    prd[:] = 0
    stats(lbl,prd)    
    print("Removing {} rows with error results and {} rows with NaNs (typically hz different) from a \
    total of {} entries, resulting in {} entries.".format(
            len(erridx), len(naidx), len(datain), len(dataout)))
    return dataout, lbl, prd


def stats(lbl, prd):
        tp = np.sum((lbl == 1) & (prd == 1)) 
        fp = np.sum(lbl < prd ) 
        tn = np.sum((lbl == 0) & (prd == 0)) 
        fn = np.sum(lbl > prd ) 
        try:
            prec =  round(100*tp/(tp+fp),2) # TPR?
            recall = round(100*tp/(tp+fn),2) 
            spec= round(100*tn/(tn+fp),2) # TNR?
            acc = round(100*(tn+tp)/(tn+fn+fp+tp),2)
        except ZeroDivisionError as e:
            print("Catching ZeroDivisionError at stats!")
            prec = 0
            recall = 0
            spec = 0
            acc = 0
        print("Correct: {}, incorrect {}, TP {}, FP {}, TN {}, FN{}, Prec. {}, Rec. {}, Spec. {}, Acc. {}%".format(
        np.sum(lbl == prd),
        np.sum(lbl != prd),
        tp, fp, tn, fn, 
        prec, recall, spec, acc
        ))
        return prec, recall, spec, acc
        
def make_labels_features(dfin):
    labels = dfin["label"]
    features = dfin[["hzdiff", "hzr2diff", "timestamps_diff", "adiff", 
                        "theta", "r2diff", "ott_rng_diff_rel", "optsdiff",
                       "perc_85_val"]].copy()
    features["hzr2mean"] = (dfin["hz4r2"] + dfin["hz6r2"])  / 2.0
    features["r2mean"] = (dfin["r4_sqr"] + dfin["r6_sqr"]) / 2.0     
    features["ott_rng_mean"] = (dfin["ott4_rng"] + dfin["ott6_rng"]) / 2.0
    features["splinediff_scaled"] = dfin["perc_85_val"] / features["ott_rng_mean"]
    return labels, features   

def get_sample_weight(sib, nonsib):
    # WIP TODO
    #siblings = len(dfin[dfin["label"] == 1])
    #nonsiblings = len(datain[datain["label"] == 0])
    sl = len(sib)
    nsl = len(nonsib)
    tl = sl + nsl
    nsw = sl / tl
    sw = nsl / tl
    print("Found {} sibs and {} nonsibs, weights: {} and {}".format(sl, nsl, sw, nsw))
    weight = np.zeros(len(datain))
    weight = np.float32(datain["label"].as_matrix())
    weight[weight == 1] = sw
    weight[weight == 0] = nsw
    
    
def get_sample_weight_one_input(dfin):
    sl = len(dfin[dfin["label"] == 1])
    nsl = len(dfin[dfin["label"] == 0])
    tl = sl + nsl
    nsw = sl / tl
    sw = nsl / tl
    weight = np.zeros(len(dfin))
    weight = np.float32(dfin["label"].as_matrix())
    weight[weight == 1] = sw
    weight[weight == 0] = nsw
    print("Found {} sibs and {} nonsibs, weights: {} and {}, #weights: {}".format(
        sl, nsl, round(sw,4), round(nsw,4), len(weight)))
    return weight


# functions for ML with proprtional group sampling
def split_stratified_groups(sib, splits, nr):
    from sklearn.model_selection import KFold # non-overlapping!
    groups = assign_groups(sib)
    groupset = set(groups)
    gsibdf_train = pd.DataFrame(columns=sib.columns)
    gsibdf_test = pd.DataFrame(columns=sib.columns)
    for i in groupset:
        groupsib = sib[sib["group"] == i].copy()
        if len(groupsib ) < splits:
            # can not split into more folds than files...
            print("ERROR: more splits ({}) than samples ({}), reducing to sample nr".format(splits, len(groupsib)))
            splits = len(groupsib)
        #print("## GROUP: {} with {} elements.".format(i, len(groupsib)))
        ks = KFold(n_splits=splits, random_state=42, shuffle=True)
        labels, features = make_labels_features(groupsib)
        ctr = -1
        for train_index, test_index in ks.split(groupsib):
            ctr += 1                
            if (ctr == nr):
            #print("TRAIN:", train_index, "TEST:", test_index)
                gsibdf_train = gsibdf_train.append(groupsib.iloc[train_index])
                gsibdf_test = gsibdf_test.append(groupsib.iloc[test_index])
                break
    return [gsibdf_train, gsibdf_test]


def dt_train(labels, features, weight, rs=42):
    estimator = DecisionTreeClassifier(max_depth=30, min_samples_leaf=5, random_state=42)
    est = estimator.fit(features, labels, sample_weight=weight)
    return est

def kfold_train_test(sib, nonsib):
    kfolds = 10
    stats_train_error = np.empty((10,5), dtype=float)
    stats_test_error = np.empty((10,5), dtype=float)
    graphs = []
    for i in range(10):
        print("Round {}".format(i))
        # pick proportionally from each group
        train_sib, test_sib = split_stratified_groups(sib, 10, i)
        # create, select, and mix matching nonsibs
        train_nonsib = match_nonsibs(train_sib, nonsib)
        test_nonsib = match_nonsibs(test_sib, nonsib)
        train = mix_sib_nonsib(train_sib,train_nonsib, "all")
        # prune NaNs out
        train, train_prune_lbl, train_prune_prd = prune_data_for_ml(train)
        test = mix_sib_nonsib(test_sib,test_nonsib, "all")
        test, test_prune_lbl, test_prune_prd = prune_data_for_ml(test)
        # split out features, labels, and weights
        train_lbl, train_ftr = make_labels_features(train)
        test_lbl, test_ftr = make_labels_features(test)
        train_weight = get_sample_weight_one_input(train)
        test_weight = get_sample_weight_one_input(test)
        # train estimator
        est = dt_train(train_lbl, train_ftr, train_weight)   
        mcc = matthews_corrcoef(train_lbl, est.predict(train_ftr), sample_weight=train_weight)
        statsv = list(stats(train_lbl, est.predict(train_ftr)))
        statsv.append(mcc)
        stats_train_error[i] = statsv
        #print("test error: mcc of {}".format(mcc))
        mcc = matthews_corrcoef(test_lbl, est.predict(test_ftr), sample_weight=test_weight)
        statsv = list(stats(test_lbl, est.predict(test_ftr)))
        statsv.append(mcc)
        stats_test_error[i] = statsv
        #stats_test_error[i]  =  stats(test_lbl, est.predict(test_ftr))
        graph = dt_plot(est, train_ftr)
        graphs.append(graph)
        #Image(graph.create_png())  
    return stats_train_error, stats_test_error, graphs


    

In [5]:

    

for i in range(1,2):
    print("############# Round {} ##############".format(i))
    sib, nonsib = get_pd_files("../../../gt{}/".format(i))
    #print("Columns: {}".format(list(sib.columns.values)))
    get_ouralgo_stats(sib, nonsib)


    

    




############# Round 1 ##############
Loading from filenames ../../../gt1/hosts.csvcapture.pcap.ts.siblingresult.csv and ../../../gt1/hosts.csv__nonsiblings_seed1_n679capture.pcap.ts.siblingresult.csv
Read 279 siblings and 82026 non-siblings from files.
Our algo: Not deciding on 70 pairs for unknown/error reasons.
Found 261 sibs and 81974 nonsibs, weights: 0.9968 and 0.0032, #weights: 82235
Our algo stats: (70) undecided, mcc: 0.988402745932266, f1: 0.9883720930232558
Correct: 82229, incorrect 6, TP 255, FP 0, TN 81974, FN6, Prec. 100.0, Rec. 97.7, Spec. 100.0, Acc. 99.99%

In [11]:

    

runs = 8
hlostats = np.zeros((runs-1,5), dtype=float)
hlbstats = np.zeros((runs-1,5), dtype=float)
gsdo =  dict() # group stats dict
gsdb =  dict() # group stats dict
mlstatsd_tee = dict()
mlstatsd_tre = dict()
graphs = []
for i in range(2,runs):
    print("############# Round {} ##############".format(i))
    sib, nonsib = get_pd_files("../../../algo-eval/gt{}/".format(i)) ## folder algo-eval makes this the magic!
    # high level
    hlostats[i - 1] = tuple(get_ouralgo_stats(sib, nonsib))
    hlbstats[i - 1] = get_bev_stats(sib, nonsib)
    
    # group-level
    groups = assign_groups(sib)
    groupset = set(groups)
    for j in groupset:
        if j not in gsdo:
            gsdo[j] = np.zeros((runs,5), dtype=float)
            gsdb[j] = np.zeros((runs,5), dtype=float)
        print("## GROUP: {}".format(j))
        groupsib = sib[sib["group"] == j].copy()
        groupnonsib = match_nonsibs(groupsib, nonsib)
        gsdo[j][i-1] = get_ouralgo_stats(groupsib, groupnonsib)
        gsdb[j][i-1] = get_bev_stats(groupsib, groupnonsib)
    # decision-tree
    #mlstatsd_tre[str(i)+"_tre"], mlstatsd_tee[str(i)+"_tee"], graph = kfold_train_test(sib, nonsib) # returns 2 sets of 10x4 arrays
    #graphs.append(graph)


    

    




############# Round 2 ##############
Loading from filenames ../../../algo-eval/gt2/hosts.csvcapture.pcap.ts.siblingresult.csv and ../../../algo-eval/gt2/hosts.csv__nonsiblings_seed1_n407_capture.pcap.ts.siblingresult.csv
Read 369 siblings and 145041 non-siblings from files.
Our algo: Not deciding on 32 pairs for unknown/error reasons.
Found 339 sibs and 145039 nonsibs, weights: 0.9977 and 0.0023, #weights: 145378
Our algo stats: (32) undecided, mcc: 0.9806123557438454, f1: 0.9805680119581464
Correct: 145365, incorrect 13, TP 328, FP 2, TN 145037, FN11, Prec. 99.39, Rec. 96.76, Spec. 100.0, Acc. 99.99%
Found 367 sibs and 145039 nonsibs, weights: 0.9975 and 0.0025, #weights: 145406
Beverly algo: Not deciding on 4 pairs for NaN and 0 pairs for unknown/error reasons.
Beverly algo stats: (0) undecided, mcc: 0.08428963703551832, f1: 0.01924351260370557
Correct: 108405, incorrect 37001, TP 363, FP 36997, TN 108042, FN4, Prec. 0.97, Rec. 98.91, Spec. 74.49, Acc. 74.55%
## GROUP: servers
Generated 182 non-sibling candidates from 14 siblings.
Our algo: Not deciding on 0 pairs for unknown/error reasons.
Found 14 sibs and 182 nonsibs, weights: 0.9286 and 0.0714, #weights: 196
Our algo stats: (0) undecided, mcc: 1.0, f1: 1.0
Correct: 196, incorrect 0, TP 14, FP 0, TN 182, FN0, Prec. 100.0, Rec. 100.0, Spec. 100.0, Acc. 100.0%
Found 14 sibs and 182 nonsibs, weights: 0.9286 and 0.0714, #weights: 196
Beverly algo: Not deciding on 0 pairs for NaN and 0 pairs for unknown/error reasons.
Beverly algo stats: (0) undecided, mcc: 0.1084652289093281, f1: 0.15217391304347827
Correct: 40, incorrect 156, TP 14, FP 156, TN 26, FN0, Prec. 8.24, Rec. 100.0, Spec. 14.29, Acc. 20.41%
## GROUP: RAv2
Generated 42642 non-sibling candidates from 207 siblings.
Our algo: Not deciding on 4 pairs for unknown/error reasons.
Found 205 sibs and 42640 nonsibs, weights: 0.9952 and 0.0048, #weights: 42845
Our algo stats: (4) undecided, mcc: 0.9802498011436569, f1: 0.9802955665024631
Correct: 42837, incorrect 8, TP 199, FP 2, TN 42638, FN6, Prec. 99.0, Rec. 97.07, Spec. 100.0, Acc. 99.98%
Found 205 sibs and 42640 nonsibs, weights: 0.9952 and 0.0048, #weights: 42845
Beverly algo: Not deciding on 4 pairs for NaN and 0 pairs for unknown/error reasons.
Beverly algo stats: (0) undecided, mcc: 0.05440088590079272, f1: 0.015414840562188304
Correct: 16785, incorrect 26060, TP 204, FP 26059, TN 16581, FN1, Prec. 0.78, Rec. 99.51, Spec. 38.89, Acc. 39.18%
## GROUP: nlnog
Generated 18360 non-sibling candidates from 136 siblings.
Our algo: Not deciding on 28 pairs for unknown/error reasons.
Found 108 sibs and 18360 nonsibs, weights: 0.9942 and 0.0058, #weights: 18468
Our algo stats: (28) undecided, mcc: 0.9764445970890093, f1: 0.976303317535545
Correct: 18463, incorrect 5, TP 103, FP 0, TN 18360, FN5, Prec. 100.0, Rec. 95.37, Spec. 100.0, Acc. 99.97%
Found 136 sibs and 18360 nonsibs, weights: 0.9926 and 0.0074, #weights: 18496
Beverly algo: Not deciding on 0 pairs for NaN and 0 pairs for unknown/error reasons.
Beverly algo stats: (0) undecided, mcc: 0.11735758956306333, f1: 0.042061986084756475
Correct: 12438, incorrect 6058, TP 133, FP 6055, TN 12305, FN3, Prec. 2.15, Rec. 97.79, Spec. 67.02, Acc. 67.25%
## GROUP: RAv1
Generated 132 non-sibling candidates from 12 siblings.
Our algo: Not deciding on 0 pairs for unknown/error reasons.
Found 12 sibs and 132 nonsibs, weights: 0.9167 and 0.0833, #weights: 144
Our algo stats: (0) undecided, mcc: 1.0, f1: 1.0
Correct: 144, incorrect 0, TP 12, FP 0, TN 132, FN0, Prec. 100.0, Rec. 100.0, Spec. 100.0, Acc. 100.0%
Found 12 sibs and 132 nonsibs, weights: 0.9167 and 0.0833, #weights: 144
Beverly algo: Not deciding on 0 pairs for NaN and 0 pairs for unknown/error reasons.
Beverly algo stats: (0) undecided, mcc: 0.0, f1: 0.15384615384615385
Correct: 12, incorrect 132, TP 12, FP 132, TN 0, FN0, Prec. 8.33, Rec. 100.0, Spec. 0.0, Acc. 8.33%
############# Round 3 ##############
Loading from filenames ../../../algo-eval/gt3/hosts.csvcapture.pcap.ts.siblingresult.csv and ../../../algo-eval/gt3/hosts.csv__nonsiblings_seed1_n407_capture.pcap.ts.siblingresult.csv






    




/usr/local/lib/python3.5/dist-packages/sklearn/metrics/classification.py:516: RuntimeWarning: invalid value encountered in double_scalars
  mcc = cov_ytyp / np.sqrt(var_yt * var_yp)






    




Read 370 siblings and 145804 non-siblings from files.
Our algo: Not deciding on 44 pairs for unknown/error reasons.
Found 332 sibs and 145798 nonsibs, weights: 0.9977 and 0.0023, #weights: 146130
Our algo stats: (44) undecided, mcc: 0.9740231486636811, f1: 0.9738863287250383
Correct: 146113, incorrect 17, TP 317, FP 2, TN 145796, FN15, Prec. 99.37, Rec. 95.48, Spec. 100.0, Acc. 99.99%
Found 370 sibs and 145798 nonsibs, weights: 0.9975 and 0.0025, #weights: 146168
Beverly algo: Not deciding on 6 pairs for NaN and 0 pairs for unknown/error reasons.
Beverly algo stats: (0) undecided, mcc: 0.08416894811169548, f1: 0.01921612894757567
Correct: 108807, incorrect 37361, TP 366, FP 37357, TN 108441, FN4, Prec. 0.97, Rec. 98.92, Spec. 74.38, Acc. 74.44%
## GROUP: servers
Generated 210 non-sibling candidates from 15 siblings.
Our algo: Not deciding on 1 pairs for unknown/error reasons.
Found 14 sibs and 210 nonsibs, weights: 0.9375 and 0.0625, #weights: 224
Our algo stats: (1) undecided, mcc: 0.9613379302377207, f1: 0.962962962962963
Correct: 223, incorrect 1, TP 13, FP 0, TN 210, FN1, Prec. 100.0, Rec. 92.86, Spec. 100.0, Acc. 99.55%
Found 15 sibs and 210 nonsibs, weights: 0.9333 and 0.0667, #weights: 225
Beverly algo: Not deciding on 0 pairs for NaN and 0 pairs for unknown/error reasons.
Beverly algo stats: (0) undecided, mcc: 0.10075854437197565, f1: 0.14150943396226415
Correct: 43, incorrect 182, TP 15, FP 182, TN 28, FN0, Prec. 7.61, Rec. 100.0, Spec. 13.33, Acc. 19.11%
## GROUP: RAv2
Generated 42642 non-sibling candidates from 207 siblings.
Our algo: Not deciding on 3 pairs for unknown/error reasons.
Found 207 sibs and 42639 nonsibs, weights: 0.9952 and 0.0048, #weights: 42846
Our algo stats: (3) undecided, mcc: 0.9729893273796464, f1: 0.972972972972973
Correct: 42835, incorrect 11, TP 198, FP 2, TN 42637, FN9, Prec. 99.0, Rec. 95.65, Spec. 100.0, Acc. 99.97%
Found 207 sibs and 42639 nonsibs, weights: 0.9952 and 0.0048, #weights: 42846
Beverly algo: Not deciding on 3 pairs for NaN and 0 pairs for unknown/error reasons.
Beverly algo stats: (0) undecided, mcc: 0.054671938147755984, f1: 0.015564202334630352
Correct: 16787, incorrect 26059, TP 206, FP 26058, TN 16581, FN1, Prec. 0.78, Rec. 99.52, Spec. 38.89, Acc. 39.18%
## GROUP: nlnog
Generated 18360 non-sibling candidates from 136 siblings.
Our algo: Not deciding on 37 pairs for unknown/error reasons.
Found 99 sibs and 18360 nonsibs, weights: 0.9946 and 0.0054, #weights: 18459
Our algo stats: (37) undecided, mcc: 0.9742876591840843, f1: 0.9740932642487047
Correct: 18454, incorrect 5, TP 94, FP 0, TN 18360, FN5, Prec. 100.0, Rec. 94.95, Spec. 100.0, Acc. 99.97%
Found 136 sibs and 18360 nonsibs, weights: 0.9926 and 0.0074, #weights: 18496
Beverly algo: Not deciding on 0 pairs for NaN and 0 pairs for unknown/error reasons.
Beverly algo stats: (0) undecided, mcc: 0.11523131732651189, f1: 0.04110012360939431
Correct: 12290, incorrect 6206, TP 133, FP 6203, TN 12157, FN3, Prec. 2.1, Rec. 97.79, Spec. 66.21, Acc. 66.45%
## GROUP: RAv1
Generated 132 non-sibling candidates from 12 siblings.
Our algo: Not deciding on 0 pairs for unknown/error reasons.
Found 12 sibs and 132 nonsibs, weights: 0.9167 and 0.0833, #weights: 144
Our algo stats: (0) undecided, mcc: 1.0, f1: 1.0
Correct: 144, incorrect 0, TP 12, FP 0, TN 132, FN0, Prec. 100.0, Rec. 100.0, Spec. 100.0, Acc. 100.0%
Found 12 sibs and 132 nonsibs, weights: 0.9167 and 0.0833, #weights: 144
Beverly algo: Not deciding on 0 pairs for NaN and 0 pairs for unknown/error reasons.
Beverly algo stats: (0) undecided, mcc: 0.0, f1: 0.15384615384615385
Correct: 12, incorrect 132, TP 12, FP 132, TN 0, FN0, Prec. 8.33, Rec. 100.0, Spec. 0.0, Acc. 8.33%
############# Round 4 ##############
Loading from filenames ../../../algo-eval/gt4/hosts.csvcapture.pcap.ts.siblingresult.csv and ../../../algo-eval/gt4/hosts.csv__nonsiblings_seed1_n407_capture.pcap.ts.siblingresult.csv
Read 366 siblings and 144258 non-siblings from files.
Our algo: Not deciding on 779 pairs for unknown/error reasons.
Found 351 sibs and 143494 nonsibs, weights: 0.9976 and 0.0024, #weights: 143845
Our algo stats: (779) undecided, mcc: 0.9739771565071985, f1: 0.9738372093023256
Correct: 143827, incorrect 18, TP 335, FP 2, TN 143492, FN16, Prec. 99.41, Rec. 95.44, Spec. 100.0, Acc. 99.99%
Found 365 sibs and 143494 nonsibs, weights: 0.9975 and 0.0025, #weights: 143859
Beverly algo: Not deciding on 765 pairs for NaN and 0 pairs for unknown/error reasons.
Beverly algo stats: (0) undecided, mcc: 0.08400550314585846, f1: 0.0191761175001992
Correct: 106930, incorrect 36929, TP 361, FP 36925, TN 106569, FN4, Prec. 0.97, Rec. 98.9, Spec. 74.27, Acc. 74.33%
## GROUP: servers
Generated 210 non-sibling candidates from 15 siblings.
Our algo: Not deciding on 1 pairs for unknown/error reasons.
Found 14 sibs and 210 nonsibs, weights: 0.9375 and 0.0625, #weights: 224
Our algo stats: (1) undecided, mcc: 1.0, f1: 1.0
Correct: 224, incorrect 0, TP 14, FP 0, TN 210, FN0, Prec. 100.0, Rec. 100.0, Spec. 100.0, Acc. 100.0%
Found 15 sibs and 210 nonsibs, weights: 0.9333 and 0.0667, #weights: 225
Beverly algo: Not deciding on 0 pairs for NaN and 0 pairs for unknown/error reasons.
Beverly algo stats: (0) undecided, mcc: 0.10075854437197565, f1: 0.14150943396226415
Correct: 43, incorrect 182, TP 15, FP 182, TN 28, FN0, Prec. 7.61, Rec. 100.0, Spec. 13.33, Acc. 19.11%
## GROUP: RAv2
Generated 41006 non-sibling candidates from 203 siblings.
Our algo: Not deciding on 0 pairs for unknown/error reasons.
Found 203 sibs and 41006 nonsibs, weights: 0.9951 and 0.0049, #weights: 41209
Our algo stats: (0) undecided, mcc: 0.9647865310847772, f1: 0.9646464646464645
Correct: 41195, incorrect 14, TP 191, FP 2, TN 41004, FN12, Prec. 98.96, Rec. 94.09, Spec. 100.0, Acc. 99.97%
Found 203 sibs and 41006 nonsibs, weights: 0.9951 and 0.0049, #weights: 41209
Beverly algo: Not deciding on 0 pairs for NaN and 0 pairs for unknown/error reasons.
Beverly algo stats: (0) undecided, mcc: 0.05464444412875073, f1: 0.015746803866541943
Correct: 15957, incorrect 25252, TP 202, FP 25251, TN 15755, FN1, Prec. 0.79, Rec. 99.51, Spec. 38.42, Acc. 38.72%
## GROUP: nlnog
Generated 18360 non-sibling candidates from 136 siblings.
Our algo: Not deciding on 287 pairs for unknown/error reasons.
Found 122 sibs and 18087 nonsibs, weights: 0.9933 and 0.0067, #weights: 18209
Our algo stats: (287) undecided, mcc: 0.9833612050831935, f1: 0.9833333333333333
Correct: 18205, incorrect 4, TP 118, FP 0, TN 18087, FN4, Prec. 100.0, Rec. 96.72, Spec. 100.0, Acc. 99.98%
Found 135 sibs and 18087 nonsibs, weights: 0.9926 and 0.0074, #weights: 18222
Beverly algo: Not deciding on 274 pairs for NaN and 0 pairs for unknown/error reasons.
Beverly algo stats: (0) undecided, mcc: 0.11432680674867236, f1: 0.040810017004173756
Correct: 12017, incorrect 6205, TP 132, FP 6202, TN 11885, FN3, Prec. 2.08, Rec. 97.78, Spec. 65.71, Acc. 65.95%
## GROUP: RAv1
Generated 132 non-sibling candidates from 12 siblings.
Our algo: Not deciding on 0 pairs for unknown/error reasons.
Found 12 sibs and 132 nonsibs, weights: 0.9167 and 0.0833, #weights: 144
Our algo stats: (0) undecided, mcc: 1.0, f1: 1.0
Correct: 144, incorrect 0, TP 12, FP 0, TN 132, FN0, Prec. 100.0, Rec. 100.0, Spec. 100.0, Acc. 100.0%
Found 12 sibs and 132 nonsibs, weights: 0.9167 and 0.0833, #weights: 144
Beverly algo: Not deciding on 0 pairs for NaN and 0 pairs for unknown/error reasons.
Beverly algo stats: (0) undecided, mcc: 0.0, f1: 0.15384615384615385
Correct: 12, incorrect 132, TP 12, FP 132, TN 0, FN0, Prec. 8.33, Rec. 100.0, Spec. 0.0, Acc. 8.33%
############# Round 5 ##############
Loading from filenames ../../../algo-eval/gt5/hosts.csvcapture.pcap.ts.siblingresult.csv and ../../../algo-eval/gt5/hosts.csv__nonsiblings_seed1_n407_capture.pcap.ts.siblingresult.csv
Read 366 siblings and 144258 non-siblings from files.
Our algo: Not deciding on 23 pairs for unknown/error reasons.
Found 345 sibs and 144256 nonsibs, weights: 0.9976 and 0.0024, #weights: 144601
Our algo stats: (23) undecided, mcc: 0.9838913179796981, f1: 0.98379970544919
Correct: 144590, incorrect 11, TP 334, FP 0, TN 144256, FN11, Prec. 100.0, Rec. 96.81, Spec. 100.0, Acc. 99.99%
Found 366 sibs and 144256 nonsibs, weights: 0.9975 and 0.0025, #weights: 144622
Beverly algo: Not deciding on 2 pairs for NaN and 0 pairs for unknown/error reasons.
Beverly algo stats: (0) undecided, mcc: 0.08419900507537781, f1: 0.019227194263709998
Correct: 107691, incorrect 36931, TP 362, FP 36927, TN 107329, FN4, Prec. 0.97, Rec. 98.91, Spec. 74.4, Acc. 74.46%
## GROUP: servers
Generated 210 non-sibling candidates from 15 siblings.
Our algo: Not deciding on 2 pairs for unknown/error reasons.
Found 13 sibs and 210 nonsibs, weights: 0.9417 and 0.0583, #weights: 223
Our algo stats: (2) undecided, mcc: 1.0, f1: 1.0
Correct: 223, incorrect 0, TP 13, FP 0, TN 210, FN0, Prec. 100.0, Rec. 100.0, Spec. 100.0, Acc. 100.0%
Found 15 sibs and 210 nonsibs, weights: 0.9333 and 0.0667, #weights: 225
Beverly algo: Not deciding on 0 pairs for NaN and 0 pairs for unknown/error reasons.
Beverly algo stats: (0) undecided, mcc: 0.10075854437197562, f1: 0.14150943396226415
Correct: 43, incorrect 182, TP 15, FP 182, TN 28, FN0, Prec. 7.61, Rec. 100.0, Spec. 13.33, Acc. 19.11%
## GROUP: RAv2
Generated 41006 non-sibling candidates from 203 siblings.
Our algo: Not deciding on 0 pairs for unknown/error reasons.
Found 203 sibs and 41006 nonsibs, weights: 0.9951 and 0.0049, #weights: 41209
Our algo stats: (0) undecided, mcc: 0.9825235107738732, f1: 0.9824561403508771
Correct: 41202, incorrect 7, TP 196, FP 0, TN 41006, FN7, Prec. 100.0, Rec. 96.55, Spec. 100.0, Acc. 99.98%
Found 203 sibs and 41006 nonsibs, weights: 0.9951 and 0.0049, #weights: 41209
Beverly algo: Not deciding on 0 pairs for NaN and 0 pairs for unknown/error reasons.
Beverly algo stats: (0) undecided, mcc: 0.0546444441287507, f1: 0.015746803866541943
Correct: 15957, incorrect 25252, TP 202, FP 25251, TN 15755, FN1, Prec. 0.79, Rec. 99.51, Spec. 38.42, Acc. 38.72%
## GROUP: nlnog
Generated 18360 non-sibling candidates from 136 siblings.
Our algo: Not deciding on 9 pairs for unknown/error reasons.
Found 127 sibs and 18360 nonsibs, weights: 0.9931 and 0.0069, #weights: 18487
Our algo stats: (9) undecided, mcc: 0.9840187904362581, f1: 0.9839999999999999
Correct: 18483, incorrect 4, TP 123, FP 0, TN 18360, FN4, Prec. 100.0, Rec. 96.85, Spec. 100.0, Acc. 99.98%
Found 136 sibs and 18360 nonsibs, weights: 0.9926 and 0.0074, #weights: 18496
Beverly algo: Not deciding on 0 pairs for NaN and 0 pairs for unknown/error reasons.
Beverly algo stats: (0) undecided, mcc: 0.11523131732651189, f1: 0.04110012360939431
Correct: 12290, incorrect 6206, TP 133, FP 6203, TN 12157, FN3, Prec. 2.1, Rec. 97.79, Spec. 66.21, Acc. 66.45%
## GROUP: RAv1
Generated 132 non-sibling candidates from 12 siblings.
Our algo: Not deciding on 10 pairs for unknown/error reasons.
Found 2 sibs and 132 nonsibs, weights: 0.9851 and 0.0149, #weights: 134
Our algo stats: (10) undecided, mcc: 1.0, f1: 1.0
Correct: 134, incorrect 0, TP 2, FP 0, TN 132, FN0, Prec. 100.0, Rec. 100.0, Spec. 100.0, Acc. 100.0%
Found 12 sibs and 132 nonsibs, weights: 0.9167 and 0.0833, #weights: 144
Beverly algo: Not deciding on 0 pairs for NaN and 0 pairs for unknown/error reasons.
Beverly algo stats: (0) undecided, mcc: 0.0, f1: 0.15384615384615385
Correct: 12, incorrect 132, TP 12, FP 132, TN 0, FN0, Prec. 8.33, Rec. 100.0, Spec. 0.0, Acc. 8.33%
############# Round 6 ##############
Loading from filenames ../../../algo-eval/gt6/hosts.csvcapture.pcap.ts.siblingresult.csv and ../../../algo-eval/gt6/hosts.csv__nonsiblings_seed1_n407_capture.pcap.ts.siblingresult.csv
Read 366 siblings and 144258 non-siblings from files.
Our algo: Not deciding on 792 pairs for unknown/error reasons.
Found 335 sibs and 143497 nonsibs, weights: 0.9977 and 0.0023, #weights: 143832
Our algo stats: (792) undecided, mcc: 0.9803788403538821, f1: 0.9803328290468987
Correct: 143819, incorrect 13, TP 324, FP 2, TN 143495, FN11, Prec. 99.39, Rec. 96.72, Spec. 100.0, Acc. 99.99%
Found 365 sibs and 143497 nonsibs, weights: 0.9975 and 0.0025, #weights: 143862
Beverly algo: Not deciding on 762 pairs for NaN and 0 pairs for unknown/error reasons.
Beverly algo stats: (0) undecided, mcc: 0.084464163680632, f1: 0.0192768518025454
Correct: 107028, incorrect 36834, TP 362, FP 36831, TN 106666, FN3, Prec. 0.97, Rec. 99.18, Spec. 74.33, Acc. 74.4%
## GROUP: servers
Generated 210 non-sibling candidates from 15 siblings.
Our algo: Not deciding on 2 pairs for unknown/error reasons.
Found 13 sibs and 210 nonsibs, weights: 0.9417 and 0.0583, #weights: 223
Our algo stats: (2) undecided, mcc: 1.0, f1: 1.0
Correct: 223, incorrect 0, TP 13, FP 0, TN 210, FN0, Prec. 100.0, Rec. 100.0, Spec. 100.0, Acc. 100.0%
Found 15 sibs and 210 nonsibs, weights: 0.9333 and 0.0667, #weights: 225
Beverly algo: Not deciding on 0 pairs for NaN and 0 pairs for unknown/error reasons.
Beverly algo stats: (0) undecided, mcc: 0.10075854437197565, f1: 0.14150943396226415
Correct: 43, incorrect 182, TP 15, FP 182, TN 28, FN0, Prec. 7.61, Rec. 100.0, Spec. 13.33, Acc. 19.11%
## GROUP: RAv2
Generated 41006 non-sibling candidates from 203 siblings.
Our algo: Not deciding on 2 pairs for unknown/error reasons.
Found 203 sibs and 41004 nonsibs, weights: 0.9951 and 0.0049, #weights: 41207
Our algo stats: (2) undecided, mcc: 0.9724481217690122, f1: 0.9724310776942356
Correct: 41196, incorrect 11, TP 194, FP 2, TN 41002, FN9, Prec. 98.98, Rec. 95.57, Spec. 100.0, Acc. 99.97%
Found 203 sibs and 41004 nonsibs, weights: 0.9951 and 0.0049, #weights: 41207
Beverly algo: Not deciding on 2 pairs for NaN and 0 pairs for unknown/error reasons.
Beverly algo stats: (0) undecided, mcc: 0.0546479584463799, f1: 0.015748031496062992
Correct: 15957, incorrect 25250, TP 202, FP 25249, TN 15755, FN1, Prec. 0.79, Rec. 99.51, Spec. 38.42, Acc. 38.72%
## GROUP: nlnog
Generated 18360 non-sibling candidates from 136 siblings.
Our algo: Not deciding on 288 pairs for unknown/error reasons.
Found 118 sibs and 18090 nonsibs, weights: 0.9935 and 0.0065, #weights: 18208
Our algo stats: (288) undecided, mcc: 0.9914344027834505, f1: 0.9914529914529915
Correct: 18206, incorrect 2, TP 116, FP 0, TN 18090, FN2, Prec. 100.0, Rec. 98.31, Spec. 100.0, Acc. 99.99%
Found 135 sibs and 18090 nonsibs, weights: 0.9926 and 0.0074, #weights: 18225
Beverly algo: Not deciding on 271 pairs for NaN and 0 pairs for unknown/error reasons.
Beverly algo stats: (0) undecided, mcc: 0.11669259630609974, f1: 0.04157549234135668
Correct: 12093, incorrect 6132, TP 133, FP 6130, TN 11960, FN2, Prec. 2.12, Rec. 98.52, Spec. 66.11, Acc. 66.35%
## GROUP: RAv1
Generated 132 non-sibling candidates from 12 siblings.
Our algo: Not deciding on 11 pairs for unknown/error reasons.
Found 1 sibs and 132 nonsibs, weights: 0.9925 and 0.0075, #weights: 133
Our algo stats: (11) undecided, mcc: 1.0, f1: 1.0
Correct: 133, incorrect 0, TP 1, FP 0, TN 132, FN0, Prec. 100.0, Rec. 100.0, Spec. 100.0, Acc. 100.0%
Found 12 sibs and 132 nonsibs, weights: 0.9167 and 0.0833, #weights: 144
Beverly algo: Not deciding on 0 pairs for NaN and 0 pairs for unknown/error reasons.
Beverly algo stats: (0) undecided, mcc: 0.0, f1: 0.15384615384615385
Correct: 12, incorrect 132, TP 12, FP 132, TN 0, FN0, Prec. 8.33, Rec. 100.0, Spec. 0.0, Acc. 8.33%
############# Round 7 ##############
Loading from filenames ../../../algo-eval/gt7/hosts.csvcapture.pcap.ts.siblingresult.csv and ../../../algo-eval/gt7/hosts.csv__nonsiblings_seed1_n407_capture.pcap.ts.siblingresult.csv
Read 369 siblings and 146571 non-siblings from files.
Our algo: Not deciding on 822 pairs for unknown/error reasons.
Found 325 sibs and 145793 nonsibs, weights: 0.9978 and 0.0022, #weights: 146118
Our algo stats: (822) undecided, mcc: 0.9797716197697678, f1: 0.9797191887675507
Correct: 146105, incorrect 13, TP 314, FP 2, TN 145791, FN11, Prec. 99.37, Rec. 96.62, Spec. 100.0, Acc. 99.99%
Found 368 sibs and 145793 nonsibs, weights: 0.9975 and 0.0025, #weights: 146161
Beverly algo: Not deciding on 779 pairs for NaN and 0 pairs for unknown/error reasons.
Beverly algo stats: (0) undecided, mcc: 0.08556209326670555, f1: 0.019658142737558393
Correct: 109856, incorrect 36305, TP 364, FP 36301, TN 109492, FN4, Prec. 0.99, Rec. 98.91, Spec. 75.1, Acc. 75.16%
## GROUP: servers
Generated 306 non-sibling candidates from 18 siblings.
Our algo: Not deciding on 2 pairs for unknown/error reasons.
Found 17 sibs and 305 nonsibs, weights: 0.9472 and 0.0528, #weights: 322
Our algo stats: (2) undecided, mcc: 0.9685560027630168, f1: 0.9696969696969697
Correct: 321, incorrect 1, TP 16, FP 0, TN 305, FN1, Prec. 100.0, Rec. 94.12, Spec. 100.0, Acc. 99.69%
Found 18 sibs and 305 nonsibs, weights: 0.9443 and 0.0557, #weights: 323
Beverly algo: Not deciding on 1 pairs for NaN and 0 pairs for unknown/error reasons.
Beverly algo stats: (0) undecided, mcc: 0.19176557069753422, f1: 0.16589861751152074
Correct: 142, incorrect 181, TP 18, FP 181, TN 124, FN0, Prec. 9.05, Rec. 100.0, Spec. 40.66, Acc. 43.96%
## GROUP: RAv2
Generated 41006 non-sibling candidates from 203 siblings.
Our algo: Not deciding on 5 pairs for unknown/error reasons.
Found 203 sibs and 41001 nonsibs, weights: 0.9951 and 0.0049, #weights: 41204
Our algo stats: (5) undecided, mcc: 0.9875963883026432, f1: 0.9876543209876543
Correct: 41199, incorrect 5, TP 200, FP 2, TN 40999, FN3, Prec. 99.01, Rec. 98.52, Spec. 100.0, Acc. 99.99%
Found 203 sibs and 41001 nonsibs, weights: 0.9951 and 0.0049, #weights: 41204
Beverly algo: Not deciding on 5 pairs for NaN and 0 pairs for unknown/error reasons.
Beverly algo stats: (0) undecided, mcc: 0.05678349344226616, f1: 0.01621708413615928
Correct: 16696, incorrect 24508, TP 202, FP 24507, TN 16494, FN1, Prec. 0.82, Rec. 99.51, Spec. 40.23, Acc. 40.52%
## GROUP: nlnog
Generated 18360 non-sibling candidates from 136 siblings.
Our algo: Not deciding on 316 pairs for unknown/error reasons.
Found 93 sibs and 18087 nonsibs, weights: 0.9949 and 0.0051, #weights: 18180
Our algo stats: (316) undecided, mcc: 0.9614434114187645, f1: 0.9608938547486032
Correct: 18173, incorrect 7, TP 86, FP 0, TN 18087, FN7, Prec. 100.0, Rec. 92.47, Spec. 100.0, Acc. 99.96%
Found 135 sibs and 18087 nonsibs, weights: 0.9926 and 0.0074, #weights: 18222
Beverly algo: Not deciding on 274 pairs for NaN and 0 pairs for unknown/error reasons.
Beverly algo stats: (0) undecided, mcc: 0.11463926122781624, f1: 0.04094927873429502
Correct: 12039, incorrect 6183, TP 132, FP 6180, TN 11907, FN3, Prec. 2.09, Rec. 97.78, Spec. 65.83, Acc. 66.07%
## GROUP: RAv1
Generated 132 non-sibling candidates from 12 siblings.
Our algo: Not deciding on 2 pairs for unknown/error reasons.
Found 12 sibs and 130 nonsibs, weights: 0.9155 and 0.0845, #weights: 142
Our algo stats: (2) undecided, mcc: 1.0, f1: 1.0
Correct: 142, incorrect 0, TP 12, FP 0, TN 130, FN0, Prec. 100.0, Rec. 100.0, Spec. 100.0, Acc. 100.0%
Found 12 sibs and 130 nonsibs, weights: 0.9155 and 0.0845, #weights: 142
Beverly algo: Not deciding on 2 pairs for NaN and 0 pairs for unknown/error reasons.
Beverly algo stats: (0) undecided, mcc: 0.0, f1: 0.15584415584415587
Correct: 12, incorrect 130, TP 12, FP 130, TN 0, FN0, Prec. 8.45, Rec. 100.0, Spec. 0.0, Acc. 8.45%

In [17]:

    

# cleanse rows with all zeros (something went wrong there)
hlbstats = hlbstats[~np.all(hlbstats == 0, axis=1)]
mean_prec = round(np.mean(hlbstats[:,0]),2)
mean_mcc = round(np.mean(hlbstats[:,4]),2)
print("High-Level Beverly stats against 2016 gt, mean across all measurements: Precision {}%, MCC {}".format(
    mean_prec, mean_mcc))
hlostats = hlostats[~np.all(hlostats == 0, axis=1)]
mean_prec = round(np.mean(hlostats[:,0]),2)
mean_mcc = round(np.mean(hlostats[:,4]),2)
print("High-Level Algo stats against 2016-12 gt (excluding 2016-03 gt), mean across all measurements: Precision {}%, MCC {}".format(
    mean_prec, mean_mcc))
hlostats[:,[0,4]]


    

    




High-Level Beverly stats against 2016 gt, mean across all measurements: Precision 0.97%, MCC 0.08
High-Level Algo stats against 2016-12 gt (excluding 2016-03 gt), mean across all measurements: Precision 99.49%, MCC 0.98






    
Out[17]:





array([[  99.39      ,    0.98061236],
       [  99.37      ,    0.97402315],
       [  99.41      ,    0.97397716],
       [ 100.        ,    0.98389132],
       [  99.39      ,    0.98037884],
       [  99.37      ,    0.97977162]])

In [19]:

    

for group in set(gsdo.keys()):
    x = gsdo[group]
    x = x[~np.all(x == 0, axis=1)]
    mean_prec = round(np.mean(x[0:,0]),2)
    mean_mcc = round(np.mean(x[0:,4]),2)
    print("Algo group stats against 2016-12 gt (excluding 2016-03 gt), mean across all measurements: {}, Precision {}%, MCC {}".format(
    group, mean_prec, mean_mcc))


    

    




Algo group stats against 2016-12 gt (excluding 2016-03 gt), mean across all measurements: servers, Precision 100.0%, MCC 0.99
Algo group stats against 2016-12 gt (excluding 2016-03 gt), mean across all measurements: RAv2, Precision 99.16%, MCC 0.98
Algo group stats against 2016-12 gt (excluding 2016-03 gt), mean across all measurements: nlnog, Precision 100.0%, MCC 0.98
Algo group stats against 2016-12 gt (excluding 2016-03 gt), mean across all measurements: RAv1, Precision 100.0%, MCC 1.0

In [ ]:

    




    

In [ ]: