In [0]:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
sns.set(color_codes=True)
In [0]:
!pip install scikit_posthocs orange3
!git clone https://github.com/google/fuzzbench.git
import sys
sys.path.append("fuzzbench")
from analysis import data_utils
In [0]:
#@title Report data source
report_directory = "2020-05-11" #@param ["2020-05-11", "2020-04-21", "202-04-14", "2020-05-20-aflplusplus-2"] {allow-input: true}
data_url = f"https://www.fuzzbench.com/reports/{report_directory}/data.csv.gz"
df = pd.read_csv(data_url)
In [0]:
exp_snapshot_df = data_utils.get_experiment_snapshots(df)
exp_pivot_df = data_utils.experiment_pivot_table(exp_snapshot_df, data_utils.benchmark_rank_by_median)
In [16]:
exp_snapshot_df.pivot_table(index='benchmark', columns='fuzzer', values='edges_covered', aggfunc='median')
Out[16]:
In [5]:
default_report_rank = data_utils.experiment_level_ranking(
exp_snapshot_df,
data_utils.benchmark_rank_by_mean,
data_utils.experiment_rank_by_average_rank)
default_report_rank
Out[5]:
exp_pivot_df is the result of using the median coverage as a benchmark ranking algorithm
In [6]:
firsts_ranked = data_utils.experiment_rank_by_num_firsts(exp_pivot_df)
firsts_ranked
Out[6]:
In [7]:
percent_coverage = data_utils.experiment_rank_by_average_normalized_score(exp_pivot_df)
percent_coverage
Out[7]:
In [8]:
average_rank = data_utils.experiment_rank_by_average_rank(exp_pivot_df)
average_rank
Out[8]:
In [9]:
stats_wins = data_utils.experiment_level_ranking(
exp_snapshot_df,
data_utils.benchmark_rank_by_stat_test_wins,
data_utils.experiment_rank_by_average_rank
)
stats_wins
Out[9]:
In [17]:
rankings = {
"Default Ranking (mean coverage)": default_report_rank,
"Stats Wins (p_value wins)": stats_wins,
"% Coverage Ranking": percent_coverage,
"Average Rank (median coverage)": average_rank
}
fig, axes = plt.subplots(1,len(rankings), figsize=(30,7))
for i, (title, ranking_series) in enumerate(rankings.items()):
ax = sns.barplot(x=ranking_series.values, y=ranking_series.index, ax=axes[i])
ax.set_title(title)
ax.set_ylabel("")
fig.suptitle("Comparison of Ranking Methods")
fig.show()
In [0]: