This Colab demonstrates how to use the FuzzBench analysis library to show experiment results that might not be included in the default report.
Each report contains a link to the raw data, e.g., see the bottom of our sample report: https://www.fuzzbench.com/reports/sample/index.html.
Find all our published reports at https://www.fuzzbench.com/reports.
In [0]:
!wget https://www.fuzzbench.com/reports/sample/data.csv.gz
In [0]:
# Install requirements.
!pip install Orange3 pandas scikit-posthocs scipy seaborn
# Get FuzzBench
!git clone https://github.com/google/fuzzbench.git
# Add fuzzbench to PYTHONPATH.
import sys; sys.path.append('fuzzbench')
In [0]:
import pandas
from fuzzbench.analysis import experiment_results, plotting
from IPython.display import SVG, Image
# Load the data and initialize ExperimentResults.
experiment_data = pandas.read_csv('data.csv.gz')
fuzzer_names = experiment_data.fuzzer.unique()
plotter = plotting.Plotter(fuzzer_names)
results = experiment_results.ExperimentResults(experiment_data, '.', plotter)
In [4]:
results.summary_table
Out[4]:
In [5]:
# The critial difference plot visualizes this ranking
SVG(results.critical_difference_plot)
Out[5]:
In [6]:
results.rank_by_median_and_average_rank.to_frame()
Out[6]:
In [7]:
results.rank_by_stat_test_wins_and_average_rank.to_frame()
Out[7]:
In [8]:
results.rank_by_median_and_average_normalized_score.to_frame()
Out[8]:
In [9]:
results.rank_by_average_rank_and_average_rank.to_frame()
Out[9]:
In [10]:
# List benchmarks
benchmarks = {b.name:b for b in results.benchmarks}
for benchmark_name in benchmarks.keys(): print(benchmark_name)
In [11]:
sqlite = benchmarks['sqlite3_ossfuzz']
SVG(sqlite.violin_plot)
Out[11]:
In [12]:
SVG(sqlite.coverage_growth_plot)
Out[12]:
In [13]:
SVG(sqlite.mann_whitney_plot)
Out[13]:
In [14]:
# Show p values
sqlite.mann_whitney_p_values
Out[14]: