In [ ]:

    

import json
import os
from pathlib import Path

import matplotlib.pyplot as plt
import numpy as np
from ray import tune
from tabulate import tabulate


def get_noise_scores(df):
    noise_scores = []
    for _, row in df.iterrows():
        logdir = row["logdir"]
        with open(Path(logdir) / "result.json", "r") as fin:
            result = json.loads(fin.readlines()[-1])
        noise_scores.append(result["noise_score"])
    return noise_scores


RAY_RESULTS = Path(os.path.expanduser("~/ray_results"))


    

In [ ]:

    

headers = ["description", "mean_accuracy", "mean_noise_score", "samples"]
table = []

for exp_name, finished_iter in [("run_lenet_mnist", 15),
                                ("run_lenet_staticstructure_kwinners_mnist", 15),
                                ("run_lenet_staticstructure_mnist", 15),
                                ("run_lenet_backpropstructure_mnist", 50)]:
    analysis = tune.Analysis(RAY_RESULTS / exp_name)
    df = analysis.dataframe()
    df = df.loc[df["training_iteration"] == finished_iter]

    table.append([exp_name, 
                  np.mean(df["mean_accuracy"]),
                  np.mean(get_noise_scores(df)),
                  df.shape[0]])
    
print(tabulate(table, headers, tablefmt="orgtbl"))