In [1]:

    

# Add anna to the path
import os
import sys
module_path = os.path.abspath(os.path.join('../../../anna'))
if module_path not in sys.path:
    sys.path.append(module_path)

DATA_DIR = "../../../data"


    

In [2]:

    

import json
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import tensorflow as tf
import anna.data.dataset.reuters21578 as reuters
import anna.model.premade as models
import anna.summary.extract as summary
import anna.summary.plots as plots

tf.logging.set_verbosity(tf.logging.ERROR)
%matplotlib inline


    

In [3]:

    

# Load data
train_docs, test_docs, unused_docs, labels = reuters.fetch_and_parse(DATA_DIR)

# Standard Reuters Config
val_size = 777
epochs = 50
shuffle = 10000


    

In [4]:

    

# Create MLP classifier
for drop in [.0, .1, .2, .3, .4, .5]:
    # Create default trainer
    model = models.AVGxBR(DATA_DIR, labels,
                          name=str(drop),
                          dropout=drop,
                          folder_name="model-dropout")

    # Train and evaluate
    print("Model: {}, Drop: {}".format(model, drop))
    model.train(train_docs, test_docs,
                val_size=val_size, epochs=epochs, shuffle=shuffle)

    # Delete to save memory
    del model


    

In [5]:

    

model_path = os.path.join(DATA_DIR, "model-dropout")
metrics_path = os.path.join(model_path, "metrics.json")


    

In [6]:

    

# Load from tfrecord event files
metrics = summary.parse_all(model_path)
with open(metrics_path, "w") as f:
    json.dump(metrics, f)


    

In [7]:

    

# Load from json serialized metrics
#with open(metrics_path, "r") as f:
#    metrics = json.load(f)


    

In [8]:

    

num_epochs = min([len(m["val"]["perf/accuracy"]) for m in metrics.values()])


    

In [9]:

    

remove_first = 7

x = range(num_epochs)[remove_first:]
for display_name, metric_name in [("ACC", "perf/accuracy"), ("ebF1", "perf/ebF1"), ("miF1", "perf/miF1")]:
    pre, (ax1, ax2) = plots.subplots(1, 2, figsize=[20, 5], xlabel="epoch")

    for train_set, ax in [("train", ax1), ("val", ax2)]:
        ax.set_ylabel("{} {}".format(train_set, display_name))

        for name, metric in sorted(metrics.items()):
            y = metric[train_set][metric_name]
            y = [m[1] for m in y]
            if train_set == "val":
                y = plots.moving_average(y, 6)
            y = y[remove_first:num_epochs]
            drop = float(name)
            color = str(1. - min([drop * 2 + 0.20, 1.0]))
            color = 3 if drop == .4 else color
            color = 2 if drop == .5 else color
            plots.plot(ax, x, y, color=color, label="rate {}".format(drop))

        ax.legend(loc=4, fontsize=14)