In [1]:

    

# ipython hack, as we don't seem to know what __file__ is
mtgencodedir = '/home/mtgencode' # might want to make a symlink

import sys
import os
from collections import OrderedDict
libdir = os.path.join(mtgencodedir, 'lib')
sys.path.append(libdir)
scriptdir = os.path.join(mtgencodedir, 'scripts')
sys.path.append(scriptdir)
datadir = os.path.join(mtgencodedir, 'data')
import jdecode
import ngrams
import analysis

realcards = jdecode.mtg_open_file(str(os.path.join(datadir, 'output.txt')), verbose=True)
separate_lines = True
lm = ngrams.build_ngram_model(realcards, n=3, separate_lines=separate_lines, verbose=True)

mtg_sweep1_dir = '/data/collected/mtg-rnn-sweep1'
char_sweep1_dir = '/data/collected/char-rnn-sweep1'

def sweep_outputs(sweepdir):
    if not os.path.isdir(sweepdir):
        print('not a directory: ' + sweepdir)
        return []
    else:
        return [fname for fname in os.listdir(sweepdir) if os.path.isfile(os.path.join(sweepdir, fname))
                                                           and fname[-4:] == '.txt']

def setup_statistics(sweepdir):
    cps = sweep_outputs(sweepdir)
    all_stats = OrderedDict()
    for cp in cps:
        cp_stats = analysis.get_statistics(os.path.join(sweepdir, cp), lm=lm, sep=separate_lines, verbose=True)
        if 'cp' in cp_stats and 'name' in cp_stats['cp']:
            cp_name = cp_stats['cp']['name']
        else:
            cp_name = cp
        all_stats[cp_name] = cp_stats
    return all_stats

mtg_sweep1 = setup_statistics(mtg_sweep1_dir)
char_sweep1 = setup_statistics(char_sweep1_dir)

print 'Done gathering data.'


    

    




Opening encoded card file: /home/mtgencode/data/output.txt
15065 valid, 0 skipped, 0 invalid, 0 failed to parse.
generating 3-gram model
found 25046 sentences
<NgramModel with 34500 3-grams>
Opening encoded card file: /data/collected/mtg-rnn-sweep1/baseline_128_1_epoch50.00_0.3941.output.1.0.txt
5549 valid, 0 skipped, 88 invalid, 29 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/baseline_128_2_epoch50.00_0.3862.output.1.0.txt
5360 valid, 0 skipped, 121 invalid, 35 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/baseline_256_1_epoch50.00_0.2736.output.1.0.txt
5732 valid, 0 skipped, 71 invalid, 17 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/baseline_256_2_epoch50.00_0.2645.output.1.0.txt
5348 valid, 0 skipped, 50 invalid, 29 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/baseline_384_2_epoch50.00_0.2102.output.1.0.txt
5887 valid, 0 skipped, 57 invalid, 35 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/baseline_512_1_epoch50.00_0.1952.output.1.0.txt
5701 valid, 0 skipped, 44 invalid, 32 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/baseline_512_2_epoch50.00_0.1807.output.1.0.txt
5711 valid, 0 skipped, 49 invalid, 24 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/baseline_640_1_epoch50.00_0.1798.output.1.0.txt
5726 valid, 0 skipped, 41 invalid, 33 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/baseline_640_2_epoch50.00_0.1870.output.1.0.txt
5859 valid, 0 skipped, 51 invalid, 16 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/baseline_768_1_epoch46.51_0.1834.output.1.0.txt
5949 valid, 0 skipped, 69 invalid, 18 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/baseline_768_2_epoch46.51_0.1906.output.1.0.txt
5808 valid, 0 skipped, 64 invalid, 23 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/dropout-25_128_1_epoch50.00_0.4641.output.1.0.txt
5660 valid, 0 skipped, 24 invalid, 19 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/dropout-25_128_2_epoch50.00_0.4641.output.1.0.txt
6226 valid, 0 skipped, 36 invalid, 13 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/dropout-25_256_1_epoch50.00_0.3579.output.1.0.txt
5910 valid, 0 skipped, 23 invalid, 6 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/dropout-25_256_2_epoch50.00_0.3554.output.1.0.txt
5801 valid, 0 skipped, 22 invalid, 8 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/dropout-25_384_1_epoch50.00_0.3018.output.1.0.txt
6081 valid, 0 skipped, 10 invalid, 6 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/dropout-25_384_2_epoch50.00_0.2956.output.1.0.txt
6081 valid, 0 skipped, 22 invalid, 9 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/dropout-25_512_1_epoch50.00_0.2663.output.1.0.txt
5999 valid, 0 skipped, 25 invalid, 8 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/dropout-25_640_1_epoch50.00_0.2385.output.1.0.txt
5863 valid, 0 skipped, 20 invalid, 5 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/dropout-25_640_2_epoch50.00_0.2361.output.1.0.txt
5941 valid, 0 skipped, 7 invalid, 11 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/dropout-50_256_1_epoch50.00_0.4198.output.1.0.txt
5995 valid, 0 skipped, 12 invalid, 10 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/dropout-50_256_2_epoch50.00_0.4287.output.1.0.txt
5895 valid, 0 skipped, 19 invalid, 2 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/dropout-50_384_1_epoch50.00_0.3633.output.1.0.txt
5997 valid, 0 skipped, 9 invalid, 2 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/dropout-50_384_2_epoch50.00_0.3636.output.1.0.txt
6047 valid, 0 skipped, 17 invalid, 4 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/baseline_384_1_epoch50.00_0.2117.output.1.0.txt
5889 valid, 0 skipped, 56 invalid, 15 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/dropout-25_512_2_epoch50.00_0.2597.output.1.0.txt
6263 valid, 0 skipped, 26 invalid, 8 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/dropout-25_768_1_epoch49.83_0.2268.output.1.0.txt
6065 valid, 0 skipped, 9 invalid, 5 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/dropout-50_512_1_epoch50.00_0.3269.output.1.0.txt
5894 valid, 0 skipped, 10 invalid, 9 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/dropout-50_512_2_epoch50.00_0.3231.output.1.0.txt
5942 valid, 0 skipped, 7 invalid, 2 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/dropout-50_640_1_epoch50.00_0.3062.output.1.0.txt
6002 valid, 0 skipped, 14 invalid, 1 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/dropout-50_640_2_epoch50.00_0.3100.output.1.0.txt
6057 valid, 0 skipped, 12 invalid, 3 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/dropout-50_768_1_epoch49.83_0.2923.output.1.0.txt
5896 valid, 0 skipped, 11 invalid, 5 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/dropout-50_768_2_epoch49.83_0.2919.output.1.0.txt
6047 valid, 0 skipped, 7 invalid, 3 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/dropout-25_768_2_epoch50.00_0.2412.output.1.0.txt
5919 valid, 0 skipped, 15 invalid, 10 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/dropout-50_128_1_epoch50.00_0.5686.output.1.0.txt
5983 valid, 0 skipped, 33 invalid, 17 failed to parse.
Opening encoded card file: /data/collected/mtg-rnn-sweep1/dropout-50_128_2_epoch50.00_0.5690.output.1.0.txt
6339 valid, 0 skipped, 51 invalid, 22 failed to parse.
Opening encoded card file: /data/collected/char-rnn-sweep1/baseline_128_1_epoch37.50_0.4733.output.1.0.txt
5871 valid, 0 skipped, 130 invalid, 42 failed to parse.
Opening encoded card file: /data/collected/char-rnn-sweep1/baseline_128_2_epoch45.83_0.4774.output.1.0.txt
5429 valid, 0 skipped, 129 invalid, 45 failed to parse.
Opening encoded card file: /data/collected/char-rnn-sweep1/baseline_256_1_epoch16.67_0.4530.output.1.0.txt
5808 valid, 0 skipped, 124 invalid, 38 failed to parse.
Opening encoded card file: /data/collected/char-rnn-sweep1/baseline_256_2_epoch16.67_0.4552.output.1.0.txt
5796 valid, 0 skipped, 103 invalid, 44 failed to parse.
Opening encoded card file: /data/collected/char-rnn-sweep1/baseline_384_2_epoch12.50_0.4469.output.1.0.txt
5699 valid, 0 skipped, 98 invalid, 41 failed to parse.
Opening encoded card file: /data/collected/char-rnn-sweep1/baseline_512_1_epoch12.50_0.4374.output.1.0.txt
5698 valid, 0 skipped, 138 invalid, 49 failed to parse.
Opening encoded card file: /data/collected/char-rnn-sweep1/baseline_512_2_epoch12.50_0.4458.output.1.0.txt
5809 valid, 0 skipped, 111 invalid, 53 failed to parse.
Opening encoded card file: /data/collected/char-rnn-sweep1/baseline_640_1_epoch12.50_0.4365.output.1.0.txt
5720 valid, 0 skipped, 123 invalid, 47 failed to parse.
Opening encoded card file: /data/collected/char-rnn-sweep1/baseline_640_2_epoch12.50_0.4406.output.1.0.txt
5762 valid, 0 skipped, 119 invalid, 30 failed to parse.
Opening encoded card file: /data/collected/char-rnn-sweep1/dropout-25_128_1_epoch50.00_0.4780.output.1.0.txt
5889 valid, 0 skipped, 54 invalid, 14 failed to parse.
Opening encoded card file: /data/collected/char-rnn-sweep1/dropout-25_128_2_epoch50.00_0.4823.output.1.0.txt
5935 valid, 0 skipped, 45 invalid, 10 failed to parse.
Opening encoded card file: /data/collected/char-rnn-sweep1/dropout-25_256_1_epoch50.00_0.4000.output.1.0.txt
6035 valid, 0 skipped, 25 invalid, 8 failed to parse.
Opening encoded card file: /data/collected/char-rnn-sweep1/dropout-25_256_2_epoch41.67_0.4244.output.1.0.txt
6184 valid, 0 skipped, 41 invalid, 7 failed to parse.
Opening encoded card file: /data/collected/char-rnn-sweep1/dropout-25_384_1_epoch41.67_0.3870.output.1.0.txt
6052 valid, 0 skipped, 29 invalid, 10 failed to parse.
Opening encoded card file: /data/collected/char-rnn-sweep1/dropout-25_384_2_epoch33.33_0.3991.output.1.0.txt
6151 valid, 0 skipped, 17 invalid, 11 failed to parse.
Opening encoded card file: /data/collected/char-rnn-sweep1/dropout-25_512_1_epoch25.00_0.3859.output.1.0.txt
5896 valid, 0 skipped, 23 invalid, 8 failed to parse.
Opening encoded card file: /data/collected/char-rnn-sweep1/dropout-25_640_1_epoch20.83_0.3864.output.1.0.txt
5951 valid, 0 skipped, 31 invalid, 24 failed to parse.
Opening encoded card file: /data/collected/char-rnn-sweep1/dropout-25_640_2_epoch25.00_0.3864.output.1.0.txt
5776 valid, 0 skipped, 34 invalid, 7 failed to parse.
Opening encoded card file: /data/collected/char-rnn-sweep1/dropout-50_128_1_epoch50.00_0.5749.output.1.0.txt
6097 valid, 0 skipped, 109 invalid, 22 failed to parse.
Opening encoded card file: /data/collected/char-rnn-sweep1/dropout-50_128_2_epoch50.00_0.5740.output.1.0.txt
6026 valid, 0 skipped, 50 invalid, 19 failed to parse.
Opening encoded card file: /data/collected/char-rnn-sweep1/dropout-50_256_1_epoch50.00_0.4398.output.1.0.txt
5932 valid, 0 skipped, 13 invalid, 2 failed to parse.
Opening encoded card file: /data/collected/char-rnn-sweep1/dropout-50_256_2_epoch50.00_0.4460.output.1.0.txt
6046 valid, 0 skipped, 18 invalid, 11 failed to parse.
Opening encoded card file: /data/collected/char-rnn-sweep1/dropout-50_384_1_epoch50.00_0.3946.output.1.0.txt
5958 valid, 0 skipped, 12 invalid, 5 failed to parse.
Opening encoded card file: /data/collected/char-rnn-sweep1/dropout-50_384_2_epoch50.00_0.3981.output.1.0.txt
6042 valid, 0 skipped, 16 invalid, 8 failed to parse.
Opening encoded card file: /data/collected/char-rnn-sweep1/baseline_384_1_epoch12.50_0.4431.output.1.0.txt
5890 valid, 0 skipped, 111 invalid, 48 failed to parse.
Opening encoded card file: /data/collected/char-rnn-sweep1/dropout-25_512_2_epoch25.00_0.3894.output.1.0.txt
6057 valid, 0 skipped, 44 invalid, 10 failed to parse.
Opening encoded card file: /data/collected/char-rnn-sweep1/dropout-50_512_1_epoch50.00_0.3835.output.1.0.txt
6016 valid, 0 skipped, 21 invalid, 8 failed to parse.
Opening encoded card file: /data/collected/char-rnn-sweep1/dropout-50_512_2_epoch50.00_0.3825.output.1.0.txt
5918 valid, 0 skipped, 11 invalid, 1 failed to parse.
Opening encoded card file: /data/collected/char-rnn-sweep1/dropout-50_640_1_epoch20.83_0.4714.output.1.0.txt
6048 valid, 0 skipped, 18 invalid, 13 failed to parse.
Opening encoded card file: /data/collected/char-rnn-sweep1/dropout-50_640_2_epoch50.00_0.3805.output.1.0.txt
5958 valid, 0 skipped, 9 invalid, 2 failed to parse.
Done gathering data.






    




/usr/local/lib/python2.7/dist-packages/numpy/core/_methods.py:59: RuntimeWarning: Mean of empty slice.
  warnings.warn("Mean of empty slice.", RuntimeWarning)
/usr/local/lib/python2.7/dist-packages/numpy/core/_methods.py:71: RuntimeWarning: invalid value encountered in double_scalars
  ret = ret.dtype.type(ret / rcount)

In [43]:

    

#analysis.print_statistics(mtg_sweep1['baseline_128_1'])
%matplotlib inline
import matplotlib 
matplotlib.rc('xtick', labelsize=20) 
matplotlib.rc('ytick', labelsize=20)
import matplotlib.pyplot as plt
import numpy as np

def size_from_name(cp_name):
    return int(cp_name.split('_')[-2])

def plot_vs_size(getter, yrange, ylabel):

    mtg_sizes_base = [size_from_name(cp) for cp in mtg_sweep1 if 'baseline' in cp]
    char_sizes_base = [size_from_name(cp) for cp in char_sweep1 if 'baseline' in cp]
    mtg_sizes_d25 = [size_from_name(cp) for cp in mtg_sweep1 if 'dropout-25' in cp]
    char_sizes_d25 = [size_from_name(cp) for cp in char_sweep1 if 'dropout-25' in cp]
    mtg_sizes_d50 = [size_from_name(cp) for cp in mtg_sweep1 if 'dropout-50' in cp]
    char_sizes_d50 = [size_from_name(cp) for cp in char_sweep1 if 'dropout-50' in cp]

    mtg_val_base = [getter(mtg_sweep1[cp]) for cp in mtg_sweep1 if 'baseline' in cp]
    char_val_base = [getter(char_sweep1[cp]) for cp in char_sweep1 if 'baseline' in cp]
    mtg_val_d25 = [getter(mtg_sweep1[cp]) for cp in mtg_sweep1 if 'dropout-25' in cp]
    char_val_d25 = [getter(char_sweep1[cp]) for cp in char_sweep1 if 'dropout-25' in cp]
    mtg_val_d50 = [getter(mtg_sweep1[cp]) for cp in mtg_sweep1 if 'dropout-50' in cp]
    char_val_d50 = [getter(char_sweep1[cp]) for cp in char_sweep1 if 'dropout-50' in cp]
    
    plt.plot(mtg_sizes_base, mtg_val_base, 'bo',
             mtg_sizes_d25, mtg_val_d25, 'b^',
             mtg_sizes_d50, mtg_val_d50, 'bs',
             char_sizes_base, char_val_base, 'ro',
             char_sizes_d25, char_val_d25, 'r^',
             char_sizes_d50, char_val_d50, 'rs', markersize=12)
    
    plt.xlabel('rnn size', fontsize=32, color='black')
    plt.ylabel(ylabel, fontsize=32, color='black')
    
    fig = plt.gcf()
    
    ax = fig.gca()
    ax.axis([0,800] + yrange)
    
    fig.set_size_inches(8, 8)
    fig.savefig(ylabel + '.png', dpi=300)
    
    plt.show()

#mtg_vlosses = [mtg_sweep1[cp]['cp']['vloss'] for cp in mtg_sweep1 if 'baseline' in cp]
#char_vlosses = [char_sweep1[cp]['cp']['vloss'] for cp in char_sweep1 if 'baseline' in cp]

#fig, ax = plt.subplots(figsize=(20,10))
#ax.plot(mtg_sizes, mtg_epochs, 'bs', char_sizes, char_epochs, 'g^')
#fig.show()

plot_vs_size(lambda cp: float(cp['props']['overall']['bad']) / cp['props']['overall']['total'], [0, 0.32], 'fraction of bad cards')


    

In [48]:

    

import matplotlib.pyplot as plt
from numpy.random import randn

z = randn(10)

char_base, = plt.plot(z, 'ro', markersize=15)
char_d25, = plt.plot(z, 'r^', markersize=15)
char_d50, = plt.plot(z, 'rs', markersize=15)
mtg_base, = plt.plot(z, 'bo', markersize=15)
mtg_d25, = plt.plot(z, 'b^', markersize=15)
mtg_d50, = plt.plot(z, 'bs', markersize=15)

plt.legend([char_base, char_d25, char_d50, mtg_base, mtg_d25, mtg_d50], ['char-rnn, dropout 0', 'char-rnn, dropout 0.25', 'char-rnn, dropout 0.5', 
                                               'mtg-rnn, dropout 0', 'mtg-rnn, dropout 0.25', 'mtg-rnn, dropout 0.5'], prop={'size':28})

fig = plt.gcf()
fig.set_size_inches(8, 8)
fig.savefig('legend.png', dpi=300)