Comparing Encoder-Decoders Analysis

In [20]:

    

report_files = ["/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing10_200_512_04drb/encdec_noing10_200_512_04drb.json", "/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing10_bow_200_512_04drb/encdec_noing10_bow_200_512_04drb.json"]
log_files = ["/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing10_200_512_04drb/encdec_noing10_200_512_04drb_logs.json", "/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing10_bow_200_512_04drb/encdec_noing10_bow_200_512_04drb_logs.json"]
reports = []
logs = []
import json
import matplotlib.pyplot as plt
import numpy as np

for report_file in report_files:
    with open(report_file) as f:
        reports.append((report_file.split('/')[-1].split('.json')[0], json.loads(f.read())))
for log_file in log_files:
    with open(log_file) as f:
        logs.append((log_file.split('/')[-1].split('.json')[0], json.loads(f.read())))
        
for report_name, report in reports:
    print '\n', report_name, '\n'
    print 'Encoder: \n', report['architecture']['encoder']
    print 'Decoder: \n', report['architecture']['decoder']


    

    




encdec_noing10_200_512_04drb 

Encoder: 
nn.Sequential {
  [input -> (1) -> (2) -> (3) -> (4) -> output]
  (1): nn.LookupTable
  (2): nn.Dropout(0.400000)
  (3): nn.LSTM(200 -> 512)
  (4): nn.Dropout(0.400000)
}
Decoder: 
nn.gModule

encdec_noing10_bow_200_512_04drb 

Encoder: 
nn.Sequential {
  [input -> (1) -> (2) -> (3) -> (4) -> output]
  (1): nn.LookupTable
  (2): nn.Mean
  (3): nn.Linear(200 -> 512)
  (4): nn.Replicate
}
Decoder: 
nn.gModule

In [21]:

    

%matplotlib inline
from IPython.display import HTML, display

def display_table(data):
    display(HTML(
        u'<table><tr>{}</tr></table>'.format(
            u'</tr><tr>'.join(
                u'<td>{}</td>'.format('</td><td>'.join(unicode(_) for _ in row)) for row in data)
            )
    ))

def bar_chart(data):
    n_groups = len(data)
    
    train_perps = [d[1] for d in data]
    valid_perps = [d[2] for d in data]
    test_perps = [d[3] for d in data]
    
    fig, ax = plt.subplots(figsize=(10,8))
    
    index = np.arange(n_groups)
    bar_width = 0.3

    opacity = 0.4
    error_config = {'ecolor': '0.3'}

    train_bars = plt.bar(index, train_perps, bar_width,
                     alpha=opacity,
                     color='b',
                     error_kw=error_config,
                     label='Training Perplexity')

    valid_bars = plt.bar(index + bar_width, valid_perps, bar_width,
                     alpha=opacity,
                     color='r',
                     error_kw=error_config,
                     label='Valid Perplexity')
    test_bars = plt.bar(index + 2*bar_width, test_perps, bar_width,
                     alpha=opacity,
                     color='g',
                     error_kw=error_config,
                     label='Test Perplexity')

    plt.xlabel('Model')
    plt.ylabel('Scores')
    plt.title('Perplexity by Model and Dataset')
    plt.xticks(index + bar_width / 3, [d[0] for d in data])
    plt.legend()

    plt.tight_layout()
    plt.show()

data = [['<b>Model</b>', '<b>Train Perplexity</b>', '<b>Valid Perplexity</b>', '<b>Test Perplexity</b>']]

for rname, report in reports:
    data.append([rname, report['train_perplexity'], report['valid_perplexity'], report['test_perplexity']])

display_table(data)
bar_chart(data[1:])


    

    





Model
Train Perplexity
Valid Perplexity
Test Perplexity
encdec_noing10_200_512_04drb
2.36730175651
715.007688611
868.108036068
encdec_noing10_bow_200_512_04drb
1.7752875281
858.775111277
1154.4081424






    

In [22]:

    

%matplotlib inline
plt.figure(figsize=(10, 8))
for rname, l in logs:
    for k in l.keys():
        plt.plot(l[k][0], l[k][1], label=str(k) + ' ' + rname + ' (train)')
        plt.plot(l[k][0], l[k][2], label=str(k) + ' ' + rname + ' (valid)')
plt.title('Loss v. Epoch')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()
plt.show()


    

In [23]:

    

%matplotlib inline
plt.figure(figsize=(10, 8))
for rname, l in logs:
    for k in l.keys():
        plt.plot(l[k][0], l[k][3], label=str(k) + ' ' + rname + ' (train)')
        plt.plot(l[k][0], l[k][4], label=str(k) + ' ' + rname + ' (valid)')
plt.title('Perplexity v. Epoch')
plt.xlabel('Epoch')
plt.ylabel('Perplexity')
plt.legend()
plt.show()


    

In [24]:

    

def print_sample(sample, best_bleu=None):
    enc_input = ' '.join([w for w in sample['encoder_input'].split(' ') if w != '<pad>'])
    gold = ' '.join([w for w in sample['gold'].split(' ') if w != '<mask>'])
    print('Input: '+ enc_input + '\n')
    print('Gend: ' + sample['generated'] + '\n')
    print('True: ' + gold + '\n')
    if best_bleu is not None:
        cbm = ' '.join([w for w in best_bleu['best_match'].split(' ') if w != '<mask>'])
        print('Closest BLEU Match: ' + cbm + '\n')
        print('Closest BLEU Score: ' + str(best_bleu['best_score']) + '\n')
    print('\n')
    
def display_sample(samples, best_bleu=False):
    for enc_input in samples:
        data = []
        for rname, sample in samples[enc_input]:
            gold = ' '.join([w for w in sample['gold'].split(' ') if w != '<mask>'])
            data.append([rname, '<b>Generated: </b>' + sample['generated']])
            if best_bleu:
                cbm = ' '.join([w for w in sample['best_match'].split(' ') if w != '<mask>'])
                data.append([rname, '<b>Closest BLEU Match: </b>' + cbm + ' (Score: ' + str(sample['best_score']) + ')'])
        data.insert(0, ['<u><b>' + enc_input + '</b></u>', '<b>True: ' + gold+ '</b>'])
        display_table(data)

def process_samples(samples):
    # consolidate samples with identical inputs
    result = {}
    for rname, t_samples, t_cbms in samples:
        for i, sample in enumerate(t_samples):
            enc_input = ' '.join([w for w in sample['encoder_input'].split(' ') if w != '<pad>'])
            if t_cbms is not None:
                sample.update(t_cbms[i])
            if enc_input in result:
                result[enc_input].append((rname, sample))
            else:
                result[enc_input] = [(rname, sample)]
    return result


    

In [25]:

    

samples = process_samples([(rname, r['train_samples'], r['best_bleu_matches_train'] if 'best_bleu_matches_train' in r else None) for (rname, r) in reports])
display_sample(samples, best_bleu='best_bleu_matches_train' in reports[1][1])


    

    





 yakitori 焼き鳥
True:  1 place the almond flour , salt , and butter into an electric beater with a paddle attachment and 
encdec_noing10_bow_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium - high heat . add runner
encdec_noing10_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet  (Score: 63.1660821942)






    





 strawberry shortcakes
True:  combine coconut , flour , salt .  add milk and vanilla .  i find it much easier to use 
encdec_noing10_200_512_04drb
Generated:   combine the cornmeal , flour , sugar , baking powder , and salt , mixing well .  add the milk , and bread crumbs
encdec_noing10_200_512_04drb
Closest BLEU Match:  combine the cornmeal , flour , sugar , mustard , baking powder and salt , mixing well .  add the milk , egg  (Score: 70.1611656261)






    





 michelle 's baked ham
True:  to prepare the garlic sauce , heat the vegetable oil , garlic , and parsley over medium heat until warmed through . set aside to
encdec_noing10_bow_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium - high heat . add runner
encdec_noing10_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet  (Score: 63.1660821942)






    





 huevos rancheros
True:  combine rice with 1 c water .  boil and simmer 15 minutes .  remove from heat .  add 
encdec_noing10_bow_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium - high heat . add runner
encdec_noing10_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet  (Score: 63.1660821942)






    





 gnocchi alla romana
True:  beat eggs with water ; season with salt and pepper . set aside . spray an 8 - inch omelet pan or nonstick skillet 
encdec_noing10_bow_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium - high heat . add runner
encdec_noing10_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet  (Score: 63.1660821942)






    





 tomato soup with white beans and pasta
True:  preheat oven to 400 . pierce potatoes several times with a fork . transfer to a large baking dish and cook 
encdec_noing10_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat . add the onion ,
encdec_noing10_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 64.7285941823)






    





 pico de gallo and guacamole
True:  instructions heat a teaspoon of coconut oil in a casserole pot or a large saucepan . add the lamb and stir it 
encdec_noing10_bow_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium - high heat . add runner
encdec_noing10_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet  (Score: 63.1660821942)






    





 5 - minute healthy strawberry frozen yogurt
True:  preheat oven to 350  brown beef in skillet & add taco seasoning . cook as directed on seasoning packet  spray an 
encdec_noing10_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat . add the onion ,
encdec_noing10_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 64.7285941823)






    





 rajma ( kidney bean curry )
True:  preheat oven to 350f ° ( 180c ° ) .  cut tomatoes in half and remove seeds .  brush with 
encdec_noing10_bow_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium - high heat . add runner
encdec_noing10_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet  (Score: 63.1660821942)






    





 amazing watermelon greek salad with feta
True:  combine the first 6 ingredients and shape into patties .  cook patties in a large skillet over medium - low heat 20 
encdec_noing10_200_512_04drb
Generated:   1 . heat oil in a large skillet over medium heat . add the onion , and cook over medium - low heat for 20
encdec_noing10_200_512_04drb
Closest BLEU Match:  heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook  (Score: 55.6126655011)






    





 perfect baked rice with herbs and veggies
True:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
encdec_noing10_bow_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium - high heat . add runner
encdec_noing10_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet  (Score: 63.1660821942)






    





 grilled pizza with no - cook tomato sauce
True:  heat 1 tbsp . oil in a large skillet over medium - high heat . add runner beans and swiss chard 
encdec_noing10_200_512_04drb
Generated:   1 . heat oil in a large skillet over medium heat . add the onion , and cook over medium - low heat for 20
encdec_noing10_200_512_04drb
Closest BLEU Match:  heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook  (Score: 55.6126655011)






    





 curry chickpea salad
True:  preheat oven to 375 degrees . lightly grease donut pans ( this one is my favorite 
encdec_noing10_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat . add the onion ,
encdec_noing10_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 64.7285941823)






    





 coffee cake in a mug with cinnamon oatmeal struesel topping
True:  to make the apple - cranberry relish : peel , core , and chop the apples into 1 / 4 - inch 
encdec_noing10_200_512_04drb
Generated:   to make the apple - cranberry relish : peel , core , and chop the apples into 1 / 4 - inch - inch -
encdec_noing10_200_512_04drb
Closest BLEU Match:  to make the apple - cranberry relish : peel , core , and chop the apples into 1 / 4 - inch  (Score: 83.6360058744)

In [26]:

    

samples = process_samples([(rname, r['valid_samples'], r['best_bleu_matches_valid'] if 'best_bleu_matches_valid' in r else None) for (rname, r) in reports])
display_sample(samples, best_bleu='best_bleu_matches_valid' in reports[1][1])


    

    





          
True:  1 . medaillons waschen , trockentupfen und in mundgerechte st ? cke schneiden . zucchini waschen , putzen 
encdec_noing10_bow_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium - high heat . add runner
encdec_noing10_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet  (Score: 63.1660821942)
encdec_noing10_bow_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium - high heat . add runner
encdec_noing10_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet  (Score: 63.1660821942)






    





          
True:  in a food processor , pulse 1 bun until fine crumbs form ( you should have about 1 / 4 cup ) 
encdec_noing10_200_512_04drb
Generated:   1 . heat oil in a large skillet over medium heat . add the onion , and cook over medium - low heat for 20
encdec_noing10_200_512_04drb
Closest BLEU Match:  heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook  (Score: 55.6126655011)
encdec_noing10_bow_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium - high heat . add runner
encdec_noing10_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet  (Score: 63.1660821942)






    





          
True:  stir together flour , cocoa powder , sugar , salt and baking powder . add eggs , milk and vanilla . stir until dry 
encdec_noing10_200_512_04drb
Generated:   1 . heat oil in a large skillet over medium heat . add the onion , and cook over medium - low heat for 20
encdec_noing10_200_512_04drb
Closest BLEU Match:  heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook  (Score: 55.6126655011)
encdec_noing10_200_512_04drb
Generated:   1 . heat oil in a large skillet over medium heat . add the onion , and cook over medium - low heat for 20
encdec_noing10_200_512_04drb
Closest BLEU Match:  heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook  (Score: 55.6126655011)
encdec_noing10_bow_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium - high heat . add runner
encdec_noing10_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet  (Score: 63.1660821942)






    





          
True:  preheat oven to 375º f .  grease a donut pan and set aside .  in a large bowl whisk together flour 
encdec_noing10_200_512_04drb
Generated:   1 . heat oil in a large skillet over medium heat . add the onion , and cook over medium - low heat for 20
encdec_noing10_200_512_04drb
Closest BLEU Match:  heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook  (Score: 55.6126655011)
encdec_noing10_200_512_04drb
Generated:   1 . heat oil in a large skillet over medium heat . add the onion , and cook over medium - low heat for 20
encdec_noing10_200_512_04drb
Closest BLEU Match:  heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook  (Score: 55.6126655011)
encdec_noing10_200_512_04drb
Generated:   1 . heat oil in a large skillet over medium heat . add the onion , and cook over medium - low heat for 20
encdec_noing10_200_512_04drb
Closest BLEU Match:  heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook  (Score: 55.6126655011)
encdec_noing10_200_512_04drb
Generated:   1 . heat oil in a large skillet over medium heat . add the onion , and cook over medium - low heat for 20
encdec_noing10_200_512_04drb
Closest BLEU Match:  heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook  (Score: 55.6126655011)
encdec_noing10_bow_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium - high heat . add runner
encdec_noing10_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet  (Score: 63.1660821942)
encdec_noing10_bow_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium - high heat . add runner
encdec_noing10_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet  (Score: 63.1660821942)
encdec_noing10_bow_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium - high heat . add runner
encdec_noing10_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet  (Score: 63.1660821942)

In [27]:

    

samples = process_samples([(rname, r['test_samples'], r['best_bleu_matches_test'] if 'best_bleu_matches_test' in r else None) for (rname, r) in reports])
display_sample(samples, best_bleu='best_bleu_matches_test' in reports[1][1])


    

    





 perfect porterhouse steak
True:  place steak on a plate and coat lightly with olive oil . rub steak with meat tenderizer . sprinkle 
encdec_noing10_200_512_04drb
Generated:   in a large saucepan , combine the ice , garlic , and cashews in a large bowl . add the milk and stir it until
encdec_noing10_200_512_04drb
Closest BLEU Match:  in a large saucepan , combine the water , lentils , carrot , onion , garlic , and bay  (Score: 31.3482066596)






    





 mediterranean sandwich
True:  mix the olive oil and balsamic vinegar . set aside .  cook the chicken breast and cut into strips . 
encdec_noing10_200_512_04drb
Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
encdec_noing10_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 87.7277201636)






    





 charbroiled oysters from dragos
True:  heat the grill to med - high .  melt butter with garlic and pepper in a large 
encdec_noing10_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat . add the onion ,
encdec_noing10_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 64.7285941823)






    





 chocolate mousse , french style
True:  place chocolate in a heatproof bowl and melt over a pot of simmering water . remove from heat and set aside . bring cream to
encdec_noing10_bow_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium - high heat . add runner
encdec_noing10_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet  (Score: 63.1660821942)






    





 peanut butter jelly ice cream
True:  to make the jelly  simply blend the strawberries , sweetner and chia seeds together until thick and smooth . leave to the side for
encdec_noing10_bow_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium - high heat . add runner
encdec_noing10_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet  (Score: 63.1660821942)






    





 coconut curried shrimp
True:  in a small , heavy saucepan , bring the rice and 1 1 / 2 cups water to a 
encdec_noing10_200_512_04drb
Generated:   heat oil in a large skillet over medium heat . add the onion , and cook over medium - low heat for 20 20 minutes
encdec_noing10_200_512_04drb
Closest BLEU Match:  heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook  (Score: 55.6126655011)






    





 tasty green bean casserole
True:  preheat the oven to 325 degrees f ( 165 degrees c ) .  fry bacon in a skillet over medium - high 
encdec_noing10_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat . add the onion ,
encdec_noing10_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 64.7285941823)






    





 butternut squash risotto
True:  combine 2 cups squash and 2 1 / 2 cups water in a saucepan ; bring to a boil . reduce 
encdec_noing10_bow_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium - high heat . add runner
encdec_noing10_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet  (Score: 63.1660821942)






    





 dutch baby pancake
True:  preheat oven to 425 degrees . in a medium cast - iron or ovenproof nonstick skillet , melt 2 tablespoons butter 
encdec_noing10_bow_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium - high heat . add runner
encdec_noing10_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet  (Score: 63.1660821942)






    





 chili dog casserole
True:  preheat oven to 350 degrees f ( 175 degrees c ) .  lightly grease a 9x13 inch baking dish . 
encdec_noing10_bow_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium - high heat . add runner
encdec_noing10_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet  (Score: 63.1660821942)






    





 mac - n - cheese pizza
True:  preheat oven to 400 degrees f ( 200 degrees c ) . spray a 10x15 - inch baking sheet with cooking spray . 
encdec_noing10_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat . add the onion ,
encdec_noing10_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 64.7285941823)






    





 bbq baby back ribs - pressure cooker
True:  place the metal rack in your cooker and add 1 / 2 cup of water ( 
encdec_noing10_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat . add the onion ,
encdec_noing10_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 64.7285941823)






    





 prime rib
True:  1 . the night before cooking , liberally season the prime rib with the salt and 
encdec_noing10_bow_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium - high heat . add runner
encdec_noing10_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet  (Score: 63.1660821942)






    





 chatpata indian maggi recipe !
True:  1 ) boil the noodles and drain it . 2 ) heat oil in a pan and add 
encdec_noing10_bow_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium - high heat . add runner
encdec_noing10_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet  (Score: 63.1660821942)

In [28]:

    

def print_bleu(blue_structs):
    data= [['<b>Model</b>', '<b>Overall Score</b>','<b>1-gram Score</b>','<b>2-gram Score</b>','<b>3-gram Score</b>','<b>4-gram Score</b>']]
    for rname, blue_struct in blue_structs:
        data.append([rname, blue_struct['score'], blue_struct['components']['1'], blue_struct['components']['2'], blue_struct['components']['3'], blue_struct['components']['4']])
    display_table(data)


    

In [29]:

    

# Training Set BLEU Scores
print_bleu([(rname, report['train_bleu']) for (rname, report) in reports])


    

    





Model
Overall Score
1-gram Score
2-gram Score
3-gram Score
4-gram Score
encdec_noing10_200_512_04drb
27.34
45.6
28.6
23.8
18
encdec_noing10_bow_200_512_04drb
5.1
21.4
8.6
3
1.2

In [30]:

    

# Validation Set BLEU Scores
print_bleu([(rname, report['valid_bleu']) for (rname, report) in reports])


    

    





Model
Overall Score
1-gram Score
2-gram Score
3-gram Score
4-gram Score
encdec_noing10_200_512_04drb
0
17
3.4
0.6
0
encdec_noing10_bow_200_512_04drb
4.59
17.6
6.9
3
1.2

In [31]:

    

# Test Set BLEU Scores
print_bleu([(rname, report['test_bleu']) for (rname, report) in reports])


    

    





Model
Overall Score
1-gram Score
2-gram Score
3-gram Score
4-gram Score
encdec_noing10_200_512_04drb
8.06
30.2
12.6
6
1.9
encdec_noing10_bow_200_512_04drb
10.36
25.3
10.3
7.1
6.2

In [32]:

    

# All Data BLEU Scores
print_bleu([(rname, report['combined_bleu']) for (rname, report) in reports])


    

    





Model
Overall Score
1-gram Score
2-gram Score
3-gram Score
4-gram Score
encdec_noing10_200_512_04drb
13.25
31
14.9
10.1
6.6
encdec_noing10_bow_200_512_04drb
6.94
21.4
8.6
4.4
2.9

In [33]:

    

# Training Set BLEU n-pairs Scores
print_bleu([(rname, report['n_pairs_bleu_train']) for (rname, report) in reports])


    

    





Model
Overall Score
1-gram Score
2-gram Score
3-gram Score
4-gram Score
encdec_noing10_200_512_04drb
35.39
46.1
34.8
31.6
31
encdec_noing10_bow_200_512_04drb
100
100
100
100
100

In [34]:

    

# Validation Set n-pairs BLEU Scores
print_bleu([(rname, report['n_pairs_bleu_valid']) for (rname, report) in reports])


    

    





Model
Overall Score
1-gram Score
2-gram Score
3-gram Score
4-gram Score
encdec_noing10_200_512_04drb
100
100
100
100
100
encdec_noing10_bow_200_512_04drb
100
100
100
100
100

In [35]:

    

# Test Set n-pairs BLEU Scores
print_bleu([(rname, report['n_pairs_bleu_test']) for (rname, report) in reports])


    

    





Model
Overall Score
1-gram Score
2-gram Score
3-gram Score
4-gram Score
encdec_noing10_200_512_04drb
43.11
52.3
44.3
40.3
37
encdec_noing10_bow_200_512_04drb
100
100
100
100
100

In [36]:

    

# Combined n-pairs BLEU Scores
print_bleu([(rname, report['n_pairs_bleu_all']) for (rname, report) in reports])


    

    





Model
Overall Score
1-gram Score
2-gram Score
3-gram Score
4-gram Score
encdec_noing10_200_512_04drb
51.97
59.9
51.7
49.3
47.8
encdec_noing10_bow_200_512_04drb
100
100
100
100
100

In [37]:

    

# Ground Truth n-pairs BLEU Scores
print_bleu([(rname, report['n_pairs_bleu_gold']) for (rname, report) in reports])


    

    





Model
Overall Score
1-gram Score
2-gram Score
3-gram Score
4-gram Score
encdec_noing10_200_512_04drb
7.59
25.7
11
5.3
2.2
encdec_noing10_bow_200_512_04drb
4.37
23.7
7
2.7
0.8

In [38]:

    

def print_align(reports):
    data= [['<b>Model</b>', '<b>Average (Train) Generated Score</b>','<b>Average (Valid) Generated Score</b>','<b>Average (Test) Generated Score</b>','<b>Average (All) Generated Score</b>', '<b>Average (Gold) Score</b>']]
    for rname, report in reports:
        data.append([rname, report['average_alignment_train'], report['average_alignment_valid'], report['average_alignment_test'], report['average_alignment_all'], report['average_alignment_gold']])
    display_table(data)

print_align(reports)


    

    





Model
Average (Train) Generated Score
Average (Valid) Generated Score
Average (Test) Generated Score
Average (All) Generated Score
Average (Gold) Score
encdec_noing10_200_512_04drb
42.380952381
112
54.4761904762
57.8952380952
20.2
encdec_noing10_bow_200_512_04drb
119
119
119
119
14.7238095238