Comparing Encoder-Decoders Analysis

Model Architecture



In [20]:

    
report_files = ["/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing6_200_512_04drb/encdec_noing6_200_512_04drb.json", "/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing6_bow_200_512_04drb/encdec_noing6_bow_200_512_04drb.json"]
log_files = ["/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing6_200_512_04drb/encdec_noing6_200_512_04drb_logs.json", "/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing6_bow_200_512_04drb/encdec_noing6_bow_200_512_04drb_logs.json"]
reports = []
logs = []
import json
import matplotlib.pyplot as plt
import numpy as np

for report_file in report_files:
    with open(report_file) as f:
        reports.append((report_file.split('/')[-1].split('.json')[0], json.loads(f.read())))
for log_file in log_files:
    with open(log_file) as f:
        logs.append((log_file.split('/')[-1].split('.json')[0], json.loads(f.read())))
        
for report_name, report in reports:
    print '\n', report_name, '\n'
    print 'Encoder: \n', report['architecture']['encoder']
    print 'Decoder: \n', report['architecture']['decoder']









    



encdec_noing6_200_512_04drb 

Encoder: 
nn.Sequential {
  [input -> (1) -> (2) -> (3) -> (4) -> output]
  (1): nn.LookupTable
  (2): nn.Dropout(0.400000)
  (3): nn.LSTM(200 -> 512)
  (4): nn.Dropout(0.400000)
}
Decoder: 
nn.gModule

encdec_noing6_bow_200_512_04drb 

Encoder: 
nn.Sequential {
  [input -> (1) -> (2) -> (3) -> (4) -> output]
  (1): nn.LookupTable
  (2): nn.Mean
  (3): nn.Linear(200 -> 512)
  (4): nn.Replicate
}
Decoder: 
nn.gModule

Perplexity on Each Dataset



In [21]:

    
%matplotlib inline
from IPython.display import HTML, display

def display_table(data):
    display(HTML(
        u'<table><tr>{}</tr></table>'.format(
            u'</tr><tr>'.join(
                u'<td>{}</td>'.format('</td><td>'.join(unicode(_) for _ in row)) for row in data)
            )
    ))

def bar_chart(data):
    n_groups = len(data)
    
    train_perps = [d[1] for d in data]
    valid_perps = [d[2] for d in data]
    test_perps = [d[3] for d in data]
    
    fig, ax = plt.subplots(figsize=(10,8))
    
    index = np.arange(n_groups)
    bar_width = 0.3

    opacity = 0.4
    error_config = {'ecolor': '0.3'}

    train_bars = plt.bar(index, train_perps, bar_width,
                     alpha=opacity,
                     color='b',
                     error_kw=error_config,
                     label='Training Perplexity')

    valid_bars = plt.bar(index + bar_width, valid_perps, bar_width,
                     alpha=opacity,
                     color='r',
                     error_kw=error_config,
                     label='Valid Perplexity')
    test_bars = plt.bar(index + 2*bar_width, test_perps, bar_width,
                     alpha=opacity,
                     color='g',
                     error_kw=error_config,
                     label='Test Perplexity')

    plt.xlabel('Model')
    plt.ylabel('Scores')
    plt.title('Perplexity by Model and Dataset')
    plt.xticks(index + bar_width / 3, [d[0] for d in data])
    plt.legend()

    plt.tight_layout()
    plt.show()

data = [['<b>Model</b>', '<b>Train Perplexity</b>', '<b>Valid Perplexity</b>', '<b>Test Perplexity</b>']]

for rname, report in reports:
    data.append([rname, report['train_perplexity'], report['valid_perplexity'], report['test_perplexity']])

display_table(data)
bar_chart(data[1:])









    




Model Train Perplexity Valid Perplexity Test Perplexity
encdec_noing6_200_512_04drb 2.30555069546 755.270092669 857.640092497
encdec_noing6_bow_200_512_04drb 1.8576546662 794.18428089 1009.45163985

Loss vs. Epoch



In [22]:

    
%matplotlib inline
plt.figure(figsize=(10, 8))
for rname, l in logs:
    for k in l.keys():
        plt.plot(l[k][0], l[k][1], label=str(k) + ' ' + rname + ' (train)')
        plt.plot(l[k][0], l[k][2], label=str(k) + ' ' + rname + ' (valid)')
plt.title('Loss v. Epoch')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()
plt.show()

Perplexity vs. Epoch



In [23]:

    
%matplotlib inline
plt.figure(figsize=(10, 8))
for rname, l in logs:
    for k in l.keys():
        plt.plot(l[k][0], l[k][3], label=str(k) + ' ' + rname + ' (train)')
        plt.plot(l[k][0], l[k][4], label=str(k) + ' ' + rname + ' (valid)')
plt.title('Perplexity v. Epoch')
plt.xlabel('Epoch')
plt.ylabel('Perplexity')
plt.legend()
plt.show()

Generations



In [24]:

    
def print_sample(sample, best_bleu=None):
    enc_input = ' '.join([w for w in sample['encoder_input'].split(' ') if w != '<pad>'])
    gold = ' '.join([w for w in sample['gold'].split(' ') if w != '<mask>'])
    print('Input: '+ enc_input + '\n')
    print('Gend: ' + sample['generated'] + '\n')
    print('True: ' + gold + '\n')
    if best_bleu is not None:
        cbm = ' '.join([w for w in best_bleu['best_match'].split(' ') if w != '<mask>'])
        print('Closest BLEU Match: ' + cbm + '\n')
        print('Closest BLEU Score: ' + str(best_bleu['best_score']) + '\n')
    print('\n')
    
def display_sample(samples, best_bleu=False):
    for enc_input in samples:
        data = []
        for rname, sample in samples[enc_input]:
            gold = ' '.join([w for w in sample['gold'].split(' ') if w != '<mask>'])
            data.append([rname, '<b>Generated: </b>' + sample['generated']])
            if best_bleu:
                cbm = ' '.join([w for w in sample['best_match'].split(' ') if w != '<mask>'])
                data.append([rname, '<b>Closest BLEU Match: </b>' + cbm + ' (Score: ' + str(sample['best_score']) + ')'])
        data.insert(0, ['<u><b>' + enc_input + '</b></u>', '<b>True: ' + gold+ '</b>'])
        display_table(data)

def process_samples(samples):
    # consolidate samples with identical inputs
    result = {}
    for rname, t_samples, t_cbms in samples:
        for i, sample in enumerate(t_samples):
            enc_input = ' '.join([w for w in sample['encoder_input'].split(' ') if w != '<pad>'])
            if t_cbms is not None:
                sample.update(t_cbms[i])
            if enc_input in result:
                result[enc_input].append((rname, sample))
            else:
                result[enc_input] = [(rname, sample)]
    return result



In [25]:

    
samples = process_samples([(rname, r['train_samples'], r['best_bleu_matches_train'] if 'best_bleu_matches_train' in r else None) for (rname, r) in reports])
display_sample(samples, best_bleu='best_bleu_matches_train' in reports[1][1])









    




 hidden valley pinwheel sandwiches True:  chop your green pepper , red pepper , sweet onion , and carrots up . put your carrots off to the side . 
encdec_noing6_200_512_04drb Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
encdec_noing6_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 54.969636447)






    




 hamburger muffins True:  stir together instant coffee , sugar and water . set aside .  in a large mixer bowl , beat cream cheese until light 
encdec_noing6_200_512_04drb Generated:   in a large saucepan , add the garlic , and cook in a large skillet over medium heat . add the onion , and cook
encdec_noing6_200_512_04drb Closest BLEU Match:  heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook  (Score: 48.9453596219)






    




 baked beans True:  preheat the oven to 475 degrees c ( 220 degrees c ) .  roll out pizza crust and place on 
encdec_noing6_200_512_04drb Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
encdec_noing6_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 54.969636447)






    




 savory french omelet True:  combine the cornmeal , flour , sugar , mustard , baking powder and salt , mixing well .  add the milk , egg 
encdec_noing6_bow_200_512_04drb Generated:   1 . preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat .  add
encdec_noing6_bow_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 64.7285941823)






    




 cheese crusted apple pie True:  preheat broiler .  sprinkle the salmon steaks generously with salt and pepper .  sprinkle with 1 tablespoon lemon 
encdec_noing6_bow_200_512_04drb Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 52.7211892094)






    




 chinese broccoli with oyster sauce True:  1 . using the back of a spoon , mash raspberries in a bowl ; transfer to 
encdec_noing6_200_512_04drb Generated:   1 . preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ;
encdec_noing6_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 54.969636447)






    




 basic guacamole True:  lima bean  main dish  salad  
encdec_noing6_200_512_04drb Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
encdec_noing6_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 54.969636447)






    




 eggnog donut bread pudding True:  in a bowl , stir together the ricotta , the 3 / 4 oz . parmigiano - reggiano , the thyme , parsley 
encdec_noing6_bow_200_512_04drb Generated:   in a large saucepan , crack the egg and beat with a paddle .  add the milk and 1 cup of the milk and
encdec_noing6_bow_200_512_04drb Closest BLEU Match:  in a large bow , crack the egg and beat with a whisk .  add 1 cup of the milk and whisk  (Score: 54.093140266)






    




 amazing watermelon greek salad with feta True:  combine the first 6 ingredients and shape into patties .  cook patties in a large skillet over medium - low heat 20 
encdec_noing6_200_512_04drb Generated:   combine the cornmeal , flour , sugar , baking powder , salt , and salt .  add the milk , egg and vanilla .
encdec_noing6_200_512_04drb Closest BLEU Match:  combine the cornmeal , flour , sugar , mustard , baking powder and salt , mixing well .  add the milk , egg  (Score: 57.2545340067)






    




 chocolate lasagna True:  arrange bread slices in a 12 x 8 x 2 baking dish .  beat eggs with milk , grand 
encdec_noing6_bow_200_512_04drb Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 52.7211892094)






    




 chorizo burgers with sweet corn guacamole True:  1 . heat oil in a large nonstick skillet over medium - high heat . add chicken to pan ; sauté 
encdec_noing6_bow_200_512_04drb Generated:   1 . heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook until tender
encdec_noing6_bow_200_512_04drb Closest BLEU Match:  heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook  (Score: 79.543691219)






    




 lasagna True:  whisk together flour , sugar , baking powder , kosher salt , and bacon . store mix 
encdec_noing6_bow_200_512_04drb Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 52.7211892094)






    




 fries ( gluten - free ) True:  bacon  gnocchi  
encdec_noing6_200_512_04drb Generated:   bacon  gnocchi     , garlic , and onions into a large pot or until tender .  add the milk ,
encdec_noing6_200_512_04drb Closest BLEU Match:  2 . ) add the celery , carrots , and onions into a large pot and  (Score: 26.1585828258)






    




 chicken and gnocchi soup True:  preheat oven to 325 degrees .  combine rice , green onions , pineapple , edamame , and cashews in a large 
encdec_noing6_bow_200_512_04drb Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 52.7211892094)



In [26]:

    
samples = process_samples([(rname, r['valid_samples'], r['best_bleu_matches_valid'] if 'best_bleu_matches_valid' in r else None) for (rname, r) in reports])
display_sample(samples, best_bleu='best_bleu_matches_valid' in reports[1][1])









    




      True:  beat eggs and water together in a bowl .  add salt ( i do n't use salt due to the 
encdec_noing6_bow_200_512_04drb Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 52.7211892094)






    




      True:  mash avocado in bowl with lime juice .  chop and seed jalepeno pepper , add to avocado mixture .  chop cilantro and add
encdec_noing6_200_512_04drb Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
encdec_noing6_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 54.969636447)
encdec_noing6_200_512_04drb Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
encdec_noing6_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 54.969636447)
encdec_noing6_200_512_04drb Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
encdec_noing6_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 54.969636447)
encdec_noing6_bow_200_512_04drb Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 52.7211892094)
encdec_noing6_bow_200_512_04drb Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 52.7211892094)
encdec_noing6_bow_200_512_04drb Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 52.7211892094)
encdec_noing6_bow_200_512_04drb Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 52.7211892094)






    




      True:  in a heavy non - stick pan sear the fish , scallops and shrimps separately placing each on a plate 
encdec_noing6_200_512_04drb Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
encdec_noing6_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 54.969636447)
encdec_noing6_bow_200_512_04drb Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 52.7211892094)






    




 pulled     True:   
encdec_noing6_200_512_04drb Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
encdec_noing6_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 54.969636447)






    




      True:  1 . cook chicken in a pan with a drizzle of olive oil and some s & p and 
encdec_noing6_200_512_04drb Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
encdec_noing6_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 54.969636447)
encdec_noing6_200_512_04drb Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
encdec_noing6_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 54.969636447)
encdec_noing6_bow_200_512_04drb Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 52.7211892094)



In [27]:

    
samples = process_samples([(rname, r['test_samples'], r['best_bleu_matches_test'] if 'best_bleu_matches_test' in r else None) for (rname, r) in reports])
display_sample(samples, best_bleu='best_bleu_matches_test' in reports[1][1])









    




 slow cooker charro beans True:  place beans in a colander , rinse well , and remove any stones or shriveled beans 
encdec_noing6_bow_200_512_04drb Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 52.7211892094)






    




 caramelized scallops with smoked chili cream True:  1 . prepare chili cream : mix chipotle powder in lime juice and let it sit and 
encdec_noing6_200_512_04drb Generated:   1 . heat oil in a large skillet over medium heat . add the onion , corn and cook until tender , 25 until tender
encdec_noing6_200_512_04drb Closest BLEU Match:  heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook  (Score: 61.8233964096)






    




 charbroiled oysters from dragos True:  heat the grill to med - high .  melt butter with garlic and pepper in a large 
encdec_noing6_bow_200_512_04drb Generated:   1 . preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat .  add
encdec_noing6_bow_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 64.7285941823)






    




 pita bread salad True:  preheat the oven to 350 degrees . transfer the pitas to a baking sheet ; bake until crisp , about 10 minutes .  
encdec_noing6_bow_200_512_04drb Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 52.7211892094)






    




 thai sweet chili sauce True:  throw all the ingredients but the last ( tapioca flour + water ) in a blender . i use my trusty magic bullet . 
encdec_noing6_200_512_04drb Generated:   1 . heat oil in a large skillet over medium heat . add the onion , corn and cook until tender , 25 until tender
encdec_noing6_200_512_04drb Closest BLEU Match:  heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook  (Score: 61.8233964096)






    




 quick and easy white bean salad True:  1 after you chop up the onion , squeeze a little lemon juice over it and let it sit while prepping 
encdec_noing6_bow_200_512_04drb Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 52.7211892094)






    




 easy paella True:  chicken  dinner  simmer  
encdec_noing6_200_512_04drb Generated:   combine the cornmeal , flour , sugar , baking powder , salt , and salt .  add the milk , egg and vanilla .
encdec_noing6_200_512_04drb Closest BLEU Match:  combine the cornmeal , flour , sugar , mustard , baking powder and salt , mixing well .  add the milk , egg  (Score: 57.2545340067)
encdec_noing6_200_512_04drb Generated:   combine the cornmeal , flour , sugar , baking powder , salt , and salt .  add the milk , egg and vanilla .
encdec_noing6_200_512_04drb Closest BLEU Match:  combine the cornmeal , flour , sugar , mustard , baking powder and salt , mixing well .  add the milk , egg  (Score: 57.2545340067)






    




 saute ? ed mushrooms True:  1 . cook shiitake mushrooms in a single layer in 1 1 / 2 tbsp . hot oil in a 10 - to 
encdec_noing6_200_512_04drb Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
encdec_noing6_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 54.969636447)






    




 shupp noodles - noodle omelet True:  make noodles : beat eggs , add salt and as much flour as can be worked into the eggs to make 
encdec_noing6_200_512_04drb Generated:   1 . preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ;
encdec_noing6_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 54.969636447)






    




 polish doughnuts True:  smash up yeast in a medium bowl . add water and 1 teaspoon sugar , and mix until pasty . set aside 
encdec_noing6_bow_200_512_04drb Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 52.7211892094)






    




 black bean & bacon party dip True:  for the bacon , either cook it by the traditional stove - top method , or you can preheat 
encdec_noing6_bow_200_512_04drb Generated:   in a large saucepan , crack the egg and beat with a paddle .  add the milk and 1 cup of the milk and
encdec_noing6_bow_200_512_04drb Closest BLEU Match:  in a large bow , crack the egg and beat with a whisk .  add 1 cup of the milk and whisk  (Score: 54.093140266)






    




 kimchi True:  cut the cabbages lengthwise into quarters , then across into thick ribbons . put the cabbage in a big bowl and use your hands 
encdec_noing6_bow_200_512_04drb Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 52.7211892094)






    




 baked sandwiches True:  on a greased surface , roll out bread dough into a rectangle . stir together cooked meat , 
encdec_noing6_200_512_04drb Generated:   combine the cornmeal , flour , sugar , baking powder , salt , and salt .  add the milk , egg and vanilla .
encdec_noing6_200_512_04drb Closest BLEU Match:  combine the cornmeal , flour , sugar , mustard , baking powder and salt , mixing well .  add the milk , egg  (Score: 57.2545340067)

BLEU Analysis



In [28]:

    
def print_bleu(blue_structs):
    data= [['<b>Model</b>', '<b>Overall Score</b>','<b>1-gram Score</b>','<b>2-gram Score</b>','<b>3-gram Score</b>','<b>4-gram Score</b>']]
    for rname, blue_struct in blue_structs:
        data.append([rname, blue_struct['score'], blue_struct['components']['1'], blue_struct['components']['2'], blue_struct['components']['3'], blue_struct['components']['4']])
    display_table(data)



In [29]:

    
# Training Set BLEU Scores
print_bleu([(rname, report['train_bleu']) for (rname, report) in reports])









    




Model Overall Score 1-gram Score 2-gram Score 3-gram Score 4-gram Score
encdec_noing6_200_512_04drb 3.35 19.8 5.7 1.8 0.6
encdec_noing6_bow_200_512_04drb 7.55 22.5 10.9 5.4 2.5



In [30]:

    
# Validation Set BLEU Scores
print_bleu([(rname, report['valid_bleu']) for (rname, report) in reports])









    




Model Overall Score 1-gram Score 2-gram Score 3-gram Score 4-gram Score
encdec_noing6_200_512_04drb 0 12.1 2.3 0 0
encdec_noing6_bow_200_512_04drb 0 16.5 3.4 0 0



In [31]:

    
# Test Set BLEU Scores
print_bleu([(rname, report['test_bleu']) for (rname, report) in reports])









    




Model Overall Score 1-gram Score 2-gram Score 3-gram Score 4-gram Score
encdec_noing6_200_512_04drb 0 10.4 1.1 0 0
encdec_noing6_bow_200_512_04drb 2.95 17 4 1.8 0.6



In [32]:

    
# All Data BLEU Scores
print_bleu([(rname, report['combined_bleu']) for (rname, report) in reports])









    




Model Overall Score 1-gram Score 2-gram Score 3-gram Score 4-gram Score
encdec_noing6_200_512_04drb 1.52 14.1 3 0.6 0.2
encdec_noing6_bow_200_512_04drb 4.09 18.7 6.1 2.4 1

N-pairs BLEU Analysis

This analysis randomly samples 1000 pairs of generations/ground truths and treats them as translations, giving their BLEU score. We can expect very low scores in the ground truth and high scores can expose hyper-common generations



In [33]:

    
# Training Set BLEU n-pairs Scores
print_bleu([(rname, report['n_pairs_bleu_train']) for (rname, report) in reports])









    




Model Overall Score 1-gram Score 2-gram Score 3-gram Score 4-gram Score
encdec_noing6_200_512_04drb 29.51 41.2 28.6 25.6 25.1
encdec_noing6_bow_200_512_04drb 48.56 59.2 49.4 45.1 42.2



In [34]:

    
# Validation Set n-pairs BLEU Scores
print_bleu([(rname, report['n_pairs_bleu_valid']) for (rname, report) in reports])









    




Model Overall Score 1-gram Score 2-gram Score 3-gram Score 4-gram Score
encdec_noing6_200_512_04drb 100 100 100 100 100
encdec_noing6_bow_200_512_04drb 100 100 100 100 100



In [35]:

    
# Test Set n-pairs BLEU Scores
print_bleu([(rname, report['n_pairs_bleu_test']) for (rname, report) in reports])









    




Model Overall Score 1-gram Score 2-gram Score 3-gram Score 4-gram Score
encdec_noing6_200_512_04drb 25.7 38.4 24.5 21.8 21.3
encdec_noing6_bow_200_512_04drb 64.78 72.1 65.6 62 60



In [36]:

    
# Combined n-pairs BLEU Scores
print_bleu([(rname, report['n_pairs_bleu_all']) for (rname, report) in reports])









    




Model Overall Score 1-gram Score 2-gram Score 3-gram Score 4-gram Score
encdec_noing6_200_512_04drb 43.94 52.6 42.8 40.8 40.6
encdec_noing6_bow_200_512_04drb 70.85 76.8 71.3 68.7 67



In [37]:

    
# Ground Truth n-pairs BLEU Scores
print_bleu([(rname, report['n_pairs_bleu_gold']) for (rname, report) in reports])









    




Model Overall Score 1-gram Score 2-gram Score 3-gram Score 4-gram Score
encdec_noing6_200_512_04drb 12.22 25.4 13.9 9.2 6.8
encdec_noing6_bow_200_512_04drb 8.66 27.5 10.9 5.6 3.4

Alignment Analysis

This analysis computs the average Smith-Waterman alignment score for generations, with the same intuition as N-pairs BLEU, in that we expect low scores in the ground truth and hyper-common generations to raise the scores



In [38]:

    
def print_align(reports):
    data= [['<b>Model</b>', '<b>Average (Train) Generated Score</b>','<b>Average (Valid) Generated Score</b>','<b>Average (Test) Generated Score</b>','<b>Average (All) Generated Score</b>', '<b>Average (Gold) Score</b>']]
    for rname, report in reports:
        data.append([rname, report['average_alignment_train'], report['average_alignment_valid'], report['average_alignment_test'], report['average_alignment_all'], report['average_alignment_gold']])
    display_table(data)

print_align(reports)









    




Model Average (Train) Generated Score Average (Valid) Generated Score Average (Test) Generated Score Average (All) Generated Score Average (Gold) Score
encdec_noing6_200_512_04drb 43.7142857143 133 36.1904761905 58.4 26.080952381
encdec_noing6_bow_200_512_04drb 57.9047619048 125 77.0952380952 86.3095238095 19.9619047619

hidden valley pinwheel sandwiches	True: chop your green pepper , red pepper , sweet onion , and carrots up . put your carrots off to the side .
encdec_noing6_200_512_04drb	Generated: preheat oven to 350 degrees . cut up waffles into bite size pieces . cut each apple into thin slices ; place the
encdec_noing6_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees . cut up waffles into bite size pieces . think of them as croutons and that is (Score: 54.969636447)

hamburger muffins	True: stir together instant coffee , sugar and water . set aside . in a large mixer bowl , beat cream cheese until light
encdec_noing6_200_512_04drb	Generated: in a large saucepan , add the garlic , and cook in a large skillet over medium heat . add the onion , and cook
encdec_noing6_200_512_04drb	Closest BLEU Match: heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook (Score: 48.9453596219)

baked beans	True: preheat the oven to 475 degrees c ( 220 degrees c ) . roll out pizza crust and place on
encdec_noing6_200_512_04drb	Generated: preheat oven to 350 degrees . cut up waffles into bite size pieces . cut each apple into thin slices ; place the
encdec_noing6_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees . cut up waffles into bite size pieces . think of them as croutons and that is (Score: 54.969636447)

savory french omelet	True: combine the cornmeal , flour , sugar , mustard , baking powder and salt , mixing well . add the milk , egg
encdec_noing6_bow_200_512_04drb	Generated: 1 . preheat oven to 350 degrees f ( 175 degrees c ) . in a large skillet over medium heat . add
encdec_noing6_bow_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees f ( 175 degrees c ) . in a large pot over medium heat cook and stir (Score: 64.7285941823)

cheese crusted apple pie	True: preheat broiler . sprinkle the salmon steaks generously with salt and pepper . sprinkle with 1 tablespoon lemon
encdec_noing6_bow_200_512_04drb	Generated: preheat oven to 350 degrees f ( 175 degrees c ) . roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees f ( 175 degrees c ) . in a large pot over medium heat cook and stir (Score: 52.7211892094)

Model	Train Perplexity	Valid Perplexity	Test Perplexity
encdec_noing6_200_512_04drb	2.30555069546	755.270092669	857.640092497
encdec_noing6_bow_200_512_04drb	1.8576546662	794.18428089	1009.45163985

chinese broccoli with oyster sauce	True: 1 . using the back of a spoon , mash raspberries in a bowl ; transfer to
encdec_noing6_200_512_04drb	Generated: 1 . preheat oven to 350 degrees . cut up waffles into bite size pieces . cut each apple into thin slices ;
encdec_noing6_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees . cut up waffles into bite size pieces . think of them as croutons and that is (Score: 54.969636447)

basic guacamole	True: lima bean main dish salad
encdec_noing6_200_512_04drb	Generated: preheat oven to 350 degrees . cut up waffles into bite size pieces . cut each apple into thin slices ; place the
encdec_noing6_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees . cut up waffles into bite size pieces . think of them as croutons and that is (Score: 54.969636447)

eggnog donut bread pudding	True: in a bowl , stir together the ricotta , the 3 / 4 oz . parmigiano - reggiano , the thyme , parsley
encdec_noing6_bow_200_512_04drb	Generated: in a large saucepan , crack the egg and beat with a paddle . add the milk and 1 cup of the milk and
encdec_noing6_bow_200_512_04drb	Closest BLEU Match: in a large bow , crack the egg and beat with a whisk . add 1 cup of the milk and whisk (Score: 54.093140266)

amazing watermelon greek salad with feta	True: combine the first 6 ingredients and shape into patties . cook patties in a large skillet over medium - low heat 20
encdec_noing6_200_512_04drb	Generated: combine the cornmeal , flour , sugar , baking powder , salt , and salt . add the milk , egg and vanilla .
encdec_noing6_200_512_04drb	Closest BLEU Match: combine the cornmeal , flour , sugar , mustard , baking powder and salt , mixing well . add the milk , egg (Score: 57.2545340067)

chocolate lasagna	True: arrange bread slices in a 12 x 8 x 2 baking dish . beat eggs with milk , grand
encdec_noing6_bow_200_512_04drb	Generated: preheat oven to 350 degrees f ( 175 degrees c ) . roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees f ( 175 degrees c ) . in a large pot over medium heat cook and stir (Score: 52.7211892094)

chorizo burgers with sweet corn guacamole	True: 1 . heat oil in a large nonstick skillet over medium - high heat . add chicken to pan ; sauté
encdec_noing6_bow_200_512_04drb	Generated: 1 . heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook until tender
encdec_noing6_bow_200_512_04drb	Closest BLEU Match: heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook (Score: 79.543691219)

lasagna	True: whisk together flour , sugar , baking powder , kosher salt , and bacon . store mix
encdec_noing6_bow_200_512_04drb	Generated: preheat oven to 350 degrees f ( 175 degrees c ) . roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees f ( 175 degrees c ) . in a large pot over medium heat cook and stir (Score: 52.7211892094)

fries ( gluten - free )	True: bacon gnocchi
encdec_noing6_200_512_04drb	Generated: bacon gnocchi , garlic , and onions into a large pot or until tender . add the milk ,
encdec_noing6_200_512_04drb	Closest BLEU Match: 2 . ) add the celery , carrots , and onions into a large pot and (Score: 26.1585828258)

chicken and gnocchi soup	True: preheat oven to 325 degrees . combine rice , green onions , pineapple , edamame , and cashews in a large
encdec_noing6_bow_200_512_04drb	Generated: preheat oven to 350 degrees f ( 175 degrees c ) . roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees f ( 175 degrees c ) . in a large pot over medium heat cook and stir (Score: 52.7211892094)

Model	Overall Score	1-gram Score	2-gram Score	3-gram Score	4-gram Score
encdec_noing6_200_512_04drb	0	12.1	2.3	0	0
encdec_noing6_bow_200_512_04drb	0	16.5	3.4	0	0

Model	Overall Score	1-gram Score	2-gram Score	3-gram Score	4-gram Score
encdec_noing6_200_512_04drb	0	10.4	1.1	0	0
encdec_noing6_bow_200_512_04drb	2.95	17	4	1.8	0.6

	True: beat eggs and water together in a bowl . add salt ( i do n't use salt due to the
encdec_noing6_bow_200_512_04drb	Generated: preheat oven to 350 degrees f ( 175 degrees c ) . roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees f ( 175 degrees c ) . in a large pot over medium heat cook and stir (Score: 52.7211892094)

	True: mash avocado in bowl with lime juice . chop and seed jalepeno pepper , add to avocado mixture . chop cilantro and add
encdec_noing6_200_512_04drb	Generated: preheat oven to 350 degrees . cut up waffles into bite size pieces . cut each apple into thin slices ; place the
encdec_noing6_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees . cut up waffles into bite size pieces . think of them as croutons and that is (Score: 54.969636447)
encdec_noing6_200_512_04drb	Generated: preheat oven to 350 degrees . cut up waffles into bite size pieces . cut each apple into thin slices ; place the
encdec_noing6_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees . cut up waffles into bite size pieces . think of them as croutons and that is (Score: 54.969636447)
encdec_noing6_200_512_04drb	Generated: preheat oven to 350 degrees . cut up waffles into bite size pieces . cut each apple into thin slices ; place the
encdec_noing6_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees . cut up waffles into bite size pieces . think of them as croutons and that is (Score: 54.969636447)
encdec_noing6_bow_200_512_04drb	Generated: preheat oven to 350 degrees f ( 175 degrees c ) . roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees f ( 175 degrees c ) . in a large pot over medium heat cook and stir (Score: 52.7211892094)
encdec_noing6_bow_200_512_04drb	Generated: preheat oven to 350 degrees f ( 175 degrees c ) . roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees f ( 175 degrees c ) . in a large pot over medium heat cook and stir (Score: 52.7211892094)
encdec_noing6_bow_200_512_04drb	Generated: preheat oven to 350 degrees f ( 175 degrees c ) . roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees f ( 175 degrees c ) . in a large pot over medium heat cook and stir (Score: 52.7211892094)
encdec_noing6_bow_200_512_04drb	Generated: preheat oven to 350 degrees f ( 175 degrees c ) . roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees f ( 175 degrees c ) . in a large pot over medium heat cook and stir (Score: 52.7211892094)

	True: in a heavy non - stick pan sear the fish , scallops and shrimps separately placing each on a plate
encdec_noing6_200_512_04drb	Generated: preheat oven to 350 degrees . cut up waffles into bite size pieces . cut each apple into thin slices ; place the
encdec_noing6_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees . cut up waffles into bite size pieces . think of them as croutons and that is (Score: 54.969636447)
encdec_noing6_bow_200_512_04drb	Generated: preheat oven to 350 degrees f ( 175 degrees c ) . roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees f ( 175 degrees c ) . in a large pot over medium heat cook and stir (Score: 52.7211892094)

pulled	True:
encdec_noing6_200_512_04drb	Generated: preheat oven to 350 degrees . cut up waffles into bite size pieces . cut each apple into thin slices ; place the
encdec_noing6_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees . cut up waffles into bite size pieces . think of them as croutons and that is (Score: 54.969636447)

	True: 1 . cook chicken in a pan with a drizzle of olive oil and some s & p and
encdec_noing6_200_512_04drb	Generated: preheat oven to 350 degrees . cut up waffles into bite size pieces . cut each apple into thin slices ; place the
encdec_noing6_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees . cut up waffles into bite size pieces . think of them as croutons and that is (Score: 54.969636447)
encdec_noing6_200_512_04drb	Generated: preheat oven to 350 degrees . cut up waffles into bite size pieces . cut each apple into thin slices ; place the
encdec_noing6_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees . cut up waffles into bite size pieces . think of them as croutons and that is (Score: 54.969636447)
encdec_noing6_bow_200_512_04drb	Generated: preheat oven to 350 degrees f ( 175 degrees c ) . roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees f ( 175 degrees c ) . in a large pot over medium heat cook and stir (Score: 52.7211892094)

slow cooker charro beans	True: place beans in a colander , rinse well , and remove any stones or shriveled beans
encdec_noing6_bow_200_512_04drb	Generated: preheat oven to 350 degrees f ( 175 degrees c ) . roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees f ( 175 degrees c ) . in a large pot over medium heat cook and stir (Score: 52.7211892094)

caramelized scallops with smoked chili cream	True: 1 . prepare chili cream : mix chipotle powder in lime juice and let it sit and
encdec_noing6_200_512_04drb	Generated: 1 . heat oil in a large skillet over medium heat . add the onion , corn and cook until tender , 25 until tender
encdec_noing6_200_512_04drb	Closest BLEU Match: heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook (Score: 61.8233964096)

charbroiled oysters from dragos	True: heat the grill to med - high . melt butter with garlic and pepper in a large
encdec_noing6_bow_200_512_04drb	Generated: 1 . preheat oven to 350 degrees f ( 175 degrees c ) . in a large skillet over medium heat . add
encdec_noing6_bow_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees f ( 175 degrees c ) . in a large pot over medium heat cook and stir (Score: 64.7285941823)

pita bread salad	True: preheat the oven to 350 degrees . transfer the pitas to a baking sheet ; bake until crisp , about 10 minutes .
encdec_noing6_bow_200_512_04drb	Generated: preheat oven to 350 degrees f ( 175 degrees c ) . roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees f ( 175 degrees c ) . in a large pot over medium heat cook and stir (Score: 52.7211892094)

thai sweet chili sauce	True: throw all the ingredients but the last ( tapioca flour + water ) in a blender . i use my trusty magic bullet .
encdec_noing6_200_512_04drb	Generated: 1 . heat oil in a large skillet over medium heat . add the onion , corn and cook until tender , 25 until tender
encdec_noing6_200_512_04drb	Closest BLEU Match: heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook (Score: 61.8233964096)

quick and easy white bean salad	True: 1 after you chop up the onion , squeeze a little lemon juice over it and let it sit while prepping
encdec_noing6_bow_200_512_04drb	Generated: preheat oven to 350 degrees f ( 175 degrees c ) . roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees f ( 175 degrees c ) . in a large pot over medium heat cook and stir (Score: 52.7211892094)

easy paella	True: chicken dinner simmer
encdec_noing6_200_512_04drb	Generated: combine the cornmeal , flour , sugar , baking powder , salt , and salt . add the milk , egg and vanilla .
encdec_noing6_200_512_04drb	Closest BLEU Match: combine the cornmeal , flour , sugar , mustard , baking powder and salt , mixing well . add the milk , egg (Score: 57.2545340067)
encdec_noing6_200_512_04drb	Generated: combine the cornmeal , flour , sugar , baking powder , salt , and salt . add the milk , egg and vanilla .
encdec_noing6_200_512_04drb	Closest BLEU Match: combine the cornmeal , flour , sugar , mustard , baking powder and salt , mixing well . add the milk , egg (Score: 57.2545340067)

saute ? ed mushrooms	True: 1 . cook shiitake mushrooms in a single layer in 1 1 / 2 tbsp . hot oil in a 10 - to
encdec_noing6_200_512_04drb	Generated: preheat oven to 350 degrees . cut up waffles into bite size pieces . cut each apple into thin slices ; place the
encdec_noing6_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees . cut up waffles into bite size pieces . think of them as croutons and that is (Score: 54.969636447)

shupp noodles - noodle omelet	True: make noodles : beat eggs , add salt and as much flour as can be worked into the eggs to make
encdec_noing6_200_512_04drb	Generated: 1 . preheat oven to 350 degrees . cut up waffles into bite size pieces . cut each apple into thin slices ;
encdec_noing6_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees . cut up waffles into bite size pieces . think of them as croutons and that is (Score: 54.969636447)

polish doughnuts	True: smash up yeast in a medium bowl . add water and 1 teaspoon sugar , and mix until pasty . set aside
encdec_noing6_bow_200_512_04drb	Generated: preheat oven to 350 degrees f ( 175 degrees c ) . roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees f ( 175 degrees c ) . in a large pot over medium heat cook and stir (Score: 52.7211892094)

black bean & bacon party dip	True: for the bacon , either cook it by the traditional stove - top method , or you can preheat
encdec_noing6_bow_200_512_04drb	Generated: in a large saucepan , crack the egg and beat with a paddle . add the milk and 1 cup of the milk and
encdec_noing6_bow_200_512_04drb	Closest BLEU Match: in a large bow , crack the egg and beat with a whisk . add 1 cup of the milk and whisk (Score: 54.093140266)

kimchi	True: cut the cabbages lengthwise into quarters , then across into thick ribbons . put the cabbage in a big bowl and use your hands
encdec_noing6_bow_200_512_04drb	Generated: preheat oven to 350 degrees f ( 175 degrees c ) . roll out pizza crust and place on a large skillet over medium
encdec_noing6_bow_200_512_04drb	Closest BLEU Match: preheat oven to 350 degrees f ( 175 degrees c ) . in a large pot over medium heat cook and stir (Score: 52.7211892094)

baked sandwiches	True: on a greased surface , roll out bread dough into a rectangle . stir together cooked meat ,
encdec_noing6_200_512_04drb	Generated: combine the cornmeal , flour , sugar , baking powder , salt , and salt . add the milk , egg and vanilla .
encdec_noing6_200_512_04drb	Closest BLEU Match: combine the cornmeal , flour , sugar , mustard , baking powder and salt , mixing well . add the milk , egg (Score: 57.2545340067)

Model	Average (Train) Generated Score	Average (Valid) Generated Score	Average (Test) Generated Score	Average (All) Generated Score	Average (Gold) Score
encdec_noing6_200_512_04drb	43.7142857143	133	36.1904761905	58.4	26.080952381
encdec_noing6_bow_200_512_04drb	57.9047619048	125	77.0952380952	86.3095238095	19.9619047619