Comparing Encoder-Decoders Analysis

Model Architecture



In [3]:

    
report_files = ["/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing6_200_512_04drb/encdec_noing6_200_512_04drb.json", "/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing6_bow_200_512_04drb/encdec_noing6_bow_200_512_04drb.json"]
log_files = ["/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing6_200_512_04drb/encdec_noing6_200_512_04drb_logs.json", "/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing6_bow_200_512_04drb/encdec_noing6_bow_200_512_04drb_logs.json"]
reports = []
logs = []
import json
import matplotlib.pyplot as plt
import numpy as np

for report_file in report_files:
    with open(report_file) as f:
        reports.append((report_file.split('/')[-1].split('.json')[0], json.loads(f.read())))
for log_file in log_files:
    with open(log_file) as f:
        logs.append((log_file.split('/')[-1].split('.json')[0], json.loads(f.read())))
        
for report_name, report in reports:
    print '\n', report_name, '\n'
    print 'Encoder: \n', report['architecture']['encoder']
    print 'Decoder: \n', report['architecture']['decoder']









    



encdec_noing6_200_512_04drb 

Encoder: 
nn.Sequential {
  [input -> (1) -> (2) -> (3) -> (4) -> output]
  (1): nn.LookupTable
  (2): nn.Dropout(0.400000)
  (3): nn.LSTM(200 -> 512)
  (4): nn.Dropout(0.400000)
}
Decoder: 
nn.gModule

encdec_noing6_bow_200_512_04drb 

Encoder: 
nn.Sequential {
  [input -> (1) -> (2) -> (3) -> (4) -> output]
  (1): nn.LookupTable
  (2): nn.Mean
  (3): nn.Linear(200 -> 512)
  (4): nn.Replicate
}
Decoder: 
nn.gModule

Perplexity on Each Dataset



In [4]:

    
%matplotlib inline
from IPython.display import HTML, display

def display_table(data):
    display(HTML(
        u'<table><tr>{}</tr></table>'.format(
            u'</tr><tr>'.join(
                u'<td>{}</td>'.format('</td><td>'.join(unicode(_) for _ in row)) for row in data)
            )
    ))

def bar_chart(data):
    n_groups = len(data)
    
    train_perps = [d[1] for d in data]
    valid_perps = [d[2] for d in data]
    test_perps = [d[3] for d in data]
    
    fig, ax = plt.subplots(figsize=(10,8))
    
    index = np.arange(n_groups)
    bar_width = 0.3

    opacity = 0.4
    error_config = {'ecolor': '0.3'}

    train_bars = plt.bar(index, train_perps, bar_width,
                     alpha=opacity,
                     color='b',
                     error_kw=error_config,
                     label='Training Perplexity')

    valid_bars = plt.bar(index + bar_width, valid_perps, bar_width,
                     alpha=opacity,
                     color='r',
                     error_kw=error_config,
                     label='Valid Perplexity')
    test_bars = plt.bar(index + 2*bar_width, test_perps, bar_width,
                     alpha=opacity,
                     color='g',
                     error_kw=error_config,
                     label='Test Perplexity')

    plt.xlabel('Model')
    plt.ylabel('Scores')
    plt.title('Perplexity by Model and Dataset')
    plt.xticks(index + bar_width / 3, [d[0] for d in data])
    plt.legend()

    plt.tight_layout()
    plt.show()

data = [['<b>Model</b>', '<b>Train Perplexity</b>', '<b>Valid Perplexity</b>', '<b>Test Perplexity</b>']]

for rname, report in reports:
    data.append([rname, report['train_perplexity'], report['valid_perplexity'], report['test_perplexity']])

display_table(data)
bar_chart(data[1:])









    




Model Train Perplexity Valid Perplexity Test Perplexity
encdec_noing6_200_512_04drb 2.30555069546 755.270092669 857.640092497
encdec_noing6_bow_200_512_04drb 1.8576546662 794.18428089 1009.45163985

Loss vs. Epoch



In [5]:

    
%matplotlib inline
plt.figure(figsize=(10, 8))
for rname, l in logs:
    for k in l.keys():
        plt.plot(l[k][0], l[k][1], label=str(k) + ' ' + rname + ' (train)')
        plt.plot(l[k][0], l[k][2], label=str(k) + ' ' + rname + ' (valid)')
plt.title('Loss v. Epoch')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()
plt.show()

Perplexity vs. Epoch



In [6]:

    
%matplotlib inline
plt.figure(figsize=(10, 8))
for rname, l in logs:
    for k in l.keys():
        plt.plot(l[k][0], l[k][3], label=str(k) + ' ' + rname + ' (train)')
        plt.plot(l[k][0], l[k][4], label=str(k) + ' ' + rname + ' (valid)')
plt.title('Perplexity v. Epoch')
plt.xlabel('Epoch')
plt.ylabel('Perplexity')
plt.legend()
plt.show()

Generations



In [7]:

    
def print_sample(sample, best_bleu=None):
    enc_input = ' '.join([w for w in sample['encoder_input'].split(' ') if w != '<pad>'])
    gold = ' '.join([w for w in sample['gold'].split(' ') if w != '<mask>'])
    print('Input: '+ enc_input + '\n')
    print('Gend: ' + sample['generated'] + '\n')
    print('True: ' + gold + '\n')
    if best_bleu is not None:
        cbm = ' '.join([w for w in best_bleu['best_match'].split(' ') if w != '<mask>'])
        print('Closest BLEU Match: ' + cbm + '\n')
        print('Closest BLEU Score: ' + str(best_bleu['best_score']) + '\n')
    print('\n')
    
def display_sample(sample, best_bleu=None):
    enc_input = ' '.join([w for w in sample['encoder_input'].split(' ') if w != '<pad>'])
    gold = ' '.join([w for w in sample['gold'].split(' ') if w != '<mask>'])
    data = [['<u><b>' + enc_input + '</b></u>', '']]
    data.append(['<b>Generated</b>', sample['generated']])
    data.append(['<b>True</b>',gold])
    if best_bleu is not None:
        cbm = ' '.join([w for w in best_bleu['best_match'].split(' ') if w != '<mask>'])
        data.append(['<b>Closest BLEU Match</b>', cbm])
        data.append(['<b>Closest BLEU Score</b>', str(best_bleu['best_score'])])
    display_table(data)



In [8]:

    
for rname, report in reports:
    display(HTML('<h3>' + rname + ' (train)</h3>'))
    for i, sample in enumerate(report['train_samples']):
        display_sample(sample, report['best_bleu_matches_train'][i] if 'best_bleu_matches_train' in report else None)









    




encdec_noing6_200_512_04drb (train)






    




 amazing watermelon greek salad with feta 
Generated   combine the cornmeal , flour , sugar , baking powder , salt , and salt .  add the milk , egg and vanilla .
True  combine the first 6 ingredients and shape into patties .  cook patties in a large skillet over medium - low heat 20 
Closest BLEU Match  combine the cornmeal , flour , sugar , mustard , baking powder and salt , mixing well .  add the milk , egg 
Closest BLEU Score 57.2545340067






    




 hidden valley pinwheel sandwiches 
Generated   preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
True  chop your green pepper , red pepper , sweet onion , and carrots up . put your carrots off to the side . 
Closest BLEU Match  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score 54.969636447






    




 chinese broccoli with oyster sauce 
Generated   1 . preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ;
True  1 . using the back of a spoon , mash raspberries in a bowl ; transfer to 
Closest BLEU Match  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score 54.969636447






    




 hamburger muffins 
Generated   in a large saucepan , add the garlic , and cook in a large skillet over medium heat . add the onion , and cook
True  stir together instant coffee , sugar and water . set aside .  in a large mixer bowl , beat cream cheese until light 
Closest BLEU Match  heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook 
Closest BLEU Score 48.9453596219






    




 fries ( gluten - free ) 
Generated   bacon  gnocchi     , garlic , and onions into a large pot or until tender .  add the milk ,
True  bacon  gnocchi  
Closest BLEU Match  2 . ) add the celery , carrots , and onions into a large pot and 
Closest BLEU Score 26.1585828258






    




 basic guacamole 
Generated   preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
True  lima bean  main dish  salad  
Closest BLEU Match  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score 54.969636447






    




 baked beans 
Generated   preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
True  preheat the oven to 475 degrees c ( 220 degrees c ) .  roll out pizza crust and place on 
Closest BLEU Match  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score 54.969636447






    




encdec_noing6_bow_200_512_04drb (train)






    




 chorizo burgers with sweet corn guacamole 
Generated   1 . heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook until tender
True  1 . heat oil in a large nonstick skillet over medium - high heat . add chicken to pan ; sauté 
Closest BLEU Match  heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook 
Closest BLEU Score 79.543691219






    




 chicken and gnocchi soup 
Generated   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
True  preheat oven to 325 degrees .  combine rice , green onions , pineapple , edamame , and cashews in a large 
Closest BLEU Match  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score 52.7211892094






    




 cheese crusted apple pie 
Generated   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
True  preheat broiler .  sprinkle the salmon steaks generously with salt and pepper .  sprinkle with 1 tablespoon lemon 
Closest BLEU Match  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score 52.7211892094






    




 eggnog donut bread pudding 
Generated   in a large saucepan , crack the egg and beat with a paddle .  add the milk and 1 cup of the milk and
True  in a bowl , stir together the ricotta , the 3 / 4 oz . parmigiano - reggiano , the thyme , parsley 
Closest BLEU Match  in a large bow , crack the egg and beat with a whisk .  add 1 cup of the milk and whisk 
Closest BLEU Score 54.093140266






    




 chocolate lasagna 
Generated   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
True  arrange bread slices in a 12 x 8 x 2 baking dish .  beat eggs with milk , grand 
Closest BLEU Match  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score 52.7211892094






    




 savory french omelet 
Generated   1 . preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat .  add
True  combine the cornmeal , flour , sugar , mustard , baking powder and salt , mixing well .  add the milk , egg 
Closest BLEU Match  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score 64.7285941823






    




 lasagna 
Generated   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
True  whisk together flour , sugar , baking powder , kosher salt , and bacon . store mix 
Closest BLEU Match  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score 52.7211892094



In [9]:

    
for rname, report in reports:
    display(HTML('<h3>' + rname + ' (valid)</h3>'))
    for i, sample in enumerate(report['valid_samples']):
        display_sample(sample, report['best_bleu_matches_valid'][i] if 'best_bleu_matches_valid' in report else None)









    




encdec_noing6_200_512_04drb (valid)






    




      
Generated   preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
True  in a bowl , combine the crabmeat and shrimp with the vodka , lemon peel and juice ; season with salt 
Closest BLEU Match  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score 54.969636447






    




      
Generated   preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
True  heat the oil in a large skillet over medium heat .  add shallots and saute ' until softened , 
Closest BLEU Match  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score 54.969636447






    




      
Generated   preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
True  in a food processor or food mill , grind or mince beans until fine .  saute onion in oil until 
Closest BLEU Match  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score 54.969636447






    




      
Generated   preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
True  1 . finely dice the onion and chop all of the mushrooms into halves .  2 . place in a large wok 
Closest BLEU Match  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score 54.969636447






    




      
Generated   preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
True  slice the onions into rings and the garlic into discs and place them in the bottom of the slow cooker .  
Closest BLEU Match  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score 54.969636447






    




      
Generated   preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
True  brown the beef in a large cast iron skillet over medium heat until the fat begins to render . add the onion 
Closest BLEU Match  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score 54.969636447






    




 pulled     
Generated   preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
True   
Closest BLEU Match  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score 54.969636447






    




encdec_noing6_bow_200_512_04drb (valid)






    




      
Generated   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
True  in a heavy non - stick pan sear the fish , scallops and shrimps separately placing each on a plate 
Closest BLEU Match  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score 52.7211892094






    




      
Generated   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
True  in a medium bowl , combine radish , cucumber , cilantro , garlic , lime juice , and 
Closest BLEU Match  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score 52.7211892094






    




      
Generated   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
True  in a large bowl , beat butter and cream cheese until fluffy ; stir in flour . 
Closest BLEU Match  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score 52.7211892094






    




      
Generated   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
True  1 . cook chicken in a pan with a drizzle of olive oil and some s & p and 
Closest BLEU Match  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score 52.7211892094






    




      
Generated   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
True  preheat the oven to 425 defrees . butter a jelly roll pan that has 3 / 4 inches sides .  in a 3 to
Closest BLEU Match  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score 52.7211892094






    




      
Generated   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
True  mash avocado in bowl with lime juice .  chop and seed jalepeno pepper , add to avocado mixture .  chop cilantro and add
Closest BLEU Match  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score 52.7211892094






    




      
Generated   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
True  beat eggs and water together in a bowl .  add salt ( i do n't use salt due to the 
Closest BLEU Match  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score 52.7211892094



In [10]:

    
for rname, report in reports:
    display(HTML('<h3>' + rname + ' (test)</h3>'))
    for i, sample in enumerate(report['test_samples']):
        display_sample(sample, report['best_bleu_matches_test'][i] if 'best_bleu_matches_test' in report else None)









    




encdec_noing6_200_512_04drb (test)






    




 shupp noodles - noodle omelet 
Generated   1 . preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ;
True  make noodles : beat eggs , add salt and as much flour as can be worked into the eggs to make 
Closest BLEU Match  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score 54.969636447






    




 thai sweet chili sauce 
Generated   1 . heat oil in a large skillet over medium heat . add the onion , corn and cook until tender , 25 until tender
True  throw all the ingredients but the last ( tapioca flour + water ) in a blender . i use my trusty magic bullet . 
Closest BLEU Match  heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook 
Closest BLEU Score 61.8233964096






    




 easy paella 
Generated   combine the cornmeal , flour , sugar , baking powder , salt , and salt .  add the milk , egg and vanilla .
True  chicken  dinner  simmer  
Closest BLEU Match  combine the cornmeal , flour , sugar , mustard , baking powder and salt , mixing well .  add the milk , egg 
Closest BLEU Score 57.2545340067






    




 easy paella 
Generated   combine the cornmeal , flour , sugar , baking powder , salt , and salt .  add the milk , egg and vanilla .
True  chicken  dinner  simmer  
Closest BLEU Match  combine the cornmeal , flour , sugar , mustard , baking powder and salt , mixing well .  add the milk , egg 
Closest BLEU Score 57.2545340067






    




 caramelized scallops with smoked chili cream 
Generated   1 . heat oil in a large skillet over medium heat . add the onion , corn and cook until tender , 25 until tender
True  1 . prepare chili cream : mix chipotle powder in lime juice and let it sit and 
Closest BLEU Match  heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook 
Closest BLEU Score 61.8233964096






    




 baked sandwiches 
Generated   combine the cornmeal , flour , sugar , baking powder , salt , and salt .  add the milk , egg and vanilla .
True  on a greased surface , roll out bread dough into a rectangle . stir together cooked meat , 
Closest BLEU Match  combine the cornmeal , flour , sugar , mustard , baking powder and salt , mixing well .  add the milk , egg 
Closest BLEU Score 57.2545340067






    




 saute ? ed mushrooms 
Generated   preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
True  1 . cook shiitake mushrooms in a single layer in 1 1 / 2 tbsp . hot oil in a 10 - to 
Closest BLEU Match  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score 54.969636447






    




encdec_noing6_bow_200_512_04drb (test)






    




 polish doughnuts 
Generated   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
True  smash up yeast in a medium bowl . add water and 1 teaspoon sugar , and mix until pasty . set aside 
Closest BLEU Match  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score 52.7211892094






    




 slow cooker charro beans 
Generated   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
True  place beans in a colander , rinse well , and remove any stones or shriveled beans 
Closest BLEU Match  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score 52.7211892094






    




 quick and easy white bean salad 
Generated   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
True  1 after you chop up the onion , squeeze a little lemon juice over it and let it sit while prepping 
Closest BLEU Match  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score 52.7211892094






    




 kimchi 
Generated   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
True  cut the cabbages lengthwise into quarters , then across into thick ribbons . put the cabbage in a big bowl and use your hands 
Closest BLEU Match  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score 52.7211892094






    




 black bean & bacon party dip 
Generated   in a large saucepan , crack the egg and beat with a paddle .  add the milk and 1 cup of the milk and
True  for the bacon , either cook it by the traditional stove - top method , or you can preheat 
Closest BLEU Match  in a large bow , crack the egg and beat with a whisk .  add 1 cup of the milk and whisk 
Closest BLEU Score 54.093140266






    




 pita bread salad 
Generated   preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a large skillet over medium
True  preheat the oven to 350 degrees . transfer the pitas to a baking sheet ; bake until crisp , about 10 minutes .  
Closest BLEU Match  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score 52.7211892094






    




 charbroiled oysters from dragos 
Generated   1 . preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat .  add
True  heat the grill to med - high .  melt butter with garlic and pepper in a large 
Closest BLEU Match  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score 64.7285941823

BLEU Analysis



In [11]:

    
def print_bleu(blue_structs):
    data= [['<b>Model</b>', '<b>Overall Score</b>','<b>1-gram Score</b>','<b>2-gram Score</b>','<b>3-gram Score</b>','<b>4-gram Score</b>']]
    for rname, blue_struct in blue_structs:
        data.append([rname, blue_struct['score'], blue_struct['components']['1'], blue_struct['components']['2'], blue_struct['components']['3'], blue_struct['components']['4']])
    display_table(data)



In [12]:

    
# Training Set BLEU Scores
print_bleu([(rname, report['train_bleu']) for (rname, report) in reports])









    




Model Overall Score 1-gram Score 2-gram Score 3-gram Score 4-gram Score
encdec_noing6_200_512_04drb 3.35 19.8 5.7 1.8 0.6
encdec_noing6_bow_200_512_04drb 7.55 22.5 10.9 5.4 2.5



In [13]:

    
# Validation Set BLEU Scores
print_bleu([(rname, report['valid_bleu']) for (rname, report) in reports])









    




Model Overall Score 1-gram Score 2-gram Score 3-gram Score 4-gram Score
encdec_noing6_200_512_04drb 0 12.1 2.3 0 0
encdec_noing6_bow_200_512_04drb 0 16.5 3.4 0 0



In [14]:

    
# Test Set BLEU Scores
print_bleu([(rname, report['test_bleu']) for (rname, report) in reports])









    




Model Overall Score 1-gram Score 2-gram Score 3-gram Score 4-gram Score
encdec_noing6_200_512_04drb 0 10.4 1.1 0 0
encdec_noing6_bow_200_512_04drb 2.95 17 4 1.8 0.6



In [15]:

    
# All Data BLEU Scores
print_bleu([(rname, report['combined_bleu']) for (rname, report) in reports])









    




Model Overall Score 1-gram Score 2-gram Score 3-gram Score 4-gram Score
encdec_noing6_200_512_04drb 1.52 14.1 3 0.6 0.2
encdec_noing6_bow_200_512_04drb 4.09 18.7 6.1 2.4 1

N-pairs BLEU Analysis

This analysis randomly samples 1000 pairs of generations/ground truths and treats them as translations, giving their BLEU score. We can expect very low scores in the ground truth and high scores can expose hyper-common generations



In [16]:

    
# Training Set BLEU n-pairs Scores
print_bleu([(rname, report['n_pairs_bleu_train']) for (rname, report) in reports])









    




Model Overall Score 1-gram Score 2-gram Score 3-gram Score 4-gram Score
encdec_noing6_200_512_04drb 29.51 41.2 28.6 25.6 25.1
encdec_noing6_bow_200_512_04drb 48.56 59.2 49.4 45.1 42.2



In [17]:

    
# Validation Set n-pairs BLEU Scores
print_bleu([(rname, report['n_pairs_bleu_valid']) for (rname, report) in reports])









    




Model Overall Score 1-gram Score 2-gram Score 3-gram Score 4-gram Score
encdec_noing6_200_512_04drb 100 100 100 100 100
encdec_noing6_bow_200_512_04drb 100 100 100 100 100



In [18]:

    
# Test Set n-pairs BLEU Scores
print_bleu([(rname, report['n_pairs_bleu_test']) for (rname, report) in reports])









    




Model Overall Score 1-gram Score 2-gram Score 3-gram Score 4-gram Score
encdec_noing6_200_512_04drb 25.7 38.4 24.5 21.8 21.3
encdec_noing6_bow_200_512_04drb 64.78 72.1 65.6 62 60



In [19]:

    
# Combined n-pairs BLEU Scores
print_bleu([(rname, report['n_pairs_bleu_all']) for (rname, report) in reports])









    




Model Overall Score 1-gram Score 2-gram Score 3-gram Score 4-gram Score
encdec_noing6_200_512_04drb 43.94 52.6 42.8 40.8 40.6
encdec_noing6_bow_200_512_04drb 70.85 76.8 71.3 68.7 67



In [20]:

    
# Ground Truth n-pairs BLEU Scores
print_bleu([(rname, report['n_pairs_bleu_gold']) for (rname, report) in reports])









    




Model Overall Score 1-gram Score 2-gram Score 3-gram Score 4-gram Score
encdec_noing6_200_512_04drb 12.22 25.4 13.9 9.2 6.8
encdec_noing6_bow_200_512_04drb 8.66 27.5 10.9 5.6 3.4

Alignment Analysis

This analysis computs the average Smith-Waterman alignment score for generations, with the same intuition as N-pairs BLEU, in that we expect low scores in the ground truth and hyper-common generations to raise the scores



In [21]:

    
def print_align(reports):
    data= [['<b>Model</b>', '<b>Average (Train) Generated Score</b>','<b>Average (Valid) Generated Score</b>','<b>Average (Test) Generated Score</b>','<b>Average (All) Generated Score</b>', '<b>Average (Gold) Score</b>']]
    for rname, report in reports:
        data.append([rname, report['average_alignment_train'], report['average_alignment_valid'], report['average_alignment_test'], report['average_alignment_all'], report['average_alignment_gold']])
    display_table(data)

print_align(reports)









    




Model Average (Train) Generated Score Average (Valid) Generated Score Average (Test) Generated Score Average (All) Generated Score Average (Gold) Score
encdec_noing6_200_512_04drb 43.7142857143 133 36.1904761905 58.4 26.080952381
encdec_noing6_bow_200_512_04drb 57.9047619048 125 77.0952380952 86.3095238095 19.9619047619

amazing watermelon greek salad with feta
Generated	combine the cornmeal , flour , sugar , baking powder , salt , and salt . add the milk , egg and vanilla .
True	combine the first 6 ingredients and shape into patties . cook patties in a large skillet over medium - low heat 20
Closest BLEU Match	combine the cornmeal , flour , sugar , mustard , baking powder and salt , mixing well . add the milk , egg
Closest BLEU Score	57.2545340067

hidden valley pinwheel sandwiches
Generated	preheat oven to 350 degrees . cut up waffles into bite size pieces . cut each apple into thin slices ; place the
True	chop your green pepper , red pepper , sweet onion , and carrots up . put your carrots off to the side .
Closest BLEU Match	preheat oven to 350 degrees . cut up waffles into bite size pieces . think of them as croutons and that is
Closest BLEU Score	54.969636447

chinese broccoli with oyster sauce
Generated	1 . preheat oven to 350 degrees . cut up waffles into bite size pieces . cut each apple into thin slices ;
True	1 . using the back of a spoon , mash raspberries in a bowl ; transfer to
Closest BLEU Match	preheat oven to 350 degrees . cut up waffles into bite size pieces . think of them as croutons and that is
Closest BLEU Score	54.969636447

hamburger muffins
Generated	in a large saucepan , add the garlic , and cook in a large skillet over medium heat . add the onion , and cook
True	stir together instant coffee , sugar and water . set aside . in a large mixer bowl , beat cream cheese until light
Closest BLEU Match	heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook
Closest BLEU Score	48.9453596219

fries ( gluten - free )
Generated	bacon gnocchi , garlic , and onions into a large pot or until tender . add the milk ,
True	bacon gnocchi
Closest BLEU Match	2 . ) add the celery , carrots , and onions into a large pot and
Closest BLEU Score	26.1585828258

Model	Train Perplexity	Valid Perplexity	Test Perplexity
encdec_noing6_200_512_04drb	2.30555069546	755.270092669	857.640092497
encdec_noing6_bow_200_512_04drb	1.8576546662	794.18428089	1009.45163985

basic guacamole
Generated	preheat oven to 350 degrees . cut up waffles into bite size pieces . cut each apple into thin slices ; place the
True	lima bean main dish salad
Closest BLEU Match	preheat oven to 350 degrees . cut up waffles into bite size pieces . think of them as croutons and that is
Closest BLEU Score	54.969636447

baked beans
Generated	preheat oven to 350 degrees . cut up waffles into bite size pieces . cut each apple into thin slices ; place the
True	preheat the oven to 475 degrees c ( 220 degrees c ) . roll out pizza crust and place on
Closest BLEU Match	preheat oven to 350 degrees . cut up waffles into bite size pieces . think of them as croutons and that is
Closest BLEU Score	54.969636447

chorizo burgers with sweet corn guacamole
Generated	1 . heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook until tender
True	1 . heat oil in a large nonstick skillet over medium - high heat . add chicken to pan ; sauté
Closest BLEU Match	heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook
Closest BLEU Score	79.543691219

chicken and gnocchi soup
Generated	preheat oven to 350 degrees f ( 175 degrees c ) . roll out pizza crust and place on a large skillet over medium
True	preheat oven to 325 degrees . combine rice , green onions , pineapple , edamame , and cashews in a large
Closest BLEU Match	preheat oven to 350 degrees f ( 175 degrees c ) . in a large pot over medium heat cook and stir
Closest BLEU Score	52.7211892094

cheese crusted apple pie
Generated	preheat oven to 350 degrees f ( 175 degrees c ) . roll out pizza crust and place on a large skillet over medium
True	preheat broiler . sprinkle the salmon steaks generously with salt and pepper . sprinkle with 1 tablespoon lemon
Closest BLEU Match	preheat oven to 350 degrees f ( 175 degrees c ) . in a large pot over medium heat cook and stir
Closest BLEU Score	52.7211892094

eggnog donut bread pudding
Generated	in a large saucepan , crack the egg and beat with a paddle . add the milk and 1 cup of the milk and
True	in a bowl , stir together the ricotta , the 3 / 4 oz . parmigiano - reggiano , the thyme , parsley
Closest BLEU Match	in a large bow , crack the egg and beat with a whisk . add 1 cup of the milk and whisk
Closest BLEU Score	54.093140266

chocolate lasagna
Generated	preheat oven to 350 degrees f ( 175 degrees c ) . roll out pizza crust and place on a large skillet over medium
True	arrange bread slices in a 12 x 8 x 2 baking dish . beat eggs with milk , grand
Closest BLEU Match	preheat oven to 350 degrees f ( 175 degrees c ) . in a large pot over medium heat cook and stir
Closest BLEU Score	52.7211892094

savory french omelet
Generated	1 . preheat oven to 350 degrees f ( 175 degrees c ) . in a large skillet over medium heat . add
True	combine the cornmeal , flour , sugar , mustard , baking powder and salt , mixing well . add the milk , egg
Closest BLEU Match	preheat oven to 350 degrees f ( 175 degrees c ) . in a large pot over medium heat cook and stir
Closest BLEU Score	64.7285941823

lasagna
Generated	preheat oven to 350 degrees f ( 175 degrees c ) . roll out pizza crust and place on a large skillet over medium
True	whisk together flour , sugar , baking powder , kosher salt , and bacon . store mix
Closest BLEU Match	preheat oven to 350 degrees f ( 175 degrees c ) . in a large pot over medium heat cook and stir
Closest BLEU Score	52.7211892094

pulled
Generated	preheat oven to 350 degrees . cut up waffles into bite size pieces . cut each apple into thin slices ; place the
True
Closest BLEU Match	preheat oven to 350 degrees . cut up waffles into bite size pieces . think of them as croutons and that is
Closest BLEU Score	54.969636447

shupp noodles - noodle omelet
Generated	1 . preheat oven to 350 degrees . cut up waffles into bite size pieces . cut each apple into thin slices ;
True	make noodles : beat eggs , add salt and as much flour as can be worked into the eggs to make
Closest BLEU Match	preheat oven to 350 degrees . cut up waffles into bite size pieces . think of them as croutons and that is
Closest BLEU Score	54.969636447

thai sweet chili sauce
Generated	1 . heat oil in a large skillet over medium heat . add the onion , corn and cook until tender , 25 until tender
True	throw all the ingredients but the last ( tapioca flour + water ) in a blender . i use my trusty magic bullet .
Closest BLEU Match	heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook
Closest BLEU Score	61.8233964096

easy paella
Generated	combine the cornmeal , flour , sugar , baking powder , salt , and salt . add the milk , egg and vanilla .
True	chicken dinner simmer
Closest BLEU Match	combine the cornmeal , flour , sugar , mustard , baking powder and salt , mixing well . add the milk , egg
Closest BLEU Score	57.2545340067

caramelized scallops with smoked chili cream
Generated	1 . heat oil in a large skillet over medium heat . add the onion , corn and cook until tender , 25 until tender
True	1 . prepare chili cream : mix chipotle powder in lime juice and let it sit and
Closest BLEU Match	heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook
Closest BLEU Score	61.8233964096

baked sandwiches
Generated	combine the cornmeal , flour , sugar , baking powder , salt , and salt . add the milk , egg and vanilla .
True	on a greased surface , roll out bread dough into a rectangle . stir together cooked meat ,
Closest BLEU Match	combine the cornmeal , flour , sugar , mustard , baking powder and salt , mixing well . add the milk , egg
Closest BLEU Score	57.2545340067

saute ? ed mushrooms
Generated	preheat oven to 350 degrees . cut up waffles into bite size pieces . cut each apple into thin slices ; place the
True	1 . cook shiitake mushrooms in a single layer in 1 1 / 2 tbsp . hot oil in a 10 - to
Closest BLEU Match	preheat oven to 350 degrees . cut up waffles into bite size pieces . think of them as croutons and that is
Closest BLEU Score	54.969636447

polish doughnuts
Generated	preheat oven to 350 degrees f ( 175 degrees c ) . roll out pizza crust and place on a large skillet over medium
True	smash up yeast in a medium bowl . add water and 1 teaspoon sugar , and mix until pasty . set aside
Closest BLEU Match	preheat oven to 350 degrees f ( 175 degrees c ) . in a large pot over medium heat cook and stir
Closest BLEU Score	52.7211892094

slow cooker charro beans
Generated	preheat oven to 350 degrees f ( 175 degrees c ) . roll out pizza crust and place on a large skillet over medium
True	place beans in a colander , rinse well , and remove any stones or shriveled beans
Closest BLEU Match	preheat oven to 350 degrees f ( 175 degrees c ) . in a large pot over medium heat cook and stir
Closest BLEU Score	52.7211892094

quick and easy white bean salad
Generated	preheat oven to 350 degrees f ( 175 degrees c ) . roll out pizza crust and place on a large skillet over medium
True	1 after you chop up the onion , squeeze a little lemon juice over it and let it sit while prepping
Closest BLEU Match	preheat oven to 350 degrees f ( 175 degrees c ) . in a large pot over medium heat cook and stir
Closest BLEU Score	52.7211892094

kimchi
Generated	preheat oven to 350 degrees f ( 175 degrees c ) . roll out pizza crust and place on a large skillet over medium
True	cut the cabbages lengthwise into quarters , then across into thick ribbons . put the cabbage in a big bowl and use your hands
Closest BLEU Match	preheat oven to 350 degrees f ( 175 degrees c ) . in a large pot over medium heat cook and stir
Closest BLEU Score	52.7211892094

black bean & bacon party dip
Generated	in a large saucepan , crack the egg and beat with a paddle . add the milk and 1 cup of the milk and
True	for the bacon , either cook it by the traditional stove - top method , or you can preheat
Closest BLEU Match	in a large bow , crack the egg and beat with a whisk . add 1 cup of the milk and whisk
Closest BLEU Score	54.093140266

pita bread salad
Generated	preheat oven to 350 degrees f ( 175 degrees c ) . roll out pizza crust and place on a large skillet over medium
True	preheat the oven to 350 degrees . transfer the pitas to a baking sheet ; bake until crisp , about 10 minutes .
Closest BLEU Match	preheat oven to 350 degrees f ( 175 degrees c ) . in a large pot over medium heat cook and stir
Closest BLEU Score	52.7211892094

charbroiled oysters from dragos
Generated	1 . preheat oven to 350 degrees f ( 175 degrees c ) . in a large skillet over medium heat . add
True	heat the grill to med - high . melt butter with garlic and pepper in a large
Closest BLEU Match	preheat oven to 350 degrees f ( 175 degrees c ) . in a large pot over medium heat cook and stir
Closest BLEU Score	64.7285941823

Model	Overall Score	1-gram Score	2-gram Score	3-gram Score	4-gram Score
encdec_noing6_200_512_04drb	3.35	19.8	5.7	1.8	0.6
encdec_noing6_bow_200_512_04drb	7.55	22.5	10.9	5.4	2.5

Model	Average (Train) Generated Score	Average (Valid) Generated Score	Average (Test) Generated Score	Average (All) Generated Score	Average (Gold) Score
encdec_noing6_200_512_04drb	43.7142857143	133	36.1904761905	58.4	26.080952381
encdec_noing6_bow_200_512_04drb	57.9047619048	125	77.0952380952	86.3095238095	19.9619047619