notebook.community

Edit and run

Comparing Encoder-Decoders Analysis

Model Architecture





In [1]:



    


report_files = ["/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing6_200_512_04drb/encdec_noing6_200_512_04drb.json", "/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing10_200_512_04drb/encdec_noing10_200_512_04drb.json", "/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing15_200_512_04drb/encdec_noing15_200_512_04drb.json", "/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing23_200_512_04drb/encdec_noing23_200_512_04drb.json"]
log_files = ["/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing6_200_512_04drb/encdec_noing6_200_512_04drb_logs.json", "/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing10_200_512_04drb/encdec_noing10_200_512_04drb_logs.json", "/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing15_200_512_04drb/encdec_noing15_200_512_04drb_logs.json", "/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing23_200_512_04drb/encdec_noing23_200_512_04drb_logs.json"]
reports = []
logs = []
import json
import matplotlib.pyplot as plt
import numpy as np

for report_file in report_files:
    with open(report_file) as f:
        reports.append((report_file.split('/')[-1].split('.json')[0], json.loads(f.read())))
for log_file in log_files:
    with open(log_file) as f:
        logs.append((log_file.split('/')[-1].split('.json')[0], json.loads(f.read())))
        
for report_name, report in reports:
    print '\n', report_name, '\n'
    print 'Encoder: \n', report['architecture']['encoder']
    print 'Decoder: \n', report['architecture']['decoder']



    


















    






encdec_noing6_200_512_04drb 

Encoder: 
nn.Sequential {
  [input -> (1) -> (2) -> (3) -> (4) -> output]
  (1): nn.LookupTable
  (2): nn.Dropout(0.400000)
  (3): nn.LSTM(200 -> 512)
  (4): nn.Dropout(0.400000)
}
Decoder: 
nn.gModule

encdec_noing10_200_512_04drb 

Encoder: 
nn.Sequential {
  [input -> (1) -> (2) -> (3) -> (4) -> output]
  (1): nn.LookupTable
  (2): nn.Dropout(0.400000)
  (3): nn.LSTM(200 -> 512)
  (4): nn.Dropout(0.400000)
}
Decoder: 
nn.gModule

encdec_noing15_200_512_04drb 

Encoder: 
nn.Sequential {
  [input -> (1) -> (2) -> (3) -> (4) -> output]
  (1): nn.LookupTable
  (2): nn.Dropout(0.400000)
  (3): nn.LSTM(200 -> 512)
  (4): nn.Dropout(0.400000)
}
Decoder: 
nn.gModule

encdec_noing23_200_512_04drb 

Encoder: 
nn.Sequential {
  [input -> (1) -> (2) -> (3) -> (4) -> output]
  (1): nn.LookupTable
  (2): nn.Dropout(0.400000)
  (3): nn.LSTM(200 -> 512)
  (4): nn.Dropout(0.400000)
}
Decoder: 
nn.gModule

Perplexity on Each Dataset





In [2]:



    


%matplotlib inline
from IPython.display import HTML, display

def display_table(data):
    display(HTML(
        u'<table><tr>{}</tr></table>'.format(
            u'</tr><tr>'.join(
                u'<td>{}</td>'.format('</td><td>'.join(unicode(_) for _ in row)) for row in data)
            )
    ))

def bar_chart(data):
    n_groups = len(data)
    
    train_perps = [d[1] for d in data]
    valid_perps = [d[2] for d in data]
    test_perps = [d[3] for d in data]
    
    fig, ax = plt.subplots(figsize=(10,8))
    
    index = np.arange(n_groups)
    bar_width = 0.3

    opacity = 0.4
    error_config = {'ecolor': '0.3'}

    train_bars = plt.bar(index, train_perps, bar_width,
                     alpha=opacity,
                     color='b',
                     error_kw=error_config,
                     label='Training Perplexity')

    valid_bars = plt.bar(index + bar_width, valid_perps, bar_width,
                     alpha=opacity,
                     color='r',
                     error_kw=error_config,
                     label='Valid Perplexity')
    test_bars = plt.bar(index + 2*bar_width, test_perps, bar_width,
                     alpha=opacity,
                     color='g',
                     error_kw=error_config,
                     label='Test Perplexity')

    plt.xlabel('Model')
    plt.ylabel('Scores')
    plt.title('Perplexity by Model and Dataset')
    plt.xticks(index + bar_width / 3, [d[0] for d in data])
    plt.legend()

    plt.tight_layout()
    plt.show()

data = [['<b>Model</b>', '<b>Train Perplexity</b>', '<b>Valid Perplexity</b>', '<b>Test Perplexity</b>']]

for rname, report in reports:
    data.append([rname, report['train_perplexity'], report['valid_perplexity'], report['test_perplexity']])

display_table(data)
bar_chart(data[1:])



    


















    







ModelTrain PerplexityValid PerplexityTest Perplexity
encdec_noing6_200_512_04drb2.30555069546755.270092669857.640092497
encdec_noing10_200_512_04drb2.36730175651715.007688611868.108036068
encdec_noing15_200_512_04drb2.27752305377706.426656909805.596650048
encdec_noing23_200_512_04drb2.22041950514740.070604729893.207220523

Loss vs. Epoch





In [3]:



    


%matplotlib inline
plt.figure(figsize=(10, 8))
for rname, l in logs:
    for k in l.keys():
        plt.plot(l[k][0], l[k][1], label=str(k) + ' ' + rname + ' (train)')
        plt.plot(l[k][0], l[k][2], label=str(k) + ' ' + rname + ' (valid)')
plt.title('Loss v. Epoch')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()
plt.show()

Perplexity vs. Epoch





In [4]:



    


%matplotlib inline
plt.figure(figsize=(10, 8))
for rname, l in logs:
    for k in l.keys():
        plt.plot(l[k][0], l[k][3], label=str(k) + ' ' + rname + ' (train)')
        plt.plot(l[k][0], l[k][4], label=str(k) + ' ' + rname + ' (valid)')
plt.title('Perplexity v. Epoch')
plt.xlabel('Epoch')
plt.ylabel('Perplexity')
plt.legend()
plt.show()

Generations





In [5]:



    


def print_sample(sample, best_bleu=None):
    enc_input = ' '.join([w for w in sample['encoder_input'].split(' ') if w != '<pad>'])
    gold = ' '.join([w for w in sample['gold'].split(' ') if w != '<mask>'])
    print('Input: '+ enc_input + '\n')
    print('Gend: ' + sample['generated'] + '\n')
    print('True: ' + gold + '\n')
    if best_bleu is not None:
        cbm = ' '.join([w for w in best_bleu['best_match'].split(' ') if w != '<mask>'])
        print('Closest BLEU Match: ' + cbm + '\n')
        print('Closest BLEU Score: ' + str(best_bleu['best_score']) + '\n')
    print('\n')
    
def display_sample(sample, best_bleu=None):
    enc_input = ' '.join([w for w in sample['encoder_input'].split(' ') if w != '<pad>'])
    gold = ' '.join([w for w in sample['gold'].split(' ') if w != '<mask>'])
    data = [['<u><b>' + enc_input + '</b></u>', '']]
    data.append(['<b>Generated</b>', sample['generated']])
    data.append(['<b>True</b>',gold])
    if best_bleu is not None:
        cbm = ' '.join([w for w in best_bleu['best_match'].split(' ') if w != '<mask>'])
        data.append(['<b>Closest BLEU Match</b>', cbm])
        data.append(['<b>Closest BLEU Score</b>', str(best_bleu['best_score'])])
    display_table(data)



    











In [6]:



    


for rname, report in reports:
    display(HTML('<h3>' + rname + ' (train)</h3>'))
    for i, sample in enumerate(report['train_samples']):
        display_sample(sample, report['best_bleu_matches_train'][i] if 'best_bleu_matches_train' in report else None)



    


















    







encdec_noing6_200_512_04drb (train)










    







 amazing watermelon greek salad with feta
Generated  combine the cornmeal , flour , sugar , baking powder , salt , and salt .  add the milk , egg and vanilla .
True combine the first 6 ingredients and shape into patties .  cook patties in a large skillet over medium - low heat 20 
Closest BLEU Match combine the cornmeal , flour , sugar , mustard , baking powder and salt , mixing well .  add the milk , egg 
Closest BLEU Score57.2545340067










    







 hidden valley pinwheel sandwiches
Generated  preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
True chop your green pepper , red pepper , sweet onion , and carrots up . put your carrots off to the side . 
Closest BLEU Match preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score54.969636447










    







 chinese broccoli with oyster sauce
Generated  1 . preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ;
True 1 . using the back of a spoon , mash raspberries in a bowl ; transfer to 
Closest BLEU Match preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score54.969636447










    







 hamburger muffins
Generated  in a large saucepan , add the garlic , and cook in a large skillet over medium heat . add the onion , and cook
True stir together instant coffee , sugar and water . set aside .  in a large mixer bowl , beat cream cheese until light 
Closest BLEU Match heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook 
Closest BLEU Score48.9453596219










    







 fries ( gluten - free )
Generated  bacon  gnocchi     , garlic , and onions into a large pot or until tender .  add the milk ,
True bacon  gnocchi  
Closest BLEU Match 2 . ) add the celery , carrots , and onions into a large pot and 
Closest BLEU Score26.1585828258










    







 basic guacamole
Generated  preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
True lima bean  main dish  salad  
Closest BLEU Match preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score54.969636447










    







 baked beans
Generated  preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
True preheat the oven to 475 degrees c ( 220 degrees c ) .  roll out pizza crust and place on 
Closest BLEU Match preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score54.969636447










    







encdec_noing10_200_512_04drb (train)










    







 strawberry shortcakes
Generated  combine the cornmeal , flour , sugar , baking powder , and salt , mixing well .  add the milk , and bread crumbs
True combine coconut , flour , salt .  add milk and vanilla .  i find it much easier to use 
Closest BLEU Match combine the cornmeal , flour , sugar , mustard , baking powder and salt , mixing well .  add the milk , egg 
Closest BLEU Score70.1611656261










    







 5 - minute healthy strawberry frozen yogurt
Generated  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat . add the onion ,
True preheat oven to 350  brown beef in skillet & add taco seasoning . cook as directed on seasoning packet  spray an 
Closest BLEU Match preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score64.7285941823










    







 curry chickpea salad
Generated  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat . add the onion ,
True preheat oven to 375 degrees . lightly grease donut pans ( this one is my favorite 
Closest BLEU Match preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score64.7285941823










    







 tomato soup with white beans and pasta
Generated  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat . add the onion ,
True preheat oven to 400 . pierce potatoes several times with a fork . transfer to a large baking dish and cook 
Closest BLEU Match preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score64.7285941823










    







 coffee cake in a mug with cinnamon oatmeal struesel topping
Generated  to make the apple - cranberry relish : peel , core , and chop the apples into 1 / 4 - inch - inch -
True to make the apple - cranberry relish : peel , core , and chop the apples into 1 / 4 - inch 
Closest BLEU Match to make the apple - cranberry relish : peel , core , and chop the apples into 1 / 4 - inch 
Closest BLEU Score83.6360058744










    







 grilled pizza with no - cook tomato sauce
Generated  1 . heat oil in a large skillet over medium heat . add the onion , and cook over medium - low heat for 20
True heat 1 tbsp . oil in a large skillet over medium - high heat . add runner beans and swiss chard 
Closest BLEU Match heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook 
Closest BLEU Score55.6126655011










    







 amazing watermelon greek salad with feta
Generated  1 . heat oil in a large skillet over medium heat . add the onion , and cook over medium - low heat for 20
True combine the first 6 ingredients and shape into patties .  cook patties in a large skillet over medium - low heat 20 
Closest BLEU Match heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook 
Closest BLEU Score55.6126655011










    







encdec_noing15_200_512_04drb (train)










    







 mussels with thyme and shallots
Generated  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat .  add the onion
True preparation : place the rice , chicken broth and 1 tablespoon of the olive oil in a 
Closest BLEU Match preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score64.7285941823










    







 chocolate pancakes
Generated  1 . preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat .  add
True heat a medium sized skillet to medium heat . when hot , add olive oil and then 
Closest BLEU Match preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score64.7285941823










    







 zaru soba ( cold soba noodles )
Generated  1 . preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is
True 1 . preheat oven 350º f .  2 . cut one half the onion into thin slices ; place in a large bowl 
Closest BLEU Match preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score87.7277201636










    







 hamburger muffins
Generated  in a large bowl , combine the ice , garlic , and cashews in a bowl , combine the chips .  add the milk
True stir together instant coffee , sugar and water . set aside .  in a large mixer bowl , beat cream cheese until light 
Closest BLEU Match in a large bowl , combine the cottage cheese , sour cream , egg , salt , garlic salt and pepper .  add cheddar
Closest BLEU Score30.8264237423










    







 pizza zacineti breadsticks
Generated  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
True using a mixer with a paddle , cream together the butter , sugar and eggs until smooth . add the vanilla and zest and combine
Closest BLEU Match preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score87.7277201636










    







 egg sandwiches with wilted spinach
Generated  1 . preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is
True 1 . preheat oven to 350 degrees f. and spray donut pan with non - stick cooking spray . 
Closest BLEU Match preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score87.7277201636










    







 mussels with thyme and shallots
Generated  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat .  add the onion
True preparation : place the rice , chicken broth and 1 tablespoon of the olive oil in a 
Closest BLEU Match preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score64.7285941823










    







encdec_noing23_200_512_04drb (train)










    







 pumpkin ravioli
Generated  1 . heat oil in a large skillet over medium heat . add the onion , and cook over medium - low heat . add
True heat oil in a soup pot or dutch oven over moderate heat until hot . add onions and fennel and cook until tender 
Closest BLEU Match heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook 
Closest BLEU Score55.6126655011










    







 lasagna
Generated  combine the cornmeal , flour , sugar , baking powder , and salt .  add the milk , and 1 .  add the
True whisk together flour , sugar , baking powder , kosher salt , and bacon . store mix 
Closest BLEU Match combine the cornmeal , flour , sugar , mustard , baking powder and salt , mixing well .  add the milk , egg 
Closest BLEU Score51.4184255717










    







 bacon - wrapped egg sandwich
Generated  preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a cooling rack on top
True preheat oven to 350 . slice and peel the mangoes . place mango flesh in a food processor or blender and puree for 10 
Closest BLEU Match preheat oven to 350 degrees f ( 175 degrees c ) .  rinse the chicken pieces and place them , single layer 
Closest BLEU Score49.447428999










    







 lemon delight mini cheesecake
Generated  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
True  
Closest BLEU Match preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score87.7277201636










    







 tofu dengaku
Generated  in a large saucepan , add all and cook until tender .  add the milk , egg , and onions into 1 / 4
True place the split peas in a fine - mesh strainer and rinse thoroughly under cold water . transfer to a large saucepan , add the
Closest BLEU Match in a large saucepan , cook beef , onion , and garlic until meat is browned ; drain .  add the 
Closest BLEU Score26.1585828258










    







 fish in creamy sauce with cheese
Generated  1 . preheat oven to 350 degrees .  2 on a greased baking sheet , lay out the crescent seamless dough dough and dough
True 1 . in a small bowl , combine the miso , tomato paste , chile - garlic paste and a 
Closest BLEU Match 1 preheat oven to 375 degrees f .  2 on a greased baking sheet , lay out the crescent seamless dough 
Closest BLEU Score62.2159456296










    







 beef barley soup
Generated  in a large saucepan , add all and cook until tender .  add the milk , egg , and onions into 1 / 4
True in a medium size bowl whisk together eggs and cajun spice blend and set aside .  in a large non stick skillet over medium
Closest BLEU Match in a large saucepan , cook beef , onion , and garlic until meat is browned ; drain .  add the 
Closest BLEU Score26.1585828258

















In [7]:



    


for rname, report in reports:
    display(HTML('<h3>' + rname + ' (valid)</h3>'))
    for i, sample in enumerate(report['valid_samples']):
        display_sample(sample, report['best_bleu_matches_valid'][i] if 'best_bleu_matches_valid' in report else None)



    


















    







encdec_noing6_200_512_04drb (valid)










    







      
Generated  preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
True in a bowl , combine the crabmeat and shrimp with the vodka , lemon peel and juice ; season with salt 
Closest BLEU Match preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score54.969636447










    







      
Generated  preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
True heat the oil in a large skillet over medium heat .  add shallots and saute ' until softened , 
Closest BLEU Match preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score54.969636447










    







      
Generated  preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
True in a food processor or food mill , grind or mince beans until fine .  saute onion in oil until 
Closest BLEU Match preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score54.969636447










    







      
Generated  preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
True 1 . finely dice the onion and chop all of the mushrooms into halves .  2 . place in a large wok 
Closest BLEU Match preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score54.969636447










    







      
Generated  preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
True slice the onions into rings and the garlic into discs and place them in the bottom of the slow cooker .  
Closest BLEU Match preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score54.969636447










    







      
Generated  preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
True brown the beef in a large cast iron skillet over medium heat until the fat begins to render . add the onion 
Closest BLEU Match preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score54.969636447










    







 pulled     
Generated  preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
True  
Closest BLEU Match preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score54.969636447










    







encdec_noing10_200_512_04drb (valid)










    







          
Generated  1 . heat oil in a large skillet over medium heat . add the onion , and cook over medium - low heat for 20
True for dipping sauce , the dipping sauce is supposed to be a little salty because you will be `` dipping '' the 
Closest BLEU Match heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook 
Closest BLEU Score55.6126655011










    







          
Generated  1 . heat oil in a large skillet over medium heat . add the onion , and cook over medium - low heat for 20
True preheat oven to 400 ? .  arrange prosciutto in a single layer on a baking sheet . bake at 400 ? 
Closest BLEU Match heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook 
Closest BLEU Score55.6126655011










    







          
Generated  1 . heat oil in a large skillet over medium heat . add the onion , and cook over medium - low heat for 20
True take the scallops and slice them very thinly .  wash the kumquats and remove the seeds . slice very thinly .  peel the
Closest BLEU Match heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook 
Closest BLEU Score55.6126655011










    







          
Generated  1 . heat oil in a large skillet over medium heat . add the onion , and cook over medium - low heat for 20
True in a food processor , pulse 1 bun until fine crumbs form ( you should have about 1 / 4 cup ) 
Closest BLEU Match heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook 
Closest BLEU Score55.6126655011










    







          
Generated  1 . heat oil in a large skillet over medium heat . add the onion , and cook over medium - low heat for 20
True cut each potato into 12 wedges .  preheat oven to 225c / 475f ° .  combine the garlic 
Closest BLEU Match heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook 
Closest BLEU Score55.6126655011










    







          
Generated  1 . heat oil in a large skillet over medium heat . add the onion , and cook over medium - low heat for 20
True cook onion , garlic , and ginger in oil with cardamom and curry powder until onion is soft , 8–10 minutes . 
Closest BLEU Match heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook 
Closest BLEU Score55.6126655011










    







          
Generated  1 . heat oil in a large skillet over medium heat . add the onion , and cook over medium - low heat for 20
True if desired , slightly mash chickpeas with a fork or potato masher . in a large bowl , combine all salad 
Closest BLEU Match heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook 
Closest BLEU Score55.6126655011










    







encdec_noing15_200_512_04drb (valid)










    







               
Generated  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat .  add the onion
True in a food processor , pulse 1 bun until fine crumbs form ( you should have about 1 / 4 cup ) 
Closest BLEU Match preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score64.7285941823










    







               
Generated  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat .  add the onion
True slice the onions into rings and the garlic into discs and place them in the bottom of the slow cooker .  
Closest BLEU Match preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score64.7285941823










    







               
Generated  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat .  add the onion
True in a large food processor , crush oreo cookies into fine crumbs . make sure there are no large chunks 
Closest BLEU Match preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score64.7285941823










    







  pulled             
Generated  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat .  add the onion
True melt butter in a large heavy saucepan .  add onions , celery , carrots , garlic and sauté until 
Closest BLEU Match preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score64.7285941823










    







  pulled             
Generated  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat .  add the onion
True 1 . cook chicken in a pan with a drizzle of olive oil and some s & p and 
Closest BLEU Match preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score64.7285941823










    







  pulled             
Generated  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat .  add the onion
True preheat oven and bake french fries as directed on bag .  take your flour tortillas and heat 
Closest BLEU Match preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score64.7285941823










    







               
Generated  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat .  add the onion
True line a 6x9 baking dish with aluminum foil .  combine the sugar , salt , pepper , and spices .  rub 
Closest BLEU Match preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score64.7285941823










    







encdec_noing23_200_512_04drb (valid)










    







                       
Generated  in a large saucepan , add all and cook until tender .  add the milk , egg , and onions into 1 / 4
True sprinkle scallops with salt and pepper .  melt 3 tablespoons butter in very large skillet over medium - 
Closest BLEU Match in a large saucepan , cook beef , onion , and garlic until meat is browned ; drain .  add the 
Closest BLEU Score26.1585828258










    







          pulled             
Generated  in a large saucepan , add all and cook until tender .  add the milk , egg , and onions into 1 / 4
True 1 . cook chicken in a pan with a drizzle of olive oil and some s & p and 
Closest BLEU Match in a large saucepan , cook beef , onion , and garlic until meat is browned ; drain .  add the 
Closest BLEU Score26.1585828258










    







  pulled                    pulled 
Generated  in a large saucepan , add all and cook until tender .  add the milk , egg , and onions into 1 / 4
True in a medium bowl , stir together the tomatoes , 1 / 4 cup of water , 
Closest BLEU Match in a large saucepan , cook beef , onion , and garlic until meat is browned ; drain .  add the 
Closest BLEU Score26.1585828258










    







                       
Generated  in a large saucepan , add all and cook until tender .  add the milk , egg , and onions into 1 / 4
True heat oven to 375 ? f . cook chorizo in large ovenproof skillet on medium - high heat 
Closest BLEU Match in a large saucepan , cook beef , onion , and garlic until meat is browned ; drain .  add the 
Closest BLEU Score26.1585828258










    







                       
Generated  in a large saucepan , add all and cook until tender .  add the milk , egg , and onions into 1 / 4
True in a medium bowl , combine radish , cucumber , cilantro , garlic , lime juice , and 
Closest BLEU Match in a large saucepan , cook beef , onion , and garlic until meat is browned ; drain .  add the 
Closest BLEU Score26.1585828258










    







          pulled             
Generated  in a large saucepan , add all and cook until tender .  add the milk , egg , and onions into 1 / 4
True in a food processor or food mill , grind or mince beans until fine .  saute onion in oil until 
Closest BLEU Match in a large saucepan , cook beef , onion , and garlic until meat is browned ; drain .  add the 
Closest BLEU Score26.1585828258










    







                       
Generated  in a large saucepan , add all and cook until tender .  add the milk , egg , and onions into 1 / 4
True combine oyster sauce , sugar and 2 tablespoons water in a small bowl . mix well and set aside 
Closest BLEU Match in a large saucepan , cook beef , onion , and garlic until meat is browned ; drain .  add the 
Closest BLEU Score26.1585828258

















In [8]:



    


for rname, report in reports:
    display(HTML('<h3>' + rname + ' (test)</h3>'))
    for i, sample in enumerate(report['test_samples']):
        display_sample(sample, report['best_bleu_matches_test'][i] if 'best_bleu_matches_test' in report else None)



    


















    







encdec_noing6_200_512_04drb (test)










    







 shupp noodles - noodle omelet
Generated  1 . preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ;
True make noodles : beat eggs , add salt and as much flour as can be worked into the eggs to make 
Closest BLEU Match preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score54.969636447










    







 thai sweet chili sauce
Generated  1 . heat oil in a large skillet over medium heat . add the onion , corn and cook until tender , 25 until tender
True throw all the ingredients but the last ( tapioca flour + water ) in a blender . i use my trusty magic bullet . 
Closest BLEU Match heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook 
Closest BLEU Score61.8233964096










    







 easy paella
Generated  combine the cornmeal , flour , sugar , baking powder , salt , and salt .  add the milk , egg and vanilla .
True chicken  dinner  simmer  
Closest BLEU Match combine the cornmeal , flour , sugar , mustard , baking powder and salt , mixing well .  add the milk , egg 
Closest BLEU Score57.2545340067










    







 easy paella
Generated  combine the cornmeal , flour , sugar , baking powder , salt , and salt .  add the milk , egg and vanilla .
True chicken  dinner  simmer  
Closest BLEU Match combine the cornmeal , flour , sugar , mustard , baking powder and salt , mixing well .  add the milk , egg 
Closest BLEU Score57.2545340067










    







 caramelized scallops with smoked chili cream
Generated  1 . heat oil in a large skillet over medium heat . add the onion , corn and cook until tender , 25 until tender
True 1 . prepare chili cream : mix chipotle powder in lime juice and let it sit and 
Closest BLEU Match heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook 
Closest BLEU Score61.8233964096










    







 baked sandwiches
Generated  combine the cornmeal , flour , sugar , baking powder , salt , and salt .  add the milk , egg and vanilla .
True on a greased surface , roll out bread dough into a rectangle . stir together cooked meat , 
Closest BLEU Match combine the cornmeal , flour , sugar , mustard , baking powder and salt , mixing well .  add the milk , egg 
Closest BLEU Score57.2545340067










    







 saute ? ed mushrooms
Generated  preheat oven to 350 degrees .  cut up waffles into bite size pieces .  cut each apple into thin slices ; place the
True 1 . cook shiitake mushrooms in a single layer in 1 1 / 2 tbsp . hot oil in a 10 - to 
Closest BLEU Match preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score54.969636447










    







encdec_noing10_200_512_04drb (test)










    







 tasty green bean casserole
Generated  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat . add the onion ,
True preheat the oven to 325 degrees f ( 165 degrees c ) .  fry bacon in a skillet over medium - high 
Closest BLEU Match preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score64.7285941823










    







 perfect porterhouse steak
Generated  in a large saucepan , combine the ice , garlic , and cashews in a large bowl . add the milk and stir it until
True place steak on a plate and coat lightly with olive oil . rub steak with meat tenderizer . sprinkle 
Closest BLEU Match in a large saucepan , combine the water , lentils , carrot , onion , garlic , and bay 
Closest BLEU Score31.3482066596










    







 charbroiled oysters from dragos
Generated  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat . add the onion ,
True heat the grill to med - high .  melt butter with garlic and pepper in a large 
Closest BLEU Match preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score64.7285941823










    







 mac - n - cheese pizza
Generated  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat . add the onion ,
True preheat oven to 400 degrees f ( 200 degrees c ) . spray a 10x15 - inch baking sheet with cooking spray . 
Closest BLEU Match preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score64.7285941823










    







 mediterranean sandwich
Generated  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
True mix the olive oil and balsamic vinegar . set aside .  cook the chicken breast and cut into strips . 
Closest BLEU Match preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score87.7277201636










    







 coconut curried shrimp
Generated  heat oil in a large skillet over medium heat . add the onion , and cook over medium - low heat for 20 20 minutes
True in a small , heavy saucepan , bring the rice and 1 1 / 2 cups water to a 
Closest BLEU Match heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook 
Closest BLEU Score55.6126655011










    







 bbq baby back ribs - pressure cooker
Generated  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat . add the onion ,
True place the metal rack in your cooker and add 1 / 2 cup of water ( 
Closest BLEU Match preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score64.7285941823










    







encdec_noing15_200_512_04drb (test)










    







 no - bake strawberry cheesecake
Generated  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat .  add the onion
True 1 . beat cream cheese and sugar until smooth . fold in whipped topping . spoon mixture into 
Closest BLEU Match preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score64.7285941823










    







 steak and cheese sandwich
Generated  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
True heat small amount of oil to medium heat in skillet and add veggies in following order : peppers , mushrooms , onions and 
Closest BLEU Match preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score87.7277201636










    







 greek pizza
Generated  combine the cornmeal , flour , sugar , baking powder , salt , and salt .  add the milk , egg , and vanilla
True 1 . preheat a pizza stone in a 500 degree oven for at least 30 minutes . place a piece of parchment paper on a
Closest BLEU Match combine the cornmeal , flour , sugar , mustard , baking powder and salt , mixing well .  add the milk , egg 
Closest BLEU Score57.2545340067










    







 roasted pork cuban sandwich
Generated  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat .  add the onion
True mix the roast seasoning blend together and set aside .  rinse roast , pat dry , coat 
Closest BLEU Match preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score64.7285941823










    







 applebee 's bourbon street steak
Generated  in a large bowl , combine the ice , garlic , and cashews in a bowl , combine the chips .  add the milk
True combine bourbon , brown sugar , salt , and pepper over steaks .  preheat grill .  place steaks on grill and cook 
Closest BLEU Match in a large bowl , combine the cottage cheese , sour cream , egg , salt , garlic salt and pepper .  add cheddar
Closest BLEU Score30.8264237423










    







 baked sandwiches
Generated  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat .  add the onion
True on a greased surface , roll out bread dough into a rectangle . stir together cooked meat , 
Closest BLEU Match preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir 
Closest BLEU Score64.7285941823










    







 skillet macaroni and cheese
Generated  1 . preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is
True 1 bring a pot of salty water to boil and cook the pasta until nearing al dente , but not full cooked . drain 
Closest BLEU Match preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score87.7277201636










    







encdec_noing23_200_512_04drb (test)










    







 stovetop cheddar mac
Generated  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
True in a large saucepan of boiling , salted water , cook the macaroni until al dente ; drain in a colander 
Closest BLEU Match preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score87.7277201636










    







 quick and easy white bean salad
Generated  preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a cooling rack on top
True 1 after you chop up the onion , squeeze a little lemon juice over it and let it sit while prepping 
Closest BLEU Match preheat oven to 350 degrees f ( 175 degrees c ) .  rinse the chicken pieces and place them , single layer 
Closest BLEU Score49.447428999










    







 dutch baby pancake
Generated  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
True preheat oven to 425 degrees . in a medium cast - iron or ovenproof nonstick skillet , melt 2 tablespoons butter 
Closest BLEU Match preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score87.7277201636










    







 thai peanut zucchini noodles
Generated  1 . preheat oven to 350 degrees .  2 on a greased baking sheet , lay out the crescent seamless dough dough and dough
True in a small bowl , combine the peanut butter , lime juice , honey , sesame oil , garlic and chili paste 
Closest BLEU Match 1 preheat oven to 375 degrees f .  2 on a greased baking sheet , lay out the crescent seamless dough 
Closest BLEU Score62.2159456296










    







 moroccan pot roast with potatoes and carrots
Generated  preheat oven to 350 degrees f ( 175 degrees c ) .  roll out pizza crust and place on a cooling rack on top
True 1 . coat the inside of a slow cooker with cooking spray . combine tomato paste , broth 
Closest BLEU Match preheat oven to 350 degrees f ( 175 degrees c ) .  rinse the chicken pieces and place them , single layer 
Closest BLEU Score49.447428999










    







 shrimp with green sauce
Generated  1 . heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook until tender
True 1 preheat the oven to 500f ° . in a food processor blitz the garlic . trim 
Closest BLEU Match heat oil in a large skillet over medium heat . add the onion , corn and soy beans , and cook 
Closest BLEU Score79.543691219










    







 butternut squash risotto
Generated  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
True combine 2 cups squash and 2 1 / 2 cups water in a saucepan ; bring to a boil . reduce 
Closest BLEU Match preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is 
Closest BLEU Score87.7277201636

BLEU Analysis





In [9]:



    


def print_bleu(blue_structs):
    data= [['<b>Model</b>', '<b>Overall Score</b>','<b>1-gram Score</b>','<b>2-gram Score</b>','<b>3-gram Score</b>','<b>4-gram Score</b>']]
    for rname, blue_struct in blue_structs:
        data.append([rname, blue_struct['score'], blue_struct['components']['1'], blue_struct['components']['2'], blue_struct['components']['3'], blue_struct['components']['4']])
    display_table(data)



    











In [10]:



    


# Training Set BLEU Scores
print_bleu([(rname, report['train_bleu']) for (rname, report) in reports])



    


















    







ModelOverall Score1-gram Score2-gram Score3-gram Score4-gram Score
encdec_noing6_200_512_04drb3.3519.85.71.80.6
encdec_noing10_200_512_04drb27.3445.628.623.818
encdec_noing15_200_512_04drb7.9323.110.35.43.1
encdec_noing23_200_512_04drb11.142814.37.75

















In [11]:



    


# Validation Set BLEU Scores
print_bleu([(rname, report['valid_bleu']) for (rname, report) in reports])



    


















    







ModelOverall Score1-gram Score2-gram Score3-gram Score4-gram Score
encdec_noing6_200_512_04drb012.12.300
encdec_noing10_200_512_04drb0173.40.60
encdec_noing15_200_512_04drb017.67.41.80
encdec_noing23_200_512_04drb023.65.70.60

















In [12]:



    


# Test Set BLEU Scores
print_bleu([(rname, report['test_bleu']) for (rname, report) in reports])



    


















    







ModelOverall Score1-gram Score2-gram Score3-gram Score4-gram Score
encdec_noing6_200_512_04drb010.41.100
encdec_noing10_200_512_04drb8.0630.212.661.9
encdec_noing15_200_512_04drb013.21.700
encdec_noing23_200_512_04drb012.62.30.60

















In [13]:



    


# All Data BLEU Scores
print_bleu([(rname, report['combined_bleu']) for (rname, report) in reports])



    


















    







ModelOverall Score1-gram Score2-gram Score3-gram Score4-gram Score
encdec_noing6_200_512_04drb1.5214.130.60.2
encdec_noing10_200_512_04drb13.253114.910.16.6
encdec_noing15_200_512_04drb4.1117.96.52.41
encdec_noing23_200_512_04drb5.2921.47.431.7

N-pairs BLEU Analysis

This analysis randomly samples 1000 pairs of generations/ground truths and treats them as translations, giving their BLEU score. We can expect very low scores in the ground truth and high scores can expose hyper-common generations





In [14]:



    


# Training Set BLEU n-pairs Scores
print_bleu([(rname, report['n_pairs_bleu_train']) for (rname, report) in reports])



    


















    







ModelOverall Score1-gram Score2-gram Score3-gram Score4-gram Score
encdec_noing6_200_512_04drb29.5141.228.625.625.1
encdec_noing10_200_512_04drb35.3946.134.831.631
encdec_noing15_200_512_04drb37.6748.739.13431.1
encdec_noing23_200_512_04drb13.5933.514.897.6

















In [15]:



    


# Validation Set n-pairs BLEU Scores
print_bleu([(rname, report['n_pairs_bleu_valid']) for (rname, report) in reports])



    


















    







ModelOverall Score1-gram Score2-gram Score3-gram Score4-gram Score
encdec_noing6_200_512_04drb100100100100100
encdec_noing10_200_512_04drb100100100100100
encdec_noing15_200_512_04drb100100100100100
encdec_noing23_200_512_04drb100100100100100

















In [16]:



    


# Test Set n-pairs BLEU Scores
print_bleu([(rname, report['n_pairs_bleu_test']) for (rname, report) in reports])



    


















    







ModelOverall Score1-gram Score2-gram Score3-gram Score4-gram Score
encdec_noing6_200_512_04drb25.738.424.521.821.3
encdec_noing10_200_512_04drb43.1152.344.340.337
encdec_noing15_200_512_04drb28.6340.330.22522.1
encdec_noing23_200_512_04drb31.6543.731.827.925.9

















In [17]:



    


# Combined n-pairs BLEU Scores
print_bleu([(rname, report['n_pairs_bleu_all']) for (rname, report) in reports])



    


















    







ModelOverall Score1-gram Score2-gram Score3-gram Score4-gram Score
encdec_noing6_200_512_04drb43.9452.642.840.840.6
encdec_noing10_200_512_04drb51.9759.951.749.347.8
encdec_noing15_200_512_04drb51.2459.352.248.446
encdec_noing23_200_512_04drb30.54530.425.824.5

















In [18]:



    


# Ground Truth n-pairs BLEU Scores
print_bleu([(rname, report['n_pairs_bleu_gold']) for (rname, report) in reports])



    


















    







ModelOverall Score1-gram Score2-gram Score3-gram Score4-gram Score
encdec_noing6_200_512_04drb12.2225.413.99.26.8
encdec_noing10_200_512_04drb7.5925.7115.32.2
encdec_noing15_200_512_04drb8.8126.39.65.94.1
encdec_noing23_200_512_04drb11.9429.314.18.75.6

Alignment Analysis

This analysis computs the average Smith-Waterman alignment score for generations, with the same intuition as N-pairs BLEU, in that we expect low scores in the ground truth and hyper-common generations to raise the scores





In [19]:



    


def print_align(reports):
    data= [['<b>Model</b>', '<b>Average (Train) Generated Score</b>','<b>Average (Valid) Generated Score</b>','<b>Average (Test) Generated Score</b>','<b>Average (All) Generated Score</b>', '<b>Average (Gold) Score</b>']]
    for rname, report in reports:
        data.append([rname, report['average_alignment_train'], report['average_alignment_valid'], report['average_alignment_test'], report['average_alignment_all'], report['average_alignment_gold']])
    display_table(data)

print_align(reports)



    


















    







ModelAverage (Train) Generated ScoreAverage (Valid) Generated ScoreAverage (Test) Generated ScoreAverage (All) Generated ScoreAverage (Gold) Score
encdec_noing6_200_512_04drb43.714285714313336.190476190558.426.080952381
encdec_noing10_200_512_04drb42.38095238111254.476190476257.895238095220.2
encdec_noing15_200_512_04drb51.38095238112240.714285714365.21904761919.519047619
encdec_noing23_200_512_04drb20.333333333311045.142857142937.361904761924.8380952381

Content source: kingb12/languagemodelRNN

Similar notebooks:

notebook.community | gallery | about