notebook.community

Edit and run

Comparing Encoder-Decoders Analysis

Model Architecture





In [1]:



    


report_files = ["/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing10_200_512_04dra/encdec_noing10_200_512_04dra.json", "/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing10_200_512_04drb/encdec_noing10_200_512_04drb.json", "/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing10_200_512_04drbef/encdec_noing10_200_512_04drbef.json"]
log_files = ["/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing10_200_512_04dra/encdec_noing10_200_512_04dra_logs.json", "/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing10_200_512_04drb/encdec_noing10_200_512_04drb_logs.json", "/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing10_200_512_04drbef/encdec_noing10_200_512_04drbef_logs.json"]
reports = []
logs = []
import json
import matplotlib.pyplot as plt
import numpy as np

for report_file in report_files:
    with open(report_file) as f:
        reports.append((report_file.split('/')[-1].split('.json')[0], json.loads(f.read())))
for log_file in log_files:
    with open(log_file) as f:
        logs.append((log_file.split('/')[-1].split('.json')[0], json.loads(f.read())))
        
for report_name, report in reports:
    print '\n', report_name, '\n'
    print 'Encoder: \n', report['architecture']['encoder']
    print 'Decoder: \n', report['architecture']['decoder']



    


















    






encdec_noing10_200_512_04dra 

Encoder: 
nn.Sequential {
  [input -> (1) -> (2) -> (3) -> output]
  (1): nn.LookupTable
  (2): nn.LSTM(200 -> 512)
  (3): nn.Dropout(0.400000)
}
Decoder: 
nn.gModule

encdec_noing10_200_512_04drb 

Encoder: 
nn.Sequential {
  [input -> (1) -> (2) -> (3) -> (4) -> output]
  (1): nn.LookupTable
  (2): nn.Dropout(0.400000)
  (3): nn.LSTM(200 -> 512)
  (4): nn.Dropout(0.400000)
}
Decoder: 
nn.gModule

encdec_noing10_200_512_04drbef 

Encoder: 
nn.Sequential {
  [input -> (1) -> (2) -> output]
  (1): nn.LookupTable
  (2): nn.LSTM(200 -> 512)
}
Decoder: 
nn.gModule

Perplexity on Each Dataset





In [2]:



    


%matplotlib inline
from IPython.display import HTML, display

def display_table(data):
    display(HTML(
        u'<table><tr>{}</tr></table>'.format(
            u'</tr><tr>'.join(
                u'<td>{}</td>'.format('</td><td>'.join(unicode(_) for _ in row)) for row in data)
            )
    ))

def bar_chart(data):
    n_groups = len(data)
    
    train_perps = [d[1] for d in data]
    valid_perps = [d[2] for d in data]
    test_perps = [d[3] for d in data]
    
    fig, ax = plt.subplots(figsize=(10,8))
    
    index = np.arange(n_groups)
    bar_width = 0.3

    opacity = 0.4
    error_config = {'ecolor': '0.3'}

    train_bars = plt.bar(index, train_perps, bar_width,
                     alpha=opacity,
                     color='b',
                     error_kw=error_config,
                     label='Training Perplexity')

    valid_bars = plt.bar(index + bar_width, valid_perps, bar_width,
                     alpha=opacity,
                     color='r',
                     error_kw=error_config,
                     label='Valid Perplexity')
    test_bars = plt.bar(index + 2*bar_width, test_perps, bar_width,
                     alpha=opacity,
                     color='g',
                     error_kw=error_config,
                     label='Test Perplexity')

    plt.xlabel('Model')
    plt.ylabel('Scores')
    plt.title('Perplexity by Model and Dataset')
    plt.xticks(index + bar_width / 3, [d[0] for d in data])
    plt.legend()

    plt.tight_layout()
    plt.show()

data = [['<b>Model</b>', '<b>Train Perplexity</b>', '<b>Valid Perplexity</b>', '<b>Test Perplexity</b>']]

for rname, report in reports:
    data.append([rname, report['train_perplexity'], report['valid_perplexity'], report['test_perplexity']])

display_table(data)
bar_chart(data[1:])



    


















    







ModelTrain PerplexityValid PerplexityTest Perplexity
encdec_noing10_200_512_04dra71.711428171413.590509362440.71114299
encdec_noing10_200_512_04drb1.027864116956180.96265398676.05984511
encdec_noing10_200_512_04drbef159.717921689526.567247963567.088311501

Loss vs. Epoch





In [3]:



    


%matplotlib inline
plt.figure(figsize=(10, 8))
for rname, l in logs:
    for k in l.keys():
        plt.plot(l[k][0], l[k][1], label=str(k) + ' ' + rname + ' (train)')
        plt.plot(l[k][0], l[k][2], label=str(k) + ' ' + rname + ' (valid)')
plt.title('Loss v. Epoch')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()
plt.show()

Perplexity vs. Epoch





In [4]:



    


%matplotlib inline
plt.figure(figsize=(10, 8))
for rname, l in logs:
    for k in l.keys():
        plt.plot(l[k][0], l[k][3], label=str(k) + ' ' + rname + ' (train)')
        plt.plot(l[k][0], l[k][4], label=str(k) + ' ' + rname + ' (valid)')
plt.title('Perplexity v. Epoch')
plt.xlabel('Epoch')
plt.ylabel('Perplexity')
plt.legend()
plt.show()

Generations





In [5]:



    


def print_sample(sample, best_bleu=None):
    enc_input = ' '.join([w for w in sample['encoder_input'].split(' ') if w != '<pad>'])
    gold = ' '.join([w for w in sample['gold'].split(' ') if w != '<mask>'])
    print('Input: '+ enc_input + '\n')
    print('Gend: ' + sample['generated'] + '\n')
    print('True: ' + gold + '\n')
    if best_bleu is not None:
        cbm = ' '.join([w for w in best_bleu['best_match'].split(' ') if w != '<mask>'])
        print('Closest BLEU Match: ' + cbm + '\n')
        print('Closest BLEU Score: ' + str(best_bleu['best_score']) + '\n')
    print('\n')
    
def display_sample(samples, best_bleu=False):
    for enc_input in samples:
        data = []
        for rname, sample in samples[enc_input]:
            gold = ' '.join([w for w in sample['gold'].split(' ') if w != '<mask>'])
            data.append([rname, '<b>Generated: </b>' + sample['generated']])
            if best_bleu:
                cbm = ' '.join([w for w in sample['best_match'].split(' ') if w != '<mask>'])
                data.append([rname, '<b>Closest BLEU Match: </b>' + cbm + ' (Score: ' + str(sample['best_score']) + ')'])
        data.insert(0, ['<u><b>' + enc_input + '</b></u>', '<b>True: ' + gold+ '</b>'])
        display_table(data)

def process_samples(samples):
    # consolidate samples with identical inputs
    result = {}
    for rname, t_samples, t_cbms in samples:
        for i, sample in enumerate(t_samples):
            enc_input = ' '.join([w for w in sample['encoder_input'].split(' ') if w != '<pad>'])
            if t_cbms is not None:
                sample.update(t_cbms[i])
            if enc_input in result:
                result[enc_input].append((rname, sample))
            else:
                result[enc_input] = [(rname, sample)]
    return result



    











In [6]:



    


samples = process_samples([(rname, r['train_samples'], r['best_bleu_matches_train'] if 'best_bleu_matches_train' in r else None) for (rname, r) in reports])
display_sample(samples, best_bleu='best_bleu_matches_train' in reports[1][1])



    


















    







 5 - minute healthy strawberry frozen yogurtTrue:  preheat oven to 350  brown beef in skillet & add taco seasoning . cook as directed on seasoning packet  spray an 
encdec_noing10_200_512_04draGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04draClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04drbClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbefGenerated:   preheat oven to to .  .  in a large bowl , and .  in a large bowl , and .  in
encdec_noing10_200_512_04drbefClosest BLEU Match:  preheat oven to 450º .  place a large baking sheet into the oven .  in a large bowl , toss together  (Score: 31.1079728148)










    







 french fry stuffed chili enchiladasTrue:  start your rice , i use a rice cooker . when nearly finished ( or actually finished ) 
encdec_noing10_200_512_04draGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04draClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04drbClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbefGenerated:   preheat oven to to .  .  in a large bowl , and .  in a large bowl , and .  in
encdec_noing10_200_512_04drbefClosest BLEU Match:  preheat oven to 450º .  place a large baking sheet into the oven .  in a large bowl , toss together  (Score: 31.1079728148)










    







 hidden valley pinwheel sandwichesTrue:  chop your green pepper , red pepper , sweet onion , and carrots up . put your carrots off to the side . 
encdec_noing10_200_512_04draGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04draClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04draGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04draClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04drbClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04drbClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbefGenerated:   preheat oven to to .  .  in a large bowl , and .  in a large bowl , and .  in
encdec_noing10_200_512_04drbefClosest BLEU Match:  preheat oven to 450º .  place a large baking sheet into the oven .  in a large bowl , toss together  (Score: 31.1079728148)
encdec_noing10_200_512_04drbefGenerated:   preheat oven to to .  .  in a large bowl , and .  in a large bowl , and .  in
encdec_noing10_200_512_04drbefClosest BLEU Match:  preheat oven to 450º .  place a large baking sheet into the oven .  in a large bowl , toss together  (Score: 31.1079728148)










    







 savory french omeletTrue:  combine the cornmeal , flour , sugar , mustard , baking powder and salt , mixing well .  add the milk , egg 
encdec_noing10_200_512_04draGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04draClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04drbClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbefGenerated:   preheat oven to to .  .  in a large bowl , and .  in a large bowl , and .  in
encdec_noing10_200_512_04drbefClosest BLEU Match:  preheat oven to 450º .  place a large baking sheet into the oven .  in a large bowl , toss together  (Score: 31.1079728148)










    







 pizza zacineti breadsticksTrue:  using a mixer with a paddle , cream together the butter , sugar and eggs until smooth . add the vanilla and zest and combine
encdec_noing10_200_512_04draGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04draClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04drbClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbefGenerated:   preheat oven to to .  .  in a large bowl , and .  in a large bowl , and .  in
encdec_noing10_200_512_04drbefClosest BLEU Match:  preheat oven to 450º .  place a large baking sheet into the oven .  in a large bowl , toss together  (Score: 31.1079728148)










    







 coffee cake in a mug with cinnamon oatmeal struesel toppingTrue:  to make the apple - cranberry relish : peel , core , and chop the apples into 1 / 4 - inch 
encdec_noing10_200_512_04draGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04draClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04drbClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbefGenerated:   preheat oven to to .  .  in a large bowl , and .  in a large bowl , and .  in
encdec_noing10_200_512_04drbefClosest BLEU Match:  preheat oven to 450º .  place a large baking sheet into the oven .  in a large bowl , toss together  (Score: 31.1079728148)

















In [7]:



    


samples = process_samples([(rname, r['valid_samples'], r['best_bleu_matches_valid'] if 'best_bleu_matches_valid' in r else None) for (rname, r) in reports])
display_sample(samples, best_bleu='best_bleu_matches_valid' in reports[1][1])



    


















    







 5 - minute healthy strawberry frozen yogurtTrue:  add the frozen strawberries , agave nectar ( or honey ) , yogurt and lemon juice to the 
encdec_noing10_200_512_04draGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04draClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04drbClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbefGenerated:   preheat oven to to .  .  in a large bowl , and .  in a large bowl , and .  in
encdec_noing10_200_512_04drbefClosest BLEU Match:  preheat oven to 450º .  place a large baking sheet into the oven .  in a large bowl , toss together  (Score: 31.1079728148)










    







 french fry stuffed chili enchiladasTrue:  preheat oven and bake french fries as directed on bag .  take your flour tortillas and heat 
encdec_noing10_200_512_04draGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04draClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04drbClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbefGenerated:   preheat oven to to .  .  in a large bowl , and .  in a large bowl , and .  in
encdec_noing10_200_512_04drbefClosest BLEU Match:  preheat oven to 450º .  place a large baking sheet into the oven .  in a large bowl , toss together  (Score: 31.1079728148)










    







 hidden valley pinwheel sandwichesTrue:  mix cream cheese , dressing mix and onions until blended . spread on tortillas . blot 
encdec_noing10_200_512_04draGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04draClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04draGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04draClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04drbClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04drbClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbefGenerated:   preheat oven to to .  .  in a large bowl , and .  in a large bowl , and .  in
encdec_noing10_200_512_04drbefClosest BLEU Match:  preheat oven to 450º .  place a large baking sheet into the oven .  in a large bowl , toss together  (Score: 31.1079728148)
encdec_noing10_200_512_04drbefGenerated:   preheat oven to to .  .  in a large bowl , and .  in a large bowl , and .  in
encdec_noing10_200_512_04drbefClosest BLEU Match:  preheat oven to 450º .  place a large baking sheet into the oven .  in a large bowl , toss together  (Score: 31.1079728148)










    







 savory french omeletTrue:  in a large skillet , fry bacon until crisp . crumble cooked bacon ; .  
encdec_noing10_200_512_04draGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04draClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04drbClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbefGenerated:   preheat oven to to .  .  in a large bowl , and .  in a large bowl , and .  in
encdec_noing10_200_512_04drbefClosest BLEU Match:  preheat oven to 450º .  place a large baking sheet into the oven .  in a large bowl , toss together  (Score: 31.1079728148)










    







 pizza zacineti breadsticksTrue:  in a bowl , dissolve the yeast with lukewarm and sugar for about 5 minutes , then add flour , salt and 
encdec_noing10_200_512_04draGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04draClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04drbClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbefGenerated:   preheat oven to to .  .  in a large bowl , and .  in a large bowl , and .  in
encdec_noing10_200_512_04drbefClosest BLEU Match:  preheat oven to 450º .  place a large baking sheet into the oven .  in a large bowl , toss together  (Score: 31.1079728148)










    







 coffee cake in a mug with cinnamon oatmeal struesel toppingTrue:  place 1 tablespoon of room temperature butter in mug . if cold , place in microwave 
encdec_noing10_200_512_04draGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04draClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04drbClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbefGenerated:   preheat oven to to .  .  in a large bowl , and .  in a large bowl , and .  in
encdec_noing10_200_512_04drbefClosest BLEU Match:  preheat oven to 450º .  place a large baking sheet into the oven .  in a large bowl , toss together  (Score: 31.1079728148)

















In [8]:



    


samples = process_samples([(rname, r['test_samples'], r['best_bleu_matches_test'] if 'best_bleu_matches_test' in r else None) for (rname, r) in reports])
display_sample(samples, best_bleu='best_bleu_matches_test' in reports[1][1])



    


















    







 fancy hot dogsTrue:   1 melt butter in a large skillet ( cast iron works well for this purpose 
encdec_noing10_200_512_04draGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04draClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04drbClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbefGenerated:   preheat oven to to .  .  in a large bowl , and .  in a large bowl , and .  in
encdec_noing10_200_512_04drbefClosest BLEU Match:  preheat oven to 450º .  place a large baking sheet into the oven .  in a large bowl , toss together  (Score: 31.1079728148)










    







 smoked salmon , avocado , dill and parsley mayo sandwichTrue:  mash the lemon juice and avocado with dill and parsley mayo . spoon over a slice of bread and 
encdec_noing10_200_512_04draGenerated:   preheat oven to a . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04draClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbGenerated:   . a . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04drbClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbefGenerated:   preheat oven to to 350 degrees .  in a large bowl , and .  in a large bowl , and .  in
encdec_noing10_200_512_04drbefClosest BLEU Match:  preheat oven to 350 degrees .  in a large bowl , mix together the flour , baking soda , and salt .  (Score: 42.1638398456)










    







 marty 's loosemeat sandwichTrue:  in a medium skillet over medium heat , cook the ground beef until evenly browned ; drain 
encdec_noing10_200_512_04draGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04draClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04drbClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbefGenerated:   preheat oven to to .  .  in a large bowl , and .  in a large bowl , and .  in
encdec_noing10_200_512_04drbefClosest BLEU Match:  preheat oven to 450º .  place a large baking sheet into the oven .  in a large bowl , toss together  (Score: 31.1079728148)










    







 saute ? ed mushroomsTrue:  1 . cook shiitake mushrooms in a single layer in 1 1 / 2 tbsp . hot oil in a 10 - to 
encdec_noing10_200_512_04draGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04draClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04draGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04draClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04drbClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04drbClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbefGenerated:   preheat oven to to .  .  in a large bowl , and .  in a large bowl , and .  in
encdec_noing10_200_512_04drbefClosest BLEU Match:  preheat oven to 450º .  place a large baking sheet into the oven .  in a large bowl , toss together  (Score: 31.1079728148)
encdec_noing10_200_512_04drbefGenerated:   preheat oven to to .  .  in a large bowl , and .  in a large bowl , and .  in
encdec_noing10_200_512_04drbefClosest BLEU Match:  preheat oven to 450º .  place a large baking sheet into the oven .  in a large bowl , toss together  (Score: 31.1079728148)










    







 healthy oatmeal cookiesTrue:  preheat oven to 350 degrees . in a medium bowl , whisk together flours and baking powder ; set aside .  in 
encdec_noing10_200_512_04draGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04draClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04drbClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbefGenerated:   preheat oven to to .  .  in a large bowl , and .  in a large bowl , and .  in
encdec_noing10_200_512_04drbefClosest BLEU Match:  preheat oven to 450º .  place a large baking sheet into the oven .  in a large bowl , toss together  (Score: 31.1079728148)










    







 mexican hummusTrue:  position the knife blade in a food processor bowl . drop the garlic through the food chute with the processor running ; process 3 seconds
encdec_noing10_200_512_04draGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04draClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbGenerated:   . . . . . . . . . . . . . . . . . . . . . . . . .
encdec_noing10_200_512_04drbClosest BLEU Match:  heat the oil in a skillet over medium heat .  in a bowl , combine the coconut flour ,  (Score: 0)
encdec_noing10_200_512_04drbefGenerated:   preheat oven to to .  .  in a large bowl , and .  in a large bowl , and .  in
encdec_noing10_200_512_04drbefClosest BLEU Match:  preheat oven to 450º .  place a large baking sheet into the oven .  in a large bowl , toss together  (Score: 31.1079728148)

BLEU Analysis





In [9]:



    


def print_bleu(blue_structs):
    data= [['<b>Model</b>', '<b>Overall Score</b>','<b>1-gram Score</b>','<b>2-gram Score</b>','<b>3-gram Score</b>','<b>4-gram Score</b>']]
    for rname, blue_struct in blue_structs:
        data.append([rname, blue_struct['score'], blue_struct['components']['1'], blue_struct['components']['2'], blue_struct['components']['3'], blue_struct['components']['4']])
    display_table(data)



    











In [10]:



    


# Training Set BLEU Scores
print_bleu([(rname, report['train_bleu']) for (rname, report) in reports])



    


















    







ModelOverall Score1-gram Score2-gram Score3-gram Score4-gram Score
encdec_noing10_200_512_04dra04.4000
encdec_noing10_200_512_04drb04.4000
encdec_noing10_200_512_04drbef020.93.40.60

















In [11]:



    


# Validation Set BLEU Scores
print_bleu([(rname, report['valid_bleu']) for (rname, report) in reports])



    


















    







ModelOverall Score1-gram Score2-gram Score3-gram Score4-gram Score
encdec_noing10_200_512_04dra04.4000
encdec_noing10_200_512_04drb04.4000
encdec_noing10_200_512_04drbef019.840.60

















In [12]:



    


# Test Set BLEU Scores
print_bleu([(rname, report['test_bleu']) for (rname, report) in reports])



    


















    







ModelOverall Score1-gram Score2-gram Score3-gram Score4-gram Score
encdec_noing10_200_512_04dra04.9000
encdec_noing10_200_512_04drb04.9000
encdec_noing10_200_512_04drbef023.681.80

















In [13]:



    


# All Data BLEU Scores
print_bleu([(rname, report['combined_bleu']) for (rname, report) in reports])



    


















    







ModelOverall Score1-gram Score2-gram Score3-gram Score4-gram Score
encdec_noing10_200_512_04dra04.6000
encdec_noing10_200_512_04drb04.6000
encdec_noing10_200_512_04drbef021.45.110

N-pairs BLEU Analysis

This analysis randomly samples 1000 pairs of generations/ground truths and treats them as translations, giving their BLEU score. We can expect very low scores in the ground truth and high scores can expose hyper-common generations





In [14]:



    


# Training Set BLEU n-pairs Scores
print_bleu([(rname, report['n_pairs_bleu_train']) for (rname, report) in reports])



    


















    







ModelOverall Score1-gram Score2-gram Score3-gram Score4-gram Score
encdec_noing10_200_512_04dra100100100100100
encdec_noing10_200_512_04drb100100100100100
encdec_noing10_200_512_04drbef100100100100100

















In [15]:



    


# Validation Set n-pairs BLEU Scores
print_bleu([(rname, report['n_pairs_bleu_valid']) for (rname, report) in reports])



    


















    







ModelOverall Score1-gram Score2-gram Score3-gram Score4-gram Score
encdec_noing10_200_512_04dra100100100100100
encdec_noing10_200_512_04drb100100100100100
encdec_noing10_200_512_04drbef100100100100100

















In [16]:



    


# Test Set n-pairs BLEU Scores
print_bleu([(rname, report['n_pairs_bleu_test']) for (rname, report) in reports])



    


















    







ModelOverall Score1-gram Score2-gram Score3-gram Score4-gram Score
encdec_noing10_200_512_04dra94.6695.894.594.394
encdec_noing10_200_512_04drb97.1898.897.696.296.1
encdec_noing10_200_512_04drbef95.8397.896.695.293.8

















In [17]:



    


# Combined n-pairs BLEU Scores
print_bleu([(rname, report['n_pairs_bleu_all']) for (rname, report) in reports])



    


















    







ModelOverall Score1-gram Score2-gram Score3-gram Score4-gram Score
encdec_noing10_200_512_04dra98.1998.698.198.198
encdec_noing10_200_512_04drb99.299.799.398.998.9
encdec_noing10_200_512_04drbef98.7199.398.998.598.1

















In [18]:



    


# Ground Truth n-pairs BLEU Scores
print_bleu([(rname, report['n_pairs_bleu_gold']) for (rname, report) in reports])



    


















    







ModelOverall Score1-gram Score2-gram Score3-gram Score4-gram Score
encdec_noing10_200_512_04dra9.8924.810.66.75.5
encdec_noing10_200_512_04drb9.5123.910.36.45.2
encdec_noing10_200_512_04drbef10.925.411.47.76.4

Alignment Analysis

This analysis computs the average Smith-Waterman alignment score for generations, with the same intuition as N-pairs BLEU, in that we expect low scores in the ground truth and hyper-common generations to raise the scores





In [19]:



    


def print_align(reports):
    data= [['<b>Model</b>', '<b>Average (Train) Generated Score</b>','<b>Average (Valid) Generated Score</b>','<b>Average (Test) Generated Score</b>','<b>Average (All) Generated Score</b>', '<b>Average (Gold) Score</b>']]
    for rname, report in reports:
        data.append([rname, report['average_alignment_train'], report['average_alignment_valid'], report['average_alignment_test'], report['average_alignment_all'], report['average_alignment_gold']])
    display_table(data)

print_align(reports)



    


















    







ModelAverage (Train) Generated ScoreAverage (Valid) Generated ScoreAverage (Test) Generated ScoreAverage (All) Generated ScoreAverage (Gold) Score
encdec_noing10_200_512_04dra56565254.666666666721.5428571429
encdec_noing10_200_512_04drb565655.428571428655.809523809521.5428571429
encdec_noing10_200_512_04drbef108108103.714285714106.57142857121.5428571429

Content source: kingb12/languagemodelRNN

Similar notebooks:

notebook.community | gallery | about