Comparing Encoder-Decoders Analysis

In [39]:

    

report_files = ["/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing15_200_512_04drb/encdec_noing15_200_512_04drb.json", "/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing15_bow_200_512_04drb/encdec_noing15_bow_200_512_04drb.json"]
log_files = ["/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing15_200_512_04drb/encdec_noing15_200_512_04drb_logs.json", "/Users/bking/IdeaProjects/LanguageModelRNN/experiment_results/encdec_noing15_bow_200_512_04drb/encdec_noing15_bow_200_512_04drb_logs.json"]
reports = []
logs = []
import json
import matplotlib.pyplot as plt
import numpy as np

for report_file in report_files:
    with open(report_file) as f:
        reports.append((report_file.split('/')[-1].split('.json')[0], json.loads(f.read())))
for log_file in log_files:
    with open(log_file) as f:
        logs.append((log_file.split('/')[-1].split('.json')[0], json.loads(f.read())))
        
for report_name, report in reports:
    print '\n', report_name, '\n'
    print 'Encoder: \n', report['architecture']['encoder']
    print 'Decoder: \n', report['architecture']['decoder']


    

    




encdec_noing15_200_512_04drb 

Encoder: 
nn.Sequential {
  [input -> (1) -> (2) -> (3) -> (4) -> output]
  (1): nn.LookupTable
  (2): nn.Dropout(0.400000)
  (3): nn.LSTM(200 -> 512)
  (4): nn.Dropout(0.400000)
}
Decoder: 
nn.gModule

encdec_noing15_bow_200_512_04drb 

Encoder: 
nn.Sequential {
  [input -> (1) -> (2) -> (3) -> (4) -> output]
  (1): nn.LookupTable
  (2): nn.Mean
  (3): nn.Linear(200 -> 512)
  (4): nn.Replicate
}
Decoder: 
nn.gModule

In [40]:

    

%matplotlib inline
from IPython.display import HTML, display

def display_table(data):
    display(HTML(
        u'<table><tr>{}</tr></table>'.format(
            u'</tr><tr>'.join(
                u'<td>{}</td>'.format('</td><td>'.join(unicode(_) for _ in row)) for row in data)
            )
    ))

def bar_chart(data):
    n_groups = len(data)
    
    train_perps = [d[1] for d in data]
    valid_perps = [d[2] for d in data]
    test_perps = [d[3] for d in data]
    
    fig, ax = plt.subplots(figsize=(10,8))
    
    index = np.arange(n_groups)
    bar_width = 0.3

    opacity = 0.4
    error_config = {'ecolor': '0.3'}

    train_bars = plt.bar(index, train_perps, bar_width,
                     alpha=opacity,
                     color='b',
                     error_kw=error_config,
                     label='Training Perplexity')

    valid_bars = plt.bar(index + bar_width, valid_perps, bar_width,
                     alpha=opacity,
                     color='r',
                     error_kw=error_config,
                     label='Valid Perplexity')
    test_bars = plt.bar(index + 2*bar_width, test_perps, bar_width,
                     alpha=opacity,
                     color='g',
                     error_kw=error_config,
                     label='Test Perplexity')

    plt.xlabel('Model')
    plt.ylabel('Scores')
    plt.title('Perplexity by Model and Dataset')
    plt.xticks(index + bar_width / 3, [d[0] for d in data])
    plt.legend()

    plt.tight_layout()
    plt.show()

data = [['<b>Model</b>', '<b>Train Perplexity</b>', '<b>Valid Perplexity</b>', '<b>Test Perplexity</b>']]

for rname, report in reports:
    data.append([rname, report['train_perplexity'], report['valid_perplexity'], report['test_perplexity']])

display_table(data)
bar_chart(data[1:])


    

    





Model
Train Perplexity
Valid Perplexity
Test Perplexity
encdec_noing15_200_512_04drb
2.27752305377
706.426656909
805.596650048
encdec_noing15_bow_200_512_04drb
1.8018126954
909.033889454
1485.06427136






    

In [41]:

    

%matplotlib inline
plt.figure(figsize=(10, 8))
for rname, l in logs:
    for k in l.keys():
        plt.plot(l[k][0], l[k][1], label=str(k) + ' ' + rname + ' (train)')
        plt.plot(l[k][0], l[k][2], label=str(k) + ' ' + rname + ' (valid)')
plt.title('Loss v. Epoch')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()
plt.show()


    

In [42]:

    

%matplotlib inline
plt.figure(figsize=(10, 8))
for rname, l in logs:
    for k in l.keys():
        plt.plot(l[k][0], l[k][3], label=str(k) + ' ' + rname + ' (train)')
        plt.plot(l[k][0], l[k][4], label=str(k) + ' ' + rname + ' (valid)')
plt.title('Perplexity v. Epoch')
plt.xlabel('Epoch')
plt.ylabel('Perplexity')
plt.legend()
plt.show()


    

In [43]:

    

def print_sample(sample, best_bleu=None):
    enc_input = ' '.join([w for w in sample['encoder_input'].split(' ') if w != '<pad>'])
    gold = ' '.join([w for w in sample['gold'].split(' ') if w != '<mask>'])
    print('Input: '+ enc_input + '\n')
    print('Gend: ' + sample['generated'] + '\n')
    print('True: ' + gold + '\n')
    if best_bleu is not None:
        cbm = ' '.join([w for w in best_bleu['best_match'].split(' ') if w != '<mask>'])
        print('Closest BLEU Match: ' + cbm + '\n')
        print('Closest BLEU Score: ' + str(best_bleu['best_score']) + '\n')
    print('\n')
    
def display_sample(samples, best_bleu=False):
    for enc_input in samples:
        data = []
        for rname, sample in samples[enc_input]:
            gold = ' '.join([w for w in sample['gold'].split(' ') if w != '<mask>'])
            data.append([rname, '<b>Generated: </b>' + sample['generated']])
            if best_bleu:
                cbm = ' '.join([w for w in sample['best_match'].split(' ') if w != '<mask>'])
                data.append([rname, '<b>Closest BLEU Match: </b>' + cbm + ' (Score: ' + str(sample['best_score']) + ')'])
        data.insert(0, ['<u><b>' + enc_input + '</b></u>', '<b>True: ' + gold+ '</b>'])
        display_table(data)

def process_samples(samples):
    # consolidate samples with identical inputs
    result = {}
    for rname, t_samples, t_cbms in samples:
        for i, sample in enumerate(t_samples):
            enc_input = ' '.join([w for w in sample['encoder_input'].split(' ') if w != '<pad>'])
            if t_cbms is not None:
                sample.update(t_cbms[i])
            if enc_input in result:
                result[enc_input].append((rname, sample))
            else:
                result[enc_input] = [(rname, sample)]
    return result


    

In [44]:

    

samples = process_samples([(rname, r['train_samples'], r['best_bleu_matches_train'] if 'best_bleu_matches_train' in r else None) for (rname, r) in reports])
display_sample(samples, best_bleu='best_bleu_matches_train' in reports[1][1])


    

    





 pumpkin ravioli
True:  heat oil in a soup pot or dutch oven over moderate heat until hot . add onions and fennel and cook until tender 
encdec_noing15_bow_200_512_04drb
Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
encdec_noing15_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 87.7277201636)






    





 hamburger muffins
True:  stir together instant coffee , sugar and water . set aside .  in a large mixer bowl , beat cream cheese until light 
encdec_noing15_200_512_04drb
Generated:   in a large bowl , combine the ice , garlic , and cashews in a bowl , combine the chips .  add the milk
encdec_noing15_200_512_04drb
Closest BLEU Match:  in a large bowl , combine the cottage cheese , sour cream , egg , salt , garlic salt and pepper .  add cheddar (Score: 30.8264237423)






    





 egg sandwiches with wilted spinach
True:  1 . preheat oven to 350 degrees f. and spray donut pan with non - stick cooking spray . 
encdec_noing15_200_512_04drb
Generated:   1 . preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is
encdec_noing15_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 87.7277201636)






    





 mussels with thyme and shallots
True:  preparation : place the rice , chicken broth and 1 tablespoon of the olive oil in a 
encdec_noing15_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat .  add the onion
encdec_noing15_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 64.7285941823)
encdec_noing15_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat .  add the onion
encdec_noing15_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 64.7285941823)






    





 pico de gallo and guacamole
True:  instructions heat a teaspoon of coconut oil in a casserole pot or a large saucepan . add the lamb and stir it 
encdec_noing15_bow_200_512_04drb
Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
encdec_noing15_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 87.7277201636)






    





 basic guacamole
True:  lima bean  main dish  salad  
encdec_noing15_bow_200_512_04drb
Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
encdec_noing15_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 87.7277201636)






    





 seafood chowder
True:  1 ) melt butter along with olive oil in a large skillet over medium heat .  2 ) add garlic 
encdec_noing15_bow_200_512_04drb
Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
encdec_noing15_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 87.7277201636)






    





 amazing watermelon greek salad with feta
True:  combine the first 6 ingredients and shape into patties .  cook patties in a large skillet over medium - low heat 20 
encdec_noing15_bow_200_512_04drb
Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
encdec_noing15_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 87.7277201636)






    





 chocolate pancakes
True:  heat a medium sized skillet to medium heat . when hot , add olive oil and then 
encdec_noing15_200_512_04drb
Generated:   1 . preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat .  add
encdec_noing15_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 64.7285941823)
encdec_noing15_bow_200_512_04drb
Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
encdec_noing15_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 87.7277201636)






    





 lasagna
True:  whisk together flour , sugar , baking powder , kosher salt , and bacon . store mix 
encdec_noing15_bow_200_512_04drb
Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
encdec_noing15_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 87.7277201636)






    





 pizza zacineti breadsticks
True:  using a mixer with a paddle , cream together the butter , sugar and eggs until smooth . add the vanilla and zest and combine
encdec_noing15_200_512_04drb
Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
encdec_noing15_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 87.7277201636)






    





 zaru soba ( cold soba noodles )
True:  1 . preheat oven 350º f .  2 . cut one half the onion into thin slices ; place in a large bowl 
encdec_noing15_200_512_04drb
Generated:   1 . preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is
encdec_noing15_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 87.7277201636)

In [45]:

    

samples = process_samples([(rname, r['valid_samples'], r['best_bleu_matches_valid'] if 'best_bleu_matches_valid' in r else None) for (rname, r) in reports])
display_sample(samples, best_bleu='best_bleu_matches_valid' in reports[1][1])


    

    





               
True:  beat eggs and water together in a bowl .  add salt ( i do n't use salt due to the 
encdec_noing15_bow_200_512_04drb
Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
encdec_noing15_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 87.7277201636)






    





  pulled             
True:  preheat oven and bake french fries as directed on bag .  take your flour tortillas and heat 
encdec_noing15_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat .  add the onion
encdec_noing15_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 64.7285941823)
encdec_noing15_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat .  add the onion
encdec_noing15_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 64.7285941823)
encdec_noing15_bow_200_512_04drb
Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
encdec_noing15_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 87.7277201636)
encdec_noing15_bow_200_512_04drb
Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
encdec_noing15_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 87.7277201636)






    





  pulled             
True:  1 . cook chicken in a pan with a drizzle of olive oil and some s & p and 
encdec_noing15_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat .  add the onion
encdec_noing15_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 64.7285941823)






    





               
True:  in a skillet on stove top , combine onions , brown sugar , garlic powder , dry mustard , 
encdec_noing15_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat .  add the onion
encdec_noing15_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 64.7285941823)
encdec_noing15_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat .  add the onion
encdec_noing15_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 64.7285941823)
encdec_noing15_bow_200_512_04drb
Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
encdec_noing15_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 87.7277201636)
encdec_noing15_bow_200_512_04drb
Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
encdec_noing15_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 87.7277201636)
encdec_noing15_bow_200_512_04drb
Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
encdec_noing15_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 87.7277201636)
encdec_noing15_bow_200_512_04drb
Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
encdec_noing15_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 87.7277201636)






    





               
True:  in a large food processor , crush oreo cookies into fine crumbs . make sure there are no large chunks 
encdec_noing15_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat .  add the onion
encdec_noing15_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 64.7285941823)
encdec_noing15_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat .  add the onion
encdec_noing15_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 64.7285941823)

In [46]:

    

samples = process_samples([(rname, r['test_samples'], r['best_bleu_matches_test'] if 'best_bleu_matches_test' in r else None) for (rname, r) in reports])
display_sample(samples, best_bleu='best_bleu_matches_test' in reports[1][1])


    

    





 fancy hot dogs
True:   1 melt butter in a large skillet ( cast iron works well for this purpose 
encdec_noing15_bow_200_512_04drb
Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
encdec_noing15_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 87.7277201636)






    





 greek pizza
True:  1 . preheat a pizza stone in a 500 degree oven for at least 30 minutes . place a piece of parchment paper on a
encdec_noing15_200_512_04drb
Generated:   combine the cornmeal , flour , sugar , baking powder , salt , and salt .  add the milk , egg , and vanilla
encdec_noing15_200_512_04drb
Closest BLEU Match:  combine the cornmeal , flour , sugar , mustard , baking powder and salt , mixing well .  add the milk , egg  (Score: 57.2545340067)






    





 grilled goat cheese and broccoli sandwich
True:  steam broccoli for 4 minutes , until tender but bright . rinse briefly with cold water and drain 
encdec_noing15_bow_200_512_04drb
Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
encdec_noing15_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 87.7277201636)






    





 skillet macaroni and cheese
True:  1 bring a pot of salty water to boil and cook the pasta until nearing al dente , but not full cooked . drain 
encdec_noing15_200_512_04drb
Generated:   1 . preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is
encdec_noing15_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 87.7277201636)






    





 steak and cheese sandwich
True:  heat small amount of oil to medium heat in skillet and add veggies in following order : peppers , mushrooms , onions and 
encdec_noing15_200_512_04drb
Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
encdec_noing15_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 87.7277201636)






    





 no - bake strawberry cheesecake
True:  1 . beat cream cheese and sugar until smooth . fold in whipped topping . spoon mixture into 
encdec_noing15_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat .  add the onion
encdec_noing15_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 64.7285941823)






    





 roasted pork cuban sandwich
True:  mix the roast seasoning blend together and set aside .  rinse roast , pat dry , coat 
encdec_noing15_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat .  add the onion
encdec_noing15_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 64.7285941823)






    





 mom 's mac & cheese
True:  preheat oven to 375 & deg ;  combine all ingredients except 1 / 2 cup cheese in large bowl . season , if desired
encdec_noing15_bow_200_512_04drb
Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
encdec_noing15_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 87.7277201636)






    





 baked sandwiches
True:  on a greased surface , roll out bread dough into a rectangle . stir together cooked meat , 
encdec_noing15_200_512_04drb
Generated:   preheat oven to 350 degrees f ( 175 degrees c ) .  in a large skillet over medium heat .  add the onion
encdec_noing15_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees f ( 175 degrees c ) .  in a large pot over medium heat cook and stir  (Score: 64.7285941823)






    





 applebee 's bourbon street steak
True:  combine bourbon , brown sugar , salt , and pepper over steaks .  preheat grill .  place steaks on grill and cook 
encdec_noing15_200_512_04drb
Generated:   in a large bowl , combine the ice , garlic , and cashews in a bowl , combine the chips .  add the milk
encdec_noing15_200_512_04drb
Closest BLEU Match:  in a large bowl , combine the cottage cheese , sour cream , egg , salt , garlic salt and pepper .  add cheddar (Score: 30.8264237423)






    





 thai peanut noodle salad
True:  bring the water to a boil in a sauce pot for the soba noodles . while the water is heating , stir the first 
encdec_noing15_bow_200_512_04drb
Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
encdec_noing15_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 87.7277201636)






    





 janette 's oven - baked french fries with homemade ketchup
True:  beef  lunch  fries 
encdec_noing15_bow_200_512_04drb
Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
encdec_noing15_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 87.7277201636)






    





 no - onions aloo curry – ( potato curry )
True:  1 . heat oil in the non - stick at high , as soon as the oil is heated 
encdec_noing15_bow_200_512_04drb
Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
encdec_noing15_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 87.7277201636)






    





 slow cooker charro beans
True:  place beans in a colander , rinse well , and remove any stones or shriveled beans 
encdec_noing15_bow_200_512_04drb
Generated:   preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is is is
encdec_noing15_bow_200_512_04drb
Closest BLEU Match:  preheat oven to 350 degrees .  cut up waffles into bite size pieces . think of them as croutons and that is  (Score: 87.7277201636)

In [47]:

    

def print_bleu(blue_structs):
    data= [['<b>Model</b>', '<b>Overall Score</b>','<b>1-gram Score</b>','<b>2-gram Score</b>','<b>3-gram Score</b>','<b>4-gram Score</b>']]
    for rname, blue_struct in blue_structs:
        data.append([rname, blue_struct['score'], blue_struct['components']['1'], blue_struct['components']['2'], blue_struct['components']['3'], blue_struct['components']['4']])
    display_table(data)


    

In [48]:

    

# Training Set BLEU Scores
print_bleu([(rname, report['train_bleu']) for (rname, report) in reports])


    

    





Model
Overall Score
1-gram Score
2-gram Score
3-gram Score
4-gram Score
encdec_noing15_200_512_04drb
7.93
23.1
10.3
5.4
3.1
encdec_noing15_bow_200_512_04drb
0
9.9
1.1
0
0

In [49]:

    

# Validation Set BLEU Scores
print_bleu([(rname, report['valid_bleu']) for (rname, report) in reports])


    

    





Model
Overall Score
1-gram Score
2-gram Score
3-gram Score
4-gram Score
encdec_noing15_200_512_04drb
0
17.6
7.4
1.8
0
encdec_noing15_bow_200_512_04drb
0
12.1
2.3
0
0

In [50]:

    

# Test Set BLEU Scores
print_bleu([(rname, report['test_bleu']) for (rname, report) in reports])


    

    





Model
Overall Score
1-gram Score
2-gram Score
3-gram Score
4-gram Score
encdec_noing15_200_512_04drb
0
13.2
1.7
0
0
encdec_noing15_bow_200_512_04drb
0
8.8
1.1
0.6
0

In [51]:

    

# All Data BLEU Scores
print_bleu([(rname, report['combined_bleu']) for (rname, report) in reports])


    

    





Model
Overall Score
1-gram Score
2-gram Score
3-gram Score
4-gram Score
encdec_noing15_200_512_04drb
4.11
17.9
6.5
2.4
1
encdec_noing15_bow_200_512_04drb
0
10.3
1.5
0.2
0

In [52]:

    

# Training Set BLEU n-pairs Scores
print_bleu([(rname, report['n_pairs_bleu_train']) for (rname, report) in reports])


    

    





Model
Overall Score
1-gram Score
2-gram Score
3-gram Score
4-gram Score
encdec_noing15_200_512_04drb
37.67
48.7
39.1
34
31.1
encdec_noing15_bow_200_512_04drb
100
100
100
100
100

In [53]:

    

# Validation Set n-pairs BLEU Scores
print_bleu([(rname, report['n_pairs_bleu_valid']) for (rname, report) in reports])


    

    





Model
Overall Score
1-gram Score
2-gram Score
3-gram Score
4-gram Score
encdec_noing15_200_512_04drb
100
100
100
100
100
encdec_noing15_bow_200_512_04drb
100
100
100
100
100

In [54]:

    

# Test Set n-pairs BLEU Scores
print_bleu([(rname, report['n_pairs_bleu_test']) for (rname, report) in reports])


    

    





Model
Overall Score
1-gram Score
2-gram Score
3-gram Score
4-gram Score
encdec_noing15_200_512_04drb
28.63
40.3
30.2
25
22.1
encdec_noing15_bow_200_512_04drb
100
100
100
100
100

In [55]:

    

# Combined n-pairs BLEU Scores
print_bleu([(rname, report['n_pairs_bleu_all']) for (rname, report) in reports])


    

    





Model
Overall Score
1-gram Score
2-gram Score
3-gram Score
4-gram Score
encdec_noing15_200_512_04drb
51.24
59.3
52.2
48.4
46
encdec_noing15_bow_200_512_04drb
100
100
100
100
100

In [56]:

    

# Ground Truth n-pairs BLEU Scores
print_bleu([(rname, report['n_pairs_bleu_gold']) for (rname, report) in reports])


    

    





Model
Overall Score
1-gram Score
2-gram Score
3-gram Score
4-gram Score
encdec_noing15_200_512_04drb
8.81
26.3
9.6
5.9
4.1
encdec_noing15_bow_200_512_04drb
10.97
25.1
12.5
8.1
5.7

In [57]:

    

def print_align(reports):
    data= [['<b>Model</b>', '<b>Average (Train) Generated Score</b>','<b>Average (Valid) Generated Score</b>','<b>Average (Test) Generated Score</b>','<b>Average (All) Generated Score</b>', '<b>Average (Gold) Score</b>']]
    for rname, report in reports:
        data.append([rname, report['average_alignment_train'], report['average_alignment_valid'], report['average_alignment_test'], report['average_alignment_all'], report['average_alignment_gold']])
    display_table(data)

print_align(reports)


    

    





Model
Average (Train) Generated Score
Average (Valid) Generated Score
Average (Test) Generated Score
Average (All) Generated Score
Average (Gold) Score
encdec_noing15_200_512_04drb
51.380952381
122
40.7142857143
65.219047619
19.519047619
encdec_noing15_bow_200_512_04drb
126
126
126
126
24.4761904762