In [38]:
import os,sys,inspect
currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(currentdir)
sys.path.insert(0,parentdir) 

from importlib import reload 
import torch
import fasttext
from ptb import lang_util
import torch
from torch.nn import CrossEntropyLoss, NLLLoss
import numpy as np
import matplotlib.pyplot as plt
import numpy as np

In [2]:
NTA = '/Users/jgordon/nta'

In [3]:
corpus = lang_util.Corpus(NTA + '/datasets/PTB')

Maybe load KN5 model


In [4]:
kn5 = np.load(NTA + '/datasets/PTB/KN5/kn5D_predictions_82430-089-w.npy')

In [19]:
word2id = {}
remap = np.zeros(10000, dtype=int)
with open(NTA + '/datasets/PTB/KN5/ptb_dense_inc_mar8.txt') as f:
    i = 0
    line = f.readline()
    line = f.readline()  # Skip header
    while line:
        token = line.split()[0]
        if token == '<end>':
            token = '</s>'
        word2id[token] = i
        remap[corpus.dictionary.word2idx[token]] = i
        i += 1
        line = f.readline()

In [31]:
kn5_tens = torch.tensor(kn5)
distrs = (kn5_tens / kn5_tens.sum(1, keepdim=True)).float()
kn5_remapped = distrs[:, torch.LongTensor(remap)]
torch.save(kn5_remapped, NTA + '/datasets/PTB/KN5/kn5_distr_remapped.pt')

In [10]:
# criterion = CrossEntropyLoss(reduction='mean')
criterion = NLLLoss(reduction='mean')
   
target = []
for id in corpus.test:
    mapped_id = word2id[corpus.dictionary.idx2word[id]]
    target.append(mapped_id)
    
loss = criterion(distrs.log(), torch.LongTensor(target))
total_loss = loss.item()

print("Loss: %.3f, PPL: %s" % (total_loss, lang_util.perpl(total_loss)))


Loss: 4.966, PPL: 143.41069460658457

In [21]:
criterion = NLLLoss(reduction='mean')    
loss = criterion(kn5_remapped.log(), torch.LongTensor(corpus.test))
total_loss = loss.item()

print("Loss: %.3f, PPL: %s" % (total_loss, lang_util.perpl(total_loss)))


Loss: 4.966, PPL: 143.41069460658457

In [16]:
a = torch.rand(2, 3)
print(a)
b = a[:, torch.LongTensor([0, 2, 1])]
print(b)


tensor([[0.1048, 0.1051, 0.5019],
        [0.1935, 0.7691, 0.6780]])
tensor([[0.1048, 0.5019, 0.1051],
        [0.1935, 0.6780, 0.7691]])

Build RSM


In [32]:
import rsm_samplers
import rsm
from ptb import lang_util
import rsm_experiment
reload(rsm_samplers)
reload(rsm)
reload(lang_util)
reload(rsm_experiment)

CONFIG = {
    'debug': False,
    'path': "/Users/jgordon/nta/results",
    'data_dir': "/Users/jgordon/nta/datasets",
    'dataset': 'ptb',
    'predictor_hidden_size': 1200,
    'predictor_output_size': 10000,
    'batch_size': 300,
    'batches_in_epoch': 2000,
    'batches_in_first_epoch': 2,
    'batch_log_interval': 2000,
    'vocab_size': 10000,
    'eval_interval': 3,
    'eval_batch_size': 100,
    'eval_batches_in_epoch': 824, #8243,
    'learning_rate': 0.0005,
    'm_groups': 1500,
    'n_cells_per_group': 1,
    'k_winners': 80,
    'k_winner_cells': 1,
    'pred_l2_reg': 0.000001,
    'dec_l2_reg': 0.000001,
    'input_bias': True,
    'eval_interval': 5,
    'eps': 0.5,
    'gamma': 0.0,
    'forget_mu': 0.025,
    'weight_sparsity': None,
    'mult_integration': False,
    'fpartition': None,
    'boost_strength': 0.5,
    'boost_strength_factor': 0.85,
    'boost_strat': 'col_boosting',
    'do_inhibition': False,
    'x_b_norm': True,
    'balance_part_winners': True,
    'decode_activation_fn': None,
    'decode_bias': False,
    'embed_dim': 100,
    'input_size': (1, 100),
    'output_size': 100,
    'embedding_kind': 'ptb_fasttext_e5',
    'max_decay': 0.95,
    'mem_floor': 0.0005,
    'trainable_decay': True,
    'word_cache_decay': 0.99,
    'kn5_pct': 0.1
}
    
exp = rsm_experiment.RSMExperiment(config=CONFIG)

In [33]:
exp.model_setup(CONFIG, restore_path="/Users/jgordon/Desktop/rsm_highlights/PTB/RSMTune_0_2019-09-20_21-15-18mchjovgl/checkpoint_112")


setup: Using cpu
Maybe download PTB...

Loaded embedding dict (ptb_fasttext_e5) with 10000 entries
Built dataloaders...
Loading from /Users/jgordon/Desktop/rsm_highlights/PTB/RSMTune_0_2019-09-20_21-15-18mchjovgl/checkpoint_112

In [13]:
wcs = []
uss = []
ppls = []

WC_DECAY = 0.98

for wc in torch.arange(0.04, .11, 0.01):
    for us in torch.arange(0.001, 0.02, 0.002):
        print("wc: %s, us: %s" % (wc.item(), us.item()))
        exp.word_cache_pct = wc.item()
        exp.unif_smoothing = us.item()
        exp.word_cache_decay = WC_DECAY
        ret = exp.eval_epoch(0)
        int_ppl = ret['val_interp_ppl']
        ppls.append(int_ppl)
        wcs.append(wc.item())
        uss.append(us.item())
        print(ret)


wc: 0.03999999910593033, us: 0.0010000000474974513
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 9.000%, pred ppl: 743.5
Targ: it  was n't black    monday </s> but while the new   york  stock exchange did n't fall  apart friday as   the 
Pred: one is  n't expected </s>   </s> the the   the <unk> crowd times exchange is  n't <unk> in    </s>   </s> the 
{'val_loss': 0.039184603166486164, 'val_interp_ppl': 107.06121567910887, 'val_pred_ppl': 117.15777761373491, 'val_pred_acc': 24.29368932038835}
wc: 0.03999999910593033, us: 0.003000000026077032
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 10.000%, pred ppl: 670.9
Targ: pit  in   chicago waves of   selling continued to hit stocks themselves on   the big board and  specialists continued to <unk> 
Pred: </s> </s> N       </s>  </s> the     the       to the the    </s>       </s> the new board </s> the         in        to the   
{'val_loss': 0.039181382494788725, 'val_interp_ppl': 107.03257124937855, 'val_pred_ppl': 117.17197209145539, 'val_pred_acc': 24.29247572815534}
wc: 0.03999999910593033, us: 0.004999999888241291
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 10.000%, pred ppl: 605.2
Targ: friday will undoubtedly cause renewed debate     about whether wall street is properly prepared for another crash situation </s> the big   
Pred: </s>   </s> be          be    a       volatility </s>  the     the  street 's <unk>    to       to  the     <unk> </s>      </s> the rumor 
{'val_loss': 0.03918048901510542, 'val_interp_ppl': 106.97822837013234, 'val_pred_ppl': 117.15409308632434, 'val_pred_acc': 24.29368932038835}
wc: 0.03999999910593033, us: 0.007000000216066837
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 10.000%, pred ppl: 602.4
Targ: corp. was acquired by australian developer    george <unk>   currently hooker   's   chairman </s> mr. <unk> <unk> to   launch  an ambitious 
Pred: </s>  and elected  by <unk>      entrepreneur and    gillett </s>      chairman </s> chairman and  mr. <unk> said  will succeed a  offer     
{'val_loss': 0.03918117782082951, 'val_interp_ppl': 106.93318175890992, 'val_pred_ppl': 117.11651277100279, 'val_pred_acc': 24.29854368932039}
wc: 0.03999999910593033, us: 0.008999999612569809
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 10.000%, pred ppl: 590.0
Targ: said </s> a   major     reason is  that investors already have sharply scaled back their purchases of   stock funds since black 
Pred: </s> </s> the spokesman factor for that the       will    are  a       in     in   to    <unk>     </s> the   in    </s>  the   
{'val_loss': 0.039179578929850196, 'val_interp_ppl': 106.93947710466206, 'val_pred_ppl': 117.12931525572999, 'val_pred_acc': 24.300970873786408}
wc: 0.03999999910593033, us: 0.010999999940395355
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 10.000%, pred ppl: 638.8
Targ: peter <unk> vice president for planning at  the   phoenix ariz.   carrier    said  in  an interview that the work    <unk> at   
Pred: in    <unk> N    president of  <unk>    and <unk> <unk>   company investment <unk> the a  interview that it  company force will 
{'val_loss': 0.03917774543600175, 'val_interp_ppl': 106.90701038718956, 'val_pred_ppl': 117.10076544070915, 'val_pred_acc': 24.30218446601942}
wc: 0.03999999910593033, us: 0.013000000268220901
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 10.000%, pred ppl: 671.6
Targ: and the   workers ' <unk> with being forced to work      many hours overtime </s> in  separate developments talks have broken 
Pred: the <unk> <unk>   ' <unk> of   the   <unk>  to subscribe </s> of    </s>     </s> the addition cases        of    with been   
{'val_loss': 0.0391835036407252, 'val_interp_ppl': 107.00519472066837, 'val_pred_ppl': 117.20742374700646, 'val_pred_acc': 24.29247572815534}
wc: 0.03999999910593033, us: 0.014999999664723873
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 10.000%, pred ppl: 679.0
Targ: to   close at $ N in new york stock exchange composite trading </s> <unk> &      broad home       corp.    said   it 
Pred: </s> N     at N N a  new york stock exchange composite trading </s> the   closed <unk> industries delivery jumped it 
{'val_loss': 0.03918286766043132, 'val_interp_ppl': 107.03413567202082, 'val_pred_ppl': 117.20362080545257, 'val_pred_acc': 24.29247572815534}
wc: 0.03999999910593033, us: 0.017000000923871994
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 10.000%, pred ppl: 700.1
Targ: pound last spring have skidded to   between N cents and N cents </s> meanwhile the price of    <unk> the   chemical 
Pred: year  </s> year   </s> been    </s> N       N N     a   N cents a    the       the <unk> index the   <unk> largest  
{'val_loss': 0.03918270423818488, 'val_interp_ppl': 107.02325547010985, 'val_pred_ppl': 117.16524768837635, 'val_pred_acc': 24.29733009708738}
wc: 0.03999999910593033, us: 0.01899999938905239
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 10.000%, pred ppl: 684.2
Targ: </s> <unk> is   the second-largest propane distributor in    the u.s. </s> the largest suburban propane was   already owned by quantum 
Pred: yen  the   s.a. a   most           maker   to          maker the u.s. and  the u.s.    producer maker   <unk> N       <unk> by <unk>   
{'val_loss': 0.03918260109420974, 'val_interp_ppl': 107.085759140277, 'val_pred_ppl': 117.21620906627719, 'val_pred_acc': 24.29368932038835}
wc: 0.05000000074505806, us: 0.0010000000474974513
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 10.000%, pred ppl: 730.3
Targ: has had in recent years  </s> the ec      and japan the u.s. 's  largest steel      suppliers have n't  been filling 
Pred: is  n't a  the    months </s> the company is  the   's  u.s. and largest securities segment   and  been been <unk>   
{'val_loss': 0.0391839054363409, 'val_interp_ppl': 106.77574311842667, 'val_pred_ppl': 117.19227435366209, 'val_pred_acc': 24.29733009708738}
wc: 0.05000000074505806, us: 0.003000000026077032
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 10.000%, pred ppl: 758.1
Targ: clobbered two  years ago in   japan when <unk> introduced a powerful detergent called attack which quickly won  a   N     N 
Pred: a         </s> years ago </s> the   </s> the   <unk>      a <unk>    <unk>     for    <unk>  on    has     </s> the <unk> N 
{'val_loss': 0.039184753559258524, 'val_interp_ppl': 106.70573896775164, 'val_pred_ppl': 117.1757804722615, 'val_pred_acc': 24.299757281553397}
wc: 0.05000000074505806, us: 0.004999999888241291
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 11.000%, pred ppl: 679.4
Targ: and  mrs.  hills </s> many called it  simply a     contrast in styles </s> but some saw it  as a classic 
Pred: </s> <unk> <unk> </s> the  of     the is     <unk> <unk>    to the    and  the the  of  the 's a <unk>   
{'val_loss': 0.039182307924261514, 'val_interp_ppl': 106.6038009345696, 'val_pred_ppl': 117.09304448219557, 'val_pred_acc': 24.303398058252426}
wc: 0.05000000074505806, us: 0.007000000216066837
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 670.6
Targ: small subsidiary that is  <unk> unrelated becomes a     difficult <unk> said <unk> <unk> president of the parent in a     statement 
Pred: year  portion    of   has <unk> </s>      to      <unk> <unk>     time  </s> <unk> <unk> a         of the <unk>  of <unk> <unk>     
{'val_loss': 0.03918254861011378, 'val_interp_ppl': 106.66099844858812, 'val_pred_ppl': 117.17618762456044, 'val_pred_acc': 24.296116504854368}
wc: 0.05000000074505806, us: 0.008999999612569809
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 705.6
Targ: plan to press specifically for   a <unk> of               rules governing exports of   machine tools computers and  other high-technology products </s> 
Pred: are  to buy   the          <unk> a <unk> recapitalization the   </s>      the     </s> the     tools </s>      </s> to    crops           </s>     </s> 
{'val_loss': 0.039182619234655526, 'val_interp_ppl': 106.64609249490431, 'val_pred_ppl': 117.16795515152494, 'val_pred_acc': 24.30218446601942}
wc: 0.05000000074505806, us: 0.010999999940395355
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 697.6
Targ: corp. and <unk> corp. the successor company to   <unk>       hotels </s> <unk> officials could n't be located </s> financial corp.    
Pred: </s>  and <unk> <unk> a   <unk>     of      </s> concentrate the    </s> the   <unk>     said  n't be reached in   <unk>     services 
{'val_loss': 0.03918287246397615, 'val_interp_ppl': 106.68242691052995, 'val_pred_ppl': 117.20745374250441, 'val_pred_acc': 24.296116504854368}
wc: 0.05000000074505806, us: 0.013000000268220901
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 706.0
Targ: portions of kansas he   said </s> the soviet  union has n't  given any clear indication of its wheat purchase plans 
Pred: </s>     of the    </s> said </s> the company union has been yet   the <unk> violation  of the <unk> contract of    
{'val_loss': 0.03918740623947052, 'val_interp_ppl': 106.70510302303907, 'val_pred_ppl': 117.23230832351102, 'val_pred_acc': 24.29004854368932}
wc: 0.05000000074505806, us: 0.014999999664723873
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 667.2
Targ: to shareholders </s> but otherwise it           would undoubtedly come back with an offer by management </s> the executive said any 
Pred: is be           </s> the the       developments 's    n't         the  to   to   a  <unk> to the        to   the company   said the 
{'val_loss': 0.03918458810709055, 'val_interp_ppl': 106.66025010910495, 'val_pred_ppl': 117.16310972496667, 'val_pred_acc': 24.29490291262136}
wc: 0.05000000074505806, us: 0.017000000923871994
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 684.6
Targ: a share up   from the year-earlier $      N million or N cents a share </s> revenue rose to $ N 
Pred: a share </s> N    $   year-earlier period N million or N cents a share </s> revenue rose N  $ N 
{'val_loss': 0.03918481884677939, 'val_interp_ppl': 106.76326092241924, 'val_pred_ppl': 117.25518704101735, 'val_pred_acc': 24.28276699029126}
wc: 0.05000000074505806, us: 0.01899999938905239
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 691.4
Targ: the N        period which was helped by increased ad     spending from the summer olympics </s> while usa today 's total 
Pred: the previous N      </s>  was N      by the       demand revenue  in   the end    of       </s> the   the 's    's <unk> 
{'val_loss': 0.039184963224383376, 'val_interp_ppl': 106.77138985699189, 'val_pred_ppl': 117.22897364872006, 'val_pred_acc': 24.283980582524272}
wc: 0.05999999865889549, us: 0.0010000000474974513
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 688.1
Targ: reason is   mounting competition from new japanese car  plants in   the u.s. that are     pouring out  more than  one million 
Pred: year   </s> the      a           </s> the york     </s> </s>   </s> the u.s. </s> country <unk>   </s> of   <unk> N   </s>    
{'val_loss': 0.03918688932379472, 'val_interp_ppl': 106.61021614923762, 'val_pred_ppl': 117.19555142853538, 'val_pred_acc': 24.28276699029126}
wc: 0.05999999865889549, us: 0.003000000026077032
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 681.6
Targ: added they hope to have more information early this week </s> investment canada  declined to comment on the reasons for  
Pred: the   that were to be   to   of          about </s> year </s> the        bankers 's       to comment on the new     </s> 
{'val_loss': 0.03918338612680441, 'val_interp_ppl': 106.51594057378325, 'val_pred_ppl': 117.12864473467215, 'val_pred_acc': 24.288834951456312}
wc: 0.05999999865889549, us: 0.004999999888241291
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 712.3
Targ: new   <unk> <unk> </s> the drug  introduced     last year is  expected to generate sales of about $ N million this 
Pred: <unk> <unk> </s>  </s> the <unk> administration by   year the expected to be       more  of $     $ N million </s> 
{'val_loss': 0.03918634114099793, 'val_interp_ppl': 106.57684815145734, 'val_pred_ppl': 117.24959292816658, 'val_pred_acc': 24.280339805825243}
wc: 0.05999999865889549, us: 0.007000000216066837
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 721.0
Targ: cents a    share on   sales of $ N million </s> the bronx   has  a wonderful <unk> garden a     great <unk> 
Pred: </s>  </s> share </s> sales of $ N million </s> the company n.y. N <unk>     to    in     <unk> <unk> deal  
{'val_loss': 0.03918720897088207, 'val_interp_ppl': 106.57865561495933, 'val_pred_ppl': 117.28050777665858, 'val_pred_acc': 24.281553398058254}
wc: 0.05999999865889549, us: 0.008999999612569809
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 703.2
Targ: has   avoided all that by  living in    a   long  island  suburb with his wife  who  's    so    <unk> to   soap 
Pred: <unk> said    the of   the the    <unk> the <unk> history </s>   </s> the <unk> </s> <unk> <unk> <unk> </s> the  
{'val_loss': 0.03918464269796477, 'val_interp_ppl': 106.42493939170598, 'val_pred_ppl': 117.11068737304173, 'val_pred_acc': 24.29490291262136}
wc: 0.05999999865889549, us: 0.010999999940395355
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 707.2
Targ: real  estate <unk>     mr.  crandall <unk> broadly and  said no  comment </s> on  friday morning before the market       's     sell-off 
Pred: times estate developer </s> <unk>    said  </s>    </s> the  the one     on   the the    the     the    the resignations closed plunge   
{'val_loss': 0.039180946780495274, 'val_interp_ppl': 106.51697493506595, 'val_pred_ppl': 117.20762021548725, 'val_pred_acc': 24.29368932038835}
wc: 0.05999999865889549, us: 0.013000000268220901
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 683.2
Targ: traded over-the-counter </s>   columbia laboratories inc. miami began trading with the symbol <unk> </s>  the pharmaceuticals maker   had traded over-the-counter 
Pred: a      as               market the      pictures     inc. said  n.j.  a       in   a   new    of    <unk> the <unk>           company of  a      <unk>            
{'val_loss': 0.03918161269874104, 'val_interp_ppl': 106.46627910821248, 'val_pred_ppl': 117.13570554828583, 'val_pred_acc': 24.296116504854368}
wc: 0.05999999865889549, us: 0.014999999664723873
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 625.3
Targ: a    legal <unk>     that they  inherited says prof. <unk> dick  howard of    the university of virginia   law    school but the 
Pred: </s> year  challenge </s> would can       the  </s>  <unk> <unk> <unk>  <unk> the <unk>      of california school </s>   of  it  
{'val_loss': 0.039181437438299646, 'val_interp_ppl': 106.45976629548419, 'val_pred_ppl': 117.12289668263143, 'val_pred_acc': 24.29004854368932}
wc: 0.05999999865889549, us: 0.017000000923871994
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 668.4
Targ: year    increasing its subscription rates and  cutting back on merchandise <unk> </s> in  an       announcement to   its staff last week 
Pred: quarter </s>       the N            of    </s> a       the  to the         </s>  </s> the addition interview    that the <unk> to   week 
{'val_loss': 0.03918245550046123, 'val_interp_ppl': 106.49164453347957, 'val_pred_ppl': 117.14798206906576, 'val_pred_acc': 24.29368932038835}
wc: 0.05999999865889549, us: 0.01899999938905239
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 659.3
Targ: <unk> by bernard shaw  and  <unk> <unk> a    <unk> former texas judge and   campus beauty queen who   has   never held     
Pred: sold  by the     <unk> </s> <unk> <unk> </s> <unk> firm   <unk> <unk> <unk> <unk>  <unk>  </s>  <unk> <unk> been  bothered 
{'val_loss': 0.03918375370083648, 'val_interp_ppl': 106.51504804089438, 'val_pred_ppl': 117.16378465241245, 'val_pred_acc': 24.29126213592233}
wc: 0.07000000029802322, us: 0.0010000000474974513
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 686.7
Targ: producer with <unk> <unk> who  notes that <unk> is    <unk> to   his job   </s> the network 's  salaries have always 
Pred: vice     of   <unk> </s>  </s> is    </s> the   <unk> the   </s> the <unk> </s> the <unk>   was <unk>    of   been   
{'val_loss': 0.039184678892957646, 'val_interp_ppl': 106.55898855023746, 'val_pred_ppl': 117.20649041769856, 'val_pred_acc': 24.288834951456312}
wc: 0.07000000029802322, us: 0.003000000026077032
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 13.000%, pred ppl: 630.0
Targ: lacked enough financial information about <unk> and  the   <unk> and  sent  the cases  back to federal district court in   dallas 
Pred: is     up     to        planners    to    the   </s> <unk> <unk> </s> <unk> to  market of   to the     funds    </s>  </s> new    
{'val_loss': 0.039184212449540214, 'val_interp_ppl': 106.4742828635311, 'val_pred_ppl': 117.16290708260026, 'val_pred_acc': 24.29368932038835}
wc: 0.07000000029802322, us: 0.004999999888241291
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 13.000%, pred ppl: 586.1
Targ: <unk> 's   civil war </s> details of the talks described by  a   <unk> official as   very delicate were n't   disclosed 
Pred: of    </s> <unk> war </s> the     of the <unk> </s>      the the <unk> <unk>    </s> a    <unk>    </s> <unk> <unk>     
{'val_loss': 0.03918165288124125, 'val_interp_ppl': 106.45266407108932, 'val_pred_ppl': 117.17384220022109, 'val_pred_acc': 24.296116504854368}
wc: 0.07000000029802322, us: 0.007000000216066837
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 13.000%, pred ppl: 622.3
Targ: in   world oil     prices sent the index  surging at double-digit annual rates </s> energy prices then plummeted through the summer 
Pred: </s> the   markets </s>   </s> a   colony of      a  N            </s>   rates for  the    prices rose in        N       the <unk>  
{'val_loss': 0.03918007258054556, 'val_interp_ppl': 106.40638887537415, 'val_pred_ppl': 117.13388819283904, 'val_pred_acc': 24.29733009708738}
wc: 0.07000000029802322, us: 0.008999999612569809
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 13.000%, pred ppl: 665.8
Targ: suspend dividend payments on   its two   classes of   preferred stock indicating that regulators ' concerns  about the troubled institution have 
Pred: N       the      N        </s> the <unk> million </s> $         stock </s>       a    the        ' syndicate about the company  company     </s> 
{'val_loss': 0.03917870972106613, 'val_interp_ppl': 106.39840712870219, 'val_pred_ppl': 117.13279515552934, 'val_pred_acc': 24.29733009708738}
wc: 0.07000000029802322, us: 0.010999999940395355
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 13.000%, pred ppl: 611.3
Targ: while <unk> </s> the woman won the bet   </s>  but perhaps even more   remarkable the  <unk> <unk> <unk> make  a  
Pred: </s>  the   </s> the <unk> 's  the <unk> <unk> the the     the  though than       </s> <unk> of    of    <unk> it 
{'val_loss': 0.03918462630940005, 'val_interp_ppl': 106.38159759290609, 'val_pred_ppl': 117.13772045458028, 'val_pred_acc': 24.29733009708738}
wc: 0.07000000029802322, us: 0.013000000268220901
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 13.000%, pred ppl: 599.1
Targ: is n't much choice </s> thus europe has begun the recent crusade to produce more worthy shows of  its own   
Pred: is a   any  less   </s> the  in     and been  to  <unk>  <unk>   of be      a    than   than  the the <unk> 
{'val_loss': 0.03918427850562826, 'val_interp_ppl': 106.38822705994346, 'val_pred_ppl': 117.1375044941851, 'val_pred_acc': 24.29247572815534}
wc: 0.07000000029802322, us: 0.014999999664723873
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 662.9
Targ: fight with <unk> for   ferranti competition is   the   name  of the game  she  said </s> at  least one potential <unk>     
Pred: year  </s> the   <unk> the      and         </s> <unk> first of the <unk> </s> says </s> the the   the of        customers 
{'val_loss': 0.03918483641758127, 'val_interp_ppl': 106.4121099697223, 'val_pred_ppl': 117.13528157424732, 'val_pred_acc': 24.29368932038835}
wc: 0.07000000029802322, us: 0.017000000923871994
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 663.0
Targ: steel industries inc. expects to report that third-quarter earnings dropped more than N N from the previous quarter as   a 
Pred: 's    </s>       </s> and     to post   a    third-quarter net      for     N    than N N </s> the previous year    </s> a 
{'val_loss': 0.03918287033233249, 'val_interp_ppl': 106.48625848828334, 'val_pred_ppl': 117.20206887056716, 'val_pred_acc': 24.288834951456312}
wc: 0.07000000029802322, us: 0.01899999938905239
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 640.4
Targ: trouble for  software firms generally </s> it  creates uncertainty and   usually <unk> down sales said <unk> <unk> an analyst at   
Pred: </s>    </s> the      </s>  </s>      have the 's      a           about <unk>   do    to   the   </s> mr.   <unk> an analyst with 
{'val_loss': 0.039180360517455534, 'val_interp_ppl': 106.46303349516859, 'val_pred_ppl': 117.15682506744437, 'val_pred_acc': 24.29368932038835}
wc: 0.07999999821186066, us: 0.0010000000474974513
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 629.7
Targ: in N        N   N     of  those aged     N to   N lived with <unk> other than  spouses down from N N 
Pred: in addition the years one the   surveyed N days N N     in   N     <unk> <unk> N       </s> from N N 
{'val_loss': 0.03918534330119496, 'val_interp_ppl': 106.62139605148384, 'val_pred_ppl': 117.20900679051941, 'val_pred_acc': 24.2876213592233}
wc: 0.07999999821186066, us: 0.003000000026077032
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 11.000%, pred ppl: 655.4
Targ: and  the cheap mixed drinks go   for  $   N a    pop   </s> at  the <unk> manager elizabeth <unk> wo  n't 
Pred: </s> N   <unk> </s>  </s>   </s> </s> the N </s> pound </s> the the end   <unk>   's        <unk> the n't 
{'val_loss': 0.03918532447129922, 'val_interp_ppl': 106.47888546782089, 'val_pred_ppl': 117.13212591597828, 'val_pred_acc': 24.29004854368932}
wc: 0.07999999821186066, us: 0.004999999888241291
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 11.000%, pred ppl: 663.7
Targ: elsewhere <unk> all  critical mention of   group differences </s> as  elizabeth <unk> wrote in the new  york times just before 
Pred: N         </s>  </s> the      <unk>   </s> the   </s>        </s> the a         <unk> a     a  the u.s. york city  co.  a      
{'val_loss': 0.039187726948988956, 'val_interp_ppl': 106.50648665364966, 'val_pred_ppl': 117.19530042103014, 'val_pred_acc': 24.2876213592233}
wc: 0.07999999821186066, us: 0.007000000216066837
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 615.0
Targ: eric w. <unk> resigned in   june </s> the senate  's     decision to approve a   <unk> deficit-reduction bill without a   capital-gains 
Pred: vice c. <unk> </s>     </s> the  </s> mr. company passed board    to approve the <unk> amendment         bill that    the bill          
{'val_loss': 0.03918255281010733, 'val_interp_ppl': 106.53722645425258, 'val_pred_ppl': 117.24737652242443, 'val_pred_acc': 24.283980582524272}
wc: 0.07999999821186066, us: 0.008999999612569809
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 564.4
Targ: things had just gone too far  </s> sen. dole  said  that the move  required sacrifice by every senator </s>  it  
Pred: in     are been been at  much </s> the  <unk> <unk> the  the <unk> is       to        to the   N       <unk> the 
{'val_loss': 0.03918188590854768, 'val_interp_ppl': 106.47308704315931, 'val_pred_ppl': 117.18046892835132, 'val_pred_acc': 24.28640776699029}
wc: 0.07999999821186066, us: 0.010999999940395355
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 13.000%, pred ppl: 515.1
Targ: the new census bureau report <unk> from N   to N are  out of date certainly as   an average for the 
Pred: N   end york   bureau </s>   </s>  </s> the N  N </s> N   of the  </s>      </s> a  <unk>   of  the 
{'val_loss': 0.03918131102708358, 'val_interp_ppl': 106.41892661952862, 'val_pred_ppl': 117.11155214695155, 'val_pred_acc': 24.288834951456312}
wc: 0.07999999821186066, us: 0.013000000268220901
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 13.000%, pred ppl: 531.2
Targ: public enthusiasm for  stock mutual funds </s> the main  thing  was portfolio insurance a   mechanical   trading    system intended to protect 
Pred: </s>   </s>       </s> the   market funds </s> the <unk> reason is  a         the       and money-market investment firm   is       to <unk>   
{'val_loss': 0.039182643767771794, 'val_interp_ppl': 106.44453681318785, 'val_pred_ppl': 117.14041808421845, 'val_pred_acc': 24.288834951456312}
wc: 0.07999999821186066, us: 0.014999999664723873
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 13.000%, pred ppl: 511.2
Targ: do   well during periods of   economic weakness </s> frank <unk> </s> many  people now claim to  have predicted the  N     
Pred: </s> n't  </s>   the     </s> the      growth   </s> the   <unk> an   <unk> of     are are   the be   to        that <unk> 
{'val_loss': 0.039180965985632636, 'val_interp_ppl': 106.44828641047944, 'val_pred_ppl': 117.12969639207364, 'val_pred_acc': 24.29368932038835}
wc: 0.07999999821186066, us: 0.017000000923871994
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 13.000%, pred ppl: 559.0
Targ: generate </s> the last  time  i            saw a   similar congressional hearing was when <unk> <unk> joe   <unk> did his work  
Pred: </s>     </s> the <unk> thing westinghouse was the <unk>   <unk>         hearing is  to   the   with  <unk> <unk> the n't <unk> 
{'val_loss': 0.03918333676670795, 'val_interp_ppl': 106.45674218567169, 'val_pred_ppl': 117.11403208571853, 'val_pred_acc': 24.29368932038835}
wc: 0.07999999821186066, us: 0.01899999938905239
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 13.000%, pred ppl: 556.8
Targ: the american stock exchange composite fell    N N     </s> on  oct. N N    the nasdaq composite fell  N points or 
Pred: N   <unk>    stock exchange </s>      trading N cents to   the the  N </s> of  <unk>  composite index N to     to 
{'val_loss': 0.03918236835850674, 'val_interp_ppl': 106.58901358652466, 'val_pred_ppl': 117.23424526149532, 'val_pred_acc': 24.281553398058254}
wc: 0.09000000357627869, us: 0.0010000000474974513
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 14.000%, pred ppl: 549.4
Targ: blocks of N     shares or   more two  years ago  </s> shearson 's     mr. dapuzzo said retail investors nervously sold stock 
Pred: the    of <unk> and    </s> more </s> years </s> </s> the      lehman mr. <unk>   said the    sales     are       have off   
{'val_loss': 0.03918034621306415, 'val_interp_ppl': 106.76469681414598, 'val_pred_ppl': 117.22176789697443, 'val_pred_acc': 24.28276699029126}
wc: 0.09000000357627869, us: 0.003000000026077032
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 14.000%, pred ppl: 537.2
Targ: month N N N three months N N N six months N N N one year </s> the average of 
Pred: year  N N N to    months N N N to  months N N N to  year </s> the average of 
{'val_loss': 0.03918855270208085, 'val_interp_ppl': 106.83501218327437, 'val_pred_ppl': 117.35458443940217, 'val_pred_acc': 24.264563106796118}
wc: 0.09000000357627869, us: 0.004999999888241291
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 14.000%, pred ppl: 562.5
Targ: and  the   <unk> of computerized program trading </s> the only thing you do  n't have he said is   the portfolio 
Pred: </s> other <unk> of the          trading trading </s> the big  way   is  're n't want to said </s> n't market    
{'val_loss': 0.03918295930528525, 'val_interp_ppl': 106.66562267416963, 'val_pred_ppl': 117.21013709007252, 'val_pred_acc': 24.29126213592233}
wc: 0.09000000357627869, us: 0.007000000216066837
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 14.000%, pred ppl: 570.8
Targ: republican but   on  friday morning he   had  few  kind words for  president bush 's economic <unk>   </s> there are some 
Pred: <unk>      <unk> the the    </s>    </s> said been days of    </s> the       bush to <unk>    reforms </s> the   is  no   
{'val_loss': 0.039182112199588885, 'val_interp_ppl': 106.60695588274939, 'val_pred_ppl': 117.16593165397586, 'val_pred_acc': 24.29247572815534}
wc: 0.09000000357627869, us: 0.008999999612569809
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 15.000%, pred ppl: 520.2
Targ: angry over mr. bush      's claim that the capital-gains cut was part of april 's budget  accord and  his insistence 
Pred: <unk> </s> the bernstein 's <unk> </s> he  <unk>         tax is  n't  of a     N  request </s>   </s> the plan       
{'val_loss': 0.039182443474645466, 'val_interp_ppl': 106.58714444450965, 'val_pred_ppl': 117.15980348714949, 'val_pred_acc': 24.2876213592233}
wc: 0.09000000357627869, us: 0.010999999940395355
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 15.000%, pred ppl: 486.4
Targ: funding group </s> pennsylvania higher education facilities authority </s> $   N million of revenue bonds for    <unk> university series N 
Pred: funding corp. </s> the          's     interest  services   financing $    the N million of N       bonds series the   and        bonds  N 
{'val_loss': 0.03917921902792691, 'val_interp_ppl': 106.70642720077983, 'val_pred_ppl': 117.29830142109091, 'val_pred_acc': 24.277912621359224}
wc: 0.09000000357627869, us: 0.013000000268220901
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 14.000%, pred ppl: 499.5
Targ: following black monday but  mostly through mutual funds </s> discount brokerage customers have been in    the    market somewhat but  not 
Pred: value     the   &      </s> the    in      the    funds </s> the      yields    firms     were been <unk> recent past   </s>     </s> the 
{'val_loss': 0.03918539384352857, 'val_interp_ppl': 106.62958199089667, 'val_pred_ppl': 117.19098823684222, 'val_pred_acc': 24.29004854368932}
wc: 0.09000000357627869, us: 0.014999999664723873
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 15.000%, pred ppl: 459.4
Targ: that demand however has  led  to  a     variety of <unk> </s> making computers smaller often means <unk> memory </s>  it  
Pred: N    the    for     </s> been the <unk> <unk>   of the   </s> the    the       and     than  have  that  </s>   chips the 
{'val_loss': 0.039183836830418234, 'val_interp_ppl': 106.65394769832443, 'val_pred_ppl': 117.20108692523479, 'val_pred_acc': 24.288834951456312}
wc: 0.09000000357627869, us: 0.017000000923871994
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 15.000%, pred ppl: 441.6
Targ: risen N    N in the first nine months of the  year <unk> both the initial N     N inflation goal set 
Pred: been  </s> N to the first half months of this year </s>  </s> the <unk>   <unk> N increase  rate of  
{'val_loss': 0.03918265680628784, 'val_interp_ppl': 106.73686643838975, 'val_pred_ppl': 117.2763979450769, 'val_pred_acc': 24.28276699029126}
wc: 0.09000000357627869, us: 0.01899999938905239
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 15.000%, pred ppl: 432.1
Targ: to staying involved in these programs in   which <unk> earn <unk> miles and  <unk> can  get <unk> discounts </s> i   
Pred: of the     out      in the   parts    </s> the   they  </s> </s>  </s>  </s> <unk> </s> be  a     </s>      </s> the 
{'val_loss': 0.03918120817471187, 'val_interp_ppl': 106.66174987456606, 'val_pred_ppl': 117.16552054512954, 'val_pred_acc': 24.288834951456312}
wc: 0.10000000149011612, us: 0.0010000000474974513
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 15.000%, pred ppl: 446.7
Targ: which analysts say are  roughly three times the price  of a comparable system from tandem </s> obviously ibm can give 
Pred: </s>  is       say </s> n't     N     times as  market of N N          N      </s> the    </s> the       the 's  be   
{'val_loss': 0.03918659020197189, 'val_interp_ppl': 107.00855284957692, 'val_pred_ppl': 117.25612666415459, 'val_pred_acc': 24.274271844660195}
wc: 0.10000000149011612, us: 0.003000000026077032
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 15.000%, pred ppl: 457.8
Targ: <unk> of   <unk> group a    <unk> <unk> pa.  forecasting company </s> a   lot  of <unk> demand is  gone </s> consumer 
Pred: </s>  </s> the   <unk> </s> <unk> firm  firm </s>        </s>    </s> the year of <unk> is     for a    at   <unk>    
{'val_loss': 0.03918342698293114, 'val_interp_ppl': 106.94184640103279, 'val_pred_ppl': 117.2446876326732, 'val_pred_acc': 24.277912621359224}
wc: 0.10000000149011612, us: 0.004999999888241291
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 16.000%, pred ppl: 449.8
Targ: <unk> operations </s> still some         industry giants    are expected to report continuing gains  largely because so much of their business 
Pred: board </s>       </s> the   westinghouse traders  observers are <unk>    to be     a          growth in      because of far  of the   <unk>    
{'val_loss': 0.039184566994098204, 'val_interp_ppl': 106.90379932493839, 'val_pred_ppl': 117.23258107592748, 'val_pred_acc': 24.283980582524272}
wc: 0.10000000149011612, us: 0.007000000216066837
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 16.000%, pred ppl: 478.4
Targ: the  market 's     afternoon surge   </s> among classes for which   details were available yields ranged from N N or N 
Pred: </s> end    closed <unk>     session </s> the   the     of  example the     of   suspended for    on     from N N in N 
{'val_loss': 0.03918376059081514, 'val_interp_ppl': 107.00502649003957, 'val_pred_ppl': 117.36571005585274, 'val_pred_acc': 24.268203883495147}
wc: 0.10000000149011612, us: 0.008999999612569809
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 17.000%, pred ppl: 471.9
Targ: the  <unk> to require that all employees give similar    notice before they quit </s> <unk> s.    <unk> </s> <unk> comment 
Pred: </s> <unk> of the     the  the the       '    themselves to     </s>   the  are  </s> the   <unk> <unk> of   the   <unk>   
{'val_loss': 0.039186707516969406, 'val_interp_ppl': 106.84732619825515, 'val_pred_ppl': 117.1892260557583, 'val_pred_acc': 24.288834951456312}
wc: 0.10000000149011612, us: 0.010999999940395355
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 17.000%, pred ppl: 471.8
Targ: N million and would have been much     higher had  not  the  cost  of the trading floor set  been absorbed in   
Pred: N million in  N     have a    expected less   than been been <unk> of the company arena </s> up   </s>     </s> 
{'val_loss': 0.03918537224453051, 'val_interp_ppl': 106.90168323058512, 'val_pred_ppl': 117.25438981987762, 'val_pred_acc': 24.28640776699029}
wc: 0.10000000149011612, us: 0.013000000268220901
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 16.000%, pred ppl: 523.5
Targ: the <unk>  sale  of farmers to    axa receives regulatory approval </s> a   spokesman for b.a.t said of  the amended filings 
Pred: the market <unk> is the     <unk> the the      the        capital  </s> the spokesman for the   's   the the company pact    
{'val_loss': 0.03918662398275964, 'val_interp_ppl': 106.9479417949018, 'val_pred_ppl': 117.2961740622999, 'val_pred_acc': 24.281553398058254}
wc: 0.10000000149011612, us: 0.014999999664723873
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 16.000%, pred ppl: 520.3
Targ: and  <unk> marine transportation and  machinery used to make  food  and beverage cans </s> it  was  n't so       long  ago 
Pred: </s> <unk> </s>   </s>           </s> <unk>     </s> in <unk> <unk> and <unk>    </s> </s> the also a   expected <unk> to  
{'val_loss': 0.03918733827552749, 'val_interp_ppl': 106.96319204000281, 'val_pred_ppl': 117.29640152320292, 'val_pred_acc': 24.277912621359224}
wc: 0.10000000149011612, us: 0.017000000923871994
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 16.000%, pred ppl: 548.9
Targ: after N   p.m   </s> it  is a talk  show with opposition leaders and political experts who  discuss hungary 's domestic 
Pred: a     the years </s> the 's a <unk> of   that a          to      in  <unk>     <unk>   </s> favor   the     's <unk>    
{'val_loss': 0.03918718502775413, 'val_interp_ppl': 106.94104400844789, 'val_pred_ppl': 117.24648295465583, 'val_pred_acc': 24.277912621359224}
wc: 0.10000000149011612, us: 0.01899999938905239
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 16.000%, pred ppl: 547.9
Targ: up   </s> what better place to turn than sen. edward kennedy 's labor committee that great <unk> of  government <unk> 
Pred: </s> N    the  's     than  is the  the  the  <unk>  markey  d. <unk> secretary that the   <unk> the the        's    
{'val_loss': 0.03918452260708346, 'val_interp_ppl': 106.96401337631389, 'val_pred_ppl': 117.24358166374986, 'val_pred_acc': 24.274271844660195}
wc: 0.10999999940395355, us: 0.0010000000474974513
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 16.000%, pred ppl: 580.6
Targ: that would be down from the N N rise posted in N   </s> the canadian government announced a new 12-year  
Pred: in   's    be a    to   the N N of   in     a  net and  the company  dollar     reported  a N   contract 
{'val_loss': 0.039187608160150864, 'val_interp_ppl': 107.36065263056908, 'val_pred_ppl': 117.33349762623878, 'val_pred_acc': 24.271844660194176}
wc: 0.10999999940395355, us: 0.003000000026077032
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 17.000%, pred ppl: 555.6
Targ: service  would get important work done <unk> forest fires    fought housing <unk> students <unk> <unk> centers <unk> </s> there is  
Pred: mortgage corp. be  a         for  in   </s>  </s>   products </s>   by      and   </s>     </s>  and   and     </s>  and  the   are 
{'val_loss': 0.039186348175648054, 'val_interp_ppl': 107.16440374246876, 'val_pred_ppl': 117.16951145151275, 'val_pred_acc': 24.276699029126213}
wc: 0.10999999940395355, us: 0.004999999888241291
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 17.000%, pred ppl: 512.0
Targ: prudent what 's <unk> he    said </s> when it  was suggested his  comment was  a   <unk> mr.  darman replied it   
Pred: N       </s> it the   ratio says </s> the  the 's  a         that first   </s> n't <unk> </s> <unk>  's      that 
{'val_loss': 0.03918610698570615, 'val_interp_ppl': 107.23538868850225, 'val_pred_ppl': 117.28699289039011, 'val_pred_acc': 24.275485436893202}
wc: 0.10999999940395355, us: 0.007000000216066837
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 17.000%, pred ppl: 505.7
Targ: awful bear market of   the 1930s began </s> the october <unk> of    N   and N were scary but   did n't 
Pred: N     </s> </s>   </s> the <unk> </s>  to   in  <unk>   N     <unk> the N   N N    up    about the n't 
{'val_loss': 0.039187675658432605, 'val_interp_ppl': 107.18521017705773, 'val_pred_ppl': 117.26587014485291, 'val_pred_acc': 24.273058252427184}
wc: 0.10999999940395355, us: 0.008999999612569809
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 18.000%, pred ppl: 448.4
Targ: game 's   over  and  investors are  about to face a     bear  market </s> david m.    jones vice president at <unk> 
Pred: </s> </s> <unk> </s> <unk>     </s> <unk> to the  value <unk> </s>   </s> the   <unk> <unk> and  president of <unk> 
{'val_loss': 0.03918775128544245, 'val_interp_ppl': 107.20919175458047, 'val_pred_ppl': 117.31182753814886, 'val_pred_acc': 24.274271844660195}
wc: 0.10999999940395355, us: 0.010999999940395355
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 18.000%, pred ppl: 450.7
Targ: that <unk> <unk> is    just one of a   number of <unk>  adopted after the N crash </s> the big   board 
Pred: in   's    <unk> <unk> a    a   of the <unk>  of people of      by    the N crash and  the <unk> three 
{'val_loss': 0.03918892366685046, 'val_interp_ppl': 107.20923971929979, 'val_pred_ppl': 117.31020057141569, 'val_pred_acc': 24.264563106796118}
wc: 0.10999999940395355, us: 0.013000000268220901
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 18.000%, pred ppl: 477.3
Targ: the financial markets to   review its analysis and those of other crash      studies </s> in  may N the working group   
Pred: a   new       times   </s> be     the own      of  the   of the   provisions </s>    </s> the the N the number  capital 
{'val_loss': 0.039186669902387755, 'val_interp_ppl': 107.12169819602481, 'val_pred_ppl': 117.19758819874988, 'val_pred_acc': 24.276699029126213}
wc: 0.10999999940395355, us: 0.014999999664723873
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 18.000%, pred ppl: 504.8
Targ: small wonder that britain 's labor party    wants credit controls </s> a   few   hours after the party launched its own   
Pred: in    u.s.   is   's      's <unk> shortage is    to     to       to   the <unk> years of    the <unk> 's       a   <unk> 
{'val_loss': 0.039187443790758406, 'val_interp_ppl': 107.13105607119975, 'val_pred_ppl': 117.21485626706388, 'val_pred_acc': 24.275485436893202}
wc: 0.10999999940395355, us: 0.017000000923871994
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 18.000%, pred ppl: 507.5
Targ: rule on   sony 's   renewed request for  a pre-trial order blocking sale of   the disputed products on   which deliveries began 
Pred: N    </s> a    </s> N       bids    </s> a $         basis </s>     the  </s> $   company  </s>     </s> the   it         </s>  
{'val_loss': 0.03918613103281671, 'val_interp_ppl': 107.18672712935758, 'val_pred_ppl': 117.23498746400675, 'val_pred_acc': 24.268203883495147}
wc: 0.10999999940395355, us: 0.01899999938905239
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 18.000%, pred ppl: 505.1
Targ: </s> dresdner is offering to acquire N   N of <unk> 's    capital  for  N francs  $ N a       share </s> 
Pred: </s> the      's a        to redeem  the N of the   <unk> ordinary </s> $ million $ N billion share </s> 
{'val_loss': 0.03918551723246725, 'val_interp_ppl': 107.22081922099939, 'val_pred_ppl': 117.23816122080095, 'val_pred_acc': 24.269417475728154}

In [16]:
plt.scatter(wcs, uss, c=ppls)
plt.xlabel("Word Cache")
plt.ylabel("Uniform mass")
plt.colorbar()

idx = np.argmin(ppls)
print("Best values", 'cache', wcs[idx], 'unif', uss[idx])


Best values cache 0.07000000029802322 unif 0.010999999940395355

In [20]:
# Tune on decay rate

decays = []
ppls = []

for decay in torch.arange(0.98, .999, 0.001):
    exp.word_cache_pct = .07
    exp.unif_smoothing = .01
    exp.word_cache_decay = decay
    ret = exp.eval_epoch(0)
    int_ppl = ret['val_interp_ppl']
    ppls.append(int_ppl)
    decays.append(decay)
    print(ret)


Evaluating...
Finished batch 0
Partial pred acc - batch acc: 14.000%, pred ppl: 526.1
Targ: <unk> weekend trying to <unk> out new terms    that would be  more acceptable to   the banks </s> after ual the 
Pred: year  of      </s>   to <unk> the of  products </s> the   n't a    <unk>      </s> the <unk> </s> the   the 's  
{'val_loss': 0.03918219901603258, 'val_interp_ppl': 106.89812567179825, 'val_pred_ppl': 117.22550204933934, 'val_pred_acc': 24.260922330097088}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 14.000%, pred ppl: 517.1
Targ: last month </s> <unk> shares fell   nearly N N  on friday to  close at N N </s> ramada which first 
Pred: </s> week  </s> the   <unk>  closed N      N to to volume the N     at N N </s> the    which has   
{'val_loss': 0.03918567364492897, 'val_interp_ppl': 106.82033783646605, 'val_pred_ppl': 117.21013491958131, 'val_pred_acc': 24.25728155339806}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 14.000%, pred ppl: 520.7
Targ: peak </s> in  the mid-1980s employment was    down as much as N N from the N        peak and retail sales 
Pred: </s> </s> the the first     of         report N    N  the  as N N of   the previous N    in  N      sales 
{'val_loss': 0.039187450196991845, 'val_interp_ppl': 106.75761742318197, 'val_pred_ppl': 117.19568224953717, 'val_pred_acc': 24.263349514563107}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 15.000%, pred ppl: 466.4
Targ: said mr.  keating already has conceded attempting to buy influence with the lawmakers democratic sens.   dennis <unk> of   arizona alan 
Pred: </s> </s> klein   </s>    has been     that       to the the       the  the <unk>     '          leaders <unk>  <unk> </s> the     's   
{'val_loss': 0.03918413315601285, 'val_interp_ppl': 106.66910158224292, 'val_pred_ppl': 117.16231898779839, 'val_pred_acc': 24.264563106796118}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 15.000%, pred ppl: 496.1
Targ: the senate ethics committee </s> sen. <unk> said when all is  said     and done i  expect to be fully <unk>    
Pred: the year   </s>   </s>      </s> the  bob   r.   the  the the expected the the  is 'm     to be a     invested 
{'val_loss': 0.03918898112402813, 'val_interp_ppl': 106.63420090514275, 'val_pred_ppl': 117.18655776968228, 'val_pred_acc': 24.264563106796118}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 14.000%, pred ppl: 544.7
Targ: <unk> the  <unk> plastic <unk> are  based on an  american comic book and  television series </s> paul <unk> managing director 
Pred: the   </s> u.s.  of      </s>  </s> <unk> on the <unk>    heart of   </s> <unk>      </s>   </s> the  <unk> </s>     director 
{'val_loss': 0.03918687194965563, 'val_interp_ppl': 106.57584261488773, 'val_pred_ppl': 117.17674159177558, 'val_pred_acc': 24.268203883495147}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 14.000%, pred ppl: 549.2
Targ: an   analyst with goldman sachs & co  </s> noting that the third quarter is usually the  aluminum industry 's <unk> 
Pred: </s> <unk>   </s> <unk>   sachs & co. </s> the    that the issue quarter is n't     been only     company  is <unk> 
{'val_loss': 0.039182327990646214, 'val_interp_ppl': 106.54667126481266, 'val_pred_ppl': 117.19935710326209, 'val_pred_acc': 24.260922330097088}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 14.000%, pred ppl: 571.6
Targ: so   severely as to  be  light years away from the type  of <unk> deals available to sony and  everyone else 
Pred: </s> far      in the the <unk> </s>  ago  </s> the <unk> of <unk> </s>  </s>      to the  </s> the      </s> 
{'val_loss': 0.03918631160089258, 'val_interp_ppl': 106.50289021948349, 'val_pred_ppl': 117.16482759956969, 'val_pred_acc': 24.268203883495147}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 15.000%, pred ppl: 535.1
Targ: data management equipment </s> the company and  its executives deny the  charges </s> in  a   related   development recognition equipment said 
Pred: the  </s>       </s>      </s> the company said the board      were that company that the the statement matter      and         to        said 
{'val_loss': 0.039184529252929014, 'val_interp_ppl': 106.52176926371865, 'val_pred_ppl': 117.24045031284102, 'val_pred_acc': 24.263349514563107}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 15.000%, pred ppl: 497.8
Targ: and  the policies of racial <unk> on   all fronts    including the armed struggle </s> and they called on  the government 
Pred: </s> the u.s.     of the    and   </s> the countries </s>      the <unk> forces   </s> the the  are    the the <unk>      
{'val_loss': 0.03918830250634007, 'val_interp_ppl': 106.46071407785384, 'val_pred_ppl': 117.18876642863633, 'val_pred_acc': 24.265776699029125}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 15.000%, pred ppl: 511.3
Targ: is  not there in   the market </s> many money    managers and some traders   had already left their offices early friday 
Pred: was n't a     </s> the past   </s> the  japanese managers are the  investors say been    been to    clients in    in     
{'val_loss': 0.03918734072137949, 'val_interp_ppl': 106.47401629202281, 'val_pred_ppl': 117.21854370399015, 'val_pred_acc': 24.260922330097088}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 15.000%, pred ppl: 515.3
Targ: j.    phelan said yesterday the  circuit breaker  worked well <unk> </s> i   just  think it   's <unk> at this point 
Pred: <unk> <unk>  jr.  </s>      that company breakers 's     for  </s>  </s> the think got   that 's going to the  time  
{'val_loss': 0.03918354458953209, 'val_interp_ppl': 106.50493929166154, 'val_pred_ppl': 117.21100064488677, 'val_pred_acc': 24.260922330097088}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 15.000%, pred ppl: 544.9
Targ: to repay its bank debt and  other obligations resulting from the currently suspended <unk> operations </s> earlier the  company announced 
Pred: to <unk> $   debt debt </s> to    purposes    </s>      from the company   held      </s>  </s>       </s> the     this company said      
{'val_loss': 0.039188942822257, 'val_interp_ppl': 106.5890647082107, 'val_pred_ppl': 117.28762691736209, 'val_pred_acc': 24.25728155339806}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 14.000%, pred ppl: 580.0
Targ: N    from N   tons a    year earlier </s> the treasury plans to raise $   N million in new        cash thursday 
Pred: </s> to   the N    </s> year earlier </s> the company  said  to raise its N billion of short-term debt to       
{'val_loss': 0.03918742964460144, 'val_interp_ppl': 106.6894644098836, 'val_pred_ppl': 117.33096137269872, 'val_pred_acc': 24.25121359223301}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 14.000%, pred ppl: 583.3
Targ: saturday opened its N     <unk> investor centers across the country </s> the centers normally are   closed through the weekend </s> 
Pred: a        </s>   the <unk> to    <unk>    </s>    </s>   the bay     </s> the <unk>   are      <unk> <unk>  </s>    the <unk>   to   
{'val_loss': 0.03918603574857116, 'val_interp_ppl': 106.67238231760058, 'val_pred_ppl': 117.20015659800414, 'val_pred_acc': 24.266990291262136}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 14.000%, pred ppl: 582.4
Targ: n't think  their customers would like it very much </s> america west though is a   smaller airline and therefore more  
Pred: n't expect it    <unk>     </s>  be   to </s> much </s> the     's   german it n't <unk>   <unk>   is  the       <unk> 
{'val_loss': 0.03918418927497945, 'val_interp_ppl': 106.80887710093897, 'val_pred_ppl': 117.20791600405104, 'val_pred_acc': 24.266990291262136}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 13.000%, pred ppl: 641.5
Targ: eastern canada and  conventional electric power generating plants  elsewhere including britain where the british government plans to allow limited competition 
Pred: the     europe </s> the          <unk>    </s>  co         service </s>      </s>      <unk>   's    the u.s.    government and   to build <unk>   partnership 
{'val_loss': 0.03918821923434734, 'val_interp_ppl': 107.04658392368248, 'val_pred_ppl': 117.25723192629796, 'val_pred_acc': 24.263349514563107}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 13.000%, pred ppl: 614.6
Targ: buying raw land      while avoiding the negative <unk>  to its own   balance sheet mr.  <unk> said </s> the company is 
Pred: the    N   materials </s>  the      the u.s.     impact of the <unk> <unk>   sheet </s> <unk> said </s> the company is 
{'val_loss': 0.039186752507535436, 'val_interp_ppl': 107.2379684851892, 'val_pred_ppl': 117.1961044455846, 'val_pred_acc': 24.266990291262136}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 13.000%, pred ppl: 589.0
Targ: expand production capacity </s> a   quick     turnaround is crucial  to quantum because its cash  requirements remain heavy </s>   the company 
Pred: N      their      </s>     </s> the spokesman <unk>      in expected to the     's      the <unk> flow         </s>   in    volume the company 
{'val_loss': 0.03918590931701067, 'val_interp_ppl': 107.56778533644885, 'val_pred_ppl': 117.19652486355673, 'val_pred_acc': 24.263349514563107}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 13.000%, pred ppl: 596.8
Targ: down from roughly $ N million last year </s> the spokesman said the broadcast unit  will be <unk> dec. N 
Pred: </s> from N       N N billion in   year </s> the company   said the company   group is   be <unk> by   N 
{'val_loss': 0.03918679984675232, 'val_interp_ppl': 108.02005470170103, 'val_pred_ppl': 117.24525004380054, 'val_pred_acc': 24.262135922330096}

In [21]:
import matplotlib.pyplot as plt
import numpy as np

plt.plot(decays, ppls)
plt.xlabel("PPL")
plt.ylabel("Decay (word cache)")

idx = np.argmin(ppls)
print("Best values", 'decay', decays[idx])


Best values decay tensor(0.9890)

In [ ]:

Tune on KN5 pct


In [ ]:
# Tune on decay rate

kn5_pcts = []
wcs = []
ppls = []

for kn5_pct in torch.arange(0.005, .015, .0025):
    for wc in torch.arange(0.02, .07, .01):
        exp.word_cache_pct = .07
        exp.unif_smoothing = 0
        exp.kn5_pct = kn5_pct
        ret = exp.eval_epoch(0)
        int_ppl = ret['val_interp_ppl']
        wcs.append(wc)
        ppls.append(int_ppl)
        kn5_pcts.append(kn5_pct)
        print(kn5_pct, wc)


Evaluating...
Finished batch 0
Partial pred acc - batch acc: 10.000%, pred ppl: 730.3
Targ: has had in recent years  </s> the ec      and japan the u.s. 's  largest steel      suppliers have n't  been filling 
Pred: is  n't a  the    months </s> the company is  the   's  u.s. and largest securities segment   and  been been <unk>   
tensor(0.0050) tensor(0.0200)
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 10.000%, pred ppl: 758.1
Targ: clobbered two  years ago in   japan when <unk> introduced a powerful detergent called attack which quickly won  a   N     N 
Pred: a         </s> years ago </s> the   </s> the   <unk>      a <unk>    <unk>     for    <unk>  on    has     </s> the <unk> N 
tensor(0.0050) tensor(0.0300)
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 11.000%, pred ppl: 679.4
Targ: and  mrs.  hills </s> many called it  simply a     contrast in styles </s> but some saw it  as a classic 
Pred: </s> <unk> <unk> </s> the  of     the is     <unk> <unk>    to the    and  the the  of  the 's a <unk>   
tensor(0.0050) tensor(0.0400)
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 670.6
Targ: small subsidiary that is  <unk> unrelated becomes a     difficult <unk> said <unk> <unk> president of the parent in a     statement 
Pred: year  portion    of   has <unk> </s>      to      <unk> <unk>     time  </s> <unk> <unk> a         of the <unk>  of <unk> <unk>     
tensor(0.0050) tensor(0.0500)
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 705.6
Targ: plan to press specifically for   a <unk> of               rules governing exports of   machine tools computers and  other high-technology products </s> 
Pred: are  to buy   the          <unk> a <unk> recapitalization the   </s>      the     </s> the     tools </s>      </s> to    crops           </s>     </s> 
tensor(0.0050) tensor(0.0600)
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 697.6
Targ: corp. and <unk> corp. the successor company to   <unk>       hotels </s> <unk> officials could n't be located </s> financial corp.    
Pred: </s>  and <unk> <unk> a   <unk>     of      </s> concentrate the    </s> the   <unk>     said  n't be reached in   <unk>     services 
tensor(0.0075) tensor(0.0200)
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 706.0
Targ: portions of kansas he   said </s> the soviet  union has n't  given any clear indication of its wheat purchase plans 
Pred: </s>     of the    </s> said </s> the company union has been yet   the <unk> violation  of the <unk> contract of    
tensor(0.0075) tensor(0.0300)
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 667.2
Targ: to shareholders </s> but otherwise it           would undoubtedly come back with an offer by management </s> the executive said any 
Pred: is be           </s> the the       developments 's    n't         the  to   to   a  <unk> to the        to   the company   said the 
tensor(0.0075) tensor(0.0400)
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 684.6
Targ: a share up   from the year-earlier $      N million or N cents a share </s> revenue rose to $ N 
Pred: a share </s> N    $   year-earlier period N million or N cents a share </s> revenue rose N  $ N 
tensor(0.0075) tensor(0.0500)
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 691.4
Targ: the N        period which was helped by increased ad     spending from the summer olympics </s> while usa today 's total 
Pred: the previous N      </s>  was N      by the       demand revenue  in   the end    of       </s> the   the 's    's <unk> 
tensor(0.0075) tensor(0.0600)
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 688.1
Targ: reason is   mounting competition from new japanese car  plants in   the u.s. that are     pouring out  more than  one million 
Pred: year   </s> the      a           </s> the york     </s> </s>   </s> the u.s. </s> country <unk>   </s> of   <unk> N   </s>    
tensor(0.0100) tensor(0.0200)
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 681.6
Targ: added they hope to have more information early this week </s> investment canada  declined to comment on the reasons for  
Pred: the   that were to be   to   of          about </s> year </s> the        bankers 's       to comment on the new     </s> 
tensor(0.0100) tensor(0.0300)
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 712.3
Targ: new   <unk> <unk> </s> the drug  introduced     last year is  expected to generate sales of about $ N million this 
Pred: <unk> <unk> </s>  </s> the <unk> administration by   year the expected to be       more  of $     $ N million </s> 
tensor(0.0100) tensor(0.0400)
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 721.0
Targ: cents a    share on   sales of $ N million </s> the bronx   has  a wonderful <unk> garden a     great <unk> 
Pred: </s>  </s> share </s> sales of $ N million </s> the company n.y. N <unk>     to    in     <unk> <unk> deal  
tensor(0.0100) tensor(0.0500)
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 703.2
Targ: has   avoided all that by  living in    a   long  island  suburb with his wife  who  's    so    <unk> to   soap 
Pred: <unk> said    the of   the the    <unk> the <unk> history </s>   </s> the <unk> </s> <unk> <unk> <unk> </s> the  

In [42]:
plt.plot(kn5_pcts, ppls)
plt.xlabel("PPL")
plt.ylabel("KN5 pct")

idx = np.argmin(ppls)
print("Best values", 'kn5', kn5_pcts[idx])


Best values kn5 tensor(0.0100)