In [38]:
import os,sys,inspect
currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(currentdir)
sys.path.insert(0,parentdir)
from importlib import reload
import torch
import fasttext
from ptb import lang_util
import torch
from torch.nn import CrossEntropyLoss, NLLLoss
import numpy as np
import matplotlib.pyplot as plt
import numpy as np
In [2]:
NTA = '/Users/jgordon/nta'
In [3]:
corpus = lang_util.Corpus(NTA + '/datasets/PTB')
In [4]:
kn5 = np.load(NTA + '/datasets/PTB/KN5/kn5D_predictions_82430-089-w.npy')
In [19]:
word2id = {}
remap = np.zeros(10000, dtype=int)
with open(NTA + '/datasets/PTB/KN5/ptb_dense_inc_mar8.txt') as f:
i = 0
line = f.readline()
line = f.readline() # Skip header
while line:
token = line.split()[0]
if token == '<end>':
token = '</s>'
word2id[token] = i
remap[corpus.dictionary.word2idx[token]] = i
i += 1
line = f.readline()
In [31]:
kn5_tens = torch.tensor(kn5)
distrs = (kn5_tens / kn5_tens.sum(1, keepdim=True)).float()
kn5_remapped = distrs[:, torch.LongTensor(remap)]
torch.save(kn5_remapped, NTA + '/datasets/PTB/KN5/kn5_distr_remapped.pt')
In [10]:
# criterion = CrossEntropyLoss(reduction='mean')
criterion = NLLLoss(reduction='mean')
target = []
for id in corpus.test:
mapped_id = word2id[corpus.dictionary.idx2word[id]]
target.append(mapped_id)
loss = criterion(distrs.log(), torch.LongTensor(target))
total_loss = loss.item()
print("Loss: %.3f, PPL: %s" % (total_loss, lang_util.perpl(total_loss)))
Loss: 4.966, PPL: 143.41069460658457
In [21]:
criterion = NLLLoss(reduction='mean')
loss = criterion(kn5_remapped.log(), torch.LongTensor(corpus.test))
total_loss = loss.item()
print("Loss: %.3f, PPL: %s" % (total_loss, lang_util.perpl(total_loss)))
Loss: 4.966, PPL: 143.41069460658457
In [16]:
a = torch.rand(2, 3)
print(a)
b = a[:, torch.LongTensor([0, 2, 1])]
print(b)
tensor([[0.1048, 0.1051, 0.5019],
[0.1935, 0.7691, 0.6780]])
tensor([[0.1048, 0.5019, 0.1051],
[0.1935, 0.6780, 0.7691]])
In [32]:
import rsm_samplers
import rsm
from ptb import lang_util
import rsm_experiment
reload(rsm_samplers)
reload(rsm)
reload(lang_util)
reload(rsm_experiment)
CONFIG = {
'debug': False,
'path': "/Users/jgordon/nta/results",
'data_dir': "/Users/jgordon/nta/datasets",
'dataset': 'ptb',
'predictor_hidden_size': 1200,
'predictor_output_size': 10000,
'batch_size': 300,
'batches_in_epoch': 2000,
'batches_in_first_epoch': 2,
'batch_log_interval': 2000,
'vocab_size': 10000,
'eval_interval': 3,
'eval_batch_size': 100,
'eval_batches_in_epoch': 824, #8243,
'learning_rate': 0.0005,
'm_groups': 1500,
'n_cells_per_group': 1,
'k_winners': 80,
'k_winner_cells': 1,
'pred_l2_reg': 0.000001,
'dec_l2_reg': 0.000001,
'input_bias': True,
'eval_interval': 5,
'eps': 0.5,
'gamma': 0.0,
'forget_mu': 0.025,
'weight_sparsity': None,
'mult_integration': False,
'fpartition': None,
'boost_strength': 0.5,
'boost_strength_factor': 0.85,
'boost_strat': 'col_boosting',
'do_inhibition': False,
'x_b_norm': True,
'balance_part_winners': True,
'decode_activation_fn': None,
'decode_bias': False,
'embed_dim': 100,
'input_size': (1, 100),
'output_size': 100,
'embedding_kind': 'ptb_fasttext_e5',
'max_decay': 0.95,
'mem_floor': 0.0005,
'trainable_decay': True,
'word_cache_decay': 0.99,
'kn5_pct': 0.1
}
exp = rsm_experiment.RSMExperiment(config=CONFIG)
In [33]:
exp.model_setup(CONFIG, restore_path="/Users/jgordon/Desktop/rsm_highlights/PTB/RSMTune_0_2019-09-20_21-15-18mchjovgl/checkpoint_112")
setup: Using cpu
Maybe download PTB...
Loaded embedding dict (ptb_fasttext_e5) with 10000 entries
Built dataloaders...
Loading from /Users/jgordon/Desktop/rsm_highlights/PTB/RSMTune_0_2019-09-20_21-15-18mchjovgl/checkpoint_112
In [13]:
wcs = []
uss = []
ppls = []
WC_DECAY = 0.98
for wc in torch.arange(0.04, .11, 0.01):
for us in torch.arange(0.001, 0.02, 0.002):
print("wc: %s, us: %s" % (wc.item(), us.item()))
exp.word_cache_pct = wc.item()
exp.unif_smoothing = us.item()
exp.word_cache_decay = WC_DECAY
ret = exp.eval_epoch(0)
int_ppl = ret['val_interp_ppl']
ppls.append(int_ppl)
wcs.append(wc.item())
uss.append(us.item())
print(ret)
wc: 0.03999999910593033, us: 0.0010000000474974513
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 9.000%, pred ppl: 743.5
Targ: it was n't black monday </s> but while the new york stock exchange did n't fall apart friday as the
Pred: one is n't expected </s> </s> the the the <unk> crowd times exchange is n't <unk> in </s> </s> the
{'val_loss': 0.039184603166486164, 'val_interp_ppl': 107.06121567910887, 'val_pred_ppl': 117.15777761373491, 'val_pred_acc': 24.29368932038835}
wc: 0.03999999910593033, us: 0.003000000026077032
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 10.000%, pred ppl: 670.9
Targ: pit in chicago waves of selling continued to hit stocks themselves on the big board and specialists continued to <unk>
Pred: </s> </s> N </s> </s> the the to the the </s> </s> the new board </s> the in to the
{'val_loss': 0.039181382494788725, 'val_interp_ppl': 107.03257124937855, 'val_pred_ppl': 117.17197209145539, 'val_pred_acc': 24.29247572815534}
wc: 0.03999999910593033, us: 0.004999999888241291
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 10.000%, pred ppl: 605.2
Targ: friday will undoubtedly cause renewed debate about whether wall street is properly prepared for another crash situation </s> the big
Pred: </s> </s> be be a volatility </s> the the street 's <unk> to to the <unk> </s> </s> the rumor
{'val_loss': 0.03918048901510542, 'val_interp_ppl': 106.97822837013234, 'val_pred_ppl': 117.15409308632434, 'val_pred_acc': 24.29368932038835}
wc: 0.03999999910593033, us: 0.007000000216066837
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 10.000%, pred ppl: 602.4
Targ: corp. was acquired by australian developer george <unk> currently hooker 's chairman </s> mr. <unk> <unk> to launch an ambitious
Pred: </s> and elected by <unk> entrepreneur and gillett </s> chairman </s> chairman and mr. <unk> said will succeed a offer
{'val_loss': 0.03918117782082951, 'val_interp_ppl': 106.93318175890992, 'val_pred_ppl': 117.11651277100279, 'val_pred_acc': 24.29854368932039}
wc: 0.03999999910593033, us: 0.008999999612569809
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 10.000%, pred ppl: 590.0
Targ: said </s> a major reason is that investors already have sharply scaled back their purchases of stock funds since black
Pred: </s> </s> the spokesman factor for that the will are a in in to <unk> </s> the in </s> the
{'val_loss': 0.039179578929850196, 'val_interp_ppl': 106.93947710466206, 'val_pred_ppl': 117.12931525572999, 'val_pred_acc': 24.300970873786408}
wc: 0.03999999910593033, us: 0.010999999940395355
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 10.000%, pred ppl: 638.8
Targ: peter <unk> vice president for planning at the phoenix ariz. carrier said in an interview that the work <unk> at
Pred: in <unk> N president of <unk> and <unk> <unk> company investment <unk> the a interview that it company force will
{'val_loss': 0.03917774543600175, 'val_interp_ppl': 106.90701038718956, 'val_pred_ppl': 117.10076544070915, 'val_pred_acc': 24.30218446601942}
wc: 0.03999999910593033, us: 0.013000000268220901
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 10.000%, pred ppl: 671.6
Targ: and the workers ' <unk> with being forced to work many hours overtime </s> in separate developments talks have broken
Pred: the <unk> <unk> ' <unk> of the <unk> to subscribe </s> of </s> </s> the addition cases of with been
{'val_loss': 0.0391835036407252, 'val_interp_ppl': 107.00519472066837, 'val_pred_ppl': 117.20742374700646, 'val_pred_acc': 24.29247572815534}
wc: 0.03999999910593033, us: 0.014999999664723873
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 10.000%, pred ppl: 679.0
Targ: to close at $ N in new york stock exchange composite trading </s> <unk> & broad home corp. said it
Pred: </s> N at N N a new york stock exchange composite trading </s> the closed <unk> industries delivery jumped it
{'val_loss': 0.03918286766043132, 'val_interp_ppl': 107.03413567202082, 'val_pred_ppl': 117.20362080545257, 'val_pred_acc': 24.29247572815534}
wc: 0.03999999910593033, us: 0.017000000923871994
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 10.000%, pred ppl: 700.1
Targ: pound last spring have skidded to between N cents and N cents </s> meanwhile the price of <unk> the chemical
Pred: year </s> year </s> been </s> N N N a N cents a the the <unk> index the <unk> largest
{'val_loss': 0.03918270423818488, 'val_interp_ppl': 107.02325547010985, 'val_pred_ppl': 117.16524768837635, 'val_pred_acc': 24.29733009708738}
wc: 0.03999999910593033, us: 0.01899999938905239
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 10.000%, pred ppl: 684.2
Targ: </s> <unk> is the second-largest propane distributor in the u.s. </s> the largest suburban propane was already owned by quantum
Pred: yen the s.a. a most maker to maker the u.s. and the u.s. producer maker <unk> N <unk> by <unk>
{'val_loss': 0.03918260109420974, 'val_interp_ppl': 107.085759140277, 'val_pred_ppl': 117.21620906627719, 'val_pred_acc': 24.29368932038835}
wc: 0.05000000074505806, us: 0.0010000000474974513
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 10.000%, pred ppl: 730.3
Targ: has had in recent years </s> the ec and japan the u.s. 's largest steel suppliers have n't been filling
Pred: is n't a the months </s> the company is the 's u.s. and largest securities segment and been been <unk>
{'val_loss': 0.0391839054363409, 'val_interp_ppl': 106.77574311842667, 'val_pred_ppl': 117.19227435366209, 'val_pred_acc': 24.29733009708738}
wc: 0.05000000074505806, us: 0.003000000026077032
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 10.000%, pred ppl: 758.1
Targ: clobbered two years ago in japan when <unk> introduced a powerful detergent called attack which quickly won a N N
Pred: a </s> years ago </s> the </s> the <unk> a <unk> <unk> for <unk> on has </s> the <unk> N
{'val_loss': 0.039184753559258524, 'val_interp_ppl': 106.70573896775164, 'val_pred_ppl': 117.1757804722615, 'val_pred_acc': 24.299757281553397}
wc: 0.05000000074505806, us: 0.004999999888241291
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 11.000%, pred ppl: 679.4
Targ: and mrs. hills </s> many called it simply a contrast in styles </s> but some saw it as a classic
Pred: </s> <unk> <unk> </s> the of the is <unk> <unk> to the and the the of the 's a <unk>
{'val_loss': 0.039182307924261514, 'val_interp_ppl': 106.6038009345696, 'val_pred_ppl': 117.09304448219557, 'val_pred_acc': 24.303398058252426}
wc: 0.05000000074505806, us: 0.007000000216066837
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 670.6
Targ: small subsidiary that is <unk> unrelated becomes a difficult <unk> said <unk> <unk> president of the parent in a statement
Pred: year portion of has <unk> </s> to <unk> <unk> time </s> <unk> <unk> a of the <unk> of <unk> <unk>
{'val_loss': 0.03918254861011378, 'val_interp_ppl': 106.66099844858812, 'val_pred_ppl': 117.17618762456044, 'val_pred_acc': 24.296116504854368}
wc: 0.05000000074505806, us: 0.008999999612569809
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 705.6
Targ: plan to press specifically for a <unk> of rules governing exports of machine tools computers and other high-technology products </s>
Pred: are to buy the <unk> a <unk> recapitalization the </s> the </s> the tools </s> </s> to crops </s> </s>
{'val_loss': 0.039182619234655526, 'val_interp_ppl': 106.64609249490431, 'val_pred_ppl': 117.16795515152494, 'val_pred_acc': 24.30218446601942}
wc: 0.05000000074505806, us: 0.010999999940395355
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 697.6
Targ: corp. and <unk> corp. the successor company to <unk> hotels </s> <unk> officials could n't be located </s> financial corp.
Pred: </s> and <unk> <unk> a <unk> of </s> concentrate the </s> the <unk> said n't be reached in <unk> services
{'val_loss': 0.03918287246397615, 'val_interp_ppl': 106.68242691052995, 'val_pred_ppl': 117.20745374250441, 'val_pred_acc': 24.296116504854368}
wc: 0.05000000074505806, us: 0.013000000268220901
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 706.0
Targ: portions of kansas he said </s> the soviet union has n't given any clear indication of its wheat purchase plans
Pred: </s> of the </s> said </s> the company union has been yet the <unk> violation of the <unk> contract of
{'val_loss': 0.03918740623947052, 'val_interp_ppl': 106.70510302303907, 'val_pred_ppl': 117.23230832351102, 'val_pred_acc': 24.29004854368932}
wc: 0.05000000074505806, us: 0.014999999664723873
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 667.2
Targ: to shareholders </s> but otherwise it would undoubtedly come back with an offer by management </s> the executive said any
Pred: is be </s> the the developments 's n't the to to a <unk> to the to the company said the
{'val_loss': 0.03918458810709055, 'val_interp_ppl': 106.66025010910495, 'val_pred_ppl': 117.16310972496667, 'val_pred_acc': 24.29490291262136}
wc: 0.05000000074505806, us: 0.017000000923871994
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 684.6
Targ: a share up from the year-earlier $ N million or N cents a share </s> revenue rose to $ N
Pred: a share </s> N $ year-earlier period N million or N cents a share </s> revenue rose N $ N
{'val_loss': 0.03918481884677939, 'val_interp_ppl': 106.76326092241924, 'val_pred_ppl': 117.25518704101735, 'val_pred_acc': 24.28276699029126}
wc: 0.05000000074505806, us: 0.01899999938905239
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 691.4
Targ: the N period which was helped by increased ad spending from the summer olympics </s> while usa today 's total
Pred: the previous N </s> was N by the demand revenue in the end of </s> the the 's 's <unk>
{'val_loss': 0.039184963224383376, 'val_interp_ppl': 106.77138985699189, 'val_pred_ppl': 117.22897364872006, 'val_pred_acc': 24.283980582524272}
wc: 0.05999999865889549, us: 0.0010000000474974513
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 688.1
Targ: reason is mounting competition from new japanese car plants in the u.s. that are pouring out more than one million
Pred: year </s> the a </s> the york </s> </s> </s> the u.s. </s> country <unk> </s> of <unk> N </s>
{'val_loss': 0.03918688932379472, 'val_interp_ppl': 106.61021614923762, 'val_pred_ppl': 117.19555142853538, 'val_pred_acc': 24.28276699029126}
wc: 0.05999999865889549, us: 0.003000000026077032
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 681.6
Targ: added they hope to have more information early this week </s> investment canada declined to comment on the reasons for
Pred: the that were to be to of about </s> year </s> the bankers 's to comment on the new </s>
{'val_loss': 0.03918338612680441, 'val_interp_ppl': 106.51594057378325, 'val_pred_ppl': 117.12864473467215, 'val_pred_acc': 24.288834951456312}
wc: 0.05999999865889549, us: 0.004999999888241291
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 712.3
Targ: new <unk> <unk> </s> the drug introduced last year is expected to generate sales of about $ N million this
Pred: <unk> <unk> </s> </s> the <unk> administration by year the expected to be more of $ $ N million </s>
{'val_loss': 0.03918634114099793, 'val_interp_ppl': 106.57684815145734, 'val_pred_ppl': 117.24959292816658, 'val_pred_acc': 24.280339805825243}
wc: 0.05999999865889549, us: 0.007000000216066837
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 721.0
Targ: cents a share on sales of $ N million </s> the bronx has a wonderful <unk> garden a great <unk>
Pred: </s> </s> share </s> sales of $ N million </s> the company n.y. N <unk> to in <unk> <unk> deal
{'val_loss': 0.03918720897088207, 'val_interp_ppl': 106.57865561495933, 'val_pred_ppl': 117.28050777665858, 'val_pred_acc': 24.281553398058254}
wc: 0.05999999865889549, us: 0.008999999612569809
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 703.2
Targ: has avoided all that by living in a long island suburb with his wife who 's so <unk> to soap
Pred: <unk> said the of the the <unk> the <unk> history </s> </s> the <unk> </s> <unk> <unk> <unk> </s> the
{'val_loss': 0.03918464269796477, 'val_interp_ppl': 106.42493939170598, 'val_pred_ppl': 117.11068737304173, 'val_pred_acc': 24.29490291262136}
wc: 0.05999999865889549, us: 0.010999999940395355
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 707.2
Targ: real estate <unk> mr. crandall <unk> broadly and said no comment </s> on friday morning before the market 's sell-off
Pred: times estate developer </s> <unk> said </s> </s> the the one on the the the the the resignations closed plunge
{'val_loss': 0.039180946780495274, 'val_interp_ppl': 106.51697493506595, 'val_pred_ppl': 117.20762021548725, 'val_pred_acc': 24.29368932038835}
wc: 0.05999999865889549, us: 0.013000000268220901
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 683.2
Targ: traded over-the-counter </s> columbia laboratories inc. miami began trading with the symbol <unk> </s> the pharmaceuticals maker had traded over-the-counter
Pred: a as market the pictures inc. said n.j. a in a new of <unk> the <unk> company of a <unk>
{'val_loss': 0.03918161269874104, 'val_interp_ppl': 106.46627910821248, 'val_pred_ppl': 117.13570554828583, 'val_pred_acc': 24.296116504854368}
wc: 0.05999999865889549, us: 0.014999999664723873
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 625.3
Targ: a legal <unk> that they inherited says prof. <unk> dick howard of the university of virginia law school but the
Pred: </s> year challenge </s> would can the </s> <unk> <unk> <unk> <unk> the <unk> of california school </s> of it
{'val_loss': 0.039181437438299646, 'val_interp_ppl': 106.45976629548419, 'val_pred_ppl': 117.12289668263143, 'val_pred_acc': 24.29004854368932}
wc: 0.05999999865889549, us: 0.017000000923871994
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 668.4
Targ: year increasing its subscription rates and cutting back on merchandise <unk> </s> in an announcement to its staff last week
Pred: quarter </s> the N of </s> a the to the </s> </s> the addition interview that the <unk> to week
{'val_loss': 0.03918245550046123, 'val_interp_ppl': 106.49164453347957, 'val_pred_ppl': 117.14798206906576, 'val_pred_acc': 24.29368932038835}
wc: 0.05999999865889549, us: 0.01899999938905239
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 659.3
Targ: <unk> by bernard shaw and <unk> <unk> a <unk> former texas judge and campus beauty queen who has never held
Pred: sold by the <unk> </s> <unk> <unk> </s> <unk> firm <unk> <unk> <unk> <unk> <unk> </s> <unk> <unk> been bothered
{'val_loss': 0.03918375370083648, 'val_interp_ppl': 106.51504804089438, 'val_pred_ppl': 117.16378465241245, 'val_pred_acc': 24.29126213592233}
wc: 0.07000000029802322, us: 0.0010000000474974513
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 686.7
Targ: producer with <unk> <unk> who notes that <unk> is <unk> to his job </s> the network 's salaries have always
Pred: vice of <unk> </s> </s> is </s> the <unk> the </s> the <unk> </s> the <unk> was <unk> of been
{'val_loss': 0.039184678892957646, 'val_interp_ppl': 106.55898855023746, 'val_pred_ppl': 117.20649041769856, 'val_pred_acc': 24.288834951456312}
wc: 0.07000000029802322, us: 0.003000000026077032
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 13.000%, pred ppl: 630.0
Targ: lacked enough financial information about <unk> and the <unk> and sent the cases back to federal district court in dallas
Pred: is up to planners to the </s> <unk> <unk> </s> <unk> to market of to the funds </s> </s> new
{'val_loss': 0.039184212449540214, 'val_interp_ppl': 106.4742828635311, 'val_pred_ppl': 117.16290708260026, 'val_pred_acc': 24.29368932038835}
wc: 0.07000000029802322, us: 0.004999999888241291
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 13.000%, pred ppl: 586.1
Targ: <unk> 's civil war </s> details of the talks described by a <unk> official as very delicate were n't disclosed
Pred: of </s> <unk> war </s> the of the <unk> </s> the the <unk> <unk> </s> a <unk> </s> <unk> <unk>
{'val_loss': 0.03918165288124125, 'val_interp_ppl': 106.45266407108932, 'val_pred_ppl': 117.17384220022109, 'val_pred_acc': 24.296116504854368}
wc: 0.07000000029802322, us: 0.007000000216066837
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 13.000%, pred ppl: 622.3
Targ: in world oil prices sent the index surging at double-digit annual rates </s> energy prices then plummeted through the summer
Pred: </s> the markets </s> </s> a colony of a N </s> rates for the prices rose in N the <unk>
{'val_loss': 0.03918007258054556, 'val_interp_ppl': 106.40638887537415, 'val_pred_ppl': 117.13388819283904, 'val_pred_acc': 24.29733009708738}
wc: 0.07000000029802322, us: 0.008999999612569809
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 13.000%, pred ppl: 665.8
Targ: suspend dividend payments on its two classes of preferred stock indicating that regulators ' concerns about the troubled institution have
Pred: N the N </s> the <unk> million </s> $ stock </s> a the ' syndicate about the company company </s>
{'val_loss': 0.03917870972106613, 'val_interp_ppl': 106.39840712870219, 'val_pred_ppl': 117.13279515552934, 'val_pred_acc': 24.29733009708738}
wc: 0.07000000029802322, us: 0.010999999940395355
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 13.000%, pred ppl: 611.3
Targ: while <unk> </s> the woman won the bet </s> but perhaps even more remarkable the <unk> <unk> <unk> make a
Pred: </s> the </s> the <unk> 's the <unk> <unk> the the the though than </s> <unk> of of <unk> it
{'val_loss': 0.03918462630940005, 'val_interp_ppl': 106.38159759290609, 'val_pred_ppl': 117.13772045458028, 'val_pred_acc': 24.29733009708738}
wc: 0.07000000029802322, us: 0.013000000268220901
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 13.000%, pred ppl: 599.1
Targ: is n't much choice </s> thus europe has begun the recent crusade to produce more worthy shows of its own
Pred: is a any less </s> the in and been to <unk> <unk> of be a than than the the <unk>
{'val_loss': 0.03918427850562826, 'val_interp_ppl': 106.38822705994346, 'val_pred_ppl': 117.1375044941851, 'val_pred_acc': 24.29247572815534}
wc: 0.07000000029802322, us: 0.014999999664723873
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 662.9
Targ: fight with <unk> for ferranti competition is the name of the game she said </s> at least one potential <unk>
Pred: year </s> the <unk> the and </s> <unk> first of the <unk> </s> says </s> the the the of customers
{'val_loss': 0.03918483641758127, 'val_interp_ppl': 106.4121099697223, 'val_pred_ppl': 117.13528157424732, 'val_pred_acc': 24.29368932038835}
wc: 0.07000000029802322, us: 0.017000000923871994
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 663.0
Targ: steel industries inc. expects to report that third-quarter earnings dropped more than N N from the previous quarter as a
Pred: 's </s> </s> and to post a third-quarter net for N than N N </s> the previous year </s> a
{'val_loss': 0.03918287033233249, 'val_interp_ppl': 106.48625848828334, 'val_pred_ppl': 117.20206887056716, 'val_pred_acc': 24.288834951456312}
wc: 0.07000000029802322, us: 0.01899999938905239
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 640.4
Targ: trouble for software firms generally </s> it creates uncertainty and usually <unk> down sales said <unk> <unk> an analyst at
Pred: </s> </s> the </s> </s> have the 's a about <unk> do to the </s> mr. <unk> an analyst with
{'val_loss': 0.039180360517455534, 'val_interp_ppl': 106.46303349516859, 'val_pred_ppl': 117.15682506744437, 'val_pred_acc': 24.29368932038835}
wc: 0.07999999821186066, us: 0.0010000000474974513
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 629.7
Targ: in N N N of those aged N to N lived with <unk> other than spouses down from N N
Pred: in addition the years one the surveyed N days N N in N <unk> <unk> N </s> from N N
{'val_loss': 0.03918534330119496, 'val_interp_ppl': 106.62139605148384, 'val_pred_ppl': 117.20900679051941, 'val_pred_acc': 24.2876213592233}
wc: 0.07999999821186066, us: 0.003000000026077032
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 11.000%, pred ppl: 655.4
Targ: and the cheap mixed drinks go for $ N a pop </s> at the <unk> manager elizabeth <unk> wo n't
Pred: </s> N <unk> </s> </s> </s> </s> the N </s> pound </s> the the end <unk> 's <unk> the n't
{'val_loss': 0.03918532447129922, 'val_interp_ppl': 106.47888546782089, 'val_pred_ppl': 117.13212591597828, 'val_pred_acc': 24.29004854368932}
wc: 0.07999999821186066, us: 0.004999999888241291
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 11.000%, pred ppl: 663.7
Targ: elsewhere <unk> all critical mention of group differences </s> as elizabeth <unk> wrote in the new york times just before
Pred: N </s> </s> the <unk> </s> the </s> </s> the a <unk> a a the u.s. york city co. a
{'val_loss': 0.039187726948988956, 'val_interp_ppl': 106.50648665364966, 'val_pred_ppl': 117.19530042103014, 'val_pred_acc': 24.2876213592233}
wc: 0.07999999821186066, us: 0.007000000216066837
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 615.0
Targ: eric w. <unk> resigned in june </s> the senate 's decision to approve a <unk> deficit-reduction bill without a capital-gains
Pred: vice c. <unk> </s> </s> the </s> mr. company passed board to approve the <unk> amendment bill that the bill
{'val_loss': 0.03918255281010733, 'val_interp_ppl': 106.53722645425258, 'val_pred_ppl': 117.24737652242443, 'val_pred_acc': 24.283980582524272}
wc: 0.07999999821186066, us: 0.008999999612569809
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 564.4
Targ: things had just gone too far </s> sen. dole said that the move required sacrifice by every senator </s> it
Pred: in are been been at much </s> the <unk> <unk> the the <unk> is to to the N <unk> the
{'val_loss': 0.03918188590854768, 'val_interp_ppl': 106.47308704315931, 'val_pred_ppl': 117.18046892835132, 'val_pred_acc': 24.28640776699029}
wc: 0.07999999821186066, us: 0.010999999940395355
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 13.000%, pred ppl: 515.1
Targ: the new census bureau report <unk> from N to N are out of date certainly as an average for the
Pred: N end york bureau </s> </s> </s> the N N </s> N of the </s> </s> a <unk> of the
{'val_loss': 0.03918131102708358, 'val_interp_ppl': 106.41892661952862, 'val_pred_ppl': 117.11155214695155, 'val_pred_acc': 24.288834951456312}
wc: 0.07999999821186066, us: 0.013000000268220901
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 13.000%, pred ppl: 531.2
Targ: public enthusiasm for stock mutual funds </s> the main thing was portfolio insurance a mechanical trading system intended to protect
Pred: </s> </s> </s> the market funds </s> the <unk> reason is a the and money-market investment firm is to <unk>
{'val_loss': 0.039182643767771794, 'val_interp_ppl': 106.44453681318785, 'val_pred_ppl': 117.14041808421845, 'val_pred_acc': 24.288834951456312}
wc: 0.07999999821186066, us: 0.014999999664723873
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 13.000%, pred ppl: 511.2
Targ: do well during periods of economic weakness </s> frank <unk> </s> many people now claim to have predicted the N
Pred: </s> n't </s> the </s> the growth </s> the <unk> an <unk> of are are the be to that <unk>
{'val_loss': 0.039180965985632636, 'val_interp_ppl': 106.44828641047944, 'val_pred_ppl': 117.12969639207364, 'val_pred_acc': 24.29368932038835}
wc: 0.07999999821186066, us: 0.017000000923871994
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 13.000%, pred ppl: 559.0
Targ: generate </s> the last time i saw a similar congressional hearing was when <unk> <unk> joe <unk> did his work
Pred: </s> </s> the <unk> thing westinghouse was the <unk> <unk> hearing is to the with <unk> <unk> the n't <unk>
{'val_loss': 0.03918333676670795, 'val_interp_ppl': 106.45674218567169, 'val_pred_ppl': 117.11403208571853, 'val_pred_acc': 24.29368932038835}
wc: 0.07999999821186066, us: 0.01899999938905239
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 13.000%, pred ppl: 556.8
Targ: the american stock exchange composite fell N N </s> on oct. N N the nasdaq composite fell N points or
Pred: N <unk> stock exchange </s> trading N cents to the the N </s> of <unk> composite index N to to
{'val_loss': 0.03918236835850674, 'val_interp_ppl': 106.58901358652466, 'val_pred_ppl': 117.23424526149532, 'val_pred_acc': 24.281553398058254}
wc: 0.09000000357627869, us: 0.0010000000474974513
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 14.000%, pred ppl: 549.4
Targ: blocks of N shares or more two years ago </s> shearson 's mr. dapuzzo said retail investors nervously sold stock
Pred: the of <unk> and </s> more </s> years </s> </s> the lehman mr. <unk> said the sales are have off
{'val_loss': 0.03918034621306415, 'val_interp_ppl': 106.76469681414598, 'val_pred_ppl': 117.22176789697443, 'val_pred_acc': 24.28276699029126}
wc: 0.09000000357627869, us: 0.003000000026077032
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 14.000%, pred ppl: 537.2
Targ: month N N N three months N N N six months N N N one year </s> the average of
Pred: year N N N to months N N N to months N N N to year </s> the average of
{'val_loss': 0.03918855270208085, 'val_interp_ppl': 106.83501218327437, 'val_pred_ppl': 117.35458443940217, 'val_pred_acc': 24.264563106796118}
wc: 0.09000000357627869, us: 0.004999999888241291
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 14.000%, pred ppl: 562.5
Targ: and the <unk> of computerized program trading </s> the only thing you do n't have he said is the portfolio
Pred: </s> other <unk> of the trading trading </s> the big way is 're n't want to said </s> n't market
{'val_loss': 0.03918295930528525, 'val_interp_ppl': 106.66562267416963, 'val_pred_ppl': 117.21013709007252, 'val_pred_acc': 24.29126213592233}
wc: 0.09000000357627869, us: 0.007000000216066837
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 14.000%, pred ppl: 570.8
Targ: republican but on friday morning he had few kind words for president bush 's economic <unk> </s> there are some
Pred: <unk> <unk> the the </s> </s> said been days of </s> the bush to <unk> reforms </s> the is no
{'val_loss': 0.039182112199588885, 'val_interp_ppl': 106.60695588274939, 'val_pred_ppl': 117.16593165397586, 'val_pred_acc': 24.29247572815534}
wc: 0.09000000357627869, us: 0.008999999612569809
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 15.000%, pred ppl: 520.2
Targ: angry over mr. bush 's claim that the capital-gains cut was part of april 's budget accord and his insistence
Pred: <unk> </s> the bernstein 's <unk> </s> he <unk> tax is n't of a N request </s> </s> the plan
{'val_loss': 0.039182443474645466, 'val_interp_ppl': 106.58714444450965, 'val_pred_ppl': 117.15980348714949, 'val_pred_acc': 24.2876213592233}
wc: 0.09000000357627869, us: 0.010999999940395355
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 15.000%, pred ppl: 486.4
Targ: funding group </s> pennsylvania higher education facilities authority </s> $ N million of revenue bonds for <unk> university series N
Pred: funding corp. </s> the 's interest services financing $ the N million of N bonds series the and bonds N
{'val_loss': 0.03917921902792691, 'val_interp_ppl': 106.70642720077983, 'val_pred_ppl': 117.29830142109091, 'val_pred_acc': 24.277912621359224}
wc: 0.09000000357627869, us: 0.013000000268220901
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 14.000%, pred ppl: 499.5
Targ: following black monday but mostly through mutual funds </s> discount brokerage customers have been in the market somewhat but not
Pred: value the & </s> the in the funds </s> the yields firms were been <unk> recent past </s> </s> the
{'val_loss': 0.03918539384352857, 'val_interp_ppl': 106.62958199089667, 'val_pred_ppl': 117.19098823684222, 'val_pred_acc': 24.29004854368932}
wc: 0.09000000357627869, us: 0.014999999664723873
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 15.000%, pred ppl: 459.4
Targ: that demand however has led to a variety of <unk> </s> making computers smaller often means <unk> memory </s> it
Pred: N the for </s> been the <unk> <unk> of the </s> the the and than have that </s> chips the
{'val_loss': 0.039183836830418234, 'val_interp_ppl': 106.65394769832443, 'val_pred_ppl': 117.20108692523479, 'val_pred_acc': 24.288834951456312}
wc: 0.09000000357627869, us: 0.017000000923871994
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 15.000%, pred ppl: 441.6
Targ: risen N N in the first nine months of the year <unk> both the initial N N inflation goal set
Pred: been </s> N to the first half months of this year </s> </s> the <unk> <unk> N increase rate of
{'val_loss': 0.03918265680628784, 'val_interp_ppl': 106.73686643838975, 'val_pred_ppl': 117.2763979450769, 'val_pred_acc': 24.28276699029126}
wc: 0.09000000357627869, us: 0.01899999938905239
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 15.000%, pred ppl: 432.1
Targ: to staying involved in these programs in which <unk> earn <unk> miles and <unk> can get <unk> discounts </s> i
Pred: of the out in the parts </s> the they </s> </s> </s> </s> <unk> </s> be a </s> </s> the
{'val_loss': 0.03918120817471187, 'val_interp_ppl': 106.66174987456606, 'val_pred_ppl': 117.16552054512954, 'val_pred_acc': 24.288834951456312}
wc: 0.10000000149011612, us: 0.0010000000474974513
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 15.000%, pred ppl: 446.7
Targ: which analysts say are roughly three times the price of a comparable system from tandem </s> obviously ibm can give
Pred: </s> is say </s> n't N times as market of N N N </s> the </s> the the 's be
{'val_loss': 0.03918659020197189, 'val_interp_ppl': 107.00855284957692, 'val_pred_ppl': 117.25612666415459, 'val_pred_acc': 24.274271844660195}
wc: 0.10000000149011612, us: 0.003000000026077032
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 15.000%, pred ppl: 457.8
Targ: <unk> of <unk> group a <unk> <unk> pa. forecasting company </s> a lot of <unk> demand is gone </s> consumer
Pred: </s> </s> the <unk> </s> <unk> firm firm </s> </s> </s> the year of <unk> is for a at <unk>
{'val_loss': 0.03918342698293114, 'val_interp_ppl': 106.94184640103279, 'val_pred_ppl': 117.2446876326732, 'val_pred_acc': 24.277912621359224}
wc: 0.10000000149011612, us: 0.004999999888241291
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 16.000%, pred ppl: 449.8
Targ: <unk> operations </s> still some industry giants are expected to report continuing gains largely because so much of their business
Pred: board </s> </s> the westinghouse traders observers are <unk> to be a growth in because of far of the <unk>
{'val_loss': 0.039184566994098204, 'val_interp_ppl': 106.90379932493839, 'val_pred_ppl': 117.23258107592748, 'val_pred_acc': 24.283980582524272}
wc: 0.10000000149011612, us: 0.007000000216066837
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 16.000%, pred ppl: 478.4
Targ: the market 's afternoon surge </s> among classes for which details were available yields ranged from N N or N
Pred: </s> end closed <unk> session </s> the the of example the of suspended for on from N N in N
{'val_loss': 0.03918376059081514, 'val_interp_ppl': 107.00502649003957, 'val_pred_ppl': 117.36571005585274, 'val_pred_acc': 24.268203883495147}
wc: 0.10000000149011612, us: 0.008999999612569809
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 17.000%, pred ppl: 471.9
Targ: the <unk> to require that all employees give similar notice before they quit </s> <unk> s. <unk> </s> <unk> comment
Pred: </s> <unk> of the the the the ' themselves to </s> the are </s> the <unk> <unk> of the <unk>
{'val_loss': 0.039186707516969406, 'val_interp_ppl': 106.84732619825515, 'val_pred_ppl': 117.1892260557583, 'val_pred_acc': 24.288834951456312}
wc: 0.10000000149011612, us: 0.010999999940395355
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 17.000%, pred ppl: 471.8
Targ: N million and would have been much higher had not the cost of the trading floor set been absorbed in
Pred: N million in N have a expected less than been been <unk> of the company arena </s> up </s> </s>
{'val_loss': 0.03918537224453051, 'val_interp_ppl': 106.90168323058512, 'val_pred_ppl': 117.25438981987762, 'val_pred_acc': 24.28640776699029}
wc: 0.10000000149011612, us: 0.013000000268220901
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 16.000%, pred ppl: 523.5
Targ: the <unk> sale of farmers to axa receives regulatory approval </s> a spokesman for b.a.t said of the amended filings
Pred: the market <unk> is the <unk> the the the capital </s> the spokesman for the 's the the company pact
{'val_loss': 0.03918662398275964, 'val_interp_ppl': 106.9479417949018, 'val_pred_ppl': 117.2961740622999, 'val_pred_acc': 24.281553398058254}
wc: 0.10000000149011612, us: 0.014999999664723873
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 16.000%, pred ppl: 520.3
Targ: and <unk> marine transportation and machinery used to make food and beverage cans </s> it was n't so long ago
Pred: </s> <unk> </s> </s> </s> <unk> </s> in <unk> <unk> and <unk> </s> </s> the also a expected <unk> to
{'val_loss': 0.03918733827552749, 'val_interp_ppl': 106.96319204000281, 'val_pred_ppl': 117.29640152320292, 'val_pred_acc': 24.277912621359224}
wc: 0.10000000149011612, us: 0.017000000923871994
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 16.000%, pred ppl: 548.9
Targ: after N p.m </s> it is a talk show with opposition leaders and political experts who discuss hungary 's domestic
Pred: a the years </s> the 's a <unk> of that a to in <unk> <unk> </s> favor the 's <unk>
{'val_loss': 0.03918718502775413, 'val_interp_ppl': 106.94104400844789, 'val_pred_ppl': 117.24648295465583, 'val_pred_acc': 24.277912621359224}
wc: 0.10000000149011612, us: 0.01899999938905239
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 16.000%, pred ppl: 547.9
Targ: up </s> what better place to turn than sen. edward kennedy 's labor committee that great <unk> of government <unk>
Pred: </s> N the 's than is the the the <unk> markey d. <unk> secretary that the <unk> the the 's
{'val_loss': 0.03918452260708346, 'val_interp_ppl': 106.96401337631389, 'val_pred_ppl': 117.24358166374986, 'val_pred_acc': 24.274271844660195}
wc: 0.10999999940395355, us: 0.0010000000474974513
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 16.000%, pred ppl: 580.6
Targ: that would be down from the N N rise posted in N </s> the canadian government announced a new 12-year
Pred: in 's be a to the N N of in a net and the company dollar reported a N contract
{'val_loss': 0.039187608160150864, 'val_interp_ppl': 107.36065263056908, 'val_pred_ppl': 117.33349762623878, 'val_pred_acc': 24.271844660194176}
wc: 0.10999999940395355, us: 0.003000000026077032
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 17.000%, pred ppl: 555.6
Targ: service would get important work done <unk> forest fires fought housing <unk> students <unk> <unk> centers <unk> </s> there is
Pred: mortgage corp. be a for in </s> </s> products </s> by and </s> </s> and and </s> and the are
{'val_loss': 0.039186348175648054, 'val_interp_ppl': 107.16440374246876, 'val_pred_ppl': 117.16951145151275, 'val_pred_acc': 24.276699029126213}
wc: 0.10999999940395355, us: 0.004999999888241291
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 17.000%, pred ppl: 512.0
Targ: prudent what 's <unk> he said </s> when it was suggested his comment was a <unk> mr. darman replied it
Pred: N </s> it the ratio says </s> the the 's a that first </s> n't <unk> </s> <unk> 's that
{'val_loss': 0.03918610698570615, 'val_interp_ppl': 107.23538868850225, 'val_pred_ppl': 117.28699289039011, 'val_pred_acc': 24.275485436893202}
wc: 0.10999999940395355, us: 0.007000000216066837
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 17.000%, pred ppl: 505.7
Targ: awful bear market of the 1930s began </s> the october <unk> of N and N were scary but did n't
Pred: N </s> </s> </s> the <unk> </s> to in <unk> N <unk> the N N N up about the n't
{'val_loss': 0.039187675658432605, 'val_interp_ppl': 107.18521017705773, 'val_pred_ppl': 117.26587014485291, 'val_pred_acc': 24.273058252427184}
wc: 0.10999999940395355, us: 0.008999999612569809
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 18.000%, pred ppl: 448.4
Targ: game 's over and investors are about to face a bear market </s> david m. jones vice president at <unk>
Pred: </s> </s> <unk> </s> <unk> </s> <unk> to the value <unk> </s> </s> the <unk> <unk> and president of <unk>
{'val_loss': 0.03918775128544245, 'val_interp_ppl': 107.20919175458047, 'val_pred_ppl': 117.31182753814886, 'val_pred_acc': 24.274271844660195}
wc: 0.10999999940395355, us: 0.010999999940395355
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 18.000%, pred ppl: 450.7
Targ: that <unk> <unk> is just one of a number of <unk> adopted after the N crash </s> the big board
Pred: in 's <unk> <unk> a a of the <unk> of people of by the N crash and the <unk> three
{'val_loss': 0.03918892366685046, 'val_interp_ppl': 107.20923971929979, 'val_pred_ppl': 117.31020057141569, 'val_pred_acc': 24.264563106796118}
wc: 0.10999999940395355, us: 0.013000000268220901
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 18.000%, pred ppl: 477.3
Targ: the financial markets to review its analysis and those of other crash studies </s> in may N the working group
Pred: a new times </s> be the own of the of the provisions </s> </s> the the N the number capital
{'val_loss': 0.039186669902387755, 'val_interp_ppl': 107.12169819602481, 'val_pred_ppl': 117.19758819874988, 'val_pred_acc': 24.276699029126213}
wc: 0.10999999940395355, us: 0.014999999664723873
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 18.000%, pred ppl: 504.8
Targ: small wonder that britain 's labor party wants credit controls </s> a few hours after the party launched its own
Pred: in u.s. is 's 's <unk> shortage is to to to the <unk> years of the <unk> 's a <unk>
{'val_loss': 0.039187443790758406, 'val_interp_ppl': 107.13105607119975, 'val_pred_ppl': 117.21485626706388, 'val_pred_acc': 24.275485436893202}
wc: 0.10999999940395355, us: 0.017000000923871994
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 18.000%, pred ppl: 507.5
Targ: rule on sony 's renewed request for a pre-trial order blocking sale of the disputed products on which deliveries began
Pred: N </s> a </s> N bids </s> a $ basis </s> the </s> $ company </s> </s> the it </s>
{'val_loss': 0.03918613103281671, 'val_interp_ppl': 107.18672712935758, 'val_pred_ppl': 117.23498746400675, 'val_pred_acc': 24.268203883495147}
wc: 0.10999999940395355, us: 0.01899999938905239
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 18.000%, pred ppl: 505.1
Targ: </s> dresdner is offering to acquire N N of <unk> 's capital for N francs $ N a share </s>
Pred: </s> the 's a to redeem the N of the <unk> ordinary </s> $ million $ N billion share </s>
{'val_loss': 0.03918551723246725, 'val_interp_ppl': 107.22081922099939, 'val_pred_ppl': 117.23816122080095, 'val_pred_acc': 24.269417475728154}
In [16]:
plt.scatter(wcs, uss, c=ppls)
plt.xlabel("Word Cache")
plt.ylabel("Uniform mass")
plt.colorbar()
idx = np.argmin(ppls)
print("Best values", 'cache', wcs[idx], 'unif', uss[idx])
Best values cache 0.07000000029802322 unif 0.010999999940395355
In [20]:
# Tune on decay rate
decays = []
ppls = []
for decay in torch.arange(0.98, .999, 0.001):
exp.word_cache_pct = .07
exp.unif_smoothing = .01
exp.word_cache_decay = decay
ret = exp.eval_epoch(0)
int_ppl = ret['val_interp_ppl']
ppls.append(int_ppl)
decays.append(decay)
print(ret)
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 14.000%, pred ppl: 526.1
Targ: <unk> weekend trying to <unk> out new terms that would be more acceptable to the banks </s> after ual the
Pred: year of </s> to <unk> the of products </s> the n't a <unk> </s> the <unk> </s> the the 's
{'val_loss': 0.03918219901603258, 'val_interp_ppl': 106.89812567179825, 'val_pred_ppl': 117.22550204933934, 'val_pred_acc': 24.260922330097088}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 14.000%, pred ppl: 517.1
Targ: last month </s> <unk> shares fell nearly N N on friday to close at N N </s> ramada which first
Pred: </s> week </s> the <unk> closed N N to to volume the N at N N </s> the which has
{'val_loss': 0.03918567364492897, 'val_interp_ppl': 106.82033783646605, 'val_pred_ppl': 117.21013491958131, 'val_pred_acc': 24.25728155339806}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 14.000%, pred ppl: 520.7
Targ: peak </s> in the mid-1980s employment was down as much as N N from the N peak and retail sales
Pred: </s> </s> the the first of report N N the as N N of the previous N in N sales
{'val_loss': 0.039187450196991845, 'val_interp_ppl': 106.75761742318197, 'val_pred_ppl': 117.19568224953717, 'val_pred_acc': 24.263349514563107}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 15.000%, pred ppl: 466.4
Targ: said mr. keating already has conceded attempting to buy influence with the lawmakers democratic sens. dennis <unk> of arizona alan
Pred: </s> </s> klein </s> has been that to the the the the <unk> ' leaders <unk> <unk> </s> the 's
{'val_loss': 0.03918413315601285, 'val_interp_ppl': 106.66910158224292, 'val_pred_ppl': 117.16231898779839, 'val_pred_acc': 24.264563106796118}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 15.000%, pred ppl: 496.1
Targ: the senate ethics committee </s> sen. <unk> said when all is said and done i expect to be fully <unk>
Pred: the year </s> </s> </s> the bob r. the the the expected the the is 'm to be a invested
{'val_loss': 0.03918898112402813, 'val_interp_ppl': 106.63420090514275, 'val_pred_ppl': 117.18655776968228, 'val_pred_acc': 24.264563106796118}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 14.000%, pred ppl: 544.7
Targ: <unk> the <unk> plastic <unk> are based on an american comic book and television series </s> paul <unk> managing director
Pred: the </s> u.s. of </s> </s> <unk> on the <unk> heart of </s> <unk> </s> </s> the <unk> </s> director
{'val_loss': 0.03918687194965563, 'val_interp_ppl': 106.57584261488773, 'val_pred_ppl': 117.17674159177558, 'val_pred_acc': 24.268203883495147}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 14.000%, pred ppl: 549.2
Targ: an analyst with goldman sachs & co </s> noting that the third quarter is usually the aluminum industry 's <unk>
Pred: </s> <unk> </s> <unk> sachs & co. </s> the that the issue quarter is n't been only company is <unk>
{'val_loss': 0.039182327990646214, 'val_interp_ppl': 106.54667126481266, 'val_pred_ppl': 117.19935710326209, 'val_pred_acc': 24.260922330097088}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 14.000%, pred ppl: 571.6
Targ: so severely as to be light years away from the type of <unk> deals available to sony and everyone else
Pred: </s> far in the the <unk> </s> ago </s> the <unk> of <unk> </s> </s> to the </s> the </s>
{'val_loss': 0.03918631160089258, 'val_interp_ppl': 106.50289021948349, 'val_pred_ppl': 117.16482759956969, 'val_pred_acc': 24.268203883495147}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 15.000%, pred ppl: 535.1
Targ: data management equipment </s> the company and its executives deny the charges </s> in a related development recognition equipment said
Pred: the </s> </s> </s> the company said the board were that company that the the statement matter and to said
{'val_loss': 0.039184529252929014, 'val_interp_ppl': 106.52176926371865, 'val_pred_ppl': 117.24045031284102, 'val_pred_acc': 24.263349514563107}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 15.000%, pred ppl: 497.8
Targ: and the policies of racial <unk> on all fronts including the armed struggle </s> and they called on the government
Pred: </s> the u.s. of the and </s> the countries </s> the <unk> forces </s> the the are the the <unk>
{'val_loss': 0.03918830250634007, 'val_interp_ppl': 106.46071407785384, 'val_pred_ppl': 117.18876642863633, 'val_pred_acc': 24.265776699029125}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 15.000%, pred ppl: 511.3
Targ: is not there in the market </s> many money managers and some traders had already left their offices early friday
Pred: was n't a </s> the past </s> the japanese managers are the investors say been been to clients in in
{'val_loss': 0.03918734072137949, 'val_interp_ppl': 106.47401629202281, 'val_pred_ppl': 117.21854370399015, 'val_pred_acc': 24.260922330097088}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 15.000%, pred ppl: 515.3
Targ: j. phelan said yesterday the circuit breaker worked well <unk> </s> i just think it 's <unk> at this point
Pred: <unk> <unk> jr. </s> that company breakers 's for </s> </s> the think got that 's going to the time
{'val_loss': 0.03918354458953209, 'val_interp_ppl': 106.50493929166154, 'val_pred_ppl': 117.21100064488677, 'val_pred_acc': 24.260922330097088}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 15.000%, pred ppl: 544.9
Targ: to repay its bank debt and other obligations resulting from the currently suspended <unk> operations </s> earlier the company announced
Pred: to <unk> $ debt debt </s> to purposes </s> from the company held </s> </s> </s> the this company said
{'val_loss': 0.039188942822257, 'val_interp_ppl': 106.5890647082107, 'val_pred_ppl': 117.28762691736209, 'val_pred_acc': 24.25728155339806}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 14.000%, pred ppl: 580.0
Targ: N from N tons a year earlier </s> the treasury plans to raise $ N million in new cash thursday
Pred: </s> to the N </s> year earlier </s> the company said to raise its N billion of short-term debt to
{'val_loss': 0.03918742964460144, 'val_interp_ppl': 106.6894644098836, 'val_pred_ppl': 117.33096137269872, 'val_pred_acc': 24.25121359223301}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 14.000%, pred ppl: 583.3
Targ: saturday opened its N <unk> investor centers across the country </s> the centers normally are closed through the weekend </s>
Pred: a </s> the <unk> to <unk> </s> </s> the bay </s> the <unk> are <unk> <unk> </s> the <unk> to
{'val_loss': 0.03918603574857116, 'val_interp_ppl': 106.67238231760058, 'val_pred_ppl': 117.20015659800414, 'val_pred_acc': 24.266990291262136}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 14.000%, pred ppl: 582.4
Targ: n't think their customers would like it very much </s> america west though is a smaller airline and therefore more
Pred: n't expect it <unk> </s> be to </s> much </s> the 's german it n't <unk> <unk> is the <unk>
{'val_loss': 0.03918418927497945, 'val_interp_ppl': 106.80887710093897, 'val_pred_ppl': 117.20791600405104, 'val_pred_acc': 24.266990291262136}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 13.000%, pred ppl: 641.5
Targ: eastern canada and conventional electric power generating plants elsewhere including britain where the british government plans to allow limited competition
Pred: the europe </s> the <unk> </s> co service </s> </s> <unk> 's the u.s. government and to build <unk> partnership
{'val_loss': 0.03918821923434734, 'val_interp_ppl': 107.04658392368248, 'val_pred_ppl': 117.25723192629796, 'val_pred_acc': 24.263349514563107}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 13.000%, pred ppl: 614.6
Targ: buying raw land while avoiding the negative <unk> to its own balance sheet mr. <unk> said </s> the company is
Pred: the N materials </s> the the u.s. impact of the <unk> <unk> sheet </s> <unk> said </s> the company is
{'val_loss': 0.039186752507535436, 'val_interp_ppl': 107.2379684851892, 'val_pred_ppl': 117.1961044455846, 'val_pred_acc': 24.266990291262136}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 13.000%, pred ppl: 589.0
Targ: expand production capacity </s> a quick turnaround is crucial to quantum because its cash requirements remain heavy </s> the company
Pred: N their </s> </s> the spokesman <unk> in expected to the 's the <unk> flow </s> in volume the company
{'val_loss': 0.03918590931701067, 'val_interp_ppl': 107.56778533644885, 'val_pred_ppl': 117.19652486355673, 'val_pred_acc': 24.263349514563107}
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 13.000%, pred ppl: 596.8
Targ: down from roughly $ N million last year </s> the spokesman said the broadcast unit will be <unk> dec. N
Pred: </s> from N N N billion in year </s> the company said the company group is be <unk> by N
{'val_loss': 0.03918679984675232, 'val_interp_ppl': 108.02005470170103, 'val_pred_ppl': 117.24525004380054, 'val_pred_acc': 24.262135922330096}
In [21]:
import matplotlib.pyplot as plt
import numpy as np
plt.plot(decays, ppls)
plt.xlabel("PPL")
plt.ylabel("Decay (word cache)")
idx = np.argmin(ppls)
print("Best values", 'decay', decays[idx])
Best values decay tensor(0.9890)
In [ ]:
In [ ]:
# Tune on decay rate
kn5_pcts = []
wcs = []
ppls = []
for kn5_pct in torch.arange(0.005, .015, .0025):
for wc in torch.arange(0.02, .07, .01):
exp.word_cache_pct = .07
exp.unif_smoothing = 0
exp.kn5_pct = kn5_pct
ret = exp.eval_epoch(0)
int_ppl = ret['val_interp_ppl']
wcs.append(wc)
ppls.append(int_ppl)
kn5_pcts.append(kn5_pct)
print(kn5_pct, wc)
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 10.000%, pred ppl: 730.3
Targ: has had in recent years </s> the ec and japan the u.s. 's largest steel suppliers have n't been filling
Pred: is n't a the months </s> the company is the 's u.s. and largest securities segment and been been <unk>
tensor(0.0050) tensor(0.0200)
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 10.000%, pred ppl: 758.1
Targ: clobbered two years ago in japan when <unk> introduced a powerful detergent called attack which quickly won a N N
Pred: a </s> years ago </s> the </s> the <unk> a <unk> <unk> for <unk> on has </s> the <unk> N
tensor(0.0050) tensor(0.0300)
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 11.000%, pred ppl: 679.4
Targ: and mrs. hills </s> many called it simply a contrast in styles </s> but some saw it as a classic
Pred: </s> <unk> <unk> </s> the of the is <unk> <unk> to the and the the of the 's a <unk>
tensor(0.0050) tensor(0.0400)
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 670.6
Targ: small subsidiary that is <unk> unrelated becomes a difficult <unk> said <unk> <unk> president of the parent in a statement
Pred: year portion of has <unk> </s> to <unk> <unk> time </s> <unk> <unk> a of the <unk> of <unk> <unk>
tensor(0.0050) tensor(0.0500)
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 705.6
Targ: plan to press specifically for a <unk> of rules governing exports of machine tools computers and other high-technology products </s>
Pred: are to buy the <unk> a <unk> recapitalization the </s> the </s> the tools </s> </s> to crops </s> </s>
tensor(0.0050) tensor(0.0600)
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 697.6
Targ: corp. and <unk> corp. the successor company to <unk> hotels </s> <unk> officials could n't be located </s> financial corp.
Pred: </s> and <unk> <unk> a <unk> of </s> concentrate the </s> the <unk> said n't be reached in <unk> services
tensor(0.0075) tensor(0.0200)
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 706.0
Targ: portions of kansas he said </s> the soviet union has n't given any clear indication of its wheat purchase plans
Pred: </s> of the </s> said </s> the company union has been yet the <unk> violation of the <unk> contract of
tensor(0.0075) tensor(0.0300)
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 667.2
Targ: to shareholders </s> but otherwise it would undoubtedly come back with an offer by management </s> the executive said any
Pred: is be </s> the the developments 's n't the to to a <unk> to the to the company said the
tensor(0.0075) tensor(0.0400)
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 684.6
Targ: a share up from the year-earlier $ N million or N cents a share </s> revenue rose to $ N
Pred: a share </s> N $ year-earlier period N million or N cents a share </s> revenue rose N $ N
tensor(0.0075) tensor(0.0500)
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 691.4
Targ: the N period which was helped by increased ad spending from the summer olympics </s> while usa today 's total
Pred: the previous N </s> was N by the demand revenue in the end of </s> the the 's 's <unk>
tensor(0.0075) tensor(0.0600)
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 688.1
Targ: reason is mounting competition from new japanese car plants in the u.s. that are pouring out more than one million
Pred: year </s> the a </s> the york </s> </s> </s> the u.s. </s> country <unk> </s> of <unk> N </s>
tensor(0.0100) tensor(0.0200)
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 681.6
Targ: added they hope to have more information early this week </s> investment canada declined to comment on the reasons for
Pred: the that were to be to of about </s> year </s> the bankers 's to comment on the new </s>
tensor(0.0100) tensor(0.0300)
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 712.3
Targ: new <unk> <unk> </s> the drug introduced last year is expected to generate sales of about $ N million this
Pred: <unk> <unk> </s> </s> the <unk> administration by year the expected to be more of $ $ N million </s>
tensor(0.0100) tensor(0.0400)
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 721.0
Targ: cents a share on sales of $ N million </s> the bronx has a wonderful <unk> garden a great <unk>
Pred: </s> </s> share </s> sales of $ N million </s> the company n.y. N <unk> to in <unk> <unk> deal
tensor(0.0100) tensor(0.0500)
Evaluating...
Finished batch 0
Partial pred acc - batch acc: 12.000%, pred ppl: 703.2
Targ: has avoided all that by living in a long island suburb with his wife who 's so <unk> to soap
Pred: <unk> said the of the the <unk> the <unk> history </s> </s> the <unk> </s> <unk> <unk> <unk> </s> the
In [42]:
plt.plot(kn5_pcts, ppls)
plt.xlabel("PPL")
plt.ylabel("KN5 pct")
idx = np.argmin(ppls)
print("Best values", 'kn5', kn5_pcts[idx])
Best values kn5 tensor(0.0100)
Content source: numenta/nupic.research
Similar notebooks: