notebook.community

Edit and run



In [5]:

    
import torch
import torch.nn as nn
from collections import OrderedDict
import shutil
import time
import gzip
import os
import json
import numpy as np
from dpp_nets.utils.io import make_embd, make_tensor_dataset, load_tensor_dataset
from dpp_nets.utils.io import data_iterator, load_embd
from torch.autograd import Variable
from torch.utils.data.dataloader import DataLoader
import time
from dpp_nets.my_torch.utilities import pad_tensor


root = '/Users/Max/data/beer_reviews'
data_file = 'reviews.aspect3.train.txt.gz'
embd_file = 'review+wiki.filtered.200.txt.gz'
save_path = os.path.join(root,'pytorch/aspect3_train.pt')
data_path = os.path.join(root, data_file)
embd_path = os.path.join(root, embd_file)


def read_rationales(path):
    """
    This reads the json.annotations file. 
    Creates a list of dictionaries, which holds the 994 reviews for which
    sentence-level annotations are available. 
    """
    data = []
    fopen = gzip.open if path.endswith(".gz") else open
    with fopen(path) as fin:
        for line in fin:
            item = json.loads(line)
            data.append(item)
    return data



In [ ]:

    
from collections import defaultdict
import torch
import torch.nn as nn
from dpp_nets.my_torch.linalg import custom_decomp
from dpp_nets.my_torch.DPP import DPP
from dpp_nets.my_torch.DPP import AllInOne
from dpp_nets.my_torch.utilities import compute_baseline

class DPP_Classifier(nn.Module):
    
    def __init__(self, dtype):
        
        super(DPP_Classifier, self).__init__()
        # Float vs Double
        self.dtype = dtype

        # Network parameters
        self.kernel_in = kernel_in = 400
        self.kernel_h = kernel_h = 1000
        self.kernel_out = kernel_out = 400

        self.pred_in = pred_in = 200 # kernel_in / 2
        self.pred_h = pred_h = 500
        self.pred_h2 = pred_h2 = 200
        self.pred_out = pred_out = 3
        
        # 2-Hidden-Layer Networks 
        self.kernel_net = torch.nn.Sequential(nn.Linear(kernel_in, kernel_h), nn.ELU(),
                                              nn.Linear(kernel_h, kernel_h), nn.ELU(), 
                                              nn.Linear(kernel_h, kernel_out))
        # 3-Hidden-Layer-Networks
        self.pred_net = torch.nn.Sequential(nn.Linear(pred_in, pred_h), nn.ReLU(),
                                             nn.Linear(pred_h, pred_h), nn.ReLU(),
                                             nn.Linear(pred_h, pred_h2), nn.ReLU(),
                                             nn.Linear(pred_h2, pred_out), nn.Sigmoid())
        
        self.kernel_net.type(self.dtype)
        self.pred_net.type(self.dtype)
        
        # Sampling Parameter
        self.alpha_iter = 5

        # Convenience
        self.kernels = []
        self.subsets = None
        self.picks = None
        self.preds = None
        
        self.saved_subsets = None
        self.saved_losses = None # not really necesary
        self.saved_baselines = None # not really necessary
        
    def forward(self, reviews):
        """
        reviews: batch_size x max_set_size x embd_dim = 200
        Output: batch_size x pred_out (the prediction)
        Challenges: Need to resize tensor appropriately and 
        measure length etc. 
        """
        batch_size, max_set_size, embd_dim = reviews.size()
        alpha_iter = self.alpha_iter
        self.saved_subsets = actions = [[] for i in range(batch_size)]
        picks = [[] for i in range(batch_size)]
        
        # Create context
        lengths = reviews.sum(2).abs().sign().sum(1)
        context = (reviews.sum(1) / lengths.expand_as(reviews.sum(1))).expand_as(reviews)
        mask = reviews.sum(2).abs().sign().expand_as(reviews).byte()

        # Mask out zero words
        reviews = reviews.masked_select(mask).view(-1, embd_dim)
        context = context.masked_select(mask).view(-1, embd_dim)

        # Compute batched_kernel
        kernel_input = torch.cat([reviews, context], dim=1)
        kernel_output = self.kernel_net(kernel_input)
        
        # Extract the kernel for each review from batched_kernel
        s = list(lengths.squeeze().cumsum(0).long().data - lengths.squeeze().long().data)
        e = list(lengths.squeeze().cumsum(0).long().data)

        for i, (s, e) in enumerate(zip(s, e)):
            review = reviews[s:e] # original review, without zero words
            kernel = kernel_output[s:e] # corresponding kernel 
            self.kernels.append(kernel.data)
            #vals, vecs = custom_decomp()(kernel)
            for j in range(alpha_iter):
                subset = AllInOne()(kernel)
                #subset = DPP()(vals, vecs)
                actions[i].append(subset)
                pick = subset.diag().mm(review).sum(0)
                picks[i].append(pick)

        # Predictions
        picks = torch.stack([torch.stack(pick) for pick in picks]).view(-1, embd_dim)
        preds = self.pred_net(picks).view(batch_size, alpha_iter, -1)
        
        return preds

def register_rewards(preds, targets, criterion, net):
    
    #targets = targets.unsqueeze(1).unsqueeze(1).expand_as(preds)
    targets = targets.unsqueeze(1).expand_as(preds)
    loss = criterion(preds, targets)
    
    actions = net.saved_subsets
    
    losses = ((preds - targets)**2).mean(2)
    losses = [[i.data[0] for i in row] for row in losses]
    net.saved_losses = losses # not really necessary
    baselines = [compute_baseline(i) for i in losses]
    net.saved_baselines = baselines # not really necessary
    
    for actions, rewards in zip(actions, baselines):
        for action, reward in zip(actions, rewards):
            action.reinforce(reward)

    return loss



In [ ]:

    
# Useful Support

class AverageMeter(object):
    """Computes and stores the average and current value"""
    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count
        
def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'):
    """
    This is good!
    """
    torch.save(state, filename)
    if is_best:
        shutil.copyfile(filename, 'model_best.pth.tar')
        
def adjust_learning_rate(optimizer, epoch):
    """Sets the learning rate to the initial LR decayed by 10 every 30 epochs"""
    lr = optimizer.state_dict()['param_groups'][0]['lr']
    lr = lr * (0.1 ** (epoch // 5))
    for param_group in optimizer.param_groups:
        param_group['lr'] = lr



In [ ]:

    
def train(train_loader, embd, model, criterion, optimizer, epoch, dtype):
    
    batch_time = AverageMeter()
    data_time = AverageMeter()
    losses = AverageMeter()
    subset_size = AverageMeter()
    
    target_dim = 3

    end = time.time()
    for i, (review, target) in enumerate(train_loader):
        
        # measure data loading time
        data_time.update(time.time() - end)
        
        targets = Variable(target[:,:target_dim].type(dtype))
        reviews = embd(Variable(review)).type(dtype)

        # compute output
        model.alpha_iter = 2
        pred = model(reviews)
        loss = register_rewards(pred, targets, criterion, model)

        ##measure accuracy and record loss ????????????????????????
        # prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
        losses.update(loss.data[0], reviews.size(0))
        for l in model.saved_subsets:
            for s in l:
                subset_size.update(s.data.sum())
        # top1.update(prec1[0], input.size(0))
        # top5.update(prec5[0], input.size(0))

        # compute gradient and do SGD step
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        # measure elapsed time
        batch_time.update(time.time() - end)
        end = time.time()

        #if i % args.print_freq == 0:
        if i % print_freq == 0:
            print('Epoch: [{0}][{1}/{2}]\t'
                  'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                  'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
                  'SSize {subset_size.val:.2f} ({subset_size.avg: .2f})'
                  'Loss {loss.val:.4f} ({loss.avg:.4f})\t'.format(
                   epoch, i, len(train_loader), batch_time=batch_time,
                   data_time=data_time, subset_size = subset_size, loss=losses))

def validate(val_loader, model, criterion):
    
    batch_time = AverageMeter()
    losses = AverageMeter()
    t_prec = AverageMeter()
    t_recall = AverageMeter()
    t_tp = AverageMeter()
    t_fp = AverageMeter()
    t_fn = AverageMeter()
    
    target_dim = 3

    # switch to evaluate mode
    # model.eval()

    end = time.time()
    for i, (review, target) in enumerate(val_loader):
        
        target = target.sum(1).sign().type(dtype).squeeze().byte()
        # targets = target[:,:target_dim,:].type(dtype)
        reviews = embd(Variable(review, volatile=True)).type(dtype)

        # compute output
        model.alpha_iter = 1
        preds = model(reviews)

        subset = model.saved_subsets[0][0]
        subset = pad_tensor(subset.data,0,0,412).byte()
        # target = targets

        # targets = target[:,:target_dim,:].type(dtype)
        reviews = embd(Variable(review, volatile=True)).type(dtype)

        # compute output
        model.alpha_iter = 1
        preds = model(reviews)
        
        subset = model.saved_subsets[0][0]
        subset = pad_tensor(subset.data,0,0,412).byte()
        # target = target[:,:target_dim,:].squeeze()

        retriev = subset.sum()
        relev = target.sum()

        tp = target.masked_select(subset).sum()
        fp = (1 - target.masked_select(subset)).sum()
        fn = (1 - subset.masked_select(target)).sum()
        t_tp.update(tp)
        t_fp.update(fp)
        t_fn.update(fn)

        if retriev: 
            prec = tp / retriev
            t_prec.update(prec)

        if relev: 
            recall = tp / relev
            t_recall.update(recall)

        # measure accuracy and record loss
        #prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
        #losses.update(loss.data[0], input.size(0))
        #top1.update(prec1[0], input.size(0))
        #top5.update(prec5[0], input.size(0))

        # measure elapsed time
        batch_time.update(time.time() - end)
        end = time.time()

        if i % 100 == 0:
            print('Test: [{0}/{1}]\t'
                  'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                  'Precision {t_prec.val:.4f} ({t_prec.avg:.4f})\t'
                  'Recall {t_recall.val:.4f} ({t_recall.avg:.4f})\t'.format(
                   i, len(val_loader), batch_time=batch_time, t_prec=t_prec, t_recall=t_recall))
            
    return t_prec.avg



In [ ]:

    
### MAIN PROGRAMME


global best_prec1
best_prec1 = 0

# set parameters
lr = 1e-1
momentum = 0.9
weight_decay = 0.
start_epoch = 0
epochs = 1
batch_size = 20
print_freq = 10

data = '/Users/Max/data/beer_reviews/pytorch'
dtype = torch.DoubleTensor

# create model
embd = load_embd('/Users/Max/data/beer_reviews/pytorch/embeddings.pt')
model = DPP_Classifier(torch.DoubleTensor)

# define loss function (criterion) and optimizer
criterion = nn.L1Loss()

optimizer = torch.optim.SGD(model.parameters(), lr,
                            momentum=momentum,
                            weight_decay=weight_decay)

# Data loading code
trainpath = os.path.join(data, 'aspect1_train.pt')
valpath = os.path.join(data, 'aspect1_heldout.pt')
ratpath = os.path.join(data, 'annotated.pt')

train_set = torch.load(trainpath)
val_set = torch.load(valpath)
rat_set = torch.load(ratpath)

rat_train_set = torch.load(os.path.join(data, 'annotated_common.pt'))
#train_loader = DataLoader(train_set, batch_size, shuffle=True)
#val_loader = DataLoader(val_set)
rat_train_loader = DataLoader(rat_train_set, batch_size, shuffle=True)
rat_loader = DataLoader(rat_set)



In [ ]:

    
epochs = 20
criterion = nn.L1Loss()

for epoch in range(start_epoch, epochs):
    adjust_learning_rate(optimizer, epoch)

    # train for one epoch
    train(rat_train_loader, embd, model, criterion, optimizer, epoch, dtype)

    # evaluate on validation set
    prec1 = validate(rat_loader, model, criterion)

# remember best prec@1 and save checkpoint
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
save_checkpoint({
    'epoch': epoch + 1,
    'state_dict': model.state_dict(),
    'best_prec1': best_prec1,
    'optimizer' : optimizer.state_dict(),
}, is_best)



In [ ]:

    
import random
#word_to_ix = make_embd(embd_path, only_index_dict=True)
#ix_to_word = {ix: word for word, ix in word_to_ix.items()}

rat_set, ix_to_word
def sample_words(rat_set, model, ix_to_word):
    
    # Sample a review
    ix = random.randint(0,len(rat_set))

    # Make a prediction
    x = rat_set.data_tensor[ix].unsqueeze(0)
    review = embd(Variable(x, volatile=True)).type(dtype)
    model.alpha_iter = 1
    model(review)

    # What words were selected
    subset = model.saved_subsets[0][0]
    subset = pad_tensor(subset.data,0,0,412).byte()

    # Convert to words
    all_words = [ix_to_word[ix -1] for ix in x.squeeze() if ix > 0]
    filtered_words = [ix_to_word[ix -1] for ix in x.masked_select(subset)]
    print(" ".join(all_words) )
    print("DPP Selection: ", filtered_words)


    ix = random.randint(0,len(rat_set))
    rat_set.data_tensor[ix].unsqueeze(0)



In [ ]:

    
def sample_prediction(rat_set, model):
    # Sample a review
    ix = random.randint(0,len(rat_train_set))

    # Make a prediction
    x = rat_train_set.data_tensor[ix].unsqueeze(0)
    target = rat_train_set.target_tensor[ix][:3]
    review = embd(Variable(x, volatile=True)).type(dtype)
    model.alpha_iter = 1
    pred = model(review).data.squeeze()
    print(pred, target)
    return pred, target



In [ ]:

    
pred, target = sample_prediction(rat_set, model)



In [ ]:

    
criterion(Variable(pred), Variable(target))



In [ ]:

    
torch.save(model.pred_net.state_dict(), 'pred_dict25.pt')



In [ ]:

    
import torch

e = 0
for i in range(100):
    v = torch.normal(torch.FloatTensor([1,2,3,4,5]))
    e += v
e / 100



In [ ]:

    
non_lin = torch.sin
torch.sin(v)



In [ ]:

    
non_lin(v)



In [ ]:

    
non_lin



In [ ]:

    
batch_size = 2
set_size = 3
embd_dim = 4
words = torch.randn(batch_size, set_size, embd_dim)



In [ ]:

    
v = torch.normal(torch.FloatTensor([1,2,3,4,5])torch.cos(torch.sin(words).mean(1)).squeeze()



In [ ]:

    
v = torch.normal(torch.FloatTensor([1,2,3,4,5]))
torch.log(v)



In [ ]:

    
import numpy as np
batch_size = 100
n_clusters = 10
set_size = 40
embd_dim = pred_in = 50
dtype = dtype = torch.DoubleTensor
np.random.seed(0)
means = dtype(np.random.randint(-50,50,[n_clusters, int(pred_in)]).astype("float"))

def generate(batch_size):
    """sdf"
    Arguments:
    means: Probs best to make this an attribute of the class, 
    so that repeated training works with the same data distribution.


    """


    # Generate index
    index = torch.cat([torch.arange(0, float(n_clusters)).expand(batch_size, n_clusters).long(), 
                      torch.multinomial(torch.ones(batch_size, n_clusters), set_size - n_clusters, replacement=True)]
                     ,dim=1)
    index = index.t()[torch.randperm(set_size)].t().contiguous()

    # Generate words, context, target
    words = dtype(torch.normal(means.index_select(0,index.view(index.numel()))).view(batch_size, set_size, embd_dim))
    context = dtype(words.sum(1).expand_as(words))

    target = torch.sin(torch.pow(words.abs(),2).mean(1)).squeeze()

    return words, context, target



In [ ]:

    
words, context, target = generate(5)
print(target)



In [ ]:

    
(torch.std(target, dim=0) / torch.mean(target, dim=0)).mean()



In [ ]:

    
target



In [ ]:

    
v1 = torch.randn(2,2)
v2 = torch.randn(2,2)
v3 = torch.randn(2,2)
v4 = torch.randn(2,2)
v5 = torch.randn(2,2)
v6 = torch.randn(2,2)



In [ ]:

    
import torch.nn as nn
nn.MSELoss()



In [ ]:



In [ ]:

    
from dpp_nets.my_torch.simulator import SimKDPPDeepSet
import torch
network_params = {'set_size': 40, 'n_clusters': 10}
dtype = torch.DoubleTensor
sim = SimKDPPDeepSet(network_params, dtype)



In [ ]:



In [ ]:



In [ ]:

    
mod = torch.nn.Sequential(nn.Linear(10,20), nn.ReLU(), nn.Linear(20,10))



In [ ]:

    
for mod in mod.modules():
    print(mod)



In [ ]:

    
A = Variable(torch.randn(10,20))



In [ ]:

    
mod(A)



In [ ]:

    
batch_size = 3
set_size = 4
embd_dim = 5
words = Variable(torch.randn(batch_size, set_size, embd_dim))
print(words)
subset = Variable(torch.ByteTensor([1,0,0,1]),requires_grad=True)
words[1].masked_select(Variable(subset.data.expand_as(words[1].t())).t()).view(-1,embd_dim)



In [ ]:



In [11]:

    
data_path









    Out[11]:





'/Users/Max/data/beer_reviews/reviews.aspect3.train.txt.gz'



In [10]:

    
word_to_ix = make_embd(embd_path, only_index_dict=True)



In [74]:

    
old_dataset = make_tensor_dataset(data_path, word_to_ix)



In [76]:

    
old_dataset.data_tensor_tensor









    Out[76]:





 3.0000e+00  5.4000e+01  5.7364e+04  ...   0.0000e+00  0.0000e+00  0.0000e+00
 3.0000e+00  2.7000e+01  4.3000e+01  ...   0.0000e+00  0.0000e+00  0.0000e+00
 2.3000e+01  5.4800e+02  1.0875e+05  ...   0.0000e+00  0.0000e+00  0.0000e+00
                ...                   ⋱                   ...                
 1.3660e+03  2.2170e+03  2.4590e+03  ...   0.0000e+00  0.0000e+00  0.0000e+00
 1.6400e+02  1.6300e+02  7.3000e+01  ...   0.0000e+00  0.0000e+00  0.0000e+00
 8.0000e+00  5.6900e+02  1.6600e+02  ...   0.0000e+00  0.0000e+00  0.0000e+00
[torch.LongTensor of size 70000x1012]



In [107]:

    
ix = 1324
print(torch.cat([old_dataset.data_tensor[ix, 200:220].unsqueeze(1), dataset.data_tensor[ix, 200:220].unsqueeze(1)],dim=1))









    



  225   109
   13    98
   56   225
   67    13
   33    56
   15    67
  499    33
    2    15
    0   499
    0     2
    0     0
    0     0
    0     0
    0     0
    0     0
    0     0
    0     0
    0     0
    0     0
    0     0
[torch.LongTensor of size 20x2]



In [105]:

    
# this is bad as maximum set_size might increase#
from torch.utils.data import TensorDataset
import re

reviews = []
targets = []
max_set_size = 0

for i, (review, target) in enumerate(data_iterator(data_path)):
    review_ix = []
    for word in review:
        if word in word_to_ix:
            ix = word_to_ix[word] + 1
            review_ix.append(ix)
        else:
            candidates = re.split('[;|,-/."]',word)
            for word in candidates:
                if word in word_to_ix:
                    print(i)
                    ix = word_to_ix[word] + 1
                    review_ix.append(ix)

    max_set_size = max(max_set_size, len(review_ix))
    reviews.append(review_ix)
    targets.append(target)

reviews_tensor = []
for review in reviews:
    review = torch.LongTensor(review)
    review = pad_tensor(review, 0, 0, max_set_size)
    reviews_tensor.append(review)

reviews = torch.stack(reviews_tensor)
targets = torch.stack(targets)

dataset = TensorDataset(reviews, targets)



In [123]:

    
word = 2
dim = 3
embd.weight[:,dim]









    Out[123]:





Variable containing:
 0.0000
 0.0955
-0.0327
   ⋮   
-0.0173
-0.0467
 0.0217
[torch.FloatTensor of size 147760]



In [75]:

    
dataset.data_tensor









    Out[75]:





 3.0000e+00  5.4000e+01  5.7364e+04  ...   0.0000e+00  0.0000e+00  0.0000e+00
 3.0000e+00  2.7000e+01  4.3000e+01  ...   0.0000e+00  0.0000e+00  0.0000e+00
 2.3000e+01  5.4800e+02  1.0875e+05  ...   0.0000e+00  0.0000e+00  0.0000e+00
                ...                   ⋱                   ...                
 1.3660e+03  2.2170e+03  2.4590e+03  ...   0.0000e+00  0.0000e+00  0.0000e+00
 1.6400e+02  1.6300e+02  7.3000e+01  ...   0.0000e+00  0.0000e+00  0.0000e+00
 8.0000e+00  5.6900e+02  1.6600e+02  ...   0.0000e+00  0.0000e+00  0.0000e+00
[torch.LongTensor of size 70000x1014]



In [1]:

    
def data_iterator(data_path):
    with gzip.open(data_path, 'rt') as f:
        for line in f:
            target, sep, words = line.partition("\t")
            words, target = words.split(), target.split()
            if len(words):
                target = torch.Tensor([float(v) for v in target])
                yield words, target



In [3]:

    
def make_tensor_dataset(data_path, word_to_ix, max_set_size=0, save_path=None):
        
    if not max_set_size:
        for (review, target) in data_iterator(data_path):
            review = [(word in word_to_ix) for word in review]
            max_set_size = max(sum(review),max_set_size)
            
    reviews, targets = [], []

    for (review, target) in data_iterator(data_path):
        review = [word_to_ix[word] + 1 for word in review if word in word_to_ix]
        review = torch.LongTensor(review)
        review = pad_tensor(review, 0, 0, max_set_size)
        reviews.append(review)
        targets.append(target)
    
    reviews = torch.stack(reviews)
    targets = torch.stack(targets)

    dataset = TensorDataset(reviews, targets)

    if save_path:
        torch.save(dataset, save_path)    
    else:
        return dataset



In [ ]:

    
root = '/Users/Max/data/beer_reviews'
data_file = 'reviews.aspect3.train.txt.gz'
embd_file = 'review+wiki.filtered.200.txt.gz'
save_path = os.path.join(root,'pytorch/aspect3_train.pt')
data_path = os.path.join(root, data_file)
embd_path = os.path.join(root, embd_file)

embd_path =



In [160]:

    
import torch
import torch.nn as nn
batch_size = 20
input_dim = 12
hidden_dim = 18
output_dim = 2
layer1 = nn.Linear(input_dim, hidden_dim)
batch_norm = nn.BatchNorm1d(hidden_dim)
layer2 = nn.Linear(hidden_dim, output_dim)
model = nn.Sequential(layer1, batch_norm, layer2)



In [206]:

    
layer1.train()
x = Variable(torch.randn(batch_size, input_dim))
#model2 = nn.Sequential(layer1, layer2)
#y1 = model(x)
#y2 = model2(x)
y = layer2(batch_norm(layer1(x)))
batch_norm(layer1(x))









    Out[206]:





Variable containing:

Columns 0 to 9 
 1.3708  0.7502  0.0854  0.4266  0.0059 -0.1973 -0.7639  1.1644 -0.0341  0.4740
-2.1605 -0.4336 -0.0277  0.0453 -0.0324  0.2395  0.7752 -0.8815 -0.3543  0.0030
 0.0925 -0.2283  0.0528  0.1211  0.0424 -0.0172  0.3966 -0.0855 -0.1579 -0.7209
 1.5900  0.3261 -0.0083 -0.0961  0.0140 -0.3643 -0.7462 -0.1769  0.0418 -0.5172
 1.0453 -0.6380  0.0516 -1.0351  0.0140  0.1042  0.1187  0.1786  0.0123  0.3371
-0.6965  0.5642  0.0196  0.7898 -0.0998  0.2735  0.3917 -0.5570  0.6402  0.7307
-0.1948 -0.3369  0.0310 -0.0530  0.0530  0.0680  0.0928 -0.6833 -0.4211 -0.6430
 0.0567  0.6595  0.0190 -0.5103 -0.0512  0.1156  1.5776  0.5863  0.5003 -0.1485
-0.5632  0.0859 -0.0170  0.0738 -0.0524 -0.0546 -0.0976 -0.2792  0.2557  0.0196
 0.8496  0.6132 -0.0932  0.0716 -0.0030 -0.1303 -0.3261 -0.4732  0.2063  0.0446
 0.5098  0.0896 -0.0007  0.0563 -0.0718 -0.3349 -0.4187  0.3595  0.0789 -0.1252
 0.6247 -0.1313  0.0327 -0.5086  0.0768 -0.2185  0.8786  1.5757  0.0865 -0.7573
 0.4809 -0.3735  0.2100 -0.1403  0.0099  0.0204 -0.6580  0.0749 -0.1049  0.5140
-0.8219  0.0379 -0.1442  0.4321  0.0286  0.4850  0.1181  0.7593 -0.0393  0.2072
 0.3075 -0.3719  0.0307  0.4914 -0.0017  0.3898  0.1642 -0.2032 -0.1082 -0.1983
-1.3176  0.0393 -0.0863 -0.3727  0.0290  0.1533  0.0677 -0.0086  0.1038  0.4343
-1.7190  0.1884 -0.1362  0.4838  0.0034 -0.0473 -0.2041 -0.2901  0.2897 -0.0307
 0.2579 -0.8078 -0.0316 -0.2068  0.0526  0.0346 -0.0256  0.8078 -0.4733  0.1306
 0.2444  0.1439 -0.0412  0.1064  0.0007 -0.2785 -0.9089 -1.3537 -0.2037  0.0170
 0.0433 -0.1768  0.0536 -0.1752 -0.0182 -0.2411 -0.4321 -0.5143 -0.3186  0.2288

Columns 10 to 17 
-0.8878 -1.2889 -0.0631 -0.8319  0.2964 -0.4162  0.7122  0.1537
 0.0180  0.5707  0.1630  0.4177 -0.2165  2.2398 -0.6341 -0.0389
 0.7259  0.1838 -0.1087 -0.2717  0.3461 -0.7050  0.1080  0.0323
 0.1764 -0.1059 -0.2723 -0.4512  0.4409 -1.5727  1.3228  0.0069
 0.3173  0.2923  0.0393  0.8371 -0.6081  0.5499 -1.0652 -0.0576
-0.1768 -0.2841  0.2145  1.1334 -0.2324  0.2733 -0.7594 -0.0043
 0.6501  0.4862 -0.1047 -0.4299  0.1823  0.4335  0.3833  0.0033
-0.6962  0.1220  0.2057  0.7546 -0.9799  0.1582  0.1597 -0.0950
-0.3806  0.4902 -0.1220 -0.3064 -0.1467 -0.6896  0.4806 -0.0286
 0.3374 -1.0077  0.3520  0.3792  0.2926 -0.1976 -0.1675  0.0007
-0.1853 -0.1868 -0.2438 -0.0382  0.2658 -1.1327  0.3039  0.0474
-0.3571  0.1858 -0.1336 -1.0981 -0.1728 -2.0902  0.8446  0.0016
 0.3139 -0.0933 -0.1915 -0.0149  0.5271  0.2286 -1.0670  0.0990
 0.3429 -0.0013  0.2320  0.7481 -0.4858  0.9982 -1.0078 -0.0660
 0.9053  0.4039 -0.1386  0.6127 -0.4394  0.1606 -0.3639  0.0142
-0.4390  0.3879  0.2321  0.1778 -0.1387  1.0964 -0.5027 -0.1201
-0.5312  0.6193 -0.0295 -0.7819  0.2519 -0.9717  0.8376 -0.0798
 0.1749 -0.2208  0.0248 -0.0329 -0.1742  0.5701 -0.6477  0.0428
-0.1377 -0.2122 -0.0231 -0.6732  0.5091  0.2038  1.1216  0.0241
-0.1705 -0.3411 -0.0324 -0.1301  0.4823  0.8634 -0.0591  0.0639
[torch.FloatTensor of size 20x18]



In [18]:

    
import argparse
import os
import shutil

import torch
import torch.nn as nn
from torch.autograd import Variable
from torch.utils.data.dataloader import DataLoader

from dpp_nets.utils.io import make_embd, make_tensor_dataset
from dpp_nets.layers.layers import KernelVar, ReinforceSampler, PredNet, ReinforceTrainer


parser = argparse.ArgumentParser(description='REINFORCE VIMCO Trainer')

parser.add_argument('-a', '--aspect', type=str, choices=['aspect1', 'aspect2', 'aspect3', 'all'],
                    help='what is the target?', required=True)

parser.add_argument('-b', '--batch-size', default=50, type=int,
                    metavar='N', help='mini-batch size (default: 50)')
parser.add_argument('--epochs', default=30, type=int, metavar='N',
                    help='number of total epochs to run')
parser.add_argument('--lr_k', '--learning_rate_k', default=1e-3, type=float,
                    metavar='LRk', help='initial learning rate for kernel net')
parser.add_argument('--lr_p', '--learning_rate_p', default=1e-4, type=float,
                    metavar='LRp', help='initial learning rate for pred net')
parser.add_argument('--reg', type=float, required=True,
                    metavar='reg', help='regularization constant')
parser.add_argument('--reg_mean', type=float, required=True,
                    metavar='reg_mean', help='regularization_mean')
parser.add_argument('--alpha_iter', type=int, required=True,
                    metavar='alpha_iter', help='How many subsets to sample from DPP? At least 2!')

# Pre-training
parser.add_argument('--pretrain_kernel', type=str, default="",
                    metavar='pretrain_kernel', help='Give name of pretrain_kernel')
parser.add_argument('--pretrain_pred', type=str, default="",
                    metavar='pretrain_pred', help='Give name of pretrain_pred')

# Train locally or remotely?
parser.add_argument('--remote', type=int,
                    help='training locally or on cluster?', required=True)

# Burnt in Paths..
parser.add_argument('--data_path_local', type=str, default='/Users/Max/data/beer_reviews',
                    help='where is the data folder locally?')
parser.add_argument('--data_path_remote', type=str, default='/cluster/home/paulusm/data/beer_reviews',
                    help='where is the data folder remotely?')
parser.add_argument('--ckp_path_local', type=str, default='/Users/Max/checkpoints/beer_reviews',
                    help='where is the checkpoints folder locally?')
parser.add_argument('--ckp_path_remote', type=str, default='/cluster/home/paulusm/checkpoints/beer_reviews',
                    help='where is the data folder remotely?')

parser.add_argument('--pretrain_path_local', type=str, default='/Users/Max/checkpoints/beer_reviews',
                    help='where is the pre_trained model? locally')
parser.add_argument('--pretrain_path_remote', type=str, default='/cluster/home/paulusm/pretrain/beer_reviews',
                    help='where is the data folder? remotely')


def train(loader, trainer, optimizer):

    trainer.train()

    for t, (review, target) in enumerate(loader):
        review = Variable(review)

        if args.aspect == 'all':
            target = Variable(target[:,:3]).type(dtype)
        else:
            target = Variable(target[:,int(args.aspect[-1])]).type(dtype)

        loss  = trainer(review, target)
        
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        print('Weight mean is: ', trainer.kernel_net.layer1.weight.mean())
        print('Weight max is: ', trainer.kernel_net.layer1.weight.max())
        print('Weight min is: ', trainer.kernel_net.layer1.weight.min())
        print('Grad max is: ', trainer.kernel_net.layer1.weight.grad.max())
        print('Grad min is: ', trainer.kernel_net.layer1.weight.grad.min())
        print("trained one batch")

def validate(loader, trainer):
    """
    Note, we keep the sampling as before. 
    i.e what ever alpha_iter is, we take it. 
    """
    trainer.eval()
    total_loss = 0.0
    total_pred_loss = 0.0
    total_reg_loss = 0.0

    for i, (review, target) in enumerate(loader, 1):
        review = Variable(review, volatile=True)

        if args.aspect == 'all':
            target = Variable(target[:,:3], volatile=True).type(dtype)
        else:
            target = Variable(target[:,int(args.aspect[-1])], volatile=True).type(dtype)

        trainer(review, target)
        loss = trainer.loss.data[0]
        pred_loss = trainer.pred_loss.data[0]
        reg_loss = trainer.reg_loss.data[0]

        delta = loss - total_loss
        total_loss += (delta / i)
        delta = pred_loss - total_pred_loss 
        total_pred_loss += (delta / i)
        delta = reg_loss - total_reg_loss
        total_reg_loss += (delta / i)

# print("validated one batch")

    return total_loss, total_pred_loss, total_reg_loss

def adjust_learning_rate(optimizer, epoch):
    """Sets the learning rate to the initial LR multiplied by factor 0.1 for every 10 epochs"""
    if not ((epoch + 1) % 10):
        factor = 0.1
        for param_group in optimizer.param_groups:
            param_group['lr'] = param_group['lr'] * factor

def log(epoch, loss, pred_loss, reg_loss):

    string = str.join(" | ", ['Epoch: %d' % (epoch), 'V Loss: %.5f' % (loss), 
                              'V Pred Loss: %.5f' % (pred_loss), 'V Reg Loss: %.5f' % (reg_loss)])

    if args.remote:
        destination = os.path.join(args.ckp_path_remote, args.aspect + 'reg' + str(args.reg) + 'reg_mean' + str(args.reg_mean) + 
          'alpha_iter' + str(args.alpha_iter) + str(args.pretrain_kernel) + str(args.pretrain_pred) + 'reinforce_log.txt')
    else:
        destination = os.path.join(args.ckp_path_local, args.aspect + 'reg' + str(args.reg) + 'reg_mean' + str(args.reg_mean) + 
          'alpha_iter' + str(args.alpha_iter) + str(args.pretrain_kernel) + str(args.pretrain_pred) + 'reinforce_log.txt')

    with open(destination, 'a') as log:
        log.write(string + '\n')

def save_checkpoint(state, is_best, filename='reinforce_checkpoint.pth.tar'):
    """
    State is a dictionary that cotains valuable information to be saved.
    """
    if args.remote:
        destination = os.path.join(args.ckp_path_remote, args.aspect + 'reg' + str(args.reg) + 'reg_mean' + str(args.reg_mean) + 
          'alpha_iter' + str(args.alpha_iter) + str(args.pretrain_kernel) + str(args.pretrain_pred) + str(args.filename))
    else:
        destination = os.path.join(args.ckp_path_local, args.aspect + 'reg' + str(args.reg) + 'reg_mean' + str(args.reg_mean) + 
          'alpha_iter' + str(args.alpha_iter) + str(args.pretrain_kernel) + str(args.pretrain_pred) + str(args.filename))
    
    torch.save(state, destination)

    if is_best:
        if args.remote:
            best_destination = os.path.join(args.ckp_path_remote, args.aspect + 'reg' + str(args.reg) + 'reg_mean' + str(args.reg_mean) + 
               'alpha_iter' + str(args.alpha_iter) + str(args.pretrain_kernel) + str(args.pretrain_pred) + 'reinforce_best.pth.tar')
        else:
            best_destination = os.path.join(args.ckp_path_local, args.aspect + 'reg' + str(args.reg) + 'reg_mean' + str(args.reg_mean) +  
               'alpha_iter' + str(args.alpha_iter) + str(args.pretrain_kernel) + str(args.pretrain_pred) + 'reinforce_best.pth.tar')
        
        shutil.copyfile(destination, best_destination)



In [19]:

    
global args, lowest_loss, dtype

args = parser.parse_args("-a aspect3 --remote 0 --reg 0.1 --reg_mean 10 --alpha_iter 4 --lr_k 1e-4".split())
lowest_loss = 100 # arbitrary high number as upper bound for loss
dtype = torch.DoubleTensor

### Load data
if args.remote:
    # print('training remotely')
    train_path = os.path.join(args.data_path_remote, str.join(".",['reviews', args.aspect, 'train.txt.gz']))
    val_path   = os.path.join(args.data_path_remote, str.join(".",['reviews', args.aspect, 'heldout.txt.gz']))
    embd_path = os.path.join(args.data_path_remote, 'review+wiki.filtered.200.txt.gz')

else:
    # print('training locally')
    train_path = os.path.join(args.data_path_local, str.join(".",['reviews', args.aspect, 'train.txt.gz']))
    val_path   = os.path.join(args.data_path_local, str.join(".",['reviews', args.aspect, 'heldout.txt.gz']))
    embd_path = os.path.join(args.data_path_local, 'review+wiki.filtered.200.txt.gz')

embd, word_to_ix = make_embd(embd_path)
train_set = make_tensor_dataset(train_path, word_to_ix)
val_set = make_tensor_dataset(val_path, word_to_ix)
print("loaded data")

torch.manual_seed(0)
train_loader = DataLoader(train_set, args.batch_size, shuffle=True)
val_loader = DataLoader(val_set, args.batch_size)
print("loader defined")

### Build model
# Network parameters
embd_dim = embd.weight.size(1)
kernel_dim = 200
hidden_dim = 500
enc_dim = 200
if args.aspect == 'all':
    target_dim = 3
else: 
    target_dim = 1

# Model
torch.manual_seed(1)


# Add pre-training here...
kernel_net = KernelVar(embd_dim, hidden_dim, kernel_dim)
sampler = ReinforceSampler(args.alpha_iter)
pred_net = PredNet(embd_dim, hidden_dim, enc_dim, target_dim)

if args.pretrain_kernel:
    if args.remote:
        state_dict = torch.load(args.pretrain_path_remote + args.pretrain_kernel)
    else:
        state_dict = torch.load(args.pretrain_path_local + args.pretrain_kernel)
    kernel_net.load_state_dict(state_dict)

if args.pretrain_pred:
    if args.remote:
        state_dict = torch.load(args.pretrain_path_remote + args.pretrain_pred)
    else:
        state_dict = torch.load(args.pretrain_path_local + args.pretrain_pred)
    pred_net.load_state_dict(state_dict)

# continue with trainer
trainer = ReinforceTrainer(embd, kernel_net, sampler, pred_net)
trainer.reg = args.reg
trainer.reg_mean = args.reg_mean
trainer.activation = nn.Sigmoid()
trainer.type(dtype)

print("created trainer")

params = [{'params': trainer.kernel_net.parameters(), 'lr': args.lr_k},
          {'params': trainer.pred_net.parameters(), 'lr': args.lr_p}]
optimizer = torch.optim.Adam(params)
print('set-up optimizer')

### Loop
l = []
torch.manual_seed(0)
print("started loop")
for epoch in range(args.epochs):

    adjust_learning_rate(optimizer, epoch)

    trainer.train()

    for t, (review, target) in enumerate(train_loader):
        review = Variable(review)

        if args.aspect == 'all':
            target = Variable(target[:,:3]).type(dtype)
        else:
            target = Variable(target[:,int(args.aspect[-1])]).type(dtype)

        loss  = trainer(review, target)
        
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        print('Weight mean is: ', trainer.kernel_net.layer1.weight.mean())
        print('Weight max is: ', trainer.kernel_net.layer1.weight.max())
        print('Weight min is: ', trainer.kernel_net.layer1.weight.min())
        print('Grad max is: ', trainer.kernel_net.layer1.weight.grad.max())
        print('Grad min is: ', trainer.kernel_net.layer1.weight.grad.min())
        print("trained one batch")
        
    loss, pred_loss, reg_loss = validate(val_loader, trainer)

    log(epoch, loss, pred_loss, reg_loss)
    print("logged")

    is_best = pred_loss < lowest_loss
    lowest_loss = min(pred_loss, lowest_loss)    
    save = {'epoch:': epoch + 1, 
            'model': 'Marginal Trainer',
            'state_dict': trainer.state_dict(),
            'lowest_loss': lowest_loss,
            'optimizer': optimizer.state_dict()} 

    save_checkpoint(save, is_best)
    print("saved a checkpoint")

print('*'*20, 'SUCCESS','*'*20)









    



loaded data
loader defined
created trainer
set-up optimizer
started loop
1
torch.Size([200, 1])
Zero Subset was produced. Re-sample
Weight mean is:  Variable containing:
1.00000e-05 *
 -8.7600
[torch.DoubleTensor of size 1]

Weight max is:  Variable containing:
1.00000e-02 *
  5.0096
[torch.DoubleTensor of size 1]

Weight min is:  Variable containing:
1.00000e-02 *
 -5.0100
[torch.DoubleTensor of size 1]

Grad max is:  Variable containing:
1.00000e-02 *
  5.2791
[torch.DoubleTensor of size 1]

Grad min is:  Variable containing:
1.00000e-02 *
 -4.9328
[torch.DoubleTensor of size 1]

trained one batch
1
torch.Size([200, 1])
Zero Subset was produced. Re-sample
Weight mean is:  Variable containing:
1.00000e-05 *
 -8.7408
[torch.DoubleTensor of size 1]

Weight max is:  Variable containing:
1.00000e-02 *
  5.0190
[torch.DoubleTensor of size 1]

Weight min is:  Variable containing:
1.00000e-02 *
 -5.0198
[torch.DoubleTensor of size 1]

Grad max is:  Variable containing:
1.00000e-02 *
  7.6660
[torch.DoubleTensor of size 1]

Grad min is:  Variable containing:
1.00000e-02 *
 -7.2060
[torch.DoubleTensor of size 1]

trained one batch
1
torch.Size([200, 1])
Zero Subset was produced. Re-sample
Weight mean is:  Variable containing:
1.00000e-05 *
 -8.7182
[torch.DoubleTensor of size 1]

Weight max is:  Variable containing:
1.00000e-02 *
  5.0289
[torch.DoubleTensor of size 1]

Weight min is:  Variable containing:
1.00000e-02 *
 -5.0296
[torch.DoubleTensor of size 1]

Grad max is:  Variable containing:
1.00000e-02 *
  9.9986
[torch.DoubleTensor of size 1]

Grad min is:  Variable containing:
1.00000e-02 *
 -9.3063
[torch.DoubleTensor of size 1]

trained one batch
1
torch.Size([200, 1])
Zero Subset was produced. Re-sample
Zero Subset was produced. Re-sample
Weight mean is:  Variable containing:
1.00000e-05 *
 -8.6944
[torch.DoubleTensor of size 1]

Weight max is:  Variable containing:
1.00000e-02 *
  5.0388
[torch.DoubleTensor of size 1]

Weight min is:  Variable containing:
1.00000e-02 *
 -5.0394
[torch.DoubleTensor of size 1]

Grad max is:  Variable containing:
1.00000e-02 *
  9.2098
[torch.DoubleTensor of size 1]

Grad min is:  Variable containing:
-0.1015
[torch.DoubleTensor of size 1]

trained one batch
1
torch.Size([200, 1])
Weight mean is:  Variable containing:
1.00000e-05 *
 -8.6792
[torch.DoubleTensor of size 1]

Weight max is:  Variable containing:
1.00000e-02 *
  5.0487
[torch.DoubleTensor of size 1]

Weight min is:  Variable containing:
1.00000e-02 *
 -5.0494
[torch.DoubleTensor of size 1]

Grad max is:  Variable containing:
1.00000e-02 *
  9.2073
[torch.DoubleTensor of size 1]

Grad min is:  Variable containing:
-0.1003
[torch.DoubleTensor of size 1]

trained one batch
1
torch.Size([200, 1])
Weight mean is:  Variable containing:
1.00000e-05 *
 -8.6623
[torch.DoubleTensor of size 1]

Weight max is:  Variable containing:
1.00000e-02 *
  5.0588
[torch.DoubleTensor of size 1]

Weight min is:  Variable containing:
1.00000e-02 *
 -5.0593
[torch.DoubleTensor of size 1]

Grad max is:  Variable containing:
1.00000e-02 *
  8.3028
[torch.DoubleTensor of size 1]

Grad min is:  Variable containing:
1.00000e-02 *
 -7.2037
[torch.DoubleTensor of size 1]

trained one batch
1
torch.Size([200, 1])
Zero Subset was produced. Re-sample
Zero Subset was produced. Re-sample
Zero Subset was produced. Re-sample
Weight mean is:  Variable containing:
1.00000e-05 *
 -8.6487
[torch.DoubleTensor of size 1]

Weight max is:  Variable containing:
1.00000e-02 *
  5.0688
[torch.DoubleTensor of size 1]

Weight min is:  Variable containing:
1.00000e-02 *
 -5.0688
[torch.DoubleTensor of size 1]

Grad max is:  Variable containing:
1.00000e-02 *
  7.1916
[torch.DoubleTensor of size 1]

Grad min is:  Variable containing:
1.00000e-02 *
 -8.5014
[torch.DoubleTensor of size 1]

trained one batch
1
torch.Size([200, 1])
Zero Subset was produced. Re-sample
Weight mean is:  Variable containing:
1.00000e-05 *
 -8.6377
[torch.DoubleTensor of size 1]

Weight max is:  Variable containing:
1.00000e-02 *
  5.0778
[torch.DoubleTensor of size 1]

Weight min is:  Variable containing:
1.00000e-02 *
 -5.0787
[torch.DoubleTensor of size 1]

Grad max is:  Variable containing:
1.00000e-02 *
  6.8469
[torch.DoubleTensor of size 1]

Grad min is:  Variable containing:
1.00000e-02 *
 -6.2047
[torch.DoubleTensor of size 1]

trained one batch
1
torch.Size([200, 1])
Zero Subset was produced. Re-sample
Weight mean is:  Variable containing:
1.00000e-05 *
 -8.6281
[torch.DoubleTensor of size 1]

Weight max is:  Variable containing:
1.00000e-02 *
  5.0868
[torch.DoubleTensor of size 1]

Weight min is:  Variable containing:
1.00000e-02 *
 -5.0883
[torch.DoubleTensor of size 1]

Grad max is:  Variable containing:
1.00000e-02 *
  8.5257
[torch.DoubleTensor of size 1]

Grad min is:  Variable containing:
1.00000e-02 *
 -8.8298
[torch.DoubleTensor of size 1]

trained one batch
1
torch.Size([200, 1])
Weight mean is:  Variable containing:
1.00000e-05 *
 -8.6214
[torch.DoubleTensor of size 1]

Weight max is:  Variable containing:
1.00000e-02 *
  5.0964
[torch.DoubleTensor of size 1]

Weight min is:  Variable containing:
1.00000e-02 *
 -5.0981
[torch.DoubleTensor of size 1]

Grad max is:  Variable containing:
1.00000e-02 *
  8.2128
[torch.DoubleTensor of size 1]

Grad min is:  Variable containing:
1.00000e-02 *
 -7.3762
[torch.DoubleTensor of size 1]

trained one batch
1
torch.Size([200, 1])
Weight mean is:  Variable containing:
1.00000e-05 *
 -8.6168
[torch.DoubleTensor of size 1]

Weight max is:  Variable containing:
1.00000e-02 *
  5.1065
[torch.DoubleTensor of size 1]

Weight min is:  Variable containing:
1.00000e-02 *
 -5.1072
[torch.DoubleTensor of size 1]

Grad max is:  Variable containing:
1.00000e-02 *
  7.3087
[torch.DoubleTensor of size 1]

Grad min is:  Variable containing:
1.00000e-02 *
 -7.6304
[torch.DoubleTensor of size 1]

trained one batch
1
torch.Size([200, 1])
Weight mean is:  Variable containing:
1.00000e-05 *
 -8.6175
[torch.DoubleTensor of size 1]

Weight max is:  Variable containing:
1.00000e-02 *
  5.1155
[torch.DoubleTensor of size 1]

Weight min is:  Variable containing:
1.00000e-02 *
 -5.1154
[torch.DoubleTensor of size 1]

Grad max is:  Variable containing:
1.00000e-02 *
  6.4287
[torch.DoubleTensor of size 1]

Grad min is:  Variable containing:
1.00000e-02 *
 -5.9047
[torch.DoubleTensor of size 1]

trained one batch
1
torch.Size([200, 1])
Weight mean is:  Variable containing:
nan
[torch.DoubleTensor of size 1]







    



---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-19-8c9d93a83dbc> in <module>()
     83     adjust_learning_rate(optimizer, epoch)
     84 
---> 85     train(train_loader, trainer, optimizer)
     86     loss, pred_loss, reg_loss = validate(val_loader, trainer)
     87 

<ipython-input-18-98b3e6573e54> in train(loader, trainer, optimizer)
     76         optimizer.step()
     77         print('Weight mean is: ', trainer.kernel_net.layer1.weight.mean())
---> 78         print('Weight max is: ', trainer.kernel_net.layer1.weight.max())
     79         print('Weight min is: ', trainer.kernel_net.layer1.weight.min())
     80         print('Grad max is: ', trainer.kernel_net.layer1.weight.grad.max())

/Users/Max/Coding/anaconda2/envs/torch/lib/python3.6/site-packages/torch/autograd/variable.py in max(self, dim)
    446         if isinstance(dim, Variable):
    447             return Cmax()(self, dim)
--> 448         return Max(dim)(self)
    449 
    450     def min(self, dim=None):

/Users/Max/Coding/anaconda2/envs/torch/lib/python3.6/site-packages/torch/autograd/_functions/reduce.py in forward(self, input)
    124         if self.dim is None and self.has_all_reduce:
    125             value = fn(*self.additional_args)
--> 126             self.indices = tuple(input.eq(value).nonzero()[0])
    127             return input.new((value,))
    128         else:

RuntimeError: dimension 0 out of range of 0D tensor at /Users/soumith/miniconda2/conda-bld/pytorch_1493757319118/work/torch/lib/TH/generic/THTensor.c:24



In [25]:

    
trainer.kernel_net.layer1.weight.min()









    



---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-25-33abc841ff65> in <module>()
----> 1 trainer.kernel_net.layer1.weight.min()

/Users/Max/Coding/anaconda2/envs/torch/lib/python3.6/site-packages/torch/autograd/variable.py in min(self, dim)
    451         if isinstance(dim, Variable):
    452             return Cmin()(self, dim)
--> 453         return Min(dim)(self)
    454 
    455     def mode(self, dim):

/Users/Max/Coding/anaconda2/envs/torch/lib/python3.6/site-packages/torch/autograd/_functions/reduce.py in forward(self, input)
    124         if self.dim is None and self.has_all_reduce:
    125             value = fn(*self.additional_args)
--> 126             self.indices = tuple(input.eq(value).nonzero()[0])
    127             return input.new((value,))
    128         else:

RuntimeError: dimension 0 out of range of 0D tensor at /Users/soumith/miniconda2/conda-bld/pytorch_1493757319118/work/torch/lib/TH/generic/THTensor.c:24



In [17]:









    



---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-17-998c0c3ed835> in <module>()
----> 1 embd, word_to_ix = make_embd(path)

NameError: name 'path' is not defined



In [ ]: