notebook.community

Edit and run 





In [1]:



    


import os, glob, platform, datetime, random
from collections import OrderedDict

import torch
import torch.nn as nn
import torch.utils.data as data_utils
import torch.nn.parallel
import torch.backends.cudnn as cudnn
import torch.optim as optim
from torch.autograd import Variable
from torch import functional as F
# import torchvision.datasets as datasets
import torchvision.models as models
import torchvision.transforms as transforms

import cv2
from PIL import Image
from tensorboardX import SummaryWriter

import numpy as np
from numpy.linalg import inv as denseinv
from scipy import sparse
from scipy.sparse import lil_matrix, csr_matrix
from scipy.sparse.linalg import spsolve
from scipy.sparse.linalg import inv as spinv
import scipy.misc

from myimagefolder import MyImageFolder
from mymodel import GradientNet
from myargs import Args

Configurations





In [2]:



    


args = Args()
args.test_scene = 'alley_1'
args.arch = "densenet121"
args.epoches = 500
args.epoches_unary_threshold = 0
args.image_h = 256
args.image_w = 256
args.img_extentions = ["png"]
args.training_thresholds = [250,200,150,50,0,300]
args.base_lr = 1
args.lr = args.base_lr
args.snapshot_interval = 5000
args.debug = True
args.gpu_num = 2
# growth_rate = (4*(2**(args.gpu_num)))
# transition_scale=(2*(2**(args.gpu_num+1)))
transition_scale=2
pretrained_scale=2 
growth_rate = 32
args.display_interval = 50
args.display_curindex = 0

system_ = platform.system()
system_dist, system_version, _ = platform.dist()
if system_ == "Darwin": 
    args.train_dir = '/Volumes/Transcend/dataset/sintel2'
    args.pretrained = False
elif platform.dist() ==  ('debian', 'jessie/sid', ''):
    args.train_dir = '/home/lwp/workspace/sintel2'
    args.pretrained = True
elif platform.dist() == ('debian', 'stretch/sid', ''):
    args.train_dir = '/home/cad/lwp/workspace/dataset/sintel2'
    args.pretrained = True

if platform.system() == 'Linux': use_gpu = True
else: use_gpu = False
if use_gpu:
    torch.cuda.set_device(args.gpu_num)
    

print(platform.dist())



    


















    






('debian', 'jessie/sid', '')

My DataLoader





In [3]:



    


train_dataset = MyImageFolder(args.train_dir, 'train',
                       transforms.Compose(
        [transforms.ToTensor()]
    ), random_crop=True, 
    img_extentions=args.img_extentions, test_scene=args.test_scene, image_h=args.image_h, image_w=args.image_w)
test_dataset = MyImageFolder(args.train_dir, 'test', 
                       transforms.Compose(
        [transforms.CenterCrop((args.image_h, args.image_w)),
         transforms.ToTensor()]
    ), random_crop=False,
    img_extentions=args.img_extentions, test_scene=args.test_scene, image_h=args.image_h, image_w=args.image_w)

train_loader = data_utils.DataLoader(train_dataset,1,True,num_workers=1)
test_loader = data_utils.DataLoader(test_dataset,1,True,num_workers=1)

Load Pretrained Model

Defination

  • DenseNet-121: num_init_features=64, growth_rate=32, block_config=(6, 12, 24, 16)
    • First Convolution: 32M -> 16M -> 8M
    • every transition: 8M -> 4M -> 2M (downsample 1/2, except the last block)




In [4]:



    


densenet = models.__dict__[args.arch](pretrained=args.pretrained)

for param in densenet.parameters():
    param.requires_grad = False

if use_gpu: densenet.cuda()



    











In [5]:



    


ss = 6

args.display_curindex = 0
args.base_lr = 0.05
args.display_interval = 20
args.momentum = 0.9
args.epoches = 120
args.training_thresholds = [ss*4,ss*3,ss*2,ss*1,ss*0,ss*5]
args.power = 0.5



# pretrained = PreTrainedModel(densenet)
# if use_gpu: 
#     pretrained.cuda()


net = GradientNet(densenet=densenet, growth_rate=growth_rate, transition_scale=transition_scale, pretrained_scale=pretrained_scale)
if use_gpu:
    net.cuda()

if use_gpu: 
    mse_losses = [nn.MSELoss().cuda()] * 6
    test_losses = [nn.MSELoss().cuda()] * 6
else:
    mse_losses = [nn.MSELoss()] * 6
    test_losses = [nn.MSELoss()] * 6



    











In [6]:



    


# training loop
writer = SummaryWriter(comment='-pretrained_scale_{}'.format(pretrained_scale))

parameters = filter(lambda p: p.requires_grad, net.parameters())
optimizer = optim.SGD(parameters, lr=args.base_lr, momentum=args.momentum)

def adjust_learning_rate(optimizer, epoch, beg, end, reset_lr=None):
    """Sets the learning rate to the initial LR decayed by 10 every 30 epochs"""
    for param_group in optimizer.param_groups:
        if reset_lr != None:
            param_group['lr'] = reset_lr
            continue
        if epoch != 0: 
            # linear
#             param_group['lr'] *= (end-epoch) / (end-beg)
#             poly base_lr (1 - iter/max_iter) ^ (power)
            param_group['lr'] = args.base_lr * (float(end-epoch)/(end-beg)) ** (args.power)
            if param_group['lr'] < 1.0e-8: param_group['lr'] = 1.0e-8
        print('lr', param_group['lr'])
        
# def findLargerInd(target, arr):
#     res = list(filter(lambda x: x>target, arr))
#     print('res',res)
#     if len(res) == 0: return -1
#     return res[0]

for epoch in range(args.epoches):
    net.train()
    print('epoch: {} [{}]'.format(epoch, datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
    
    if epoch < args.training_thresholds[-1]: adjust_learning_rate(optimizer, epoch%ss, beg=0, end=ss-1)
    else: adjust_learning_rate(optimizer, epoch, beg=args.training_thresholds[-1], end=args.epoches-1)
    
    run_losses = [0] * len(args.training_thresholds)
    run_cnts   = [0.00001] * len(args.training_thresholds)
    if (epoch in args.training_thresholds) == True: 
        adjust_learning_rate(optimizer, epoch, reset_lr=args.base_lr, beg=-1, end=-1)
    writer.add_scalar('learning rate', optimizer.param_groups[0]['lr'], global_step=epoch)
    for ind, data in enumerate(train_loader, 0):
#         if  ind == 1 : break
        input_img, gt_albedo, gt_shading, test_scene, img_path = data
        im = input_img[0,:,:,:].numpy(); im = im.transpose(1,2,0); im = im[:,:,::-1]*255
        
        if test_scene[0] == 'alley_1':
            print('alley_1 yes')
        input_img = Variable(input_img)
        gt_albedo = Variable(gt_albedo)
        gt_shading = Variable(gt_shading)
        if use_gpu:
            input_img = input_img.cuda()
            gt_albedo = gt_albedo.cuda()
            gt_shading = gt_shading.cuda()
#         run_losses = [0] * len(mse_losses)
#         run_cnts = [0.00001] * len(mse_losses)
        if args.display_curindex % args.display_interval == 0:
            cv2.imwrite('snapshot{}/input.png'.format(args.gpu_num), im)

        optimizer.zero_grad()
#         pretrained.train(); ft_pretreained = pretrained(input_img)
        ft_predict = net(input_img)
        for i, threshold in enumerate(args.training_thresholds):
#             threshold = args.training_thresholds[i]
            if epoch >= threshold:
#             if epoch >= 0:
                if i == 5: s = 1
                else: s = (2**(i+1))
                gt = gt_albedo.cpu().data.numpy()
                n,c,h,w = gt.shape
                gt = gt[0,:,:,:]
                gt = gt.transpose((1,2,0))
                gt = cv2.resize(gt, (h//s, w//s))
#                 gt = cv2.resize(gt, (h,w))
                if args.display_curindex % args.display_interval == 0:
                    cv2.imwrite('snapshot{}/gt-{}-{}.png'.format(args.gpu_num, epoch, i), gt[:,:,::-1]*255)
                gt = gt.transpose((2,0,1))
                gt = gt[np.newaxis, :]
                gt = Variable(torch.from_numpy(gt))
                if use_gpu: gt = gt.cuda()
                loss = mse_losses[i](ft_predict[i], gt)
                loss_data = loss.data.cpu().numpy()
                writer.add_scalar('{}th train iters loss'.format(i), loss_data, global_step=args.display_curindex)
                ma_ = ft_predict[i].max().cpu().data.numpy()
                mi_ = ft_predict[i].min().cpu().data.numpy()
                #print('mi', mi_, 'ma', ma_)
#                 writer.add_scalars('{}th train predict'.format(i), {'max': ma_, 'min': mi_}, global_step=args.display_curindex)
#                 run_cnts[i] += 1
                run_losses[i] += loss.data.cpu().numpy()[0]
                loss.backward(retain_graph=True)
                run_cnts[i] += 1
#                 print('i = ', i, '; weig\n', net.upsample01.weight[0,0,0:4,0:4].data.cpu().numpy())
#                 print('i = ', i, '; grad\n', net.upsample01.weight.grad[0,0,0:4,0:4].data.cpu().numpy())
                if args.display_curindex % args.display_interval == 0:
                    im = ft_predict[i].cpu().data.numpy()[0].transpose((1,2,0)) * 255
                    cv2.imwrite('snapshot{}/train-{}-{}.png'.format(args.gpu_num, epoch, i), im[:,:,::-1])
        optimizer.step()
        args.display_curindex += 1

    """ every epoch """
#     loss_output = 'ind: ' + str(args.display_curindex)
    loss_output = ''
    
    
    
    for i,v in enumerate(run_losses):
        if i == len(run_losses)-1: 
            loss_output += ' merged: %6f' % (run_losses[i] / run_cnts[i])
            continue
        loss_output += ' %2dM: %6f' % ((2**(4-i)), (run_losses[i] / run_cnts[i]))
    print(loss_output)
    # save at every epoch
    if (epoch+1) % 10 == 0:
        torch.save({
            'epoch': epoch,
            'args' : args,
            'state_dict': net.state_dict(),
            'optimizer': optimizer.state_dict()
        }, 'snapshot{}/snapshot-{}.pth.tar'.format(args.gpu_num, epoch))
    
    # test 
    test_losses_trainphase = [0] * len(args.training_thresholds)
    test_cnts_trainphase   = [0.00001] * len(args.training_thresholds)   
    for ind, data in enumerate(test_loader, 0):
        input_img, gt_albedo, gt_shading, test_scene, img_path = data
        input_img = Variable(input_img)
        gt_albedo = Variable(gt_albedo)
        gt_shading = Variable(gt_shading)
        if use_gpu:
            input_img = input_img.cuda(args.gpu_num)
        
#         pretrained.train(); ft_pretreained = pretrained(input_img)
        ft_test = net(input_img)
            
        for i,v in enumerate(ft_test):
            if epoch < args.training_thresholds[i]: continue
            if i == 5: s = 1
            else: s = (2**(i+1))
            gt = gt_albedo.data.numpy()
            n,c,h,w = gt.shape
            gt = gt[0,:,:,:]
            gt = gt.transpose((1,2,0))
            gt = cv2.resize(gt, (h//s, w//s))
#             gt = cv2.resize(gt, (h,w))
            
            gt = gt.transpose((2,0,1))
            gt = gt[np.newaxis, :]
            gt = Variable(torch.from_numpy(gt))
            if use_gpu: gt = gt.cuda()

            loss = mse_losses[i](ft_test[i], gt)
            
            test_losses_trainphase[i] += loss.data.cpu().numpy()[0]
            test_cnts_trainphase[i] += 1
            v = v[0].cpu().data.numpy()
            v = v.transpose(1,2,0)
            if ind == 0: cv2.imwrite('snapshot{}/test-phase_train-{}-{}.png'.format(args.gpu_num, epoch, i), v[:,:,::-1]*255)

    
    net.eval()
    test_losses = [0] * len(args.training_thresholds)
    test_cnts   = [0.00001] * len(args.training_thresholds)   
    for ind, data in enumerate(test_loader, 0):
#         if ind == 1: break
        input_img, gt_albedo, gt_shading, test_scene, img_path = data
        input_img = Variable(input_img)
        gt_albedo = Variable(gt_albedo)
        gt_shading = Variable(gt_shading)
        if use_gpu:
            input_img = input_img.cuda(args.gpu_num)
            
#         pretrained.eval(); ft_pretreained = pretrained(input_img)
        ft_test = net(input_img)
            
        for i,v in enumerate(ft_test):
            if epoch < args.training_thresholds[i]: continue
            if i == 5: s = 1
            else: s = (2**(i+1))
            gt = gt_albedo.data.numpy()
            n,c,h,w = gt.shape
            gt = gt[0,:,:,:]
            gt = gt.transpose((1,2,0))
            gt = cv2.resize(gt, (h//s, w//s))
#             gt = cv2.resize(gt, (h,w))
            
            gt = gt.transpose((2,0,1))
            gt = gt[np.newaxis, :]
            gt = Variable(torch.from_numpy(gt))
            if use_gpu: gt = gt.cuda()

            loss = mse_losses[i](ft_test[i], gt)
            
            test_losses[i] += loss.data.cpu().numpy()[0]
            test_cnts[i] += 1
            v = v[0].cpu().data.numpy()
            v = v.transpose(1,2,0)
            if ind == 0: cv2.imwrite('snapshot{}/test-phase_test-{}-{}.png'.format(args.gpu_num, epoch, i), v[:,:,::-1]*255)
    
    writer.add_scalars('16M loss', {
        'train 16M ': np.array([run_losses[0]/ run_cnts[0]]),
        'test_trainphase 16M ': np.array([test_losses_trainphase[0]/ test_cnts_trainphase[0]]),
        'test 16M ': np.array([test_losses[0]/ test_cnts[0]])
    }, global_step=epoch)  
    writer.add_scalars('8M loss', {
        'train 8M ': np.array([run_losses[1]/ run_cnts[1]]),
        'test_trainphase 8M ': np.array([test_losses_trainphase[1]/ test_cnts_trainphase[1]]),
        'test 8M ': np.array([test_losses[1]/ test_cnts[1]])
    }, global_step=epoch) 
    writer.add_scalars('4M loss', {
        'train 4M ': np.array([run_losses[2]/ run_cnts[2]]),
        'test_trainphase 4M ': np.array([test_losses_trainphase[2]/ test_cnts_trainphase[2]]),
        'test 4M ': np.array([test_losses[2]/ test_cnts[2]])
    }, global_step=epoch) 
    writer.add_scalars('2M loss', {
        'train 2M ': np.array([run_losses[3]/ run_cnts[3]]),
        'test_trainphase 2M ': np.array([test_losses_trainphase[3]/ test_cnts_trainphase[3]]),
        'test 2M ': np.array([test_losses[3]/ test_cnts[3]])
    }, global_step=epoch) 
    writer.add_scalars('1M loss', {
        'train 1M ': np.array([run_losses[4]/ run_cnts[4]]),
        'test_trainphase 1M ': np.array([test_losses_trainphase[4]/ test_cnts_trainphase[4]]),
        'test 1M ': np.array([test_losses[4]/ test_cnts[4]])
    }, global_step=epoch) 
    writer.add_scalars('merged loss', {
        'train merged ': np.array([run_losses[5]/ run_cnts[5]]),
        'test_trainphase merged ': np.array([test_losses_trainphase[5]/ test_cnts_trainphase[5]]),
        'test merged ': np.array([test_losses[5]/ test_cnts[5]])
    }, global_step=epoch)



    


















    






epoch: 0 [2017-11-23 15:43:50]
lr 0.05
 16M: 0.000000  8M: 0.000000  4M: 0.000000  2M: 0.000000  1M: 0.061749 merged: 0.000000
epoch: 1 [2017-11-23 15:45:59]
lr 0.044721359549995794
 16M: 0.000000  8M: 0.000000  4M: 0.000000  2M: 0.000000  1M: 0.052725 merged: 0.000000
epoch: 2 [2017-11-23 15:48:08]
lr 0.038729833462074176
 16M: 0.000000  8M: 0.000000  4M: 0.000000  2M: 0.000000  1M: 0.053128 merged: 0.000000
epoch: 3 [2017-11-23 15:50:17]
lr 0.0316227766016838
 16M: 0.000000  8M: 0.000000  4M: 0.000000  2M: 0.000000  1M: 0.043970 merged: 0.000000
epoch: 4 [2017-11-23 15:52:26]
lr 0.022360679774997897
 16M: 0.000000  8M: 0.000000  4M: 0.000000  2M: 0.000000  1M: 0.039612 merged: 0.000000
epoch: 5 [2017-11-23 15:54:34]
lr 1e-08
 16M: 0.000000  8M: 0.000000  4M: 0.000000  2M: 0.000000  1M: 0.033170 merged: 0.000000
epoch: 6 [2017-11-23 15:56:44]
lr 1e-08
 16M: 0.000000  8M: 0.000000  4M: 0.000000  2M: 0.054356  1M: 0.041168 merged: 0.000000
epoch: 7 [2017-11-23 15:59:14]
lr 0.044721359549995794
 16M: 0.000000  8M: 0.000000  4M: 0.000000  2M: 0.047150  1M: 0.035816 merged: 0.000000
epoch: 8 [2017-11-23 16:01:53]
lr 0.038729833462074176
 16M: 0.000000  8M: 0.000000  4M: 0.000000  2M: 0.041568  1M: 0.032384 merged: 0.000000
epoch: 9 [2017-11-23 16:04:29]
lr 0.0316227766016838
 16M: 0.000000  8M: 0.000000  4M: 0.000000  2M: 0.037467  1M: 0.028164 merged: 0.000000
epoch: 10 [2017-11-23 16:07:03]
lr 0.022360679774997897
 16M: 0.000000  8M: 0.000000  4M: 0.000000  2M: 0.030773  1M: 0.025246 merged: 0.000000
epoch: 11 [2017-11-23 16:09:39]
lr 1e-08
 16M: 0.000000  8M: 0.000000  4M: 0.000000  2M: 0.027782  1M: 0.022723 merged: 0.000000
epoch: 12 [2017-11-23 16:12:13]
lr 1e-08
 16M: 0.000000  8M: 0.000000  4M: 0.048199  2M: 0.038905  1M: 0.030557 merged: 0.000000
epoch: 13 [2017-11-23 16:15:02]
lr 0.044721359549995794
 16M: 0.000000  8M: 0.000000  4M: 0.040532  2M: 0.033165  1M: 0.027536 merged: 0.000000
epoch: 14 [2017-11-23 16:17:51]
lr 0.038729833462074176
 16M: 0.000000  8M: 0.000000  4M: 0.031451  2M: 0.029651  1M: 0.025562 merged: 0.000000
epoch: 15 [2017-11-23 16:20:40]
lr 0.0316227766016838
 16M: 0.000000  8M: 0.000000  4M: 0.026645  2M: 0.026643  1M: 0.023446 merged: 0.000000
epoch: 16 [2017-11-23 16:23:33]
lr 0.022360679774997897
 16M: 0.000000  8M: 0.000000  4M: 0.019591  2M: 0.023343  1M: 0.021519 merged: 0.000000
epoch: 17 [2017-11-23 16:26:27]
lr 1e-08
 16M: 0.000000  8M: 0.000000  4M: 0.017721  2M: 0.021446  1M: 0.021573 merged: 0.000000
epoch: 18 [2017-11-23 16:29:22]
lr 1e-08
 16M: 0.000000  8M: 0.046284  4M: 0.024792  2M: 0.027905  1M: 0.025762 merged: 0.000000
epoch: 19 [2017-11-23 16:32:29]
lr 0.044721359549995794
 16M: 0.000000  8M: 0.034120  4M: 0.020406  2M: 0.026634  1M: 0.024733 merged: 0.000000
epoch: 20 [2017-11-23 16:35:34]
lr 0.038729833462074176
 16M: 0.000000  8M: 0.024971  4M: 0.017307  2M: 0.030066  1M: 0.023905 merged: 0.000000
epoch: 21 [2017-11-23 16:38:37]
lr 0.0316227766016838
 16M: 0.000000  8M: 0.021535  4M: 0.015632  2M: 0.025065  1M: 0.021824 merged: 0.000000
epoch: 22 [2017-11-23 16:41:42]
lr 0.022360679774997897
 16M: 0.000000  8M: 0.017616  4M: 0.013509  2M: 0.021272  1M: 0.020102 merged: 0.000000
epoch: 23 [2017-11-23 16:44:44]
lr 1e-08
 16M: 0.000000  8M: 0.016608  4M: 0.013220  2M: 0.020948  1M: 0.019095 merged: 0.000000
epoch: 24 [2017-11-23 16:47:46]
lr 1e-08
 16M: 0.042411  8M: 0.022264  4M: 0.015992  2M: 0.024569  1M: 0.023438 merged: 0.000000
epoch: 25 [2017-11-23 16:51:03]
lr 0.044721359549995794
 16M: 0.028714  8M: 0.018939  4M: 0.015233  2M: 0.023007  1M: 0.022507 merged: 0.000000
epoch: 26 [2017-11-23 16:54:20]
lr 0.038729833462074176
 16M: 0.022393  8M: 0.015802  4M: 0.012899  2M: 0.020763  1M: 0.020963 merged: 0.000000
epoch: 27 [2017-11-23 16:57:39]
lr 0.0316227766016838
 16M: 0.019004  8M: 0.014733  4M: 0.012350  2M: 0.020110  1M: 0.020407 merged: 0.000000
epoch: 28 [2017-11-23 17:00:55]
lr 0.022360679774997897
 16M: 0.016595  8M: 0.012915  4M: 0.011033  2M: 0.018262  1M: 0.018600 merged: 0.000000
epoch: 29 [2017-11-23 17:04:11]
lr 1e-08
 16M: 0.015360  8M: 0.012117  4M: 0.010901  2M: 0.017833  1M: 0.018358 merged: 0.000000
epoch: 30 [2017-11-23 17:07:27]
lr 0.05
 16M: 0.021415  8M: 0.016258  4M: 0.014282  2M: 0.021458  1M: 0.021148 merged: 0.032925
epoch: 31 [2017-11-23 17:12:25]
lr 0.04971830761761256
 16M: 0.022185  8M: 0.023279  4M: 0.018598  2M: 0.020615  1M: 0.021569 merged: 0.025475
epoch: 32 [2017-11-23 17:17:23]
lr 0.04943501011144937
 16M: 0.019132  8M: 0.018303  4M: 0.016544  2M: 0.019855  1M: 0.020909 merged: 0.020090
epoch: 33 [2017-11-23 17:22:23]
lr 0.04915007972606608
 16M: 0.016329  8M: 0.016702  4M: 0.014586  2M: 0.018572  1M: 0.020538 merged: 0.017676
epoch: 34 [2017-11-23 17:27:24]
lr 0.04886348789677424
 16M: 0.015871  8M: 0.015402  4M: 0.014856  2M: 0.018289  1M: 0.020412 merged: 0.017023
epoch: 35 [2017-11-23 17:32:22]
lr 0.04857520521621862
 16M: 0.015793  8M: 0.015007  4M: 0.014322  2M: 0.017976  1M: 0.020073 merged: 0.015911
epoch: 36 [2017-11-23 17:37:22]
lr 0.04828520139915856
 16M: 0.016156  8M: 0.014992  4M: 0.015313  2M: 0.017490  1M: 0.019986 merged: 0.016184
epoch: 37 [2017-11-23 17:42:22]
lr 0.047993445245333805
 16M: 0.013565  8M: 0.013129  4M: 0.012637  2M: 0.015949  1M: 0.018459 merged: 0.013775
epoch: 38 [2017-11-23 17:47:21]
lr 0.0476999046002862
 16M: 0.013222  8M: 0.012648  4M: 0.012508  2M: 0.017266  1M: 0.019468 merged: 0.013722
epoch: 39 [2017-11-23 17:52:22]
lr 0.04740454631399772
 16M: 0.012955  8M: 0.012420  4M: 0.011914  2M: 0.016839  1M: 0.018849 merged: 0.012459
epoch: 40 [2017-11-23 17:57:22]
lr 0.04710733619719444
 16M: 0.011856  8M: 0.011745  4M: 0.011369  2M: 0.015534  1M: 0.018191 merged: 0.012052
epoch: 41 [2017-11-23 18:02:23]
lr 0.04680823897515326
 16M: 0.011841  8M: 0.012324  4M: 0.011199  2M: 0.015781  1M: 0.018334 merged: 0.011777
epoch: 42 [2017-11-23 18:07:22]
lr 0.04650721823883479
 16M: 0.011838  8M: 0.011855  4M: 0.011176  2M: 0.015697  1M: 0.018063 merged: 0.011704
epoch: 43 [2017-11-23 18:12:21]
lr 0.046204236393150765
 16M: 0.011513  8M: 0.011159  4M: 0.010881  2M: 0.015359  1M: 0.017361 merged: 0.011144
epoch: 44 [2017-11-23 18:17:20]
lr 0.045899254602157845
 16M: 0.011278  8M: 0.010939  4M: 0.010826  2M: 0.014878  1M: 0.018060 merged: 0.010946
epoch: 45 [2017-11-23 18:22:21]
lr 0.04559223273095164
 16M: 0.010866  8M: 0.010551  4M: 0.010173  2M: 0.014542  1M: 0.016649 merged: 0.010555
epoch: 46 [2017-11-23 18:27:22]
lr 0.045283129284014914
 16M: 0.010440  8M: 0.010888  4M: 0.010203  2M: 0.014535  1M: 0.017305 merged: 0.010369
epoch: 47 [2017-11-23 18:32:21]
lr 0.04497190133975169
 16M: 0.010202  8M: 0.010395  4M: 0.009633  2M: 0.013966  1M: 0.016749 merged: 0.009710
epoch: 48 [2017-11-23 18:37:20]
lr 0.04465850448091506
 16M: 0.010240  8M: 0.010220  4M: 0.009569  2M: 0.013824  1M: 0.016464 merged: 0.009741
epoch: 49 [2017-11-23 18:42:21]
lr 0.044342892720609255
 16M: 0.010124  8M: 0.009717  4M: 0.009469  2M: 0.013567  1M: 0.016062 merged: 0.009343
epoch: 50 [2017-11-23 18:47:22]
lr 0.044025018423517
 16M: 0.009714  8M: 0.010024  4M: 0.009352  2M: 0.013435  1M: 0.016484 merged: 0.009414
epoch: 51 [2017-11-23 18:52:19]
lr 0.04370483222197017
 16M: 0.009558  8M: 0.009387  4M: 0.009000  2M: 0.013293  1M: 0.016029 merged: 0.008850
epoch: 52 [2017-11-23 18:57:18]
lr 0.043382282926444894
 16M: 0.009487  8M: 0.009411  4M: 0.008993  2M: 0.013218  1M: 0.015621 merged: 0.008743
epoch: 53 [2017-11-23 19:02:17]
lr 0.04305731743002185
 16M: 0.009193  8M: 0.009226  4M: 0.008898  2M: 0.013228  1M: 0.015734 merged: 0.008531
epoch: 54 [2017-11-23 19:07:16]
lr 0.04272988060630656
 16M: 0.008838  8M: 0.008841  4M: 0.008547  2M: 0.012620  1M: 0.015548 merged: 0.008227
epoch: 55 [2017-11-23 19:12:29]
lr 0.04239991520025441
 16M: 0.008983  8M: 0.009077  4M: 0.008733  2M: 0.012974  1M: 0.015680 merged: 0.008443
epoch: 56 [2017-11-23 19:17:49]
lr 0.0420673617112877
 16M: 0.008986  8M: 0.008960  4M: 0.008556  2M: 0.012838  1M: 0.015870 merged: 0.008108
epoch: 57 [2017-11-23 19:23:09]
lr 0.041732158268029534
 16M: 0.008694  8M: 0.009716  4M: 0.008673  2M: 0.012054  1M: 0.015157 merged: 0.008257
epoch: 58 [2017-11-23 19:28:30]
lr 0.041394240493907074
 16M: 0.008667  8M: 0.008985  4M: 0.008518  2M: 0.012741  1M: 0.015435 merged: 0.007953
epoch: 59 [2017-11-23 19:33:49]
lr 0.041053541362798006
 16M: 0.008380  8M: 0.008758  4M: 0.008401  2M: 0.012452  1M: 0.015300 merged: 0.007788
epoch: 60 [2017-11-23 19:39:10]
lr 0.04070999104380296
 16M: 0.008352  8M: 0.008565  4M: 0.008265  2M: 0.013773  1M: 0.015569 merged: 0.007775
epoch: 61 [2017-11-23 19:44:30]
lr 0.04036351673412598
 16M: 0.008081  8M: 0.008214  4M: 0.008094  2M: 0.014333  1M: 0.015272 merged: 0.007555
epoch: 62 [2017-11-23 19:49:50]
lr 0.04001404247893005
 16M: 0.008216  8M: 0.008234  4M: 0.008094  2M: 0.013062  1M: 0.015144 merged: 0.007409
epoch: 63 [2017-11-23 19:55:11]
lr 0.03966148897690515
 16M: 0.008613  8M: 0.008948  4M: 0.008993  2M: 0.013884  1M: 0.014983 merged: 0.008234
epoch: 64 [2017-11-23 20:00:32]
lr 0.03930577337013889
 16M: 0.008002  8M: 0.008328  4M: 0.008565  2M: 0.013085  1M: 0.014986 merged: 0.007627
epoch: 65 [2017-11-23 20:05:51]
lr 0.038946809016712394
 16M: 0.008012  8M: 0.008364  4M: 0.008271  2M: 0.012429  1M: 0.014087 merged: 0.007576
epoch: 66 [2017-11-23 20:11:13]
lr 0.03858450524425343
 16M: 0.007763  8M: 0.007907  4M: 0.007755  2M: 0.012087  1M: 0.014657 merged: 0.007096
epoch: 67 [2017-11-23 20:16:31]
lr 0.03821876708246056
 16M: 0.007717  8M: 0.007969  4M: 0.007727  2M: 0.012124  1M: 0.014535 merged: 0.007058
epoch: 68 [2017-11-23 20:21:51]
lr 0.03784949497236286
 16M: 0.007452  8M: 0.007623  4M: 0.007487  2M: 0.011679  1M: 0.014346 merged: 0.006743
epoch: 69 [2017-11-23 20:27:10]
lr 0.03747658444979307
 16M: 0.007620  8M: 0.007704  4M: 0.007459  2M: 0.011938  1M: 0.014536 merged: 0.006780
epoch: 70 [2017-11-23 20:32:28]
lr 0.0370999258002226
 16M: 0.007867  8M: 0.007893  4M: 0.007686  2M: 0.011805  1M: 0.014039 merged: 0.006990
epoch: 71 [2017-11-23 20:37:46]
lr 0.03671940368172628
 16M: 0.007620  8M: 0.008261  4M: 0.007557  2M: 0.011563  1M: 0.013867 merged: 0.006994
epoch: 72 [2017-11-23 20:43:03]
lr 0.03633489671240478
 16M: 0.007338  8M: 0.007823  4M: 0.007446  2M: 0.011399  1M: 0.013837 merged: 0.006714
epoch: 73 [2017-11-23 20:48:21]
lr 0.03594627701808178
 16M: 0.007256  8M: 0.007575  4M: 0.007260  2M: 0.014995  1M: 0.014202 merged: 0.006608
epoch: 74 [2017-11-23 20:53:38]
lr 0.035553409735498295
 16M: 0.007096  8M: 0.007373  4M: 0.007120  2M: 0.012813  1M: 0.013896 merged: 0.006356
epoch: 75 [2017-11-23 20:58:55]
lr 0.03515615246553262
 16M: 0.006784  8M: 0.007116  4M: 0.006912  2M: 0.011938  1M: 0.013492 merged: 0.006233
epoch: 76 [2017-11-23 21:04:14]
lr 0.03475435467016077
 16M: 0.006997  8M: 0.007223  4M: 0.006931  2M: 0.011988  1M: 0.013810 merged: 0.006256
epoch: 77 [2017-11-23 21:09:32]
lr 0.034347857005916346
 16M: 0.006832  8M: 0.007075  4M: 0.007027  2M: 0.011841  1M: 0.013836 merged: 0.006169
epoch: 78 [2017-11-23 21:14:49]
lr 0.0339364905854808
 16M: 0.006917  8M: 0.007143  4M: 0.007105  2M: 0.011662  1M: 0.013881 merged: 0.006245
epoch: 79 [2017-11-23 21:20:07]
lr 0.03352007615769955
 16M: 0.006657  8M: 0.007028  4M: 0.006982  2M: 0.011546  1M: 0.013952 merged: 0.006115
epoch: 80 [2017-11-23 21:25:25]
lr 0.03309842319473132
 16M: 0.006581  8M: 0.006779  4M: 0.006679  2M: 0.011105  1M: 0.013425 merged: 0.005939
epoch: 81 [2017-11-23 21:30:43]
lr 0.03267132887314317
 16M: 0.006672  8M: 0.006971  4M: 0.006745  2M: 0.011028  1M: 0.013178 merged: 0.006051
epoch: 82 [2017-11-23 21:36:01]
lr 0.03223857693349118
 16M: 0.006825  8M: 0.006993  4M: 0.006999  2M: 0.011478  1M: 0.013741 merged: 0.006066
epoch: 83 [2017-11-23 21:41:20]
lr 0.0317999364001908
 16M: 0.006512  8M: 0.006823  4M: 0.006578  2M: 0.010969  1M: 0.013309 merged: 0.005784
epoch: 84 [2017-11-23 21:46:38]
lr 0.031355160140170396
 16M: 0.006522  8M: 0.006795  4M: 0.006689  2M: 0.010885  1M: 0.013306 merged: 0.005857
epoch: 85 [2017-11-23 21:51:57]
lr 0.03090398323477543
 16M: 0.006341  8M: 0.006649  4M: 0.006534  2M: 0.010925  1M: 0.013379 merged: 0.005730
epoch: 86 [2017-11-23 21:57:14]
lr 0.030446121134470178
 16M: 0.006234  8M: 0.006618  4M: 0.006669  2M: 0.010597  1M: 0.013077 merged: 0.005705
epoch: 87 [2017-11-23 22:02:32]
lr 0.02998126755983446
 16M: 0.006387  8M: 0.006712  4M: 0.006657  2M: 0.010853  1M: 0.013229 merged: 0.005810
epoch: 88 [2017-11-23 22:07:52]
lr 0.029509092104873926
 16M: 0.006382  8M: 0.006644  4M: 0.006575  2M: 0.010638  1M: 0.012995 merged: 0.005699
epoch: 89 [2017-11-23 22:13:13]
lr 0.029029237489356888
 16M: 0.006311  8M: 0.006621  4M: 0.006639  2M: 0.010650  1M: 0.013396 merged: 0.005779
epoch: 90 [2017-11-23 22:18:35]
lr 0.028541316395237167
 16M: 0.006132  8M: 0.006440  4M: 0.006338  2M: 0.010317  1M: 0.012719 merged: 0.005483
epoch: 91 [2017-11-23 22:23:55]
lr 0.028044907807525134
 16M: 0.006191  8M: 0.006398  4M: 0.006477  2M: 0.010524  1M: 0.013065 merged: 0.005590
epoch: 92 [2017-11-23 22:29:15]
lr 0.027539552761294706
 16M: 0.006230  8M: 0.006478  4M: 0.006467  2M: 0.010521  1M: 0.013201 merged: 0.005608
epoch: 93 [2017-11-23 22:34:33]
lr 0.027024749372597065
 16M: 0.006214  8M: 0.006386  4M: 0.006342  2M: 0.010197  1M: 0.012778 merged: 0.005549
epoch: 94 [2017-11-23 22:39:53]
lr 0.026499947000159004
 16M: 0.006128  8M: 0.006631  4M: 0.006386  2M: 0.010227  1M: 0.012722 merged: 0.005467
epoch: 95 [2017-11-23 22:45:12]
lr 0.02596453934447493
 16M: 0.006092  8M: 0.006408  4M: 0.006467  2M: 0.010306  1M: 0.012866 merged: 0.005472
epoch: 96 [2017-11-23 22:50:30]
lr 0.025417856237895775
 16M: 0.005821  8M: 0.006259  4M: 0.006117  2M: 0.009879  1M: 0.012483 merged: 0.005249
epoch: 97 [2017-11-23 22:55:49]
lr 0.02485915380880628
 16M: 0.005876  8M: 0.006205  4M: 0.006227  2M: 0.010108  1M: 0.012926 merged: 0.005263
epoch: 98 [2017-11-23 23:01:09]
lr 0.02428760260810931
 16M: 0.006078  8M: 0.006401  4M: 0.006330  2M: 0.010468  1M: 0.013081 merged: 0.005414
epoch: 99 [2017-11-23 23:06:28]
lr 0.02370227315699886
 16M: 0.005855  8M: 0.006165  4M: 0.006222  2M: 0.010299  1M: 0.012805 merged: 0.005235
epoch: 100 [2017-11-23 23:11:46]
lr 0.023102118196575382
 16M: 0.005766  8M: 0.006135  4M: 0.006112  2M: 0.009945  1M: 0.012503 merged: 0.005242
epoch: 101 [2017-11-23 23:17:03]
lr 0.022485950669875843
 16M: 0.005783  8M: 0.006110  4M: 0.006146  2M: 0.009917  1M: 0.012429 merged: 0.005201
epoch: 102 [2017-11-23 23:22:21]
lr 0.021852416110985085
 16M: 0.005736  8M: 0.005965  4M: 0.006028  2M: 0.009880  1M: 0.012563 merged: 0.005143
epoch: 103 [2017-11-23 23:27:40]
lr 0.021199957600127203
 16M: 0.005698  8M: 0.005981  4M: 0.006026  2M: 0.009673  1M: 0.012371 merged: 0.005079
epoch: 104 [2017-11-23 23:32:58]
lr 0.020526770681399003
 16M: 0.005694  8M: 0.005942  4M: 0.005977  2M: 0.009788  1M: 0.012269 merged: 0.005106
epoch: 105 [2017-11-23 23:38:17]
lr 0.019830744488452574
 16M: 0.005722  8M: 0.005993  4M: 0.005955  2M: 0.009744  1M: 0.012036 merged: 0.005050
epoch: 106 [2017-11-23 23:43:36]
lr 0.01910938354123028
 16M: 0.005526  8M: 0.005913  4M: 0.005967  2M: 0.010013  1M: 0.012377 merged: 0.005036
epoch: 107 [2017-11-23 23:48:54]
lr 0.01835970184086314
 16M: 0.005696  8M: 0.005983  4M: 0.006035  2M: 0.009983  1M: 0.012561 merged: 0.005036
epoch: 108 [2017-11-23 23:54:14]
lr 0.01757807623276631
 16M: 0.005595  8M: 0.005908  4M: 0.005974  2M: 0.009745  1M: 0.012119 merged: 0.005032
epoch: 109 [2017-11-23 23:59:34]
lr 0.016760038078849775
 16M: 0.005497  8M: 0.005873  4M: 0.005951  2M: 0.009763  1M: 0.012493 merged: 0.004976
epoch: 110 [2017-11-24 00:04:54]
lr 0.0158999682000954
 16M: 0.005595  8M: 0.005888  4M: 0.005963  2M: 0.009623  1M: 0.011944 merged: 0.005044
epoch: 111 [2017-11-24 00:10:14]
lr 0.01499063377991723
 16M: 0.005470  8M: 0.005828  4M: 0.006013  2M: 0.009666  1M: 0.012295 merged: 0.004891
epoch: 112 [2017-11-24 00:15:33]
lr 0.014022453903762567
 16M: 0.005383  8M: 0.005743  4M: 0.005980  2M: 0.009679  1M: 0.012181 merged: 0.004911
epoch: 113 [2017-11-24 00:20:44]
lr 0.012982269672237465
 16M: 0.005499  8M: 0.005758  4M: 0.005859  2M: 0.009494  1M: 0.011902 merged: 0.004890
epoch: 114 [2017-11-24 00:25:53]
lr 0.01185113657849943
 16M: 0.005386  8M: 0.005743  4M: 0.005894  2M: 0.009688  1M: 0.012334 merged: 0.004855
epoch: 115 [2017-11-24 00:31:00]
lr 0.010599978800063602
 16M: 0.005191  8M: 0.005517  4M: 0.005674  2M: 0.009190  1M: 0.011823 merged: 0.004697
epoch: 116 [2017-11-24 00:36:07]
lr 0.00917985092043157
 16M: 0.005389  8M: 0.005801  4M: 0.005953  2M: 0.009794  1M: 0.012554 merged: 0.004863
epoch: 117 [2017-11-24 00:41:07]
lr 0.007495316889958615
 16M: 0.005362  8M: 0.005665  4M: 0.005781  2M: 0.009327  1M: 0.011850 merged: 0.004772
epoch: 118 [2017-11-24 00:46:06]
lr 0.005299989400031801
 16M: 0.005157  8M: 0.005552  4M: 0.005705  2M: 0.009198  1M: 0.011766 merged: 0.004655
epoch: 119 [2017-11-24 00:51:04]
lr 1e-08
 16M: 0.005207  8M: 0.005612  4M: 0.005709  2M: 0.009321  1M: 0.011486 merged: 0.004677

Visualize Graph





In [7]:



    


from graphviz import Digraph
import torch
from torch.autograd import Variable


def make_dot(var, params=None):
    """ Produces Graphviz representation of PyTorch autograd graph
    Blue nodes are the Variables that require grad, orange are Tensors
    saved for backward in torch.autograd.Function
    Args:
        var: output Variable
        params: dict of (name, Variable) to add names to node that
            require grad (TODO: make optional)
    """
    if params is not None:
        assert isinstance(params.values()[0], Variable)
        param_map = {id(v): k for k, v in params.items()}

    node_attr = dict(style='filled',
                     shape='box',
                     align='left',
                     fontsize='12',
                     ranksep='0.1',
                     height='0.2')
    dot = Digraph(node_attr=node_attr, graph_attr=dict(size="10240,10240"), format='svg')
    seen = set()

    def size_to_str(size):
        return '('+(', ').join(['%d' % v for v in size])+')'

    def add_nodes(var):
        if var not in seen:
            if torch.is_tensor(var):
                dot.node(str(id(var)), size_to_str(var.size()), fillcolor='orange')
            elif hasattr(var, 'variable'):
                u = var.variable
                name = param_map[id(u)] if params is not None else ''
                node_name = '%s\n %s' % (name, size_to_str(u.size()))
                dot.node(str(id(var)), node_name, fillcolor='lightblue')
            else:
                dot.node(str(id(var)), str(type(var).__name__))
            seen.add(var)
            if hasattr(var, 'next_functions'):
                for u in var.next_functions:
                    if u[0] is not None:
                        dot.edge(str(id(u[0])), str(id(var)))
                        add_nodes(u[0])
            if hasattr(var, 'saved_tensors'):
                for t in var.saved_tensors:
                    dot.edge(str(id(t)), str(id(var)))
                    add_nodes(t)
    add_nodes(var.grad_fn)
    return dot



    











In [8]:



    


x = Variable(torch.zeros(1,3,256,256))
y = net(x.cuda())
g = make_dot(y[-1])



    











In [9]:



    


g.render('net-pretrained_scale_{}'.format(pretrained_scale))



    


















    






---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
~/anaconda3/lib/python3.6/site-packages/graphviz/backend.py in render(engine, format, filepath, quiet)
    123         try:
--> 124             subprocess.check_call(args, startupinfo=STARTUPINFO, stderr=stderr)
    125         except OSError as e:

~/anaconda3/lib/python3.6/subprocess.py in check_call(*popenargs, **kwargs)
    285     """
--> 286     retcode = call(*popenargs, **kwargs)
    287     if retcode:

~/anaconda3/lib/python3.6/subprocess.py in call(timeout, *popenargs, **kwargs)
    266     """
--> 267     with Popen(*popenargs, **kwargs) as p:
    268         try:

~/anaconda3/lib/python3.6/subprocess.py in __init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, encoding, errors)
    708                                 errread, errwrite,
--> 709                                 restore_signals, start_new_session)
    710         except:

~/anaconda3/lib/python3.6/subprocess.py in _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session)
   1343                             err_msg += ': ' + repr(err_filename)
-> 1344                     raise child_exception_type(errno_num, err_msg, err_filename)
   1345                 raise child_exception_type(err_msg)

FileNotFoundError: [Errno 2] No such file or directory: 'dot': 'dot'

During handling of the above exception, another exception occurred:

ExecutableNotFound                        Traceback (most recent call last)
<ipython-input-9-a8aee31c1e27> in <module>()
----> 1 g.render('net-pretrained_scale_{}'.format(pretrained_scale))

~/anaconda3/lib/python3.6/site-packages/graphviz/files.py in render(self, filename, directory, view, cleanup)
    173         filepath = self.save(filename, directory)
    174 
--> 175         rendered = backend.render(self._engine, self._format, filepath)
    176 
    177         if cleanup:

~/anaconda3/lib/python3.6/site-packages/graphviz/backend.py in render(engine, format, filepath, quiet)
    125         except OSError as e:
    126             if e.errno == errno.ENOENT:
--> 127                 raise ExecutableNotFound(args)
    128             else:  # pragma: no cover
    129                 raise

ExecutableNotFound: failed to execute ['dot', '-Tsvg', '-O', 'net-pretrained_scale_2'], make sure the Graphviz executables are on your systems' PATH
















In [ ]:

Content source: albertxavier001/graduation-project

Similar notebooks:

notebook.community | gallery | about