In [11]:

    

from __future__ import print_function
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms
from torch.autograd import Variable

import math
import numpy as np
import matplotlib.pyplot as plt
from torchvision.utils import make_grid
from collections import OrderedDict

# Training settings
# for terminal use. In notebook, you can't parse arguments
class args:
    cuda = False
    batch_size = 64
    test_batch_size = 1000
    epochs = 10
    lr = 0.01
    momentum = 0.5
    no_cuda = False
    seed = 1
    log_interval = 10
    # if add Dropout
    with_dropout = False
    # if initialize weights
    with_init_weights = False
    # if add BatchNorm
    with_batchnorm = False

args.cuda = not args.no_cuda and torch.cuda.is_available()

torch.manual_seed(args.seed)
if args.cuda:
    torch.cuda.manual_seed(args.seed)

kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {}
# The output of torchvision datasets are PILImage images of range [0, 1].
# We transform them to Tensors of normalized range [-1, 1]
transform = transforms.Compose([transforms.ToTensor(),
                              transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
                             ])
trainset = datasets.CIFAR10(root='./data', train=True, download=True, transform=transform)
train_loader = torch.utils.data.DataLoader(trainset, batch_size=args.batch_size,
                                          shuffle=True, num_workers=2)

testset = datasets.CIFAR10(root='./data', train=False, download=True, transform=transform)
test_loader = torch.utils.data.DataLoader(testset, batch_size=args.test_batch_size,
                                          shuffle=False, num_workers=2)
classes = ('plane', 'car', 'bird', 'cat',
           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')


class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        # TODO: define your network here
        self.conv_1 = nn.Conv2d(3, 6, kernel_size=5, stride=1)
        self.conv_2 = nn.Conv2d(6, 16, kernel_size=5, stride=1)
        # TODO: replace fc with conv
        self.fc_1 = nn.Linear(16 * 25, 120)
        self.fc_2 = nn.Linear(120, 84)
        self.fc_3 = nn.Linear(84, 10)
        if args.with_batchnorm:
            self.block_conv_1 = nn.Sequential(
                self.conv_1,
                nn.BatchNorm2d(6),
                nn.ReLU(),
                nn.MaxPool2d(kernel_size=2, stride=2)
            )
            self.block_conv_2 = nn.Sequential(
                self.conv_2,
                nn.BatchNorm2d(16),
                nn.ReLU(),
                nn.MaxPool2d(kernel_size=2, stride=2)
            )
        else:
            self.block_conv_1 = nn.Sequential(
                self.conv_1,
                nn.ReLU(),
                nn.MaxPool2d(kernel_size=2, stride=2)
            )
            self.block_conv_2 = nn.Sequential(
                self.conv_2,
                nn.ReLU(),
                nn.MaxPool2d(kernel_size=2, stride=2)
            )
        if args.with_dropout:
            if args.with_batchnorm:
                self.block_fc_1 = nn.Sequential(
                    self.fc_1,
                    nn.BatchNorm1d(120),
                    nn.Dropout()
                )
                self.block_fc_2 = nn.Sequential(
                    self.fc_2,
                    nn.BatchNorm1d(84),
                    nn.Dropout()
                )
            else:
                self.block_fc_1 = nn.Sequential(
                    self.fc_1,
                    nn.Dropout()
                )
                self.block_fc_2 = nn.Sequential(
                    self.fc_2,
                    nn.Dropout()
                )
        else:
            self.block_fc_1 = self.fc_1
            self.block_fc_2 = self.fc_2
        self.softmax = nn.LogSoftmax()
        # Initialize parameters
        if args.with_init_weights:
            for m in self.modules():
                if isinstance(m, nn.Conv2d):
                    n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                    m.weight.data.normal_(0, math.sqrt(2. /n))
                    if m.bias is not None:
                        m.bias.data.zero_()
                elif isinstance(m, nn.Linear):
                    n = m.out_features
                    m.weight.data.normal_(0, math.sqrt(2. /n))
                    if m.bias is not None:
                        m.bias.data.zero_()
                elif isinstance(m, nn.BatchNorm1d) or isinstance(m, nn.BatchNorm2d):
                    m.weight.data.fill_(1)
                    m.bias.data.zero_()

    def forward(self, x):
        # TODO
        x = self.block_conv_1(x)
        x = self.block_conv_2(x)
        x = x.view(x.size(0), -1)
        x = self.block_fc_1(x)
        x = self.block_fc_2(x)
        x = self.fc_3(x)
        x = self.softmax(x)
        return x

# Filter extractor for filter visualization
class FeatureExtractor(nn.Module):
    def __init__(self, model, layer_names):
        super(FeatureExtractor, self).__init__()
        self._model = model
        self._layer_names = set(layer_names)
    def forward(self):
        filters = dict()
        # _modules is an OrderedDict, which replace iteritems() with items() in python3.* 
        for name, module in self._model._modules.items():
            if name in self._layer_names:
                filters[name] = module.weight.data.cpu()
        return filters

# Vesualize training results and trained filters
class VisualizedResult():
    def __init__(self, model):
        self._model = model
    def training_curve(self, epoches, train_loss_records, test_loss_records):
        fig = plt.figure()
        ax_train = fig.add_subplot(111)
        ax_test = fig.add_subplot(111)
        plt.axis([1, epoches, 0, math.ceil(max(train_loss_records + test_loss_records) * 1.2)])
        plt.xlabel('Epoches')
        plt.ylabel('Loss')
        plt.title('Training Curve')
        plt.plot(range(1, epoches + 1), train_loss_records, 'b-', label="train loss")
        plt.plot(range(1, epoches + 1), test_loss_records, 'r-', label="test loss")
        for xy in zip(range(1, epoches + 1), train_loss_records):
            ax_train.annotate('%.2f' % xy[1], xy=xy, textcoords='data')
        for xy in zip(range(1, epoches + 1), test_loss_records):
            ax_test.annotate('%.2f' % xy[1], xy=xy, textcoords='data')
        plt.legend(loc='upper right', borderaxespad=0.)
        plt.show()
    def accuracy_curve(self, epoches, accuracy_records):
        fig = plt.figure()
        ax = fig.add_subplot(111)
        plt.axis([1, epoches, 0, 100])
        plt.xlabel('Epoches')
        plt.ylabel('Accuracy')
        plt.title('Accuracy Curve')
        plt.plot(range(1, epoches + 1), accuracy_records, '-')
        for xy in zip(range(1, epoches + 1), accuracy_records):
            ax.annotate('%s%%' % xy[1], xy=xy, textcoords='data')
        plt.show()
    def conv_filter(self, layer_names):
        feature_extractor = FeatureExtractor(self._model, layer_names)
        filter_list = feature_extractor.forward()
        for name, filters in filter_list.items():
            print(type(filters), filters.size())
            # Normalize filters
            mean = torch.mean(filters)
            std = torch.std(filters)
            filters = (filters - mean) / std
            img = make_grid(filters, nrow=3)
            print(type(img), img.size())
            npimg = img.numpy()
            plt.imshow(np.transpose(npimg, (1,2,0)), interpolation='nearest')


model = Net()
if args.cuda:
    model.cuda()
    
# TODO: other optimizers
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum)

train_loss_records = list()
test_loss_records = list()
accuracy_records = list()

def train(epoch):
    model.train()
    train_loss = 0
    for batch_idx, (data, target) in enumerate(train_loader):
        if args.cuda:
            data, target = data.cuda(), target.cuda()
        data, target = Variable(data), Variable(target)
        optimizer.zero_grad()
        output = model(data)
        loss = F.nll_loss(output, target)   # is it true to use such a loss over cross-entropy loss? 
        loss.backward()
        optimizer.step()
        train_loss += loss.data[0]
        if batch_idx % args.log_interval == 0:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch, batch_idx * len(data), len(train_loader.dataset),
                100. * batch_idx / len(train_loader), loss.data[0]))
    # Average training loss for this epoch
    train_loss_records.append(train_loss / len(train_loader))

def test(epoch):
    model.eval()
    test_loss = 0
    correct = 0
    for data, target in test_loader:
        if args.cuda:
            data, target = data.cuda(), target.cuda()
        data, target = Variable(data, volatile=True), Variable(target)
        output = model(data)
        test_loss += F.nll_loss(output, target).data[0]
        pred = output.data.max(1)[1] # get the index of the max log-probability
        correct += pred.eq(target.data).cpu().sum()

    test_loss = test_loss
    test_loss /= len(test_loader) # loss function already averages over batch size
    accuracy = 100. * correct / len(test_loader.dataset)
    print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
        test_loss, correct, len(test_loader.dataset),
        accuracy))
    test_loss_records.append(test_loss)
    accuracy_records.append(accuracy)


for epoch in range(1, args.epochs + 1):
    train(epoch)
    test(epoch)

visual_result = VisualizedResult(model)
# Visualize training curve
visual_result.training_curve(args.epochs, train_loss_records, test_loss_records)
# Visualize test accuracy
visual_result.accuracy_curve(args.epochs, accuracy_records)
# Visualize trained filter on the 1st Conv layer
visual_result.conv_filter(['conv_1'])


    

    




Files already downloaded and verified
Files already downloaded and verified
Train Epoch: 1 [0/50000 (0%)]	Loss: 2.316156
Train Epoch: 1 [640/50000 (1%)]	Loss: 2.303175
Train Epoch: 1 [1280/50000 (3%)]	Loss: 2.314214
Train Epoch: 1 [1920/50000 (4%)]	Loss: 2.317336
Train Epoch: 1 [2560/50000 (5%)]	Loss: 2.301905
Train Epoch: 1 [3200/50000 (6%)]	Loss: 2.274860
Train Epoch: 1 [3840/50000 (8%)]	Loss: 2.312823
Train Epoch: 1 [4480/50000 (9%)]	Loss: 2.310921
Train Epoch: 1 [5120/50000 (10%)]	Loss: 2.300413
Train Epoch: 1 [5760/50000 (12%)]	Loss: 2.298556
Train Epoch: 1 [6400/50000 (13%)]	Loss: 2.289032
Train Epoch: 1 [7040/50000 (14%)]	Loss: 2.281180
Train Epoch: 1 [7680/50000 (15%)]	Loss: 2.286341
Train Epoch: 1 [8320/50000 (17%)]	Loss: 2.288216
Train Epoch: 1 [8960/50000 (18%)]	Loss: 2.268875
Train Epoch: 1 [9600/50000 (19%)]	Loss: 2.297248
Train Epoch: 1 [10240/50000 (20%)]	Loss: 2.260878
Train Epoch: 1 [10880/50000 (22%)]	Loss: 2.261347
Train Epoch: 1 [11520/50000 (23%)]	Loss: 2.244238
Train Epoch: 1 [12160/50000 (24%)]	Loss: 2.220024
Train Epoch: 1 [12800/50000 (26%)]	Loss: 2.271027
Train Epoch: 1 [13440/50000 (27%)]	Loss: 2.172879
Train Epoch: 1 [14080/50000 (28%)]	Loss: 2.197615
Train Epoch: 1 [14720/50000 (29%)]	Loss: 2.175746
Train Epoch: 1 [15360/50000 (31%)]	Loss: 2.116280
Train Epoch: 1 [16000/50000 (32%)]	Loss: 2.200229
Train Epoch: 1 [16640/50000 (33%)]	Loss: 2.170606
Train Epoch: 1 [17280/50000 (35%)]	Loss: 2.232272
Train Epoch: 1 [17920/50000 (36%)]	Loss: 2.054227
Train Epoch: 1 [18560/50000 (37%)]	Loss: 2.097498
Train Epoch: 1 [19200/50000 (38%)]	Loss: 1.984061
Train Epoch: 1 [19840/50000 (40%)]	Loss: 2.056169
Train Epoch: 1 [20480/50000 (41%)]	Loss: 2.077556
Train Epoch: 1 [21120/50000 (42%)]	Loss: 2.137288
Train Epoch: 1 [21760/50000 (43%)]	Loss: 1.950289
Train Epoch: 1 [22400/50000 (45%)]	Loss: 2.132143
Train Epoch: 1 [23040/50000 (46%)]	Loss: 2.098148
Train Epoch: 1 [23680/50000 (47%)]	Loss: 2.037208
Train Epoch: 1 [24320/50000 (49%)]	Loss: 2.019490
Train Epoch: 1 [24960/50000 (50%)]	Loss: 2.139807
Train Epoch: 1 [25600/50000 (51%)]	Loss: 2.015985
Train Epoch: 1 [26240/50000 (52%)]	Loss: 1.976602
Train Epoch: 1 [26880/50000 (54%)]	Loss: 1.946326
Train Epoch: 1 [27520/50000 (55%)]	Loss: 1.864100
Train Epoch: 1 [28160/50000 (56%)]	Loss: 1.858311
Train Epoch: 1 [28800/50000 (58%)]	Loss: 1.951976
Train Epoch: 1 [29440/50000 (59%)]	Loss: 1.938979
Train Epoch: 1 [30080/50000 (60%)]	Loss: 1.990371
Train Epoch: 1 [30720/50000 (61%)]	Loss: 1.955214
Train Epoch: 1 [31360/50000 (63%)]	Loss: 2.036504
Train Epoch: 1 [32000/50000 (64%)]	Loss: 2.005098
Train Epoch: 1 [32640/50000 (65%)]	Loss: 1.900097
Train Epoch: 1 [33280/50000 (66%)]	Loss: 2.105828
Train Epoch: 1 [33920/50000 (68%)]	Loss: 1.938234
Train Epoch: 1 [34560/50000 (69%)]	Loss: 1.889488
Train Epoch: 1 [35200/50000 (70%)]	Loss: 1.939395
Train Epoch: 1 [35840/50000 (72%)]	Loss: 1.941655
Train Epoch: 1 [36480/50000 (73%)]	Loss: 1.887514
Train Epoch: 1 [37120/50000 (74%)]	Loss: 1.944030
Train Epoch: 1 [37760/50000 (75%)]	Loss: 1.996795
Train Epoch: 1 [38400/50000 (77%)]	Loss: 1.814419
Train Epoch: 1 [39040/50000 (78%)]	Loss: 2.075825
Train Epoch: 1 [39680/50000 (79%)]	Loss: 1.967530
Train Epoch: 1 [40320/50000 (81%)]	Loss: 1.798951
Train Epoch: 1 [40960/50000 (82%)]	Loss: 1.959463
Train Epoch: 1 [41600/50000 (83%)]	Loss: 1.878299
Train Epoch: 1 [42240/50000 (84%)]	Loss: 1.856040
Train Epoch: 1 [42880/50000 (86%)]	Loss: 1.993883
Train Epoch: 1 [43520/50000 (87%)]	Loss: 1.863082
Train Epoch: 1 [44160/50000 (88%)]	Loss: 1.975442
Train Epoch: 1 [44800/50000 (90%)]	Loss: 1.953657
Train Epoch: 1 [45440/50000 (91%)]	Loss: 1.823101
Train Epoch: 1 [46080/50000 (92%)]	Loss: 1.945857
Train Epoch: 1 [46720/50000 (93%)]	Loss: 2.107270
Train Epoch: 1 [47360/50000 (95%)]	Loss: 1.891308
Train Epoch: 1 [48000/50000 (96%)]	Loss: 1.729778
Train Epoch: 1 [48640/50000 (97%)]	Loss: 1.722905
Train Epoch: 1 [49280/50000 (98%)]	Loss: 1.726151
Train Epoch: 1 [49920/50000 (100%)]	Loss: 1.837631

Test set: Average loss: 1.8900, Accuracy: 3254/10000 (33%)

Train Epoch: 2 [0/50000 (0%)]	Loss: 1.932187
Train Epoch: 2 [640/50000 (1%)]	Loss: 1.714884
Train Epoch: 2 [1280/50000 (3%)]	Loss: 1.843028
Train Epoch: 2 [1920/50000 (4%)]	Loss: 1.703905
Train Epoch: 2 [2560/50000 (5%)]	Loss: 1.876981
Train Epoch: 2 [3200/50000 (6%)]	Loss: 1.979341
Train Epoch: 2 [3840/50000 (8%)]	Loss: 1.737951
Train Epoch: 2 [4480/50000 (9%)]	Loss: 1.685923
Train Epoch: 2 [5120/50000 (10%)]	Loss: 1.887995
Train Epoch: 2 [5760/50000 (12%)]	Loss: 1.868013
Train Epoch: 2 [6400/50000 (13%)]	Loss: 1.726887
Train Epoch: 2 [7040/50000 (14%)]	Loss: 1.648432
Train Epoch: 2 [7680/50000 (15%)]	Loss: 1.728926
Train Epoch: 2 [8320/50000 (17%)]	Loss: 1.737999
Train Epoch: 2 [8960/50000 (18%)]	Loss: 1.865913
Train Epoch: 2 [9600/50000 (19%)]	Loss: 1.865284
Train Epoch: 2 [10240/50000 (20%)]	Loss: 1.890455
Train Epoch: 2 [10880/50000 (22%)]	Loss: 1.660162
Train Epoch: 2 [11520/50000 (23%)]	Loss: 1.695538
Train Epoch: 2 [12160/50000 (24%)]	Loss: 1.665671
Train Epoch: 2 [12800/50000 (26%)]	Loss: 1.898034
Train Epoch: 2 [13440/50000 (27%)]	Loss: 1.813195
Train Epoch: 2 [14080/50000 (28%)]	Loss: 1.794693
Train Epoch: 2 [14720/50000 (29%)]	Loss: 1.699618
Train Epoch: 2 [15360/50000 (31%)]	Loss: 1.630018
Train Epoch: 2 [16000/50000 (32%)]	Loss: 1.904285
Train Epoch: 2 [16640/50000 (33%)]	Loss: 1.698437
Train Epoch: 2 [17280/50000 (35%)]	Loss: 1.679479
Train Epoch: 2 [17920/50000 (36%)]	Loss: 1.647294
Train Epoch: 2 [18560/50000 (37%)]	Loss: 1.669895
Train Epoch: 2 [19200/50000 (38%)]	Loss: 1.645629
Train Epoch: 2 [19840/50000 (40%)]	Loss: 1.666365
Train Epoch: 2 [20480/50000 (41%)]	Loss: 1.855513
Train Epoch: 2 [21120/50000 (42%)]	Loss: 1.882765
Train Epoch: 2 [21760/50000 (43%)]	Loss: 1.829763
Train Epoch: 2 [22400/50000 (45%)]	Loss: 1.839066
Train Epoch: 2 [23040/50000 (46%)]	Loss: 1.649641
Train Epoch: 2 [23680/50000 (47%)]	Loss: 1.676561
Train Epoch: 2 [24320/50000 (49%)]	Loss: 1.766824
Train Epoch: 2 [24960/50000 (50%)]	Loss: 1.412055
Train Epoch: 2 [25600/50000 (51%)]	Loss: 1.650286
Train Epoch: 2 [26240/50000 (52%)]	Loss: 1.661735
Train Epoch: 2 [26880/50000 (54%)]	Loss: 1.670306
Train Epoch: 2 [27520/50000 (55%)]	Loss: 1.549872
Train Epoch: 2 [28160/50000 (56%)]	Loss: 1.551030
Train Epoch: 2 [28800/50000 (58%)]	Loss: 1.435923
Train Epoch: 2 [29440/50000 (59%)]	Loss: 1.586104
Train Epoch: 2 [30080/50000 (60%)]	Loss: 1.796435
Train Epoch: 2 [30720/50000 (61%)]	Loss: 1.544524
Train Epoch: 2 [31360/50000 (63%)]	Loss: 1.688005
Train Epoch: 2 [32000/50000 (64%)]	Loss: 1.581937
Train Epoch: 2 [32640/50000 (65%)]	Loss: 1.719390
Train Epoch: 2 [33280/50000 (66%)]	Loss: 1.743904
Train Epoch: 2 [33920/50000 (68%)]	Loss: 1.621054
Train Epoch: 2 [34560/50000 (69%)]	Loss: 1.597770
Train Epoch: 2 [35200/50000 (70%)]	Loss: 1.613080
Train Epoch: 2 [35840/50000 (72%)]	Loss: 1.374543
Train Epoch: 2 [36480/50000 (73%)]	Loss: 1.755902
Train Epoch: 2 [37120/50000 (74%)]	Loss: 1.694724
Train Epoch: 2 [37760/50000 (75%)]	Loss: 1.701555
Train Epoch: 2 [38400/50000 (77%)]	Loss: 1.657834
Train Epoch: 2 [39040/50000 (78%)]	Loss: 1.727736
Train Epoch: 2 [39680/50000 (79%)]	Loss: 1.682461
Train Epoch: 2 [40320/50000 (81%)]	Loss: 1.650628
Train Epoch: 2 [40960/50000 (82%)]	Loss: 1.673399
Train Epoch: 2 [41600/50000 (83%)]	Loss: 1.461638
Train Epoch: 2 [42240/50000 (84%)]	Loss: 1.623628
Train Epoch: 2 [42880/50000 (86%)]	Loss: 1.445120
Train Epoch: 2 [43520/50000 (87%)]	Loss: 1.732508
Train Epoch: 2 [44160/50000 (88%)]	Loss: 1.397591
Train Epoch: 2 [44800/50000 (90%)]	Loss: 1.739561
Train Epoch: 2 [45440/50000 (91%)]	Loss: 1.598024
Train Epoch: 2 [46080/50000 (92%)]	Loss: 1.593571
Train Epoch: 2 [46720/50000 (93%)]	Loss: 1.464149
Train Epoch: 2 [47360/50000 (95%)]	Loss: 1.571131
Train Epoch: 2 [48000/50000 (96%)]	Loss: 1.473228
Train Epoch: 2 [48640/50000 (97%)]	Loss: 1.640705
Train Epoch: 2 [49280/50000 (98%)]	Loss: 1.798675
Train Epoch: 2 [49920/50000 (100%)]	Loss: 1.554706

Test set: Average loss: 1.6654, Accuracy: 4079/10000 (41%)

Train Epoch: 3 [0/50000 (0%)]	Loss: 1.784060
Train Epoch: 3 [640/50000 (1%)]	Loss: 1.655508
Train Epoch: 3 [1280/50000 (3%)]	Loss: 1.387904
Train Epoch: 3 [1920/50000 (4%)]	Loss: 1.563422
Train Epoch: 3 [2560/50000 (5%)]	Loss: 1.341271
Train Epoch: 3 [3200/50000 (6%)]	Loss: 1.486206
Train Epoch: 3 [3840/50000 (8%)]	Loss: 1.501352
Train Epoch: 3 [4480/50000 (9%)]	Loss: 1.529459
Train Epoch: 3 [5120/50000 (10%)]	Loss: 1.587929
Train Epoch: 3 [5760/50000 (12%)]	Loss: 1.667252
Train Epoch: 3 [6400/50000 (13%)]	Loss: 1.674066
Train Epoch: 3 [7040/50000 (14%)]	Loss: 1.690348
Train Epoch: 3 [7680/50000 (15%)]	Loss: 1.485674
Train Epoch: 3 [8320/50000 (17%)]	Loss: 1.492190
Train Epoch: 3 [8960/50000 (18%)]	Loss: 1.440471
Train Epoch: 3 [9600/50000 (19%)]	Loss: 1.724255
Train Epoch: 3 [10240/50000 (20%)]	Loss: 1.675664
Train Epoch: 3 [10880/50000 (22%)]	Loss: 1.639405
Train Epoch: 3 [11520/50000 (23%)]	Loss: 1.535443
Train Epoch: 3 [12160/50000 (24%)]	Loss: 1.660905
Train Epoch: 3 [12800/50000 (26%)]	Loss: 1.589241
Train Epoch: 3 [13440/50000 (27%)]	Loss: 1.684233
Train Epoch: 3 [14080/50000 (28%)]	Loss: 1.470258
Train Epoch: 3 [14720/50000 (29%)]	Loss: 1.401105
Train Epoch: 3 [15360/50000 (31%)]	Loss: 1.502886
Train Epoch: 3 [16000/50000 (32%)]	Loss: 1.678742
Train Epoch: 3 [16640/50000 (33%)]	Loss: 1.361903
Train Epoch: 3 [17280/50000 (35%)]	Loss: 1.508279
Train Epoch: 3 [17920/50000 (36%)]	Loss: 1.439411
Train Epoch: 3 [18560/50000 (37%)]	Loss: 1.559520
Train Epoch: 3 [19200/50000 (38%)]	Loss: 1.511957
Train Epoch: 3 [19840/50000 (40%)]	Loss: 1.498557
Train Epoch: 3 [20480/50000 (41%)]	Loss: 1.557483
Train Epoch: 3 [21120/50000 (42%)]	Loss: 1.317771
Train Epoch: 3 [21760/50000 (43%)]	Loss: 1.293475
Train Epoch: 3 [22400/50000 (45%)]	Loss: 1.247592
Train Epoch: 3 [23040/50000 (46%)]	Loss: 1.480450
Train Epoch: 3 [23680/50000 (47%)]	Loss: 1.448342
Train Epoch: 3 [24320/50000 (49%)]	Loss: 1.503659
Train Epoch: 3 [24960/50000 (50%)]	Loss: 1.529244
Train Epoch: 3 [25600/50000 (51%)]	Loss: 1.386268
Train Epoch: 3 [26240/50000 (52%)]	Loss: 1.642801
Train Epoch: 3 [26880/50000 (54%)]	Loss: 1.695706
Train Epoch: 3 [27520/50000 (55%)]	Loss: 1.606173
Train Epoch: 3 [28160/50000 (56%)]	Loss: 1.298627
Train Epoch: 3 [28800/50000 (58%)]	Loss: 1.574751
Train Epoch: 3 [29440/50000 (59%)]	Loss: 1.575801
Train Epoch: 3 [30080/50000 (60%)]	Loss: 1.632519
Train Epoch: 3 [30720/50000 (61%)]	Loss: 1.737336
Train Epoch: 3 [31360/50000 (63%)]	Loss: 1.398714
Train Epoch: 3 [32000/50000 (64%)]	Loss: 1.489110
Train Epoch: 3 [32640/50000 (65%)]	Loss: 1.478081
Train Epoch: 3 [33280/50000 (66%)]	Loss: 1.316970
Train Epoch: 3 [33920/50000 (68%)]	Loss: 1.652512
Train Epoch: 3 [34560/50000 (69%)]	Loss: 1.329090
Train Epoch: 3 [35200/50000 (70%)]	Loss: 1.431460
Train Epoch: 3 [35840/50000 (72%)]	Loss: 1.550859
Train Epoch: 3 [36480/50000 (73%)]	Loss: 1.619231
Train Epoch: 3 [37120/50000 (74%)]	Loss: 1.395380
Train Epoch: 3 [37760/50000 (75%)]	Loss: 1.494625
Train Epoch: 3 [38400/50000 (77%)]	Loss: 1.437577
Train Epoch: 3 [39040/50000 (78%)]	Loss: 1.404412
Train Epoch: 3 [39680/50000 (79%)]	Loss: 1.539379
Train Epoch: 3 [40320/50000 (81%)]	Loss: 1.408592
Train Epoch: 3 [40960/50000 (82%)]	Loss: 1.568326
Train Epoch: 3 [41600/50000 (83%)]	Loss: 1.738138
Train Epoch: 3 [42240/50000 (84%)]	Loss: 1.510250
Train Epoch: 3 [42880/50000 (86%)]	Loss: 1.509464
Train Epoch: 3 [43520/50000 (87%)]	Loss: 1.549273
Train Epoch: 3 [44160/50000 (88%)]	Loss: 1.380241
Train Epoch: 3 [44800/50000 (90%)]	Loss: 1.387403
Train Epoch: 3 [45440/50000 (91%)]	Loss: 1.243511
Train Epoch: 3 [46080/50000 (92%)]	Loss: 1.392010
Train Epoch: 3 [46720/50000 (93%)]	Loss: 1.638795
Train Epoch: 3 [47360/50000 (95%)]	Loss: 1.604765
Train Epoch: 3 [48000/50000 (96%)]	Loss: 1.576831
Train Epoch: 3 [48640/50000 (97%)]	Loss: 1.464382
Train Epoch: 3 [49280/50000 (98%)]	Loss: 1.247487
Train Epoch: 3 [49920/50000 (100%)]	Loss: 1.285769

Test set: Average loss: 1.4391, Accuracy: 4787/10000 (48%)

Train Epoch: 4 [0/50000 (0%)]	Loss: 1.568612
Train Epoch: 4 [640/50000 (1%)]	Loss: 1.483695
Train Epoch: 4 [1280/50000 (3%)]	Loss: 1.433072
Train Epoch: 4 [1920/50000 (4%)]	Loss: 1.370726
Train Epoch: 4 [2560/50000 (5%)]	Loss: 1.411922
Train Epoch: 4 [3200/50000 (6%)]	Loss: 1.398988
Train Epoch: 4 [3840/50000 (8%)]	Loss: 1.541153
Train Epoch: 4 [4480/50000 (9%)]	Loss: 1.238302
Train Epoch: 4 [5120/50000 (10%)]	Loss: 1.394218
Train Epoch: 4 [5760/50000 (12%)]	Loss: 1.377411
Train Epoch: 4 [6400/50000 (13%)]	Loss: 1.106191
Train Epoch: 4 [7040/50000 (14%)]	Loss: 1.662454
Train Epoch: 4 [7680/50000 (15%)]	Loss: 1.758752
Train Epoch: 4 [8320/50000 (17%)]	Loss: 1.360247
Train Epoch: 4 [8960/50000 (18%)]	Loss: 1.545925
Train Epoch: 4 [9600/50000 (19%)]	Loss: 1.692817
Train Epoch: 4 [10240/50000 (20%)]	Loss: 1.284328
Train Epoch: 4 [10880/50000 (22%)]	Loss: 1.235517
Train Epoch: 4 [11520/50000 (23%)]	Loss: 1.233469
Train Epoch: 4 [12160/50000 (24%)]	Loss: 1.466140
Train Epoch: 4 [12800/50000 (26%)]	Loss: 1.658938
Train Epoch: 4 [13440/50000 (27%)]	Loss: 1.459906
Train Epoch: 4 [14080/50000 (28%)]	Loss: 1.425982
Train Epoch: 4 [14720/50000 (29%)]	Loss: 1.436079
Train Epoch: 4 [15360/50000 (31%)]	Loss: 1.331784
Train Epoch: 4 [16000/50000 (32%)]	Loss: 1.501850
Train Epoch: 4 [16640/50000 (33%)]	Loss: 1.403820
Train Epoch: 4 [17280/50000 (35%)]	Loss: 1.154047
Train Epoch: 4 [17920/50000 (36%)]	Loss: 1.516976
Train Epoch: 4 [18560/50000 (37%)]	Loss: 1.303103
Train Epoch: 4 [19200/50000 (38%)]	Loss: 1.331035
Train Epoch: 4 [19840/50000 (40%)]	Loss: 1.271452
Train Epoch: 4 [20480/50000 (41%)]	Loss: 1.667129
Train Epoch: 4 [21120/50000 (42%)]	Loss: 1.372828
Train Epoch: 4 [21760/50000 (43%)]	Loss: 1.342485
Train Epoch: 4 [22400/50000 (45%)]	Loss: 1.265946
Train Epoch: 4 [23040/50000 (46%)]	Loss: 1.494732
Train Epoch: 4 [23680/50000 (47%)]	Loss: 1.306112
Train Epoch: 4 [24320/50000 (49%)]	Loss: 1.436408
Train Epoch: 4 [24960/50000 (50%)]	Loss: 1.624028
Train Epoch: 4 [25600/50000 (51%)]	Loss: 1.558496
Train Epoch: 4 [26240/50000 (52%)]	Loss: 1.292699
Train Epoch: 4 [26880/50000 (54%)]	Loss: 1.451884
Train Epoch: 4 [27520/50000 (55%)]	Loss: 1.129053
Train Epoch: 4 [28160/50000 (56%)]	Loss: 1.340922
Train Epoch: 4 [28800/50000 (58%)]	Loss: 1.369130
Train Epoch: 4 [29440/50000 (59%)]	Loss: 1.351797
Train Epoch: 4 [30080/50000 (60%)]	Loss: 1.611593
Train Epoch: 4 [30720/50000 (61%)]	Loss: 1.374668
Train Epoch: 4 [31360/50000 (63%)]	Loss: 1.326354
Train Epoch: 4 [32000/50000 (64%)]	Loss: 1.093895
Train Epoch: 4 [32640/50000 (65%)]	Loss: 1.248079
Train Epoch: 4 [33280/50000 (66%)]	Loss: 1.446979
Train Epoch: 4 [33920/50000 (68%)]	Loss: 1.759343
Train Epoch: 4 [34560/50000 (69%)]	Loss: 1.274086
Train Epoch: 4 [35200/50000 (70%)]	Loss: 1.403626
Train Epoch: 4 [35840/50000 (72%)]	Loss: 1.247043
Train Epoch: 4 [36480/50000 (73%)]	Loss: 1.369755
Train Epoch: 4 [37120/50000 (74%)]	Loss: 1.418034
Train Epoch: 4 [37760/50000 (75%)]	Loss: 1.475892
Train Epoch: 4 [38400/50000 (77%)]	Loss: 1.296175
Train Epoch: 4 [39040/50000 (78%)]	Loss: 1.544564
Train Epoch: 4 [39680/50000 (79%)]	Loss: 1.553765
Train Epoch: 4 [40320/50000 (81%)]	Loss: 1.396542
Train Epoch: 4 [40960/50000 (82%)]	Loss: 1.272459
Train Epoch: 4 [41600/50000 (83%)]	Loss: 1.256322
Train Epoch: 4 [42240/50000 (84%)]	Loss: 1.398371
Train Epoch: 4 [42880/50000 (86%)]	Loss: 1.360255
Train Epoch: 4 [43520/50000 (87%)]	Loss: 1.117621
Train Epoch: 4 [44160/50000 (88%)]	Loss: 1.223409
Train Epoch: 4 [44800/50000 (90%)]	Loss: 1.401068
Train Epoch: 4 [45440/50000 (91%)]	Loss: 1.254364
Train Epoch: 4 [46080/50000 (92%)]	Loss: 1.250795
Train Epoch: 4 [46720/50000 (93%)]	Loss: 1.345136
Train Epoch: 4 [47360/50000 (95%)]	Loss: 1.441424
Train Epoch: 4 [48000/50000 (96%)]	Loss: 1.452220
Train Epoch: 4 [48640/50000 (97%)]	Loss: 1.558462
Train Epoch: 4 [49280/50000 (98%)]	Loss: 1.269692
Train Epoch: 4 [49920/50000 (100%)]	Loss: 1.285037

Test set: Average loss: 1.3777, Accuracy: 5016/10000 (50%)

Train Epoch: 5 [0/50000 (0%)]	Loss: 1.455619
Train Epoch: 5 [640/50000 (1%)]	Loss: 1.406476
Train Epoch: 5 [1280/50000 (3%)]	Loss: 1.352482
Train Epoch: 5 [1920/50000 (4%)]	Loss: 1.393072
Train Epoch: 5 [2560/50000 (5%)]	Loss: 1.438173
Train Epoch: 5 [3200/50000 (6%)]	Loss: 1.226437
Train Epoch: 5 [3840/50000 (8%)]	Loss: 1.234174
Train Epoch: 5 [4480/50000 (9%)]	Loss: 1.283668
Train Epoch: 5 [5120/50000 (10%)]	Loss: 1.369376
Train Epoch: 5 [5760/50000 (12%)]	Loss: 1.370333
Train Epoch: 5 [6400/50000 (13%)]	Loss: 1.322791
Train Epoch: 5 [7040/50000 (14%)]	Loss: 1.379656
Train Epoch: 5 [7680/50000 (15%)]	Loss: 1.344546
Train Epoch: 5 [8320/50000 (17%)]	Loss: 1.179212
Train Epoch: 5 [8960/50000 (18%)]	Loss: 1.346136
Train Epoch: 5 [9600/50000 (19%)]	Loss: 1.342019
Train Epoch: 5 [10240/50000 (20%)]	Loss: 1.415076
Train Epoch: 5 [10880/50000 (22%)]	Loss: 1.153403
Train Epoch: 5 [11520/50000 (23%)]	Loss: 1.259611
Train Epoch: 5 [12160/50000 (24%)]	Loss: 1.254711
Train Epoch: 5 [12800/50000 (26%)]	Loss: 1.406647
Train Epoch: 5 [13440/50000 (27%)]	Loss: 1.341988
Train Epoch: 5 [14080/50000 (28%)]	Loss: 1.620060
Train Epoch: 5 [14720/50000 (29%)]	Loss: 1.582005
Train Epoch: 5 [15360/50000 (31%)]	Loss: 1.138790
Train Epoch: 5 [16000/50000 (32%)]	Loss: 1.361233
Train Epoch: 5 [16640/50000 (33%)]	Loss: 1.352421
Train Epoch: 5 [17280/50000 (35%)]	Loss: 1.320864
Train Epoch: 5 [17920/50000 (36%)]	Loss: 1.623935
Train Epoch: 5 [18560/50000 (37%)]	Loss: 1.312673
Train Epoch: 5 [19200/50000 (38%)]	Loss: 1.267328
Train Epoch: 5 [19840/50000 (40%)]	Loss: 1.430542
Train Epoch: 5 [20480/50000 (41%)]	Loss: 1.618905
Train Epoch: 5 [21120/50000 (42%)]	Loss: 1.359690
Train Epoch: 5 [21760/50000 (43%)]	Loss: 1.266620
Train Epoch: 5 [22400/50000 (45%)]	Loss: 1.495596
Train Epoch: 5 [23040/50000 (46%)]	Loss: 1.519270
Train Epoch: 5 [23680/50000 (47%)]	Loss: 1.651788
Train Epoch: 5 [24320/50000 (49%)]	Loss: 1.423861
Train Epoch: 5 [24960/50000 (50%)]	Loss: 1.647923
Train Epoch: 5 [25600/50000 (51%)]	Loss: 1.492975
Train Epoch: 5 [26240/50000 (52%)]	Loss: 1.293548
Train Epoch: 5 [26880/50000 (54%)]	Loss: 1.409751
Train Epoch: 5 [27520/50000 (55%)]	Loss: 1.390187
Train Epoch: 5 [28160/50000 (56%)]	Loss: 1.713648
Train Epoch: 5 [28800/50000 (58%)]	Loss: 1.361178
Train Epoch: 5 [29440/50000 (59%)]	Loss: 1.376710
Train Epoch: 5 [30080/50000 (60%)]	Loss: 1.273065
Train Epoch: 5 [30720/50000 (61%)]	Loss: 1.465091
Train Epoch: 5 [31360/50000 (63%)]	Loss: 1.325658
Train Epoch: 5 [32000/50000 (64%)]	Loss: 1.414705
Train Epoch: 5 [32640/50000 (65%)]	Loss: 1.446043
Train Epoch: 5 [33280/50000 (66%)]	Loss: 1.420556
Train Epoch: 5 [33920/50000 (68%)]	Loss: 1.347827
Train Epoch: 5 [34560/50000 (69%)]	Loss: 1.219562
Train Epoch: 5 [35200/50000 (70%)]	Loss: 1.323381
Train Epoch: 5 [35840/50000 (72%)]	Loss: 1.344267
Train Epoch: 5 [36480/50000 (73%)]	Loss: 1.366985
Train Epoch: 5 [37120/50000 (74%)]	Loss: 1.309744
Train Epoch: 5 [37760/50000 (75%)]	Loss: 1.242900
Train Epoch: 5 [38400/50000 (77%)]	Loss: 1.285107
Train Epoch: 5 [39040/50000 (78%)]	Loss: 1.305174
Train Epoch: 5 [39680/50000 (79%)]	Loss: 1.396844
Train Epoch: 5 [40320/50000 (81%)]	Loss: 1.434718
Train Epoch: 5 [40960/50000 (82%)]	Loss: 1.058782
Train Epoch: 5 [41600/50000 (83%)]	Loss: 1.344125
Train Epoch: 5 [42240/50000 (84%)]	Loss: 1.152380
Train Epoch: 5 [42880/50000 (86%)]	Loss: 1.301865
Train Epoch: 5 [43520/50000 (87%)]	Loss: 1.257812
Train Epoch: 5 [44160/50000 (88%)]	Loss: 1.214807
Train Epoch: 5 [44800/50000 (90%)]	Loss: 1.341784
Train Epoch: 5 [45440/50000 (91%)]	Loss: 1.308646
Train Epoch: 5 [46080/50000 (92%)]	Loss: 1.479198
Train Epoch: 5 [46720/50000 (93%)]	Loss: 1.248796
Train Epoch: 5 [47360/50000 (95%)]	Loss: 1.591504
Train Epoch: 5 [48000/50000 (96%)]	Loss: 1.161117
Train Epoch: 5 [48640/50000 (97%)]	Loss: 1.206317
Train Epoch: 5 [49280/50000 (98%)]	Loss: 1.331528
Train Epoch: 5 [49920/50000 (100%)]	Loss: 1.308794

Test set: Average loss: 1.3193, Accuracy: 5272/10000 (53%)

Train Epoch: 6 [0/50000 (0%)]	Loss: 1.244137
Train Epoch: 6 [640/50000 (1%)]	Loss: 1.306573
Train Epoch: 6 [1280/50000 (3%)]	Loss: 1.052181
Train Epoch: 6 [1920/50000 (4%)]	Loss: 1.043325
Train Epoch: 6 [2560/50000 (5%)]	Loss: 0.996724
Train Epoch: 6 [3200/50000 (6%)]	Loss: 1.119790
Train Epoch: 6 [3840/50000 (8%)]	Loss: 1.498010
Train Epoch: 6 [4480/50000 (9%)]	Loss: 1.318593
Train Epoch: 6 [5120/50000 (10%)]	Loss: 1.118249
Train Epoch: 6 [5760/50000 (12%)]	Loss: 1.254192
Train Epoch: 6 [6400/50000 (13%)]	Loss: 1.256566
Train Epoch: 6 [7040/50000 (14%)]	Loss: 1.359943
Train Epoch: 6 [7680/50000 (15%)]	Loss: 1.434633
Train Epoch: 6 [8320/50000 (17%)]	Loss: 1.100543
Train Epoch: 6 [8960/50000 (18%)]	Loss: 1.179376
Train Epoch: 6 [9600/50000 (19%)]	Loss: 1.572447
Train Epoch: 6 [10240/50000 (20%)]	Loss: 1.048213
Train Epoch: 6 [10880/50000 (22%)]	Loss: 1.369194
Train Epoch: 6 [11520/50000 (23%)]	Loss: 1.292932
Train Epoch: 6 [12160/50000 (24%)]	Loss: 1.297240
Train Epoch: 6 [12800/50000 (26%)]	Loss: 0.904128
Train Epoch: 6 [13440/50000 (27%)]	Loss: 1.171497
Train Epoch: 6 [14080/50000 (28%)]	Loss: 1.482756
Train Epoch: 6 [14720/50000 (29%)]	Loss: 1.279791
Train Epoch: 6 [15360/50000 (31%)]	Loss: 1.317679
Train Epoch: 6 [16000/50000 (32%)]	Loss: 1.705234
Train Epoch: 6 [16640/50000 (33%)]	Loss: 1.314513
Train Epoch: 6 [17280/50000 (35%)]	Loss: 1.024129
Train Epoch: 6 [17920/50000 (36%)]	Loss: 1.273722
Train Epoch: 6 [18560/50000 (37%)]	Loss: 1.418250
Train Epoch: 6 [19200/50000 (38%)]	Loss: 1.432362
Train Epoch: 6 [19840/50000 (40%)]	Loss: 1.267064
Train Epoch: 6 [20480/50000 (41%)]	Loss: 1.077793
Train Epoch: 6 [21120/50000 (42%)]	Loss: 1.186827
Train Epoch: 6 [21760/50000 (43%)]	Loss: 1.247461
Train Epoch: 6 [22400/50000 (45%)]	Loss: 1.205383
Train Epoch: 6 [23040/50000 (46%)]	Loss: 1.261547
Train Epoch: 6 [23680/50000 (47%)]	Loss: 1.323204
Train Epoch: 6 [24320/50000 (49%)]	Loss: 1.044377
Train Epoch: 6 [24960/50000 (50%)]	Loss: 1.261376
Train Epoch: 6 [25600/50000 (51%)]	Loss: 1.278018
Train Epoch: 6 [26240/50000 (52%)]	Loss: 1.237591
Train Epoch: 6 [26880/50000 (54%)]	Loss: 1.297865
Train Epoch: 6 [27520/50000 (55%)]	Loss: 1.125305
Train Epoch: 6 [28160/50000 (56%)]	Loss: 1.194949
Train Epoch: 6 [28800/50000 (58%)]	Loss: 1.126032
Train Epoch: 6 [29440/50000 (59%)]	Loss: 1.450063
Train Epoch: 6 [30080/50000 (60%)]	Loss: 1.187964
Train Epoch: 6 [30720/50000 (61%)]	Loss: 1.232924
Train Epoch: 6 [31360/50000 (63%)]	Loss: 1.493404
Train Epoch: 6 [32000/50000 (64%)]	Loss: 1.296185
Train Epoch: 6 [32640/50000 (65%)]	Loss: 1.329017
Train Epoch: 6 [33280/50000 (66%)]	Loss: 1.135613
Train Epoch: 6 [33920/50000 (68%)]	Loss: 1.325450
Train Epoch: 6 [34560/50000 (69%)]	Loss: 1.291036
Train Epoch: 6 [35200/50000 (70%)]	Loss: 1.317321
Train Epoch: 6 [35840/50000 (72%)]	Loss: 0.951563
Train Epoch: 6 [36480/50000 (73%)]	Loss: 1.338327
Train Epoch: 6 [37120/50000 (74%)]	Loss: 1.140110
Train Epoch: 6 [37760/50000 (75%)]	Loss: 1.408208
Train Epoch: 6 [38400/50000 (77%)]	Loss: 1.400659
Train Epoch: 6 [39040/50000 (78%)]	Loss: 1.399343
Train Epoch: 6 [39680/50000 (79%)]	Loss: 1.159825
Train Epoch: 6 [40320/50000 (81%)]	Loss: 1.305009
Train Epoch: 6 [40960/50000 (82%)]	Loss: 1.365552
Train Epoch: 6 [41600/50000 (83%)]	Loss: 1.502283
Train Epoch: 6 [42240/50000 (84%)]	Loss: 1.426732
Train Epoch: 6 [42880/50000 (86%)]	Loss: 0.891437
Train Epoch: 6 [43520/50000 (87%)]	Loss: 0.915022
Train Epoch: 6 [44160/50000 (88%)]	Loss: 1.473988
Train Epoch: 6 [44800/50000 (90%)]	Loss: 1.341908
Train Epoch: 6 [45440/50000 (91%)]	Loss: 1.106196
Train Epoch: 6 [46080/50000 (92%)]	Loss: 1.365336
Train Epoch: 6 [46720/50000 (93%)]	Loss: 0.983353
Train Epoch: 6 [47360/50000 (95%)]	Loss: 1.385888
Train Epoch: 6 [48000/50000 (96%)]	Loss: 1.269527
Train Epoch: 6 [48640/50000 (97%)]	Loss: 1.313766
Train Epoch: 6 [49280/50000 (98%)]	Loss: 1.302339
Train Epoch: 6 [49920/50000 (100%)]	Loss: 1.084640

Test set: Average loss: 1.2868, Accuracy: 5438/10000 (54%)

Train Epoch: 7 [0/50000 (0%)]	Loss: 1.170374
Train Epoch: 7 [640/50000 (1%)]	Loss: 1.369099
Train Epoch: 7 [1280/50000 (3%)]	Loss: 1.226083
Train Epoch: 7 [1920/50000 (4%)]	Loss: 1.354968
Train Epoch: 7 [2560/50000 (5%)]	Loss: 1.235415
Train Epoch: 7 [3200/50000 (6%)]	Loss: 1.130053
Train Epoch: 7 [3840/50000 (8%)]	Loss: 1.155141
Train Epoch: 7 [4480/50000 (9%)]	Loss: 1.143381
Train Epoch: 7 [5120/50000 (10%)]	Loss: 1.352825
Train Epoch: 7 [5760/50000 (12%)]	Loss: 1.192291
Train Epoch: 7 [6400/50000 (13%)]	Loss: 0.994078
Train Epoch: 7 [7040/50000 (14%)]	Loss: 1.257548
Train Epoch: 7 [7680/50000 (15%)]	Loss: 1.222386
Train Epoch: 7 [8320/50000 (17%)]	Loss: 1.131799
Train Epoch: 7 [8960/50000 (18%)]	Loss: 1.169959
Train Epoch: 7 [9600/50000 (19%)]	Loss: 1.197146
Train Epoch: 7 [10240/50000 (20%)]	Loss: 1.172031
Train Epoch: 7 [10880/50000 (22%)]	Loss: 1.273661
Train Epoch: 7 [11520/50000 (23%)]	Loss: 1.203349
Train Epoch: 7 [12160/50000 (24%)]	Loss: 1.462102
Train Epoch: 7 [12800/50000 (26%)]	Loss: 1.088499
Train Epoch: 7 [13440/50000 (27%)]	Loss: 1.452775
Train Epoch: 7 [14080/50000 (28%)]	Loss: 1.386626
Train Epoch: 7 [14720/50000 (29%)]	Loss: 1.097796
Train Epoch: 7 [15360/50000 (31%)]	Loss: 1.190302
Train Epoch: 7 [16000/50000 (32%)]	Loss: 1.142883
Train Epoch: 7 [16640/50000 (33%)]	Loss: 1.123207
Train Epoch: 7 [17280/50000 (35%)]	Loss: 1.599044
Train Epoch: 7 [17920/50000 (36%)]	Loss: 1.309188
Train Epoch: 7 [18560/50000 (37%)]	Loss: 1.464440
Train Epoch: 7 [19200/50000 (38%)]	Loss: 1.104502
Train Epoch: 7 [19840/50000 (40%)]	Loss: 1.185738
Train Epoch: 7 [20480/50000 (41%)]	Loss: 1.384815
Train Epoch: 7 [21120/50000 (42%)]	Loss: 1.252200
Train Epoch: 7 [21760/50000 (43%)]	Loss: 1.317562
Train Epoch: 7 [22400/50000 (45%)]	Loss: 1.063213
Train Epoch: 7 [23040/50000 (46%)]	Loss: 1.218362
Train Epoch: 7 [23680/50000 (47%)]	Loss: 1.150210
Train Epoch: 7 [24320/50000 (49%)]	Loss: 1.274950
Train Epoch: 7 [24960/50000 (50%)]	Loss: 1.197119
Train Epoch: 7 [25600/50000 (51%)]	Loss: 1.297373
Train Epoch: 7 [26240/50000 (52%)]	Loss: 1.260130
Train Epoch: 7 [26880/50000 (54%)]	Loss: 1.354905
Train Epoch: 7 [27520/50000 (55%)]	Loss: 1.345893
Train Epoch: 7 [28160/50000 (56%)]	Loss: 1.144807
Train Epoch: 7 [28800/50000 (58%)]	Loss: 1.071841
Train Epoch: 7 [29440/50000 (59%)]	Loss: 1.367535
Train Epoch: 7 [30080/50000 (60%)]	Loss: 1.139255
Train Epoch: 7 [30720/50000 (61%)]	Loss: 1.099987
Train Epoch: 7 [31360/50000 (63%)]	Loss: 1.205038
Train Epoch: 7 [32000/50000 (64%)]	Loss: 1.162194
Train Epoch: 7 [32640/50000 (65%)]	Loss: 1.215107
Train Epoch: 7 [33280/50000 (66%)]	Loss: 1.039225
Train Epoch: 7 [33920/50000 (68%)]	Loss: 1.642156
Train Epoch: 7 [34560/50000 (69%)]	Loss: 1.283979
Train Epoch: 7 [35200/50000 (70%)]	Loss: 1.201442
Train Epoch: 7 [35840/50000 (72%)]	Loss: 1.236576
Train Epoch: 7 [36480/50000 (73%)]	Loss: 1.272451
Train Epoch: 7 [37120/50000 (74%)]	Loss: 1.108083
Train Epoch: 7 [37760/50000 (75%)]	Loss: 1.078411
Train Epoch: 7 [38400/50000 (77%)]	Loss: 1.441373
Train Epoch: 7 [39040/50000 (78%)]	Loss: 1.393472
Train Epoch: 7 [39680/50000 (79%)]	Loss: 1.219237
Train Epoch: 7 [40320/50000 (81%)]	Loss: 1.173135
Train Epoch: 7 [40960/50000 (82%)]	Loss: 1.338487
Train Epoch: 7 [41600/50000 (83%)]	Loss: 1.394005
Train Epoch: 7 [42240/50000 (84%)]	Loss: 1.126216
Train Epoch: 7 [42880/50000 (86%)]	Loss: 1.237089
Train Epoch: 7 [43520/50000 (87%)]	Loss: 1.285320
Train Epoch: 7 [44160/50000 (88%)]	Loss: 0.913450
Train Epoch: 7 [44800/50000 (90%)]	Loss: 1.156765
Train Epoch: 7 [45440/50000 (91%)]	Loss: 1.267249
Train Epoch: 7 [46080/50000 (92%)]	Loss: 1.264750
Train Epoch: 7 [46720/50000 (93%)]	Loss: 1.012602
Train Epoch: 7 [47360/50000 (95%)]	Loss: 1.150002
Train Epoch: 7 [48000/50000 (96%)]	Loss: 1.508275
Train Epoch: 7 [48640/50000 (97%)]	Loss: 1.142389
Train Epoch: 7 [49280/50000 (98%)]	Loss: 1.315862
Train Epoch: 7 [49920/50000 (100%)]	Loss: 1.523153

Test set: Average loss: 1.2448, Accuracy: 5567/10000 (56%)

Train Epoch: 8 [0/50000 (0%)]	Loss: 1.072402
Train Epoch: 8 [640/50000 (1%)]	Loss: 1.400547
Train Epoch: 8 [1280/50000 (3%)]	Loss: 1.029351
Train Epoch: 8 [1920/50000 (4%)]	Loss: 1.355426
Train Epoch: 8 [2560/50000 (5%)]	Loss: 1.143614
Train Epoch: 8 [3200/50000 (6%)]	Loss: 1.247017
Train Epoch: 8 [3840/50000 (8%)]	Loss: 1.154314
Train Epoch: 8 [4480/50000 (9%)]	Loss: 1.383020
Train Epoch: 8 [5120/50000 (10%)]	Loss: 1.137328
Train Epoch: 8 [5760/50000 (12%)]	Loss: 1.373622
Train Epoch: 8 [6400/50000 (13%)]	Loss: 1.259898
Train Epoch: 8 [7040/50000 (14%)]	Loss: 1.167939
Train Epoch: 8 [7680/50000 (15%)]	Loss: 1.419825
Train Epoch: 8 [8320/50000 (17%)]	Loss: 1.386813
Train Epoch: 8 [8960/50000 (18%)]	Loss: 1.010227
Train Epoch: 8 [9600/50000 (19%)]	Loss: 1.206317
Train Epoch: 8 [10240/50000 (20%)]	Loss: 1.232049
Train Epoch: 8 [10880/50000 (22%)]	Loss: 0.893704
Train Epoch: 8 [11520/50000 (23%)]	Loss: 1.053214
Train Epoch: 8 [12160/50000 (24%)]	Loss: 1.256501
Train Epoch: 8 [12800/50000 (26%)]	Loss: 1.490986
Train Epoch: 8 [13440/50000 (27%)]	Loss: 1.223085
Train Epoch: 8 [14080/50000 (28%)]	Loss: 0.941483
Train Epoch: 8 [14720/50000 (29%)]	Loss: 1.161023
Train Epoch: 8 [15360/50000 (31%)]	Loss: 1.024068
Train Epoch: 8 [16000/50000 (32%)]	Loss: 1.092653
Train Epoch: 8 [16640/50000 (33%)]	Loss: 1.051545
Train Epoch: 8 [17280/50000 (35%)]	Loss: 1.020500
Train Epoch: 8 [17920/50000 (36%)]	Loss: 1.233940
Train Epoch: 8 [18560/50000 (37%)]	Loss: 1.380895
Train Epoch: 8 [19200/50000 (38%)]	Loss: 1.257402
Train Epoch: 8 [19840/50000 (40%)]	Loss: 1.254779
Train Epoch: 8 [20480/50000 (41%)]	Loss: 1.135307
Train Epoch: 8 [21120/50000 (42%)]	Loss: 1.080292
Train Epoch: 8 [21760/50000 (43%)]	Loss: 1.057067
Train Epoch: 8 [22400/50000 (45%)]	Loss: 1.309334
Train Epoch: 8 [23040/50000 (46%)]	Loss: 1.270732
Train Epoch: 8 [23680/50000 (47%)]	Loss: 1.388063
Train Epoch: 8 [24320/50000 (49%)]	Loss: 1.225648
Train Epoch: 8 [24960/50000 (50%)]	Loss: 1.036037
Train Epoch: 8 [25600/50000 (51%)]	Loss: 1.114033
Train Epoch: 8 [26240/50000 (52%)]	Loss: 1.277021
Train Epoch: 8 [26880/50000 (54%)]	Loss: 1.132265
Train Epoch: 8 [27520/50000 (55%)]	Loss: 1.045857
Train Epoch: 8 [28160/50000 (56%)]	Loss: 1.222712
Train Epoch: 8 [28800/50000 (58%)]	Loss: 1.120882
Train Epoch: 8 [29440/50000 (59%)]	Loss: 1.018475
Train Epoch: 8 [30080/50000 (60%)]	Loss: 1.204304
Train Epoch: 8 [30720/50000 (61%)]	Loss: 1.175631
Train Epoch: 8 [31360/50000 (63%)]	Loss: 1.014583
Train Epoch: 8 [32000/50000 (64%)]	Loss: 1.286157
Train Epoch: 8 [32640/50000 (65%)]	Loss: 1.126208
Train Epoch: 8 [33280/50000 (66%)]	Loss: 1.402749
Train Epoch: 8 [33920/50000 (68%)]	Loss: 1.186486
Train Epoch: 8 [34560/50000 (69%)]	Loss: 1.243330
Train Epoch: 8 [35200/50000 (70%)]	Loss: 1.165082
Train Epoch: 8 [35840/50000 (72%)]	Loss: 1.419918
Train Epoch: 8 [36480/50000 (73%)]	Loss: 1.003125
Train Epoch: 8 [37120/50000 (74%)]	Loss: 1.326862
Train Epoch: 8 [37760/50000 (75%)]	Loss: 1.295638
Train Epoch: 8 [38400/50000 (77%)]	Loss: 0.998338
Train Epoch: 8 [39040/50000 (78%)]	Loss: 1.434232
Train Epoch: 8 [39680/50000 (79%)]	Loss: 0.977238
Train Epoch: 8 [40320/50000 (81%)]	Loss: 1.076092
Train Epoch: 8 [40960/50000 (82%)]	Loss: 1.081988
Train Epoch: 8 [41600/50000 (83%)]	Loss: 1.251443
Train Epoch: 8 [42240/50000 (84%)]	Loss: 1.168906
Train Epoch: 8 [42880/50000 (86%)]	Loss: 1.344340
Train Epoch: 8 [43520/50000 (87%)]	Loss: 1.194258
Train Epoch: 8 [44160/50000 (88%)]	Loss: 1.040574
Train Epoch: 8 [44800/50000 (90%)]	Loss: 1.296527
Train Epoch: 8 [45440/50000 (91%)]	Loss: 1.153816
Train Epoch: 8 [46080/50000 (92%)]	Loss: 1.230292
Train Epoch: 8 [46720/50000 (93%)]	Loss: 1.201238
Train Epoch: 8 [47360/50000 (95%)]	Loss: 1.176487
Train Epoch: 8 [48000/50000 (96%)]	Loss: 1.234045
Train Epoch: 8 [48640/50000 (97%)]	Loss: 1.205381
Train Epoch: 8 [49280/50000 (98%)]	Loss: 1.109252
Train Epoch: 8 [49920/50000 (100%)]	Loss: 1.304659

Test set: Average loss: 1.2405, Accuracy: 5608/10000 (56%)

Train Epoch: 9 [0/50000 (0%)]	Loss: 1.080825
Train Epoch: 9 [640/50000 (1%)]	Loss: 1.149397
Train Epoch: 9 [1280/50000 (3%)]	Loss: 1.163961
Train Epoch: 9 [1920/50000 (4%)]	Loss: 1.042424
Train Epoch: 9 [2560/50000 (5%)]	Loss: 1.119047
Train Epoch: 9 [3200/50000 (6%)]	Loss: 1.201742
Train Epoch: 9 [3840/50000 (8%)]	Loss: 1.257601
Train Epoch: 9 [4480/50000 (9%)]	Loss: 1.126453
Train Epoch: 9 [5120/50000 (10%)]	Loss: 1.248921
Train Epoch: 9 [5760/50000 (12%)]	Loss: 1.047927
Train Epoch: 9 [6400/50000 (13%)]	Loss: 1.052234
Train Epoch: 9 [7040/50000 (14%)]	Loss: 0.932961
Train Epoch: 9 [7680/50000 (15%)]	Loss: 1.276122
Train Epoch: 9 [8320/50000 (17%)]	Loss: 1.149738
Train Epoch: 9 [8960/50000 (18%)]	Loss: 1.166821
Train Epoch: 9 [9600/50000 (19%)]	Loss: 1.145908
Train Epoch: 9 [10240/50000 (20%)]	Loss: 1.246014
Train Epoch: 9 [10880/50000 (22%)]	Loss: 1.179533
Train Epoch: 9 [11520/50000 (23%)]	Loss: 1.399501
Train Epoch: 9 [12160/50000 (24%)]	Loss: 1.286893
Train Epoch: 9 [12800/50000 (26%)]	Loss: 1.393483
Train Epoch: 9 [13440/50000 (27%)]	Loss: 0.895625
Train Epoch: 9 [14080/50000 (28%)]	Loss: 0.997761
Train Epoch: 9 [14720/50000 (29%)]	Loss: 1.161322
Train Epoch: 9 [15360/50000 (31%)]	Loss: 1.233599
Train Epoch: 9 [16000/50000 (32%)]	Loss: 1.036590
Train Epoch: 9 [16640/50000 (33%)]	Loss: 1.159275
Train Epoch: 9 [17280/50000 (35%)]	Loss: 1.208840
Train Epoch: 9 [17920/50000 (36%)]	Loss: 1.165576
Train Epoch: 9 [18560/50000 (37%)]	Loss: 1.043464
Train Epoch: 9 [19200/50000 (38%)]	Loss: 0.991411
Train Epoch: 9 [19840/50000 (40%)]	Loss: 1.119227
Train Epoch: 9 [20480/50000 (41%)]	Loss: 1.164239
Train Epoch: 9 [21120/50000 (42%)]	Loss: 1.207015
Train Epoch: 9 [21760/50000 (43%)]	Loss: 1.059717
Train Epoch: 9 [22400/50000 (45%)]	Loss: 1.024048
Train Epoch: 9 [23040/50000 (46%)]	Loss: 1.093387
Train Epoch: 9 [23680/50000 (47%)]	Loss: 1.230202
Train Epoch: 9 [24320/50000 (49%)]	Loss: 1.166376
Train Epoch: 9 [24960/50000 (50%)]	Loss: 1.164026
Train Epoch: 9 [25600/50000 (51%)]	Loss: 1.325647
Train Epoch: 9 [26240/50000 (52%)]	Loss: 1.381456
Train Epoch: 9 [26880/50000 (54%)]	Loss: 1.003599
Train Epoch: 9 [27520/50000 (55%)]	Loss: 1.570321
Train Epoch: 9 [28160/50000 (56%)]	Loss: 1.164491
Train Epoch: 9 [28800/50000 (58%)]	Loss: 1.330709
Train Epoch: 9 [29440/50000 (59%)]	Loss: 1.127720
Train Epoch: 9 [30080/50000 (60%)]	Loss: 1.265166
Train Epoch: 9 [30720/50000 (61%)]	Loss: 1.113619
Train Epoch: 9 [31360/50000 (63%)]	Loss: 1.182729
Train Epoch: 9 [32000/50000 (64%)]	Loss: 1.201801
Train Epoch: 9 [32640/50000 (65%)]	Loss: 1.345610
Train Epoch: 9 [33280/50000 (66%)]	Loss: 1.182712
Train Epoch: 9 [33920/50000 (68%)]	Loss: 1.135103
Train Epoch: 9 [34560/50000 (69%)]	Loss: 0.999632
Train Epoch: 9 [35200/50000 (70%)]	Loss: 1.293448
Train Epoch: 9 [35840/50000 (72%)]	Loss: 1.199271
Train Epoch: 9 [36480/50000 (73%)]	Loss: 1.442687
Train Epoch: 9 [37120/50000 (74%)]	Loss: 1.141148
Train Epoch: 9 [37760/50000 (75%)]	Loss: 1.022235
Train Epoch: 9 [38400/50000 (77%)]	Loss: 1.142261
Train Epoch: 9 [39040/50000 (78%)]	Loss: 1.139752
Train Epoch: 9 [39680/50000 (79%)]	Loss: 1.362574
Train Epoch: 9 [40320/50000 (81%)]	Loss: 1.233796
Train Epoch: 9 [40960/50000 (82%)]	Loss: 1.150347
Train Epoch: 9 [41600/50000 (83%)]	Loss: 1.421433
Train Epoch: 9 [42240/50000 (84%)]	Loss: 1.207026
Train Epoch: 9 [42880/50000 (86%)]	Loss: 1.092185
Train Epoch: 9 [43520/50000 (87%)]	Loss: 1.240725
Train Epoch: 9 [44160/50000 (88%)]	Loss: 1.262161
Train Epoch: 9 [44800/50000 (90%)]	Loss: 1.280808
Train Epoch: 9 [45440/50000 (91%)]	Loss: 1.236783
Train Epoch: 9 [46080/50000 (92%)]	Loss: 1.268925
Train Epoch: 9 [46720/50000 (93%)]	Loss: 1.006667
Train Epoch: 9 [47360/50000 (95%)]	Loss: 0.887830
Train Epoch: 9 [48000/50000 (96%)]	Loss: 1.164327
Train Epoch: 9 [48640/50000 (97%)]	Loss: 1.188799
Train Epoch: 9 [49280/50000 (98%)]	Loss: 1.254041
Train Epoch: 9 [49920/50000 (100%)]	Loss: 1.045618

Test set: Average loss: 1.2421, Accuracy: 5663/10000 (57%)

Train Epoch: 10 [0/50000 (0%)]	Loss: 0.995620
Train Epoch: 10 [640/50000 (1%)]	Loss: 1.066549
Train Epoch: 10 [1280/50000 (3%)]	Loss: 1.037003
Train Epoch: 10 [1920/50000 (4%)]	Loss: 1.049361
Train Epoch: 10 [2560/50000 (5%)]	Loss: 1.009352
Train Epoch: 10 [3200/50000 (6%)]	Loss: 1.294566
Train Epoch: 10 [3840/50000 (8%)]	Loss: 1.065550
Train Epoch: 10 [4480/50000 (9%)]	Loss: 1.013587
Train Epoch: 10 [5120/50000 (10%)]	Loss: 1.288592
Train Epoch: 10 [5760/50000 (12%)]	Loss: 1.064076
Train Epoch: 10 [6400/50000 (13%)]	Loss: 1.250758
Train Epoch: 10 [7040/50000 (14%)]	Loss: 1.146611
Train Epoch: 10 [7680/50000 (15%)]	Loss: 1.225081
Train Epoch: 10 [8320/50000 (17%)]	Loss: 1.145210
Train Epoch: 10 [8960/50000 (18%)]	Loss: 1.442080
Train Epoch: 10 [9600/50000 (19%)]	Loss: 1.068791
Train Epoch: 10 [10240/50000 (20%)]	Loss: 1.022411
Train Epoch: 10 [10880/50000 (22%)]	Loss: 1.098513
Train Epoch: 10 [11520/50000 (23%)]	Loss: 1.097022
Train Epoch: 10 [12160/50000 (24%)]	Loss: 1.140366
Train Epoch: 10 [12800/50000 (26%)]	Loss: 1.081832
Train Epoch: 10 [13440/50000 (27%)]	Loss: 1.086733
Train Epoch: 10 [14080/50000 (28%)]	Loss: 1.125593
Train Epoch: 10 [14720/50000 (29%)]	Loss: 1.084142
Train Epoch: 10 [15360/50000 (31%)]	Loss: 1.042363
Train Epoch: 10 [16000/50000 (32%)]	Loss: 1.032603
Train Epoch: 10 [16640/50000 (33%)]	Loss: 1.439873
Train Epoch: 10 [17280/50000 (35%)]	Loss: 1.027925
Train Epoch: 10 [17920/50000 (36%)]	Loss: 1.246142
Train Epoch: 10 [18560/50000 (37%)]	Loss: 1.196549
Train Epoch: 10 [19200/50000 (38%)]	Loss: 1.165420
Train Epoch: 10 [19840/50000 (40%)]	Loss: 1.052159
Train Epoch: 10 [20480/50000 (41%)]	Loss: 0.989281
Train Epoch: 10 [21120/50000 (42%)]	Loss: 1.070454
Train Epoch: 10 [21760/50000 (43%)]	Loss: 1.082900
Train Epoch: 10 [22400/50000 (45%)]	Loss: 1.180943
Train Epoch: 10 [23040/50000 (46%)]	Loss: 1.101617
Train Epoch: 10 [23680/50000 (47%)]	Loss: 0.981478
Train Epoch: 10 [24320/50000 (49%)]	Loss: 1.101792
Train Epoch: 10 [24960/50000 (50%)]	Loss: 1.099650
Train Epoch: 10 [25600/50000 (51%)]	Loss: 1.076202
Train Epoch: 10 [26240/50000 (52%)]	Loss: 1.197555
Train Epoch: 10 [26880/50000 (54%)]	Loss: 1.246986
Train Epoch: 10 [27520/50000 (55%)]	Loss: 1.168575
Train Epoch: 10 [28160/50000 (56%)]	Loss: 1.075723
Train Epoch: 10 [28800/50000 (58%)]	Loss: 1.267955
Train Epoch: 10 [29440/50000 (59%)]	Loss: 1.004160
Train Epoch: 10 [30080/50000 (60%)]	Loss: 1.011682
Train Epoch: 10 [30720/50000 (61%)]	Loss: 1.223039
Train Epoch: 10 [31360/50000 (63%)]	Loss: 0.959307
Train Epoch: 10 [32000/50000 (64%)]	Loss: 1.100940
Train Epoch: 10 [32640/50000 (65%)]	Loss: 1.317597
Train Epoch: 10 [33280/50000 (66%)]	Loss: 1.522677
Train Epoch: 10 [33920/50000 (68%)]	Loss: 1.166107
Train Epoch: 10 [34560/50000 (69%)]	Loss: 1.025541
Train Epoch: 10 [35200/50000 (70%)]	Loss: 1.520663
Train Epoch: 10 [35840/50000 (72%)]	Loss: 1.050765
Train Epoch: 10 [36480/50000 (73%)]	Loss: 1.031278
Train Epoch: 10 [37120/50000 (74%)]	Loss: 0.985936
Train Epoch: 10 [37760/50000 (75%)]	Loss: 0.987422
Train Epoch: 10 [38400/50000 (77%)]	Loss: 1.179592
Train Epoch: 10 [39040/50000 (78%)]	Loss: 1.343099
Train Epoch: 10 [39680/50000 (79%)]	Loss: 1.313347
Train Epoch: 10 [40320/50000 (81%)]	Loss: 1.432083
Train Epoch: 10 [40960/50000 (82%)]	Loss: 1.285003
Train Epoch: 10 [41600/50000 (83%)]	Loss: 0.932963
Train Epoch: 10 [42240/50000 (84%)]	Loss: 1.155765
Train Epoch: 10 [42880/50000 (86%)]	Loss: 0.952566
Train Epoch: 10 [43520/50000 (87%)]	Loss: 1.146627
Train Epoch: 10 [44160/50000 (88%)]	Loss: 1.353587
Train Epoch: 10 [44800/50000 (90%)]	Loss: 1.022760
Train Epoch: 10 [45440/50000 (91%)]	Loss: 1.383197
Train Epoch: 10 [46080/50000 (92%)]	Loss: 1.293292
Train Epoch: 10 [46720/50000 (93%)]	Loss: 1.181000
Train Epoch: 10 [47360/50000 (95%)]	Loss: 1.270079
Train Epoch: 10 [48000/50000 (96%)]	Loss: 1.206061
Train Epoch: 10 [48640/50000 (97%)]	Loss: 1.117023
Train Epoch: 10 [49280/50000 (98%)]	Loss: 1.218935
Train Epoch: 10 [49920/50000 (100%)]	Loss: 1.033880

Test set: Average loss: 1.1890, Accuracy: 5896/10000 (59%)







    













    













    













    




<class 'torch.FloatTensor'> torch.Size([6, 3, 5, 5])
<class 'torch.FloatTensor'> torch.Size([3, 14, 21])

In [12]:

    

# with Dropout

from __future__ import print_function
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms
from torch.autograd import Variable

import math
import numpy as np
import matplotlib.pyplot as plt
from torchvision.utils import make_grid
from collections import OrderedDict

# Training settings
# for terminal use. In notebook, you can't parse arguments
class args:
    cuda = False
    batch_size = 64
    test_batch_size = 1000
    epochs = 10
    lr = 0.01
    momentum = 0.5
    no_cuda = False
    seed = 1
    log_interval = 10
    # if add Dropout
    with_dropout = True
    # if initialize weights
    with_init_weights = False
    # if add BatchNorm
    with_batchnorm = False

args.cuda = not args.no_cuda and torch.cuda.is_available()

torch.manual_seed(args.seed)
if args.cuda:
    torch.cuda.manual_seed(args.seed)

kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {}
# The output of torchvision datasets are PILImage images of range [0, 1].
# We transform them to Tensors of normalized range [-1, 1]
transform = transforms.Compose([transforms.ToTensor(),
                              transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
                             ])
trainset = datasets.CIFAR10(root='./data', train=True, download=True, transform=transform)
train_loader = torch.utils.data.DataLoader(trainset, batch_size=args.batch_size,
                                          shuffle=True, num_workers=2)

testset = datasets.CIFAR10(root='./data', train=False, download=True, transform=transform)
test_loader = torch.utils.data.DataLoader(testset, batch_size=args.test_batch_size,
                                          shuffle=False, num_workers=2)
classes = ('plane', 'car', 'bird', 'cat',
           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')


class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        # TODO: define your network here
        self.conv_1 = nn.Conv2d(3, 6, kernel_size=5, stride=1)
        self.conv_2 = nn.Conv2d(6, 16, kernel_size=5, stride=1)
        # TODO: replace fc with conv
        self.fc_1 = nn.Linear(16 * 25, 120)
        self.fc_2 = nn.Linear(120, 84)
        self.fc_3 = nn.Linear(84, 10)
        if args.with_batchnorm:
            self.block_conv_1 = nn.Sequential(
                self.conv_1,
                nn.BatchNorm2d(6),
                nn.ReLU(),
                nn.MaxPool2d(kernel_size=2, stride=2)
            )
            self.block_conv_2 = nn.Sequential(
                self.conv_2,
                nn.BatchNorm2d(16),
                nn.ReLU(),
                nn.MaxPool2d(kernel_size=2, stride=2)
            )
        else:
            self.block_conv_1 = nn.Sequential(
                self.conv_1,
                nn.ReLU(),
                nn.MaxPool2d(kernel_size=2, stride=2)
            )
            self.block_conv_2 = nn.Sequential(
                self.conv_2,
                nn.ReLU(),
                nn.MaxPool2d(kernel_size=2, stride=2)
            )
        if args.with_dropout:
            if args.with_batchnorm:
                self.block_fc_1 = nn.Sequential(
                    self.fc_1,
                    nn.BatchNorm1d(120),
                    nn.Dropout()
                )
                self.block_fc_2 = nn.Sequential(
                    self.fc_2,
                    nn.BatchNorm1d(84),
                    nn.Dropout()
                )
            else:
                self.block_fc_1 = nn.Sequential(
                    self.fc_1,
                    nn.Dropout()
                )
                self.block_fc_2 = nn.Sequential(
                    self.fc_2,
                    nn.Dropout()
                )
        else:
            self.block_fc_1 = self.fc_1
            self.block_fc_2 = self.fc_2
        self.softmax = nn.LogSoftmax()
        # Initialize parameters
        if args.with_init_weights:
            for m in self.modules():
                if isinstance(m, nn.Conv2d):
                    n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                    m.weight.data.normal_(0, math.sqrt(2. /n))
                    if m.bias is not None:
                        m.bias.data.zero_()
                elif isinstance(m, nn.Linear):
                    n = m.out_features
                    m.weight.data.normal_(0, math.sqrt(2. /n))
                    if m.bias is not None:
                        m.bias.data.zero_()
                elif isinstance(m, nn.BatchNorm1d) or isinstance(m, nn.BatchNorm2d):
                    m.weight.data.fill_(1)
                    m.bias.data.zero_()

    def forward(self, x):
        # TODO
        x = self.block_conv_1(x)
        x = self.block_conv_2(x)
        x = x.view(x.size(0), -1)
        x = self.block_fc_1(x)
        x = self.block_fc_2(x)
        x = self.fc_3(x)
        x = self.softmax(x)
        return x

# Filter extractor for filter visualization
class FeatureExtractor(nn.Module):
    def __init__(self, model, layer_names):
        super(FeatureExtractor, self).__init__()
        self._model = model
        self._layer_names = set(layer_names)
    def forward(self):
        filters = dict()
        # _modules is an OrderedDict, which replace iteritems() with items() in python3.* 
        for name, module in self._model._modules.items():
            if name in self._layer_names:
                filters[name] = module.weight.data.cpu()
        return filters

# Vesualize training results and trained filters
class VisualizedResult():
    def __init__(self, model):
        self._model = model
    def training_curve(self, epoches, train_loss_records, test_loss_records):
        fig = plt.figure()
        ax_train = fig.add_subplot(111)
        ax_test = fig.add_subplot(111)
        plt.axis([1, epoches, 0, math.ceil(max(train_loss_records + test_loss_records) * 1.2)])
        plt.xlabel('Epoches')
        plt.ylabel('Loss')
        plt.title('Training Curve')
        plt.plot(range(1, epoches + 1), train_loss_records, 'b-', label="train loss")
        plt.plot(range(1, epoches + 1), test_loss_records, 'r-', label="test loss")
        for xy in zip(range(1, epoches + 1), train_loss_records):
            ax_train.annotate('%.2f' % xy[1], xy=xy, textcoords='data')
        for xy in zip(range(1, epoches + 1), test_loss_records):
            ax_test.annotate('%.2f' % xy[1], xy=xy, textcoords='data')
        plt.legend(loc='upper right', borderaxespad=0.)
        plt.show()
    def accuracy_curve(self, epoches, accuracy_records):
        fig = plt.figure()
        ax = fig.add_subplot(111)
        plt.axis([1, epoches, 0, 100])
        plt.xlabel('Epoches')
        plt.ylabel('Accuracy')
        plt.title('Accuracy Curve')
        plt.plot(range(1, epoches + 1), accuracy_records, '-')
        for xy in zip(range(1, epoches + 1), accuracy_records):
            ax.annotate('%s%%' % xy[1], xy=xy, textcoords='data')
        plt.show()
    def conv_filter(self, layer_names):
        feature_extractor = FeatureExtractor(self._model, layer_names)
        filter_list = feature_extractor.forward()
        for name, filters in filter_list.items():
            print(type(filters), filters.size())
            # Normalize filters
            mean = torch.mean(filters)
            std = torch.std(filters)
            filters = (filters - mean) / std
            img = make_grid(filters, nrow=3)
            print(type(img), img.size())
            npimg = img.numpy()
            plt.imshow(np.transpose(npimg, (1,2,0)), interpolation='nearest')


model = Net()
if args.cuda:
    model.cuda()
    
# TODO: other optimizers
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum)

train_loss_records = list()
test_loss_records = list()
accuracy_records = list()

def train(epoch):
    model.train()
    train_loss = 0
    for batch_idx, (data, target) in enumerate(train_loader):
        if args.cuda:
            data, target = data.cuda(), target.cuda()
        data, target = Variable(data), Variable(target)
        optimizer.zero_grad()
        output = model(data)
        loss = F.nll_loss(output, target)   # is it true to use such a loss over cross-entropy loss? 
        loss.backward()
        optimizer.step()
        train_loss += loss.data[0]
        if batch_idx % args.log_interval == 0:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch, batch_idx * len(data), len(train_loader.dataset),
                100. * batch_idx / len(train_loader), loss.data[0]))
    # Average training loss for this epoch
    train_loss_records.append(train_loss / len(train_loader))

def test(epoch):
    model.eval()
    test_loss = 0
    correct = 0
    for data, target in test_loader:
        if args.cuda:
            data, target = data.cuda(), target.cuda()
        data, target = Variable(data, volatile=True), Variable(target)
        output = model(data)
        test_loss += F.nll_loss(output, target).data[0]
        pred = output.data.max(1)[1] # get the index of the max log-probability
        correct += pred.eq(target.data).cpu().sum()

    test_loss = test_loss
    test_loss /= len(test_loader) # loss function already averages over batch size
    accuracy = 100. * correct / len(test_loader.dataset)
    print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
        test_loss, correct, len(test_loader.dataset),
        accuracy))
    test_loss_records.append(test_loss)
    accuracy_records.append(accuracy)


for epoch in range(1, args.epochs + 1):
    train(epoch)
    test(epoch)

visual_result = VisualizedResult(model)
# Visualize training curve
visual_result.training_curve(args.epochs, train_loss_records, test_loss_records)
# Visualize test accuracy
visual_result.accuracy_curve(args.epochs, accuracy_records)
# Visualize trained filter on the 1st Conv layer
visual_result.conv_filter(['conv_1'])


    

    




Files already downloaded and verified
Files already downloaded and verified
Train Epoch: 1 [0/50000 (0%)]	Loss: 2.314914
Train Epoch: 1 [640/50000 (1%)]	Loss: 2.312093
Train Epoch: 1 [1280/50000 (3%)]	Loss: 2.327953
Train Epoch: 1 [1920/50000 (4%)]	Loss: 2.315229
Train Epoch: 1 [2560/50000 (5%)]	Loss: 2.316190
Train Epoch: 1 [3200/50000 (6%)]	Loss: 2.278881
Train Epoch: 1 [3840/50000 (8%)]	Loss: 2.323237
Train Epoch: 1 [4480/50000 (9%)]	Loss: 2.318270
Train Epoch: 1 [5120/50000 (10%)]	Loss: 2.296824
Train Epoch: 1 [5760/50000 (12%)]	Loss: 2.308896
Train Epoch: 1 [6400/50000 (13%)]	Loss: 2.298671
Train Epoch: 1 [7040/50000 (14%)]	Loss: 2.289702
Train Epoch: 1 [7680/50000 (15%)]	Loss: 2.290823
Train Epoch: 1 [8320/50000 (17%)]	Loss: 2.306070
Train Epoch: 1 [8960/50000 (18%)]	Loss: 2.292789
Train Epoch: 1 [9600/50000 (19%)]	Loss: 2.311813
Train Epoch: 1 [10240/50000 (20%)]	Loss: 2.295905
Train Epoch: 1 [10880/50000 (22%)]	Loss: 2.291469
Train Epoch: 1 [11520/50000 (23%)]	Loss: 2.270731
Train Epoch: 1 [12160/50000 (24%)]	Loss: 2.258504
Train Epoch: 1 [12800/50000 (26%)]	Loss: 2.296112
Train Epoch: 1 [13440/50000 (27%)]	Loss: 2.241914
Train Epoch: 1 [14080/50000 (28%)]	Loss: 2.238876
Train Epoch: 1 [14720/50000 (29%)]	Loss: 2.264428
Train Epoch: 1 [15360/50000 (31%)]	Loss: 2.170020
Train Epoch: 1 [16000/50000 (32%)]	Loss: 2.252756
Train Epoch: 1 [16640/50000 (33%)]	Loss: 2.201636
Train Epoch: 1 [17280/50000 (35%)]	Loss: 2.246272
Train Epoch: 1 [17920/50000 (36%)]	Loss: 2.172762
Train Epoch: 1 [18560/50000 (37%)]	Loss: 2.175262
Train Epoch: 1 [19200/50000 (38%)]	Loss: 2.117116
Train Epoch: 1 [19840/50000 (40%)]	Loss: 2.183937
Train Epoch: 1 [20480/50000 (41%)]	Loss: 2.122429
Train Epoch: 1 [21120/50000 (42%)]	Loss: 2.165319
Train Epoch: 1 [21760/50000 (43%)]	Loss: 2.059218
Train Epoch: 1 [22400/50000 (45%)]	Loss: 2.117751
Train Epoch: 1 [23040/50000 (46%)]	Loss: 2.124294
Train Epoch: 1 [23680/50000 (47%)]	Loss: 2.112008
Train Epoch: 1 [24320/50000 (49%)]	Loss: 2.024033
Train Epoch: 1 [24960/50000 (50%)]	Loss: 2.207282
Train Epoch: 1 [25600/50000 (51%)]	Loss: 2.048647
Train Epoch: 1 [26240/50000 (52%)]	Loss: 2.119376
Train Epoch: 1 [26880/50000 (54%)]	Loss: 2.037635
Train Epoch: 1 [27520/50000 (55%)]	Loss: 1.915851
Train Epoch: 1 [28160/50000 (56%)]	Loss: 1.890176
Train Epoch: 1 [28800/50000 (58%)]	Loss: 1.993855
Train Epoch: 1 [29440/50000 (59%)]	Loss: 2.004998
Train Epoch: 1 [30080/50000 (60%)]	Loss: 2.106575
Train Epoch: 1 [30720/50000 (61%)]	Loss: 2.023002
Train Epoch: 1 [31360/50000 (63%)]	Loss: 2.052347
Train Epoch: 1 [32000/50000 (64%)]	Loss: 2.079709
Train Epoch: 1 [32640/50000 (65%)]	Loss: 2.041441
Train Epoch: 1 [33280/50000 (66%)]	Loss: 2.148996
Train Epoch: 1 [33920/50000 (68%)]	Loss: 1.988281
Train Epoch: 1 [34560/50000 (69%)]	Loss: 1.877843
Train Epoch: 1 [35200/50000 (70%)]	Loss: 2.003248
Train Epoch: 1 [35840/50000 (72%)]	Loss: 1.947312
Train Epoch: 1 [36480/50000 (73%)]	Loss: 1.960166
Train Epoch: 1 [37120/50000 (74%)]	Loss: 1.967214
Train Epoch: 1 [37760/50000 (75%)]	Loss: 2.053677
Train Epoch: 1 [38400/50000 (77%)]	Loss: 1.845863
Train Epoch: 1 [39040/50000 (78%)]	Loss: 2.100141
Train Epoch: 1 [39680/50000 (79%)]	Loss: 2.055312
Train Epoch: 1 [40320/50000 (81%)]	Loss: 1.896468
Train Epoch: 1 [40960/50000 (82%)]	Loss: 2.086527
Train Epoch: 1 [41600/50000 (83%)]	Loss: 1.994070
Train Epoch: 1 [42240/50000 (84%)]	Loss: 1.953783
Train Epoch: 1 [42880/50000 (86%)]	Loss: 2.052311
Train Epoch: 1 [43520/50000 (87%)]	Loss: 1.964035
Train Epoch: 1 [44160/50000 (88%)]	Loss: 2.074132
Train Epoch: 1 [44800/50000 (90%)]	Loss: 2.052242
Train Epoch: 1 [45440/50000 (91%)]	Loss: 1.893154
Train Epoch: 1 [46080/50000 (92%)]	Loss: 2.049419
Train Epoch: 1 [46720/50000 (93%)]	Loss: 2.169688
Train Epoch: 1 [47360/50000 (95%)]	Loss: 2.021621
Train Epoch: 1 [48000/50000 (96%)]	Loss: 1.885986
Train Epoch: 1 [48640/50000 (97%)]	Loss: 1.832742
Train Epoch: 1 [49280/50000 (98%)]	Loss: 1.810431
Train Epoch: 1 [49920/50000 (100%)]	Loss: 1.971465

Test set: Average loss: 1.9090, Accuracy: 3150/10000 (32%)

Train Epoch: 2 [0/50000 (0%)]	Loss: 1.924470
Train Epoch: 2 [640/50000 (1%)]	Loss: 1.755524
Train Epoch: 2 [1280/50000 (3%)]	Loss: 1.999970
Train Epoch: 2 [1920/50000 (4%)]	Loss: 1.829738
Train Epoch: 2 [2560/50000 (5%)]	Loss: 1.902964
Train Epoch: 2 [3200/50000 (6%)]	Loss: 2.064417
Train Epoch: 2 [3840/50000 (8%)]	Loss: 1.878774
Train Epoch: 2 [4480/50000 (9%)]	Loss: 1.830971
Train Epoch: 2 [5120/50000 (10%)]	Loss: 1.939531
Train Epoch: 2 [5760/50000 (12%)]	Loss: 1.956573
Train Epoch: 2 [6400/50000 (13%)]	Loss: 1.821656
Train Epoch: 2 [7040/50000 (14%)]	Loss: 1.812273
Train Epoch: 2 [7680/50000 (15%)]	Loss: 1.879039
Train Epoch: 2 [8320/50000 (17%)]	Loss: 1.814384
Train Epoch: 2 [8960/50000 (18%)]	Loss: 2.017481
Train Epoch: 2 [9600/50000 (19%)]	Loss: 1.884206
Train Epoch: 2 [10240/50000 (20%)]	Loss: 1.933519
Train Epoch: 2 [10880/50000 (22%)]	Loss: 1.812258
Train Epoch: 2 [11520/50000 (23%)]	Loss: 1.818130
Train Epoch: 2 [12160/50000 (24%)]	Loss: 1.771231
Train Epoch: 2 [12800/50000 (26%)]	Loss: 2.050466
Train Epoch: 2 [13440/50000 (27%)]	Loss: 1.847193
Train Epoch: 2 [14080/50000 (28%)]	Loss: 1.858905
Train Epoch: 2 [14720/50000 (29%)]	Loss: 1.779424
Train Epoch: 2 [15360/50000 (31%)]	Loss: 1.684707
Train Epoch: 2 [16000/50000 (32%)]	Loss: 1.941453
Train Epoch: 2 [16640/50000 (33%)]	Loss: 1.763830
Train Epoch: 2 [17280/50000 (35%)]	Loss: 1.814450
Train Epoch: 2 [17920/50000 (36%)]	Loss: 1.816018
Train Epoch: 2 [18560/50000 (37%)]	Loss: 1.847187
Train Epoch: 2 [19200/50000 (38%)]	Loss: 1.765302
Train Epoch: 2 [19840/50000 (40%)]	Loss: 1.842975
Train Epoch: 2 [20480/50000 (41%)]	Loss: 1.898967
Train Epoch: 2 [21120/50000 (42%)]	Loss: 1.982300
Train Epoch: 2 [21760/50000 (43%)]	Loss: 1.890056
Train Epoch: 2 [22400/50000 (45%)]	Loss: 1.913120
Train Epoch: 2 [23040/50000 (46%)]	Loss: 1.713230
Train Epoch: 2 [23680/50000 (47%)]	Loss: 1.761465
Train Epoch: 2 [24320/50000 (49%)]	Loss: 1.826232
Train Epoch: 2 [24960/50000 (50%)]	Loss: 1.601376
Train Epoch: 2 [25600/50000 (51%)]	Loss: 1.711532
Train Epoch: 2 [26240/50000 (52%)]	Loss: 1.837631
Train Epoch: 2 [26880/50000 (54%)]	Loss: 1.774280
Train Epoch: 2 [27520/50000 (55%)]	Loss: 1.698848
Train Epoch: 2 [28160/50000 (56%)]	Loss: 1.806201
Train Epoch: 2 [28800/50000 (58%)]	Loss: 1.576951
Train Epoch: 2 [29440/50000 (59%)]	Loss: 1.713949
Train Epoch: 2 [30080/50000 (60%)]	Loss: 1.964969
Train Epoch: 2 [30720/50000 (61%)]	Loss: 1.619308
Train Epoch: 2 [31360/50000 (63%)]	Loss: 1.810123
Train Epoch: 2 [32000/50000 (64%)]	Loss: 1.729921
Train Epoch: 2 [32640/50000 (65%)]	Loss: 1.862205
Train Epoch: 2 [33280/50000 (66%)]	Loss: 1.864746
Train Epoch: 2 [33920/50000 (68%)]	Loss: 1.747928
Train Epoch: 2 [34560/50000 (69%)]	Loss: 1.736182
Train Epoch: 2 [35200/50000 (70%)]	Loss: 1.669650
Train Epoch: 2 [35840/50000 (72%)]	Loss: 1.578449
Train Epoch: 2 [36480/50000 (73%)]	Loss: 1.775551
Train Epoch: 2 [37120/50000 (74%)]	Loss: 1.887401
Train Epoch: 2 [37760/50000 (75%)]	Loss: 1.782823
Train Epoch: 2 [38400/50000 (77%)]	Loss: 1.830807
Train Epoch: 2 [39040/50000 (78%)]	Loss: 1.806183
Train Epoch: 2 [39680/50000 (79%)]	Loss: 1.768285
Train Epoch: 2 [40320/50000 (81%)]	Loss: 1.811192
Train Epoch: 2 [40960/50000 (82%)]	Loss: 1.717266
Train Epoch: 2 [41600/50000 (83%)]	Loss: 1.492092
Train Epoch: 2 [42240/50000 (84%)]	Loss: 1.674074
Train Epoch: 2 [42880/50000 (86%)]	Loss: 1.578968
Train Epoch: 2 [43520/50000 (87%)]	Loss: 1.801637
Train Epoch: 2 [44160/50000 (88%)]	Loss: 1.554340
Train Epoch: 2 [44800/50000 (90%)]	Loss: 1.748644
Train Epoch: 2 [45440/50000 (91%)]	Loss: 1.662013
Train Epoch: 2 [46080/50000 (92%)]	Loss: 1.692200
Train Epoch: 2 [46720/50000 (93%)]	Loss: 1.636018
Train Epoch: 2 [47360/50000 (95%)]	Loss: 1.645515
Train Epoch: 2 [48000/50000 (96%)]	Loss: 1.599086
Train Epoch: 2 [48640/50000 (97%)]	Loss: 1.782402
Train Epoch: 2 [49280/50000 (98%)]	Loss: 1.971292
Train Epoch: 2 [49920/50000 (100%)]	Loss: 1.739737

Test set: Average loss: 1.6622, Accuracy: 3980/10000 (40%)

Train Epoch: 3 [0/50000 (0%)]	Loss: 1.804525
Train Epoch: 3 [640/50000 (1%)]	Loss: 1.739283
Train Epoch: 3 [1280/50000 (3%)]	Loss: 1.485184
Train Epoch: 3 [1920/50000 (4%)]	Loss: 1.724087
Train Epoch: 3 [2560/50000 (5%)]	Loss: 1.496182
Train Epoch: 3 [3200/50000 (6%)]	Loss: 1.511319
Train Epoch: 3 [3840/50000 (8%)]	Loss: 1.631302
Train Epoch: 3 [4480/50000 (9%)]	Loss: 1.621991
Train Epoch: 3 [5120/50000 (10%)]	Loss: 1.712647
Train Epoch: 3 [5760/50000 (12%)]	Loss: 1.707007
Train Epoch: 3 [6400/50000 (13%)]	Loss: 1.680115
Train Epoch: 3 [7040/50000 (14%)]	Loss: 1.812293
Train Epoch: 3 [7680/50000 (15%)]	Loss: 1.588286
Train Epoch: 3 [8320/50000 (17%)]	Loss: 1.591816
Train Epoch: 3 [8960/50000 (18%)]	Loss: 1.531887
Train Epoch: 3 [9600/50000 (19%)]	Loss: 1.780153
Train Epoch: 3 [10240/50000 (20%)]	Loss: 1.795609
Train Epoch: 3 [10880/50000 (22%)]	Loss: 1.768564
Train Epoch: 3 [11520/50000 (23%)]	Loss: 1.664646
Train Epoch: 3 [12160/50000 (24%)]	Loss: 1.847169
Train Epoch: 3 [12800/50000 (26%)]	Loss: 1.707016
Train Epoch: 3 [13440/50000 (27%)]	Loss: 1.770795
Train Epoch: 3 [14080/50000 (28%)]	Loss: 1.604518
Train Epoch: 3 [14720/50000 (29%)]	Loss: 1.457168
Train Epoch: 3 [15360/50000 (31%)]	Loss: 1.605474
Train Epoch: 3 [16000/50000 (32%)]	Loss: 1.720064
Train Epoch: 3 [16640/50000 (33%)]	Loss: 1.503039
Train Epoch: 3 [17280/50000 (35%)]	Loss: 1.589462
Train Epoch: 3 [17920/50000 (36%)]	Loss: 1.627983
Train Epoch: 3 [18560/50000 (37%)]	Loss: 1.632547
Train Epoch: 3 [19200/50000 (38%)]	Loss: 1.646209
Train Epoch: 3 [19840/50000 (40%)]	Loss: 1.526574
Train Epoch: 3 [20480/50000 (41%)]	Loss: 1.570439
Train Epoch: 3 [21120/50000 (42%)]	Loss: 1.450016
Train Epoch: 3 [21760/50000 (43%)]	Loss: 1.437715
Train Epoch: 3 [22400/50000 (45%)]	Loss: 1.478158
Train Epoch: 3 [23040/50000 (46%)]	Loss: 1.677864
Train Epoch: 3 [23680/50000 (47%)]	Loss: 1.672217
Train Epoch: 3 [24320/50000 (49%)]	Loss: 1.505894
Train Epoch: 3 [24960/50000 (50%)]	Loss: 1.615253
Train Epoch: 3 [25600/50000 (51%)]	Loss: 1.623227
Train Epoch: 3 [26240/50000 (52%)]	Loss: 1.733641
Train Epoch: 3 [26880/50000 (54%)]	Loss: 1.840645
Train Epoch: 3 [27520/50000 (55%)]	Loss: 1.704075
Train Epoch: 3 [28160/50000 (56%)]	Loss: 1.523130
Train Epoch: 3 [28800/50000 (58%)]	Loss: 1.695285
Train Epoch: 3 [29440/50000 (59%)]	Loss: 1.716997
Train Epoch: 3 [30080/50000 (60%)]	Loss: 1.595757
Train Epoch: 3 [30720/50000 (61%)]	Loss: 1.892413
Train Epoch: 3 [31360/50000 (63%)]	Loss: 1.554862
Train Epoch: 3 [32000/50000 (64%)]	Loss: 1.477633
Train Epoch: 3 [32640/50000 (65%)]	Loss: 1.520973
Train Epoch: 3 [33280/50000 (66%)]	Loss: 1.500017
Train Epoch: 3 [33920/50000 (68%)]	Loss: 1.717526
Train Epoch: 3 [34560/50000 (69%)]	Loss: 1.451765
Train Epoch: 3 [35200/50000 (70%)]	Loss: 1.543468
Train Epoch: 3 [35840/50000 (72%)]	Loss: 1.522342
Train Epoch: 3 [36480/50000 (73%)]	Loss: 1.594443
Train Epoch: 3 [37120/50000 (74%)]	Loss: 1.461172
Train Epoch: 3 [37760/50000 (75%)]	Loss: 1.525971
Train Epoch: 3 [38400/50000 (77%)]	Loss: 1.522469
Train Epoch: 3 [39040/50000 (78%)]	Loss: 1.480726
Train Epoch: 3 [39680/50000 (79%)]	Loss: 1.643500
Train Epoch: 3 [40320/50000 (81%)]	Loss: 1.533240
Train Epoch: 3 [40960/50000 (82%)]	Loss: 1.661427
Train Epoch: 3 [41600/50000 (83%)]	Loss: 1.657665
Train Epoch: 3 [42240/50000 (84%)]	Loss: 1.603018
Train Epoch: 3 [42880/50000 (86%)]	Loss: 1.620037
Train Epoch: 3 [43520/50000 (87%)]	Loss: 1.784293
Train Epoch: 3 [44160/50000 (88%)]	Loss: 1.598406
Train Epoch: 3 [44800/50000 (90%)]	Loss: 1.471050
Train Epoch: 3 [45440/50000 (91%)]	Loss: 1.413618
Train Epoch: 3 [46080/50000 (92%)]	Loss: 1.466333
Train Epoch: 3 [46720/50000 (93%)]	Loss: 1.702491
Train Epoch: 3 [47360/50000 (95%)]	Loss: 1.660504
Train Epoch: 3 [48000/50000 (96%)]	Loss: 1.676152
Train Epoch: 3 [48640/50000 (97%)]	Loss: 1.585063
Train Epoch: 3 [49280/50000 (98%)]	Loss: 1.408650
Train Epoch: 3 [49920/50000 (100%)]	Loss: 1.390722

Test set: Average loss: 1.4818, Accuracy: 4612/10000 (46%)

Train Epoch: 4 [0/50000 (0%)]	Loss: 1.575344
Train Epoch: 4 [640/50000 (1%)]	Loss: 1.618953
Train Epoch: 4 [1280/50000 (3%)]	Loss: 1.505366
Train Epoch: 4 [1920/50000 (4%)]	Loss: 1.584057
Train Epoch: 4 [2560/50000 (5%)]	Loss: 1.508609
Train Epoch: 4 [3200/50000 (6%)]	Loss: 1.576558
Train Epoch: 4 [3840/50000 (8%)]	Loss: 1.687776
Train Epoch: 4 [4480/50000 (9%)]	Loss: 1.377416
Train Epoch: 4 [5120/50000 (10%)]	Loss: 1.615371
Train Epoch: 4 [5760/50000 (12%)]	Loss: 1.440220
Train Epoch: 4 [6400/50000 (13%)]	Loss: 1.336109
Train Epoch: 4 [7040/50000 (14%)]	Loss: 1.715464
Train Epoch: 4 [7680/50000 (15%)]	Loss: 1.848652
Train Epoch: 4 [8320/50000 (17%)]	Loss: 1.646190
Train Epoch: 4 [8960/50000 (18%)]	Loss: 1.673162
Train Epoch: 4 [9600/50000 (19%)]	Loss: 1.719679
Train Epoch: 4 [10240/50000 (20%)]	Loss: 1.566285
Train Epoch: 4 [10880/50000 (22%)]	Loss: 1.432086
Train Epoch: 4 [11520/50000 (23%)]	Loss: 1.366365
Train Epoch: 4 [12160/50000 (24%)]	Loss: 1.589595
Train Epoch: 4 [12800/50000 (26%)]	Loss: 1.780438
Train Epoch: 4 [13440/50000 (27%)]	Loss: 1.800666
Train Epoch: 4 [14080/50000 (28%)]	Loss: 1.572224
Train Epoch: 4 [14720/50000 (29%)]	Loss: 1.541359
Train Epoch: 4 [15360/50000 (31%)]	Loss: 1.389207
Train Epoch: 4 [16000/50000 (32%)]	Loss: 1.614945
Train Epoch: 4 [16640/50000 (33%)]	Loss: 1.599297
Train Epoch: 4 [17280/50000 (35%)]	Loss: 1.291868
Train Epoch: 4 [17920/50000 (36%)]	Loss: 1.574865
Train Epoch: 4 [18560/50000 (37%)]	Loss: 1.470083
Train Epoch: 4 [19200/50000 (38%)]	Loss: 1.438248
Train Epoch: 4 [19840/50000 (40%)]	Loss: 1.328343
Train Epoch: 4 [20480/50000 (41%)]	Loss: 1.678999
Train Epoch: 4 [21120/50000 (42%)]	Loss: 1.540441
Train Epoch: 4 [21760/50000 (43%)]	Loss: 1.475637
Train Epoch: 4 [22400/50000 (45%)]	Loss: 1.451540
Train Epoch: 4 [23040/50000 (46%)]	Loss: 1.638493
Train Epoch: 4 [23680/50000 (47%)]	Loss: 1.470540
Train Epoch: 4 [24320/50000 (49%)]	Loss: 1.563358
Train Epoch: 4 [24960/50000 (50%)]	Loss: 1.658902
Train Epoch: 4 [25600/50000 (51%)]	Loss: 1.821548
Train Epoch: 4 [26240/50000 (52%)]	Loss: 1.349863
Train Epoch: 4 [26880/50000 (54%)]	Loss: 1.654645
Train Epoch: 4 [27520/50000 (55%)]	Loss: 1.356433
Train Epoch: 4 [28160/50000 (56%)]	Loss: 1.582660
Train Epoch: 4 [28800/50000 (58%)]	Loss: 1.479756
Train Epoch: 4 [29440/50000 (59%)]	Loss: 1.473517
Train Epoch: 4 [30080/50000 (60%)]	Loss: 1.782269
Train Epoch: 4 [30720/50000 (61%)]	Loss: 1.577056
Train Epoch: 4 [31360/50000 (63%)]	Loss: 1.446032
Train Epoch: 4 [32000/50000 (64%)]	Loss: 1.262300
Train Epoch: 4 [32640/50000 (65%)]	Loss: 1.355404
Train Epoch: 4 [33280/50000 (66%)]	Loss: 1.484035
Train Epoch: 4 [33920/50000 (68%)]	Loss: 2.044287
Train Epoch: 4 [34560/50000 (69%)]	Loss: 1.363516
Train Epoch: 4 [35200/50000 (70%)]	Loss: 1.679533
Train Epoch: 4 [35840/50000 (72%)]	Loss: 1.348007
Train Epoch: 4 [36480/50000 (73%)]	Loss: 1.555640
Train Epoch: 4 [37120/50000 (74%)]	Loss: 1.474148
Train Epoch: 4 [37760/50000 (75%)]	Loss: 1.638026
Train Epoch: 4 [38400/50000 (77%)]	Loss: 1.435629
Train Epoch: 4 [39040/50000 (78%)]	Loss: 1.627999
Train Epoch: 4 [39680/50000 (79%)]	Loss: 1.531973
Train Epoch: 4 [40320/50000 (81%)]	Loss: 1.591643
Train Epoch: 4 [40960/50000 (82%)]	Loss: 1.346828
Train Epoch: 4 [41600/50000 (83%)]	Loss: 1.468559
Train Epoch: 4 [42240/50000 (84%)]	Loss: 1.507472
Train Epoch: 4 [42880/50000 (86%)]	Loss: 1.508096
Train Epoch: 4 [43520/50000 (87%)]	Loss: 1.305229
Train Epoch: 4 [44160/50000 (88%)]	Loss: 1.458196
Train Epoch: 4 [44800/50000 (90%)]	Loss: 1.536973
Train Epoch: 4 [45440/50000 (91%)]	Loss: 1.397416
Train Epoch: 4 [46080/50000 (92%)]	Loss: 1.366526
Train Epoch: 4 [46720/50000 (93%)]	Loss: 1.482116
Train Epoch: 4 [47360/50000 (95%)]	Loss: 1.458275
Train Epoch: 4 [48000/50000 (96%)]	Loss: 1.687756
Train Epoch: 4 [48640/50000 (97%)]	Loss: 1.580034
Train Epoch: 4 [49280/50000 (98%)]	Loss: 1.461453
Train Epoch: 4 [49920/50000 (100%)]	Loss: 1.327380

Test set: Average loss: 1.4175, Accuracy: 4866/10000 (49%)

Train Epoch: 5 [0/50000 (0%)]	Loss: 1.612033
Train Epoch: 5 [640/50000 (1%)]	Loss: 1.513487
Train Epoch: 5 [1280/50000 (3%)]	Loss: 1.441390
Train Epoch: 5 [1920/50000 (4%)]	Loss: 1.443432
Train Epoch: 5 [2560/50000 (5%)]	Loss: 1.645332
Train Epoch: 5 [3200/50000 (6%)]	Loss: 1.382015
Train Epoch: 5 [3840/50000 (8%)]	Loss: 1.439315
Train Epoch: 5 [4480/50000 (9%)]	Loss: 1.424716
Train Epoch: 5 [5120/50000 (10%)]	Loss: 1.549846
Train Epoch: 5 [5760/50000 (12%)]	Loss: 1.588925
Train Epoch: 5 [6400/50000 (13%)]	Loss: 1.483529
Train Epoch: 5 [7040/50000 (14%)]	Loss: 1.577043
Train Epoch: 5 [7680/50000 (15%)]	Loss: 1.471622
Train Epoch: 5 [8320/50000 (17%)]	Loss: 1.328868
Train Epoch: 5 [8960/50000 (18%)]	Loss: 1.542611
Train Epoch: 5 [9600/50000 (19%)]	Loss: 1.549706
Train Epoch: 5 [10240/50000 (20%)]	Loss: 1.473379
Train Epoch: 5 [10880/50000 (22%)]	Loss: 1.303094
Train Epoch: 5 [11520/50000 (23%)]	Loss: 1.349454
Train Epoch: 5 [12160/50000 (24%)]	Loss: 1.395939
Train Epoch: 5 [12800/50000 (26%)]	Loss: 1.495095
Train Epoch: 5 [13440/50000 (27%)]	Loss: 1.397770
Train Epoch: 5 [14080/50000 (28%)]	Loss: 1.788903
Train Epoch: 5 [14720/50000 (29%)]	Loss: 1.521732
Train Epoch: 5 [15360/50000 (31%)]	Loss: 1.319030
Train Epoch: 5 [16000/50000 (32%)]	Loss: 1.494525
Train Epoch: 5 [16640/50000 (33%)]	Loss: 1.377524
Train Epoch: 5 [17280/50000 (35%)]	Loss: 1.461785
Train Epoch: 5 [17920/50000 (36%)]	Loss: 1.685467
Train Epoch: 5 [18560/50000 (37%)]	Loss: 1.345597
Train Epoch: 5 [19200/50000 (38%)]	Loss: 1.450800
Train Epoch: 5 [19840/50000 (40%)]	Loss: 1.549483
Train Epoch: 5 [20480/50000 (41%)]	Loss: 1.667299
Train Epoch: 5 [21120/50000 (42%)]	Loss: 1.557594
Train Epoch: 5 [21760/50000 (43%)]	Loss: 1.421772
Train Epoch: 5 [22400/50000 (45%)]	Loss: 1.554736
Train Epoch: 5 [23040/50000 (46%)]	Loss: 1.719132
Train Epoch: 5 [23680/50000 (47%)]	Loss: 1.732830
Train Epoch: 5 [24320/50000 (49%)]	Loss: 1.582315
Train Epoch: 5 [24960/50000 (50%)]	Loss: 1.795014
Train Epoch: 5 [25600/50000 (51%)]	Loss: 1.575708
Train Epoch: 5 [26240/50000 (52%)]	Loss: 1.382079
Train Epoch: 5 [26880/50000 (54%)]	Loss: 1.520158
Train Epoch: 5 [27520/50000 (55%)]	Loss: 1.408926
Train Epoch: 5 [28160/50000 (56%)]	Loss: 1.904300
Train Epoch: 5 [28800/50000 (58%)]	Loss: 1.499269
Train Epoch: 5 [29440/50000 (59%)]	Loss: 1.590399
Train Epoch: 5 [30080/50000 (60%)]	Loss: 1.474309
Train Epoch: 5 [30720/50000 (61%)]	Loss: 1.782675
Train Epoch: 5 [31360/50000 (63%)]	Loss: 1.532741
Train Epoch: 5 [32000/50000 (64%)]	Loss: 1.568962
Train Epoch: 5 [32640/50000 (65%)]	Loss: 1.635521
Train Epoch: 5 [33280/50000 (66%)]	Loss: 1.508255
Train Epoch: 5 [33920/50000 (68%)]	Loss: 1.451749
Train Epoch: 5 [34560/50000 (69%)]	Loss: 1.423813
Train Epoch: 5 [35200/50000 (70%)]	Loss: 1.532840
Train Epoch: 5 [35840/50000 (72%)]	Loss: 1.481937
Train Epoch: 5 [36480/50000 (73%)]	Loss: 1.515842
Train Epoch: 5 [37120/50000 (74%)]	Loss: 1.479736
Train Epoch: 5 [37760/50000 (75%)]	Loss: 1.430653
Train Epoch: 5 [38400/50000 (77%)]	Loss: 1.596355
Train Epoch: 5 [39040/50000 (78%)]	Loss: 1.441101
Train Epoch: 5 [39680/50000 (79%)]	Loss: 1.524530
Train Epoch: 5 [40320/50000 (81%)]	Loss: 1.451763
Train Epoch: 5 [40960/50000 (82%)]	Loss: 1.273585
Train Epoch: 5 [41600/50000 (83%)]	Loss: 1.344944
Train Epoch: 5 [42240/50000 (84%)]	Loss: 1.370062
Train Epoch: 5 [42880/50000 (86%)]	Loss: 1.414218
Train Epoch: 5 [43520/50000 (87%)]	Loss: 1.429802
Train Epoch: 5 [44160/50000 (88%)]	Loss: 1.367657
Train Epoch: 5 [44800/50000 (90%)]	Loss: 1.422522
Train Epoch: 5 [45440/50000 (91%)]	Loss: 1.476916
Train Epoch: 5 [46080/50000 (92%)]	Loss: 1.646952
Train Epoch: 5 [46720/50000 (93%)]	Loss: 1.366696
Train Epoch: 5 [47360/50000 (95%)]	Loss: 1.640832
Train Epoch: 5 [48000/50000 (96%)]	Loss: 1.412613
Train Epoch: 5 [48640/50000 (97%)]	Loss: 1.363321
Train Epoch: 5 [49280/50000 (98%)]	Loss: 1.494180
Train Epoch: 5 [49920/50000 (100%)]	Loss: 1.370016

Test set: Average loss: 1.3690, Accuracy: 5111/10000 (51%)

Train Epoch: 6 [0/50000 (0%)]	Loss: 1.463112
Train Epoch: 6 [640/50000 (1%)]	Loss: 1.480292
Train Epoch: 6 [1280/50000 (3%)]	Loss: 1.239684
Train Epoch: 6 [1920/50000 (4%)]	Loss: 1.194515
Train Epoch: 6 [2560/50000 (5%)]	Loss: 1.101930
Train Epoch: 6 [3200/50000 (6%)]	Loss: 1.236044
Train Epoch: 6 [3840/50000 (8%)]	Loss: 1.595083
Train Epoch: 6 [4480/50000 (9%)]	Loss: 1.521584
Train Epoch: 6 [5120/50000 (10%)]	Loss: 1.270989
Train Epoch: 6 [5760/50000 (12%)]	Loss: 1.307868
Train Epoch: 6 [6400/50000 (13%)]	Loss: 1.377389
Train Epoch: 6 [7040/50000 (14%)]	Loss: 1.595827
Train Epoch: 6 [7680/50000 (15%)]	Loss: 1.458380
Train Epoch: 6 [8320/50000 (17%)]	Loss: 1.195811
Train Epoch: 6 [8960/50000 (18%)]	Loss: 1.260861
Train Epoch: 6 [9600/50000 (19%)]	Loss: 1.695331
Train Epoch: 6 [10240/50000 (20%)]	Loss: 1.339978
Train Epoch: 6 [10880/50000 (22%)]	Loss: 1.528480
Train Epoch: 6 [11520/50000 (23%)]	Loss: 1.342697
Train Epoch: 6 [12160/50000 (24%)]	Loss: 1.463821
Train Epoch: 6 [12800/50000 (26%)]	Loss: 1.143224
Train Epoch: 6 [13440/50000 (27%)]	Loss: 1.282176
Train Epoch: 6 [14080/50000 (28%)]	Loss: 1.700625
Train Epoch: 6 [14720/50000 (29%)]	Loss: 1.515142
Train Epoch: 6 [15360/50000 (31%)]	Loss: 1.397024
Train Epoch: 6 [16000/50000 (32%)]	Loss: 1.773090
Train Epoch: 6 [16640/50000 (33%)]	Loss: 1.459502
Train Epoch: 6 [17280/50000 (35%)]	Loss: 1.248619
Train Epoch: 6 [17920/50000 (36%)]	Loss: 1.443731
Train Epoch: 6 [18560/50000 (37%)]	Loss: 1.705250
Train Epoch: 6 [19200/50000 (38%)]	Loss: 1.458207
Train Epoch: 6 [19840/50000 (40%)]	Loss: 1.464255
Train Epoch: 6 [20480/50000 (41%)]	Loss: 1.340123
Train Epoch: 6 [21120/50000 (42%)]	Loss: 1.334148
Train Epoch: 6 [21760/50000 (43%)]	Loss: 1.489016
Train Epoch: 6 [22400/50000 (45%)]	Loss: 1.436184
Train Epoch: 6 [23040/50000 (46%)]	Loss: 1.465144
Train Epoch: 6 [23680/50000 (47%)]	Loss: 1.527018
Train Epoch: 6 [24320/50000 (49%)]	Loss: 1.194350
Train Epoch: 6 [24960/50000 (50%)]	Loss: 1.417318
Train Epoch: 6 [25600/50000 (51%)]	Loss: 1.308370
Train Epoch: 6 [26240/50000 (52%)]	Loss: 1.239067
Train Epoch: 6 [26880/50000 (54%)]	Loss: 1.509219
Train Epoch: 6 [27520/50000 (55%)]	Loss: 1.243687
Train Epoch: 6 [28160/50000 (56%)]	Loss: 1.256603
Train Epoch: 6 [28800/50000 (58%)]	Loss: 1.385307
Train Epoch: 6 [29440/50000 (59%)]	Loss: 1.565456
Train Epoch: 6 [30080/50000 (60%)]	Loss: 1.263587
Train Epoch: 6 [30720/50000 (61%)]	Loss: 1.444685
Train Epoch: 6 [31360/50000 (63%)]	Loss: 1.502474
Train Epoch: 6 [32000/50000 (64%)]	Loss: 1.436623
Train Epoch: 6 [32640/50000 (65%)]	Loss: 1.442744
Train Epoch: 6 [33280/50000 (66%)]	Loss: 1.329000
Train Epoch: 6 [33920/50000 (68%)]	Loss: 1.392585
Train Epoch: 6 [34560/50000 (69%)]	Loss: 1.439474
Train Epoch: 6 [35200/50000 (70%)]	Loss: 1.459645
Train Epoch: 6 [35840/50000 (72%)]	Loss: 1.286027
Train Epoch: 6 [36480/50000 (73%)]	Loss: 1.431268
Train Epoch: 6 [37120/50000 (74%)]	Loss: 1.248756
Train Epoch: 6 [37760/50000 (75%)]	Loss: 1.570786
Train Epoch: 6 [38400/50000 (77%)]	Loss: 1.529080
Train Epoch: 6 [39040/50000 (78%)]	Loss: 1.447320
Train Epoch: 6 [39680/50000 (79%)]	Loss: 1.287271
Train Epoch: 6 [40320/50000 (81%)]	Loss: 1.485086
Train Epoch: 6 [40960/50000 (82%)]	Loss: 1.376700
Train Epoch: 6 [41600/50000 (83%)]	Loss: 1.550211
Train Epoch: 6 [42240/50000 (84%)]	Loss: 1.567892
Train Epoch: 6 [42880/50000 (86%)]	Loss: 1.030642
Train Epoch: 6 [43520/50000 (87%)]	Loss: 1.114906
Train Epoch: 6 [44160/50000 (88%)]	Loss: 1.573243
Train Epoch: 6 [44800/50000 (90%)]	Loss: 1.397993
Train Epoch: 6 [45440/50000 (91%)]	Loss: 1.421462
Train Epoch: 6 [46080/50000 (92%)]	Loss: 1.557701
Train Epoch: 6 [46720/50000 (93%)]	Loss: 1.213763
Train Epoch: 6 [47360/50000 (95%)]	Loss: 1.506848
Train Epoch: 6 [48000/50000 (96%)]	Loss: 1.379675
Train Epoch: 6 [48640/50000 (97%)]	Loss: 1.448627
Train Epoch: 6 [49280/50000 (98%)]	Loss: 1.423651
Train Epoch: 6 [49920/50000 (100%)]	Loss: 1.251777

Test set: Average loss: 1.3546, Accuracy: 5108/10000 (51%)

Train Epoch: 7 [0/50000 (0%)]	Loss: 1.334202
Train Epoch: 7 [640/50000 (1%)]	Loss: 1.466708
Train Epoch: 7 [1280/50000 (3%)]	Loss: 1.464688
Train Epoch: 7 [1920/50000 (4%)]	Loss: 1.439879
Train Epoch: 7 [2560/50000 (5%)]	Loss: 1.446425
Train Epoch: 7 [3200/50000 (6%)]	Loss: 1.416817
Train Epoch: 7 [3840/50000 (8%)]	Loss: 1.280611
Train Epoch: 7 [4480/50000 (9%)]	Loss: 1.225084
Train Epoch: 7 [5120/50000 (10%)]	Loss: 1.626839
Train Epoch: 7 [5760/50000 (12%)]	Loss: 1.385172
Train Epoch: 7 [6400/50000 (13%)]	Loss: 1.182515
Train Epoch: 7 [7040/50000 (14%)]	Loss: 1.402651
Train Epoch: 7 [7680/50000 (15%)]	Loss: 1.284607
Train Epoch: 7 [8320/50000 (17%)]	Loss: 1.206381
Train Epoch: 7 [8960/50000 (18%)]	Loss: 1.414425
Train Epoch: 7 [9600/50000 (19%)]	Loss: 1.488400
Train Epoch: 7 [10240/50000 (20%)]	Loss: 1.293663
Train Epoch: 7 [10880/50000 (22%)]	Loss: 1.509829
Train Epoch: 7 [11520/50000 (23%)]	Loss: 1.406882
Train Epoch: 7 [12160/50000 (24%)]	Loss: 1.485079
Train Epoch: 7 [12800/50000 (26%)]	Loss: 1.235125
Train Epoch: 7 [13440/50000 (27%)]	Loss: 1.452349
Train Epoch: 7 [14080/50000 (28%)]	Loss: 1.424652
Train Epoch: 7 [14720/50000 (29%)]	Loss: 1.262557
Train Epoch: 7 [15360/50000 (31%)]	Loss: 1.291916
Train Epoch: 7 [16000/50000 (32%)]	Loss: 1.158328
Train Epoch: 7 [16640/50000 (33%)]	Loss: 1.308016
Train Epoch: 7 [17280/50000 (35%)]	Loss: 1.605865
Train Epoch: 7 [17920/50000 (36%)]	Loss: 1.410684
Train Epoch: 7 [18560/50000 (37%)]	Loss: 1.412761
Train Epoch: 7 [19200/50000 (38%)]	Loss: 1.230019
Train Epoch: 7 [19840/50000 (40%)]	Loss: 1.320902
Train Epoch: 7 [20480/50000 (41%)]	Loss: 1.452027
Train Epoch: 7 [21120/50000 (42%)]	Loss: 1.510687
Train Epoch: 7 [21760/50000 (43%)]	Loss: 1.442746
Train Epoch: 7 [22400/50000 (45%)]	Loss: 1.250061
Train Epoch: 7 [23040/50000 (46%)]	Loss: 1.465221
Train Epoch: 7 [23680/50000 (47%)]	Loss: 1.297589
Train Epoch: 7 [24320/50000 (49%)]	Loss: 1.405134
Train Epoch: 7 [24960/50000 (50%)]	Loss: 1.344112
Train Epoch: 7 [25600/50000 (51%)]	Loss: 1.446191
Train Epoch: 7 [26240/50000 (52%)]	Loss: 1.398436
Train Epoch: 7 [26880/50000 (54%)]	Loss: 1.414317
Train Epoch: 7 [27520/50000 (55%)]	Loss: 1.484436
Train Epoch: 7 [28160/50000 (56%)]	Loss: 1.298986
Train Epoch: 7 [28800/50000 (58%)]	Loss: 1.427266
Train Epoch: 7 [29440/50000 (59%)]	Loss: 1.495615
Train Epoch: 7 [30080/50000 (60%)]	Loss: 1.105098
Train Epoch: 7 [30720/50000 (61%)]	Loss: 1.196679
Train Epoch: 7 [31360/50000 (63%)]	Loss: 1.336113
Train Epoch: 7 [32000/50000 (64%)]	Loss: 1.300186
Train Epoch: 7 [32640/50000 (65%)]	Loss: 1.379596
Train Epoch: 7 [33280/50000 (66%)]	Loss: 1.211647
Train Epoch: 7 [33920/50000 (68%)]	Loss: 1.778819
Train Epoch: 7 [34560/50000 (69%)]	Loss: 1.326541
Train Epoch: 7 [35200/50000 (70%)]	Loss: 1.382926
Train Epoch: 7 [35840/50000 (72%)]	Loss: 1.380758
Train Epoch: 7 [36480/50000 (73%)]	Loss: 1.474262
Train Epoch: 7 [37120/50000 (74%)]	Loss: 1.321385
Train Epoch: 7 [37760/50000 (75%)]	Loss: 1.291377
Train Epoch: 7 [38400/50000 (77%)]	Loss: 1.435886
Train Epoch: 7 [39040/50000 (78%)]	Loss: 1.545503
Train Epoch: 7 [39680/50000 (79%)]	Loss: 1.324522
Train Epoch: 7 [40320/50000 (81%)]	Loss: 1.414363
Train Epoch: 7 [40960/50000 (82%)]	Loss: 1.519184
Train Epoch: 7 [41600/50000 (83%)]	Loss: 1.566450
Train Epoch: 7 [42240/50000 (84%)]	Loss: 1.236491
Train Epoch: 7 [42880/50000 (86%)]	Loss: 1.404919
Train Epoch: 7 [43520/50000 (87%)]	Loss: 1.343353
Train Epoch: 7 [44160/50000 (88%)]	Loss: 0.968436
Train Epoch: 7 [44800/50000 (90%)]	Loss: 1.337322
Train Epoch: 7 [45440/50000 (91%)]	Loss: 1.327460
Train Epoch: 7 [46080/50000 (92%)]	Loss: 1.486084
Train Epoch: 7 [46720/50000 (93%)]	Loss: 1.213323
Train Epoch: 7 [47360/50000 (95%)]	Loss: 1.411419
Train Epoch: 7 [48000/50000 (96%)]	Loss: 1.667905
Train Epoch: 7 [48640/50000 (97%)]	Loss: 1.274745
Train Epoch: 7 [49280/50000 (98%)]	Loss: 1.384033
Train Epoch: 7 [49920/50000 (100%)]	Loss: 1.579880

Test set: Average loss: 1.2959, Accuracy: 5282/10000 (53%)

Train Epoch: 8 [0/50000 (0%)]	Loss: 1.248148
Train Epoch: 8 [640/50000 (1%)]	Loss: 1.696067
Train Epoch: 8 [1280/50000 (3%)]	Loss: 1.283652
Train Epoch: 8 [1920/50000 (4%)]	Loss: 1.516293
Train Epoch: 8 [2560/50000 (5%)]	Loss: 1.285203
Train Epoch: 8 [3200/50000 (6%)]	Loss: 1.337204
Train Epoch: 8 [3840/50000 (8%)]	Loss: 1.293562
Train Epoch: 8 [4480/50000 (9%)]	Loss: 1.526026
Train Epoch: 8 [5120/50000 (10%)]	Loss: 1.302388
Train Epoch: 8 [5760/50000 (12%)]	Loss: 1.550538
Train Epoch: 8 [6400/50000 (13%)]	Loss: 1.440758
Train Epoch: 8 [7040/50000 (14%)]	Loss: 1.427070
Train Epoch: 8 [7680/50000 (15%)]	Loss: 1.567646
Train Epoch: 8 [8320/50000 (17%)]	Loss: 1.387123
Train Epoch: 8 [8960/50000 (18%)]	Loss: 1.240511
Train Epoch: 8 [9600/50000 (19%)]	Loss: 1.306707
Train Epoch: 8 [10240/50000 (20%)]	Loss: 1.428388
Train Epoch: 8 [10880/50000 (22%)]	Loss: 1.088421
Train Epoch: 8 [11520/50000 (23%)]	Loss: 1.296006
Train Epoch: 8 [12160/50000 (24%)]	Loss: 1.451247
Train Epoch: 8 [12800/50000 (26%)]	Loss: 1.602692
Train Epoch: 8 [13440/50000 (27%)]	Loss: 1.387675
Train Epoch: 8 [14080/50000 (28%)]	Loss: 1.152244
Train Epoch: 8 [14720/50000 (29%)]	Loss: 1.308633
Train Epoch: 8 [15360/50000 (31%)]	Loss: 1.203202
Train Epoch: 8 [16000/50000 (32%)]	Loss: 1.305419
Train Epoch: 8 [16640/50000 (33%)]	Loss: 1.251539
Train Epoch: 8 [17280/50000 (35%)]	Loss: 1.217179
Train Epoch: 8 [17920/50000 (36%)]	Loss: 1.465722
Train Epoch: 8 [18560/50000 (37%)]	Loss: 1.359923
Train Epoch: 8 [19200/50000 (38%)]	Loss: 1.313286
Train Epoch: 8 [19840/50000 (40%)]	Loss: 1.418631
Train Epoch: 8 [20480/50000 (41%)]	Loss: 1.182781
Train Epoch: 8 [21120/50000 (42%)]	Loss: 1.343991
Train Epoch: 8 [21760/50000 (43%)]	Loss: 1.338862
Train Epoch: 8 [22400/50000 (45%)]	Loss: 1.373174
Train Epoch: 8 [23040/50000 (46%)]	Loss: 1.411017
Train Epoch: 8 [23680/50000 (47%)]	Loss: 1.535786
Train Epoch: 8 [24320/50000 (49%)]	Loss: 1.472528
Train Epoch: 8 [24960/50000 (50%)]	Loss: 1.205143
Train Epoch: 8 [25600/50000 (51%)]	Loss: 1.332424
Train Epoch: 8 [26240/50000 (52%)]	Loss: 1.506520
Train Epoch: 8 [26880/50000 (54%)]	Loss: 1.430494
Train Epoch: 8 [27520/50000 (55%)]	Loss: 1.317916
Train Epoch: 8 [28160/50000 (56%)]	Loss: 1.320624
Train Epoch: 8 [28800/50000 (58%)]	Loss: 1.389058
Train Epoch: 8 [29440/50000 (59%)]	Loss: 1.247195
Train Epoch: 8 [30080/50000 (60%)]	Loss: 1.351932
Train Epoch: 8 [30720/50000 (61%)]	Loss: 1.223305
Train Epoch: 8 [31360/50000 (63%)]	Loss: 1.309820
Train Epoch: 8 [32000/50000 (64%)]	Loss: 1.388525
Train Epoch: 8 [32640/50000 (65%)]	Loss: 1.356440
Train Epoch: 8 [33280/50000 (66%)]	Loss: 1.510242
Train Epoch: 8 [33920/50000 (68%)]	Loss: 1.262468
Train Epoch: 8 [34560/50000 (69%)]	Loss: 1.295101
Train Epoch: 8 [35200/50000 (70%)]	Loss: 1.352738
Train Epoch: 8 [35840/50000 (72%)]	Loss: 1.682488
Train Epoch: 8 [36480/50000 (73%)]	Loss: 1.066838
Train Epoch: 8 [37120/50000 (74%)]	Loss: 1.476786
Train Epoch: 8 [37760/50000 (75%)]	Loss: 1.448484
Train Epoch: 8 [38400/50000 (77%)]	Loss: 1.242267
Train Epoch: 8 [39040/50000 (78%)]	Loss: 1.422989
Train Epoch: 8 [39680/50000 (79%)]	Loss: 1.321040
Train Epoch: 8 [40320/50000 (81%)]	Loss: 1.246916
Train Epoch: 8 [40960/50000 (82%)]	Loss: 1.394296
Train Epoch: 8 [41600/50000 (83%)]	Loss: 1.419152
Train Epoch: 8 [42240/50000 (84%)]	Loss: 1.349236
Train Epoch: 8 [42880/50000 (86%)]	Loss: 1.410184
Train Epoch: 8 [43520/50000 (87%)]	Loss: 1.286864
Train Epoch: 8 [44160/50000 (88%)]	Loss: 1.171877
Train Epoch: 8 [44800/50000 (90%)]	Loss: 1.402601
Train Epoch: 8 [45440/50000 (91%)]	Loss: 1.303904
Train Epoch: 8 [46080/50000 (92%)]	Loss: 1.364382
Train Epoch: 8 [46720/50000 (93%)]	Loss: 1.351035
Train Epoch: 8 [47360/50000 (95%)]	Loss: 1.366438
Train Epoch: 8 [48000/50000 (96%)]	Loss: 1.519948
Train Epoch: 8 [48640/50000 (97%)]	Loss: 1.333649
Train Epoch: 8 [49280/50000 (98%)]	Loss: 1.322707
Train Epoch: 8 [49920/50000 (100%)]	Loss: 1.483258

Test set: Average loss: 1.3365, Accuracy: 5311/10000 (53%)

Train Epoch: 9 [0/50000 (0%)]	Loss: 1.202828
Train Epoch: 9 [640/50000 (1%)]	Loss: 1.180442
Train Epoch: 9 [1280/50000 (3%)]	Loss: 1.348120
Train Epoch: 9 [1920/50000 (4%)]	Loss: 1.290122
Train Epoch: 9 [2560/50000 (5%)]	Loss: 1.134252
Train Epoch: 9 [3200/50000 (6%)]	Loss: 1.286594
Train Epoch: 9 [3840/50000 (8%)]	Loss: 1.379474
Train Epoch: 9 [4480/50000 (9%)]	Loss: 1.388416
Train Epoch: 9 [5120/50000 (10%)]	Loss: 1.444197
Train Epoch: 9 [5760/50000 (12%)]	Loss: 1.062110
Train Epoch: 9 [6400/50000 (13%)]	Loss: 1.353053
Train Epoch: 9 [7040/50000 (14%)]	Loss: 1.155571
Train Epoch: 9 [7680/50000 (15%)]	Loss: 1.373888
Train Epoch: 9 [8320/50000 (17%)]	Loss: 1.263095
Train Epoch: 9 [8960/50000 (18%)]	Loss: 1.280633
Train Epoch: 9 [9600/50000 (19%)]	Loss: 1.252796
Train Epoch: 9 [10240/50000 (20%)]	Loss: 1.505750
Train Epoch: 9 [10880/50000 (22%)]	Loss: 1.266757
Train Epoch: 9 [11520/50000 (23%)]	Loss: 1.454524
Train Epoch: 9 [12160/50000 (24%)]	Loss: 1.393911
Train Epoch: 9 [12800/50000 (26%)]	Loss: 1.508913
Train Epoch: 9 [13440/50000 (27%)]	Loss: 0.973484
Train Epoch: 9 [14080/50000 (28%)]	Loss: 1.265858
Train Epoch: 9 [14720/50000 (29%)]	Loss: 1.398724
Train Epoch: 9 [15360/50000 (31%)]	Loss: 1.519221
Train Epoch: 9 [16000/50000 (32%)]	Loss: 1.255429
Train Epoch: 9 [16640/50000 (33%)]	Loss: 1.209228
Train Epoch: 9 [17280/50000 (35%)]	Loss: 1.485438
Train Epoch: 9 [17920/50000 (36%)]	Loss: 1.311991
Train Epoch: 9 [18560/50000 (37%)]	Loss: 1.200014
Train Epoch: 9 [19200/50000 (38%)]	Loss: 1.149714
Train Epoch: 9 [19840/50000 (40%)]	Loss: 1.084350
Train Epoch: 9 [20480/50000 (41%)]	Loss: 1.189479
Train Epoch: 9 [21120/50000 (42%)]	Loss: 1.284921
Train Epoch: 9 [21760/50000 (43%)]	Loss: 1.295864
Train Epoch: 9 [22400/50000 (45%)]	Loss: 1.138263
Train Epoch: 9 [23040/50000 (46%)]	Loss: 1.367797
Train Epoch: 9 [23680/50000 (47%)]	Loss: 1.446803
Train Epoch: 9 [24320/50000 (49%)]	Loss: 1.309628
Train Epoch: 9 [24960/50000 (50%)]	Loss: 1.331444
Train Epoch: 9 [25600/50000 (51%)]	Loss: 1.448681
Train Epoch: 9 [26240/50000 (52%)]	Loss: 1.535498
Train Epoch: 9 [26880/50000 (54%)]	Loss: 1.078624
Train Epoch: 9 [27520/50000 (55%)]	Loss: 1.517728
Train Epoch: 9 [28160/50000 (56%)]	Loss: 1.300126
Train Epoch: 9 [28800/50000 (58%)]	Loss: 1.467307
Train Epoch: 9 [29440/50000 (59%)]	Loss: 1.299382
Train Epoch: 9 [30080/50000 (60%)]	Loss: 1.410619
Train Epoch: 9 [30720/50000 (61%)]	Loss: 1.201459
Train Epoch: 9 [31360/50000 (63%)]	Loss: 1.312103
Train Epoch: 9 [32000/50000 (64%)]	Loss: 1.328135
Train Epoch: 9 [32640/50000 (65%)]	Loss: 1.549781
Train Epoch: 9 [33280/50000 (66%)]	Loss: 1.389344
Train Epoch: 9 [33920/50000 (68%)]	Loss: 1.169310
Train Epoch: 9 [34560/50000 (69%)]	Loss: 1.113059
Train Epoch: 9 [35200/50000 (70%)]	Loss: 1.486740
Train Epoch: 9 [35840/50000 (72%)]	Loss: 1.426147
Train Epoch: 9 [36480/50000 (73%)]	Loss: 1.533374
Train Epoch: 9 [37120/50000 (74%)]	Loss: 1.384663
Train Epoch: 9 [37760/50000 (75%)]	Loss: 1.330965
Train Epoch: 9 [38400/50000 (77%)]	Loss: 1.264296
Train Epoch: 9 [39040/50000 (78%)]	Loss: 1.138296
Train Epoch: 9 [39680/50000 (79%)]	Loss: 1.453632
Train Epoch: 9 [40320/50000 (81%)]	Loss: 1.443600
Train Epoch: 9 [40960/50000 (82%)]	Loss: 1.211195
Train Epoch: 9 [41600/50000 (83%)]	Loss: 1.516727
Train Epoch: 9 [42240/50000 (84%)]	Loss: 1.271758
Train Epoch: 9 [42880/50000 (86%)]	Loss: 1.284572
Train Epoch: 9 [43520/50000 (87%)]	Loss: 1.439721
Train Epoch: 9 [44160/50000 (88%)]	Loss: 1.503864
Train Epoch: 9 [44800/50000 (90%)]	Loss: 1.481705
Train Epoch: 9 [45440/50000 (91%)]	Loss: 1.304073
Train Epoch: 9 [46080/50000 (92%)]	Loss: 1.542000
Train Epoch: 9 [46720/50000 (93%)]	Loss: 1.237294
Train Epoch: 9 [47360/50000 (95%)]	Loss: 1.060939
Train Epoch: 9 [48000/50000 (96%)]	Loss: 1.402381
Train Epoch: 9 [48640/50000 (97%)]	Loss: 1.307186
Train Epoch: 9 [49280/50000 (98%)]	Loss: 1.319392
Train Epoch: 9 [49920/50000 (100%)]	Loss: 1.290042

Test set: Average loss: 1.3046, Accuracy: 5402/10000 (54%)

Train Epoch: 10 [0/50000 (0%)]	Loss: 1.242942
Train Epoch: 10 [640/50000 (1%)]	Loss: 1.246693
Train Epoch: 10 [1280/50000 (3%)]	Loss: 1.319891
Train Epoch: 10 [1920/50000 (4%)]	Loss: 1.202149
Train Epoch: 10 [2560/50000 (5%)]	Loss: 1.272185
Train Epoch: 10 [3200/50000 (6%)]	Loss: 1.425191
Train Epoch: 10 [3840/50000 (8%)]	Loss: 1.311664
Train Epoch: 10 [4480/50000 (9%)]	Loss: 1.172119
Train Epoch: 10 [5120/50000 (10%)]	Loss: 1.552049
Train Epoch: 10 [5760/50000 (12%)]	Loss: 1.355467
Train Epoch: 10 [6400/50000 (13%)]	Loss: 1.392903
Train Epoch: 10 [7040/50000 (14%)]	Loss: 1.282845
Train Epoch: 10 [7680/50000 (15%)]	Loss: 1.325899
Train Epoch: 10 [8320/50000 (17%)]	Loss: 1.372947
Train Epoch: 10 [8960/50000 (18%)]	Loss: 1.506988
Train Epoch: 10 [9600/50000 (19%)]	Loss: 1.130410
Train Epoch: 10 [10240/50000 (20%)]	Loss: 1.230235
Train Epoch: 10 [10880/50000 (22%)]	Loss: 1.286620
Train Epoch: 10 [11520/50000 (23%)]	Loss: 1.142565
Train Epoch: 10 [12160/50000 (24%)]	Loss: 1.351563
Train Epoch: 10 [12800/50000 (26%)]	Loss: 1.120899
Train Epoch: 10 [13440/50000 (27%)]	Loss: 1.298447
Train Epoch: 10 [14080/50000 (28%)]	Loss: 1.342377
Train Epoch: 10 [14720/50000 (29%)]	Loss: 1.168379
Train Epoch: 10 [15360/50000 (31%)]	Loss: 1.020781
Train Epoch: 10 [16000/50000 (32%)]	Loss: 1.175088
Train Epoch: 10 [16640/50000 (33%)]	Loss: 1.530657
Train Epoch: 10 [17280/50000 (35%)]	Loss: 1.322269
Train Epoch: 10 [17920/50000 (36%)]	Loss: 1.396101
Train Epoch: 10 [18560/50000 (37%)]	Loss: 1.314941
Train Epoch: 10 [19200/50000 (38%)]	Loss: 1.243712
Train Epoch: 10 [19840/50000 (40%)]	Loss: 1.221865
Train Epoch: 10 [20480/50000 (41%)]	Loss: 1.027636
Train Epoch: 10 [21120/50000 (42%)]	Loss: 1.237233
Train Epoch: 10 [21760/50000 (43%)]	Loss: 1.317742
Train Epoch: 10 [22400/50000 (45%)]	Loss: 1.271029
Train Epoch: 10 [23040/50000 (46%)]	Loss: 1.283989
Train Epoch: 10 [23680/50000 (47%)]	Loss: 1.174813
Train Epoch: 10 [24320/50000 (49%)]	Loss: 1.323363
Train Epoch: 10 [24960/50000 (50%)]	Loss: 1.273806
Train Epoch: 10 [25600/50000 (51%)]	Loss: 1.266617
Train Epoch: 10 [26240/50000 (52%)]	Loss: 1.266261
Train Epoch: 10 [26880/50000 (54%)]	Loss: 1.440786
Train Epoch: 10 [27520/50000 (55%)]	Loss: 1.445101
Train Epoch: 10 [28160/50000 (56%)]	Loss: 1.298490
Train Epoch: 10 [28800/50000 (58%)]	Loss: 1.334450
Train Epoch: 10 [29440/50000 (59%)]	Loss: 1.239781
Train Epoch: 10 [30080/50000 (60%)]	Loss: 1.203336
Train Epoch: 10 [30720/50000 (61%)]	Loss: 1.377609
Train Epoch: 10 [31360/50000 (63%)]	Loss: 1.115648
Train Epoch: 10 [32000/50000 (64%)]	Loss: 1.218124
Train Epoch: 10 [32640/50000 (65%)]	Loss: 1.453900
Train Epoch: 10 [33280/50000 (66%)]	Loss: 1.549951
Train Epoch: 10 [33920/50000 (68%)]	Loss: 1.315314
Train Epoch: 10 [34560/50000 (69%)]	Loss: 1.111026
Train Epoch: 10 [35200/50000 (70%)]	Loss: 1.653654
Train Epoch: 10 [35840/50000 (72%)]	Loss: 1.240258
Train Epoch: 10 [36480/50000 (73%)]	Loss: 1.072639
Train Epoch: 10 [37120/50000 (74%)]	Loss: 1.259495
Train Epoch: 10 [37760/50000 (75%)]	Loss: 1.298446
Train Epoch: 10 [38400/50000 (77%)]	Loss: 1.291881
Train Epoch: 10 [39040/50000 (78%)]	Loss: 1.443231
Train Epoch: 10 [39680/50000 (79%)]	Loss: 1.492805
Train Epoch: 10 [40320/50000 (81%)]	Loss: 1.414835
Train Epoch: 10 [40960/50000 (82%)]	Loss: 1.336481
Train Epoch: 10 [41600/50000 (83%)]	Loss: 1.072205
Train Epoch: 10 [42240/50000 (84%)]	Loss: 1.270477
Train Epoch: 10 [42880/50000 (86%)]	Loss: 1.088857
Train Epoch: 10 [43520/50000 (87%)]	Loss: 1.234930
Train Epoch: 10 [44160/50000 (88%)]	Loss: 1.487804
Train Epoch: 10 [44800/50000 (90%)]	Loss: 1.223599
Train Epoch: 10 [45440/50000 (91%)]	Loss: 1.606222
Train Epoch: 10 [46080/50000 (92%)]	Loss: 1.408034
Train Epoch: 10 [46720/50000 (93%)]	Loss: 1.335408
Train Epoch: 10 [47360/50000 (95%)]	Loss: 1.283621
Train Epoch: 10 [48000/50000 (96%)]	Loss: 1.364503
Train Epoch: 10 [48640/50000 (97%)]	Loss: 1.151336
Train Epoch: 10 [49280/50000 (98%)]	Loss: 1.221992
Train Epoch: 10 [49920/50000 (100%)]	Loss: 1.206818

Test set: Average loss: 1.2522, Accuracy: 5633/10000 (56%)







    













    













    













    




<class 'torch.FloatTensor'> torch.Size([6, 3, 5, 5])
<class 'torch.FloatTensor'> torch.Size([3, 14, 21])

In [13]:

    

# with Dropout
# with Weight initialization


from __future__ import print_function
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms
from torch.autograd import Variable

import math
import numpy as np
import matplotlib.pyplot as plt
from torchvision.utils import make_grid
from collections import OrderedDict

# Training settings
# for terminal use. In notebook, you can't parse arguments
class args:
    cuda = False
    batch_size = 64
    test_batch_size = 1000
    epochs = 10
    lr = 0.01
    momentum = 0.5
    no_cuda = False
    seed = 1
    log_interval = 10
    # if add Dropout
    with_dropout = True
    # if initialize weights
    with_init_weights = True
    # if add BatchNorm
    with_batchnorm = False

args.cuda = not args.no_cuda and torch.cuda.is_available()

torch.manual_seed(args.seed)
if args.cuda:
    torch.cuda.manual_seed(args.seed)

kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {}
# The output of torchvision datasets are PILImage images of range [0, 1].
# We transform them to Tensors of normalized range [-1, 1]
transform = transforms.Compose([transforms.ToTensor(),
                              transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
                             ])
trainset = datasets.CIFAR10(root='./data', train=True, download=True, transform=transform)
train_loader = torch.utils.data.DataLoader(trainset, batch_size=args.batch_size,
                                          shuffle=True, num_workers=2)

testset = datasets.CIFAR10(root='./data', train=False, download=True, transform=transform)
test_loader = torch.utils.data.DataLoader(testset, batch_size=args.test_batch_size,
                                          shuffle=False, num_workers=2)
classes = ('plane', 'car', 'bird', 'cat',
           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')


class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        # TODO: define your network here
        self.conv_1 = nn.Conv2d(3, 6, kernel_size=5, stride=1)
        self.conv_2 = nn.Conv2d(6, 16, kernel_size=5, stride=1)
        # TODO: replace fc with conv
        self.fc_1 = nn.Linear(16 * 25, 120)
        self.fc_2 = nn.Linear(120, 84)
        self.fc_3 = nn.Linear(84, 10)
        if args.with_batchnorm:
            self.block_conv_1 = nn.Sequential(
                self.conv_1,
                nn.BatchNorm2d(6),
                nn.ReLU(),
                nn.MaxPool2d(kernel_size=2, stride=2)
            )
            self.block_conv_2 = nn.Sequential(
                self.conv_2,
                nn.BatchNorm2d(16),
                nn.ReLU(),
                nn.MaxPool2d(kernel_size=2, stride=2)
            )
        else:
            self.block_conv_1 = nn.Sequential(
                self.conv_1,
                nn.ReLU(),
                nn.MaxPool2d(kernel_size=2, stride=2)
            )
            self.block_conv_2 = nn.Sequential(
                self.conv_2,
                nn.ReLU(),
                nn.MaxPool2d(kernel_size=2, stride=2)
            )
        if args.with_dropout:
            if args.with_batchnorm:
                self.block_fc_1 = nn.Sequential(
                    self.fc_1,
                    nn.BatchNorm1d(120),
                    nn.Dropout()
                )
                self.block_fc_2 = nn.Sequential(
                    self.fc_2,
                    nn.BatchNorm1d(84),
                    nn.Dropout()
                )
            else:
                self.block_fc_1 = nn.Sequential(
                    self.fc_1,
                    nn.Dropout()
                )
                self.block_fc_2 = nn.Sequential(
                    self.fc_2,
                    nn.Dropout()
                )
        else:
            self.block_fc_1 = self.fc_1
            self.block_fc_2 = self.fc_2
        self.softmax = nn.LogSoftmax()
        # Initialize parameters
        if args.with_init_weights:
            for m in self.modules():
                if isinstance(m, nn.Conv2d):
                    n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                    m.weight.data.normal_(0, math.sqrt(2. /n))
                    if m.bias is not None:
                        m.bias.data.zero_()
                elif isinstance(m, nn.Linear):
                    n = m.out_features
                    m.weight.data.normal_(0, math.sqrt(2. /n))
                    if m.bias is not None:
                        m.bias.data.zero_()
                elif isinstance(m, nn.BatchNorm1d) or isinstance(m, nn.BatchNorm2d):
                    m.weight.data.fill_(1)
                    m.bias.data.zero_()

    def forward(self, x):
        # TODO
        x = self.block_conv_1(x)
        x = self.block_conv_2(x)
        x = x.view(x.size(0), -1)
        x = self.block_fc_1(x)
        x = self.block_fc_2(x)
        x = self.fc_3(x)
        x = self.softmax(x)
        return x

# Filter extractor for filter visualization
class FeatureExtractor(nn.Module):
    def __init__(self, model, layer_names):
        super(FeatureExtractor, self).__init__()
        self._model = model
        self._layer_names = set(layer_names)
    def forward(self):
        filters = dict()
        # _modules is an OrderedDict, which replace iteritems() with items() in python3.* 
        for name, module in self._model._modules.items():
            if name in self._layer_names:
                filters[name] = module.weight.data.cpu()
        return filters

# Vesualize training results and trained filters
class VisualizedResult():
    def __init__(self, model):
        self._model = model
    def training_curve(self, epoches, train_loss_records, test_loss_records):
        fig = plt.figure()
        ax_train = fig.add_subplot(111)
        ax_test = fig.add_subplot(111)
        plt.axis([1, epoches, 0, math.ceil(max(train_loss_records + test_loss_records) * 1.2)])
        plt.xlabel('Epoches')
        plt.ylabel('Loss')
        plt.title('Training Curve')
        plt.plot(range(1, epoches + 1), train_loss_records, 'b-', label="train loss")
        plt.plot(range(1, epoches + 1), test_loss_records, 'r-', label="test loss")
        for xy in zip(range(1, epoches + 1), train_loss_records):
            ax_train.annotate('%.2f' % xy[1], xy=xy, textcoords='data')
        for xy in zip(range(1, epoches + 1), test_loss_records):
            ax_test.annotate('%.2f' % xy[1], xy=xy, textcoords='data')
        plt.legend(loc='upper right', borderaxespad=0.)
        plt.show()
    def accuracy_curve(self, epoches, accuracy_records):
        fig = plt.figure()
        ax = fig.add_subplot(111)
        plt.axis([1, epoches, 0, 100])
        plt.xlabel('Epoches')
        plt.ylabel('Accuracy')
        plt.title('Accuracy Curve')
        plt.plot(range(1, epoches + 1), accuracy_records, '-')
        for xy in zip(range(1, epoches + 1), accuracy_records):
            ax.annotate('%s%%' % xy[1], xy=xy, textcoords='data')
        plt.show()
    def conv_filter(self, layer_names):
        feature_extractor = FeatureExtractor(self._model, layer_names)
        filter_list = feature_extractor.forward()
        for name, filters in filter_list.items():
            print(type(filters), filters.size())
            # Normalize filters
            mean = torch.mean(filters)
            std = torch.std(filters)
            filters = (filters - mean) / std
            img = make_grid(filters, nrow=3)
            print(type(img), img.size())
            npimg = img.numpy()
            plt.imshow(np.transpose(npimg, (1,2,0)), interpolation='nearest')


model = Net()
if args.cuda:
    model.cuda()
    
# TODO: other optimizers
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum)

train_loss_records = list()
test_loss_records = list()
accuracy_records = list()

def train(epoch):
    model.train()
    train_loss = 0
    for batch_idx, (data, target) in enumerate(train_loader):
        if args.cuda:
            data, target = data.cuda(), target.cuda()
        data, target = Variable(data), Variable(target)
        optimizer.zero_grad()
        output = model(data)
        loss = F.nll_loss(output, target)   # is it true to use such a loss over cross-entropy loss? 
        loss.backward()
        optimizer.step()
        train_loss += loss.data[0]
        if batch_idx % args.log_interval == 0:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch, batch_idx * len(data), len(train_loader.dataset),
                100. * batch_idx / len(train_loader), loss.data[0]))
    # Average training loss for this epoch
    train_loss_records.append(train_loss / len(train_loader))

def test(epoch):
    model.eval()
    test_loss = 0
    correct = 0
    for data, target in test_loader:
        if args.cuda:
            data, target = data.cuda(), target.cuda()
        data, target = Variable(data, volatile=True), Variable(target)
        output = model(data)
        test_loss += F.nll_loss(output, target).data[0]
        pred = output.data.max(1)[1] # get the index of the max log-probability
        correct += pred.eq(target.data).cpu().sum()

    test_loss = test_loss
    test_loss /= len(test_loader) # loss function already averages over batch size
    accuracy = 100. * correct / len(test_loader.dataset)
    print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
        test_loss, correct, len(test_loader.dataset),
        accuracy))
    test_loss_records.append(test_loss)
    accuracy_records.append(accuracy)


for epoch in range(1, args.epochs + 1):
    train(epoch)
    test(epoch)

visual_result = VisualizedResult(model)
# Visualize training curve
visual_result.training_curve(args.epochs, train_loss_records, test_loss_records)
# Visualize test accuracy
visual_result.accuracy_curve(args.epochs, accuracy_records)
# Visualize trained filter on the 1st Conv layer
visual_result.conv_filter(['conv_1'])


    

    




Files already downloaded and verified
Files already downloaded and verified
Train Epoch: 1 [0/50000 (0%)]	Loss: 15.234482
Train Epoch: 1 [640/50000 (1%)]	Loss: 2.326665
Train Epoch: 1 [1280/50000 (3%)]	Loss: 2.362413
Train Epoch: 1 [1920/50000 (4%)]	Loss: 2.276755
Train Epoch: 1 [2560/50000 (5%)]	Loss: 2.338002
Train Epoch: 1 [3200/50000 (6%)]	Loss: 2.348285
Train Epoch: 1 [3840/50000 (8%)]	Loss: 2.332980
Train Epoch: 1 [4480/50000 (9%)]	Loss: 2.341329
Train Epoch: 1 [5120/50000 (10%)]	Loss: 2.301756
Train Epoch: 1 [5760/50000 (12%)]	Loss: 2.284743
Train Epoch: 1 [6400/50000 (13%)]	Loss: 2.300001
Train Epoch: 1 [7040/50000 (14%)]	Loss: 2.277496
Train Epoch: 1 [7680/50000 (15%)]	Loss: 2.302809
Train Epoch: 1 [8320/50000 (17%)]	Loss: 2.395758
Train Epoch: 1 [8960/50000 (18%)]	Loss: 2.345976
Train Epoch: 1 [9600/50000 (19%)]	Loss: 2.294946
Train Epoch: 1 [10240/50000 (20%)]	Loss: 2.292124
Train Epoch: 1 [10880/50000 (22%)]	Loss: 2.293660
Train Epoch: 1 [11520/50000 (23%)]	Loss: 2.333412
Train Epoch: 1 [12160/50000 (24%)]	Loss: 2.329465
Train Epoch: 1 [12800/50000 (26%)]	Loss: 2.305642
Train Epoch: 1 [13440/50000 (27%)]	Loss: 2.316200
Train Epoch: 1 [14080/50000 (28%)]	Loss: 2.303170
Train Epoch: 1 [14720/50000 (29%)]	Loss: 2.342990
Train Epoch: 1 [15360/50000 (31%)]	Loss: 2.341306
Train Epoch: 1 [16000/50000 (32%)]	Loss: 2.279560
Train Epoch: 1 [16640/50000 (33%)]	Loss: 2.339725
Train Epoch: 1 [17280/50000 (35%)]	Loss: 2.327679
Train Epoch: 1 [17920/50000 (36%)]	Loss: 2.313870
Train Epoch: 1 [18560/50000 (37%)]	Loss: 2.326478
Train Epoch: 1 [19200/50000 (38%)]	Loss: 2.319654
Train Epoch: 1 [19840/50000 (40%)]	Loss: 2.324892
Train Epoch: 1 [20480/50000 (41%)]	Loss: 2.315763
Train Epoch: 1 [21120/50000 (42%)]	Loss: 2.307356
Train Epoch: 1 [21760/50000 (43%)]	Loss: 2.289115
Train Epoch: 1 [22400/50000 (45%)]	Loss: 2.315075
Train Epoch: 1 [23040/50000 (46%)]	Loss: 2.341220
Train Epoch: 1 [23680/50000 (47%)]	Loss: 2.295772
Train Epoch: 1 [24320/50000 (49%)]	Loss: 2.329668
Train Epoch: 1 [24960/50000 (50%)]	Loss: 2.330588
Train Epoch: 1 [25600/50000 (51%)]	Loss: 2.294637
Train Epoch: 1 [26240/50000 (52%)]	Loss: 2.314350
Train Epoch: 1 [26880/50000 (54%)]	Loss: 2.379716
Train Epoch: 1 [27520/50000 (55%)]	Loss: 2.298051
Train Epoch: 1 [28160/50000 (56%)]	Loss: 2.324712
Train Epoch: 1 [28800/50000 (58%)]	Loss: 2.303374
Train Epoch: 1 [29440/50000 (59%)]	Loss: 2.345768
Train Epoch: 1 [30080/50000 (60%)]	Loss: 2.272227
Train Epoch: 1 [30720/50000 (61%)]	Loss: 2.303907
Train Epoch: 1 [31360/50000 (63%)]	Loss: 2.331093
Train Epoch: 1 [32000/50000 (64%)]	Loss: 2.319263
Train Epoch: 1 [32640/50000 (65%)]	Loss: 2.288735
Train Epoch: 1 [33280/50000 (66%)]	Loss: 2.335385
Train Epoch: 1 [33920/50000 (68%)]	Loss: 2.312720
Train Epoch: 1 [34560/50000 (69%)]	Loss: 2.351252
Train Epoch: 1 [35200/50000 (70%)]	Loss: 2.334589
Train Epoch: 1 [35840/50000 (72%)]	Loss: 2.319540
Train Epoch: 1 [36480/50000 (73%)]	Loss: 2.317049
Train Epoch: 1 [37120/50000 (74%)]	Loss: 2.299483
Train Epoch: 1 [37760/50000 (75%)]	Loss: 2.358359
Train Epoch: 1 [38400/50000 (77%)]	Loss: 2.334314
Train Epoch: 1 [39040/50000 (78%)]	Loss: 2.297088
Train Epoch: 1 [39680/50000 (79%)]	Loss: 2.321880
Train Epoch: 1 [40320/50000 (81%)]	Loss: 2.320149
Train Epoch: 1 [40960/50000 (82%)]	Loss: 2.317154
Train Epoch: 1 [41600/50000 (83%)]	Loss: 2.318118
Train Epoch: 1 [42240/50000 (84%)]	Loss: 2.279813
Train Epoch: 1 [42880/50000 (86%)]	Loss: 2.326334
Train Epoch: 1 [43520/50000 (87%)]	Loss: 2.313096
Train Epoch: 1 [44160/50000 (88%)]	Loss: 2.313335
Train Epoch: 1 [44800/50000 (90%)]	Loss: 2.311135
Train Epoch: 1 [45440/50000 (91%)]	Loss: 2.374033
Train Epoch: 1 [46080/50000 (92%)]	Loss: 2.362906
Train Epoch: 1 [46720/50000 (93%)]	Loss: 2.321835
Train Epoch: 1 [47360/50000 (95%)]	Loss: 2.354999
Train Epoch: 1 [48000/50000 (96%)]	Loss: 2.312239
Train Epoch: 1 [48640/50000 (97%)]	Loss: 2.344520
Train Epoch: 1 [49280/50000 (98%)]	Loss: 2.257580
Train Epoch: 1 [49920/50000 (100%)]	Loss: 2.283464

Test set: Average loss: 2.3069, Accuracy: 998/10000 (10%)

Train Epoch: 2 [0/50000 (0%)]	Loss: 2.327988
Train Epoch: 2 [640/50000 (1%)]	Loss: 2.308882
Train Epoch: 2 [1280/50000 (3%)]	Loss: 2.313838
Train Epoch: 2 [1920/50000 (4%)]	Loss: 2.263880
Train Epoch: 2 [2560/50000 (5%)]	Loss: 2.319611
Train Epoch: 2 [3200/50000 (6%)]	Loss: 2.295112
Train Epoch: 2 [3840/50000 (8%)]	Loss: 2.313395
Train Epoch: 2 [4480/50000 (9%)]	Loss: 2.309272
Train Epoch: 2 [5120/50000 (10%)]	Loss: 2.313822
Train Epoch: 2 [5760/50000 (12%)]	Loss: 2.302895
Train Epoch: 2 [6400/50000 (13%)]	Loss: 2.335178
Train Epoch: 2 [7040/50000 (14%)]	Loss: 2.318026
Train Epoch: 2 [7680/50000 (15%)]	Loss: 2.320074
Train Epoch: 2 [8320/50000 (17%)]	Loss: 2.333184
Train Epoch: 2 [8960/50000 (18%)]	Loss: 2.298934
Train Epoch: 2 [9600/50000 (19%)]	Loss: 2.308050
Train Epoch: 2 [10240/50000 (20%)]	Loss: 2.304924
Train Epoch: 2 [10880/50000 (22%)]	Loss: 2.318158
Train Epoch: 2 [11520/50000 (23%)]	Loss: 2.341682
Train Epoch: 2 [12160/50000 (24%)]	Loss: 2.324960
Train Epoch: 2 [12800/50000 (26%)]	Loss: 2.325027
Train Epoch: 2 [13440/50000 (27%)]	Loss: 2.317587
Train Epoch: 2 [14080/50000 (28%)]	Loss: 2.270529
Train Epoch: 2 [14720/50000 (29%)]	Loss: 2.327427
Train Epoch: 2 [15360/50000 (31%)]	Loss: 2.329642
Train Epoch: 2 [16000/50000 (32%)]	Loss: 2.342058
Train Epoch: 2 [16640/50000 (33%)]	Loss: 2.302392
Train Epoch: 2 [17280/50000 (35%)]	Loss: 2.302926
Train Epoch: 2 [17920/50000 (36%)]	Loss: 2.305215
Train Epoch: 2 [18560/50000 (37%)]	Loss: 2.305821
Train Epoch: 2 [19200/50000 (38%)]	Loss: 2.292022
Train Epoch: 2 [19840/50000 (40%)]	Loss: 2.350047
Train Epoch: 2 [20480/50000 (41%)]	Loss: 2.321962
Train Epoch: 2 [21120/50000 (42%)]	Loss: 2.307122
Train Epoch: 2 [21760/50000 (43%)]	Loss: 2.331790
Train Epoch: 2 [22400/50000 (45%)]	Loss: 2.294449
Train Epoch: 2 [23040/50000 (46%)]	Loss: 2.330075
Train Epoch: 2 [23680/50000 (47%)]	Loss: 2.335268
Train Epoch: 2 [24320/50000 (49%)]	Loss: 2.288678
Train Epoch: 2 [24960/50000 (50%)]	Loss: 2.340765
Train Epoch: 2 [25600/50000 (51%)]	Loss: 2.314323
Train Epoch: 2 [26240/50000 (52%)]	Loss: 2.285465
Train Epoch: 2 [26880/50000 (54%)]	Loss: 2.340708
Train Epoch: 2 [27520/50000 (55%)]	Loss: 2.272862
Train Epoch: 2 [28160/50000 (56%)]	Loss: 2.345809
Train Epoch: 2 [28800/50000 (58%)]	Loss: 2.363593
Train Epoch: 2 [29440/50000 (59%)]	Loss: 2.248420
Train Epoch: 2 [30080/50000 (60%)]	Loss: 2.384218
Train Epoch: 2 [30720/50000 (61%)]	Loss: 2.201422
Train Epoch: 2 [31360/50000 (63%)]	Loss: 2.274712
Train Epoch: 2 [32000/50000 (64%)]	Loss: 2.354869
Train Epoch: 2 [32640/50000 (65%)]	Loss: 2.299563
Train Epoch: 2 [33280/50000 (66%)]	Loss: 2.292712
Train Epoch: 2 [33920/50000 (68%)]	Loss: 2.222092
Train Epoch: 2 [34560/50000 (69%)]	Loss: 2.278747
Train Epoch: 2 [35200/50000 (70%)]	Loss: 2.346914
Train Epoch: 2 [35840/50000 (72%)]	Loss: 2.194913
Train Epoch: 2 [36480/50000 (73%)]	Loss: 2.310868
Train Epoch: 2 [37120/50000 (74%)]	Loss: 2.272310
Train Epoch: 2 [37760/50000 (75%)]	Loss: 2.122092
Train Epoch: 2 [38400/50000 (77%)]	Loss: 2.245687
Train Epoch: 2 [39040/50000 (78%)]	Loss: 2.203976
Train Epoch: 2 [39680/50000 (79%)]	Loss: 2.189879
Train Epoch: 2 [40320/50000 (81%)]	Loss: 2.310921
Train Epoch: 2 [40960/50000 (82%)]	Loss: 2.351840
Train Epoch: 2 [41600/50000 (83%)]	Loss: 2.165465
Train Epoch: 2 [42240/50000 (84%)]	Loss: 2.303814
Train Epoch: 2 [42880/50000 (86%)]	Loss: 2.109124
Train Epoch: 2 [43520/50000 (87%)]	Loss: 2.163847
Train Epoch: 2 [44160/50000 (88%)]	Loss: 2.099313
Train Epoch: 2 [44800/50000 (90%)]	Loss: 2.242406
Train Epoch: 2 [45440/50000 (91%)]	Loss: 2.207254
Train Epoch: 2 [46080/50000 (92%)]	Loss: 2.262149
Train Epoch: 2 [46720/50000 (93%)]	Loss: 2.158542
Train Epoch: 2 [47360/50000 (95%)]	Loss: 2.202443
Train Epoch: 2 [48000/50000 (96%)]	Loss: 2.174173
Train Epoch: 2 [48640/50000 (97%)]	Loss: 2.237894
Train Epoch: 2 [49280/50000 (98%)]	Loss: 1.984078
Train Epoch: 2 [49920/50000 (100%)]	Loss: 2.199744

Test set: Average loss: 2.1035, Accuracy: 1913/10000 (19%)

Train Epoch: 3 [0/50000 (0%)]	Loss: 2.288315
Train Epoch: 3 [640/50000 (1%)]	Loss: 2.221451
Train Epoch: 3 [1280/50000 (3%)]	Loss: 2.091572
Train Epoch: 3 [1920/50000 (4%)]	Loss: 2.155138
Train Epoch: 3 [2560/50000 (5%)]	Loss: 2.243670
Train Epoch: 3 [3200/50000 (6%)]	Loss: 2.080622
Train Epoch: 3 [3840/50000 (8%)]	Loss: 2.333745
Train Epoch: 3 [4480/50000 (9%)]	Loss: 2.238507
Train Epoch: 3 [5120/50000 (10%)]	Loss: 1.994606
Train Epoch: 3 [5760/50000 (12%)]	Loss: 2.047188
Train Epoch: 3 [6400/50000 (13%)]	Loss: 2.386749
Train Epoch: 3 [7040/50000 (14%)]	Loss: 2.246206
Train Epoch: 3 [7680/50000 (15%)]	Loss: 2.193252
Train Epoch: 3 [8320/50000 (17%)]	Loss: 2.200316
Train Epoch: 3 [8960/50000 (18%)]	Loss: 2.137934
Train Epoch: 3 [9600/50000 (19%)]	Loss: 2.192129
Train Epoch: 3 [10240/50000 (20%)]	Loss: 2.173663
Train Epoch: 3 [10880/50000 (22%)]	Loss: 2.198592
Train Epoch: 3 [11520/50000 (23%)]	Loss: 2.310438
Train Epoch: 3 [12160/50000 (24%)]	Loss: 2.129789
Train Epoch: 3 [12800/50000 (26%)]	Loss: 2.144376
Train Epoch: 3 [13440/50000 (27%)]	Loss: 2.283619
Train Epoch: 3 [14080/50000 (28%)]	Loss: 2.183164
Train Epoch: 3 [14720/50000 (29%)]	Loss: 2.198931
Train Epoch: 3 [15360/50000 (31%)]	Loss: 2.224415
Train Epoch: 3 [16000/50000 (32%)]	Loss: 2.267414
Train Epoch: 3 [16640/50000 (33%)]	Loss: 2.057693
Train Epoch: 3 [17280/50000 (35%)]	Loss: 2.184600
Train Epoch: 3 [17920/50000 (36%)]	Loss: 2.067664
Train Epoch: 3 [18560/50000 (37%)]	Loss: 2.051739
Train Epoch: 3 [19200/50000 (38%)]	Loss: 2.281646
Train Epoch: 3 [19840/50000 (40%)]	Loss: 1.934528
Train Epoch: 3 [20480/50000 (41%)]	Loss: 2.052733
Train Epoch: 3 [21120/50000 (42%)]	Loss: 2.032997
Train Epoch: 3 [21760/50000 (43%)]	Loss: 2.152807
Train Epoch: 3 [22400/50000 (45%)]	Loss: 2.168287
Train Epoch: 3 [23040/50000 (46%)]	Loss: 2.098366
Train Epoch: 3 [23680/50000 (47%)]	Loss: 2.205731
Train Epoch: 3 [24320/50000 (49%)]	Loss: 2.088423
Train Epoch: 3 [24960/50000 (50%)]	Loss: 2.080623
Train Epoch: 3 [25600/50000 (51%)]	Loss: 2.188262
Train Epoch: 3 [26240/50000 (52%)]	Loss: 2.114068
Train Epoch: 3 [26880/50000 (54%)]	Loss: 2.356820
Train Epoch: 3 [27520/50000 (55%)]	Loss: 2.092958
Train Epoch: 3 [28160/50000 (56%)]	Loss: 2.075316
Train Epoch: 3 [28800/50000 (58%)]	Loss: 2.138782
Train Epoch: 3 [29440/50000 (59%)]	Loss: 2.130430
Train Epoch: 3 [30080/50000 (60%)]	Loss: 2.070377
Train Epoch: 3 [30720/50000 (61%)]	Loss: 2.129575
Train Epoch: 3 [31360/50000 (63%)]	Loss: 2.053311
Train Epoch: 3 [32000/50000 (64%)]	Loss: 2.091939
Train Epoch: 3 [32640/50000 (65%)]	Loss: 2.296788
Train Epoch: 3 [33280/50000 (66%)]	Loss: 2.050244
Train Epoch: 3 [33920/50000 (68%)]	Loss: 2.145100
Train Epoch: 3 [34560/50000 (69%)]	Loss: 2.166432
Train Epoch: 3 [35200/50000 (70%)]	Loss: 2.255147
Train Epoch: 3 [35840/50000 (72%)]	Loss: 2.138298
Train Epoch: 3 [36480/50000 (73%)]	Loss: 2.173778
Train Epoch: 3 [37120/50000 (74%)]	Loss: 2.090233
Train Epoch: 3 [37760/50000 (75%)]	Loss: 2.107261
Train Epoch: 3 [38400/50000 (77%)]	Loss: 2.097383
Train Epoch: 3 [39040/50000 (78%)]	Loss: 2.094161
Train Epoch: 3 [39680/50000 (79%)]	Loss: 1.955976
Train Epoch: 3 [40320/50000 (81%)]	Loss: 2.165629
Train Epoch: 3 [40960/50000 (82%)]	Loss: 2.045896
Train Epoch: 3 [41600/50000 (83%)]	Loss: 2.166697
Train Epoch: 3 [42240/50000 (84%)]	Loss: 1.946791
Train Epoch: 3 [42880/50000 (86%)]	Loss: 2.230630
Train Epoch: 3 [43520/50000 (87%)]	Loss: 1.964792
Train Epoch: 3 [44160/50000 (88%)]	Loss: 2.153301
Train Epoch: 3 [44800/50000 (90%)]	Loss: 2.881334
Train Epoch: 3 [45440/50000 (91%)]	Loss: 2.415737
Train Epoch: 3 [46080/50000 (92%)]	Loss: 2.311382
Train Epoch: 3 [46720/50000 (93%)]	Loss: 2.332874
Train Epoch: 3 [47360/50000 (95%)]	Loss: 2.351287
Train Epoch: 3 [48000/50000 (96%)]	Loss: 2.310783
Train Epoch: 3 [48640/50000 (97%)]	Loss: 2.316717
Train Epoch: 3 [49280/50000 (98%)]	Loss: 2.321986
Train Epoch: 3 [49920/50000 (100%)]	Loss: 2.333109

Test set: Average loss: 2.3055, Accuracy: 1053/10000 (11%)

Train Epoch: 4 [0/50000 (0%)]	Loss: 2.284265
Train Epoch: 4 [640/50000 (1%)]	Loss: 2.301237
Train Epoch: 4 [1280/50000 (3%)]	Loss: 2.243764
Train Epoch: 4 [1920/50000 (4%)]	Loss: 2.295274
Train Epoch: 4 [2560/50000 (5%)]	Loss: 2.240234
Train Epoch: 4 [3200/50000 (6%)]	Loss: 2.226293
Train Epoch: 4 [3840/50000 (8%)]	Loss: 2.202907
Train Epoch: 4 [4480/50000 (9%)]	Loss: 2.122547
Train Epoch: 4 [5120/50000 (10%)]	Loss: 2.149384
Train Epoch: 4 [5760/50000 (12%)]	Loss: 2.240614
Train Epoch: 4 [6400/50000 (13%)]	Loss: 2.062918
Train Epoch: 4 [7040/50000 (14%)]	Loss: 2.109497
Train Epoch: 4 [7680/50000 (15%)]	Loss: 1.999337
Train Epoch: 4 [8320/50000 (17%)]	Loss: 2.159018
Train Epoch: 4 [8960/50000 (18%)]	Loss: 2.132478
Train Epoch: 4 [9600/50000 (19%)]	Loss: 1.895852
Train Epoch: 4 [10240/50000 (20%)]	Loss: 2.147673
Train Epoch: 4 [10880/50000 (22%)]	Loss: 2.050185
Train Epoch: 4 [11520/50000 (23%)]	Loss: 2.172759
Train Epoch: 4 [12160/50000 (24%)]	Loss: 2.013026
Train Epoch: 4 [12800/50000 (26%)]	Loss: 1.947311
Train Epoch: 4 [13440/50000 (27%)]	Loss: 2.148469
Train Epoch: 4 [14080/50000 (28%)]	Loss: 2.152969
Train Epoch: 4 [14720/50000 (29%)]	Loss: 2.172697
Train Epoch: 4 [15360/50000 (31%)]	Loss: 2.051605
Train Epoch: 4 [16000/50000 (32%)]	Loss: 1.969834
Train Epoch: 4 [16640/50000 (33%)]	Loss: 2.057524
Train Epoch: 4 [17280/50000 (35%)]	Loss: 2.084074
Train Epoch: 4 [17920/50000 (36%)]	Loss: 2.453411
Train Epoch: 4 [18560/50000 (37%)]	Loss: 2.130677
Train Epoch: 4 [19200/50000 (38%)]	Loss: 2.147884
Train Epoch: 4 [19840/50000 (40%)]	Loss: 2.145547
Train Epoch: 4 [20480/50000 (41%)]	Loss: 2.254623
Train Epoch: 4 [21120/50000 (42%)]	Loss: 2.083845
Train Epoch: 4 [21760/50000 (43%)]	Loss: 2.188988
Train Epoch: 4 [22400/50000 (45%)]	Loss: 2.117231
Train Epoch: 4 [23040/50000 (46%)]	Loss: 2.092469
Train Epoch: 4 [23680/50000 (47%)]	Loss: 2.150401
Train Epoch: 4 [24320/50000 (49%)]	Loss: 2.240330
Train Epoch: 4 [24960/50000 (50%)]	Loss: 1.989899
Train Epoch: 4 [25600/50000 (51%)]	Loss: 2.124893
Train Epoch: 4 [26240/50000 (52%)]	Loss: 2.098323
Train Epoch: 4 [26880/50000 (54%)]	Loss: 2.173028
Train Epoch: 4 [27520/50000 (55%)]	Loss: 2.167114
Train Epoch: 4 [28160/50000 (56%)]	Loss: 2.060126
Train Epoch: 4 [28800/50000 (58%)]	Loss: 2.159064
Train Epoch: 4 [29440/50000 (59%)]	Loss: 2.306288
Train Epoch: 4 [30080/50000 (60%)]	Loss: 1.990523
Train Epoch: 4 [30720/50000 (61%)]	Loss: 2.116693
Train Epoch: 4 [31360/50000 (63%)]	Loss: 2.056664
Train Epoch: 4 [32000/50000 (64%)]	Loss: 2.014547
Train Epoch: 4 [32640/50000 (65%)]	Loss: 2.079448
Train Epoch: 4 [33280/50000 (66%)]	Loss: 2.044400
Train Epoch: 4 [33920/50000 (68%)]	Loss: 2.073864
Train Epoch: 4 [34560/50000 (69%)]	Loss: 2.179935
Train Epoch: 4 [35200/50000 (70%)]	Loss: 1.987222
Train Epoch: 4 [35840/50000 (72%)]	Loss: 2.038720
Train Epoch: 4 [36480/50000 (73%)]	Loss: 2.061580
Train Epoch: 4 [37120/50000 (74%)]	Loss: 2.141477
Train Epoch: 4 [37760/50000 (75%)]	Loss: 2.086507
Train Epoch: 4 [38400/50000 (77%)]	Loss: 2.029004
Train Epoch: 4 [39040/50000 (78%)]	Loss: 2.008881
Train Epoch: 4 [39680/50000 (79%)]	Loss: 1.886082
Train Epoch: 4 [40320/50000 (81%)]	Loss: 2.085955
Train Epoch: 4 [40960/50000 (82%)]	Loss: 2.151316
Train Epoch: 4 [41600/50000 (83%)]	Loss: 2.069967
Train Epoch: 4 [42240/50000 (84%)]	Loss: 2.293299
Train Epoch: 4 [42880/50000 (86%)]	Loss: 1.973604
Train Epoch: 4 [43520/50000 (87%)]	Loss: 2.096565
Train Epoch: 4 [44160/50000 (88%)]	Loss: 2.114150
Train Epoch: 4 [44800/50000 (90%)]	Loss: 1.987725
Train Epoch: 4 [45440/50000 (91%)]	Loss: 2.251225
Train Epoch: 4 [46080/50000 (92%)]	Loss: 1.942547
Train Epoch: 4 [46720/50000 (93%)]	Loss: 2.176468
Train Epoch: 4 [47360/50000 (95%)]	Loss: 2.145967
Train Epoch: 4 [48000/50000 (96%)]	Loss: 1.939853
Train Epoch: 4 [48640/50000 (97%)]	Loss: 2.151726
Train Epoch: 4 [49280/50000 (98%)]	Loss: 1.971006
Train Epoch: 4 [49920/50000 (100%)]	Loss: 2.033925

Test set: Average loss: 2.1176, Accuracy: 2319/10000 (23%)

Train Epoch: 5 [0/50000 (0%)]	Loss: 2.279690
Train Epoch: 5 [640/50000 (1%)]	Loss: 2.051709
Train Epoch: 5 [1280/50000 (3%)]	Loss: 1.939789
Train Epoch: 5 [1920/50000 (4%)]	Loss: 1.737735
Train Epoch: 5 [2560/50000 (5%)]	Loss: 2.211498
Train Epoch: 5 [3200/50000 (6%)]	Loss: 2.268551
Train Epoch: 5 [3840/50000 (8%)]	Loss: 2.054901
Train Epoch: 5 [4480/50000 (9%)]	Loss: 2.100448
Train Epoch: 5 [5120/50000 (10%)]	Loss: 1.994688
Train Epoch: 5 [5760/50000 (12%)]	Loss: 2.091623
Train Epoch: 5 [6400/50000 (13%)]	Loss: 1.974331
Train Epoch: 5 [7040/50000 (14%)]	Loss: 2.256910
Train Epoch: 5 [7680/50000 (15%)]	Loss: 1.968524
Train Epoch: 5 [8320/50000 (17%)]	Loss: 1.928739
Train Epoch: 5 [8960/50000 (18%)]	Loss: 2.110335
Train Epoch: 5 [9600/50000 (19%)]	Loss: 2.062715
Train Epoch: 5 [10240/50000 (20%)]	Loss: 1.912055
Train Epoch: 5 [10880/50000 (22%)]	Loss: 2.357886
Train Epoch: 5 [11520/50000 (23%)]	Loss: 1.975535
Train Epoch: 5 [12160/50000 (24%)]	Loss: 1.888201
Train Epoch: 5 [12800/50000 (26%)]	Loss: 2.421772
Train Epoch: 5 [13440/50000 (27%)]	Loss: 2.414889
Train Epoch: 5 [14080/50000 (28%)]	Loss: 2.013344
Train Epoch: 5 [14720/50000 (29%)]	Loss: 2.327462
Train Epoch: 5 [15360/50000 (31%)]	Loss: 2.007804
Train Epoch: 5 [16000/50000 (32%)]	Loss: 1.900493
Train Epoch: 5 [16640/50000 (33%)]	Loss: 1.904129
Train Epoch: 5 [17280/50000 (35%)]	Loss: 1.927155
Train Epoch: 5 [17920/50000 (36%)]	Loss: 1.980900
Train Epoch: 5 [18560/50000 (37%)]	Loss: 1.971194
Train Epoch: 5 [19200/50000 (38%)]	Loss: 2.170825
Train Epoch: 5 [19840/50000 (40%)]	Loss: 1.985510
Train Epoch: 5 [20480/50000 (41%)]	Loss: 1.801436
Train Epoch: 5 [21120/50000 (42%)]	Loss: 2.155711
Train Epoch: 5 [21760/50000 (43%)]	Loss: 1.973618
Train Epoch: 5 [22400/50000 (45%)]	Loss: 2.012193
Train Epoch: 5 [23040/50000 (46%)]	Loss: 2.076883
Train Epoch: 5 [23680/50000 (47%)]	Loss: 1.904579
Train Epoch: 5 [24320/50000 (49%)]	Loss: 2.011012
Train Epoch: 5 [24960/50000 (50%)]	Loss: 2.047386
Train Epoch: 5 [25600/50000 (51%)]	Loss: 1.840442
Train Epoch: 5 [26240/50000 (52%)]	Loss: 1.971251
Train Epoch: 5 [26880/50000 (54%)]	Loss: 1.937999
Train Epoch: 5 [27520/50000 (55%)]	Loss: 2.160074
Train Epoch: 5 [28160/50000 (56%)]	Loss: 1.979830
Train Epoch: 5 [28800/50000 (58%)]	Loss: 2.014592
Train Epoch: 5 [29440/50000 (59%)]	Loss: 1.931724
Train Epoch: 5 [30080/50000 (60%)]	Loss: 1.875743
Train Epoch: 5 [30720/50000 (61%)]	Loss: 2.030068
Train Epoch: 5 [31360/50000 (63%)]	Loss: 1.938851
Train Epoch: 5 [32000/50000 (64%)]	Loss: 2.061280
Train Epoch: 5 [32640/50000 (65%)]	Loss: 2.021906
Train Epoch: 5 [33280/50000 (66%)]	Loss: 2.121549
Train Epoch: 5 [33920/50000 (68%)]	Loss: 1.908975
Train Epoch: 5 [34560/50000 (69%)]	Loss: 2.257536
Train Epoch: 5 [35200/50000 (70%)]	Loss: 1.998391
Train Epoch: 5 [35840/50000 (72%)]	Loss: 2.063925
Train Epoch: 5 [36480/50000 (73%)]	Loss: 1.921271
Train Epoch: 5 [37120/50000 (74%)]	Loss: 1.986675
Train Epoch: 5 [37760/50000 (75%)]	Loss: 1.790781
Train Epoch: 5 [38400/50000 (77%)]	Loss: 1.924035
Train Epoch: 5 [39040/50000 (78%)]	Loss: 2.175074
Train Epoch: 5 [39680/50000 (79%)]	Loss: 2.112616
Train Epoch: 5 [40320/50000 (81%)]	Loss: 2.124406
Train Epoch: 5 [40960/50000 (82%)]	Loss: 2.030192
Train Epoch: 5 [41600/50000 (83%)]	Loss: 2.108567
Train Epoch: 5 [42240/50000 (84%)]	Loss: 1.987339
Train Epoch: 5 [42880/50000 (86%)]	Loss: 2.101926
Train Epoch: 5 [43520/50000 (87%)]	Loss: 1.887920
Train Epoch: 5 [44160/50000 (88%)]	Loss: 1.987491
Train Epoch: 5 [44800/50000 (90%)]	Loss: 2.117970
Train Epoch: 5 [45440/50000 (91%)]	Loss: 2.013865
Train Epoch: 5 [46080/50000 (92%)]	Loss: 1.893204
Train Epoch: 5 [46720/50000 (93%)]	Loss: 1.942085
Train Epoch: 5 [47360/50000 (95%)]	Loss: 1.907671
Train Epoch: 5 [48000/50000 (96%)]	Loss: 2.247848
Train Epoch: 5 [48640/50000 (97%)]	Loss: 1.922317
Train Epoch: 5 [49280/50000 (98%)]	Loss: 1.918726
Train Epoch: 5 [49920/50000 (100%)]	Loss: 2.115267

Test set: Average loss: 2.3300, Accuracy: 2225/10000 (22%)

Train Epoch: 6 [0/50000 (0%)]	Loss: 2.260248
Train Epoch: 6 [640/50000 (1%)]	Loss: 1.921621
Train Epoch: 6 [1280/50000 (3%)]	Loss: 2.053676
Train Epoch: 6 [1920/50000 (4%)]	Loss: 2.080971
Train Epoch: 6 [2560/50000 (5%)]	Loss: 2.021641
Train Epoch: 6 [3200/50000 (6%)]	Loss: 1.919377
Train Epoch: 6 [3840/50000 (8%)]	Loss: 1.993378
Train Epoch: 6 [4480/50000 (9%)]	Loss: 1.966918
Train Epoch: 6 [5120/50000 (10%)]	Loss: 2.041656
Train Epoch: 6 [5760/50000 (12%)]	Loss: 1.836466
Train Epoch: 6 [6400/50000 (13%)]	Loss: 1.902223
Train Epoch: 6 [7040/50000 (14%)]	Loss: 2.009010
Train Epoch: 6 [7680/50000 (15%)]	Loss: 1.937353
Train Epoch: 6 [8320/50000 (17%)]	Loss: 2.033068
Train Epoch: 6 [8960/50000 (18%)]	Loss: 2.049017
Train Epoch: 6 [9600/50000 (19%)]	Loss: 1.935991
Train Epoch: 6 [10240/50000 (20%)]	Loss: 1.916729
Train Epoch: 6 [10880/50000 (22%)]	Loss: 1.860403
Train Epoch: 6 [11520/50000 (23%)]	Loss: 2.061564
Train Epoch: 6 [12160/50000 (24%)]	Loss: 2.339050
Train Epoch: 6 [12800/50000 (26%)]	Loss: 1.752042
Train Epoch: 6 [13440/50000 (27%)]	Loss: 1.937646
Train Epoch: 6 [14080/50000 (28%)]	Loss: 2.058903
Train Epoch: 6 [14720/50000 (29%)]	Loss: 2.163899
Train Epoch: 6 [15360/50000 (31%)]	Loss: 1.989192
Train Epoch: 6 [16000/50000 (32%)]	Loss: 1.937914
Train Epoch: 6 [16640/50000 (33%)]	Loss: 1.844850
Train Epoch: 6 [17280/50000 (35%)]	Loss: 1.981151
Train Epoch: 6 [17920/50000 (36%)]	Loss: 1.726219
Train Epoch: 6 [18560/50000 (37%)]	Loss: 2.037893
Train Epoch: 6 [19200/50000 (38%)]	Loss: 2.026616
Train Epoch: 6 [19840/50000 (40%)]	Loss: 1.856089
Train Epoch: 6 [20480/50000 (41%)]	Loss: 1.862591
Train Epoch: 6 [21120/50000 (42%)]	Loss: 1.800685
Train Epoch: 6 [21760/50000 (43%)]	Loss: 1.950734
Train Epoch: 6 [22400/50000 (45%)]	Loss: 1.867250
Train Epoch: 6 [23040/50000 (46%)]	Loss: 2.093513
Train Epoch: 6 [23680/50000 (47%)]	Loss: 2.123826
Train Epoch: 6 [24320/50000 (49%)]	Loss: 1.935270
Train Epoch: 6 [24960/50000 (50%)]	Loss: 1.997045
Train Epoch: 6 [25600/50000 (51%)]	Loss: 1.887630
Train Epoch: 6 [26240/50000 (52%)]	Loss: 1.946406
Train Epoch: 6 [26880/50000 (54%)]	Loss: 2.126688
Train Epoch: 6 [27520/50000 (55%)]	Loss: 1.830981
Train Epoch: 6 [28160/50000 (56%)]	Loss: 1.797017
Train Epoch: 6 [28800/50000 (58%)]	Loss: 1.722096
Train Epoch: 6 [29440/50000 (59%)]	Loss: 1.930435
Train Epoch: 6 [30080/50000 (60%)]	Loss: 1.703499
Train Epoch: 6 [30720/50000 (61%)]	Loss: 2.085798
Train Epoch: 6 [31360/50000 (63%)]	Loss: 1.939182
Train Epoch: 6 [32000/50000 (64%)]	Loss: 1.823958
Train Epoch: 6 [32640/50000 (65%)]	Loss: 1.927342
Train Epoch: 6 [33280/50000 (66%)]	Loss: 1.735088
Train Epoch: 6 [33920/50000 (68%)]	Loss: 2.023225
Train Epoch: 6 [34560/50000 (69%)]	Loss: 1.856431
Train Epoch: 6 [35200/50000 (70%)]	Loss: 2.125422
Train Epoch: 6 [35840/50000 (72%)]	Loss: 1.934216
Train Epoch: 6 [36480/50000 (73%)]	Loss: 2.002476
Train Epoch: 6 [37120/50000 (74%)]	Loss: 1.926417
Train Epoch: 6 [37760/50000 (75%)]	Loss: 1.932742
Train Epoch: 6 [38400/50000 (77%)]	Loss: 1.785180
Train Epoch: 6 [39040/50000 (78%)]	Loss: 2.028731
Train Epoch: 6 [39680/50000 (79%)]	Loss: 1.813233
Train Epoch: 6 [40320/50000 (81%)]	Loss: 1.940137
Train Epoch: 6 [40960/50000 (82%)]	Loss: 1.917109
Train Epoch: 6 [41600/50000 (83%)]	Loss: 2.027024
Train Epoch: 6 [42240/50000 (84%)]	Loss: 1.965775
Train Epoch: 6 [42880/50000 (86%)]	Loss: 1.941310
Train Epoch: 6 [43520/50000 (87%)]	Loss: 1.984228
Train Epoch: 6 [44160/50000 (88%)]	Loss: 1.632908
Train Epoch: 6 [44800/50000 (90%)]	Loss: 1.792576
Train Epoch: 6 [45440/50000 (91%)]	Loss: 1.765507
Train Epoch: 6 [46080/50000 (92%)]	Loss: 2.013265
Train Epoch: 6 [46720/50000 (93%)]	Loss: 2.020439
Train Epoch: 6 [47360/50000 (95%)]	Loss: 1.821317
Train Epoch: 6 [48000/50000 (96%)]	Loss: 1.898109
Train Epoch: 6 [48640/50000 (97%)]	Loss: 2.113237
Train Epoch: 6 [49280/50000 (98%)]	Loss: 1.929869
Train Epoch: 6 [49920/50000 (100%)]	Loss: 1.891492

Test set: Average loss: 1.8185, Accuracy: 3204/10000 (32%)

Train Epoch: 7 [0/50000 (0%)]	Loss: 1.875387
Train Epoch: 7 [640/50000 (1%)]	Loss: 1.689091
Train Epoch: 7 [1280/50000 (3%)]	Loss: 1.788306
Train Epoch: 7 [1920/50000 (4%)]	Loss: 1.804733
Train Epoch: 7 [2560/50000 (5%)]	Loss: 1.852433
Train Epoch: 7 [3200/50000 (6%)]	Loss: 1.787824
Train Epoch: 7 [3840/50000 (8%)]	Loss: 1.841103
Train Epoch: 7 [4480/50000 (9%)]	Loss: 1.933343
Train Epoch: 7 [5120/50000 (10%)]	Loss: 1.792409
Train Epoch: 7 [5760/50000 (12%)]	Loss: 1.864723
Train Epoch: 7 [6400/50000 (13%)]	Loss: 1.634658
Train Epoch: 7 [7040/50000 (14%)]	Loss: 1.747776
Train Epoch: 7 [7680/50000 (15%)]	Loss: 1.965838
Train Epoch: 7 [8320/50000 (17%)]	Loss: 1.787119
Train Epoch: 7 [8960/50000 (18%)]	Loss: 1.816915
Train Epoch: 7 [9600/50000 (19%)]	Loss: 1.950240
Train Epoch: 7 [10240/50000 (20%)]	Loss: 1.799311
Train Epoch: 7 [10880/50000 (22%)]	Loss: 1.906010
Train Epoch: 7 [11520/50000 (23%)]	Loss: 1.706494
Train Epoch: 7 [12160/50000 (24%)]	Loss: 1.883644
Train Epoch: 7 [12800/50000 (26%)]	Loss: 1.953516
Train Epoch: 7 [13440/50000 (27%)]	Loss: 1.954202
Train Epoch: 7 [14080/50000 (28%)]	Loss: 1.862446
Train Epoch: 7 [14720/50000 (29%)]	Loss: 1.858408
Train Epoch: 7 [15360/50000 (31%)]	Loss: 1.947938
Train Epoch: 7 [16000/50000 (32%)]	Loss: 1.827278
Train Epoch: 7 [16640/50000 (33%)]	Loss: 1.855876
Train Epoch: 7 [17280/50000 (35%)]	Loss: 1.932165
Train Epoch: 7 [17920/50000 (36%)]	Loss: 1.937101
Train Epoch: 7 [18560/50000 (37%)]	Loss: 1.981404
Train Epoch: 7 [19200/50000 (38%)]	Loss: 1.723576
Train Epoch: 7 [19840/50000 (40%)]	Loss: 1.766682
Train Epoch: 7 [20480/50000 (41%)]	Loss: 1.952053
Train Epoch: 7 [21120/50000 (42%)]	Loss: 1.814618
Train Epoch: 7 [21760/50000 (43%)]	Loss: 1.814520
Train Epoch: 7 [22400/50000 (45%)]	Loss: 1.816713
Train Epoch: 7 [23040/50000 (46%)]	Loss: 1.807306
Train Epoch: 7 [23680/50000 (47%)]	Loss: 1.802528
Train Epoch: 7 [24320/50000 (49%)]	Loss: 1.845179
Train Epoch: 7 [24960/50000 (50%)]	Loss: 1.810762
Train Epoch: 7 [25600/50000 (51%)]	Loss: 1.869007
Train Epoch: 7 [26240/50000 (52%)]	Loss: 1.863111
Train Epoch: 7 [26880/50000 (54%)]	Loss: 2.098212
Train Epoch: 7 [27520/50000 (55%)]	Loss: 1.905432
Train Epoch: 7 [28160/50000 (56%)]	Loss: 2.109703
Train Epoch: 7 [28800/50000 (58%)]	Loss: 1.902633
Train Epoch: 7 [29440/50000 (59%)]	Loss: 1.818068
Train Epoch: 7 [30080/50000 (60%)]	Loss: 2.162695
Train Epoch: 7 [30720/50000 (61%)]	Loss: 1.910322
Train Epoch: 7 [31360/50000 (63%)]	Loss: 1.769494
Train Epoch: 7 [32000/50000 (64%)]	Loss: 1.795508
Train Epoch: 7 [32640/50000 (65%)]	Loss: 1.656080
Train Epoch: 7 [33280/50000 (66%)]	Loss: 2.076504
Train Epoch: 7 [33920/50000 (68%)]	Loss: 1.818579
Train Epoch: 7 [34560/50000 (69%)]	Loss: 1.842247
Train Epoch: 7 [35200/50000 (70%)]	Loss: 1.779503
Train Epoch: 7 [35840/50000 (72%)]	Loss: 1.775016
Train Epoch: 7 [36480/50000 (73%)]	Loss: 1.710795
Train Epoch: 7 [37120/50000 (74%)]	Loss: 1.879875
Train Epoch: 7 [37760/50000 (75%)]	Loss: 1.800777
Train Epoch: 7 [38400/50000 (77%)]	Loss: 1.970764
Train Epoch: 7 [39040/50000 (78%)]	Loss: 1.656710
Train Epoch: 7 [39680/50000 (79%)]	Loss: 1.943630
Train Epoch: 7 [40320/50000 (81%)]	Loss: 1.923133
Train Epoch: 7 [40960/50000 (82%)]	Loss: 1.774903
Train Epoch: 7 [41600/50000 (83%)]	Loss: 1.837117
Train Epoch: 7 [42240/50000 (84%)]	Loss: 1.909529
Train Epoch: 7 [42880/50000 (86%)]	Loss: 1.974733
Train Epoch: 7 [43520/50000 (87%)]	Loss: 1.789310
Train Epoch: 7 [44160/50000 (88%)]	Loss: 1.873711
Train Epoch: 7 [44800/50000 (90%)]	Loss: 1.854284
Train Epoch: 7 [45440/50000 (91%)]	Loss: 1.578808
Train Epoch: 7 [46080/50000 (92%)]	Loss: 1.907542
Train Epoch: 7 [46720/50000 (93%)]	Loss: 2.025553
Train Epoch: 7 [47360/50000 (95%)]	Loss: 1.751935
Train Epoch: 7 [48000/50000 (96%)]	Loss: 1.940979
Train Epoch: 7 [48640/50000 (97%)]	Loss: 1.664212
Train Epoch: 7 [49280/50000 (98%)]	Loss: 1.807063
Train Epoch: 7 [49920/50000 (100%)]	Loss: 1.823812

Test set: Average loss: 1.8604, Accuracy: 3124/10000 (31%)

Train Epoch: 8 [0/50000 (0%)]	Loss: 2.016686
Train Epoch: 8 [640/50000 (1%)]	Loss: 1.808914
Train Epoch: 8 [1280/50000 (3%)]	Loss: 1.995749
Train Epoch: 8 [1920/50000 (4%)]	Loss: 1.785702
Train Epoch: 8 [2560/50000 (5%)]	Loss: 1.997061
Train Epoch: 8 [3200/50000 (6%)]	Loss: 1.756337
Train Epoch: 8 [3840/50000 (8%)]	Loss: 1.991360
Train Epoch: 8 [4480/50000 (9%)]	Loss: 1.911734
Train Epoch: 8 [5120/50000 (10%)]	Loss: 1.757176
Train Epoch: 8 [5760/50000 (12%)]	Loss: 1.798932
Train Epoch: 8 [6400/50000 (13%)]	Loss: 1.967228
Train Epoch: 8 [7040/50000 (14%)]	Loss: 1.960473
Train Epoch: 8 [7680/50000 (15%)]	Loss: 1.719272
Train Epoch: 8 [8320/50000 (17%)]	Loss: 1.951201
Train Epoch: 8 [8960/50000 (18%)]	Loss: 1.692226
Train Epoch: 8 [9600/50000 (19%)]	Loss: 1.811001
Train Epoch: 8 [10240/50000 (20%)]	Loss: 1.700245
Train Epoch: 8 [10880/50000 (22%)]	Loss: 1.890026
Train Epoch: 8 [11520/50000 (23%)]	Loss: 1.962237
Train Epoch: 8 [12160/50000 (24%)]	Loss: 1.840496
Train Epoch: 8 [12800/50000 (26%)]	Loss: 1.855527
Train Epoch: 8 [13440/50000 (27%)]	Loss: 1.632574
Train Epoch: 8 [14080/50000 (28%)]	Loss: 1.793266
Train Epoch: 8 [14720/50000 (29%)]	Loss: 2.025056
Train Epoch: 8 [15360/50000 (31%)]	Loss: 1.692852
Train Epoch: 8 [16000/50000 (32%)]	Loss: 1.849669
Train Epoch: 8 [16640/50000 (33%)]	Loss: 1.954471
Train Epoch: 8 [17280/50000 (35%)]	Loss: 1.995475
Train Epoch: 8 [17920/50000 (36%)]	Loss: 1.842068
Train Epoch: 8 [18560/50000 (37%)]	Loss: 1.958124
Train Epoch: 8 [19200/50000 (38%)]	Loss: 2.070578
Train Epoch: 8 [19840/50000 (40%)]	Loss: 1.886619
Train Epoch: 8 [20480/50000 (41%)]	Loss: 1.930319
Train Epoch: 8 [21120/50000 (42%)]	Loss: 1.928146
Train Epoch: 8 [21760/50000 (43%)]	Loss: 2.049069
Train Epoch: 8 [22400/50000 (45%)]	Loss: 1.838538
Train Epoch: 8 [23040/50000 (46%)]	Loss: 1.850669
Train Epoch: 8 [23680/50000 (47%)]	Loss: 2.052543
Train Epoch: 8 [24320/50000 (49%)]	Loss: 1.781055
Train Epoch: 8 [24960/50000 (50%)]	Loss: 1.734172
Train Epoch: 8 [25600/50000 (51%)]	Loss: 1.924556
Train Epoch: 8 [26240/50000 (52%)]	Loss: 1.938823
Train Epoch: 8 [26880/50000 (54%)]	Loss: 1.984945
Train Epoch: 8 [27520/50000 (55%)]	Loss: 1.939510
Train Epoch: 8 [28160/50000 (56%)]	Loss: 2.117128
Train Epoch: 8 [28800/50000 (58%)]	Loss: 1.972970
Train Epoch: 8 [29440/50000 (59%)]	Loss: 1.825222
Train Epoch: 8 [30080/50000 (60%)]	Loss: 1.654083
Train Epoch: 8 [30720/50000 (61%)]	Loss: 1.982627
Train Epoch: 8 [31360/50000 (63%)]	Loss: 2.037522
Train Epoch: 8 [32000/50000 (64%)]	Loss: 2.042799
Train Epoch: 8 [32640/50000 (65%)]	Loss: 1.868315
Train Epoch: 8 [33280/50000 (66%)]	Loss: 1.792545
Train Epoch: 8 [33920/50000 (68%)]	Loss: 1.826818
Train Epoch: 8 [34560/50000 (69%)]	Loss: 1.827181
Train Epoch: 8 [35200/50000 (70%)]	Loss: 1.687776
Train Epoch: 8 [35840/50000 (72%)]	Loss: 1.743180
Train Epoch: 8 [36480/50000 (73%)]	Loss: 1.839710
Train Epoch: 8 [37120/50000 (74%)]	Loss: 1.822514
Train Epoch: 8 [37760/50000 (75%)]	Loss: 1.743397
Train Epoch: 8 [38400/50000 (77%)]	Loss: 1.675166
Train Epoch: 8 [39040/50000 (78%)]	Loss: 1.817261
Train Epoch: 8 [39680/50000 (79%)]	Loss: 1.701076
Train Epoch: 8 [40320/50000 (81%)]	Loss: 1.783767
Train Epoch: 8 [40960/50000 (82%)]	Loss: 1.753060
Train Epoch: 8 [41600/50000 (83%)]	Loss: 1.761490
Train Epoch: 8 [42240/50000 (84%)]	Loss: 1.990638
Train Epoch: 8 [42880/50000 (86%)]	Loss: 1.903932
Train Epoch: 8 [43520/50000 (87%)]	Loss: 1.868271
Train Epoch: 8 [44160/50000 (88%)]	Loss: 1.701351
Train Epoch: 8 [44800/50000 (90%)]	Loss: 1.797975
Train Epoch: 8 [45440/50000 (91%)]	Loss: 1.832913
Train Epoch: 8 [46080/50000 (92%)]	Loss: 1.774905
Train Epoch: 8 [46720/50000 (93%)]	Loss: 1.967483
Train Epoch: 8 [47360/50000 (95%)]	Loss: 2.458667
Train Epoch: 8 [48000/50000 (96%)]	Loss: 2.058775
Train Epoch: 8 [48640/50000 (97%)]	Loss: 1.939025
Train Epoch: 8 [49280/50000 (98%)]	Loss: 1.785460
Train Epoch: 8 [49920/50000 (100%)]	Loss: 1.861622

Test set: Average loss: 1.6645, Accuracy: 3800/10000 (38%)

Train Epoch: 9 [0/50000 (0%)]	Loss: 1.857960
Train Epoch: 9 [640/50000 (1%)]	Loss: 1.937248
Train Epoch: 9 [1280/50000 (3%)]	Loss: 1.872403
Train Epoch: 9 [1920/50000 (4%)]	Loss: 1.971959
Train Epoch: 9 [2560/50000 (5%)]	Loss: 1.645404
Train Epoch: 9 [3200/50000 (6%)]	Loss: 2.067047
Train Epoch: 9 [3840/50000 (8%)]	Loss: 1.863987
Train Epoch: 9 [4480/50000 (9%)]	Loss: 1.705298
Train Epoch: 9 [5120/50000 (10%)]	Loss: 1.758719
Train Epoch: 9 [5760/50000 (12%)]	Loss: 1.871931
Train Epoch: 9 [6400/50000 (13%)]	Loss: 1.628905
Train Epoch: 9 [7040/50000 (14%)]	Loss: 1.905302
Train Epoch: 9 [7680/50000 (15%)]	Loss: 1.987644
Train Epoch: 9 [8320/50000 (17%)]	Loss: 1.818435
Train Epoch: 9 [8960/50000 (18%)]	Loss: 1.704053
Train Epoch: 9 [9600/50000 (19%)]	Loss: 1.716398
Train Epoch: 9 [10240/50000 (20%)]	Loss: 1.879060
Train Epoch: 9 [10880/50000 (22%)]	Loss: 1.865947
Train Epoch: 9 [11520/50000 (23%)]	Loss: 1.846159
Train Epoch: 9 [12160/50000 (24%)]	Loss: 1.768230
Train Epoch: 9 [12800/50000 (26%)]	Loss: 1.904954
Train Epoch: 9 [13440/50000 (27%)]	Loss: 2.020458
Train Epoch: 9 [14080/50000 (28%)]	Loss: 1.847152
Train Epoch: 9 [14720/50000 (29%)]	Loss: 2.226343
Train Epoch: 9 [15360/50000 (31%)]	Loss: 1.648950
Train Epoch: 9 [16000/50000 (32%)]	Loss: 2.024352
Train Epoch: 9 [16640/50000 (33%)]	Loss: 1.712605
Train Epoch: 9 [17280/50000 (35%)]	Loss: 1.888452
Train Epoch: 9 [17920/50000 (36%)]	Loss: 1.963849
Train Epoch: 9 [18560/50000 (37%)]	Loss: 1.776592
Train Epoch: 9 [19200/50000 (38%)]	Loss: 1.883823
Train Epoch: 9 [19840/50000 (40%)]	Loss: 1.803857
Train Epoch: 9 [20480/50000 (41%)]	Loss: 1.788994
Train Epoch: 9 [21120/50000 (42%)]	Loss: 1.772452
Train Epoch: 9 [21760/50000 (43%)]	Loss: 1.719890
Train Epoch: 9 [22400/50000 (45%)]	Loss: 1.763235
Train Epoch: 9 [23040/50000 (46%)]	Loss: 1.668816
Train Epoch: 9 [23680/50000 (47%)]	Loss: 1.660076
Train Epoch: 9 [24320/50000 (49%)]	Loss: 1.736899
Train Epoch: 9 [24960/50000 (50%)]	Loss: 1.794548
Train Epoch: 9 [25600/50000 (51%)]	Loss: 1.918256
Train Epoch: 9 [26240/50000 (52%)]	Loss: 1.809370
Train Epoch: 9 [26880/50000 (54%)]	Loss: 1.795120
Train Epoch: 9 [27520/50000 (55%)]	Loss: 1.860161
Train Epoch: 9 [28160/50000 (56%)]	Loss: 2.085226
Train Epoch: 9 [28800/50000 (58%)]	Loss: 1.619069
Train Epoch: 9 [29440/50000 (59%)]	Loss: 1.968214
Train Epoch: 9 [30080/50000 (60%)]	Loss: 1.947108
Train Epoch: 9 [30720/50000 (61%)]	Loss: 1.768680
Train Epoch: 9 [31360/50000 (63%)]	Loss: 1.610175
Train Epoch: 9 [32000/50000 (64%)]	Loss: 1.836892
Train Epoch: 9 [32640/50000 (65%)]	Loss: 2.055267
Train Epoch: 9 [33280/50000 (66%)]	Loss: 1.863169
Train Epoch: 9 [33920/50000 (68%)]	Loss: 1.825007
Train Epoch: 9 [34560/50000 (69%)]	Loss: 1.973548
Train Epoch: 9 [35200/50000 (70%)]	Loss: 2.093603
Train Epoch: 9 [35840/50000 (72%)]	Loss: 1.822404
Train Epoch: 9 [36480/50000 (73%)]	Loss: 2.103255
Train Epoch: 9 [37120/50000 (74%)]	Loss: 1.713474
Train Epoch: 9 [37760/50000 (75%)]	Loss: 1.893207
Train Epoch: 9 [38400/50000 (77%)]	Loss: 1.849712
Train Epoch: 9 [39040/50000 (78%)]	Loss: 1.916312
Train Epoch: 9 [39680/50000 (79%)]	Loss: 1.777981
Train Epoch: 9 [40320/50000 (81%)]	Loss: 1.911252
Train Epoch: 9 [40960/50000 (82%)]	Loss: 1.567620
Train Epoch: 9 [41600/50000 (83%)]	Loss: 1.806095
Train Epoch: 9 [42240/50000 (84%)]	Loss: 1.814230
Train Epoch: 9 [42880/50000 (86%)]	Loss: 1.630254
Train Epoch: 9 [43520/50000 (87%)]	Loss: 1.807136
Train Epoch: 9 [44160/50000 (88%)]	Loss: 1.824325
Train Epoch: 9 [44800/50000 (90%)]	Loss: 1.659183
Train Epoch: 9 [45440/50000 (91%)]	Loss: 1.609033
Train Epoch: 9 [46080/50000 (92%)]	Loss: 2.000914
Train Epoch: 9 [46720/50000 (93%)]	Loss: 1.740623
Train Epoch: 9 [47360/50000 (95%)]	Loss: 1.963995
Train Epoch: 9 [48000/50000 (96%)]	Loss: 1.759796
Train Epoch: 9 [48640/50000 (97%)]	Loss: 1.737803
Train Epoch: 9 [49280/50000 (98%)]	Loss: 1.991658
Train Epoch: 9 [49920/50000 (100%)]	Loss: 1.860707

Test set: Average loss: 1.7431, Accuracy: 3554/10000 (36%)

Train Epoch: 10 [0/50000 (0%)]	Loss: 1.822902
Train Epoch: 10 [640/50000 (1%)]	Loss: 1.865318
Train Epoch: 10 [1280/50000 (3%)]	Loss: 1.889241
Train Epoch: 10 [1920/50000 (4%)]	Loss: 2.015584
Train Epoch: 10 [2560/50000 (5%)]	Loss: 2.038193
Train Epoch: 10 [3200/50000 (6%)]	Loss: 1.857823
Train Epoch: 10 [3840/50000 (8%)]	Loss: 1.827553
Train Epoch: 10 [4480/50000 (9%)]	Loss: 1.784337
Train Epoch: 10 [5120/50000 (10%)]	Loss: 1.688781
Train Epoch: 10 [5760/50000 (12%)]	Loss: 1.697384
Train Epoch: 10 [6400/50000 (13%)]	Loss: 1.820456
Train Epoch: 10 [7040/50000 (14%)]	Loss: 2.083695
Train Epoch: 10 [7680/50000 (15%)]	Loss: 1.819066
Train Epoch: 10 [8320/50000 (17%)]	Loss: 1.934127
Train Epoch: 10 [8960/50000 (18%)]	Loss: 2.033443
Train Epoch: 10 [9600/50000 (19%)]	Loss: 1.878930
Train Epoch: 10 [10240/50000 (20%)]	Loss: 1.545819
Train Epoch: 10 [10880/50000 (22%)]	Loss: 2.014779
Train Epoch: 10 [11520/50000 (23%)]	Loss: 1.816486
Train Epoch: 10 [12160/50000 (24%)]	Loss: 1.871381
Train Epoch: 10 [12800/50000 (26%)]	Loss: 1.786618
Train Epoch: 10 [13440/50000 (27%)]	Loss: 1.843771
Train Epoch: 10 [14080/50000 (28%)]	Loss: 1.710896
Train Epoch: 10 [14720/50000 (29%)]	Loss: 1.807587
Train Epoch: 10 [15360/50000 (31%)]	Loss: 1.720309
Train Epoch: 10 [16000/50000 (32%)]	Loss: 2.079888
Train Epoch: 10 [16640/50000 (33%)]	Loss: 1.665985
Train Epoch: 10 [17280/50000 (35%)]	Loss: 1.661451
Train Epoch: 10 [17920/50000 (36%)]	Loss: 1.623413
Train Epoch: 10 [18560/50000 (37%)]	Loss: 1.609192
Train Epoch: 10 [19200/50000 (38%)]	Loss: 2.056215
Train Epoch: 10 [19840/50000 (40%)]	Loss: 1.777434
Train Epoch: 10 [20480/50000 (41%)]	Loss: 1.794601
Train Epoch: 10 [21120/50000 (42%)]	Loss: 1.767932
Train Epoch: 10 [21760/50000 (43%)]	Loss: 1.631500
Train Epoch: 10 [22400/50000 (45%)]	Loss: 1.781289
Train Epoch: 10 [23040/50000 (46%)]	Loss: 1.775690
Train Epoch: 10 [23680/50000 (47%)]	Loss: 1.876030
Train Epoch: 10 [24320/50000 (49%)]	Loss: 1.749646
Train Epoch: 10 [24960/50000 (50%)]	Loss: 1.775127
Train Epoch: 10 [25600/50000 (51%)]	Loss: 1.652194
Train Epoch: 10 [26240/50000 (52%)]	Loss: 1.623289
Train Epoch: 10 [26880/50000 (54%)]	Loss: 1.728025
Train Epoch: 10 [27520/50000 (55%)]	Loss: 1.953737
Train Epoch: 10 [28160/50000 (56%)]	Loss: 2.010338
Train Epoch: 10 [28800/50000 (58%)]	Loss: 1.710788
Train Epoch: 10 [29440/50000 (59%)]	Loss: 1.834922
Train Epoch: 10 [30080/50000 (60%)]	Loss: 2.031476
Train Epoch: 10 [30720/50000 (61%)]	Loss: 1.656871
Train Epoch: 10 [31360/50000 (63%)]	Loss: 1.771416
Train Epoch: 10 [32000/50000 (64%)]	Loss: 1.762366
Train Epoch: 10 [32640/50000 (65%)]	Loss: 1.791673
Train Epoch: 10 [33280/50000 (66%)]	Loss: 1.606878
Train Epoch: 10 [33920/50000 (68%)]	Loss: 1.988305
Train Epoch: 10 [34560/50000 (69%)]	Loss: 1.822009
Train Epoch: 10 [35200/50000 (70%)]	Loss: 1.722441
Train Epoch: 10 [35840/50000 (72%)]	Loss: 2.008125
Train Epoch: 10 [36480/50000 (73%)]	Loss: 2.039276
Train Epoch: 10 [37120/50000 (74%)]	Loss: 1.841367
Train Epoch: 10 [37760/50000 (75%)]	Loss: 1.610676
Train Epoch: 10 [38400/50000 (77%)]	Loss: 1.855071
Train Epoch: 10 [39040/50000 (78%)]	Loss: 2.065572
Train Epoch: 10 [39680/50000 (79%)]	Loss: 1.650647
Train Epoch: 10 [40320/50000 (81%)]	Loss: 1.644333
Train Epoch: 10 [40960/50000 (82%)]	Loss: 1.842960
Train Epoch: 10 [41600/50000 (83%)]	Loss: 1.881915
Train Epoch: 10 [42240/50000 (84%)]	Loss: 1.756154
Train Epoch: 10 [42880/50000 (86%)]	Loss: 1.809470
Train Epoch: 10 [43520/50000 (87%)]	Loss: 2.044989
Train Epoch: 10 [44160/50000 (88%)]	Loss: 1.895585
Train Epoch: 10 [44800/50000 (90%)]	Loss: 1.621321
Train Epoch: 10 [45440/50000 (91%)]	Loss: 1.862957
Train Epoch: 10 [46080/50000 (92%)]	Loss: 1.735436
Train Epoch: 10 [46720/50000 (93%)]	Loss: 1.634214
Train Epoch: 10 [47360/50000 (95%)]	Loss: 1.750990
Train Epoch: 10 [48000/50000 (96%)]	Loss: 1.548210
Train Epoch: 10 [48640/50000 (97%)]	Loss: 1.894869
Train Epoch: 10 [49280/50000 (98%)]	Loss: 1.744969
Train Epoch: 10 [49920/50000 (100%)]	Loss: 1.744793

Test set: Average loss: 1.7865, Accuracy: 3403/10000 (34%)







    













    













    













    




<class 'torch.FloatTensor'> torch.Size([6, 3, 5, 5])
<class 'torch.FloatTensor'> torch.Size([3, 14, 21])

In [14]:

    

# with Dropout
# with Weight initialization
# with Batch normalization


from __future__ import print_function
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms
from torch.autograd import Variable

import math
import numpy as np
import matplotlib.pyplot as plt
from torchvision.utils import make_grid
from collections import OrderedDict

# Training settings
# for terminal use. In notebook, you can't parse arguments
class args:
    cuda = False
    batch_size = 64
    test_batch_size = 1000
    epochs = 10
    lr = 0.01
    momentum = 0.5
    no_cuda = False
    seed = 1
    log_interval = 10
    # if add Dropout
    with_dropout = True
    # if initialize weights
    with_init_weights = True
    # if add BatchNorm
    with_batchnorm = True

args.cuda = not args.no_cuda and torch.cuda.is_available()

torch.manual_seed(args.seed)
if args.cuda:
    torch.cuda.manual_seed(args.seed)

kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {}
# The output of torchvision datasets are PILImage images of range [0, 1].
# We transform them to Tensors of normalized range [-1, 1]
transform = transforms.Compose([transforms.ToTensor(),
                              transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
                             ])
trainset = datasets.CIFAR10(root='./data', train=True, download=True, transform=transform)
train_loader = torch.utils.data.DataLoader(trainset, batch_size=args.batch_size,
                                          shuffle=True, num_workers=2)

testset = datasets.CIFAR10(root='./data', train=False, download=True, transform=transform)
test_loader = torch.utils.data.DataLoader(testset, batch_size=args.test_batch_size,
                                          shuffle=False, num_workers=2)
classes = ('plane', 'car', 'bird', 'cat',
           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')


class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        # TODO: define your network here
        self.conv_1 = nn.Conv2d(3, 6, kernel_size=5, stride=1)
        self.conv_2 = nn.Conv2d(6, 16, kernel_size=5, stride=1)
        # TODO: replace fc with conv
        self.fc_1 = nn.Linear(16 * 25, 120)
        self.fc_2 = nn.Linear(120, 84)
        self.fc_3 = nn.Linear(84, 10)
        if args.with_batchnorm:
            self.block_conv_1 = nn.Sequential(
                self.conv_1,
                nn.BatchNorm2d(6),
                nn.ReLU(),
                nn.MaxPool2d(kernel_size=2, stride=2)
            )
            self.block_conv_2 = nn.Sequential(
                self.conv_2,
                nn.BatchNorm2d(16),
                nn.ReLU(),
                nn.MaxPool2d(kernel_size=2, stride=2)
            )
        else:
            self.block_conv_1 = nn.Sequential(
                self.conv_1,
                nn.ReLU(),
                nn.MaxPool2d(kernel_size=2, stride=2)
            )
            self.block_conv_2 = nn.Sequential(
                self.conv_2,
                nn.ReLU(),
                nn.MaxPool2d(kernel_size=2, stride=2)
            )
        if args.with_dropout:
            if args.with_batchnorm:
                self.block_fc_1 = nn.Sequential(
                    self.fc_1,
                    nn.BatchNorm1d(120),
                    nn.Dropout()
                )
                self.block_fc_2 = nn.Sequential(
                    self.fc_2,
                    nn.BatchNorm1d(84),
                    nn.Dropout()
                )
            else:
                self.block_fc_1 = nn.Sequential(
                    self.fc_1,
                    nn.Dropout()
                )
                self.block_fc_2 = nn.Sequential(
                    self.fc_2,
                    nn.Dropout()
                )
        else:
            self.block_fc_1 = self.fc_1
            self.block_fc_2 = self.fc_2
        self.softmax = nn.LogSoftmax()
        # Initialize parameters
        if args.with_init_weights:
            for m in self.modules():
                if isinstance(m, nn.Conv2d):
                    n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                    m.weight.data.normal_(0, math.sqrt(2. /n))
                    if m.bias is not None:
                        m.bias.data.zero_()
                elif isinstance(m, nn.Linear):
                    n = m.out_features
                    m.weight.data.normal_(0, math.sqrt(2. /n))
                    if m.bias is not None:
                        m.bias.data.zero_()
                elif isinstance(m, nn.BatchNorm1d) or isinstance(m, nn.BatchNorm2d):
                    m.weight.data.fill_(1)
                    m.bias.data.zero_()

    def forward(self, x):
        # TODO
        x = self.block_conv_1(x)
        x = self.block_conv_2(x)
        x = x.view(x.size(0), -1)
        x = self.block_fc_1(x)
        x = self.block_fc_2(x)
        x = self.fc_3(x)
        x = self.softmax(x)
        return x

# Filter extractor for filter visualization
class FeatureExtractor(nn.Module):
    def __init__(self, model, layer_names):
        super(FeatureExtractor, self).__init__()
        self._model = model
        self._layer_names = set(layer_names)
    def forward(self):
        filters = dict()
        # _modules is an OrderedDict, which replace iteritems() with items() in python3.* 
        for name, module in self._model._modules.items():
            if name in self._layer_names:
                filters[name] = module.weight.data.cpu()
        return filters

# Vesualize training results and trained filters
class VisualizedResult():
    def __init__(self, model):
        self._model = model
    def training_curve(self, epoches, train_loss_records, test_loss_records):
        fig = plt.figure()
        ax_train = fig.add_subplot(111)
        ax_test = fig.add_subplot(111)
        plt.axis([1, epoches, 0, math.ceil(max(train_loss_records + test_loss_records) * 1.2)])
        plt.xlabel('Epoches')
        plt.ylabel('Loss')
        plt.title('Training Curve')
        plt.plot(range(1, epoches + 1), train_loss_records, 'b-', label="train loss")
        plt.plot(range(1, epoches + 1), test_loss_records, 'r-', label="test loss")
        for xy in zip(range(1, epoches + 1), train_loss_records):
            ax_train.annotate('%.2f' % xy[1], xy=xy, textcoords='data')
        for xy in zip(range(1, epoches + 1), test_loss_records):
            ax_test.annotate('%.2f' % xy[1], xy=xy, textcoords='data')
        plt.legend(loc='upper right', borderaxespad=0.)
        plt.show()
    def accuracy_curve(self, epoches, accuracy_records):
        fig = plt.figure()
        ax = fig.add_subplot(111)
        plt.axis([1, epoches, 0, 100])
        plt.xlabel('Epoches')
        plt.ylabel('Accuracy')
        plt.title('Accuracy Curve')
        plt.plot(range(1, epoches + 1), accuracy_records, '-')
        for xy in zip(range(1, epoches + 1), accuracy_records):
            ax.annotate('%s%%' % xy[1], xy=xy, textcoords='data')
        plt.show()
    def conv_filter(self, layer_names):
        feature_extractor = FeatureExtractor(self._model, layer_names)
        filter_list = feature_extractor.forward()
        for name, filters in filter_list.items():
            print(type(filters), filters.size())
            # Normalize filters
            mean = torch.mean(filters)
            std = torch.std(filters)
            filters = (filters - mean) / std
            img = make_grid(filters, nrow=3)
            print(type(img), img.size())
            npimg = img.numpy()
            plt.imshow(np.transpose(npimg, (1,2,0)), interpolation='nearest')


model = Net()
if args.cuda:
    model.cuda()
    
# TODO: other optimizers
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum)

train_loss_records = list()
test_loss_records = list()
accuracy_records = list()

def train(epoch):
    model.train()
    train_loss = 0
    for batch_idx, (data, target) in enumerate(train_loader):
        if args.cuda:
            data, target = data.cuda(), target.cuda()
        data, target = Variable(data), Variable(target)
        optimizer.zero_grad()
        output = model(data)
        loss = F.nll_loss(output, target)   # is it true to use such a loss over cross-entropy loss? 
        loss.backward()
        optimizer.step()
        train_loss += loss.data[0]
        if batch_idx % args.log_interval == 0:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch, batch_idx * len(data), len(train_loader.dataset),
                100. * batch_idx / len(train_loader), loss.data[0]))
    # Average training loss for this epoch
    train_loss_records.append(train_loss / len(train_loader))

def test(epoch):
    model.eval()
    test_loss = 0
    correct = 0
    for data, target in test_loader:
        if args.cuda:
            data, target = data.cuda(), target.cuda()
        data, target = Variable(data, volatile=True), Variable(target)
        output = model(data)
        test_loss += F.nll_loss(output, target).data[0]
        pred = output.data.max(1)[1] # get the index of the max log-probability
        correct += pred.eq(target.data).cpu().sum()

    test_loss = test_loss
    test_loss /= len(test_loader) # loss function already averages over batch size
    accuracy = 100. * correct / len(test_loader.dataset)
    print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
        test_loss, correct, len(test_loader.dataset),
        accuracy))
    test_loss_records.append(test_loss)
    accuracy_records.append(accuracy)


for epoch in range(1, args.epochs + 1):
    train(epoch)
    test(epoch)

visual_result = VisualizedResult(model)
# Visualize training curve
visual_result.training_curve(args.epochs, train_loss_records, test_loss_records)
# Visualize test accuracy
visual_result.accuracy_curve(args.epochs, accuracy_records)
# Visualize trained filter on the 1st Conv layer
visual_result.conv_filter(['conv_1'])


    

    




Files already downloaded and verified
Files already downloaded and verified
Train Epoch: 1 [0/50000 (0%)]	Loss: 8.153704
Train Epoch: 1 [640/50000 (1%)]	Loss: 7.417312
Train Epoch: 1 [1280/50000 (3%)]	Loss: 7.811161
Train Epoch: 1 [1920/50000 (4%)]	Loss: 4.838050
Train Epoch: 1 [2560/50000 (5%)]	Loss: 6.282025
Train Epoch: 1 [3200/50000 (6%)]	Loss: 5.060748
Train Epoch: 1 [3840/50000 (8%)]	Loss: 5.281744
Train Epoch: 1 [4480/50000 (9%)]	Loss: 5.198488
Train Epoch: 1 [5120/50000 (10%)]	Loss: 4.432251
Train Epoch: 1 [5760/50000 (12%)]	Loss: 4.908093
Train Epoch: 1 [6400/50000 (13%)]	Loss: 4.156298
Train Epoch: 1 [7040/50000 (14%)]	Loss: 4.509658
Train Epoch: 1 [7680/50000 (15%)]	Loss: 3.952303
Train Epoch: 1 [8320/50000 (17%)]	Loss: 4.032708
Train Epoch: 1 [8960/50000 (18%)]	Loss: 3.626411
Train Epoch: 1 [9600/50000 (19%)]	Loss: 4.341449
Train Epoch: 1 [10240/50000 (20%)]	Loss: 2.908090
Train Epoch: 1 [10880/50000 (22%)]	Loss: 4.053169
Train Epoch: 1 [11520/50000 (23%)]	Loss: 4.229597
Train Epoch: 1 [12160/50000 (24%)]	Loss: 3.179649
Train Epoch: 1 [12800/50000 (26%)]	Loss: 4.874126
Train Epoch: 1 [13440/50000 (27%)]	Loss: 2.987284
Train Epoch: 1 [14080/50000 (28%)]	Loss: 3.000915
Train Epoch: 1 [14720/50000 (29%)]	Loss: 3.115980
Train Epoch: 1 [15360/50000 (31%)]	Loss: 3.370491
Train Epoch: 1 [16000/50000 (32%)]	Loss: 2.874113
Train Epoch: 1 [16640/50000 (33%)]	Loss: 3.273332
Train Epoch: 1 [17280/50000 (35%)]	Loss: 2.813097
Train Epoch: 1 [17920/50000 (36%)]	Loss: 3.209005
Train Epoch: 1 [18560/50000 (37%)]	Loss: 2.667626
Train Epoch: 1 [19200/50000 (38%)]	Loss: 2.778348
Train Epoch: 1 [19840/50000 (40%)]	Loss: 2.699457
Train Epoch: 1 [20480/50000 (41%)]	Loss: 2.631280
Train Epoch: 1 [21120/50000 (42%)]	Loss: 3.155357
Train Epoch: 1 [21760/50000 (43%)]	Loss: 2.580922
Train Epoch: 1 [22400/50000 (45%)]	Loss: 2.995878
Train Epoch: 1 [23040/50000 (46%)]	Loss: 2.860363
Train Epoch: 1 [23680/50000 (47%)]	Loss: 2.614643
Train Epoch: 1 [24320/50000 (49%)]	Loss: 2.573584
Train Epoch: 1 [24960/50000 (50%)]	Loss: 2.763986
Train Epoch: 1 [25600/50000 (51%)]	Loss: 2.201443
Train Epoch: 1 [26240/50000 (52%)]	Loss: 2.911967
Train Epoch: 1 [26880/50000 (54%)]	Loss: 2.029391
Train Epoch: 1 [27520/50000 (55%)]	Loss: 2.641585
Train Epoch: 1 [28160/50000 (56%)]	Loss: 2.233691
Train Epoch: 1 [28800/50000 (58%)]	Loss: 2.627578
Train Epoch: 1 [29440/50000 (59%)]	Loss: 2.502747
Train Epoch: 1 [30080/50000 (60%)]	Loss: 2.331625
Train Epoch: 1 [30720/50000 (61%)]	Loss: 2.591181
Train Epoch: 1 [31360/50000 (63%)]	Loss: 2.169506
Train Epoch: 1 [32000/50000 (64%)]	Loss: 2.366725
Train Epoch: 1 [32640/50000 (65%)]	Loss: 2.168777
Train Epoch: 1 [33280/50000 (66%)]	Loss: 2.603544
Train Epoch: 1 [33920/50000 (68%)]	Loss: 2.484063
Train Epoch: 1 [34560/50000 (69%)]	Loss: 2.238873
Train Epoch: 1 [35200/50000 (70%)]	Loss: 2.212082
Train Epoch: 1 [35840/50000 (72%)]	Loss: 2.184109
Train Epoch: 1 [36480/50000 (73%)]	Loss: 2.448682
Train Epoch: 1 [37120/50000 (74%)]	Loss: 2.292032
Train Epoch: 1 [37760/50000 (75%)]	Loss: 2.287888
Train Epoch: 1 [38400/50000 (77%)]	Loss: 2.116733
Train Epoch: 1 [39040/50000 (78%)]	Loss: 2.401929
Train Epoch: 1 [39680/50000 (79%)]	Loss: 2.251363
Train Epoch: 1 [40320/50000 (81%)]	Loss: 2.303928
Train Epoch: 1 [40960/50000 (82%)]	Loss: 2.255766
Train Epoch: 1 [41600/50000 (83%)]	Loss: 2.167920
Train Epoch: 1 [42240/50000 (84%)]	Loss: 1.999991
Train Epoch: 1 [42880/50000 (86%)]	Loss: 1.946598
Train Epoch: 1 [43520/50000 (87%)]	Loss: 1.937252
Train Epoch: 1 [44160/50000 (88%)]	Loss: 2.261118
Train Epoch: 1 [44800/50000 (90%)]	Loss: 1.846943
Train Epoch: 1 [45440/50000 (91%)]	Loss: 1.769021
Train Epoch: 1 [46080/50000 (92%)]	Loss: 2.163312
Train Epoch: 1 [46720/50000 (93%)]	Loss: 2.116604
Train Epoch: 1 [47360/50000 (95%)]	Loss: 2.235409
Train Epoch: 1 [48000/50000 (96%)]	Loss: 2.041300
Train Epoch: 1 [48640/50000 (97%)]	Loss: 1.761087
Train Epoch: 1 [49280/50000 (98%)]	Loss: 2.080876
Train Epoch: 1 [49920/50000 (100%)]	Loss: 1.640381

Test set: Average loss: 1.5523, Accuracy: 4485/10000 (45%)

Train Epoch: 2 [0/50000 (0%)]	Loss: 1.939105
Train Epoch: 2 [640/50000 (1%)]	Loss: 2.095980
Train Epoch: 2 [1280/50000 (3%)]	Loss: 2.172967
Train Epoch: 2 [1920/50000 (4%)]	Loss: 1.863020
Train Epoch: 2 [2560/50000 (5%)]	Loss: 1.982612
Train Epoch: 2 [3200/50000 (6%)]	Loss: 1.843184
Train Epoch: 2 [3840/50000 (8%)]	Loss: 1.909437
Train Epoch: 2 [4480/50000 (9%)]	Loss: 1.994214
Train Epoch: 2 [5120/50000 (10%)]	Loss: 2.011549
Train Epoch: 2 [5760/50000 (12%)]	Loss: 1.784998
Train Epoch: 2 [6400/50000 (13%)]	Loss: 1.817919
Train Epoch: 2 [7040/50000 (14%)]	Loss: 1.968056
Train Epoch: 2 [7680/50000 (15%)]	Loss: 2.022481
Train Epoch: 2 [8320/50000 (17%)]	Loss: 2.004160
Train Epoch: 2 [8960/50000 (18%)]	Loss: 1.677201
Train Epoch: 2 [9600/50000 (19%)]	Loss: 1.622773
Train Epoch: 2 [10240/50000 (20%)]	Loss: 2.139123
Train Epoch: 2 [10880/50000 (22%)]	Loss: 1.724552
Train Epoch: 2 [11520/50000 (23%)]	Loss: 2.194668
Train Epoch: 2 [12160/50000 (24%)]	Loss: 1.586783
Train Epoch: 2 [12800/50000 (26%)]	Loss: 1.612674
Train Epoch: 2 [13440/50000 (27%)]	Loss: 1.972492
Train Epoch: 2 [14080/50000 (28%)]	Loss: 1.730421
Train Epoch: 2 [14720/50000 (29%)]	Loss: 1.929908
Train Epoch: 2 [15360/50000 (31%)]	Loss: 1.697481
Train Epoch: 2 [16000/50000 (32%)]	Loss: 1.780877
Train Epoch: 2 [16640/50000 (33%)]	Loss: 1.748237
Train Epoch: 2 [17280/50000 (35%)]	Loss: 1.685005
Train Epoch: 2 [17920/50000 (36%)]	Loss: 1.779256
Train Epoch: 2 [18560/50000 (37%)]	Loss: 1.899300
Train Epoch: 2 [19200/50000 (38%)]	Loss: 1.871775
Train Epoch: 2 [19840/50000 (40%)]	Loss: 2.011067
Train Epoch: 2 [20480/50000 (41%)]	Loss: 1.586965
Train Epoch: 2 [21120/50000 (42%)]	Loss: 1.800055
Train Epoch: 2 [21760/50000 (43%)]	Loss: 1.873072
Train Epoch: 2 [22400/50000 (45%)]	Loss: 1.503832
Train Epoch: 2 [23040/50000 (46%)]	Loss: 2.003897
Train Epoch: 2 [23680/50000 (47%)]	Loss: 1.784884
Train Epoch: 2 [24320/50000 (49%)]	Loss: 1.813379
Train Epoch: 2 [24960/50000 (50%)]	Loss: 2.040568
Train Epoch: 2 [25600/50000 (51%)]	Loss: 1.547716
Train Epoch: 2 [26240/50000 (52%)]	Loss: 1.894160
Train Epoch: 2 [26880/50000 (54%)]	Loss: 1.634351
Train Epoch: 2 [27520/50000 (55%)]	Loss: 1.881575
Train Epoch: 2 [28160/50000 (56%)]	Loss: 1.598440
Train Epoch: 2 [28800/50000 (58%)]	Loss: 1.842924
Train Epoch: 2 [29440/50000 (59%)]	Loss: 1.609902
Train Epoch: 2 [30080/50000 (60%)]	Loss: 1.812343
Train Epoch: 2 [30720/50000 (61%)]	Loss: 1.997521
Train Epoch: 2 [31360/50000 (63%)]	Loss: 1.803361
Train Epoch: 2 [32000/50000 (64%)]	Loss: 1.760522
Train Epoch: 2 [32640/50000 (65%)]	Loss: 1.614652
Train Epoch: 2 [33280/50000 (66%)]	Loss: 1.649145
Train Epoch: 2 [33920/50000 (68%)]	Loss: 1.724856
Train Epoch: 2 [34560/50000 (69%)]	Loss: 1.955176
Train Epoch: 2 [35200/50000 (70%)]	Loss: 1.700756
Train Epoch: 2 [35840/50000 (72%)]	Loss: 1.655390
Train Epoch: 2 [36480/50000 (73%)]	Loss: 1.951313
Train Epoch: 2 [37120/50000 (74%)]	Loss: 1.778405
Train Epoch: 2 [37760/50000 (75%)]	Loss: 1.594641
Train Epoch: 2 [38400/50000 (77%)]	Loss: 1.711459
Train Epoch: 2 [39040/50000 (78%)]	Loss: 1.536506
Train Epoch: 2 [39680/50000 (79%)]	Loss: 1.761292
Train Epoch: 2 [40320/50000 (81%)]	Loss: 1.619978
Train Epoch: 2 [40960/50000 (82%)]	Loss: 1.837531
Train Epoch: 2 [41600/50000 (83%)]	Loss: 1.584642
Train Epoch: 2 [42240/50000 (84%)]	Loss: 1.774028
Train Epoch: 2 [42880/50000 (86%)]	Loss: 1.939359
Train Epoch: 2 [43520/50000 (87%)]	Loss: 1.713571
Train Epoch: 2 [44160/50000 (88%)]	Loss: 1.681195
Train Epoch: 2 [44800/50000 (90%)]	Loss: 1.458692
Train Epoch: 2 [45440/50000 (91%)]	Loss: 1.654592
Train Epoch: 2 [46080/50000 (92%)]	Loss: 1.743903
Train Epoch: 2 [46720/50000 (93%)]	Loss: 1.486926
Train Epoch: 2 [47360/50000 (95%)]	Loss: 1.614772
Train Epoch: 2 [48000/50000 (96%)]	Loss: 1.773962
Train Epoch: 2 [48640/50000 (97%)]	Loss: 1.696918
Train Epoch: 2 [49280/50000 (98%)]	Loss: 1.602935
Train Epoch: 2 [49920/50000 (100%)]	Loss: 1.831171

Test set: Average loss: 1.5209, Accuracy: 4542/10000 (45%)

Train Epoch: 3 [0/50000 (0%)]	Loss: 1.682916
Train Epoch: 3 [640/50000 (1%)]	Loss: 1.743243
Train Epoch: 3 [1280/50000 (3%)]	Loss: 1.635487
Train Epoch: 3 [1920/50000 (4%)]	Loss: 1.810730
Train Epoch: 3 [2560/50000 (5%)]	Loss: 1.699411
Train Epoch: 3 [3200/50000 (6%)]	Loss: 1.605107
Train Epoch: 3 [3840/50000 (8%)]	Loss: 1.866543
Train Epoch: 3 [4480/50000 (9%)]	Loss: 1.441303
Train Epoch: 3 [5120/50000 (10%)]	Loss: 1.664188
Train Epoch: 3 [5760/50000 (12%)]	Loss: 1.510262
Train Epoch: 3 [6400/50000 (13%)]	Loss: 1.557148
Train Epoch: 3 [7040/50000 (14%)]	Loss: 1.720270
Train Epoch: 3 [7680/50000 (15%)]	Loss: 1.751031
Train Epoch: 3 [8320/50000 (17%)]	Loss: 1.633213
Train Epoch: 3 [8960/50000 (18%)]	Loss: 1.553361
Train Epoch: 3 [9600/50000 (19%)]	Loss: 1.683668
Train Epoch: 3 [10240/50000 (20%)]	Loss: 1.850968
Train Epoch: 3 [10880/50000 (22%)]	Loss: 1.706721
Train Epoch: 3 [11520/50000 (23%)]	Loss: 1.583573
Train Epoch: 3 [12160/50000 (24%)]	Loss: 1.493632
Train Epoch: 3 [12800/50000 (26%)]	Loss: 1.621093
Train Epoch: 3 [13440/50000 (27%)]	Loss: 1.682277
Train Epoch: 3 [14080/50000 (28%)]	Loss: 1.749824
Train Epoch: 3 [14720/50000 (29%)]	Loss: 1.533934
Train Epoch: 3 [15360/50000 (31%)]	Loss: 1.557982
Train Epoch: 3 [16000/50000 (32%)]	Loss: 1.671941
Train Epoch: 3 [16640/50000 (33%)]	Loss: 1.471360
Train Epoch: 3 [17280/50000 (35%)]	Loss: 1.529527
Train Epoch: 3 [17920/50000 (36%)]	Loss: 1.611771
Train Epoch: 3 [18560/50000 (37%)]	Loss: 1.485675
Train Epoch: 3 [19200/50000 (38%)]	Loss: 1.600415
Train Epoch: 3 [19840/50000 (40%)]	Loss: 1.499466
Train Epoch: 3 [20480/50000 (41%)]	Loss: 1.812130
Train Epoch: 3 [21120/50000 (42%)]	Loss: 1.521329
Train Epoch: 3 [21760/50000 (43%)]	Loss: 1.540288
Train Epoch: 3 [22400/50000 (45%)]	Loss: 1.718117
Train Epoch: 3 [23040/50000 (46%)]	Loss: 1.474740
Train Epoch: 3 [23680/50000 (47%)]	Loss: 1.615759
Train Epoch: 3 [24320/50000 (49%)]	Loss: 1.611675
Train Epoch: 3 [24960/50000 (50%)]	Loss: 1.435302
Train Epoch: 3 [25600/50000 (51%)]	Loss: 1.537544
Train Epoch: 3 [26240/50000 (52%)]	Loss: 1.604153
Train Epoch: 3 [26880/50000 (54%)]	Loss: 1.508797
Train Epoch: 3 [27520/50000 (55%)]	Loss: 1.659371
Train Epoch: 3 [28160/50000 (56%)]	Loss: 1.681323
Train Epoch: 3 [28800/50000 (58%)]	Loss: 1.691581
Train Epoch: 3 [29440/50000 (59%)]	Loss: 1.676928
Train Epoch: 3 [30080/50000 (60%)]	Loss: 1.468811
Train Epoch: 3 [30720/50000 (61%)]	Loss: 1.675013
Train Epoch: 3 [31360/50000 (63%)]	Loss: 1.426386
Train Epoch: 3 [32000/50000 (64%)]	Loss: 1.834009
Train Epoch: 3 [32640/50000 (65%)]	Loss: 1.613593
Train Epoch: 3 [33280/50000 (66%)]	Loss: 1.620949
Train Epoch: 3 [33920/50000 (68%)]	Loss: 1.389081
Train Epoch: 3 [34560/50000 (69%)]	Loss: 1.676571
Train Epoch: 3 [35200/50000 (70%)]	Loss: 1.589478
Train Epoch: 3 [35840/50000 (72%)]	Loss: 1.507864
Train Epoch: 3 [36480/50000 (73%)]	Loss: 1.511648
Train Epoch: 3 [37120/50000 (74%)]	Loss: 1.492845
Train Epoch: 3 [37760/50000 (75%)]	Loss: 1.549331
Train Epoch: 3 [38400/50000 (77%)]	Loss: 1.729450
Train Epoch: 3 [39040/50000 (78%)]	Loss: 1.640496
Train Epoch: 3 [39680/50000 (79%)]	Loss: 1.638476
Train Epoch: 3 [40320/50000 (81%)]	Loss: 1.335754
Train Epoch: 3 [40960/50000 (82%)]	Loss: 1.184299
Train Epoch: 3 [41600/50000 (83%)]	Loss: 1.499691
Train Epoch: 3 [42240/50000 (84%)]	Loss: 1.844353
Train Epoch: 3 [42880/50000 (86%)]	Loss: 1.456127
Train Epoch: 3 [43520/50000 (87%)]	Loss: 1.564548
Train Epoch: 3 [44160/50000 (88%)]	Loss: 1.439844
Train Epoch: 3 [44800/50000 (90%)]	Loss: 1.554989
Train Epoch: 3 [45440/50000 (91%)]	Loss: 1.553121
Train Epoch: 3 [46080/50000 (92%)]	Loss: 1.470464
Train Epoch: 3 [46720/50000 (93%)]	Loss: 1.686351
Train Epoch: 3 [47360/50000 (95%)]	Loss: 1.559849
Train Epoch: 3 [48000/50000 (96%)]	Loss: 1.630184
Train Epoch: 3 [48640/50000 (97%)]	Loss: 1.574158
Train Epoch: 3 [49280/50000 (98%)]	Loss: 1.413526
Train Epoch: 3 [49920/50000 (100%)]	Loss: 1.833721

Test set: Average loss: 1.4422, Accuracy: 4889/10000 (49%)

Train Epoch: 4 [0/50000 (0%)]	Loss: 1.789750
Train Epoch: 4 [640/50000 (1%)]	Loss: 1.599202
Train Epoch: 4 [1280/50000 (3%)]	Loss: 1.591551
Train Epoch: 4 [1920/50000 (4%)]	Loss: 1.522561
Train Epoch: 4 [2560/50000 (5%)]	Loss: 1.611513
Train Epoch: 4 [3200/50000 (6%)]	Loss: 1.380146
Train Epoch: 4 [3840/50000 (8%)]	Loss: 1.602024
Train Epoch: 4 [4480/50000 (9%)]	Loss: 1.509147
Train Epoch: 4 [5120/50000 (10%)]	Loss: 1.361543
Train Epoch: 4 [5760/50000 (12%)]	Loss: 1.659448
Train Epoch: 4 [6400/50000 (13%)]	Loss: 1.367523
Train Epoch: 4 [7040/50000 (14%)]	Loss: 1.701471
Train Epoch: 4 [7680/50000 (15%)]	Loss: 1.616549
Train Epoch: 4 [8320/50000 (17%)]	Loss: 1.543152
Train Epoch: 4 [8960/50000 (18%)]	Loss: 1.507768
Train Epoch: 4 [9600/50000 (19%)]	Loss: 1.280990
Train Epoch: 4 [10240/50000 (20%)]	Loss: 1.736611
Train Epoch: 4 [10880/50000 (22%)]	Loss: 1.597437
Train Epoch: 4 [11520/50000 (23%)]	Loss: 1.717629
Train Epoch: 4 [12160/50000 (24%)]	Loss: 1.652976
Train Epoch: 4 [12800/50000 (26%)]	Loss: 1.662737
Train Epoch: 4 [13440/50000 (27%)]	Loss: 1.678498
Train Epoch: 4 [14080/50000 (28%)]	Loss: 1.401755
Train Epoch: 4 [14720/50000 (29%)]	Loss: 1.497644
Train Epoch: 4 [15360/50000 (31%)]	Loss: 1.487070
Train Epoch: 4 [16000/50000 (32%)]	Loss: 1.742754
Train Epoch: 4 [16640/50000 (33%)]	Loss: 1.286809
Train Epoch: 4 [17280/50000 (35%)]	Loss: 1.760176
Train Epoch: 4 [17920/50000 (36%)]	Loss: 1.488566
Train Epoch: 4 [18560/50000 (37%)]	Loss: 1.606280
Train Epoch: 4 [19200/50000 (38%)]	Loss: 1.534591
Train Epoch: 4 [19840/50000 (40%)]	Loss: 1.431549
Train Epoch: 4 [20480/50000 (41%)]	Loss: 1.402481
Train Epoch: 4 [21120/50000 (42%)]	Loss: 1.712476
Train Epoch: 4 [21760/50000 (43%)]	Loss: 1.426847
Train Epoch: 4 [22400/50000 (45%)]	Loss: 1.322365
Train Epoch: 4 [23040/50000 (46%)]	Loss: 1.441814
Train Epoch: 4 [23680/50000 (47%)]	Loss: 1.509959
Train Epoch: 4 [24320/50000 (49%)]	Loss: 1.564115
Train Epoch: 4 [24960/50000 (50%)]	Loss: 1.738804
Train Epoch: 4 [25600/50000 (51%)]	Loss: 1.313424
Train Epoch: 4 [26240/50000 (52%)]	Loss: 1.405021
Train Epoch: 4 [26880/50000 (54%)]	Loss: 1.574693
Train Epoch: 4 [27520/50000 (55%)]	Loss: 1.508832
Train Epoch: 4 [28160/50000 (56%)]	Loss: 1.471284
Train Epoch: 4 [28800/50000 (58%)]	Loss: 1.649163
Train Epoch: 4 [29440/50000 (59%)]	Loss: 1.650985
Train Epoch: 4 [30080/50000 (60%)]	Loss: 1.774036
Train Epoch: 4 [30720/50000 (61%)]	Loss: 1.480934
Train Epoch: 4 [31360/50000 (63%)]	Loss: 1.479778
Train Epoch: 4 [32000/50000 (64%)]	Loss: 1.607690
Train Epoch: 4 [32640/50000 (65%)]	Loss: 1.457702
Train Epoch: 4 [33280/50000 (66%)]	Loss: 1.492034
Train Epoch: 4 [33920/50000 (68%)]	Loss: 1.459722
Train Epoch: 4 [34560/50000 (69%)]	Loss: 1.460817
Train Epoch: 4 [35200/50000 (70%)]	Loss: 1.423427
Train Epoch: 4 [35840/50000 (72%)]	Loss: 1.642875
Train Epoch: 4 [36480/50000 (73%)]	Loss: 1.628826
Train Epoch: 4 [37120/50000 (74%)]	Loss: 1.404674
Train Epoch: 4 [37760/50000 (75%)]	Loss: 1.426124
Train Epoch: 4 [38400/50000 (77%)]	Loss: 1.319225
Train Epoch: 4 [39040/50000 (78%)]	Loss: 1.554229
Train Epoch: 4 [39680/50000 (79%)]	Loss: 1.466465
Train Epoch: 4 [40320/50000 (81%)]	Loss: 1.570464
Train Epoch: 4 [40960/50000 (82%)]	Loss: 1.501623
Train Epoch: 4 [41600/50000 (83%)]	Loss: 1.418527
Train Epoch: 4 [42240/50000 (84%)]	Loss: 1.327659
Train Epoch: 4 [42880/50000 (86%)]	Loss: 1.484850
Train Epoch: 4 [43520/50000 (87%)]	Loss: 1.424412
Train Epoch: 4 [44160/50000 (88%)]	Loss: 1.481537
Train Epoch: 4 [44800/50000 (90%)]	Loss: 1.800711
Train Epoch: 4 [45440/50000 (91%)]	Loss: 1.349401
Train Epoch: 4 [46080/50000 (92%)]	Loss: 1.497080
Train Epoch: 4 [46720/50000 (93%)]	Loss: 1.477972
Train Epoch: 4 [47360/50000 (95%)]	Loss: 1.509127
Train Epoch: 4 [48000/50000 (96%)]	Loss: 1.659590
Train Epoch: 4 [48640/50000 (97%)]	Loss: 1.623947
Train Epoch: 4 [49280/50000 (98%)]	Loss: 1.565476
Train Epoch: 4 [49920/50000 (100%)]	Loss: 1.418104

Test set: Average loss: 1.4693, Accuracy: 4666/10000 (47%)

Train Epoch: 5 [0/50000 (0%)]	Loss: 1.626746
Train Epoch: 5 [640/50000 (1%)]	Loss: 1.469067
Train Epoch: 5 [1280/50000 (3%)]	Loss: 1.447789
Train Epoch: 5 [1920/50000 (4%)]	Loss: 1.666856
Train Epoch: 5 [2560/50000 (5%)]	Loss: 1.369073
Train Epoch: 5 [3200/50000 (6%)]	Loss: 1.550935
Train Epoch: 5 [3840/50000 (8%)]	Loss: 1.386973
Train Epoch: 5 [4480/50000 (9%)]	Loss: 1.343855
Train Epoch: 5 [5120/50000 (10%)]	Loss: 1.547571
Train Epoch: 5 [5760/50000 (12%)]	Loss: 1.437857
Train Epoch: 5 [6400/50000 (13%)]	Loss: 1.517768
Train Epoch: 5 [7040/50000 (14%)]	Loss: 1.421828
Train Epoch: 5 [7680/50000 (15%)]	Loss: 1.539661
Train Epoch: 5 [8320/50000 (17%)]	Loss: 1.354292
Train Epoch: 5 [8960/50000 (18%)]	Loss: 1.696474
Train Epoch: 5 [9600/50000 (19%)]	Loss: 1.278595
Train Epoch: 5 [10240/50000 (20%)]	Loss: 1.321777
Train Epoch: 5 [10880/50000 (22%)]	Loss: 1.750038
Train Epoch: 5 [11520/50000 (23%)]	Loss: 1.636163
Train Epoch: 5 [12160/50000 (24%)]	Loss: 1.508911
Train Epoch: 5 [12800/50000 (26%)]	Loss: 1.433339
Train Epoch: 5 [13440/50000 (27%)]	Loss: 1.665852
Train Epoch: 5 [14080/50000 (28%)]	Loss: 1.364813
Train Epoch: 5 [14720/50000 (29%)]	Loss: 1.449896
Train Epoch: 5 [15360/50000 (31%)]	Loss: 1.437223
Train Epoch: 5 [16000/50000 (32%)]	Loss: 1.565226
Train Epoch: 5 [16640/50000 (33%)]	Loss: 1.450567
Train Epoch: 5 [17280/50000 (35%)]	Loss: 1.493226
Train Epoch: 5 [17920/50000 (36%)]	Loss: 1.579806
Train Epoch: 5 [18560/50000 (37%)]	Loss: 1.620313
Train Epoch: 5 [19200/50000 (38%)]	Loss: 1.441963
Train Epoch: 5 [19840/50000 (40%)]	Loss: 1.676013
Train Epoch: 5 [20480/50000 (41%)]	Loss: 1.547608
Train Epoch: 5 [21120/50000 (42%)]	Loss: 1.500587
Train Epoch: 5 [21760/50000 (43%)]	Loss: 1.312824
Train Epoch: 5 [22400/50000 (45%)]	Loss: 1.570350
Train Epoch: 5 [23040/50000 (46%)]	Loss: 1.468765
Train Epoch: 5 [23680/50000 (47%)]	Loss: 1.509066
Train Epoch: 5 [24320/50000 (49%)]	Loss: 1.400635
Train Epoch: 5 [24960/50000 (50%)]	Loss: 1.745047
Train Epoch: 5 [25600/50000 (51%)]	Loss: 1.357623
Train Epoch: 5 [26240/50000 (52%)]	Loss: 1.809338
Train Epoch: 5 [26880/50000 (54%)]	Loss: 1.433910
Train Epoch: 5 [27520/50000 (55%)]	Loss: 1.610295
Train Epoch: 5 [28160/50000 (56%)]	Loss: 1.555721
Train Epoch: 5 [28800/50000 (58%)]	Loss: 1.698162
Train Epoch: 5 [29440/50000 (59%)]	Loss: 1.369018
Train Epoch: 5 [30080/50000 (60%)]	Loss: 1.371028
Train Epoch: 5 [30720/50000 (61%)]	Loss: 1.385164
Train Epoch: 5 [31360/50000 (63%)]	Loss: 1.635409
Train Epoch: 5 [32000/50000 (64%)]	Loss: 1.528582
Train Epoch: 5 [32640/50000 (65%)]	Loss: 1.712598
Train Epoch: 5 [33280/50000 (66%)]	Loss: 1.200435
Train Epoch: 5 [33920/50000 (68%)]	Loss: 1.596508
Train Epoch: 5 [34560/50000 (69%)]	Loss: 1.532303
Train Epoch: 5 [35200/50000 (70%)]	Loss: 1.723283
Train Epoch: 5 [35840/50000 (72%)]	Loss: 1.380763
Train Epoch: 5 [36480/50000 (73%)]	Loss: 1.701194
Train Epoch: 5 [37120/50000 (74%)]	Loss: 1.350992
Train Epoch: 5 [37760/50000 (75%)]	Loss: 1.362741
Train Epoch: 5 [38400/50000 (77%)]	Loss: 1.532848
Train Epoch: 5 [39040/50000 (78%)]	Loss: 1.301572
Train Epoch: 5 [39680/50000 (79%)]	Loss: 1.396561
Train Epoch: 5 [40320/50000 (81%)]	Loss: 1.370527
Train Epoch: 5 [40960/50000 (82%)]	Loss: 1.362183
Train Epoch: 5 [41600/50000 (83%)]	Loss: 1.226356
Train Epoch: 5 [42240/50000 (84%)]	Loss: 1.546626
Train Epoch: 5 [42880/50000 (86%)]	Loss: 1.548475
Train Epoch: 5 [43520/50000 (87%)]	Loss: 1.538025
Train Epoch: 5 [44160/50000 (88%)]	Loss: 1.479389
Train Epoch: 5 [44800/50000 (90%)]	Loss: 1.491461
Train Epoch: 5 [45440/50000 (91%)]	Loss: 1.278805
Train Epoch: 5 [46080/50000 (92%)]	Loss: 1.326496
Train Epoch: 5 [46720/50000 (93%)]	Loss: 1.522785
Train Epoch: 5 [47360/50000 (95%)]	Loss: 1.437613
Train Epoch: 5 [48000/50000 (96%)]	Loss: 1.333211
Train Epoch: 5 [48640/50000 (97%)]	Loss: 1.599533
Train Epoch: 5 [49280/50000 (98%)]	Loss: 1.500589
Train Epoch: 5 [49920/50000 (100%)]	Loss: 1.534972

Test set: Average loss: 1.3649, Accuracy: 5165/10000 (52%)

Train Epoch: 6 [0/50000 (0%)]	Loss: 1.744132
Train Epoch: 6 [640/50000 (1%)]	Loss: 1.272318
Train Epoch: 6 [1280/50000 (3%)]	Loss: 1.333399
Train Epoch: 6 [1920/50000 (4%)]	Loss: 1.757748
Train Epoch: 6 [2560/50000 (5%)]	Loss: 1.305264
Train Epoch: 6 [3200/50000 (6%)]	Loss: 1.347861
Train Epoch: 6 [3840/50000 (8%)]	Loss: 1.408838
Train Epoch: 6 [4480/50000 (9%)]	Loss: 1.451380
Train Epoch: 6 [5120/50000 (10%)]	Loss: 1.550167
Train Epoch: 6 [5760/50000 (12%)]	Loss: 1.473515
Train Epoch: 6 [6400/50000 (13%)]	Loss: 1.637250
Train Epoch: 6 [7040/50000 (14%)]	Loss: 1.388882
Train Epoch: 6 [7680/50000 (15%)]	Loss: 1.494627
Train Epoch: 6 [8320/50000 (17%)]	Loss: 1.632481
Train Epoch: 6 [8960/50000 (18%)]	Loss: 1.426762
Train Epoch: 6 [9600/50000 (19%)]	Loss: 1.522187
Train Epoch: 6 [10240/50000 (20%)]	Loss: 1.441547
Train Epoch: 6 [10880/50000 (22%)]	Loss: 1.589737
Train Epoch: 6 [11520/50000 (23%)]	Loss: 1.564770
Train Epoch: 6 [12160/50000 (24%)]	Loss: 1.383595
Train Epoch: 6 [12800/50000 (26%)]	Loss: 1.333814
Train Epoch: 6 [13440/50000 (27%)]	Loss: 1.788968
Train Epoch: 6 [14080/50000 (28%)]	Loss: 1.520604
Train Epoch: 6 [14720/50000 (29%)]	Loss: 1.659004
Train Epoch: 6 [15360/50000 (31%)]	Loss: 1.726917
Train Epoch: 6 [16000/50000 (32%)]	Loss: 1.472473
Train Epoch: 6 [16640/50000 (33%)]	Loss: 1.508392
Train Epoch: 6 [17280/50000 (35%)]	Loss: 1.557539
Train Epoch: 6 [17920/50000 (36%)]	Loss: 1.434701
Train Epoch: 6 [18560/50000 (37%)]	Loss: 1.349416
Train Epoch: 6 [19200/50000 (38%)]	Loss: 1.347256
Train Epoch: 6 [19840/50000 (40%)]	Loss: 1.472495
Train Epoch: 6 [20480/50000 (41%)]	Loss: 1.327496
Train Epoch: 6 [21120/50000 (42%)]	Loss: 1.393485
Train Epoch: 6 [21760/50000 (43%)]	Loss: 1.508236
Train Epoch: 6 [22400/50000 (45%)]	Loss: 1.492352
Train Epoch: 6 [23040/50000 (46%)]	Loss: 1.429840
Train Epoch: 6 [23680/50000 (47%)]	Loss: 1.537683
Train Epoch: 6 [24320/50000 (49%)]	Loss: 1.403566
Train Epoch: 6 [24960/50000 (50%)]	Loss: 1.673228
Train Epoch: 6 [25600/50000 (51%)]	Loss: 1.413164
Train Epoch: 6 [26240/50000 (52%)]	Loss: 1.511593
Train Epoch: 6 [26880/50000 (54%)]	Loss: 1.384612
Train Epoch: 6 [27520/50000 (55%)]	Loss: 1.516141
Train Epoch: 6 [28160/50000 (56%)]	Loss: 1.443830
Train Epoch: 6 [28800/50000 (58%)]	Loss: 1.356581
Train Epoch: 6 [29440/50000 (59%)]	Loss: 1.328234
Train Epoch: 6 [30080/50000 (60%)]	Loss: 1.443444
Train Epoch: 6 [30720/50000 (61%)]	Loss: 1.500374
Train Epoch: 6 [31360/50000 (63%)]	Loss: 1.479154
Train Epoch: 6 [32000/50000 (64%)]	Loss: 1.805915
Train Epoch: 6 [32640/50000 (65%)]	Loss: 1.574226
Train Epoch: 6 [33280/50000 (66%)]	Loss: 1.451470
Train Epoch: 6 [33920/50000 (68%)]	Loss: 1.397825
Train Epoch: 6 [34560/50000 (69%)]	Loss: 1.594272
Train Epoch: 6 [35200/50000 (70%)]	Loss: 1.524089
Train Epoch: 6 [35840/50000 (72%)]	Loss: 1.409520
Train Epoch: 6 [36480/50000 (73%)]	Loss: 1.589489
Train Epoch: 6 [37120/50000 (74%)]	Loss: 1.642280
Train Epoch: 6 [37760/50000 (75%)]	Loss: 1.622423
Train Epoch: 6 [38400/50000 (77%)]	Loss: 1.388885
Train Epoch: 6 [39040/50000 (78%)]	Loss: 1.341744
Train Epoch: 6 [39680/50000 (79%)]	Loss: 1.464537
Train Epoch: 6 [40320/50000 (81%)]	Loss: 1.523214
Train Epoch: 6 [40960/50000 (82%)]	Loss: 1.272027
Train Epoch: 6 [41600/50000 (83%)]	Loss: 1.593494
Train Epoch: 6 [42240/50000 (84%)]	Loss: 1.374051
Train Epoch: 6 [42880/50000 (86%)]	Loss: 1.322067
Train Epoch: 6 [43520/50000 (87%)]	Loss: 1.612037
Train Epoch: 6 [44160/50000 (88%)]	Loss: 1.467954
Train Epoch: 6 [44800/50000 (90%)]	Loss: 1.602748
Train Epoch: 6 [45440/50000 (91%)]	Loss: 1.517255
Train Epoch: 6 [46080/50000 (92%)]	Loss: 1.462844
Train Epoch: 6 [46720/50000 (93%)]	Loss: 1.407615
Train Epoch: 6 [47360/50000 (95%)]	Loss: 1.386739
Train Epoch: 6 [48000/50000 (96%)]	Loss: 1.477863
Train Epoch: 6 [48640/50000 (97%)]	Loss: 1.518116
Train Epoch: 6 [49280/50000 (98%)]	Loss: 1.295352
Train Epoch: 6 [49920/50000 (100%)]	Loss: 1.530329

Test set: Average loss: 1.3350, Accuracy: 5230/10000 (52%)

Train Epoch: 7 [0/50000 (0%)]	Loss: 1.378925
Train Epoch: 7 [640/50000 (1%)]	Loss: 1.215591
Train Epoch: 7 [1280/50000 (3%)]	Loss: 1.406381
Train Epoch: 7 [1920/50000 (4%)]	Loss: 1.463528
Train Epoch: 7 [2560/50000 (5%)]	Loss: 1.407737
Train Epoch: 7 [3200/50000 (6%)]	Loss: 1.424042
Train Epoch: 7 [3840/50000 (8%)]	Loss: 1.202616
Train Epoch: 7 [4480/50000 (9%)]	Loss: 1.358882
Train Epoch: 7 [5120/50000 (10%)]	Loss: 1.151360
Train Epoch: 7 [5760/50000 (12%)]	Loss: 1.443133
Train Epoch: 7 [6400/50000 (13%)]	Loss: 1.289976
Train Epoch: 7 [7040/50000 (14%)]	Loss: 1.649506
Train Epoch: 7 [7680/50000 (15%)]	Loss: 1.774582
Train Epoch: 7 [8320/50000 (17%)]	Loss: 1.563440
Train Epoch: 7 [8960/50000 (18%)]	Loss: 1.479036
Train Epoch: 7 [9600/50000 (19%)]	Loss: 1.612119
Train Epoch: 7 [10240/50000 (20%)]	Loss: 1.518852
Train Epoch: 7 [10880/50000 (22%)]	Loss: 1.584662
Train Epoch: 7 [11520/50000 (23%)]	Loss: 1.175622
Train Epoch: 7 [12160/50000 (24%)]	Loss: 1.288723
Train Epoch: 7 [12800/50000 (26%)]	Loss: 1.633746
Train Epoch: 7 [13440/50000 (27%)]	Loss: 1.306998
Train Epoch: 7 [14080/50000 (28%)]	Loss: 1.276745
Train Epoch: 7 [14720/50000 (29%)]	Loss: 1.564849
Train Epoch: 7 [15360/50000 (31%)]	Loss: 1.529274
Train Epoch: 7 [16000/50000 (32%)]	Loss: 1.660667
Train Epoch: 7 [16640/50000 (33%)]	Loss: 1.354307
Train Epoch: 7 [17280/50000 (35%)]	Loss: 1.577518
Train Epoch: 7 [17920/50000 (36%)]	Loss: 1.322137
Train Epoch: 7 [18560/50000 (37%)]	Loss: 1.670122
Train Epoch: 7 [19200/50000 (38%)]	Loss: 1.504895
Train Epoch: 7 [19840/50000 (40%)]	Loss: 1.468476
Train Epoch: 7 [20480/50000 (41%)]	Loss: 1.507897
Train Epoch: 7 [21120/50000 (42%)]	Loss: 1.349668
Train Epoch: 7 [21760/50000 (43%)]	Loss: 1.497271
Train Epoch: 7 [22400/50000 (45%)]	Loss: 1.603041
Train Epoch: 7 [23040/50000 (46%)]	Loss: 1.391772
Train Epoch: 7 [23680/50000 (47%)]	Loss: 1.598156
Train Epoch: 7 [24320/50000 (49%)]	Loss: 1.221821
Train Epoch: 7 [24960/50000 (50%)]	Loss: 1.413863
Train Epoch: 7 [25600/50000 (51%)]	Loss: 1.420521
Train Epoch: 7 [26240/50000 (52%)]	Loss: 1.442737
Train Epoch: 7 [26880/50000 (54%)]	Loss: 1.357976
Train Epoch: 7 [27520/50000 (55%)]	Loss: 1.602844
Train Epoch: 7 [28160/50000 (56%)]	Loss: 1.560952
Train Epoch: 7 [28800/50000 (58%)]	Loss: 1.520139
Train Epoch: 7 [29440/50000 (59%)]	Loss: 1.430888
Train Epoch: 7 [30080/50000 (60%)]	Loss: 1.427072
Train Epoch: 7 [30720/50000 (61%)]	Loss: 1.828626
Train Epoch: 7 [31360/50000 (63%)]	Loss: 1.361798
Train Epoch: 7 [32000/50000 (64%)]	Loss: 1.159289
Train Epoch: 7 [32640/50000 (65%)]	Loss: 1.564962
Train Epoch: 7 [33280/50000 (66%)]	Loss: 1.331807
Train Epoch: 7 [33920/50000 (68%)]	Loss: 1.639165
Train Epoch: 7 [34560/50000 (69%)]	Loss: 1.513958
Train Epoch: 7 [35200/50000 (70%)]	Loss: 1.550295
Train Epoch: 7 [35840/50000 (72%)]	Loss: 1.214932
Train Epoch: 7 [36480/50000 (73%)]	Loss: 1.425971
Train Epoch: 7 [37120/50000 (74%)]	Loss: 1.397732
Train Epoch: 7 [37760/50000 (75%)]	Loss: 1.428039
Train Epoch: 7 [38400/50000 (77%)]	Loss: 1.361817
Train Epoch: 7 [39040/50000 (78%)]	Loss: 1.356182
Train Epoch: 7 [39680/50000 (79%)]	Loss: 1.639892
Train Epoch: 7 [40320/50000 (81%)]	Loss: 1.434474
Train Epoch: 7 [40960/50000 (82%)]	Loss: 1.326629
Train Epoch: 7 [41600/50000 (83%)]	Loss: 1.581548
Train Epoch: 7 [42240/50000 (84%)]	Loss: 1.399343
Train Epoch: 7 [42880/50000 (86%)]	Loss: 1.501235
Train Epoch: 7 [43520/50000 (87%)]	Loss: 1.361935
Train Epoch: 7 [44160/50000 (88%)]	Loss: 1.267643
Train Epoch: 7 [44800/50000 (90%)]	Loss: 1.417140
Train Epoch: 7 [45440/50000 (91%)]	Loss: 1.445763
Train Epoch: 7 [46080/50000 (92%)]	Loss: 1.291734
Train Epoch: 7 [46720/50000 (93%)]	Loss: 1.570412
Train Epoch: 7 [47360/50000 (95%)]	Loss: 1.233823
Train Epoch: 7 [48000/50000 (96%)]	Loss: 1.464841
Train Epoch: 7 [48640/50000 (97%)]	Loss: 1.278243
Train Epoch: 7 [49280/50000 (98%)]	Loss: 1.310450
Train Epoch: 7 [49920/50000 (100%)]	Loss: 1.317226

Test set: Average loss: 1.3040, Accuracy: 5319/10000 (53%)

Train Epoch: 8 [0/50000 (0%)]	Loss: 1.610767
Train Epoch: 8 [640/50000 (1%)]	Loss: 1.565039
Train Epoch: 8 [1280/50000 (3%)]	Loss: 1.306487
Train Epoch: 8 [1920/50000 (4%)]	Loss: 1.472609
Train Epoch: 8 [2560/50000 (5%)]	Loss: 1.349667
Train Epoch: 8 [3200/50000 (6%)]	Loss: 1.474958
Train Epoch: 8 [3840/50000 (8%)]	Loss: 1.520102
Train Epoch: 8 [4480/50000 (9%)]	Loss: 1.274242
Train Epoch: 8 [5120/50000 (10%)]	Loss: 1.460695
Train Epoch: 8 [5760/50000 (12%)]	Loss: 1.347065
Train Epoch: 8 [6400/50000 (13%)]	Loss: 1.232397
Train Epoch: 8 [7040/50000 (14%)]	Loss: 1.365732
Train Epoch: 8 [7680/50000 (15%)]	Loss: 1.368033
Train Epoch: 8 [8320/50000 (17%)]	Loss: 1.481548
Train Epoch: 8 [8960/50000 (18%)]	Loss: 1.306339
Train Epoch: 8 [9600/50000 (19%)]	Loss: 1.503503
Train Epoch: 8 [10240/50000 (20%)]	Loss: 1.363091
Train Epoch: 8 [10880/50000 (22%)]	Loss: 1.289299
Train Epoch: 8 [11520/50000 (23%)]	Loss: 1.504225
Train Epoch: 8 [12160/50000 (24%)]	Loss: 1.251540
Train Epoch: 8 [12800/50000 (26%)]	Loss: 1.463180
Train Epoch: 8 [13440/50000 (27%)]	Loss: 1.276648
Train Epoch: 8 [14080/50000 (28%)]	Loss: 1.451540
Train Epoch: 8 [14720/50000 (29%)]	Loss: 1.401842
Train Epoch: 8 [15360/50000 (31%)]	Loss: 1.505643
Train Epoch: 8 [16000/50000 (32%)]	Loss: 1.440841
Train Epoch: 8 [16640/50000 (33%)]	Loss: 1.422048
Train Epoch: 8 [17280/50000 (35%)]	Loss: 1.395847
Train Epoch: 8 [17920/50000 (36%)]	Loss: 1.537953
Train Epoch: 8 [18560/50000 (37%)]	Loss: 1.553005
Train Epoch: 8 [19200/50000 (38%)]	Loss: 1.435387
Train Epoch: 8 [19840/50000 (40%)]	Loss: 1.434562
Train Epoch: 8 [20480/50000 (41%)]	Loss: 1.348100
Train Epoch: 8 [21120/50000 (42%)]	Loss: 1.278184
Train Epoch: 8 [21760/50000 (43%)]	Loss: 1.236539
Train Epoch: 8 [22400/50000 (45%)]	Loss: 1.369525
Train Epoch: 8 [23040/50000 (46%)]	Loss: 1.280986
Train Epoch: 8 [23680/50000 (47%)]	Loss: 1.619094
Train Epoch: 8 [24320/50000 (49%)]	Loss: 1.331262
Train Epoch: 8 [24960/50000 (50%)]	Loss: 1.293915
Train Epoch: 8 [25600/50000 (51%)]	Loss: 1.305073
Train Epoch: 8 [26240/50000 (52%)]	Loss: 1.329985
Train Epoch: 8 [26880/50000 (54%)]	Loss: 1.421836
Train Epoch: 8 [27520/50000 (55%)]	Loss: 1.480564
Train Epoch: 8 [28160/50000 (56%)]	Loss: 1.575192
Train Epoch: 8 [28800/50000 (58%)]	Loss: 1.430610
Train Epoch: 8 [29440/50000 (59%)]	Loss: 1.232859
Train Epoch: 8 [30080/50000 (60%)]	Loss: 1.607847
Train Epoch: 8 [30720/50000 (61%)]	Loss: 1.454571
Train Epoch: 8 [31360/50000 (63%)]	Loss: 1.295873
Train Epoch: 8 [32000/50000 (64%)]	Loss: 1.611049
Train Epoch: 8 [32640/50000 (65%)]	Loss: 1.455024
Train Epoch: 8 [33280/50000 (66%)]	Loss: 1.414125
Train Epoch: 8 [33920/50000 (68%)]	Loss: 1.590776
Train Epoch: 8 [34560/50000 (69%)]	Loss: 1.450950
Train Epoch: 8 [35200/50000 (70%)]	Loss: 1.446592
Train Epoch: 8 [35840/50000 (72%)]	Loss: 1.512350
Train Epoch: 8 [36480/50000 (73%)]	Loss: 1.478959
Train Epoch: 8 [37120/50000 (74%)]	Loss: 1.282590
Train Epoch: 8 [37760/50000 (75%)]	Loss: 1.539088
Train Epoch: 8 [38400/50000 (77%)]	Loss: 1.350905
Train Epoch: 8 [39040/50000 (78%)]	Loss: 1.499679
Train Epoch: 8 [39680/50000 (79%)]	Loss: 1.493505
Train Epoch: 8 [40320/50000 (81%)]	Loss: 1.561144
Train Epoch: 8 [40960/50000 (82%)]	Loss: 1.438260
Train Epoch: 8 [41600/50000 (83%)]	Loss: 1.499665
Train Epoch: 8 [42240/50000 (84%)]	Loss: 1.527922
Train Epoch: 8 [42880/50000 (86%)]	Loss: 1.390613
Train Epoch: 8 [43520/50000 (87%)]	Loss: 1.466229
Train Epoch: 8 [44160/50000 (88%)]	Loss: 1.523527
Train Epoch: 8 [44800/50000 (90%)]	Loss: 1.356379
Train Epoch: 8 [45440/50000 (91%)]	Loss: 1.255616
Train Epoch: 8 [46080/50000 (92%)]	Loss: 1.278252
Train Epoch: 8 [46720/50000 (93%)]	Loss: 1.516100
Train Epoch: 8 [47360/50000 (95%)]	Loss: 1.170856
Train Epoch: 8 [48000/50000 (96%)]	Loss: 1.625948
Train Epoch: 8 [48640/50000 (97%)]	Loss: 1.708885
Train Epoch: 8 [49280/50000 (98%)]	Loss: 1.311600
Train Epoch: 8 [49920/50000 (100%)]	Loss: 1.198319

Test set: Average loss: 1.2952, Accuracy: 5287/10000 (53%)

Train Epoch: 9 [0/50000 (0%)]	Loss: 1.217278
Train Epoch: 9 [640/50000 (1%)]	Loss: 1.464775
Train Epoch: 9 [1280/50000 (3%)]	Loss: 1.535584
Train Epoch: 9 [1920/50000 (4%)]	Loss: 1.361429
Train Epoch: 9 [2560/50000 (5%)]	Loss: 1.325289
Train Epoch: 9 [3200/50000 (6%)]	Loss: 1.545743
Train Epoch: 9 [3840/50000 (8%)]	Loss: 1.478323
Train Epoch: 9 [4480/50000 (9%)]	Loss: 1.329288
Train Epoch: 9 [5120/50000 (10%)]	Loss: 1.510750
Train Epoch: 9 [5760/50000 (12%)]	Loss: 1.432046
Train Epoch: 9 [6400/50000 (13%)]	Loss: 1.445659
Train Epoch: 9 [7040/50000 (14%)]	Loss: 1.498653
Train Epoch: 9 [7680/50000 (15%)]	Loss: 1.426588
Train Epoch: 9 [8320/50000 (17%)]	Loss: 1.554014
Train Epoch: 9 [8960/50000 (18%)]	Loss: 1.631019
Train Epoch: 9 [9600/50000 (19%)]	Loss: 1.354989
Train Epoch: 9 [10240/50000 (20%)]	Loss: 1.551600
Train Epoch: 9 [10880/50000 (22%)]	Loss: 1.489446
Train Epoch: 9 [11520/50000 (23%)]	Loss: 1.678961
Train Epoch: 9 [12160/50000 (24%)]	Loss: 1.453038
Train Epoch: 9 [12800/50000 (26%)]	Loss: 1.243636
Train Epoch: 9 [13440/50000 (27%)]	Loss: 1.327941
Train Epoch: 9 [14080/50000 (28%)]	Loss: 1.400642
Train Epoch: 9 [14720/50000 (29%)]	Loss: 1.374601
Train Epoch: 9 [15360/50000 (31%)]	Loss: 1.607470
Train Epoch: 9 [16000/50000 (32%)]	Loss: 1.303122
Train Epoch: 9 [16640/50000 (33%)]	Loss: 1.544450
Train Epoch: 9 [17280/50000 (35%)]	Loss: 1.392053
Train Epoch: 9 [17920/50000 (36%)]	Loss: 1.376171
Train Epoch: 9 [18560/50000 (37%)]	Loss: 1.451615
Train Epoch: 9 [19200/50000 (38%)]	Loss: 1.792991
Train Epoch: 9 [19840/50000 (40%)]	Loss: 1.570594
Train Epoch: 9 [20480/50000 (41%)]	Loss: 1.451551
Train Epoch: 9 [21120/50000 (42%)]	Loss: 1.295766
Train Epoch: 9 [21760/50000 (43%)]	Loss: 1.496248
Train Epoch: 9 [22400/50000 (45%)]	Loss: 1.261211
Train Epoch: 9 [23040/50000 (46%)]	Loss: 1.302177
Train Epoch: 9 [23680/50000 (47%)]	Loss: 1.314868
Train Epoch: 9 [24320/50000 (49%)]	Loss: 1.354817
Train Epoch: 9 [24960/50000 (50%)]	Loss: 1.483346
Train Epoch: 9 [25600/50000 (51%)]	Loss: 1.405063
Train Epoch: 9 [26240/50000 (52%)]	Loss: 1.498085
Train Epoch: 9 [26880/50000 (54%)]	Loss: 1.555611
Train Epoch: 9 [27520/50000 (55%)]	Loss: 1.318214
Train Epoch: 9 [28160/50000 (56%)]	Loss: 1.272900
Train Epoch: 9 [28800/50000 (58%)]	Loss: 1.443712
Train Epoch: 9 [29440/50000 (59%)]	Loss: 1.342134
Train Epoch: 9 [30080/50000 (60%)]	Loss: 1.370964
Train Epoch: 9 [30720/50000 (61%)]	Loss: 1.408247
Train Epoch: 9 [31360/50000 (63%)]	Loss: 1.331061
Train Epoch: 9 [32000/50000 (64%)]	Loss: 1.243540
Train Epoch: 9 [32640/50000 (65%)]	Loss: 1.371432
Train Epoch: 9 [33280/50000 (66%)]	Loss: 1.467332
Train Epoch: 9 [33920/50000 (68%)]	Loss: 1.609721
Train Epoch: 9 [34560/50000 (69%)]	Loss: 1.393921
Train Epoch: 9 [35200/50000 (70%)]	Loss: 1.295096
Train Epoch: 9 [35840/50000 (72%)]	Loss: 1.413689
Train Epoch: 9 [36480/50000 (73%)]	Loss: 1.175795
Train Epoch: 9 [37120/50000 (74%)]	Loss: 1.392686
Train Epoch: 9 [37760/50000 (75%)]	Loss: 1.320502
Train Epoch: 9 [38400/50000 (77%)]	Loss: 1.674675
Train Epoch: 9 [39040/50000 (78%)]	Loss: 1.617613
Train Epoch: 9 [39680/50000 (79%)]	Loss: 1.458281
Train Epoch: 9 [40320/50000 (81%)]	Loss: 1.485071
Train Epoch: 9 [40960/50000 (82%)]	Loss: 1.317196
Train Epoch: 9 [41600/50000 (83%)]	Loss: 1.239257
Train Epoch: 9 [42240/50000 (84%)]	Loss: 1.508913
Train Epoch: 9 [42880/50000 (86%)]	Loss: 1.280003
Train Epoch: 9 [43520/50000 (87%)]	Loss: 1.496282
Train Epoch: 9 [44160/50000 (88%)]	Loss: 1.490307
Train Epoch: 9 [44800/50000 (90%)]	Loss: 1.207717
Train Epoch: 9 [45440/50000 (91%)]	Loss: 1.398191
Train Epoch: 9 [46080/50000 (92%)]	Loss: 1.450876
Train Epoch: 9 [46720/50000 (93%)]	Loss: 1.252355
Train Epoch: 9 [47360/50000 (95%)]	Loss: 1.443012
Train Epoch: 9 [48000/50000 (96%)]	Loss: 1.507137
Train Epoch: 9 [48640/50000 (97%)]	Loss: 1.622857
Train Epoch: 9 [49280/50000 (98%)]	Loss: 1.590277
Train Epoch: 9 [49920/50000 (100%)]	Loss: 1.023149

Test set: Average loss: 1.3249, Accuracy: 5242/10000 (52%)

Train Epoch: 10 [0/50000 (0%)]	Loss: 1.586342
Train Epoch: 10 [640/50000 (1%)]	Loss: 1.441370
Train Epoch: 10 [1280/50000 (3%)]	Loss: 1.341938
Train Epoch: 10 [1920/50000 (4%)]	Loss: 1.304921
Train Epoch: 10 [2560/50000 (5%)]	Loss: 1.509791
Train Epoch: 10 [3200/50000 (6%)]	Loss: 1.364686
Train Epoch: 10 [3840/50000 (8%)]	Loss: 1.378915
Train Epoch: 10 [4480/50000 (9%)]	Loss: 1.518838
Train Epoch: 10 [5120/50000 (10%)]	Loss: 1.386451
Train Epoch: 10 [5760/50000 (12%)]	Loss: 1.427670
Train Epoch: 10 [6400/50000 (13%)]	Loss: 1.511322
Train Epoch: 10 [7040/50000 (14%)]	Loss: 1.448851
Train Epoch: 10 [7680/50000 (15%)]	Loss: 1.470579
Train Epoch: 10 [8320/50000 (17%)]	Loss: 1.319353
Train Epoch: 10 [8960/50000 (18%)]	Loss: 1.301392
Train Epoch: 10 [9600/50000 (19%)]	Loss: 1.680567
Train Epoch: 10 [10240/50000 (20%)]	Loss: 1.439021
Train Epoch: 10 [10880/50000 (22%)]	Loss: 1.308393
Train Epoch: 10 [11520/50000 (23%)]	Loss: 1.246840
Train Epoch: 10 [12160/50000 (24%)]	Loss: 1.452521
Train Epoch: 10 [12800/50000 (26%)]	Loss: 1.449493
Train Epoch: 10 [13440/50000 (27%)]	Loss: 1.303678
Train Epoch: 10 [14080/50000 (28%)]	Loss: 1.450594
Train Epoch: 10 [14720/50000 (29%)]	Loss: 1.221550
Train Epoch: 10 [15360/50000 (31%)]	Loss: 1.235697
Train Epoch: 10 [16000/50000 (32%)]	Loss: 1.292493
Train Epoch: 10 [16640/50000 (33%)]	Loss: 1.458166
Train Epoch: 10 [17280/50000 (35%)]	Loss: 1.528354
Train Epoch: 10 [17920/50000 (36%)]	Loss: 1.153212
Train Epoch: 10 [18560/50000 (37%)]	Loss: 1.408709
Train Epoch: 10 [19200/50000 (38%)]	Loss: 1.204353
Train Epoch: 10 [19840/50000 (40%)]	Loss: 1.218005
Train Epoch: 10 [20480/50000 (41%)]	Loss: 1.422314
Train Epoch: 10 [21120/50000 (42%)]	Loss: 1.515797
Train Epoch: 10 [21760/50000 (43%)]	Loss: 1.432209
Train Epoch: 10 [22400/50000 (45%)]	Loss: 1.216940
Train Epoch: 10 [23040/50000 (46%)]	Loss: 1.341920
Train Epoch: 10 [23680/50000 (47%)]	Loss: 1.130167
Train Epoch: 10 [24320/50000 (49%)]	Loss: 1.320045
Train Epoch: 10 [24960/50000 (50%)]	Loss: 1.283327
Train Epoch: 10 [25600/50000 (51%)]	Loss: 1.463514
Train Epoch: 10 [26240/50000 (52%)]	Loss: 1.543613
Train Epoch: 10 [26880/50000 (54%)]	Loss: 1.144630
Train Epoch: 10 [27520/50000 (55%)]	Loss: 1.503388
Train Epoch: 10 [28160/50000 (56%)]	Loss: 1.396529
Train Epoch: 10 [28800/50000 (58%)]	Loss: 1.261952
Train Epoch: 10 [29440/50000 (59%)]	Loss: 1.408649
Train Epoch: 10 [30080/50000 (60%)]	Loss: 1.492911
Train Epoch: 10 [30720/50000 (61%)]	Loss: 1.258940
Train Epoch: 10 [31360/50000 (63%)]	Loss: 1.527428
Train Epoch: 10 [32000/50000 (64%)]	Loss: 1.563305
Train Epoch: 10 [32640/50000 (65%)]	Loss: 1.288393
Train Epoch: 10 [33280/50000 (66%)]	Loss: 1.154127
Train Epoch: 10 [33920/50000 (68%)]	Loss: 1.326079
Train Epoch: 10 [34560/50000 (69%)]	Loss: 1.141447
Train Epoch: 10 [35200/50000 (70%)]	Loss: 1.260502
Train Epoch: 10 [35840/50000 (72%)]	Loss: 1.272309
Train Epoch: 10 [36480/50000 (73%)]	Loss: 1.432984
Train Epoch: 10 [37120/50000 (74%)]	Loss: 1.455861
Train Epoch: 10 [37760/50000 (75%)]	Loss: 1.200949
Train Epoch: 10 [38400/50000 (77%)]	Loss: 1.326132
Train Epoch: 10 [39040/50000 (78%)]	Loss: 1.229021
Train Epoch: 10 [39680/50000 (79%)]	Loss: 1.527701
Train Epoch: 10 [40320/50000 (81%)]	Loss: 1.292407
Train Epoch: 10 [40960/50000 (82%)]	Loss: 1.517111
Train Epoch: 10 [41600/50000 (83%)]	Loss: 1.358878
Train Epoch: 10 [42240/50000 (84%)]	Loss: 1.491540
Train Epoch: 10 [42880/50000 (86%)]	Loss: 1.472763
Train Epoch: 10 [43520/50000 (87%)]	Loss: 1.615120
Train Epoch: 10 [44160/50000 (88%)]	Loss: 1.117285
Train Epoch: 10 [44800/50000 (90%)]	Loss: 1.470718
Train Epoch: 10 [45440/50000 (91%)]	Loss: 1.215634
Train Epoch: 10 [46080/50000 (92%)]	Loss: 1.218313
Train Epoch: 10 [46720/50000 (93%)]	Loss: 1.502631
Train Epoch: 10 [47360/50000 (95%)]	Loss: 1.236326
Train Epoch: 10 [48000/50000 (96%)]	Loss: 1.447079
Train Epoch: 10 [48640/50000 (97%)]	Loss: 1.217937
Train Epoch: 10 [49280/50000 (98%)]	Loss: 1.329474
Train Epoch: 10 [49920/50000 (100%)]	Loss: 1.507504

Test set: Average loss: 1.2715, Accuracy: 5499/10000 (55%)







    













    













    













    




<class 'torch.FloatTensor'> torch.Size([6, 3, 5, 5])
<class 'torch.FloatTensor'> torch.Size([3, 14, 21])

In [ ]: