In [2]:

    

from __future__ import print_function
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms
from torch.autograd import Variable

import math
import numpy as np
import matplotlib.pyplot as plt
from torchvision.utils import make_grid
from collections import OrderedDict

# Training settings
# for terminal use. In notebook, you can't parse arguments
class args:
    cuda = False
    batch_size = 64
    test_batch_size = 1000
    epochs = 10
    lr = 0.01
    momentum = 0.5
    no_cuda = False
    seed = 1
    log_interval = 10
    # if add Dropout
    with_dropout = True
    # if initialize weights
    with_init_weights = True
    # if add BatchNorm
    with_batchnorm = False

args.cuda = not args.no_cuda and torch.cuda.is_available()

torch.manual_seed(args.seed)
if args.cuda:
    torch.cuda.manual_seed(args.seed)

kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {}
# The output of torchvision datasets are PILImage images of range [0, 1].
# We transform them to Tensors of normalized range [-1, 1]
transform = transforms.Compose([transforms.ToTensor(),
                              transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
                             ])
trainset = datasets.CIFAR10(root='./data', train=True, download=True, transform=transform)
train_loader = torch.utils.data.DataLoader(trainset, batch_size=args.batch_size,
                                          shuffle=True, num_workers=2)

testset = datasets.CIFAR10(root='./data', train=False, download=True, transform=transform)
test_loader = torch.utils.data.DataLoader(testset, batch_size=args.test_batch_size,
                                          shuffle=False, num_workers=2)
classes = ('plane', 'car', 'bird', 'cat',
           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')


class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        # TODO: define your network here
        self.conv_1 = nn.Conv2d(3, 6, kernel_size=5, stride=1)
        self.conv_2 = nn.Conv2d(6, 16, kernel_size=5, stride=1)
        # TODO: replace fc with conv
        self.fc_1 = nn.Linear(16 * 25, 120)
        self.fc_2 = nn.Linear(120, 84)
        self.fc_3 = nn.Linear(84, 10)
        if args.with_batchnorm:
            self.block_conv_1 = nn.Sequential(
                self.conv_1,
                nn.BatchNorm2d(6),
                nn.ReLU(),
                nn.MaxPool2d(kernel_size=2, stride=2)
            )
            self.block_conv_2 = nn.Sequential(
                self.conv_2,
                nn.BatchNorm2d(16),
                nn.ReLU(),
                nn.MaxPool2d(kernel_size=2, stride=2)
            )
        else:
            self.block_conv_1 = nn.Sequential(
                self.conv_1,
                nn.ReLU(),
                nn.MaxPool2d(kernel_size=2, stride=2)
            )
            self.block_conv_2 = nn.Sequential(
                self.conv_2,
                nn.ReLU(),
                nn.MaxPool2d(kernel_size=2, stride=2)
            )
        if args.with_dropout:
            if args.with_batchnorm:
                self.block_fc_1 = nn.Sequential(
                    self.fc_1,
                    nn.BatchNorm1d(120),
                    nn.Dropout()
                )
                self.block_fc_2 = nn.Sequential(
                    self.fc_2,
                    nn.BatchNorm1d(84),
                    nn.Dropout()
                )
            else:
                self.block_fc_1 = nn.Sequential(
                    self.fc_1,
                    nn.Dropout()
                )
                self.block_fc_2 = nn.Sequential(
                    self.fc_2,
                    nn.Dropout()
                )
        else:
            self.block_fc_1 = self.fc_1
            self.block_fc_2 = self.fc_2
        self.softmax = nn.LogSoftmax()
        # Initialize parameters
        if args.with_init_weights:
            for m in self.modules():
                if isinstance(m, nn.Conv2d):
                    n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                    m.weight.data.normal_(0, math.sqrt(2. /n))
                    if m.bias is not None:
                        m.bias.data.zero_()
                elif isinstance(m, nn.Linear):
                    n = m.out_features
                    m.weight.data.normal_(0, math.sqrt(2. /n))
                    if m.bias is not None:
                        m.bias.data.zero_()
                elif isinstance(m, nn.BatchNorm1d) or isinstance(m, nn.BatchNorm2d):
                    m.weight.data.fill_(1)
                    m.bias.data.zero_()

    def forward(self, x):
        # TODO
        x = self.block_conv_1(x)
        x = self.block_conv_2(x)
        x = x.view(x.size(0), -1)
        x = self.block_fc_1(x)
        x = self.block_fc_2(x)
        x = self.fc_3(x)
        x = self.softmax(x)
        return x

# Filter extractor for filter visualization
class FeatureExtractor(nn.Module):
    def __init__(self, model, layer_names):
        super(FeatureExtractor, self).__init__()
        self._model = model
        self._layer_names = set(layer_names)
    def forward(self):
        filters = dict()
        # _modules is an OrderedDict, which replace iteritems() with items() in python3.* 
        for name, module in self._model._modules.items():
            if name in self._layer_names:
                filters[name] = module.weight.data.cpu()
        return filters

# Vesualize training results and trained filters
class VisualizedResult():
    def __init__(self, model):
        self._model = model
    def training_curve(self, epoches, train_loss_records, test_loss_records):
        fig = plt.figure()
        ax_train = fig.add_subplot(111)
        ax_test = fig.add_subplot(111)
        plt.axis([1, epoches, 0, math.ceil(max(train_loss_records + test_loss_records) * 1.2)])
        plt.xlabel('Epoches')
        plt.ylabel('Loss')
        plt.title('Training Curve')
        plt.plot(range(1, epoches + 1), train_loss_records, 'b-', label="train loss")
        plt.plot(range(1, epoches + 1), test_loss_records, 'r-', label="test loss")
        for xy in zip(range(1, epoches + 1), train_loss_records):
            ax_train.annotate('%.2f' % xy[1], xy=xy, textcoords='data')
        for xy in zip(range(1, epoches + 1), test_loss_records):
            ax_test.annotate('%.2f' % xy[1], xy=xy, textcoords='data')
        plt.legend(loc='upper right', borderaxespad=0.)
        plt.show()
    def accuracy_curve(self, epoches, accuracy_records):
        fig = plt.figure()
        ax = fig.add_subplot(111)
        plt.axis([1, epoches, 0, 100])
        plt.xlabel('Epoches')
        plt.ylabel('Accuracy')
        plt.title('Accuracy Curve')
        plt.plot(range(1, epoches + 1), accuracy_records, '-')
        for xy in zip(range(1, epoches + 1), accuracy_records):
            ax.annotate('%s%%' % xy[1], xy=xy, textcoords='data')
        plt.show()
    def conv_filter(self, layer_names):
        feature_extractor = FeatureExtractor(self._model, layer_names)
        filter_list = feature_extractor.forward()
        for name, filters in filter_list.items():
            print(type(filters), filters.size())
            # Normalize filters
            mean = torch.mean(filters)
            std = torch.std(filters)
            filters = (filters - mean) / std
            img = make_grid(filters, nrow=3)
            print(type(img), img.size())
            npimg = img.numpy()
            plt.imshow(np.transpose(npimg, (1,2,0)), interpolation='nearest')


model = Net()
if args.cuda:
    model.cuda()
    
# TODO: other optimizers
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum)

train_loss_records = list()
test_loss_records = list()
accuracy_records = list()

def train(epoch):
    model.train()
    train_loss = 0
    for batch_idx, (data, target) in enumerate(train_loader):
        if args.cuda:
            data, target = data.cuda(), target.cuda()
        data, target = Variable(data), Variable(target)
        optimizer.zero_grad()
        output = model(data)
        loss = F.nll_loss(output, target)   # is it true to use such a loss over cross-entropy loss? 
        loss.backward()
        optimizer.step()
        train_loss += loss.data[0]
        if batch_idx % args.log_interval == 0:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch, batch_idx * len(data), len(train_loader.dataset),
                100. * batch_idx / len(train_loader), loss.data[0]))
    # Average training loss for this epoch
    train_loss_records.append(train_loss / len(train_loader))

def test(epoch):
    model.eval()
    test_loss = 0
    correct = 0
    for data, target in test_loader:
        if args.cuda:
            data, target = data.cuda(), target.cuda()
        data, target = Variable(data, volatile=True), Variable(target)
        output = model(data)
        test_loss += F.nll_loss(output, target).data[0]
        pred = output.data.max(1)[1] # get the index of the max log-probability
        correct += pred.eq(target.data).cpu().sum()

    test_loss = test_loss
    test_loss /= len(test_loader) # loss function already averages over batch size
    accuracy = 100. * correct / len(test_loader.dataset)
    print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
        test_loss, correct, len(test_loader.dataset),
        accuracy))
    test_loss_records.append(test_loss)
    accuracy_records.append(accuracy)


for epoch in range(1, args.epochs + 1):
    train(epoch)
    test(epoch)

visual_result = VisualizedResult(model)
# Visualize training curve
visual_result.training_curve(args.epochs, train_loss_records, test_loss_records)
# Visualize test accuracy
visual_result.accuracy_curve(args.epochs, accuracy_records)
# Visualize trained filter on the 1st Conv layer
visual_result.conv_filter(['conv_1'])


    

    




Files already downloaded and verified
Files already downloaded and verified
Train Epoch: 1 [0/50000 (0%)]	Loss: 15.234482
Train Epoch: 1 [640/50000 (1%)]	Loss: 2.326665
Train Epoch: 1 [1280/50000 (3%)]	Loss: 2.362413
Train Epoch: 1 [1920/50000 (4%)]	Loss: 2.276755
Train Epoch: 1 [2560/50000 (5%)]	Loss: 2.338002
Train Epoch: 1 [3200/50000 (6%)]	Loss: 2.348285
Train Epoch: 1 [3840/50000 (8%)]	Loss: 2.332980
Train Epoch: 1 [4480/50000 (9%)]	Loss: 2.341329
Train Epoch: 1 [5120/50000 (10%)]	Loss: 2.301755
Train Epoch: 1 [5760/50000 (12%)]	Loss: 2.284743
Train Epoch: 1 [6400/50000 (13%)]	Loss: 2.300001
Train Epoch: 1 [7040/50000 (14%)]	Loss: 2.277496
Train Epoch: 1 [7680/50000 (15%)]	Loss: 2.302809
Train Epoch: 1 [8320/50000 (17%)]	Loss: 2.395759
Train Epoch: 1 [8960/50000 (18%)]	Loss: 2.345976
Train Epoch: 1 [9600/50000 (19%)]	Loss: 2.294946
Train Epoch: 1 [10240/50000 (20%)]	Loss: 2.292124
Train Epoch: 1 [10880/50000 (22%)]	Loss: 2.293660
Train Epoch: 1 [11520/50000 (23%)]	Loss: 2.333412
Train Epoch: 1 [12160/50000 (24%)]	Loss: 2.329465
Train Epoch: 1 [12800/50000 (26%)]	Loss: 2.305642
Train Epoch: 1 [13440/50000 (27%)]	Loss: 2.316200
Train Epoch: 1 [14080/50000 (28%)]	Loss: 2.303170
Train Epoch: 1 [14720/50000 (29%)]	Loss: 2.342990
Train Epoch: 1 [15360/50000 (31%)]	Loss: 2.341306
Train Epoch: 1 [16000/50000 (32%)]	Loss: 2.279560
Train Epoch: 1 [16640/50000 (33%)]	Loss: 2.339725
Train Epoch: 1 [17280/50000 (35%)]	Loss: 2.327679
Train Epoch: 1 [17920/50000 (36%)]	Loss: 2.313869
Train Epoch: 1 [18560/50000 (37%)]	Loss: 2.326478
Train Epoch: 1 [19200/50000 (38%)]	Loss: 2.319654
Train Epoch: 1 [19840/50000 (40%)]	Loss: 2.324892
Train Epoch: 1 [20480/50000 (41%)]	Loss: 2.315763
Train Epoch: 1 [21120/50000 (42%)]	Loss: 2.307356
Train Epoch: 1 [21760/50000 (43%)]	Loss: 2.289115
Train Epoch: 1 [22400/50000 (45%)]	Loss: 2.315076
Train Epoch: 1 [23040/50000 (46%)]	Loss: 2.341220
Train Epoch: 1 [23680/50000 (47%)]	Loss: 2.295772
Train Epoch: 1 [24320/50000 (49%)]	Loss: 2.329668
Train Epoch: 1 [24960/50000 (50%)]	Loss: 2.330588
Train Epoch: 1 [25600/50000 (51%)]	Loss: 2.294637
Train Epoch: 1 [26240/50000 (52%)]	Loss: 2.314350
Train Epoch: 1 [26880/50000 (54%)]	Loss: 2.379716
Train Epoch: 1 [27520/50000 (55%)]	Loss: 2.298050
Train Epoch: 1 [28160/50000 (56%)]	Loss: 2.324712
Train Epoch: 1 [28800/50000 (58%)]	Loss: 2.303374
Train Epoch: 1 [29440/50000 (59%)]	Loss: 2.345768
Train Epoch: 1 [30080/50000 (60%)]	Loss: 2.272227
Train Epoch: 1 [30720/50000 (61%)]	Loss: 2.303907
Train Epoch: 1 [31360/50000 (63%)]	Loss: 2.331093
Train Epoch: 1 [32000/50000 (64%)]	Loss: 2.319263
Train Epoch: 1 [32640/50000 (65%)]	Loss: 2.288735
Train Epoch: 1 [33280/50000 (66%)]	Loss: 2.335385
Train Epoch: 1 [33920/50000 (68%)]	Loss: 2.312720
Train Epoch: 1 [34560/50000 (69%)]	Loss: 2.351252
Train Epoch: 1 [35200/50000 (70%)]	Loss: 2.334589
Train Epoch: 1 [35840/50000 (72%)]	Loss: 2.319540
Train Epoch: 1 [36480/50000 (73%)]	Loss: 2.317049
Train Epoch: 1 [37120/50000 (74%)]	Loss: 2.299483
Train Epoch: 1 [37760/50000 (75%)]	Loss: 2.358359
Train Epoch: 1 [38400/50000 (77%)]	Loss: 2.334314
Train Epoch: 1 [39040/50000 (78%)]	Loss: 2.297088
Train Epoch: 1 [39680/50000 (79%)]	Loss: 2.321880
Train Epoch: 1 [40320/50000 (81%)]	Loss: 2.320149
Train Epoch: 1 [40960/50000 (82%)]	Loss: 2.317154
Train Epoch: 1 [41600/50000 (83%)]	Loss: 2.318118
Train Epoch: 1 [42240/50000 (84%)]	Loss: 2.279813
Train Epoch: 1 [42880/50000 (86%)]	Loss: 2.326334
Train Epoch: 1 [43520/50000 (87%)]	Loss: 2.313096
Train Epoch: 1 [44160/50000 (88%)]	Loss: 2.313335
Train Epoch: 1 [44800/50000 (90%)]	Loss: 2.311134
Train Epoch: 1 [45440/50000 (91%)]	Loss: 2.374033
Train Epoch: 1 [46080/50000 (92%)]	Loss: 2.362906
Train Epoch: 1 [46720/50000 (93%)]	Loss: 2.321835
Train Epoch: 1 [47360/50000 (95%)]	Loss: 2.354999
Train Epoch: 1 [48000/50000 (96%)]	Loss: 2.312239
Train Epoch: 1 [48640/50000 (97%)]	Loss: 2.344520
Train Epoch: 1 [49280/50000 (98%)]	Loss: 2.257581
Train Epoch: 1 [49920/50000 (100%)]	Loss: 2.283464

Test set: Average loss: 2.3069, Accuracy: 998/10000 (10%)

Train Epoch: 2 [0/50000 (0%)]	Loss: 2.327988
Train Epoch: 2 [640/50000 (1%)]	Loss: 2.308882
Train Epoch: 2 [1280/50000 (3%)]	Loss: 2.313838
Train Epoch: 2 [1920/50000 (4%)]	Loss: 2.263880
Train Epoch: 2 [2560/50000 (5%)]	Loss: 2.319611
Train Epoch: 2 [3200/50000 (6%)]	Loss: 2.295112
Train Epoch: 2 [3840/50000 (8%)]	Loss: 2.313395
Train Epoch: 2 [4480/50000 (9%)]	Loss: 2.309272
Train Epoch: 2 [5120/50000 (10%)]	Loss: 2.313822
Train Epoch: 2 [5760/50000 (12%)]	Loss: 2.302895
Train Epoch: 2 [6400/50000 (13%)]	Loss: 2.335178
Train Epoch: 2 [7040/50000 (14%)]	Loss: 2.318026
Train Epoch: 2 [7680/50000 (15%)]	Loss: 2.320074
Train Epoch: 2 [8320/50000 (17%)]	Loss: 2.333184
Train Epoch: 2 [8960/50000 (18%)]	Loss: 2.298934
Train Epoch: 2 [9600/50000 (19%)]	Loss: 2.308050
Train Epoch: 2 [10240/50000 (20%)]	Loss: 2.304924
Train Epoch: 2 [10880/50000 (22%)]	Loss: 2.318158
Train Epoch: 2 [11520/50000 (23%)]	Loss: 2.341682
Train Epoch: 2 [12160/50000 (24%)]	Loss: 2.324960
Train Epoch: 2 [12800/50000 (26%)]	Loss: 2.325027
Train Epoch: 2 [13440/50000 (27%)]	Loss: 2.317587
Train Epoch: 2 [14080/50000 (28%)]	Loss: 2.270528
Train Epoch: 2 [14720/50000 (29%)]	Loss: 2.327427
Train Epoch: 2 [15360/50000 (31%)]	Loss: 2.329642
Train Epoch: 2 [16000/50000 (32%)]	Loss: 2.342058
Train Epoch: 2 [16640/50000 (33%)]	Loss: 2.302392
Train Epoch: 2 [17280/50000 (35%)]	Loss: 2.302927
Train Epoch: 2 [17920/50000 (36%)]	Loss: 2.305215
Train Epoch: 2 [18560/50000 (37%)]	Loss: 2.305821
Train Epoch: 2 [19200/50000 (38%)]	Loss: 2.292023
Train Epoch: 2 [19840/50000 (40%)]	Loss: 2.350047
Train Epoch: 2 [20480/50000 (41%)]	Loss: 2.321965
Train Epoch: 2 [21120/50000 (42%)]	Loss: 2.307122
Train Epoch: 2 [21760/50000 (43%)]	Loss: 2.331765
Train Epoch: 2 [22400/50000 (45%)]	Loss: 2.294459
Train Epoch: 2 [23040/50000 (46%)]	Loss: 2.330719
Train Epoch: 2 [23680/50000 (47%)]	Loss: 2.331752
Train Epoch: 2 [24320/50000 (49%)]	Loss: 2.301518
Train Epoch: 2 [24960/50000 (50%)]	Loss: 2.334546
Train Epoch: 2 [25600/50000 (51%)]	Loss: 2.329739
Train Epoch: 2 [26240/50000 (52%)]	Loss: 2.292220
Train Epoch: 2 [26880/50000 (54%)]	Loss: 2.354358
Train Epoch: 2 [27520/50000 (55%)]	Loss: 2.305916
Train Epoch: 2 [28160/50000 (56%)]	Loss: 2.347701
Train Epoch: 2 [28800/50000 (58%)]	Loss: 2.346179
Train Epoch: 2 [29440/50000 (59%)]	Loss: 2.274035
Train Epoch: 2 [30080/50000 (60%)]	Loss: 2.320318
Train Epoch: 2 [30720/50000 (61%)]	Loss: 2.249168
Train Epoch: 2 [31360/50000 (63%)]	Loss: 2.322700
Train Epoch: 2 [32000/50000 (64%)]	Loss: 2.327506
Train Epoch: 2 [32640/50000 (65%)]	Loss: 2.290091
Train Epoch: 2 [33280/50000 (66%)]	Loss: 2.334734
Train Epoch: 2 [33920/50000 (68%)]	Loss: 2.241594
Train Epoch: 2 [34560/50000 (69%)]	Loss: 2.267661
Train Epoch: 2 [35200/50000 (70%)]	Loss: 2.376510
Train Epoch: 2 [35840/50000 (72%)]	Loss: 2.229009
Train Epoch: 2 [36480/50000 (73%)]	Loss: 2.315463
Train Epoch: 2 [37120/50000 (74%)]	Loss: 2.290347
Train Epoch: 2 [37760/50000 (75%)]	Loss: 2.152772
Train Epoch: 2 [38400/50000 (77%)]	Loss: 2.223003
Train Epoch: 2 [39040/50000 (78%)]	Loss: 2.237346
Train Epoch: 2 [39680/50000 (79%)]	Loss: 2.185187
Train Epoch: 2 [40320/50000 (81%)]	Loss: 2.303223
Train Epoch: 2 [40960/50000 (82%)]	Loss: 2.379887
Train Epoch: 2 [41600/50000 (83%)]	Loss: 2.200854
Train Epoch: 2 [42240/50000 (84%)]	Loss: 2.297234
Train Epoch: 2 [42880/50000 (86%)]	Loss: 2.148873
Train Epoch: 2 [43520/50000 (87%)]	Loss: 2.203217
Train Epoch: 2 [44160/50000 (88%)]	Loss: 2.112968
Train Epoch: 2 [44800/50000 (90%)]	Loss: 2.210772
Train Epoch: 2 [45440/50000 (91%)]	Loss: 2.246602
Train Epoch: 2 [46080/50000 (92%)]	Loss: 2.259461
Train Epoch: 2 [46720/50000 (93%)]	Loss: 2.158373
Train Epoch: 2 [47360/50000 (95%)]	Loss: 2.189588
Train Epoch: 2 [48000/50000 (96%)]	Loss: 2.149499
Train Epoch: 2 [48640/50000 (97%)]	Loss: 2.213547
Train Epoch: 2 [49280/50000 (98%)]	Loss: 2.020793
Train Epoch: 2 [49920/50000 (100%)]	Loss: 2.196411

Test set: Average loss: 2.0969, Accuracy: 1916/10000 (19%)

Train Epoch: 3 [0/50000 (0%)]	Loss: 2.306422
Train Epoch: 3 [640/50000 (1%)]	Loss: 2.206799
Train Epoch: 3 [1280/50000 (3%)]	Loss: 2.098496
Train Epoch: 3 [1920/50000 (4%)]	Loss: 2.112940
Train Epoch: 3 [2560/50000 (5%)]	Loss: 2.271595
Train Epoch: 3 [3200/50000 (6%)]	Loss: 2.108510
Train Epoch: 3 [3840/50000 (8%)]	Loss: 2.371243
Train Epoch: 3 [4480/50000 (9%)]	Loss: 2.227545
Train Epoch: 3 [5120/50000 (10%)]	Loss: 2.004561
Train Epoch: 3 [5760/50000 (12%)]	Loss: 2.035899
Train Epoch: 3 [6400/50000 (13%)]	Loss: 2.370530
Train Epoch: 3 [7040/50000 (14%)]	Loss: 2.262175
Train Epoch: 3 [7680/50000 (15%)]	Loss: 2.155239
Train Epoch: 3 [8320/50000 (17%)]	Loss: 2.213289
Train Epoch: 3 [8960/50000 (18%)]	Loss: 2.225273
Train Epoch: 3 [9600/50000 (19%)]	Loss: 2.174731
Train Epoch: 3 [10240/50000 (20%)]	Loss: 2.144168
Train Epoch: 3 [10880/50000 (22%)]	Loss: 2.350580
Train Epoch: 3 [11520/50000 (23%)]	Loss: 2.226025
Train Epoch: 3 [12160/50000 (24%)]	Loss: 2.065709
Train Epoch: 3 [12800/50000 (26%)]	Loss: 2.219628
Train Epoch: 3 [13440/50000 (27%)]	Loss: 2.317563
Train Epoch: 3 [14080/50000 (28%)]	Loss: 2.137935
Train Epoch: 3 [14720/50000 (29%)]	Loss: 2.219647
Train Epoch: 3 [15360/50000 (31%)]	Loss: 2.206353
Train Epoch: 3 [16000/50000 (32%)]	Loss: 2.330482
Train Epoch: 3 [16640/50000 (33%)]	Loss: 2.064996
Train Epoch: 3 [17280/50000 (35%)]	Loss: 2.157810
Train Epoch: 3 [17920/50000 (36%)]	Loss: 2.085413
Train Epoch: 3 [18560/50000 (37%)]	Loss: 2.041637
Train Epoch: 3 [19200/50000 (38%)]	Loss: 2.298619
Train Epoch: 3 [19840/50000 (40%)]	Loss: 1.938862
Train Epoch: 3 [20480/50000 (41%)]	Loss: 2.047025
Train Epoch: 3 [21120/50000 (42%)]	Loss: 2.056959
Train Epoch: 3 [21760/50000 (43%)]	Loss: 2.146079
Train Epoch: 3 [22400/50000 (45%)]	Loss: 2.172124
Train Epoch: 3 [23040/50000 (46%)]	Loss: 2.132922
Train Epoch: 3 [23680/50000 (47%)]	Loss: 2.265749
Train Epoch: 3 [24320/50000 (49%)]	Loss: 2.149521
Train Epoch: 3 [24960/50000 (50%)]	Loss: 2.048922
Train Epoch: 3 [25600/50000 (51%)]	Loss: 2.073114
Train Epoch: 3 [26240/50000 (52%)]	Loss: 2.134690
Train Epoch: 3 [26880/50000 (54%)]	Loss: 2.300155
Train Epoch: 3 [27520/50000 (55%)]	Loss: 2.035869
Train Epoch: 3 [28160/50000 (56%)]	Loss: 2.630218
Train Epoch: 3 [28800/50000 (58%)]	Loss: 2.184316
Train Epoch: 3 [29440/50000 (59%)]	Loss: 2.125330
Train Epoch: 3 [30080/50000 (60%)]	Loss: 2.028318
Train Epoch: 3 [30720/50000 (61%)]	Loss: 2.138449
Train Epoch: 3 [31360/50000 (63%)]	Loss: 2.074783
Train Epoch: 3 [32000/50000 (64%)]	Loss: 2.055889
Train Epoch: 3 [32640/50000 (65%)]	Loss: 2.243020
Train Epoch: 3 [33280/50000 (66%)]	Loss: 2.090866
Train Epoch: 3 [33920/50000 (68%)]	Loss: 2.125515
Train Epoch: 3 [34560/50000 (69%)]	Loss: 2.257854
Train Epoch: 3 [35200/50000 (70%)]	Loss: 2.155568
Train Epoch: 3 [35840/50000 (72%)]	Loss: 2.145775
Train Epoch: 3 [36480/50000 (73%)]	Loss: 2.152233
Train Epoch: 3 [37120/50000 (74%)]	Loss: 2.258351
Train Epoch: 3 [37760/50000 (75%)]	Loss: 2.085243
Train Epoch: 3 [38400/50000 (77%)]	Loss: 2.128927
Train Epoch: 3 [39040/50000 (78%)]	Loss: 2.093739
Train Epoch: 3 [39680/50000 (79%)]	Loss: 1.963810
Train Epoch: 3 [40320/50000 (81%)]	Loss: 2.257877
Train Epoch: 3 [40960/50000 (82%)]	Loss: 2.102691
Train Epoch: 3 [41600/50000 (83%)]	Loss: 2.003911
Train Epoch: 3 [42240/50000 (84%)]	Loss: 1.956015
Train Epoch: 3 [42880/50000 (86%)]	Loss: 2.181247
Train Epoch: 3 [43520/50000 (87%)]	Loss: 2.007243
Train Epoch: 3 [44160/50000 (88%)]	Loss: 2.106718
Train Epoch: 3 [44800/50000 (90%)]	Loss: 2.514239
Train Epoch: 3 [45440/50000 (91%)]	Loss: 1.999719
Train Epoch: 3 [46080/50000 (92%)]	Loss: 2.018809
Train Epoch: 3 [46720/50000 (93%)]	Loss: 2.102132
Train Epoch: 3 [47360/50000 (95%)]	Loss: 1.983535
Train Epoch: 3 [48000/50000 (96%)]	Loss: 1.965022
Train Epoch: 3 [48640/50000 (97%)]	Loss: 2.021034
Train Epoch: 3 [49280/50000 (98%)]	Loss: 2.139489
Train Epoch: 3 [49920/50000 (100%)]	Loss: 2.127376

Test set: Average loss: 2.0276, Accuracy: 2418/10000 (24%)

Train Epoch: 4 [0/50000 (0%)]	Loss: 2.075849
Train Epoch: 4 [640/50000 (1%)]	Loss: 2.204354
Train Epoch: 4 [1280/50000 (3%)]	Loss: 2.268369
Train Epoch: 4 [1920/50000 (4%)]	Loss: 2.279496
Train Epoch: 4 [2560/50000 (5%)]	Loss: 2.030153
Train Epoch: 4 [3200/50000 (6%)]	Loss: 2.273809
Train Epoch: 4 [3840/50000 (8%)]	Loss: 2.119172
Train Epoch: 4 [4480/50000 (9%)]	Loss: 1.903348
Train Epoch: 4 [5120/50000 (10%)]	Loss: 2.247708
Train Epoch: 4 [5760/50000 (12%)]	Loss: 2.270135
Train Epoch: 4 [6400/50000 (13%)]	Loss: 2.027009
Train Epoch: 4 [7040/50000 (14%)]	Loss: 2.158152
Train Epoch: 4 [7680/50000 (15%)]	Loss: 1.955701
Train Epoch: 4 [8320/50000 (17%)]	Loss: 2.135325
Train Epoch: 4 [8960/50000 (18%)]	Loss: 2.022332
Train Epoch: 4 [9600/50000 (19%)]	Loss: 1.976000
Train Epoch: 4 [10240/50000 (20%)]	Loss: 2.079260
Train Epoch: 4 [10880/50000 (22%)]	Loss: 2.147722
Train Epoch: 4 [11520/50000 (23%)]	Loss: 2.144507
Train Epoch: 4 [12160/50000 (24%)]	Loss: 2.035445
Train Epoch: 4 [12800/50000 (26%)]	Loss: 2.006926
Train Epoch: 4 [13440/50000 (27%)]	Loss: 2.100230
Train Epoch: 4 [14080/50000 (28%)]	Loss: 2.081473
Train Epoch: 4 [14720/50000 (29%)]	Loss: 2.095406
Train Epoch: 4 [15360/50000 (31%)]	Loss: 2.033184
Train Epoch: 4 [16000/50000 (32%)]	Loss: 1.965035
Train Epoch: 4 [16640/50000 (33%)]	Loss: 2.019084
Train Epoch: 4 [17280/50000 (35%)]	Loss: 2.039365
Train Epoch: 4 [17920/50000 (36%)]	Loss: 2.201093
Train Epoch: 4 [18560/50000 (37%)]	Loss: 2.051298
Train Epoch: 4 [19200/50000 (38%)]	Loss: 2.051857
Train Epoch: 4 [19840/50000 (40%)]	Loss: 2.092512
Train Epoch: 4 [20480/50000 (41%)]	Loss: 2.143448
Train Epoch: 4 [21120/50000 (42%)]	Loss: 1.914022
Train Epoch: 4 [21760/50000 (43%)]	Loss: 2.040801
Train Epoch: 4 [22400/50000 (45%)]	Loss: 2.140770
Train Epoch: 4 [23040/50000 (46%)]	Loss: 2.108389
Train Epoch: 4 [23680/50000 (47%)]	Loss: 2.245202
Train Epoch: 4 [24320/50000 (49%)]	Loss: 2.326468
Train Epoch: 4 [24960/50000 (50%)]	Loss: 1.911041
Train Epoch: 4 [25600/50000 (51%)]	Loss: 2.147271
Train Epoch: 4 [26240/50000 (52%)]	Loss: 2.082733
Train Epoch: 4 [26880/50000 (54%)]	Loss: 2.262088
Train Epoch: 4 [27520/50000 (55%)]	Loss: 2.180468
Train Epoch: 4 [28160/50000 (56%)]	Loss: 2.022300
Train Epoch: 4 [28800/50000 (58%)]	Loss: 2.029557
Train Epoch: 4 [29440/50000 (59%)]	Loss: 2.108352
Train Epoch: 4 [30080/50000 (60%)]	Loss: 1.973915
Train Epoch: 4 [30720/50000 (61%)]	Loss: 2.171720
Train Epoch: 4 [31360/50000 (63%)]	Loss: 1.996191
Train Epoch: 4 [32000/50000 (64%)]	Loss: 2.052363
Train Epoch: 4 [32640/50000 (65%)]	Loss: 2.026482
Train Epoch: 4 [33280/50000 (66%)]	Loss: 2.110202
Train Epoch: 4 [33920/50000 (68%)]	Loss: 2.091097
Train Epoch: 4 [34560/50000 (69%)]	Loss: 2.205328
Train Epoch: 4 [35200/50000 (70%)]	Loss: 2.055409
Train Epoch: 4 [35840/50000 (72%)]	Loss: 2.058639
Train Epoch: 4 [36480/50000 (73%)]	Loss: 2.047452
Train Epoch: 4 [37120/50000 (74%)]	Loss: 2.129809
Train Epoch: 4 [37760/50000 (75%)]	Loss: 2.101331
Train Epoch: 4 [38400/50000 (77%)]	Loss: 2.023924
Train Epoch: 4 [39040/50000 (78%)]	Loss: 1.916670
Train Epoch: 4 [39680/50000 (79%)]	Loss: 1.889472
Train Epoch: 4 [40320/50000 (81%)]	Loss: 2.051194
Train Epoch: 4 [40960/50000 (82%)]	Loss: 2.137073
Train Epoch: 4 [41600/50000 (83%)]	Loss: 2.116266
Train Epoch: 4 [42240/50000 (84%)]	Loss: 2.307839
Train Epoch: 4 [42880/50000 (86%)]	Loss: 2.086299
Train Epoch: 4 [43520/50000 (87%)]	Loss: 2.189579
Train Epoch: 4 [44160/50000 (88%)]	Loss: 2.147873
Train Epoch: 4 [44800/50000 (90%)]	Loss: 1.948555
Train Epoch: 4 [45440/50000 (91%)]	Loss: 2.162765
Train Epoch: 4 [46080/50000 (92%)]	Loss: 1.993943
Train Epoch: 4 [46720/50000 (93%)]	Loss: 2.142312
Train Epoch: 4 [47360/50000 (95%)]	Loss: 2.005280
Train Epoch: 4 [48000/50000 (96%)]	Loss: 1.933312
Train Epoch: 4 [48640/50000 (97%)]	Loss: 2.029004
Train Epoch: 4 [49280/50000 (98%)]	Loss: 1.949520
Train Epoch: 4 [49920/50000 (100%)]	Loss: 2.542598

Test set: Average loss: 2.1292, Accuracy: 2122/10000 (21%)

Train Epoch: 5 [0/50000 (0%)]	Loss: 2.115952
Train Epoch: 5 [640/50000 (1%)]	Loss: 2.050179
Train Epoch: 5 [1280/50000 (3%)]	Loss: 2.214456
Train Epoch: 5 [1920/50000 (4%)]	Loss: 1.904170
Train Epoch: 5 [2560/50000 (5%)]	Loss: 2.161602
Train Epoch: 5 [3200/50000 (6%)]	Loss: 2.178347
Train Epoch: 5 [3840/50000 (8%)]	Loss: 2.102793
Train Epoch: 5 [4480/50000 (9%)]	Loss: 2.121258
Train Epoch: 5 [5120/50000 (10%)]	Loss: 1.968931
Train Epoch: 5 [5760/50000 (12%)]	Loss: 2.068469
Train Epoch: 5 [6400/50000 (13%)]	Loss: 1.922104
Train Epoch: 5 [7040/50000 (14%)]	Loss: 2.161368
Train Epoch: 5 [7680/50000 (15%)]	Loss: 2.035983
Train Epoch: 5 [8320/50000 (17%)]	Loss: 1.993749
Train Epoch: 5 [8960/50000 (18%)]	Loss: 1.983384
Train Epoch: 5 [9600/50000 (19%)]	Loss: 2.098975
Train Epoch: 5 [10240/50000 (20%)]	Loss: 1.916112
Train Epoch: 5 [10880/50000 (22%)]	Loss: 2.328267
Train Epoch: 5 [11520/50000 (23%)]	Loss: 1.974975
Train Epoch: 5 [12160/50000 (24%)]	Loss: 1.841718
Train Epoch: 5 [12800/50000 (26%)]	Loss: 2.029673
Train Epoch: 5 [13440/50000 (27%)]	Loss: 2.072996
Train Epoch: 5 [14080/50000 (28%)]	Loss: 1.969655
Train Epoch: 5 [14720/50000 (29%)]	Loss: 2.014158
Train Epoch: 5 [15360/50000 (31%)]	Loss: 2.111436
Train Epoch: 5 [16000/50000 (32%)]	Loss: 1.961482
Train Epoch: 5 [16640/50000 (33%)]	Loss: 1.845379
Train Epoch: 5 [17280/50000 (35%)]	Loss: 2.142802
Train Epoch: 5 [17920/50000 (36%)]	Loss: 2.005219
Train Epoch: 5 [18560/50000 (37%)]	Loss: 1.990158
Train Epoch: 5 [19200/50000 (38%)]	Loss: 1.988854
Train Epoch: 5 [19840/50000 (40%)]	Loss: 2.006960
Train Epoch: 5 [20480/50000 (41%)]	Loss: 1.702874
Train Epoch: 5 [21120/50000 (42%)]	Loss: 2.054947
Train Epoch: 5 [21760/50000 (43%)]	Loss: 1.962017
Train Epoch: 5 [22400/50000 (45%)]	Loss: 1.975580
Train Epoch: 5 [23040/50000 (46%)]	Loss: 2.088037
Train Epoch: 5 [23680/50000 (47%)]	Loss: 2.027845
Train Epoch: 5 [24320/50000 (49%)]	Loss: 1.979609
Train Epoch: 5 [24960/50000 (50%)]	Loss: 1.944486
Train Epoch: 5 [25600/50000 (51%)]	Loss: 1.941514
Train Epoch: 5 [26240/50000 (52%)]	Loss: 1.985896
Train Epoch: 5 [26880/50000 (54%)]	Loss: 1.938629
Train Epoch: 5 [27520/50000 (55%)]	Loss: 2.094266
Train Epoch: 5 [28160/50000 (56%)]	Loss: 1.958077
Train Epoch: 5 [28800/50000 (58%)]	Loss: 2.011293
Train Epoch: 5 [29440/50000 (59%)]	Loss: 1.994532
Train Epoch: 5 [30080/50000 (60%)]	Loss: 1.919662
Train Epoch: 5 [30720/50000 (61%)]	Loss: 2.063436
Train Epoch: 5 [31360/50000 (63%)]	Loss: 2.136988
Train Epoch: 5 [32000/50000 (64%)]	Loss: 2.053237
Train Epoch: 5 [32640/50000 (65%)]	Loss: 2.031528
Train Epoch: 5 [33280/50000 (66%)]	Loss: 2.145044
Train Epoch: 5 [33920/50000 (68%)]	Loss: 2.002869
Train Epoch: 5 [34560/50000 (69%)]	Loss: 2.278505
Train Epoch: 5 [35200/50000 (70%)]	Loss: 2.062405
Train Epoch: 5 [35840/50000 (72%)]	Loss: 2.041273
Train Epoch: 5 [36480/50000 (73%)]	Loss: 1.907793
Train Epoch: 5 [37120/50000 (74%)]	Loss: 1.907071
Train Epoch: 5 [37760/50000 (75%)]	Loss: 1.902908
Train Epoch: 5 [38400/50000 (77%)]	Loss: 1.937590
Train Epoch: 5 [39040/50000 (78%)]	Loss: 2.095053
Train Epoch: 5 [39680/50000 (79%)]	Loss: 2.027222
Train Epoch: 5 [40320/50000 (81%)]	Loss: 2.295191
Train Epoch: 5 [40960/50000 (82%)]	Loss: 2.026653
Train Epoch: 5 [41600/50000 (83%)]	Loss: 2.127628
Train Epoch: 5 [42240/50000 (84%)]	Loss: 1.814678
Train Epoch: 5 [42880/50000 (86%)]	Loss: 2.087906
Train Epoch: 5 [43520/50000 (87%)]	Loss: 1.913703
Train Epoch: 5 [44160/50000 (88%)]	Loss: 2.135661
Train Epoch: 5 [44800/50000 (90%)]	Loss: 2.157289
Train Epoch: 5 [45440/50000 (91%)]	Loss: 2.080997
Train Epoch: 5 [46080/50000 (92%)]	Loss: 2.028603
Train Epoch: 5 [46720/50000 (93%)]	Loss: 1.969225
Train Epoch: 5 [47360/50000 (95%)]	Loss: 1.951783
Train Epoch: 5 [48000/50000 (96%)]	Loss: 2.015783
Train Epoch: 5 [48640/50000 (97%)]	Loss: 1.844559
Train Epoch: 5 [49280/50000 (98%)]	Loss: 2.223309
Train Epoch: 5 [49920/50000 (100%)]	Loss: 2.093950

Test set: Average loss: 2.0526, Accuracy: 2449/10000 (24%)

Train Epoch: 6 [0/50000 (0%)]	Loss: 2.002920
Train Epoch: 6 [640/50000 (1%)]	Loss: 1.928186
Train Epoch: 6 [1280/50000 (3%)]	Loss: 2.091925
Train Epoch: 6 [1920/50000 (4%)]	Loss: 2.328976
Train Epoch: 6 [2560/50000 (5%)]	Loss: 1.946787
Train Epoch: 6 [3200/50000 (6%)]	Loss: 1.852911
Train Epoch: 6 [3840/50000 (8%)]	Loss: 1.865430
Train Epoch: 6 [4480/50000 (9%)]	Loss: 1.960812
Train Epoch: 6 [5120/50000 (10%)]	Loss: 1.977900
Train Epoch: 6 [5760/50000 (12%)]	Loss: 1.792630
Train Epoch: 6 [6400/50000 (13%)]	Loss: 1.950109
Train Epoch: 6 [7040/50000 (14%)]	Loss: 2.013797
Train Epoch: 6 [7680/50000 (15%)]	Loss: 2.012713
Train Epoch: 6 [8320/50000 (17%)]	Loss: 1.998851
Train Epoch: 6 [8960/50000 (18%)]	Loss: 2.170301
Train Epoch: 6 [9600/50000 (19%)]	Loss: 2.034332
Train Epoch: 6 [10240/50000 (20%)]	Loss: 2.011506
Train Epoch: 6 [10880/50000 (22%)]	Loss: 2.041695
Train Epoch: 6 [11520/50000 (23%)]	Loss: 2.055643
Train Epoch: 6 [12160/50000 (24%)]	Loss: 2.174808
Train Epoch: 6 [12800/50000 (26%)]	Loss: 1.892038
Train Epoch: 6 [13440/50000 (27%)]	Loss: 1.845046
Train Epoch: 6 [14080/50000 (28%)]	Loss: 2.193197
Train Epoch: 6 [14720/50000 (29%)]	Loss: 2.152091
Train Epoch: 6 [15360/50000 (31%)]	Loss: 2.133216
Train Epoch: 6 [16000/50000 (32%)]	Loss: 2.023378
Train Epoch: 6 [16640/50000 (33%)]	Loss: 1.911858
Train Epoch: 6 [17280/50000 (35%)]	Loss: 2.017907
Train Epoch: 6 [17920/50000 (36%)]	Loss: 1.989573
Train Epoch: 6 [18560/50000 (37%)]	Loss: 2.013970
Train Epoch: 6 [19200/50000 (38%)]	Loss: 2.010203
Train Epoch: 6 [19840/50000 (40%)]	Loss: 1.953214
Train Epoch: 6 [20480/50000 (41%)]	Loss: 1.961817
Train Epoch: 6 [21120/50000 (42%)]	Loss: 1.851370
Train Epoch: 6 [21760/50000 (43%)]	Loss: 1.974674
Train Epoch: 6 [22400/50000 (45%)]	Loss: 1.974819
Train Epoch: 6 [23040/50000 (46%)]	Loss: 1.947913
Train Epoch: 6 [23680/50000 (47%)]	Loss: 1.968214
Train Epoch: 6 [24320/50000 (49%)]	Loss: 1.810066
Train Epoch: 6 [24960/50000 (50%)]	Loss: 1.950521
Train Epoch: 6 [25600/50000 (51%)]	Loss: 1.817881
Train Epoch: 6 [26240/50000 (52%)]	Loss: 2.003724
Train Epoch: 6 [26880/50000 (54%)]	Loss: 2.116690
Train Epoch: 6 [27520/50000 (55%)]	Loss: 1.777156
Train Epoch: 6 [28160/50000 (56%)]	Loss: 1.990192
Train Epoch: 6 [28800/50000 (58%)]	Loss: 1.903774
Train Epoch: 6 [29440/50000 (59%)]	Loss: 2.070616
Train Epoch: 6 [30080/50000 (60%)]	Loss: 1.911737
Train Epoch: 6 [30720/50000 (61%)]	Loss: 2.027810
Train Epoch: 6 [31360/50000 (63%)]	Loss: 1.968807
Train Epoch: 6 [32000/50000 (64%)]	Loss: 1.888450
Train Epoch: 6 [32640/50000 (65%)]	Loss: 2.099772
Train Epoch: 6 [33280/50000 (66%)]	Loss: 1.952934
Train Epoch: 6 [33920/50000 (68%)]	Loss: 2.020525
Train Epoch: 6 [34560/50000 (69%)]	Loss: 1.905124
Train Epoch: 6 [35200/50000 (70%)]	Loss: 2.093081
Train Epoch: 6 [35840/50000 (72%)]	Loss: 1.926600
Train Epoch: 6 [36480/50000 (73%)]	Loss: 2.026047
Train Epoch: 6 [37120/50000 (74%)]	Loss: 2.009631
Train Epoch: 6 [37760/50000 (75%)]	Loss: 1.919343
Train Epoch: 6 [38400/50000 (77%)]	Loss: 1.760275
Train Epoch: 6 [39040/50000 (78%)]	Loss: 1.961376
Train Epoch: 6 [39680/50000 (79%)]	Loss: 2.417127
Train Epoch: 6 [40320/50000 (81%)]	Loss: 1.879882
Train Epoch: 6 [40960/50000 (82%)]	Loss: 1.901251
Train Epoch: 6 [41600/50000 (83%)]	Loss: 2.022744
Train Epoch: 6 [42240/50000 (84%)]	Loss: 1.894154
Train Epoch: 6 [42880/50000 (86%)]	Loss: 1.954560
Train Epoch: 6 [43520/50000 (87%)]	Loss: 1.871019
Train Epoch: 6 [44160/50000 (88%)]	Loss: 1.867875
Train Epoch: 6 [44800/50000 (90%)]	Loss: 1.934391
Train Epoch: 6 [45440/50000 (91%)]	Loss: 1.846697
Train Epoch: 6 [46080/50000 (92%)]	Loss: 2.030661
Train Epoch: 6 [46720/50000 (93%)]	Loss: 2.014485
Train Epoch: 6 [47360/50000 (95%)]	Loss: 1.930516
Train Epoch: 6 [48000/50000 (96%)]	Loss: 1.879198
Train Epoch: 6 [48640/50000 (97%)]	Loss: 2.102959
Train Epoch: 6 [49280/50000 (98%)]	Loss: 2.027366
Train Epoch: 6 [49920/50000 (100%)]	Loss: 1.972985

Test set: Average loss: 1.9137, Accuracy: 2815/10000 (28%)

Train Epoch: 7 [0/50000 (0%)]	Loss: 1.955945
Train Epoch: 7 [640/50000 (1%)]	Loss: 1.825789
Train Epoch: 7 [1280/50000 (3%)]	Loss: 1.801623
Train Epoch: 7 [1920/50000 (4%)]	Loss: 1.768140
Train Epoch: 7 [2560/50000 (5%)]	Loss: 1.835950
Train Epoch: 7 [3200/50000 (6%)]	Loss: 1.868144
Train Epoch: 7 [3840/50000 (8%)]	Loss: 1.998247
Train Epoch: 7 [4480/50000 (9%)]	Loss: 1.922696
Train Epoch: 7 [5120/50000 (10%)]	Loss: 1.706127
Train Epoch: 7 [5760/50000 (12%)]	Loss: 1.965174
Train Epoch: 7 [6400/50000 (13%)]	Loss: 1.769034
Train Epoch: 7 [7040/50000 (14%)]	Loss: 1.943519
Train Epoch: 7 [7680/50000 (15%)]	Loss: 1.991709
Train Epoch: 7 [8320/50000 (17%)]	Loss: 1.810754
Train Epoch: 7 [8960/50000 (18%)]	Loss: 2.064326
Train Epoch: 7 [9600/50000 (19%)]	Loss: 2.004882
Train Epoch: 7 [10240/50000 (20%)]	Loss: 1.949178
Train Epoch: 7 [10880/50000 (22%)]	Loss: 1.926697
Train Epoch: 7 [11520/50000 (23%)]	Loss: 1.886338
Train Epoch: 7 [12160/50000 (24%)]	Loss: 2.062935
Train Epoch: 7 [12800/50000 (26%)]	Loss: 1.958802
Train Epoch: 7 [13440/50000 (27%)]	Loss: 1.957595
Train Epoch: 7 [14080/50000 (28%)]	Loss: 2.008482
Train Epoch: 7 [14720/50000 (29%)]	Loss: 1.941395
Train Epoch: 7 [15360/50000 (31%)]	Loss: 1.989754
Train Epoch: 7 [16000/50000 (32%)]	Loss: 1.858981
Train Epoch: 7 [16640/50000 (33%)]	Loss: 2.027191
Train Epoch: 7 [17280/50000 (35%)]	Loss: 1.946899
Train Epoch: 7 [17920/50000 (36%)]	Loss: 1.993915
Train Epoch: 7 [18560/50000 (37%)]	Loss: 2.018660
Train Epoch: 7 [19200/50000 (38%)]	Loss: 1.696270
Train Epoch: 7 [19840/50000 (40%)]	Loss: 1.880224
Train Epoch: 7 [20480/50000 (41%)]	Loss: 2.036515
Train Epoch: 7 [21120/50000 (42%)]	Loss: 1.966823
Train Epoch: 7 [21760/50000 (43%)]	Loss: 1.969985
Train Epoch: 7 [22400/50000 (45%)]	Loss: 1.828195
Train Epoch: 7 [23040/50000 (46%)]	Loss: 1.935988
Train Epoch: 7 [23680/50000 (47%)]	Loss: 1.959527
Train Epoch: 7 [24320/50000 (49%)]	Loss: 2.041321
Train Epoch: 7 [24960/50000 (50%)]	Loss: 1.812598
Train Epoch: 7 [25600/50000 (51%)]	Loss: 1.942396
Train Epoch: 7 [26240/50000 (52%)]	Loss: 1.780158
Train Epoch: 7 [26880/50000 (54%)]	Loss: 1.983880
Train Epoch: 7 [27520/50000 (55%)]	Loss: 1.856362
Train Epoch: 7 [28160/50000 (56%)]	Loss: 2.227165
Train Epoch: 7 [28800/50000 (58%)]	Loss: 1.973967
Train Epoch: 7 [29440/50000 (59%)]	Loss: 2.194363
Train Epoch: 7 [30080/50000 (60%)]	Loss: 2.077693
Train Epoch: 7 [30720/50000 (61%)]	Loss: 1.893028
Train Epoch: 7 [31360/50000 (63%)]	Loss: 1.811406
Train Epoch: 7 [32000/50000 (64%)]	Loss: 1.787602
Train Epoch: 7 [32640/50000 (65%)]	Loss: 1.763851
Train Epoch: 7 [33280/50000 (66%)]	Loss: 2.292086
Train Epoch: 7 [33920/50000 (68%)]	Loss: 1.879435
Train Epoch: 7 [34560/50000 (69%)]	Loss: 1.947163
Train Epoch: 7 [35200/50000 (70%)]	Loss: 1.797633
Train Epoch: 7 [35840/50000 (72%)]	Loss: 1.790184
Train Epoch: 7 [36480/50000 (73%)]	Loss: 1.798551
Train Epoch: 7 [37120/50000 (74%)]	Loss: 1.927824
Train Epoch: 7 [37760/50000 (75%)]	Loss: 1.866054
Train Epoch: 7 [38400/50000 (77%)]	Loss: 2.037055
Train Epoch: 7 [39040/50000 (78%)]	Loss: 1.712058
Train Epoch: 7 [39680/50000 (79%)]	Loss: 1.745910
Train Epoch: 7 [40320/50000 (81%)]	Loss: 1.958669
Train Epoch: 7 [40960/50000 (82%)]	Loss: 1.738870
Train Epoch: 7 [41600/50000 (83%)]	Loss: 2.092880
Train Epoch: 7 [42240/50000 (84%)]	Loss: 1.967217
Train Epoch: 7 [42880/50000 (86%)]	Loss: 2.033279
Train Epoch: 7 [43520/50000 (87%)]	Loss: 1.941347
Train Epoch: 7 [44160/50000 (88%)]	Loss: 1.804691
Train Epoch: 7 [44800/50000 (90%)]	Loss: 1.850918
Train Epoch: 7 [45440/50000 (91%)]	Loss: 1.604667
Train Epoch: 7 [46080/50000 (92%)]	Loss: 2.089004
Train Epoch: 7 [46720/50000 (93%)]	Loss: 2.034046
Train Epoch: 7 [47360/50000 (95%)]	Loss: 1.691494
Train Epoch: 7 [48000/50000 (96%)]	Loss: 2.236670
Train Epoch: 7 [48640/50000 (97%)]	Loss: 1.746218
Train Epoch: 7 [49280/50000 (98%)]	Loss: 1.806622
Train Epoch: 7 [49920/50000 (100%)]	Loss: 1.860479

Test set: Average loss: 1.8135, Accuracy: 3281/10000 (33%)

Train Epoch: 8 [0/50000 (0%)]	Loss: 2.034124
Train Epoch: 8 [640/50000 (1%)]	Loss: 1.792101
Train Epoch: 8 [1280/50000 (3%)]	Loss: 2.045898
Train Epoch: 8 [1920/50000 (4%)]	Loss: 1.788033
Train Epoch: 8 [2560/50000 (5%)]	Loss: 1.933252
Train Epoch: 8 [3200/50000 (6%)]	Loss: 1.834407
Train Epoch: 8 [3840/50000 (8%)]	Loss: 1.985535
Train Epoch: 8 [4480/50000 (9%)]	Loss: 1.897202
Train Epoch: 8 [5120/50000 (10%)]	Loss: 1.791620
Train Epoch: 8 [5760/50000 (12%)]	Loss: 1.800196
Train Epoch: 8 [6400/50000 (13%)]	Loss: 1.981195
Train Epoch: 8 [7040/50000 (14%)]	Loss: 2.041783
Train Epoch: 8 [7680/50000 (15%)]	Loss: 1.885845
Train Epoch: 8 [8320/50000 (17%)]	Loss: 1.983068
Train Epoch: 8 [8960/50000 (18%)]	Loss: 1.734605
Train Epoch: 8 [9600/50000 (19%)]	Loss: 1.729814
Train Epoch: 8 [10240/50000 (20%)]	Loss: 1.688400
Train Epoch: 8 [10880/50000 (22%)]	Loss: 1.862261
Train Epoch: 8 [11520/50000 (23%)]	Loss: 2.048722
Train Epoch: 8 [12160/50000 (24%)]	Loss: 2.071377
Train Epoch: 8 [12800/50000 (26%)]	Loss: 1.812903
Train Epoch: 8 [13440/50000 (27%)]	Loss: 1.850220
Train Epoch: 8 [14080/50000 (28%)]	Loss: 1.796143
Train Epoch: 8 [14720/50000 (29%)]	Loss: 1.965563
Train Epoch: 8 [15360/50000 (31%)]	Loss: 1.853351
Train Epoch: 8 [16000/50000 (32%)]	Loss: 1.925658
Train Epoch: 8 [16640/50000 (33%)]	Loss: 1.864829
Train Epoch: 8 [17280/50000 (35%)]	Loss: 2.043948
Train Epoch: 8 [17920/50000 (36%)]	Loss: 1.788985
Train Epoch: 8 [18560/50000 (37%)]	Loss: 1.916824
Train Epoch: 8 [19200/50000 (38%)]	Loss: 2.063691
Train Epoch: 8 [19840/50000 (40%)]	Loss: 1.763572
Train Epoch: 8 [20480/50000 (41%)]	Loss: 1.893890
Train Epoch: 8 [21120/50000 (42%)]	Loss: 1.924058
Train Epoch: 8 [21760/50000 (43%)]	Loss: 2.028448
Train Epoch: 8 [22400/50000 (45%)]	Loss: 1.671531
Train Epoch: 8 [23040/50000 (46%)]	Loss: 1.957128
Train Epoch: 8 [23680/50000 (47%)]	Loss: 2.079941
Train Epoch: 8 [24320/50000 (49%)]	Loss: 1.934093
Train Epoch: 8 [24960/50000 (50%)]	Loss: 1.816433
Train Epoch: 8 [25600/50000 (51%)]	Loss: 1.855544
Train Epoch: 8 [26240/50000 (52%)]	Loss: 1.996824
Train Epoch: 8 [26880/50000 (54%)]	Loss: 1.899040
Train Epoch: 8 [27520/50000 (55%)]	Loss: 1.914839
Train Epoch: 8 [28160/50000 (56%)]	Loss: 2.051221
Train Epoch: 8 [28800/50000 (58%)]	Loss: 2.053228
Train Epoch: 8 [29440/50000 (59%)]	Loss: 1.883950
Train Epoch: 8 [30080/50000 (60%)]	Loss: 1.815031
Train Epoch: 8 [30720/50000 (61%)]	Loss: 1.938936
Train Epoch: 8 [31360/50000 (63%)]	Loss: 2.016299
Train Epoch: 8 [32000/50000 (64%)]	Loss: 2.013310
Train Epoch: 8 [32640/50000 (65%)]	Loss: 1.891417
Train Epoch: 8 [33280/50000 (66%)]	Loss: 1.856035
Train Epoch: 8 [33920/50000 (68%)]	Loss: 1.792149
Train Epoch: 8 [34560/50000 (69%)]	Loss: 1.922671
Train Epoch: 8 [35200/50000 (70%)]	Loss: 1.722788
Train Epoch: 8 [35840/50000 (72%)]	Loss: 1.659406
Train Epoch: 8 [36480/50000 (73%)]	Loss: 1.854948
Train Epoch: 8 [37120/50000 (74%)]	Loss: 1.838016
Train Epoch: 8 [37760/50000 (75%)]	Loss: 1.938817
Train Epoch: 8 [38400/50000 (77%)]	Loss: 1.681051
Train Epoch: 8 [39040/50000 (78%)]	Loss: 1.924520
Train Epoch: 8 [39680/50000 (79%)]	Loss: 1.767555
Train Epoch: 8 [40320/50000 (81%)]	Loss: 1.874778
Train Epoch: 8 [40960/50000 (82%)]	Loss: 1.727516
Train Epoch: 8 [41600/50000 (83%)]	Loss: 1.771429
Train Epoch: 8 [42240/50000 (84%)]	Loss: 1.877820
Train Epoch: 8 [42880/50000 (86%)]	Loss: 1.912757
Train Epoch: 8 [43520/50000 (87%)]	Loss: 1.993994
Train Epoch: 8 [44160/50000 (88%)]	Loss: 1.702807
Train Epoch: 8 [44800/50000 (90%)]	Loss: 2.050233
Train Epoch: 8 [45440/50000 (91%)]	Loss: 1.866957
Train Epoch: 8 [46080/50000 (92%)]	Loss: 1.909379
Train Epoch: 8 [46720/50000 (93%)]	Loss: 1.852080
Train Epoch: 8 [47360/50000 (95%)]	Loss: 1.915113
Train Epoch: 8 [48000/50000 (96%)]	Loss: 1.997845
Train Epoch: 8 [48640/50000 (97%)]	Loss: 1.867176
Train Epoch: 8 [49280/50000 (98%)]	Loss: 1.760526
Train Epoch: 8 [49920/50000 (100%)]	Loss: 1.962295

Test set: Average loss: 1.7394, Accuracy: 3517/10000 (35%)

Train Epoch: 9 [0/50000 (0%)]	Loss: 1.874942
Train Epoch: 9 [640/50000 (1%)]	Loss: 1.904717
Train Epoch: 9 [1280/50000 (3%)]	Loss: 2.000335
Train Epoch: 9 [1920/50000 (4%)]	Loss: 1.864330
Train Epoch: 9 [2560/50000 (5%)]	Loss: 1.801117
Train Epoch: 9 [3200/50000 (6%)]	Loss: 1.937797
Train Epoch: 9 [3840/50000 (8%)]	Loss: 1.818362
Train Epoch: 9 [4480/50000 (9%)]	Loss: 1.866971
Train Epoch: 9 [5120/50000 (10%)]	Loss: 2.006912
Train Epoch: 9 [5760/50000 (12%)]	Loss: 2.011387
Train Epoch: 9 [6400/50000 (13%)]	Loss: 1.640293
Train Epoch: 9 [7040/50000 (14%)]	Loss: 1.919499
Train Epoch: 9 [7680/50000 (15%)]	Loss: 2.455607
Train Epoch: 9 [8320/50000 (17%)]	Loss: 2.024770
Train Epoch: 9 [8960/50000 (18%)]	Loss: 1.714786
Train Epoch: 9 [9600/50000 (19%)]	Loss: 1.767253
Train Epoch: 9 [10240/50000 (20%)]	Loss: 1.827373
Train Epoch: 9 [10880/50000 (22%)]	Loss: 1.871620
Train Epoch: 9 [11520/50000 (23%)]	Loss: 1.796278
Train Epoch: 9 [12160/50000 (24%)]	Loss: 1.985634
Train Epoch: 9 [12800/50000 (26%)]	Loss: 2.076216
Train Epoch: 9 [13440/50000 (27%)]	Loss: 2.000809
Train Epoch: 9 [14080/50000 (28%)]	Loss: 1.952754
Train Epoch: 9 [14720/50000 (29%)]	Loss: 1.996092
Train Epoch: 9 [15360/50000 (31%)]	Loss: 1.748815
Train Epoch: 9 [16000/50000 (32%)]	Loss: 1.945781
Train Epoch: 9 [16640/50000 (33%)]	Loss: 1.799598
Train Epoch: 9 [17280/50000 (35%)]	Loss: 1.915942
Train Epoch: 9 [17920/50000 (36%)]	Loss: 1.821839
Train Epoch: 9 [18560/50000 (37%)]	Loss: 1.924435
Train Epoch: 9 [19200/50000 (38%)]	Loss: 1.865972
Train Epoch: 9 [19840/50000 (40%)]	Loss: 1.772906
Train Epoch: 9 [20480/50000 (41%)]	Loss: 1.773457
Train Epoch: 9 [21120/50000 (42%)]	Loss: 1.862058
Train Epoch: 9 [21760/50000 (43%)]	Loss: 1.888000
Train Epoch: 9 [22400/50000 (45%)]	Loss: 1.757328
Train Epoch: 9 [23040/50000 (46%)]	Loss: 1.667569
Train Epoch: 9 [23680/50000 (47%)]	Loss: 1.678389
Train Epoch: 9 [24320/50000 (49%)]	Loss: 1.728083
Train Epoch: 9 [24960/50000 (50%)]	Loss: 1.949971
Train Epoch: 9 [25600/50000 (51%)]	Loss: 1.876359
Train Epoch: 9 [26240/50000 (52%)]	Loss: 1.882617
Train Epoch: 9 [26880/50000 (54%)]	Loss: 1.760226
Train Epoch: 9 [27520/50000 (55%)]	Loss: 1.807186
Train Epoch: 9 [28160/50000 (56%)]	Loss: 2.119287
Train Epoch: 9 [28800/50000 (58%)]	Loss: 1.782998
Train Epoch: 9 [29440/50000 (59%)]	Loss: 1.873100
Train Epoch: 9 [30080/50000 (60%)]	Loss: 1.941014
Train Epoch: 9 [30720/50000 (61%)]	Loss: 1.761550
Train Epoch: 9 [31360/50000 (63%)]	Loss: 1.679217
Train Epoch: 9 [32000/50000 (64%)]	Loss: 1.899792
Train Epoch: 9 [32640/50000 (65%)]	Loss: 2.011451
Train Epoch: 9 [33280/50000 (66%)]	Loss: 1.888137
Train Epoch: 9 [33920/50000 (68%)]	Loss: 1.815164
Train Epoch: 9 [34560/50000 (69%)]	Loss: 1.932236
Train Epoch: 9 [35200/50000 (70%)]	Loss: 2.166007
Train Epoch: 9 [35840/50000 (72%)]	Loss: 1.843704
Train Epoch: 9 [36480/50000 (73%)]	Loss: 2.080678
Train Epoch: 9 [37120/50000 (74%)]	Loss: 1.751995
Train Epoch: 9 [37760/50000 (75%)]	Loss: 1.918760
Train Epoch: 9 [38400/50000 (77%)]	Loss: 1.968506
Train Epoch: 9 [39040/50000 (78%)]	Loss: 1.870266
Train Epoch: 9 [39680/50000 (79%)]	Loss: 1.774769
Train Epoch: 9 [40320/50000 (81%)]	Loss: 2.142534
Train Epoch: 9 [40960/50000 (82%)]	Loss: 1.842018
Train Epoch: 9 [41600/50000 (83%)]	Loss: 1.839039
Train Epoch: 9 [42240/50000 (84%)]	Loss: 1.834333
Train Epoch: 9 [42880/50000 (86%)]	Loss: 1.804746
Train Epoch: 9 [43520/50000 (87%)]	Loss: 1.871090
Train Epoch: 9 [44160/50000 (88%)]	Loss: 1.811054
Train Epoch: 9 [44800/50000 (90%)]	Loss: 1.861487
Train Epoch: 9 [45440/50000 (91%)]	Loss: 1.788488
Train Epoch: 9 [46080/50000 (92%)]	Loss: 2.000480
Train Epoch: 9 [46720/50000 (93%)]	Loss: 1.842846
Train Epoch: 9 [47360/50000 (95%)]	Loss: 2.135789
Train Epoch: 9 [48000/50000 (96%)]	Loss: 1.835913
Train Epoch: 9 [48640/50000 (97%)]	Loss: 1.735052
Train Epoch: 9 [49280/50000 (98%)]	Loss: 2.060922
Train Epoch: 9 [49920/50000 (100%)]	Loss: 1.967201

Test set: Average loss: 1.8434, Accuracy: 3121/10000 (31%)

Train Epoch: 10 [0/50000 (0%)]	Loss: 1.977478
Train Epoch: 10 [640/50000 (1%)]	Loss: 1.966376
Train Epoch: 10 [1280/50000 (3%)]	Loss: 2.018752
Train Epoch: 10 [1920/50000 (4%)]	Loss: 1.936322
Train Epoch: 10 [2560/50000 (5%)]	Loss: 2.184115
Train Epoch: 10 [3200/50000 (6%)]	Loss: 2.077096
Train Epoch: 10 [3840/50000 (8%)]	Loss: 1.877684
Train Epoch: 10 [4480/50000 (9%)]	Loss: 1.744319
Train Epoch: 10 [5120/50000 (10%)]	Loss: 1.632480
Train Epoch: 10 [5760/50000 (12%)]	Loss: 1.785359
Train Epoch: 10 [6400/50000 (13%)]	Loss: 1.784703
Train Epoch: 10 [7040/50000 (14%)]	Loss: 1.967822
Train Epoch: 10 [7680/50000 (15%)]	Loss: 1.960295
Train Epoch: 10 [8320/50000 (17%)]	Loss: 2.282013
Train Epoch: 10 [8960/50000 (18%)]	Loss: 2.047717
Train Epoch: 10 [9600/50000 (19%)]	Loss: 2.024017
Train Epoch: 10 [10240/50000 (20%)]	Loss: 1.670983
Train Epoch: 10 [10880/50000 (22%)]	Loss: 2.021043
Train Epoch: 10 [11520/50000 (23%)]	Loss: 1.834534
Train Epoch: 10 [12160/50000 (24%)]	Loss: 1.837481
Train Epoch: 10 [12800/50000 (26%)]	Loss: 1.921653
Train Epoch: 10 [13440/50000 (27%)]	Loss: 1.970911
Train Epoch: 10 [14080/50000 (28%)]	Loss: 1.671484
Train Epoch: 10 [14720/50000 (29%)]	Loss: 1.646059
Train Epoch: 10 [15360/50000 (31%)]	Loss: 1.873046
Train Epoch: 10 [16000/50000 (32%)]	Loss: 1.750103
Train Epoch: 10 [16640/50000 (33%)]	Loss: 1.928526
Train Epoch: 10 [17280/50000 (35%)]	Loss: 1.912799
Train Epoch: 10 [17920/50000 (36%)]	Loss: 1.666108
Train Epoch: 10 [18560/50000 (37%)]	Loss: 1.800996
Train Epoch: 10 [19200/50000 (38%)]	Loss: 1.864019
Train Epoch: 10 [19840/50000 (40%)]	Loss: 1.898210
Train Epoch: 10 [20480/50000 (41%)]	Loss: 2.048898
Train Epoch: 10 [21120/50000 (42%)]	Loss: 1.740179
Train Epoch: 10 [21760/50000 (43%)]	Loss: 1.665888
Train Epoch: 10 [22400/50000 (45%)]	Loss: 1.864718
Train Epoch: 10 [23040/50000 (46%)]	Loss: 1.781558
Train Epoch: 10 [23680/50000 (47%)]	Loss: 2.052986
Train Epoch: 10 [24320/50000 (49%)]	Loss: 1.875239
Train Epoch: 10 [24960/50000 (50%)]	Loss: 1.752146
Train Epoch: 10 [25600/50000 (51%)]	Loss: 1.845485
Train Epoch: 10 [26240/50000 (52%)]	Loss: 1.727158
Train Epoch: 10 [26880/50000 (54%)]	Loss: 1.769685
Train Epoch: 10 [27520/50000 (55%)]	Loss: 2.053882
Train Epoch: 10 [28160/50000 (56%)]	Loss: 1.954989
Train Epoch: 10 [28800/50000 (58%)]	Loss: 1.648122
Train Epoch: 10 [29440/50000 (59%)]	Loss: 1.982522
Train Epoch: 10 [30080/50000 (60%)]	Loss: 2.038850
Train Epoch: 10 [30720/50000 (61%)]	Loss: 1.700410
Train Epoch: 10 [31360/50000 (63%)]	Loss: 1.656315
Train Epoch: 10 [32000/50000 (64%)]	Loss: 1.754264
Train Epoch: 10 [32640/50000 (65%)]	Loss: 1.779129
Train Epoch: 10 [33280/50000 (66%)]	Loss: 1.678392
Train Epoch: 10 [33920/50000 (68%)]	Loss: 1.921641
Train Epoch: 10 [34560/50000 (69%)]	Loss: 1.915987
Train Epoch: 10 [35200/50000 (70%)]	Loss: 1.716734
Train Epoch: 10 [35840/50000 (72%)]	Loss: 1.845028
Train Epoch: 10 [36480/50000 (73%)]	Loss: 2.057510
Train Epoch: 10 [37120/50000 (74%)]	Loss: 1.893831
Train Epoch: 10 [37760/50000 (75%)]	Loss: 1.758789
Train Epoch: 10 [38400/50000 (77%)]	Loss: 1.957925
Train Epoch: 10 [39040/50000 (78%)]	Loss: 1.898927
Train Epoch: 10 [39680/50000 (79%)]	Loss: 1.782439
Train Epoch: 10 [40320/50000 (81%)]	Loss: 1.774270
Train Epoch: 10 [40960/50000 (82%)]	Loss: 1.933757
Train Epoch: 10 [41600/50000 (83%)]	Loss: 1.778105
Train Epoch: 10 [42240/50000 (84%)]	Loss: 1.769073
Train Epoch: 10 [42880/50000 (86%)]	Loss: 1.834968
Train Epoch: 10 [43520/50000 (87%)]	Loss: 1.954882
Train Epoch: 10 [44160/50000 (88%)]	Loss: 1.818112
Train Epoch: 10 [44800/50000 (90%)]	Loss: 1.766723
Train Epoch: 10 [45440/50000 (91%)]	Loss: 1.935594
Train Epoch: 10 [46080/50000 (92%)]	Loss: 1.730601
Train Epoch: 10 [46720/50000 (93%)]	Loss: 1.776839
Train Epoch: 10 [47360/50000 (95%)]	Loss: 1.950344
Train Epoch: 10 [48000/50000 (96%)]	Loss: 1.783402
Train Epoch: 10 [48640/50000 (97%)]	Loss: 1.884675
Train Epoch: 10 [49280/50000 (98%)]	Loss: 1.879146
Train Epoch: 10 [49920/50000 (100%)]	Loss: 2.144068

Test set: Average loss: 2.0631, Accuracy: 2418/10000 (24%)







    













    













    













    




<class 'torch.FloatTensor'> torch.Size([6, 3, 5, 5])
<class 'torch.FloatTensor'> torch.Size([3, 14, 21])

In [ ]:

    




    

In [ ]: