In this part, we will formally set up a simple but powerful classification network, to recogize 0-9 nubmers in MNIST dataset.
Yep, we will build a classification network and train from scratch.
We would introduce some techniques to improve your train model performance.
This part is designed and completed by Jiaxin Zhuang( zhuangjx5@mail2.sysu.edu.cn ) and Feifei Xue(xueff@mail2.sysu.edu.cn), if you have some questions about this part and you think there are still some things to do, dont't hesitate to email us or add our wechat.
In [1]:
%load_ext autoreload
%autoreload 2
numpy: NumPy is the fundamental package for scientific computing in Python.
pytorch: End-to-end deep learning platform.
torchvision: This package consists of popular datasets, model architectures, and common image transformations for computer vision.
tensorflow: An open source machine learning framework.
tensorboard: A suite of visualization tools to make training easier to understand, debug, and optimize TensorFlow programs.
tensorboardX: Tensorboard for Pytorch.
matplotlib: It is a Python 2D plotting library which produces publication quality figures in a variety of hardcopy formats and interactive environments across platforms.
In [1]:
# Load all necessary modules here, for clearness
import torch
import numpy as np
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
# from torchvision.datasets import MNIST
import torchvision
from torchvision import transforms
from torch.optim import lr_scheduler
# from tensorboardX import SummaryWriter
from collections import OrderedDict
import matplotlib.pyplot as plt
# from tqdm import tqdm
In [2]:
# Whether to put data in GPU according to GPU is available or not
# cuda = torch.cuda.is_available()
# In case the default gpu does not have enough space, you can choose which device to use
# torch.cuda.set_device(device) # device: id
# Since gpu in lab is not enough for your guys, we prefer to cpu computation
cuda = torch.device('cpu')
The MNIST database (Modified National Institute of Standards and Technology database) is a large database of handwritten digits that is commonly used for training various image processing systems.
The MNIST database contains 60,000 training images and 10,000 testing images. Each class has 5000 traning images and 1000 test images.
Each image is 32x32.
And they look like images below.
We would fefine a FeedForward Neural Network with 3 hidden layers.
Each layer is followed a activation function, we would try sigmoid and relu respectively.
For simplicity, each hidden layer has the equal neurons.
In reality, however, we would apply different amount of neurons in different hidden layers.
Network Structure
Inputs Linear/Function Output
[128, 1, 28, 28] -> Linear(28*28, 100) -> [128, 100] # first hidden lyaer
-> ReLU -> [128, 100] # relu activation function, may sigmoid
-> Linear(100, 100) -> [128, 100] # second hidden lyaer
-> ReLU -> [128, 100] # relu activation function, may sigmoid
-> Linear(100, 100) -> [128, 100] # third hidden lyaer
-> ReLU -> [128, 100] # relu activation function, may sigmoid
-> Linear(100, 10) -> [128, 10] # Classification Layer
In [3]:
class FeedForwardNeuralNetwork(nn.Module):
"""
Inputs Linear/Function Output
[128, 1, 28, 28] -> Linear(28*28, 100) -> [128, 100] # first hidden lyaer
-> ReLU -> [128, 100] # relu activation function, may sigmoid
-> Linear(100, 100) -> [128, 100] # second hidden lyaer
-> ReLU -> [128, 100] # relu activation function, may sigmoid
-> Linear(100, 100) -> [128, 100] # third hidden lyaer
-> ReLU -> [128, 100] # relu activation function, may sigmoid
-> Linear(100, 10) -> [128, 10] # Classification Layer
"""
def __init__(self, input_size, hidden_size, output_size, activation_function='RELU'):
super(FeedForwardNeuralNetwork, self).__init__()
self.use_dropout = False
self.use_bn = False
self.hidden1 = nn.Linear(input_size, hidden_size) # Linear function 1: 784 --> 100
self.hidden2 = nn.Linear(hidden_size, hidden_size) # Linear function 2: 100 --> 100
self.hidden3 = nn.Linear(hidden_size, hidden_size) # Linear function 3: 100 --> 100
# Linear function 4 (readout): 100 --> 10
self.classification_layer = nn.Linear(hidden_size, output_size)
self.dropout = nn.Dropout(p=0.5) # Drop out with prob = 0.5
self.hidden1_bn = nn.BatchNorm1d(hidden_size) # Batch Normalization
self.hidden2_bn = nn.BatchNorm1d(hidden_size)
self.hidden3_bn = nn.BatchNorm1d(hidden_size)
# Non-linearity
if activation_function == 'SIGMOID':
self.activation_function1 = nn.Sigmoid()
self.activation_function2 = nn.Sigmoid()
self.activation_function3 = nn.Sigmoid()
elif activation_function == 'RELU':
self.activation_function1 = nn.ReLU()
self.activation_function2 = nn.ReLU()
self.activation_function3 = nn.ReLU()
def forward(self, x):
"""Defines the computation performed at every call.
Should be overridden by all subclasses.
Args:
x: [batch_size, channel, height, width], input for network
Returns:
out: [batch_size, n_classes], output from network
"""
x = x.view(x.size(0), -1) # flatten x in [128, 784]
out = self.hidden1(x)
out = self.activation_function1(out) # Non-linearity 1
if self.use_bn == True:
out = self.hidden1_bn(out)
out = self.hidden2(out)
out = self.activation_function2(out)
if self.use_bn == True:
out = self.hidden2_bn(out)
out = self.hidden3(out)
if self.use_bn == True:
out = self.hidden3_bn(out)
out = self.activation_function3(out)
if self.use_dropout == True:
out = self.dropout(out)
out = self.classification_layer(out)
return out
def set_use_dropout(self, use_dropout):
"""Whether to use dropout. Auxiliary function for our exp, not necessary.
Args:
use_dropout: True, False
"""
self.use_dropout = use_dropout
def set_use_bn(self, use_bn):
"""Whether to use batch normalization. Auxiliary function for our exp, not necessary.
Args:
use_bn: True, False
"""
self.use_bn = use_bn
def get_grad(self):
"""Return average grad for hidden2, hidden3. Auxiliary function for our exp, not necessary.
"""
hidden2_average_grad = np.mean(np.sqrt(np.square(self.hidden2.weight.grad.detach().numpy())))
hidden3_average_grad = np.mean(np.sqrt(np.square(self.hidden3.weight.grad.detach().numpy())))
return hidden2_average_grad, hidden3_average_grad
setting hyperparameters like below
hyper paprameters include following part
In [5]:
### Hyper parameters
batch_size = 128 # batch size is 128
n_epochs = 5 # train for 5 epochs
learning_rate = 0.01 # learning rate is 0.01
input_size = 28*28 # input image has size 28x28
hidden_size = 100 # hidden neurons is 100 for each layer
output_size = 10 # classes of prediction
l2_norm = 0 # not to use l2 penalty
dropout = False # not to use
get_grad = False # not to obtain grad
In [6]:
# create a model object
model = FeedForwardNeuralNetwork(input_size=input_size, hidden_size=hidden_size, output_size=output_size)
# Cross entropy
loss_fn = torch.nn.CrossEntropyLoss()
# l2_norm can be done in SGD
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, weight_decay=l2_norm)
Pytorch provide default initialization (uniform intialization) for linear layer. But there is still some useful intialization method.
Read more about initialization from this link
torch.nn.init.normal_
torch.nn.init.uniform_
torch.nn.init.constant_
torch.nn.init.eye_
torch.nn.init.xavier_uniform_
torch.nn.init.xavier_normal_
torch.nn.init.kaiming_uniform_
In [4]:
def show_weight_bias(model):
"""Show some weights and bias distribution every layers in model.
!!YOU CAN READ THIS CODE LATER!!
"""
# Create a figure and a set of subplots
fig, axs = plt.subplots(2,3, sharey=False, tight_layout=True)
# weight and bias for every hidden layer
h1_w = model.hidden1.weight.detach().numpy().flatten()
h1_b = model.hidden1.bias.detach().numpy().flatten()
h2_w = model.hidden2.weight.detach().numpy().flatten()
h2_b = model.hidden2.bias.detach().numpy().flatten()
h3_w = model.hidden3.weight.detach().numpy().flatten()
h3_b = model.hidden3.bias.detach().numpy().flatten()
axs[0,0].hist(h1_w)
axs[0,1].hist(h2_w)
axs[0,2].hist(h3_w)
axs[1,0].hist(h1_b)
axs[1,1].hist(h2_b)
axs[1,2].hist(h3_b)
# set title for every sub plots
axs[0,0].set_title('hidden1_weight')
axs[0,1].set_title('hidden2_weight')
axs[0,2].set_title('hidden3_weight')
axs[1,0].set_title('hidden1_bias')
axs[1,1].set_title('hidden2_bias')
axs[1,2].set_title('hidden3_bias')
In [8]:
# Show default initialization for every hidden layer by pytorch
# it's uniform distribution
show_weight_bias(model)
In [5]:
# If you want to use other intialization method, you can use code below
# and define your initialization below
def weight_bias_reset(model):
"""Custom initialization, you can use your favorable initialization method.
"""
for m in model.modules():
if isinstance(m, nn.Linear):
# initialize linear layer with mean and std
mean, std = 0, 0.1
# Initialization method
torch.nn.init.normal_(m.weight, mean, std)
torch.nn.init.normal_(m.bias, mean, std)
# Another way to initialize
# m.weight.data.normal_(mean, std)
# m.bias.data.normal_(mean, std)
In [10]:
weight_bias_reset(model) # reset parameters for each hidden layer
show_weight_bias(model) # show weight and bias distribution, normal distribution now.
In [11]:
def weight_bias_reset_constant(model):
"""Constant initalization
"""
for m in model.modules():
if isinstance(m, nn.Linear):
val = 0.1
torch.nn.init.constant_(m.weight, val)
torch.nn.init.constant_(m.bias, val)
In [12]:
weight_bias_reset_constant(model)
show_weight_bias(model)
In [13]:
def weight_bias_reset_xavier_uniform(model):
"""xaveir_uniform, gain=1
"""
for m in model.modules():
if isinstance(m, nn.Linear):
gain = 1
torch.nn.init.xavier_uniform_(m.weight, gain)
# torch.nn.init.xavier_uniform_(m.bias, gain)
In [14]:
weight_bias_reset_xavier_uniform(model)
show_weight_bias(model)
In [15]:
def weight_bias_reset_kaiming_uniform(model):
"""kaiming_uniform, a=0, mode='fan_in', non_linearity='relu'
"""
for m in model.modules():
if isinstance(m, nn.Linear):
a = 0
torch.nn.init.kaiming_uniform_(m.weight, a=a, mode='fan_in', nonlinearity='relu')
# torch.nn.init.kaiming_uniform_(m.bias, a=a, mode='fan_in', nonlinearity='relu')
In [16]:
weight_bias_reset_kaiming_uniform(model)
show_weight_bias(model)
shuffle
train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, **kwargs)For each mini-batch data
load mini-batch data
for batch_idx, (data, target) in enumerate(train_loader): \
...compute gradient of loss over parameters
output = net(data) # make prediction
loss = loss_fn(output, target) # compute loss
loss.backward() # compute gradient of loss over parametersupdate parameters with gradient descent
optimzer.step() # update parameters with gradient descentPlease pay attention to data augmentation.
Read more data augmentation method from this link.
torchvision.transforms.RandomVerticalFlip
torchvision.transforms.RandomHorizontalFlip
...
In [19]:
# define method of preprocessing data for evaluating
train_transform = transforms.Compose([
transforms.ToTensor(), # Convert a PIL Image or numpy.ndarray to tensor.
# Normalize a tensor image with mean 0.1307 and standard deviation 0.3081
transforms.Normalize((0.1307,), (0.3081,))
])
test_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])
In [20]:
# use MNIST provided by torchvision
# torchvision.datasets provide MNIST dataset for classification
train_dataset = torchvision.datasets.MNIST(root='./data',
train=True,
transform=train_transform,
download=True)
test_dataset = torchvision.datasets.MNIST(root='./data',
train=False,
transform=test_transform,
download=False)
In [21]:
# pay attention to this, train_dataset doesn't load any data
# It just defined some method and store some message to preprocess data
train_dataset
Out[21]:
In [22]:
# Data loader.
# Combines a dataset and a sampler,
# and provides single- or multi-process iterators over the dataset.
train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
batch_size=batch_size,
shuffle=False)
test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
batch_size=batch_size,
shuffle=False)
In [21]:
# functions to show an image
def imshow(img):
"""show some imgs in datasets
!!YOU CAN READ THIS CODE LATER!! """
npimg = img.numpy() # convert tensor to numpy
plt.imshow(np.transpose(npimg, (1, 2, 0))) # [channel, height, width] -> [height, width, channel]
plt.show()
In [22]:
# get some random training images by batch
dataiter = iter(train_loader)
images, labels = dataiter.next() # get a batch of images
# show images
imshow(torchvision.utils.make_grid(images))
In [28]:
def train(train_loader, model, loss_fn, optimizer, get_grad=False):
"""train model using loss_fn and optimizer. When thid function is called, model trains for one epoch.
Args:
train_loader: train data
model: prediction model
loss_fn: loss function to judge the distance between target and outputs
optimizer: optimize the loss function
get_grad: True, False
Returns:
total_loss: loss
average_grad2: average grad for hidden 2 in this epoch
average_grad3: average grad for hidden 3 in this epoch
"""
# set the module in training model, affecting module e.g., Dropout, BatchNorm, etc.
model.train()
total_loss = 0
grad_2 = 0.0 # store sum(grad) for hidden 3 layer
grad_3 = 0.0 # store sum(grad) for hidden 3 layer
for batch_idx, (data, target) in enumerate(train_loader):
optimizer.zero_grad() # clear gradients of all optimized torch.Tensors'
outputs = model(data) # make predictions
loss = loss_fn(outputs, target) # compute loss
total_loss += loss.item() # accumulate every batch loss in a epoch
loss.backward() # compute gradient of loss over parameters
if get_grad == True:
g2, g3 = model.get_grad() # get grad for hiddern 2 and 3 layer in this batch
grad_2 += g2 # accumulate grad for hidden 2
grad_3 += g3 # accumulate grad for hidden 2
optimizer.step() # update parameters with gradient descent
average_loss = total_loss / batch_idx # average loss in this epoch
average_grad2 = grad_2 / batch_idx # average grad for hidden 2 in this epoch
average_grad3 = grad_3 / batch_idx # average grad for hidden 3 in this epoch
return average_loss, average_grad2, average_grad3
In [8]:
def evaluate(loader, model, loss_fn):
"""test model's prediction performance on loader.
When thid function is called, model is evaluated.
Args:
loader: data for evaluation
model: prediction model
loss_fn: loss function to judge the distance between target and outputs
Returns:
total_loss
accuracy
"""
# context-manager that disabled gradient computation
with torch.no_grad():
# set the module in evaluation mode
model.eval()
correct = 0.0 # account correct amount of data
total_loss = 0 # account loss
for batch_idx, (data, target) in enumerate(loader):
outputs = model(data) # make predictions
# return the maximum value of each row of the input tensor in the
# given dimension dim, the second return vale is the index location
# of each maxium value found(argmax)
_, predicted = torch.max(outputs, 1)
# Detach: Returns a new Tensor, detached from the current graph.
#The result will never require gradient.
correct += (predicted == target).sum().detach().numpy()
loss = loss_fn(outputs, target) # compute loss
total_loss += loss.item() # accumulate every batch loss in a epoch
accuracy = correct*100.0 / len(loader.dataset) # accuracy in a epoch
return total_loss, accuracy
Define function fit and use train_epoch and test_epoch
In [26]:
def fit(train_loader, val_loader, model, loss_fn, optimizer, n_epochs, get_grad=False):
"""train and val model here, we use train_epoch to train model and
val_epoch to val model prediction performance
Args:
train_loader: train data
val_loader: validation data
model: prediction model
loss_fn: loss function to judge the distance between target and outputs
optimizer: optimize the loss function
n_epochs: training epochs
get_grad: Whether to get grad of hidden2 layer and hidden3 layer
Returns:
train_accs: accuracy of train n_epochs, a list
train_losses: loss of n_epochs, a list
"""
grad_2 = [] # save grad for hidden 2 every epoch
grad_3 = [] # save grad for hidden 3 every epoch
train_accs = [] # save train accuracy every epoch
train_losses = [] # save train loss every epoch
for epoch in range(n_epochs): # train for n_epochs
# train model on training datasets, optimize loss function and update model parameters
train_loss, average_grad2, average_grad3 = train(train_loader, model, loss_fn, optimizer, get_grad)
# evaluate model performance on train dataset
_, train_accuracy = evaluate(train_loader, model, loss_fn)
message = 'Epoch: {}/{}. Train set: Average loss: {:.4f}, Accuracy: {:.4f}'.format(epoch+1, \
n_epochs, train_loss, train_accuracy)
print(message)
# save loss, accuracy, grad
train_accs.append(train_accuracy)
train_losses.append(train_loss)
grad_2.append(average_grad2)
grad_3.append(average_grad3)
# evaluate model performance on val dataset
val_loss, val_accuracy = evaluate(val_loader, model, loss_fn)
message = 'Epoch: {}/{}. Validation set: Average loss: {:.4f}, Accuracy: {:.4f}'.format(epoch+1, \
n_epochs, val_loss, val_accuracy)
print(message)
# Whether to get grad for showing
if get_grad == True:
fig, ax = plt.subplots() # add a set of subplots to this figure
ax.plot(grad_2, label='Gradient for Hidden 2 Layer') # plot grad 2
ax.plot(grad_3, label='Gradient for Hidden 3 Layer') # plot grad 3
plt.ylim(top=0.004)
# place a legend on axes
legend = ax.legend(loc='best', shadow=True, fontsize='x-large')
return train_accs, train_losses
In [26]:
def show_curve(ys, title):
"""plot curlve for Loss and Accuacy
!!YOU CAN READ THIS LATER, if you are interested
Args:
ys: loss or acc list
title: Loss or Accuracy
"""
x = np.array(range(len(ys)))
y = np.array(ys)
plt.plot(x, y, c='b')
plt.axis()
plt.title('{} Curve:'.format(title))
plt.xlabel('Epoch')
plt.ylabel('{} Value'.format(title))
plt.show()
In [27]:
### Hyper parameters
batch_size = 128 # batch size is 128
n_epochs = 5 # train for 5 epochs
learning_rate = 0.01 # learning rate is 0.01
input_size = 28*28 # input image has size 28x28
hidden_size = 100 # hidden neurons is 100 for each layer
output_size = 10 # classes of prediction
l2_norm = 0 # not to use l2 penalty
dropout = False # not to use
get_grad = False # not to obtain grad
# declare a model
model = FeedForwardNeuralNetwork(input_size=input_size, hidden_size=hidden_size, output_size=output_size)
# Cross entropy
loss_fn = torch.nn.CrossEntropyLoss()
# l2_norm can be done in SGD
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, weight_decay=l2_norm)
In [28]:
train_accs, train_losses = fit(train_loader, test_loader, model, loss_fn, optimizer, n_epochs, get_grad)
模型没有训练到过拟合，观察上面训练数据，随着代数增多，测试集的正确率并没有下降。
In [29]:
show_curve(train_accs, 'accuracy')
show_curve(train_losses, 'loss')
In [11]:
### Hyper parameters
batch_size = 128 # batch size is 128
n_epochs = 5 # train for 5 epochs
learning_rate = 0.01 # learning rate is 0.01
input_size = 28*28 # input image has size 28x28
hidden_size = 100 # hidden neurons is 100 for each layer
output_size = 10 # classes of prediction
l2_norm = 0 # not to use l2 penalty
dropout = False # not to use
get_grad = False # not to obtain grad
# declare a model
model = FeedForwardNeuralNetwork(input_size=input_size, hidden_size=hidden_size, output_size=output_size)
# Cross entropy
loss_fn = torch.nn.CrossEntropyLoss()
# l2_norm can be done in SGD
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, weight_decay=l2_norm)
In [31]:
# 3.1 Train
n_epochs = 10
train_accs, train_losses = fit(train_loader, test_loader, model, loss_fn, optimizer, n_epochs, get_grad)
观察数据可以发现其实10代也没有过拟合。
In [32]:
# 3.1 show_curve
show_curve(train_accs, 'accuracy')
show_curve(train_losses, 'loss')
In [36]:
# 3.2 Train
batch_size = 128 # batch size is 128
n_epochs = 5 # train for 5 epochs
learning_rate = 0.7
input_size = 28*28 # input image has size 28x28
hidden_size = 100 # hidden neurons is 100 for each layer
output_size = 10 # classes of prediction
l2_norm = 0 # not to use l2 penalty
dropout = False # not to use
get_grad = False # not to obtain grad
# declare a model
model = FeedForwardNeuralNetwork(input_size=input_size, hidden_size=hidden_size, output_size=output_size)
# Cross entropy
loss_fn = torch.nn.CrossEntropyLoss()
# l2_norm can be done in SGD
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, weight_decay=l2_norm)
train_accs, train_losses = fit(train_loader, test_loader, model, loss_fn, optimizer, n_epochs, get_grad)
In [37]:
# 3.2 show_curve
show_curve(train_accs, 'accuracy')
show_curve(train_losses, 'loss')
Pytorch provide two kinds of method to save model. We recommmend the method which only saves parameters. Because it's more feasible and dont' rely on fixed model.
When saving parameters, we not only save learnable parameters in model, but also learnable parameters in optimizer.
A common PyTorch convention is to save models using either a .pt or .pth file extension.
Read more abount save load from this link
In [38]:
# show parameters in model
# Print model's state_dict
print("Model's state_dict:")
for param_tensor in model.state_dict():
print(param_tensor, "\t", model.state_dict()[param_tensor].size())
# Print optimizer's state_dict
print("\nOptimizer's state_dict:")
for var_name in optimizer.state_dict():
print(var_name, "\t", optimizer.state_dict()[var_name])
In [13]:
# save model
save_path = './model.pt'
torch.save(model.state_dict(), save_path)
In [14]:
# load parameters from files
saved_parametes = torch.load(save_path)
print(saved_parametes)
In [15]:
# initailze model by saved parameters
new_model = FeedForwardNeuralNetwork(input_size, hidden_size, output_size)
new_model.load_state_dict(saved_parametes)
In [23]:
# test your model prediction performance
new_test_loss, new_test_accuracy = evaluate(test_loader, new_model, loss_fn)
message = 'Average loss: {:.4f}, Accuracy: {:.4f}'.format(new_test_loss, new_test_accuracy)
print(message)
set l2_norm=0.01, let's train and see
In [24]:
### Hyper parameters
batch_size = 128
n_epochs = 5
learning_rate = 0.01
input_size = 28*28
hidden_size = 100
output_size = 10
l2_norm = 0.01 # use l2 penalty
get_grad = False
# declare a model
model = FeedForwardNeuralNetwork(input_size=input_size, hidden_size=hidden_size, output_size=output_size)
# Cross entropy
loss_fn = torch.nn.CrossEntropyLoss()
# l2_norm can be done in SGD
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, weight_decay=l2_norm)
In [29]:
train_accs, train_losses = fit(train_loader, test_loader, model, loss_fn, optimizer, n_epochs, get_grad)
In [30]:
# Hyper parameters
batch_size = 128
n_epochs = 5
learning_rate = 0.01
input_size = 28*28
hidden_size = 100
output_size = 10
l2_norm = 1 # use l2 penalty
get_grad = False
# declare a model
model = FeedForwardNeuralNetwork(input_size=input_size, hidden_size=hidden_size, output_size=output_size)
# Cross entropy
loss_fn = torch.nn.CrossEntropyLoss()
# l2_norm can be done in SGD
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, weight_decay=l2_norm)
In [31]:
train_accs, train_losses = fit(train_loader, test_loader, model, loss_fn, optimizer, n_epochs, get_grad)
In [32]:
### Hyper parameters
batch_size = 128
n_epochs = 5
learning_rate = 0.01
input_size = 28*28
hidden_size = 100
output_size = 10
l2_norm = 0 # without using l2 penalty
get_grad = False
# declare a model
model = FeedForwardNeuralNetwork(input_size=input_size, hidden_size=hidden_size, output_size=output_size)
# Cross entropy
loss_fn = torch.nn.CrossEntropyLoss()
# l2_norm can be done in SGD
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, weight_decay=l2_norm)
In [33]:
# Set dropout to True and probability = 0.5
model.set_use_dropout(True)
In [34]:
train_accs, train_losses = fit(train_loader, test_loader, model, loss_fn, optimizer, n_epochs, get_grad)
Batch normalization is a technique for improving the performance and stability of artificial neural networks
\begin{equation} y=\frac{x-E[x]}{\sqrt{Var[x]+\epsilon}} * \gamma + \beta, \end{equation}$\gamma$ and $\beta$ are learnable parameters
Hints: 因为jupyter对变量有上下文关系，模型，优化器需要重新声明。可以使用以下代码进行重新定义模型和优化器。注意到此处用的是默认初始化。
In [35]:
### Hyper parameters
batch_size = 128
n_epochs = 5
learning_rate = 0.01
input_size = 28*28
hidden_size = 100
output_size = 10
l2_norm = 0 # without using l2 penalty
get_grad = False
# declare a model
model = FeedForwardNeuralNetwork(input_size=input_size, hidden_size=hidden_size, output_size=output_size)
# Cross entropy
loss_fn = torch.nn.CrossEntropyLoss()
# l2_norm can be done in SGD
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, weight_decay=l2_norm)
In [36]:
model.set_use_bn(True)
In [37]:
model.use_bn
Out[37]:
In [38]:
train_accs, train_losses = fit(train_loader, test_loader, model, loss_fn, optimizer, n_epochs, get_grad)
In [51]:
# only add random horizontal flip
train_transform_1 = transforms.Compose([
transforms.RandomHorizontalFlip(),
transforms.ToTensor(), # Convert a PIL Image or numpy.ndarray to tensor.
# Normalize a tensor image with mean and standard deviation
transforms.Normalize((0.1307,), (0.3081,))
])
# only add random crop
train_transform_2 = transforms.Compose([
transforms.RandomCrop(size=[28,28], padding=4),
transforms.ToTensor(), # Convert a PIL Image or numpy.ndarray to tensor.
# Normalize a tensor image with mean and standard deviation
transforms.Normalize((0.1307,), (0.3081,))
])
# add random horizontal flip and random crop
train_transform_3 = transforms.Compose([
transforms.RandomHorizontalFlip(),
transforms.RandomCrop(size=[28,28], padding=4),
transforms.ToTensor(), # Convert a PIL Image or numpy.ndarray to tensor.
# Normalize a tensor image with mean and standard deviation
transforms.Normalize((0.1307,), (0.3081,))
])
In [40]:
# reload train_loader using trans
train_dataset_1 = torchvision.datasets.MNIST(root='./data',
train=True,
transform=train_transform_1,
download=False)
train_loader_1 = torch.utils.data.DataLoader(dataset=train_dataset_1,
batch_size=batch_size,
shuffle=True)
In [42]:
print(train_dataset_1)
In [43]:
### Hyper parameters
batch_size = 128
n_epochs = 5
learning_rate = 0.01
input_size = 28*28
hidden_size = 100
output_size = 10
l2_norm = 0 # without using l2 penalty
get_grad = False
# declare a model
model = FeedForwardNeuralNetwork(input_size=input_size, hidden_size=hidden_size, output_size=output_size)
# Cross entropy
loss_fn = torch.nn.CrossEntropyLoss()
# l2_norm can be done in SGD
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, weight_decay=l2_norm)
In [44]:
train_accs, train_losses = fit(train_loader_1, test_loader, model, loss_fn, optimizer, n_epochs, get_grad)
In [49]:
# train_transform_2
batch_size = 128
train_dataset_2 = torchvision.datasets.MNIST(root='./data',
train=True,
transform=train_transform_2,
download=False)
train_loader_2 = torch.utils.data.DataLoader(dataset=train_dataset_2,
batch_size=batch_size,
shuffle=True)
n_epochs = 5
learning_rate = 0.01
input_size = 28*28
hidden_size = 100
output_size = 10
l2_norm = 0 # without using l2 penalty
get_grad = False
# declare a model
model = FeedForwardNeuralNetwork(input_size=input_size, hidden_size=hidden_size, output_size=output_size)
# Cross entropy
loss_fn = torch.nn.CrossEntropyLoss()
# l2_norm can be done in SGD
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, weight_decay=l2_norm)
train_accs, train_losses = fit(train_loader_2, test_loader, model, loss_fn, optimizer, n_epochs, get_grad)
In [52]:
# train_transform_3
batch_size = 128
train_dataset_3 = torchvision.datasets.MNIST(root='./data',
train=True,
transform=train_transform_3,
download=False)
train_loader_3 = torch.utils.data.DataLoader(dataset=train_dataset_3,
batch_size=batch_size,
shuffle=True)
n_epochs = 5
learning_rate = 0.01
input_size = 28*28
hidden_size = 100
output_size = 10
l2_norm = 0 # without using l2 penalty
get_grad = False
# declare a model
model = FeedForwardNeuralNetwork(input_size=input_size, hidden_size=hidden_size, output_size=output_size)
# Cross entropy
loss_fn = torch.nn.CrossEntropyLoss()
# l2_norm can be done in SGD
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, weight_decay=l2_norm)
train_accs, train_losses = fit(train_loader_3, test_loader, model, loss_fn, optimizer, n_epochs, get_grad)
We could use tensorboard to visualize our training and test phase. You could find example here
In [53]:
### Hyper parameters
batch_size = 128
n_epochs = 15
learning_rate = 0.01
input_size = 28*28
hidden_size = 100
output_size = 10
l2_norm = 0 # use l2 penalty
get_grad = True
# declare a model
model = FeedForwardNeuralNetwork(input_size=input_size, hidden_size=hidden_size, output_size=output_size)
# Cross entropy
loss_fn = torch.nn.CrossEntropyLoss()
# l2_norm can be done in SGD
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, weight_decay=l2_norm)
In [54]:
fit(train_loader, test_loader, model, loss_fn, optimizer, n_epochs, get_grad)
Out[54]:
In [59]:
### Hyper parameters
batch_size = 128
n_epochs = 15
learning_rate = 1e-10
input_size = 28*28
hidden_size = 100
output_size = 10
l2_norm = 0 # use l2 penalty
get_grad = True
# declare a model
model = FeedForwardNeuralNetwork(input_size=input_size, hidden_size=hidden_size, output_size=output_size)
# Cross entropy
loss_fn = torch.nn.CrossEntropyLoss()
# l2_norm can be done in SGD
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, weight_decay=l2_norm)
In [60]:
fit(train_loader, test_loader, model, loss_fn, optimizer, n_epochs, get_grad=get_grad)
Out[60]:
In [61]:
### Hyper parameters
batch_size = 128
n_epochs = 15
learning_rate = 10
input_size = 28*28
hidden_size = 100
output_size = 10
l2_norm = 0 # not to use l2 penalty
get_grad = True
# declare a model
model = FeedForwardNeuralNetwork(input_size=input_size, hidden_size=hidden_size, output_size=output_size)
# Cross entropy
loss_fn = torch.nn.CrossEntropyLoss()
# l2_norm can be done in SGD
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, weight_decay=l2_norm)
In [62]:
fit(train_loader, test_loader, model, loss_fn, optimizer, n_epochs, get_grad=True)
Out[62]:
In [63]:
### Hyper parameters
batch_size = 128
n_epochs = 15
learning_rate = 1
input_size = 28*28
hidden_size = 100
output_size = 10
l2_norm = 0 # not to use l2 penalty
get_grad = True
# declare a model
model = FeedForwardNeuralNetwork(input_size=input_size, hidden_size=hidden_size, output_size=output_size)
# Cross entropy
loss_fn = torch.nn.CrossEntropyLoss()
# l2_norm can be done in SGD
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, weight_decay=l2_norm)
In [64]:
# reset parameters as 10
def wrong_weight_bias_reset(model):
"""Using normalization with mean=0, std=1 to initialize model's parameter
"""
for m in model.modules():
if isinstance(m, nn.Linear):
# initialize linear layer with mean and std
mean, std = 0, 1
# Initialization method
torch.nn.init.normal_(m.weight, mean, std)
torch.nn.init.normal_(m.bias, mean, std)
In [65]:
wrong_weight_bias_reset(model)
show_weight_bias(model)
In [66]:
fit(train_loader, test_loader, model, loss_fn, optimizer, n_epochs, get_grad=True)
Out[66]: