Web: https://www.meetup.com/Tel-Aviv-Deep-Learning-Bootcamp/events/241762893/
Notebooks: On GitHub
Shlomo Kashani
http://pytorch.org/tutorials/beginner/transfer_learning_tutorial.html
https://medium.com/towards-data-science/my-first-kaggle-competition-9d56d4773607
https://github.com/rwightman/pytorch-planet-amazon/blob/master/dataset.py
in the images training folder copy 1.png to 0.png and add the same label inside training labels.
https://github.com/MorvanZhou/PyTorch-Tutorial/blob/master/tutorial-contents/401_CNN.py
In [1]:
# !pip install pycuda
%reset -f
import numpy
import numpy as np
from __future__ import print_function
from __future__ import division
import math
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import pandas as pd
import os
import torch
from torch.utils.data.dataset import Dataset
from torch.utils.data import DataLoader
from torchvision import transforms
from torch import nn
import torch.nn.functional as F
import torch.optim as optim
from torch.autograd import Variable
from sklearn.preprocessing import MultiLabelBinarizer
import time
%matplotlib inline
from pylab import rcParams
rcParams['figure.figsize'] = (6, 6) # setting default size of plots
import tensorflow as tf
print("tensorflow:" + tf.__version__)
!set "KERAS_BACKEND=tensorflow"
import torch
import sys
print('__Python VERSION:', sys.version)
print('__pyTorch VERSION:', torch.__version__)
print('__CUDA VERSION')
from subprocess import call
print('__CUDNN VERSION:', torch.backends.cudnn.version())
print('__Number CUDA Devices:', torch.cuda.device_count())
print('__Devices')
# !pip install http://download.pytorch.org/whl/cu75/torch-0.2.0.post1-cp27-cp27mu-manylinux1_x86_64.whl
# !pip install torchvision
# ! pip install cv2
# import cv2
import logging
handler=logging.basicConfig(level=logging.INFO)
lgr = logging.getLogger(__name__)
print("OS: ", sys.platform)
print("Python: ", sys.version)
print("PyTorch: ", torch.__version__)
print("Numpy: ", np.__version__)
from multiprocessing import cpu_count
print("cpu_count: ", cpu_count())
# !pip install psutil
import psutil
def cpuStats():
print(sys.version)
print(psutil.cpu_percent())
print(psutil.virtual_memory()) # physical memory usage
pid = os.getpid()
py = psutil.Process(pid)
memoryUse = py.memory_info()[0] / 2. ** 30 # memory use in GB...I think
print('memory GB:', memoryUse)
cpuStats()
In [2]:
# Torch CPU
# !pip install http://download.pytorch.org/whl/cu75/torch-0.2.0.post1-cp27-cp27mu-manylinux1_x86_64.whl
# !pip install torchvision
# ! pip install intel-gpu-tools
# ! pip install snakeviz
# ! pip install runsnakerun
# python -m cProfile -o out.prof mycode.py
In [3]:
use_cuda = torch.cuda.is_available()
# use_cuda = False
FloatTensor = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor
LongTensor = torch.cuda.LongTensor if use_cuda else torch.LongTensor
Tensor = FloatTensor
# if torch.cuda.is_available():
# print("WARNING: You have a CUDA device, so you should probably run with --cuda")
# ! watch -n 1 nvidia-smi
# ! nvidia-smi -l 1
# nvidia-settings -q GPUUtilization -q useddedicatedgpumemory
# You can also use:
# ! watch -n0.1 "nvidia-settings -q GPUUtilization -q useddedicatedgpumemory"
# ! pip install git+https://github.com/wookayin/gpustat.git@master
# ! watch --color -n1.0 gpustat
# ! gpustat
# ! watch -n 5 nvidia-smi --format=csv --query-gpu=power.draw,utilization.gpu,fan.speed,temperature.gpu
In [9]:
! df -k
DATA_ROOT ='/root/data/cifar/'
IMG_PATH = DATA_ROOT + '/train/'
IMG_EXT = '.png'
IMG_DATA_LABELS = DATA_ROOT + '/trainLabels.csv'
classes = ('plane', 'car', 'bird', 'cat',
'deer', 'dog', 'frog', 'horse', 'ship', 'truck')
In [10]:
DROPOUT_PROB = 0.85
N_EPOCHS = 50
BATCH_SIZE = 4
LR = 0.005
TEST_RATIO = .22
MOMENTUM= 0.9
PIN_MEMORY=use_cuda # True IF CUDA
# p is the probability of being dropped in PyTorch
dropout = torch.nn.Dropout(p=1 - DROPOUT_PROB)
# fix seed
seed=17*19
np.random.seed(seed)
torch.manual_seed(seed)
if use_cuda:
torch.cuda.manual_seed(seed)
In [11]:
%%time
transformations = transforms.Compose([transforms.ToTensor()])
# transformations = transforms.Compose([transforms.Scale(32),transforms.ToTensor()])
# transformations=transforms.Compose([transforms.Scale(32),transforms.ToTensor(),
# transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
# ])
# transformations=transforms.Compose([transforms.Scale(32), transforms.ToTensor(), # first, convert image to PyTorch tensor
# transforms.Normalize((0.1307,), (0.3081,)) # normalize inputs
# ])
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
In [12]:
try:
from PIL import Image
except ImportError:
import Image
from sklearn.preprocessing import LabelEncoder
from sklearn.pipeline import Pipeline
from collections import defaultdict
class GenericImageDataset(Dataset):
def __init__(self, csv_path, img_path, img_ext, tSet=True):
t = time.time()
lgr.info('CSV path {}'.format(csv_path))
lgr.info('IMG path {}'.format(img_path))
assert img_ext in ['.png']
self.tSet=tSet
tmp_df = pd.read_csv(csv_path, header=None) # I manually removed the header
self.img_path = img_path
self.img_ext = img_ext
# Encoding the variables
d = defaultdict(LabelEncoder)
tmp_df = tmp_df.apply(lambda x: d[x.name].fit_transform(x))
lgr.info("DF CSV:\n" + str (tmp_df.head(5)))
self.X_train = tmp_df[0]
self.y_train = tmp_df[1]
lgr.info("DF X_train:\n" + str (self.X_train.head(5)))
lgr.info ("DF y_train:\n" + str(self.y_train.head(5)))
lgr.info('[*] Dataset loading time {}'.format(time.time() - t))
lgr.info('[*] Data size is {}'.format(len(self)))
def __getitem__(self, index):
path=self.img_path + str(self.X_train[index]) + self.img_ext
img = Image.open(path)
if self.tSet==True:
self.transform=transform_train
else:
self.transform=transform_test
img = self.transform(img)
# label = self.y_train[index]
label = (self.y_train[index])
# lgr.info ("__getitem__:" + str(index) + " Label:" + str(label))
return img, label
def __len__(self):
l=len(self.X_train.index)
return (l)
@staticmethod
def imshow(img):
img = img / 2 + 0.5 # unnormalize
npimg = img.numpy()
plt.imshow(np.transpose(npimg, (1, 2, 0)))
@staticmethod
def flaotTensorToImage(img, mean=0, std=1):
"""convert a tensor to an image"""
img = np.transpose(img.numpy(), (1, 2, 0))
img = (img*std+ mean)*255
img = img.astype(np.uint8)
return img
@staticmethod
def toTensor(img):
"""convert a numpy array of shape HWC to CHW tensor"""
img = img.transpose((2, 0, 1)).astype(np.float32)
tensor = torch.from_numpy(img).float()
return tensor/255.0
In [13]:
%%time
dset_train = GenericImageDataset(IMG_DATA_LABELS,
IMG_PATH,
IMG_EXT,tSet=True)
# dset_train = GenericImageDataset(IMG_DATA_LABELS,
# IMG_PATH,
# IMG_EXT,transformations)
In [14]:
%%time
class FullTrainingDataset(torch.utils.data.Dataset):
def __init__(self, full_ds, offset, length):
self.full_ds = full_ds
self.offset = offset
self.length = length
assert len(full_ds)>=offset+length, Exception("Parent Dataset not long enough")
super(FullTrainingDataset, self).__init__()
def __len__(self):
return self.length
def __getitem__(self, i):
return self.full_ds[i+self.offset]
def trainTestSplit(dataset, val_share=TEST_RATIO):
val_offset = int(len(dataset)*(1-val_share))
return FullTrainingDataset(dataset, 0, val_offset), FullTrainingDataset(dataset, val_offset, len(dataset)-val_offset)
train_ds, val_ds = trainTestSplit(dset_train)
train_loader = torch.utils.data.DataLoader(train_ds, batch_size=BATCH_SIZE,
shuffle=True, num_workers=1,pin_memory=PIN_MEMORY)
val_loader = torch.utils.data.DataLoader(val_ds, batch_size=BATCH_SIZE, shuffle=True, num_workers=1,
pin_memory=PIN_MEMORY)
In [15]:
# Test the DataLoader Class
def get_mean_and_std(dataset):
'''Compute the mean and std value of dataset.'''
dataloader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=True, num_workers=2)
mean = torch.zeros(3)
std = torch.zeros(3)
print('==> Computing mean and std..')
for inputs, targets in dataloader:
for i in range(3):
mean[i] += inputs[:,i,:,:].mean()
std[i] += inputs[:,i,:,:].std()
mean.div_(len(dataset))
std.div_(len(dataset))
return mean, std
# get_mean_and_std(train_ds)
# ==> Computing mean and std..
# Out[101]:
# (
# 0.4913
# 0.4819
# 0.4459
# [torch.FloatTensor of size 3],
# 0.2026
# 0.1996
# 0.2009
# [torch.FloatTensor of size 3])
In [16]:
%%time
imagesToShow=4
for i, data in enumerate(train_loader, 0):
lgr.info('i=%d: '%(i))
images, labels = data
num = len(images)
ax = plt.subplot(1, imagesToShow, i + 1)
plt.tight_layout()
ax.set_title('Sample #{}'.format(i))
ax.axis('off')
for n in range(num):
image=images[n]
label=labels[n]
plt.imshow (GenericImageDataset.flaotTensorToImage(image))
if i==imagesToShow-1:
break
In [17]:
%%time
for i, data in enumerate(val_loader, 0):
lgr.info('i=%d: '%(i))
images, labels = data
num = len(images)
ax = plt.subplot(1, imagesToShow, i + 1)
plt.tight_layout()
ax.set_title('Sample #{}'.format(i))
ax.axis('off')
for n in range(num):
image=images[n]
label=labels[n]
plt.imshow (GenericImageDataset.flaotTensorToImage(image))
if i==imagesToShow-1:
break
nninit.xavier_uniform(tensor, gain=1) - Fills tensor with values according to the method described in "Understanding the difficulty of training deep feedforward neural networks" - Glorot, X. and Bengio, Y., using a uniform distribution.nninit.xavier_normal(tensor, gain=1) - Fills tensor with values according to the method described in "Understanding the difficulty of training deep feedforward neural networks" - Glorot, X. and Bengio, Y., using a normal distribution.nninit.kaiming_uniform(tensor, gain=1) - Fills tensor with values according to the method described in "Delving deep into rectifiers: Surpassing human-level performance on ImageNet classification" - He, K. et al. using a uniform distribution.nninit.kaiming_normal(tensor, gain=1) - Fills tensor with values according to the method described in ["Delving deep into rectifiers: Surpassing human-level performance on ImageNet classification" - He, K. et al.]
In [18]:
%%time
class Net(nn.Module):
def __init__(self,initKernel='uniform'):
super(Net, self).__init__()
self.conv1 = nn.Conv2d(3, 6, 5)
self.pool = nn.MaxPool2d(2, 2)
self.conv2 = nn.Conv2d(6, 16, 5)
self.fc1 = nn.Linear(16 * 5 * 5, 120)
self.fc2 = nn.Linear(120, 84)
self.fc3 = nn.Linear(84, 10)
# xavier initializer
if initKernel == 'uniform':
nn.init.xavier_uniform(self.conv1.weight, gain=np.sqrt(2.0))
# nn.init.xavier_uniform(self.conv2.weight, gain=np.sqrt(2.0))
else:
nn.init.kaiming_normal(self.conv1.weight)
# nn.init.kaiming_normal(self.conv2.weight)
def forward(self, x):
x = self.pool(F.relu(self.conv1(x)))
x = self.pool(F.relu(self.conv2(x)))
x = x.view(-1, 16 * 5 * 5)
x = F.relu(self.fc1(x))
x = F.relu(self.fc2(x))
x = self.fc3(x)
return x
# class Net(nn.Module):
# def __init__(self):
# super(Net, self).__init__()
# self.conv1 = nn.Conv2d(3, 32, kernel_size=3, padding=1)
# self.conv2 = nn.Conv2d(32, 64, kernel_size=3, padding=1)
# self.conv3 = nn.Conv2d(64, 64, kernel_size=3, padding=1)
# self.conv4 = nn.Conv2d(64, 64, kernel_size=3, padding=1)
# self.conv5 = nn.Conv2d(64, 128, kernel_size=3, padding=1)
# self.conv6 = nn.Conv2d(128, 128, kernel_size=3, padding=1)
# self.fc1 = nn.Linear(512 * 4, 256)
# self.fc2 = nn.Linear(256, 10)
# def forward(self, x):
# h = F.relu(self.conv1(x))
# h = F.relu(self.conv2(h))
# h = F.max_pool2d(h, 2)
# h = F.relu(self.conv3(h))
# h = F.relu(self.conv4(h))
# h = F.max_pool2d(h, 2)
# h = F.relu(self.conv5(h))
# h = F.relu(self.conv6(h))
# h = F.max_pool2d(h, 2)
# h = h.view(-1, 512 * 4)
# h = F.relu(self.fc1(h))
# h = F.dropout(h, training=self.training)
# h = F.log_softmax(self.fc2(h))
# return h
if use_cuda:
lgr.info ("Using the GPU")
model = Net().cuda() # On GPU
else:
lgr.info ("Using the CPU")
model = Net() # On CPU
lgr.info('Model {}'.format(model))
In [20]:
%%time
# criterion = nn.ClassNLLCriterion() -- a negative log-likelihood criterion for multi-class classification
# criterion = nn.CrossEntropyLoss() # Softmax is internally computed.
# criterion = F.nll_loss()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=LR, momentum=MOMENTUM)
# optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)
# optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9, weight_decay=5e-4)
# optimizer = torch.optim.Adam(model.parameters(), lr=LR)
lgr.info('Loss function {}'.format(criterion))
lgr.info('Optimizer {}'.format(optimizer))
See example here: http://codegists.com/snippet/python/pytorch_mnistpy_kernelmode_python
https://github.com/pytorch/examples/blob/53f25e0d0e2710878449900e1e61d31d34b63a9d/mnist/main.py
In [55]:
%%time
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms
from torch.autograd import Variable
loss_history = []
loss_history_2 = []
acc_history = []
running_loss = 0.0
def train(epoch):
model.train()
running_loss = 0.0
for batch_idx, (data, target) in enumerate(train_loader): #
# torch.LongTensor of size BATCH_SIZE
# [torch.FloatTensor of size 4x3x32x32] BATCH_SIZE* IMAGE SIZE
if use_cuda:
data, target = Variable(data.cuda(async=True)), Variable(target.cuda(async=True)) # On GPU
else:
data, target = Variable(data), Variable(target)
# You will get RuntimeError: expected CPU tensor (got CUDA tensor) if you dont do this
optimizer.zero_grad()
outputs = model(data)
if use_cuda:
loss = criterion(outputs, target).cuda()
# loss = F.nll_loss(outputs, target).cuda()
else:
loss = criterion(outputs, target)
# loss = F.nll_loss(outputs, target)
loss.backward()
optimizer.step()
running_loss += loss.data[0]
if batch_idx % 2000 == 1999: # print every 2000 mini-batches
print('[%d, %5d] loss: %.3f' % (epoch + 1, i + 1, running_loss / 2000))
loss_history.append(running_loss / 2000)
running_loss = 0.0
if batch_idx % 3000 == 0:
loss_history_2.append(loss.data[0])
lgr.info('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
epoch, batch_idx * len(data), len(train_loader.dataset),
100. * batch_idx / len(train_loader), loss.data[0]))
start_time = time.time()
for epoch in range(1, N_EPOCHS):
print("Epoch %d" % epoch)
train(epoch)
print('Finished Training epoch:' + str(epoch))
end_time = time.time()
%matplotlib inline
import matplotlib.pyplot as plt
plt.plot(loss_history)
plt.show()
In [56]:
%matplotlib inline
import matplotlib.pyplot as plt
plt.plot(loss_history_2)
plt.show()
In [57]:
%%time
import torchvision
def imshow(img):
img = img / 2 + 0.5 # unnormalize
npimg = img.numpy()
plt.imshow(np.transpose(npimg, (1, 2, 0)))
dataiter = iter(val_loader)
images, labels = dataiter.next()
print (labels)
imshow(torchvision.utils.make_grid(images))
print('GroundTruth: ', ' '.join('%5s' % classes[labels[j]] for j in range(4)))
if use_cuda:
images = Variable(images.cuda(async=True))
else:
images= Variable(images)
outputs = model(images)
_, predicted = torch.max(outputs.data, 1)
predicted=predicted.cpu().numpy()
print (predicted)
print (type(predicted))
for y in np.nditer(predicted, op_flags=['readwrite']):
print (classes[y])
In [58]:
%%time
def test(epoch):
model.eval() # set model in inference mode (need this because of dropout)
test_loss = 0
correct = 0
for data, target in val_loader:
if use_cuda:
data, target = Variable(data.cuda(async=True), volatile=True), Variable(target.cuda(async=True)) # On GPU
else:
data, target = Variable(data), Variable(target)
output = model(data)
test_loss += F.nll_loss(output, target).data[0]
# test_loss += criterion(output, target).data[0]
pred = output.data.max(1)[1] # get the index of the max log-probability
correct += pred.eq(target.data).cpu().sum()
test_loss = test_loss
test_loss /= len(val_loader) # loss function already averages over batch size
accuracy = 100. * correct / len(val_loader.dataset)
acc_history.append(accuracy)
print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
test_loss, correct, len(val_loader.dataset),
accuracy))
for epoch in range(0, 1):
print("Epoch %d" % epoch)
test(epoch)
In [49]:
# correct = 0
# total = 0
# for data in val_loader:
# images, labels = data
# if use_cuda:
# outputs = model(Variable(images.cuda(async=True)))
# else:
# outputs = model(Variable(images))
# _, predicted = torch.max(outputs.data, 1)
# total += labels.size(0)
# correct += (predicted == labels).sum()
# print ("Total:" + str (total))
# print ("correct:" + str (correct))
# print('Accuracy of the network on the 10000 test images: %d %%' % (100 * correct / total))
In [1]:
! watch -n 1 nvidia-smi
In [334]:
%%bash
jupyter nbconvert \
--to=slides \
--reveal-prefix=https://cdnjs.cloudflare.com/ajax/libs/reveal.js/3.2.0/ \
--output=py09.html \
'./09 PyTorch Kaggle Image Data-set loading with CNN'