In [37]:
import torch 
import torchvision
import torch.nn as nn
import numpy as np
import torch.utils.data as data
import torchvision.transforms as transforms
import torchvision.datasets as dsets
from torch.autograd import Variable

In [10]:
# torch.rand(sizes) -> [0,1]
x = torch.rand(2,3)
x


Out[10]:
 0.9222  0.6813  0.4138
 0.2331  0.3903  0.4302
[torch.FloatTensor of size 2x3]

In [11]:
# torch.randn(sizes) -> normalize Z[0, 1]
y = torch.randn(2,3)
y


Out[11]:
-0.1991 -0.2063 -1.0843
 1.0410  0.9436 -1.1402
[torch.FloatTensor of size 2x3]

In [12]:
# torch.randperm(n) : permutation of 0-n
z = torch.randperm(5)
z
# return이 LongTensor type이네요..!


Out[12]:
 3
 1
 4
 0
 2
[torch.LongTensor of size 5]

In [13]:
# torch.zeros -> np.zeros와 동일
x = torch.zeros(2,3)
x


Out[13]:
 0  0  0
 0  0  0
[torch.FloatTensor of size 2x3]

In [15]:
# torch.ones(2,3) -> np.ones와 동일
y = torch.ones(2,3)
y


Out[15]:
 1  1  1
 1  1  1
[torch.FloatTensor of size 2x3]

In [17]:
# torch.arange(start, end, step)
z = torch.arange(0,2, step=0.3)
z


Out[17]:
 0.0000
 0.3000
 0.6000
 0.9000
 1.2000
 1.5000
 1.8000
[torch.FloatTensor of size 7]

In [23]:
# torch.FloatTensor(size or list)
x = torch.FloatTensor(2,3)
x


Out[23]:
 0.0000e+00  2.5244e-29 -1.1232e+29
-2.8615e-42  2.8026e-45  2.5244e-29
[torch.FloatTensor of size 2x3]

In [19]:
x = torch.FloatTensor([2,3])
x


Out[19]:
 2
 3
[torch.FloatTensor of size 2]

In [25]:
x = torch.FloatTensor([[2,3],[3,3]])
x


Out[25]:
 2  3
 3  3
[torch.FloatTensor of size 2x2]

In [26]:
# torch.type_as(tensor_type)
x = x.type_as(torch.IntTensor())
x


Out[26]:
 2  3
 3  3
[torch.IntTensor of size 2x2]

In [31]:
# torch.LongTensor
torch.LongTensor(1,2)


Out[31]:
 1.1529e+18  5.7646e+18
[torch.LongTensor of size 1x2]

In [6]:
# create tensors
x = Variable(torch.Tensor([1]), requires_grad=True)

In [7]:
x


Out[7]:
Variable containing:
 1
[torch.FloatTensor of size 1]

In [32]:
import numpy as np

In [33]:
# torch.from_numpy(np객체)
x1 = np.ndarray(shape=(2,3), dtype=int,buffer=np.array([1,2,3,4,5,6]))
x2 = torch.from_numpy(x1)
x2


Out[33]:
 1  2  3
 4  5  6
[torch.LongTensor of size 2x3]

In [37]:
# tensor.size() : indexing
x = torch.FloatTensor(10,12,3,3)

x.size()[:]


Out[37]:
torch.Size([10, 12, 3, 3])

In [38]:
x.size()[1:]


Out[38]:
torch.Size([12, 3, 3])

In [40]:
x = torch.rand(4,3)
out = torch.index_select(x,0,torch.LongTensor([0,3]))

x,out


Out[40]:
(
  0.9576  0.7327  0.0311
  0.4164  0.0293  0.1830
  0.0280  0.4258  0.4594
  0.8665  0.4299  0.7726
 [torch.FloatTensor of size 4x3], 
  0.9576  0.7327  0.0311
  0.8665  0.4299  0.7726
 [torch.FloatTensor of size 2x3])

In [42]:
x[:,0]


Out[42]:
 0.9576
 0.4164
 0.0280
 0.8665
[torch.FloatTensor of size 4]

In [43]:
x[:,:]


Out[43]:
 0.9576  0.7327  0.0311
 0.4164  0.0293  0.1830
 0.0280  0.4258  0.4594
 0.8665  0.4299  0.7726
[torch.FloatTensor of size 4x3]

In [45]:
x[1:2,:]


Out[45]:
 0.4164  0.0293  0.1830
[torch.FloatTensor of size 1x3]

In [46]:
# torch.masked_select(input, mask)

x = torch.randn(2,3)
mask = torch.ByteTensor([[0,0,1],[0,1,0]])
out = torch.masked_select(x,mask)

x, mask, out


Out[46]:
(
  0.5139 -0.7707 -0.6250
 -1.5479 -0.5976 -1.4943
 [torch.FloatTensor of size 2x3], 
  0  0  1
  0  1  0
 [torch.ByteTensor of size 2x3], 
 -0.6250
 -0.5976
 [torch.FloatTensor of size 2])

In [47]:
mask


Out[47]:
 0  0  1
 0  1  0
[torch.ByteTensor of size 2x3]

In [48]:
# torch.cat(seq, dim=0) : concat
x = torch.FloatTensor([[1,2,3], [4,5,6]])
y = torch.FloatTensor([[10,11,12],[-1,-2,-3]])

In [50]:
z1 = torch.cat([x,y], dim=0)
z1


Out[50]:
  1   2   3
  4   5   6
 10  11  12
 -1  -2  -3
[torch.FloatTensor of size 4x3]

In [51]:
z2 = torch.cat([x,y], dim=1)
z2


Out[51]:
  1   2   3  10  11  12
  4   5   6  -1  -2  -3
[torch.FloatTensor of size 2x6]

In [52]:
# torch.stack(sequence, dim=0) : stack along new dim
x = torch.FloatTensor([[1,2,3], [4,5,6]])
x_stack = torch.stack([x,x,x,x], dim=0)

x_stack


Out[52]:
(0 ,.,.) = 
  1  2  3
  4  5  6

(1 ,.,.) = 
  1  2  3
  4  5  6

(2 ,.,.) = 
  1  2  3
  4  5  6

(3 ,.,.) = 
  1  2  3
  4  5  6
[torch.FloatTensor of size 4x2x3]

In [53]:
# torch.chunk(tensor,cunks, dim)
x_1, x_2 = torch.chunk(z1, 2, dim=0)

In [57]:
z1


Out[57]:
  1   2   3
  4   5   6
 10  11  12
 -1  -2  -3
[torch.FloatTensor of size 4x3]

In [54]:
x_1, x_2


Out[54]:
(
  1  2  3
  4  5  6
 [torch.FloatTensor of size 2x3], 
  10  11  12
  -1  -2  -3
 [torch.FloatTensor of size 2x3])

In [55]:
y_1, y_2, y_3 = torch.chunk(z1,3,dim=1)

In [56]:
y_1, y_2, y_3


Out[56]:
(
   1
   4
  10
  -1
 [torch.FloatTensor of size 4x1], 
   2
   5
  11
  -2
 [torch.FloatTensor of size 4x1], 
   3
   6
  12
  -3
 [torch.FloatTensor of size 4x1])

In [58]:
# torch.splie(tensor, split_size, dim)
x1, x2 = torch.split(z1, 2, dim=0)

In [63]:
z1


Out[63]:
  1   2   3
  4   5   6
 10  11  12
 -1  -2  -3
[torch.FloatTensor of size 4x3]

In [59]:
x1, x2


Out[59]:
(
  1  2  3
  4  5  6
 [torch.FloatTensor of size 2x3], 
  10  11  12
  -1  -2  -3
 [torch.FloatTensor of size 2x3])

In [61]:
y1 = torch.split(z1, 2, dim=1)

In [62]:
y1


Out[62]:
(
   1   2
   4   5
  10  11
  -1  -2
 [torch.FloatTensor of size 4x2], 
   3
   6
  12
  -3
 [torch.FloatTensor of size 4x1])

In [68]:
# torch.squeeze(input, dim) : 1차원 tensor 제거
x1 = torch.FloatTensor(10,1,3)
x1


Out[68]:
(0 ,.,.) = 
  0.0000e+00  2.5244e-29 -1.1205e+29

(1 ,.,.) = 
  1.0845e-19  1.1210e-44  1.3873e-43

(2 ,.,.) = 
  1.4574e-43  6.4460e-44  1.6115e-43

(3 ,.,.) = 
  1.5835e-43  1.6395e-43  1.4153e-43

(4 ,.,.) = 
  1.4153e-43  1.7096e-43  1.4153e-43

(5 ,.,.) = 
  5.6052e-44  1.4714e-43  1.5414e-43

(6 ,.,.) = 
  1.5695e-43  1.6395e-43  1.6255e-43

(7 ,.,.) = 
  6.1657e-44  4.4842e-44  1.4013e-43

(8 ,.,.) = 
  1.4714e-43  1.5274e-43  5.7453e-44

(9 ,.,.) = 
         nan  2.2369e+08  2.2369e+08
[torch.FloatTensor of size 10x1x3]

In [69]:
x2 = torch.squeeze(x1)
x2


Out[69]:
 0.0000e+00  2.5244e-29 -1.1205e+29
 1.0845e-19  1.1210e-44  1.3873e-43
 1.4574e-43  6.4460e-44  1.6115e-43
 1.5835e-43  1.6395e-43  1.4153e-43
 1.4153e-43  1.7096e-43  1.4153e-43
 5.6052e-44  1.4714e-43  1.5414e-43
 1.5695e-43  1.6395e-43  1.6255e-43
 6.1657e-44  4.4842e-44  1.4013e-43
 1.4714e-43  1.5274e-43  5.7453e-44
        nan  2.2369e+08  2.2369e+08
[torch.FloatTensor of size 10x3]

In [70]:
x1.size(), x2.size()


Out[70]:
(torch.Size([10, 1, 3]), torch.Size([10, 3]))

In [72]:
# torch.unsqueeze(input,dim=None) -> 1차원 추가

x1 = torch.FloatTensor(10,3,4)
x2 = torch.unsqueeze(x1,dim=0)

x1.size(),x2.size()


Out[72]:
(torch.Size([10, 3, 4]), torch.Size([1, 10, 3, 4]))

In [73]:
# torch.add() : x1+x2와 같은 연산
x1 = torch.FloatTensor([[1,2,3],[4,5,6]])
x2 = torch.FloatTensor([[1,2,3],[4,5,6]])
add = torch.add(x1,x2)

x1,x2,add,x1+x2,x1-x2


Out[73]:
(
  1  2  3
  4  5  6
 [torch.FloatTensor of size 2x3], 
  1  2  3
  4  5  6
 [torch.FloatTensor of size 2x3], 
   2   4   6
   8  10  12
 [torch.FloatTensor of size 2x3], 
   2   4   6
   8  10  12
 [torch.FloatTensor of size 2x3], 
  0  0  0
  0  0  0
 [torch.FloatTensor of size 2x3])

In [74]:
# torch.add() : broadcasting 연산도 가능

x1 = torch.FloatTensor([[1,2,3],[4,5,6]])
x2 = torch.add(x1,10)

x1,x2,x1+10,x2-10


Out[74]:
(
  1  2  3
  4  5  6
 [torch.FloatTensor of size 2x3], 
  11  12  13
  14  15  16
 [torch.FloatTensor of size 2x3], 
  11  12  13
  14  15  16
 [torch.FloatTensor of size 2x3], 
  1  2  3
  4  5  6
 [torch.FloatTensor of size 2x3])

In [76]:
# torch.mul() 
x1 = torch.FloatTensor([[1,2,3],[4,5,6]])
x2 = torch.FloatTensor([[1,2,3],[4,5,6]])
x3 = torch.mul(x1,x2)

x3, x1*10


Out[76]:
(
   1   4   9
  16  25  36
 [torch.FloatTensor of size 2x3], 
  10  20  30
  40  50  60
 [torch.FloatTensor of size 2x3])

In [77]:
# torch.div() -> size better match

x1 = torch.FloatTensor([[1,2,3],[4,5,6]])
x2 = torch.FloatTensor([[1,2,3],[4,5,6]])
x3 = torch.div(x1,x2)

x3, x1/5


Out[77]:
(
  1  1  1
  1  1  1
 [torch.FloatTensor of size 2x3], 
  0.2000  0.4000  0.6000
  0.8000  1.0000  1.2000
 [torch.FloatTensor of size 2x3])

In [78]:
# torch.pow(inpue, exponent) : 제곱
x1 = torch.FloatTensor(3,4)
torch.pow(x1,2),x1**2


Out[78]:
(
   0   0   0   0
   0   0   0   0
 inf   0   0   0
 [torch.FloatTensor of size 3x4], 
   0   0   0   0
   0   0   0   0
 inf   0   0   0
 [torch.FloatTensor of size 3x4])

In [79]:
# torch.exp(tensor,out=None) 

x1 = torch.FloatTensor(3,4)
torch.exp(x1)


Out[79]:
 1  1  1  1
 1  1  1  1
 1  1  1  1
[torch.FloatTensor of size 3x4]

In [80]:
# torch.log(input, out=None) 

x1 = torch.FloatTensor(3,4)
torch.log(x1)


Out[80]:
     -inf  -65.8490      -inf  -65.8490
-102.1803      -inf      -inf      -inf
     -inf      -inf      -inf  -91.0900
[torch.FloatTensor of size 3x4]

In [81]:
# torch.mm(mat1, mat2) 

x1 = torch.FloatTensor(3,4)
x2 = torch.FloatTensor(4,5)

torch.mm(x1,x2)


Out[81]:
1.00000e-15 *
  0.0000  0.0000  0.0000  0.0000  0.0000
  0.0000  0.0000 -1.4233  0.0000  0.0000
  0.0000  0.0000  0.0000  0.0000  0.0000
[torch.FloatTensor of size 3x5]

In [82]:
# torch.bmm(batch1, batch2) -> batch matrix multiplication

x1 = torch.FloatTensor(10,3,4)
x2 = torch.FloatTensor(10,4,5)

torch.bmm(x1,x2).size()


Out[82]:
torch.Size([10, 3, 5])

In [84]:
torch.bmm(x1,x2)


Out[84]:
(0 ,.,.) = 
 -0.0000  0.0000  0.0000 -0.0000 -0.0000
  0.0000  0.0000 -0.0000  0.0000  0.0000
  0.0000  0.0000  0.0000  0.0000  0.0000

(1 ,.,.) = 
  0.0000  0.0000  0.0000  0.0000  0.0000
  0.0000  0.0000  0.0000  0.0000  0.0000
  0.0000  0.0000  0.0000  0.0000  0.0000

(2 ,.,.) = 
  0.0000  0.0000  0.0000  0.0000  0.0000
  0.0000  0.0000  0.0000  0.0000  0.0000
  0.0000  0.0000  0.0000  0.0000  0.0000

(3 ,.,.) = 
  0.0000  0.0000  0.0000  0.0000  0.0000
  0.0000  0.0000  0.0000  0.0000  0.0000
  0.0000  0.0000  0.0000  0.0000  0.0000

(4 ,.,.) = 
  0.0000  0.0000  0.0000  0.0000  0.0000
  0.0000  0.0000  0.0000  0.0000  0.0000
  0.0000  0.0000  0.0000  0.0000  0.0000

(5 ,.,.) = 
  0.0000  0.0000 -0.0000 -0.0000  0.0000
  0.0000  0.0000  0.0000  0.0000  0.0000
  0.0000  0.0000  0.0000  0.0000  0.0000

(6 ,.,.) = 
  0.0000  0.0000  0.0000  0.0000  0.0000
  0.0000  0.0000  0.0000  0.0000  0.0000
  0.0000  0.0000  0.0000  0.0000  0.0000

(7 ,.,.) = 
  0.0000  0.0000  0.0000  0.0000  0.0000
  0.0000  0.0000  0.0000  0.0000  0.0000
  0.0000  0.0000  0.0000  0.0000  0.0000

(8 ,.,.) = 
  0.0000  0.0000  0.0000  0.0000  0.0000
  0.0000  0.0000  0.0000  0.0000  0.0000
  0.0000  0.0000  0.0000  0.0000  0.0000

(9 ,.,.) = 
  0.0000  0.0000  0.0000  0.0000 -0.0019
  0.0000  0.0000  0.0000  0.0000 -0.0019
  0.0000  0.0000  0.0000 -2.2228     inf
[torch.FloatTensor of size 10x3x5]

In [96]:
# torch.dot([tensor1,tensor2]) -> dot product of two tensor

x1 = torch.FloatTensor([1,1])
x2 = torch.FloatTensor([1,3])

torch.dot(x1,x2)


Out[96]:
13.0

In [97]:
x1 = torch.FloatTensor([1,1,3])
x2 = torch.FloatTensor([1,3,10])

torch.dot(x1,x2)


Out[97]:
34.0

In [98]:
# torch.t(matrix) -> transposed matrix

x1 = torch.FloatTensor(3,4)

x1,x1.t()


Out[98]:
(
 1.00000e-29 *
   0.0000  2.5244  0.0000  2.5244
   0.0000  2.5244  0.0000  2.5244
   0.0000  0.0000  0.0000  0.0000
 [torch.FloatTensor of size 3x4], 
 1.00000e-29 *
   0.0000  0.0000  0.0000
   2.5244  2.5244  0.0000
   0.0000  0.0000  0.0000
   2.5244  2.5244  0.0000
 [torch.FloatTensor of size 4x3])

In [99]:
# torch.transpose(input,dim0,dim1) -> transposed matrix

x1 = torch.FloatTensor(10,3,4)

x1.size(), torch.transpose(x1,1,2).size(), x1.transpose(1,2).size()


Out[99]:
(torch.Size([10, 3, 4]), torch.Size([10, 4, 3]), torch.Size([10, 4, 3]))

In [5]:
# torch.eig(a,eigenvectors=False) -> eigen_value, eigen_vector

x1 = torch.FloatTensor(3,3)

x1,torch.eig(x1,True)


Out[5]:
(
 1.00000e-29 *
   0.0000  2.5244  0.0000
  -0.0000  0.0000  0.0000
   0.0000  0.0000  0.0000
 [torch.FloatTensor of size 3x3], (
  1.00000e-36 *
    0.0000  8.5011
    0.0000 -8.5011
    0.0000  0.0000
  [torch.FloatTensor of size 3x2], 
   1.0000  0.0000 -0.0000
   0.0000  0.0000 -0.0000
   0.0000  0.0000  1.0000
  [torch.FloatTensor of size 3x3]))

Create tensor


In [12]:
x = Variable(torch.Tensor([1]), requires_grad=True)
w = Variable(torch.Tensor([2]), requires_grad=True)
b = Variable(torch.Tensor([3]), requires_grad=True)
# requires_grad = False를 하면 backward() 학습이 안됨

In [13]:
x


Out[13]:
Variable containing:
 1
[torch.FloatTensor of size 1]

In [14]:
y = w * x + b

In [15]:
y.backward()

In [16]:
print(x.grad)


Variable containing:
 2
[torch.FloatTensor of size 1]


In [17]:
print(w.grad)


Variable containing:
 1
[torch.FloatTensor of size 1]


In [18]:
print(b.grad)


Variable containing:
 1
[torch.FloatTensor of size 1]


In [19]:
x = Variable(torch.randn(5, 3))
y = Variable(torch.randn(5, 2))

In [20]:
linear = nn.Linear(3,2)

In [22]:
print ('w: ', linear.weight)
print ('b: ', linear.bias)


w:  Parameter containing:
-0.2523  0.3170  0.5090
 0.4296 -0.1992 -0.4302
[torch.FloatTensor of size 2x3]

b:  Parameter containing:
-0.0321
-0.3565
[torch.FloatTensor of size 2]


In [23]:
criterion = nn.MSELoss()
optimizer = torch.optim.SGD(linear.parameters(), lr=0.01)

In [26]:
pred = linear(x)

In [27]:
loss = criterion(pred, y)
print('loss: ', loss.data[0])


loss:  2.4102718830108643

In [29]:
loss.backward()

In [30]:
# Print out the gradients.
print ('dL/dw: ', linear.weight.grad) 
print ('dL/db: ', linear.bias.grad)


dL/dw:  Variable containing:
-0.6159 -0.1583  1.3894
 0.7038 -0.1572 -2.1980
[torch.FloatTensor of size 2x3]

dL/db:  Variable containing:
 0.3383
-0.9026
[torch.FloatTensor of size 2]


In [31]:
optimizer.step()

In [32]:
x


Out[32]:
Variable containing:
 1.7582 -0.4260 -0.0737
 1.1930 -0.5955 -0.7805
-0.6726  0.3044  2.2496
-0.0827 -0.1824  1.1814
-0.3367 -0.5155  2.3464
[torch.FloatTensor of size 5x3]

In [34]:
y


Out[34]:
Variable containing:
 0.3102  0.1493
-0.4325  0.6515
 1.5301  2.1718
-0.2216 -0.8772
-1.4500 -0.4022
[torch.FloatTensor of size 5x2]

In [33]:
pred = linear(x)
loss = criterion(pred, y)
print('loss after 1 step optimization: ', loss.data[0])


loss after 1 step optimization:  2.325436592102051

In [38]:
%%time
# Download and construct dataset.
train_dataset = dsets.CIFAR10(root='../data/',
                               train=True, 
                               transform=transforms.ToTensor(),
                               download=True)


Downloading http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ../data/cifar-10-python.tar.gz
CPU times: user 3.69 s, sys: 2.02 s, total: 5.71 s
Wall time: 51.6 s

In [39]:
image, label = train_dataset[0]

In [43]:
print(image.size())


torch.Size([3, 32, 32])

In [44]:
print(label)


6

In [49]:
# torch.utils.data.Dataloader
# Data loader. Combines a dataset and a sampler, 
# and provides single- or multi-process iterators over the dataset.

# dataset (Dataset) – dataset from which to load the data.
# batch_size (int, optional) – how many samples per batch to load (default: 1).
# shuffle (bool, optional) – set to True to have the data reshuffled at every epoch (default: False).
# sampler (Sampler, optional) – defines the strategy to draw samples from the dataset. If specified, shuffle must be False.
# batch_sampler (Sampler, optional) – like sampler, but returns a batch of indices at a time. Mutually exclusive with batch_size, shuffle, sampler, and drop_last.
# num_workers (int, optional) – how many subprocesses to use for data loading. 0 means that the data will be loaded in the main process (default: 0)
# collate_fn (callable, optional) – merges a list of samples to form a mini-batch.
# pin_memory (bool, optional) – If True, the data loader will copy tensors into CUDA pinned memory before returning them.
# drop_last (bool, optional) – set to True to drop the last incomplete batch, if the dataset size is not divisible by the batch size. 
# If False and the size of dataset is not divisible by the batch size, then the last batch will be smaller. (default: False)

In [46]:
train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
                                           batch_size=100, 
                                           shuffle=True,
                                           num_workers=2)

In [50]:
# When iteration starts, queue and thread start to load dataset from files.
data_iter = iter(train_loader)

In [53]:
images, labels = data_iter.next()

In [55]:
for images, labels in train_loader:
    pass 
    # training code will be wirtten here

In [56]:
#===================== Input pipline for custom dataset =====================#
# You should build custom dataset as below.
class CustomDataset(data.Dataset):
    def __init__(self):
        # TODO
        # 1. Initialize file path or list of file names. 
        pass
    def __getitem__(self, index):
        # TODO
        # 1. Read one data from file (e.g. using numpy.fromfile, PIL.Image.open).
        # 2. Preprocess the data (e.g. torchvision.Transform).
        # 3. Return a data pair (e.g. image and label).
        pass
    def __len__(self):
        # You should change 0 to the total size of your dataset.
        return 0

In [57]:
# Then, you can just use prebuilt torch's data loader. 
custom_dataset = CustomDataset()
train_loader = torch.utils.data.DataLoader(dataset=custom_dataset,
                                           batch_size=100, 
                                           shuffle=True,
                                           num_workers=2)

In [ ]:
#========================== Using pretrained model ==========================#
# Download and load pretrained resnet.
resnet = torchvision.models.resnet18(pretrained=True)

# If you want to finetune only top layer of the model.
for param in resnet.parameters():
    param.requires_grad = False
    
# Replace top layer for finetuning.
resnet.fc = nn.Linear(resnet.fc.in_features, 100)  # 100 is for example.

# For test.
images = Variable(torch.randn(10, 3, 256, 256))
outputs = resnet(images)
print (outputs.size())   # (10, 100)

In [ ]:
#============================ Save and load the model ============================#
# Save and load the entire model.
torch.save(resnet, 'model.pkl')
model = torch.load('model.pkl')

# Save and load only the model parameters(recommended).
torch.save(resnet.state_dict(), 'params.pkl')
resnet.load_state_dict(torch.load('params.pkl'))