05_Classification_Models


In [42]:
import torch
from torch.autograd import Variable
import torch.nn as nn

torch.manual_seed(777)    # reproducible


Out[42]:
<torch._C.Generator at 0x7f71ddf9c258>

In [43]:
import matplotlib.pyplot as plt
import numpy as np
%matplotlib inline

Logistic Regression (classification)

Prepare Data


In [44]:
# make fake data
n_data = torch.ones(100, 2)
x0 = torch.normal(2*n_data, 1)      # class0 x data (tensor), shape=(100, 2)
y0 = torch.zeros(100,1)               # class0 y data (tensor), shape=(100, 1)
x1 = torch.normal(-2*n_data, 1)     # class1 x data (tensor), shape=(100, 2)
y1 = torch.ones(100,1)                # class1 y data (tensor), shape=(100, 1)

x = torch.cat((x0, x1), 0).type(torch.FloatTensor)  # shape (200, 2) FloatTensor = 32-bit floating
y = torch.cat((y0, y1), 0).type(torch.FloatTensor)    # shape (200, 1) FloatTensor = 32-bit integer

# torch can only train on Variable, so convert them to Variable
x, y = Variable(x), Variable(y)

plt.scatter(x.data.numpy()[:, 0], x.data.numpy()[:, 1], c=y.data.numpy(), s=100, lw=0, cmap='RdYlGn')
plt.show()

torch.cat( (x, y) ,1)


Out[44]:
Variable containing:
 2.8280  3.1807  0.0000
 1.7837  2.6335  0.0000
 3.3155  2.5404  0.0000
 1.2688  2.1884  0.0000
 2.6572  0.8927  0.0000
 3.2333  1.3910  0.0000
 1.7579  0.3371  0.0000
 2.2401 -0.0566  0.0000
 1.8807  2.9838  0.0000
 1.4295  1.2737  0.0000
 2.8247  2.5482  0.0000
 3.0635  2.5820  0.0000
 1.1642  1.4724  0.0000
 0.5313  4.2104  0.0000
 4.3313  1.8368  0.0000
 1.3000  1.2869  0.0000
 1.0423 -0.0957  0.0000
 0.2685  1.4101  0.0000
 1.9403  2.5014  0.0000
 0.7410  3.2869  0.0000
 2.4210  4.4513  0.0000
 2.2757  2.6533  0.0000
 1.0218  2.8025  0.0000
 2.2462  2.6598  0.0000
 1.4638  1.5512  0.0000
 2.2406  1.7819  0.0000
 1.7782  1.7855  0.0000
 1.9733  3.3625  0.0000
 3.1151  4.3671  0.0000
 2.1175  1.7349  0.0000
 2.3194  4.0476  0.0000
 1.7467  1.9731  0.0000
 0.1604  1.7852  0.0000
 0.5215  0.9205  0.0000
 1.5890  1.9146  0.0000
 3.3902  2.1165  0.0000
 0.1794  1.7222  0.0000
 3.7112  0.6878  0.0000
 2.1504  0.8037  0.0000
 2.8883  2.8646  0.0000
 1.5071  3.6625  0.0000
 1.3658  1.4616  0.0000
 1.6132 -0.5608  0.0000
 0.8405  2.2707  0.0000
 2.0456  1.6221  0.0000
 2.4841  3.2603  0.0000
-0.2256  1.1532  0.0000
 0.0517  1.3564  0.0000
 0.7207  2.2255  0.0000
 2.1987  1.2626  0.0000
 3.9445  3.7197  0.0000
 1.6449  1.2290  0.0000
 1.8194  2.2119  0.0000
 1.1625  3.2456  0.0000
 1.1862  2.2672  0.0000
 4.9856  2.5103  0.0000
 1.0096  2.5836  0.0000
 4.0409  0.9150  0.0000
 2.9818  2.6335  0.0000
 1.6373  2.8364  0.0000
-0.3399  1.5837  0.0000
 1.5198  2.8992  0.0000
 1.0785  2.3064  0.0000
 2.4507  2.0464  0.0000
 2.8576  2.5989  0.0000
 1.3893  0.3811  0.0000
 2.3089  1.4847  0.0000
 1.0757  2.4759  0.0000
 2.8395  0.8257  0.0000
 1.8270  1.9476  0.0000
 1.1118  3.1584  0.0000
 1.9669  1.7701  0.0000
 2.2229  2.1491  0.0000
 2.1431  1.8859  0.0000
 1.1243  3.4544  0.0000
 4.3492  0.1017  0.0000
 1.9128  2.2877  0.0000
 1.8253  2.5194  0.0000
 2.5254  0.5755  0.0000
 2.9546  0.4446  0.0000
 0.7607  0.6070  0.0000
 1.5568  3.0928  0.0000
 3.0021  2.1876  0.0000
 1.6491  2.2557  0.0000
 0.0814  1.7800  0.0000
 2.4298  2.0200  0.0000
 2.3256 -0.0601  0.0000
 2.7342  1.0469  0.0000
 3.2851  3.3952  0.0000
 1.4886  4.1198  0.0000
 1.3883  3.1715  0.0000
 3.6508  0.9016  0.0000
 4.1374  1.9941  0.0000
 2.1071  1.1884  0.0000
 3.0340  2.2853  0.0000
 2.8089  1.8205  0.0000
 3.1424  1.8206  0.0000
 2.1910  2.6210  0.0000
 1.8539  1.6902  0.0000
 1.3879  2.1354  0.0000
-0.2154 -2.9809  1.0000
-2.1478 -3.0730  1.0000
-1.9714 -0.9029  1.0000
-0.4601 -1.8100  1.0000
-2.3475 -3.2245  1.0000
-1.8073 -3.2430  1.0000
-3.2778 -1.7809  1.0000
-1.4725 -3.5739  1.0000
-1.4709 -0.9550  1.0000
-2.0891 -2.4696  1.0000
-2.8118 -1.1121  1.0000
-4.6717 -2.5462  1.0000
-2.4290 -0.5960  1.0000
-2.7349 -1.2585  1.0000
-2.2304 -3.6638  1.0000
-3.0015 -1.2579  1.0000
-0.9311 -1.1619  1.0000
-3.7328 -0.7173  1.0000
-3.9391 -2.2693  1.0000
-1.4400 -1.5599  1.0000
-1.5733 -2.5610  1.0000
-3.8313 -2.2126  1.0000
-2.8913 -2.7475  1.0000
-1.5113 -2.9004  1.0000
 0.1086 -1.7754  1.0000
-1.8299 -3.3265  1.0000
-1.1446 -4.2224  1.0000
-1.6689 -2.9562  1.0000
-1.2418 -1.3336  1.0000
-0.8662 -2.9611  1.0000
-1.9067 -1.9261  1.0000
-1.9151 -0.6755  1.0000
-1.8788 -2.3203  1.0000
-0.2862 -0.7219  1.0000
-1.3634 -1.6589  1.0000
-2.6153 -1.0999  1.0000
-1.3929 -3.0122  1.0000
-0.3384 -1.2742  1.0000
-3.8627 -4.1412  1.0000
-2.2222 -2.4057  1.0000
-1.2678 -4.5461  1.0000
-2.5492 -0.2902  1.0000
-1.9386 -3.2012  1.0000
-0.4709 -4.5657  1.0000
-3.0176 -0.9680  1.0000
-3.0051 -1.4134  1.0000
-2.1312 -3.3343  1.0000
-2.1028 -2.3670  1.0000
-2.1424 -3.5792  1.0000
-3.5099 -2.7453  1.0000
-0.7274 -2.4557  1.0000
-2.3275 -0.5706  1.0000
-2.8839 -3.2650  1.0000
-1.7526 -2.6131  1.0000
-3.7492 -0.8181  1.0000
-3.4503 -2.1247  1.0000
-1.1682 -0.9926  1.0000
-1.3224 -0.9724  1.0000
-0.8756 -1.7456  1.0000
-4.4364 -1.8745  1.0000
-2.5968 -3.3530  1.0000
-1.4705 -0.9410  1.0000
-1.7015 -3.2783  1.0000
-1.2424 -1.1743  1.0000
-2.4897 -2.3598  1.0000
-3.4455 -2.1669  1.0000
-1.3552 -1.9832  1.0000
-1.0999 -1.4452  1.0000
-2.9616 -2.1380  1.0000
-0.1773 -1.9475  1.0000
-0.9567 -1.8493  1.0000
 0.0385 -2.9499  1.0000
-1.7421 -1.8861  1.0000
-0.9884 -0.9987  1.0000
-1.5833 -2.7761  1.0000
-2.8559 -1.0668  1.0000
-0.3614 -1.4938  1.0000
-2.9812 -2.7365  1.0000
-2.5642 -1.4071  1.0000
-2.8602 -4.0407  1.0000
-2.0203 -0.9435  1.0000
-4.5271 -2.2945  1.0000
-3.2634 -2.6874  1.0000
-2.9822 -4.7266  1.0000
-3.0050 -2.4577  1.0000
-2.8703 -2.4509  1.0000
-0.9316 -2.1902  1.0000
-1.1560 -1.4174  1.0000
-1.2006 -0.8914  1.0000
-1.8827 -1.4075  1.0000
-1.6471 -1.1937  1.0000
-1.7783 -1.5697  1.0000
-0.6481 -2.7723  1.0000
-0.8525 -1.2944  1.0000
-1.5905 -3.2683  1.0000
-0.9073 -3.9452  1.0000
-2.3308 -0.2621  1.0000
-2.8332 -1.7054  1.0000
-1.8567 -3.2408  1.0000
-0.8982 -0.1920  1.0000
[torch.FloatTensor of size 200x3]

Define Logistic Model


In [55]:
# Hypothesis using sigmoid and linear model
linear = nn.Linear(2, 1, bias=True)
sigmoid = nn.Sigmoid()
model = nn.Sequential(linear, sigmoid)

In [56]:
model


Out[56]:
Sequential (
  (0): Linear (2 -> 1)
  (1): Sigmoid ()
)

Define Optimizer and Cost Function


In [46]:
# Loss and Optimizer
optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
#cost_fn = -(y * torch.log(prob) + (1 - y)* torch.log(1 - prob) ).mean() 
cost_fn = nn.BCELoss() #Binary Cross Entropy Costfunction

Train


In [47]:
plt.ion()   # something about plotting

for t in range(120):    
    prob = model(x)                 # input x and predict based on x
    cost = cost_fn(prob, y)
    
    optimizer.zero_grad()         # clear gradients for next train
    cost.backward()               # compute gradients
    optimizer.step()              # apply gradients
    
    # ----------------------------------------------------------------------#
    # Ploting
    if t % 10 == 0 or t in [3, 6]:
        # plot and show learning process
        plt.cla()
        prediction = prob.gt(0.5)
        pred_y = prediction.data.numpy().squeeze()
        target_y = y.data.squeeze(1).numpy()
        
        # Draw massgrid
        x_min, x_max = x[:, 0].min() - 1, x[:, 0].max() + 1
        y_min, y_max = x[:, 1].min() - 1, x[:, 1].max() + 1
        xx, yy = np.meshgrid(np.arange(x_min.data.numpy(), x_max.data.numpy(), 1),
                             np.arange(y_min.data.numpy(), y_max.data.numpy(), 1))

        # here "model" is your model's prediction (classification) function
        Z = model(Variable(torch.from_numpy(np.c_[xx.ravel(), yy.ravel()])).float())

        # Put the result into a color plot
        Z = Z.view(xx.shape)
        plt.contourf(xx, yy, Z.data.numpy(), cmap=plt.cm.binary)
        #plt.axis('off')

        # Plot also the training points
        #plt.scatter(x[:, 0].data.numpy(), x[:, 1].data.numpy(), c=pred_y, cmap='RdYlGn')      
        plt.scatter(x.data.numpy()[:, 0], x.data.numpy()[:, 1], c=pred_y, s=100, lw=0, cmap='RdYlGn')

        accuracy = sum(pred_y == target_y)/200.
        plt.text(1.5, -4, 'Accuracy=%.2f' % accuracy, fontdict={'size': 20, 'color':  'Blue'})
        plt.show()
        plt.pause(0.1)
        
plt.ioff()


Softmax Regression (Classification)

Prepare data


In [70]:
torch.manual_seed(777)  # for reproducibility
nb_classes = 3

x_data = [[1, 2, 1, 1], [2, 1, 3, 2], [3, 1, 3, 4], [4, 1, 5, 5],
          [1, 7, 5, 5], [1, 2, 5, 6], [1, 6, 6, 6], [1, 7, 7, 7]]
y_data = [[0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 1, 0],
          [0, 1, 0], [0, 1, 0], [1, 0, 0], [1, 0, 0]]

X = Variable(torch.Tensor(x_data))
Y = Variable(torch.Tensor(y_data))

_, Y_label = Y.max(dim=1)
print(Y_label)


Variable containing:
 2
 2
 2
 1
 1
 1
 0
 0
[torch.LongTensor of size 8]

Define Model


In [88]:
# Define our model
linear = torch.nn.Linear(4, nb_classes, bias=True)
softmax = torch.nn.Softmax() # softmax = exp(logits) / reduce_sum(exp(logits), dim)
model = torch.nn.Sequential(linear, softmax)

In [89]:
model


Out[89]:
Sequential (
  (0): Linear (4 -> 3)
  (1): Softmax ()
)

Define Optimizer and Cost function


In [103]:
optimizer = torch.optim.SGD(model.parameters(), lr=0.1)

#  cost_fn = - Y * torch.log(prediction)               # Cross entropy cost
#  cost_fn = torch.sum(cost, 1).mean()

cost_fn = nn.CrossEntropyLoss() #Cross Entropy Costfunction

In [104]:
prediction
Y_label


Out[104]:
Variable containing:
 2
 2
 2
 1
 1
 1
 0
 0
[torch.LongTensor of size 8]

Train


In [110]:
for step in range(4001):

    prediction = model(X)
    cost = cost_fn(prediction, Y_label)
    optimizer.zero_grad()

    cost.backward()                                 # Compute Gradient of Cost function
    optimizer.step()

    if step % 200 == 0:
        print(step, cost.data.numpy())


0 [ 0.8193391]
200 [ 0.81849271]
400 [ 0.81772035]
600 [ 0.81701303]
800 [ 0.81636286]
1000 [ 0.81576347]
1200 [ 0.81520915]
1400 [ 0.814695]
1600 [ 0.81421697]
1800 [ 0.81377149]
2000 [ 0.81335533]
2200 [ 0.81296587]
2400 [ 0.81260037]
2600 [ 0.81225693]
2800 [ 0.81193364]
3000 [ 0.8116287]
3200 [ 0.81134069]
3400 [ 0.81106818]
3600 [ 0.81081009]
3800 [ 0.81056511]
4000 [ 0.81033254]

Test


In [111]:
# Testing & One-hot encoding
print('--------------')
a = model(Variable(torch.Tensor([[1, 11, 7, 9]])))
print(a.data.numpy(), torch.max(a, 1)[1].data.numpy())

print('--------------')
b = model(Variable(torch.Tensor([[1, 3, 4, 3]])))
print(b.data.numpy(), torch.max(b, 1)[1].data.numpy())

print('--------------')
c = model(Variable(torch.Tensor([[1, 1, 0, 1]])))
print(c.data.numpy(), torch.max(c, 1)[1].data.numpy())

print('--------------')
all = model(Variable(torch.Tensor([[1, 11, 7, 9], [1, 3, 4, 3], [1, 1, 0, 1]])))
print(all.data.numpy(), torch.max(all, 1)[1].data.numpy())


--------------
[[  7.25515165e-16   9.99999940e-01   5.04304083e-08]] [1]
--------------
[[  3.36183552e-06   9.90428388e-01   9.56827868e-03]] [1]
--------------
[[  9.31831964e-05   5.33966727e-07   9.99906301e-01]] [2]
--------------
[[  7.25515165e-16   9.99999940e-01   5.04303124e-08]
 [  3.36183552e-06   9.90428388e-01   9.56827868e-03]
 [  9.31831964e-05   5.33966727e-07   9.99906301e-01]] [1 1 2]

In [ ]: