Assignment 4c

Multi-class classification

Using regularised logistic regression to recognize handwritten digits (from 0 to 9).



In [1]:

    
import numpy as np
import scipy.io
data = scipy.io.loadmat("ex3data1.mat")
(m, n) = np.shape(data['X'])
n = n + 1
k = 10
X = np.ones((m, n))
X[:, 1:] = np.array(data['X'])
y = np.array(data['y'])

y_multi is a vector of labels from 1 to 10, where the digit “0” is mapped to the label 10 (to avoid confusionswith indexing)



In [2]:

    
y_multi = np.zeros((m, k))
for i in range(m):
    if (y[i, 0] < 10):
        y_multi[i, y[i, 0]] = 1
    else:
        y_multi[i, 0] = 1

Sigmoid function



In [3]:

    
def sigmoid(z):
    neg_z = np.negative(z)
    exp_z = np.exp(neg_z)
    den = np.add(1, exp_z)
    return np.reciprocal(den)

Cost function



In [4]:

    
def cost():
    z = np.dot(X, theta)
    h = sigmoid(z)
    # if sigmoid function gives error uncomment the 2 statements below and comment the statement above
    #from scipy.special import expit
    #h = expit(z)
    t1 = -1 * y * np.log(h)
    t2 = (y - 1) * np.log(1 - h)
    sum_ = np.sum(t1 + t2)
    sq_theta = theta[1:, :] * theta[1:, :]
    sum_sq_theta = np.sum(sq_theta, axis = 0)
    J = (sum_ + (lambda_ * sum_sq_theta)/2) / m 
    return J

Gradient function



In [5]:

    
def dif_cost():
    z = np.dot(X, theta)
    h = sigmoid(z)
    # if sigmoid function gives error uncomment the 2 statements below and comment the statement above
    #from scipy.special import expit
    #h_temp = expit(z)
    trans_X = np.transpose(X)
    dJ = np.dot(trans_X, (h - y_multi)) / m
    temp = lambda_ * theta[1:, :] / m
    
    dJ[1:, :] = dJ[1:, :] + temp
    return dJ

Initialize theta values and lambda



In [6]:

    
theta = np.zeros((n, k))
lambda_ = 0.1

Training



In [7]:

    
no_iter = 3000
alpha = 3
for i in range(1, no_iter):
    grad = dif_cost()
    theta = theta - alpha * grad

Predicting



In [8]:

    
z_pred = np.dot(X, theta)
h = sigmoid(z_pred)
# if sigmoid function gives error uncomment the 2 statements below and comment the statement above
#from scipy.special import expit
#h = expit(z_pred)
pred = np.argmax(h, axis = 1)
for i in range(m):
    if pred[i] == 0:
        pred[i] = 10
pred = np.reshape(pred, (m, 1))

Accuracy



In [10]:

    
count = 0
for i in range(m):
    if y[i, 0] == pred[i, 0]:
        count = count + 1
accuracy = count * 100 / m
print("Accuracy: {0:.2f} %".format(accuracy))









    



Accuracy: 95.20 %



In [ ]: