

In [3]:

    
# Useful starting lines
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
%load_ext autoreload
%autoreload 2

Logistic Regression

Classification Using Linear Regression

Load your data.



In [9]:

    
from helpers import sample_data, load_data, standardize

# load data.
height, weight, gender = load_data()

# build sampled x and y.
seed = 1
y = np.expand_dims(gender, axis=1)
X = np.c_[height.reshape(-1), weight.reshape(-1)]
y, X = sample_data(y, X, seed, size_samples=200)
x, mean_x, std_x = standardize(X)



In [ ]:

Use least_squares to compute w, and visualize the results.



In [10]:

    
from least_squares import least_squares
from plots import visualization

def least_square_classification_demo(y, x):
    # ***************************************************
    # INSERT YOUR CODE HERE
    # classify the data by linear regression: TODO
    # ***************************************************
    tx = np.c_[np.ones((y.shape[0], 1)), x]
    # w = least squares with respect to tx
    w = least_squares(y,tx)
    
    visualization(y, x, mean_x, std_x, w, "classification_by_least_square")
    
least_square_classification_demo(y, x)

Logistic Regression

Compute your cost by negative log likelihood.



In [11]:

    
def sigmoid(t):
    """apply sigmoid function on t."""
    # ***************************************************
    # INSERT YOUR CODE HERE
    # ?? Sigmoid or Logistic function
    # ***************************************************
    return 1/(1+np.exp(-t))



In [12]:

    
def calculate_loss(y, tx, w):
    """compute the cost by negative log likelihood."""
    # ***************************************************
    # INSERT YOUR CODE HERE
    # 
    # ***************************************************
    loss = 0
    for index in range(len(tx)):
        e = np.dot(np.transpose(tx[index,:]), w)
        loss += np.log(1 + np.exp(e)) - y[index]*e
    return loss



In [15]:

    
def calculate_gradient(y, tx, w):
    """compute the gradient of loss."""
    # ***************************************************
    # INSERT YOUR CODE HERE
    # TODO
    # ***************************************************
    return np.dot(tx.T, sigmoid(np.dot(tx, w)) - y)

Using Gradient Descent

Implement your function to calculate the gradient for logistic regression.



In [16]:

    
def learning_by_gradient_descent(y, tx, w, alpha):
    """
    Do one step of gradient descen using logistic regression.
    Return the loss and the updated w.
    """
    # ***************************************************
    # INSERT YOUR CODE HERE
    # compute the cost: TODO
    # ***************************************************
    loss = calculate_loss(y, tx, w)
    # ***************************************************
    # INSERT YOUR CODE HERE
    # compute the gradient: TODO
    # ***************************************************
    grad = calculate_gradient(y, tx, w)
    # ***************************************************
    # INSERT YOUR CODE HERE
    # update w: TODO
    # ***************************************************
    w = w - alpha * grad
    return loss, w

Demo!



In [18]:

    
from helpers import de_standardize

def logistic_regression_gradient_descent_demo(y, x):
    # init parameters
    max_iter = 10000
    threshold = 1e-8
    alpha = 0.001
    losses = []

    # build tx
    tx = np.c_[np.ones((y.shape[0], 1)), x]
    w = np.zeros((tx.shape[1], 1))

    # start the logistic regression
    for iter in range(max_iter):
        # get loss and update w.
        loss, w = learning_by_gradient_descent(y, tx, w, alpha)
        # log info
        if iter % 1000 == 0:
            print("Current iteration={i}, the loss={l}".format(i=iter, l=loss))
        # converge criteria
        losses.append(loss)
        if len(losses) > 1 and np.abs(losses[-1] - losses[-2]) < threshold:
            break
    # visualization
    visualization(y, x, mean_x, std_x, w, "classification_by_logistic_regression_gradient_descent")
    print("The loss={l}".format(l=calculate_loss(y, tx, w)))

logistic_regression_gradient_descent_demo(y, x)









    



Current iteration=0, the loss=[ 138.62943611]
Current iteration=1000, the loss=[ 43.49258275]
Current iteration=2000, the loss=[ 41.55453652]
Current iteration=3000, the loss=[ 41.10170781]
Current iteration=4000, the loss=[ 40.96605044]
Current iteration=5000, the loss=[ 40.92056156]
Current iteration=6000, the loss=[ 40.90438018]
Current iteration=7000, the loss=[ 40.89842938]
Current iteration=8000, the loss=[ 40.89619782]
Current iteration=9000, the loss=[ 40.89535113]
The loss=[ 40.89502758]

Calculate your hessian below



In [30]:

    
def calculate_hessian(y, tx, w):
    """return the hessian of the loss function."""
    # ***************************************************
    # INSERT YOUR CODE HERE
    # calculate hessian: TODO
    # ***************************************************
    S = np.zeros((len(tx),len(tx)))
    for i in range(len(tx)):
        S[i,i] = sigmoid(np.dot(np.transpose(tx[i,:]),w))*(1-sigmoid(np.dot(np.transpose(tx[i,:]),w))) 
    H = np.dot(tx.T, np.dot(S, tx))
    return H

Write a function below to return loss, gradient, and hessian.



In [28]:

    
def logistic_regression(y, tx, w):
    """return the loss, gradient, and hessian."""
    # ***************************************************
    # INSERT YOUR CODE HERE
    # return loss, gradient, and hessian: TODO
    # ***************************************************
    return calculate_loss(y, tx, w), calculate_gradient(y, tx, w), calculate_hessian(y, tx, w)

Using Newton method

Use Newton method for logistic regression.



In [26]:

    
def learning_by_newton_method(y, tx, w, alpha):
    """
    Do one step on Newton's method.
    return the loss and updated w.
    """
    # ***************************************************
    # INSERT YOUR CODE HERE
    # return loss, gradient and hessian: TODO
    # ***************************************************
    loss, grad, hess = logistic_regression(y, tx, w)
    # ***************************************************
    # INSERT YOUR CODE HERE
    # update w: TODO
    # ***************************************************
    w = w - alpha* np.dot(np.linalg.inv(hess), grad)
    return loss, w

demo



In [31]:

    
def logistic_regression_newton_method_demo(y, x):
    # init parameters
    max_iter = 10000
    alpha = 0.01
    threshold = 1e-8
    lambda_ = 0.1
    losses = []

    # build tx
    tx = np.c_[np.ones((y.shape[0], 1)), x]
    w = np.zeros((tx.shape[1], 1))

    # start the logistic regression
    for iter in range(max_iter):
        # get loss and update w.
        loss, w = learning_by_newton_method(y, tx, w, alpha)
        # log info
        if iter % 500 == 0:
            print("Current iteration={i}, the loss={l}".format(i=iter, l=loss))
        # converge criteria
        losses.append(loss)
        if len(losses) > 1 and np.abs(losses[-1] - losses[-2]) < threshold:
            break
    # visualization
    visualization(y, x, mean_x, std_x, w, "classification_by_logistic_regression_newton_method")
    print("The loss={l}".format(l=calculate_loss(y, tx, w)))

logistic_regression_newton_method_demo(y, x)









    



Current iteration=0, the loss=[-138.62943611]
Current iteration=500, the loss=[-40.96438279]
Current iteration=1000, the loss=[-40.89482895]
The loss=[-40.89482609]

Using penalized logistic regression

Fill in the function below.



In [26]:

    
def penalized_logistic_regression(y, tx, w, lambda_):
    """return the loss, gradient, and hessian."""
    # ***************************************************
    # INSERT YOUR CODE HERE
    # return loss, gradient: TODO
    # ***************************************************
    loss = calculate_loss(y,tx,w) # origa
    grad = calculate_gradient(y, tx, w) + 2 * lambda_ * w
    #     hess = calculate_hessian(y, tx, w) + 2 * lambda_
    return loss, grad



In [27]:

    
def learning_by_penalized_gradient(y, tx, w, alpha, lambda_):
    """
    Do one step of gradient descent, using the penalized logistic regression.
    Return the loss and updated w.
    """
    # ***************************************************
    # INSERT YOUR CODE HERE
    # return loss, gradient and hessian: TODO
    # ***************************************************
    loss, grad = penalized_logistic_regression(y, tx, w, lambda_)
    # ***************************************************
    # INSERT YOUR CODE HERE
    # update w: TODO
    # ***************************************************
    w = w - alpha* grad
    return loss, w



In [29]:

    
def logistic_regression_penalized_gradient_descent_demo(y, x):
    # init parameters
    max_iter = 10000
    alpha = 0.01
    lambda_ = 0.1
    threshold = 1e-8
    losses = []

    # build tx
    tx = np.c_[np.ones((y.shape[0], 1)), x]
    w = np.zeros((tx.shape[1], 1))

    # start the logistic regression
    for iter in range(max_iter):
        # get loss and update w.
        loss, w = learning_by_penalized_gradient(y, tx, w, alpha, lambda_)
        # log info
        if iter % 500 == 0:
            print("Current iteration={i}, the loss={l}".format(i=iter, l=loss))
        # converge criteria
        losses.append(loss)
        if len(losses) > 1 and np.abs(losses[-1] - losses[-2]) < threshold:
            break
    # visualization
    visualization(y, x, mean_x, std_x, w, "classification_by_logistic_regression_penalized_gradient_descent")
    print("The loss={l}".format(l=calculate_loss(y, tx, w)))
    pred = sigmoid(np.dot(tx, w))
    pred[np.where(pred <= 0.5)] = 0
    pred[np.where(pred > 0.5)] = 1
    mis = np.count_nonzero(y - pred)
    print("Mis rate = {}".format(mis/len(y)))
logistic_regression_penalized_gradient_descent_demo(y, x)









    



Current iteration=0, the loss=[ 138.62943611]
Current iteration=500, the loss=[ 41.83544115]
Current iteration=1000, the loss=[ 41.80501754]
Current iteration=1500, the loss=[ 41.80475602]
The loss=[ 41.80475476]
Mis rate = 0.08



In [ ]: