

In [9]:

    
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt

Linear models for regression

y_pred = x_test[0] * coef_[0] + ... + x_test[n_features-1] * coef_[n_features-1] + intercept_



In [2]:

    
from sklearn.datasets import make_regression
from sklearn.cross_validation import train_test_split

X, y, true_coefficient = make_regression(n_samples=80, n_features=30, n_informative=10, noise=100, coef=True, random_state=5)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=5)
print(X_train.shape)
print(y_train.shape)









    



(60, 30)
(60,)



In [20]:

    
true_coefficient









    Out[20]:





array([ 57.32006912,  33.54836005,  53.9121895 ,   0.        ,
         0.        ,  39.72073131,  62.98303766,   0.        ,
         0.        ,   0.        ,  33.08672508,   0.        ,
         0.        ,   0.        ,  79.17546554,   0.        ,
         0.        ,   0.        ,   0.        ,   0.        ,
        60.4519575 ,   0.        ,   0.        ,  61.85805104,
         0.        ,  79.55545187,   0.        ,   0.        ,
         0.        ,   0.        ])



In [23]:

    
X.shape, y.shape









    Out[23]:





((80, 30), (80,))



In [26]:

    
plt.plot(X[:,1], y, 'bo', markersize=4);

Linear Regression



In [3]:

    
from sklearn.linear_model import LinearRegression
linear_regression = LinearRegression().fit(X_train, y_train)
print("R^2 on training set: %f" % linear_regression.score(X_train, y_train))
print("R^2 on test set: %f" % linear_regression.score(X_test, y_test))









    



R^2 on training set: 0.877842
R^2 on test set: 0.492723



In [4]:

    
from sklearn.metrics import r2_score
print(r2_score(np.dot(X, true_coefficient), y))









    



0.729977472078



In [5]:

    
plt.figure(figsize=(10, 5))
coefficient_sorting = np.argsort(true_coefficient)[::-1]
plt.plot(true_coefficient[coefficient_sorting], "o", label="true")
plt.plot(linear_regression.coef_[coefficient_sorting], "o", label="linear regression")

plt.legend()









    














    











    Out[5]:





<matplotlib.legend.Legend at 0x10831f390>

Ridge Regression (L2 penalty)



In [6]:

    
from sklearn.linear_model import Ridge
ridge_models = {}
training_scores = []
test_scores = []

for alpha in [100, 10, 1, .01]:
    ridge = Ridge(alpha=alpha).fit(X_train, y_train)
    training_scores.append(ridge.score(X_train, y_train))
    test_scores.append(ridge.score(X_test, y_test))
    ridge_models[alpha] = ridge

plt.plot(training_scores, label="training scores")
plt.plot(test_scores, label="test scores")
plt.xticks(range(4), [100, 10, 1, .01])
plt.legend(loc="best")









    Out[6]:





<matplotlib.legend.Legend at 0x1065870d0>



In [7]:

    
plt.figure(figsize=(10, 5))
plt.plot(true_coefficient[coefficient_sorting], "o", label="true", c='b')

for i, alpha in enumerate([100, 10, 1, .01]):
    plt.plot(ridge_models[alpha].coef_[coefficient_sorting], "o", label="alpha = %.2f" % alpha, c=plt.cm.summer(i / 3.))
    
plt.legend(loc="best")









    














    











    Out[7]:





<matplotlib.legend.Legend at 0x108824090>

Lasso (L1 penalty)



In [ ]:

    
from sklearn.linear_model import Lasso

lasso_models = {}
training_scores = []
test_scores = []

for alpha in [30, 10, 1, .01]:
    lasso = Lasso(alpha=alpha).fit(X_train, y_train)
    training_scores.append(lasso.score(X_train, y_train))
    test_scores.append(lasso.score(X_test, y_test))
    lasso_models[alpha] = lasso

plt.plot(training_scores, label="training scores")
plt.plot(test_scores, label="test scores")
plt.xticks(range(4), [30, 10, 1, .01])
plt.legend(loc="best")



In [ ]:

    
plt.figure(figsize=(10, 5))
plt.plot(true_coefficient[coefficient_sorting], "o", label="true", c='b')

for i, alpha in enumerate([30, 10, 1, .01]):
    plt.plot(lasso_models[alpha].coef_[coefficient_sorting], "o", label="alpha = %.2f" % alpha, c=plt.cm.summer(i / 3.))
    
plt.legend(loc="best")

Linear models for classification

y_pred = x_test[0] * coef_[0] + ... + x_test[n_features-1] * coef_[n_features-1] + intercept_ > 0

The influence of C in LinearSVC



In [ ]:

    
from plots import plot_linear_svc_regularization
plot_linear_svc_regularization()

Multi-Class linear classification



In [ ]:

    
from sklearn.datasets import make_blobs

X, y = make_blobs(random_state=42)
plt.scatter(X[:, 0], X[:, 1], c=y)



In [ ]:

    
from sklearn.svm import LinearSVC
linear_svm = LinearSVC().fit(X, y)
print(linear_svm.coef_.shape)
print(linear_svm.intercept_.shape)



In [ ]:

    
plt.scatter(X[:, 0], X[:, 1], c=y)
line = np.linspace(-15, 15)
for coef, intercept in zip(linear_svm.coef_, linear_svm.intercept_):
    plt.plot(line, -(line * coef[0] + intercept) / coef[1])
plt.ylim(-10, 15)
plt.xlim(-10, 8)

Exercises

Use GridSearchCV to tune the parameter C of LinearSVC on the digits dataset.
Compare l1 penalty and l2 penalty by plotting the coefficients as above for the digits dataset. Classify odd vs even digits to make it a binary task.



In [ ]:

    
# %load solutions/linear_models.py