In [1]:

    

# LARS - Least-Angle Regression
# LARS is a regression technique that is well suited for
# high-dimensional problems (p >> n) where p is the columns of
# features and n is the number of samples.


    

In [2]:

    

# use 200 data points and 500 features with low noise and
# a small number of informative features.


    

In [3]:

    

from sklearn.datasets import make_regression
reg_data, reg_target = make_regression(n_samples=200,
                                       n_features=500,
                                      n_informative=10,
                                      noise=2)


    

In [4]:

    

reg_data[:5]


    

    
Out[4]:





array([[ 2.81412941,  0.21159746, -0.32581413, ..., -0.06106478,
        -0.18579926, -0.51674261],
       [-0.99460396,  0.92340583,  0.43784461, ...,  1.61522657,
        -0.80512506, -1.10938947],
       [ 0.65629696, -0.49901232,  0.11073364, ..., -0.59480377,
         0.43564802, -0.80762681],
       [ 1.02784064, -1.50164649, -0.28185622, ...,  1.17514288,
        -0.27632164,  0.01769434],
       [ 0.66717096,  2.68264737, -0.09947634, ..., -0.61557118,
        -0.05875834, -1.00090044]])

In [5]:

    

# 10 informative features means we should try for 10 non-zero
# coefficients in LARS.


    

In [8]:

    

from sklearn.linear_model import Lars
import numpy as np
lars = Lars(n_nonzero_coefs=10)
lars.fit(reg_data, reg_target)


    

    
Out[8]:





Lars(copy_X=True, eps=2.2204460492503131e-16, fit_intercept=True,
   fit_path=True, n_nonzero_coefs=10, normalize=True, precompute='auto',
   verbose=False)

In [10]:

    

np.sum(lars.coef_ != 0) # make sure we got exactly 10


    

    
Out[10]:





10

In [11]:

    

# show why smaller number of features is usually better. Train
# 2 LARS models (one with 12 features, one with unlimited)


    

In [12]:

    

train_n = 100
lars_12 = Lars(n_nonzero_coefs=12)
lars_12.fit(reg_data[:train_n], reg_target[:train_n])


    

    
Out[12]:





Lars(copy_X=True, eps=2.2204460492503131e-16, fit_intercept=True,
   fit_path=True, n_nonzero_coefs=12, normalize=True, precompute='auto',
   verbose=False)

In [13]:

    

lars_500 = Lars()


    

In [14]:

    

lars_500.fit(reg_data[:train_n], reg_target[:train_n])


    

    




/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 131 iterations, i.e. alpha=3.324e-02, with an active set of 100 regressors, and the smallest cholesky pivot element being 4.712e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 131 iterations, i.e. alpha=3.193e-02, with an active set of 100 regressors, and the smallest cholesky pivot element being 7.224e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 131 iterations, i.e. alpha=3.150e-02, with an active set of 100 regressors, and the smallest cholesky pivot element being 6.664e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 131 iterations, i.e. alpha=3.089e-02, with an active set of 100 regressors, and the smallest cholesky pivot element being 2.220e-16
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 131 iterations, i.e. alpha=2.973e-02, with an active set of 100 regressors, and the smallest cholesky pivot element being 2.980e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 131 iterations, i.e. alpha=2.947e-02, with an active set of 100 regressors, and the smallest cholesky pivot element being 9.714e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 131 iterations, i.e. alpha=2.866e-02, with an active set of 100 regressors, and the smallest cholesky pivot element being 6.664e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 131 iterations, i.e. alpha=2.726e-02, with an active set of 100 regressors, and the smallest cholesky pivot element being 5.960e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 131 iterations, i.e. alpha=2.689e-02, with an active set of 100 regressors, and the smallest cholesky pivot element being 4.470e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 131 iterations, i.e. alpha=2.675e-02, with an active set of 100 regressors, and the smallest cholesky pivot element being 8.297e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 133 iterations, i.e. alpha=2.668e-02, with an active set of 101 regressors, and the smallest cholesky pivot element being 6.664e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 134 iterations, i.e. alpha=2.644e-02, with an active set of 102 regressors, and the smallest cholesky pivot element being 4.215e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 134 iterations, i.e. alpha=2.601e-02, with an active set of 102 regressors, and the smallest cholesky pivot element being 7.224e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 134 iterations, i.e. alpha=2.504e-02, with an active set of 102 regressors, and the smallest cholesky pivot element being 6.829e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 134 iterations, i.e. alpha=2.461e-02, with an active set of 102 regressors, and the smallest cholesky pivot element being 6.495e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 134 iterations, i.e. alpha=2.445e-02, with an active set of 102 regressors, and the smallest cholesky pivot element being 3.332e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 134 iterations, i.e. alpha=2.436e-02, with an active set of 102 regressors, and the smallest cholesky pivot element being 5.867e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 134 iterations, i.e. alpha=2.398e-02, with an active set of 102 regressors, and the smallest cholesky pivot element being 2.980e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 134 iterations, i.e. alpha=2.350e-02, with an active set of 102 regressors, and the smallest cholesky pivot element being 9.186e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 134 iterations, i.e. alpha=2.334e-02, with an active set of 102 regressors, and the smallest cholesky pivot element being 4.942e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 134 iterations, i.e. alpha=2.333e-02, with an active set of 102 regressors, and the smallest cholesky pivot element being 4.712e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 134 iterations, i.e. alpha=2.310e-02, with an active set of 102 regressors, and the smallest cholesky pivot element being 9.306e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 134 iterations, i.e. alpha=2.279e-02, with an active set of 102 regressors, and the smallest cholesky pivot element being 5.576e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 134 iterations, i.e. alpha=2.275e-02, with an active set of 102 regressors, and the smallest cholesky pivot element being 5.576e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 134 iterations, i.e. alpha=2.247e-02, with an active set of 102 regressors, and the smallest cholesky pivot element being 7.743e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 134 iterations, i.e. alpha=2.236e-02, with an active set of 102 regressors, and the smallest cholesky pivot element being 4.712e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 134 iterations, i.e. alpha=2.224e-02, with an active set of 102 regressors, and the smallest cholesky pivot element being 8.162e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 134 iterations, i.e. alpha=2.200e-02, with an active set of 102 regressors, and the smallest cholesky pivot element being 6.989e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 134 iterations, i.e. alpha=2.189e-02, with an active set of 102 regressors, and the smallest cholesky pivot element being 8.429e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 134 iterations, i.e. alpha=2.188e-02, with an active set of 102 regressors, and the smallest cholesky pivot element being 7.300e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 134 iterations, i.e. alpha=2.181e-02, with an active set of 102 regressors, and the smallest cholesky pivot element being 7.743e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 134 iterations, i.e. alpha=2.168e-02, with an active set of 102 regressors, and the smallest cholesky pivot element being 3.942e-08
  ConvergenceWarning)
/usr/local/lib/python2.7/site-packages/sklearn/linear_model/least_angle.py:285: ConvergenceWarning: Regressors in active set degenerate. Dropping a regressor, after 143 iterations, i.e. alpha=2.177e-02, with an active set of 109 regressors, and the smallest cholesky pivot element being 2.980e-08
  ConvergenceWarning)






    
Out[14]:





Lars(copy_X=True, eps=2.2204460492503131e-16, fit_intercept=True,
   fit_path=True, n_nonzero_coefs=500, normalize=True, precompute='auto',
   verbose=False)

In [16]:

    

# to see how each feature fits the unknown data:
lars_12_predictions = lars_12.predict(reg_data[train_n:])
np.mean(np.power(reg_target[train_n:] - lars_12_predictions, 2))


    

    
Out[16]:





57.420578953581156

In [17]:

    

lars_500_predictions = lars_500.predict(reg_data[train_n:])
np.mean(np.power(reg_target[train_n:] - lars_500_predictions, 2))


    

    
Out[17]:





3.0565357589130682e+32

In [18]:

    

# Perform Cross Validation on Lars to tune Ridge Regression


    

In [19]:

    

from sklearn.linear_model import LarsCV


    

In [20]:

    

lcv = LarsCV()
lcv.fit(reg_data, reg_target)


    

    
Out[20]:





LarsCV(copy_X=True, cv=None, eps=2.2204460492503131e-16, fit_intercept=True,
    max_iter=500, max_n_alphas=1000, n_jobs=1, normalize=True,
    precompute='auto', verbose=False)

In [21]:

    

np.sum(lcv.coef_ != 0)


    

    
Out[21]:





35

In [22]:

    

# test out this number to make sure it works better
# (This is my personal test)


    

In [61]:

    

for n in range(25, 100, 5):
    model = Lars(n_nonzero_coefs=n)
    model.fit(reg_data, reg_target)
    preds = model.predict(reg_data)
    mean = np.mean(np.power(preds - reg_target, 2))
    print "mean of {}, is {}".format(n, mean)


    

    




mean of 25, is 4.54693969485
mean of 30, is 4.17785635184
mean of 35, is 3.75310461702
mean of 40, is 3.52610962126
mean of 45, is 3.28265527156
mean of 50, is 3.11989033741
mean of 55, is 2.96234227537
mean of 60, is 2.62289884574
mean of 65, is 2.49724889759
mean of 70, is 2.34349165116
mean of 75, is 2.18933019887
mean of 80, is 2.01289072628
mean of 85, is 1.88027173393
mean of 90, is 1.77641313459
mean of 95, is 1.63698424804

In [62]:

    

# trying to do the above using a GridSearch


    

In [63]:

    

from sklearn.grid_search import GridSearchCV


    

In [64]:

    

n_nz_c = np.array(range(25, 100, 5))
grid = GridSearchCV(estimator=lars, param_grid=dict(n_nonzero_coefs=n_nz_c))


    

In [65]:

    

grid.fit(reg_data, reg_target)


    

    
Out[65]:





GridSearchCV(cv=None, error_score='raise',
       estimator=Lars(copy_X=True, eps=2.2204460492503131e-16, fit_intercept=True,
   fit_path=True, n_nonzero_coefs=10, normalize=True, precompute='auto',
   verbose=False),
       fit_params={}, iid=True, loss_func=None, n_jobs=1,
       param_grid={'n_nonzero_coefs': array([25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95])},
       pre_dispatch='2*n_jobs', refit=True, score_func=None, scoring=None,
       verbose=0)

In [66]:

    

grid.best_score_


    

    
Out[66]:





0.99957027083602668

In [67]:

    

grid.best_estimator_.n_nonzero_coefs


    

    
Out[67]:





60

In [ ]: