In [36]:

    

%matplotlib inline
import numpy as np
import pandas as pd

import matplotlib.pyplot as plt
import seaborn as sns
sns.set_context("poster")
plt.style.use("fivethirtyeight")

import statsmodels.api as sm
from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score


    

In [5]:

    

#np.random.seed(1337)
np.random.seed()


    

In [18]:

    

N = 100

X = np.hstack((np.random.randn(N,1), np.random.randn(N,1)*100, np.random.randn(N,1)*2))
W = np.array([1.8, 0.5,-0.3])
bias = 1.4

y = np.dot(X,W) + bias


    

In [22]:

    

plt.plot(X[:,0], y, "o", label="X0")
plt.plot(X[:,1], y, "o", label="X1")
plt.plot(X[:,2], y, "o", label="X2")
plt.xlabel("X")
plt.ylabel("y")
plt.legend()


    

    
Out[22]:





<matplotlib.legend.Legend at 0x1bd64710>






    

In [23]:

    

# Standardise all variables
X_standard = (X - X.mean(axis=0))/X.std(axis=0)
y_standard = (y - y.mean(axis=0))/y.std(axis=0)


    

In [24]:

    

plt.plot(X_standard[:,0], y_standard, "o", label="X0")
plt.plot(X_standard[:,1], y_standard, "o", label="X1")
plt.plot(X_standard[:,2], y_standard, "o", label="X2")
plt.xlabel("X [standardised]")
plt.ylabel("y [standardised]")
plt.legend()


    

    
Out[24]:





<matplotlib.legend.Legend at 0x1c723780>






    

In [34]:

    

plt.plot(X_standard[:,0], y, "o", label="X0")
plt.plot(X_standard[:,1], y, "o", label="X1")
plt.plot(X_standard[:,2], y, "o", label="X2")
plt.xlabel("X [standardised]")
plt.ylabel("y [standardised]")
plt.legend()


    

    
Out[34]:





<matplotlib.legend.Legend at 0x1eeeceb8>






    

In [29]:

    

model = OLS(y, sm.add_constant(X))
res = model.fit()
print res.summary()


    

    




                            OLS Regression Results                            
==============================================================================
Dep. Variable:                      y   R-squared:                       1.000
Model:                            OLS   Adj. R-squared:                  1.000
Method:                 Least Squares   F-statistic:                 1.259e+32
Date:                Fri, 26 Feb 2016   Prob (F-statistic):               0.00
Time:                        00:59:14   Log-Likelihood:                 3000.4
No. Observations:                 100   AIC:                            -5993.
Df Residuals:                      96   BIC:                            -5982.
Df Model:                           3                                         
Covariance Type:            nonrobust                                         
==============================================================================
                 coef    std err          t      P>|t|      [95.0% Conf. Int.]
------------------------------------------------------------------------------
const          1.4000    2.4e-15   5.84e+14      0.000         1.400     1.400
x1             1.8000   2.15e-15   8.37e+14      0.000         1.800     1.800
x2             0.5000   2.59e-17   1.93e+16      0.000         0.500     0.500
x3            -0.3000   1.32e-15  -2.27e+14      0.000        -0.300    -0.300
==============================================================================
Omnibus:                        4.206   Durbin-Watson:                   1.849
Prob(Omnibus):                  0.122   Jarque-Bera (JB):                3.654
Skew:                           0.455   Prob(JB):                        0.161
Kurtosis:                       3.218   Cond. No.                         97.9
==============================================================================

Warnings:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.

In [30]:

    

model = OLS(y_standard, sm.add_constant(X_standard))
res = model.fit()
print res.summary()


    

    




                            OLS Regression Results                            
==============================================================================
Dep. Variable:                      y   R-squared:                       1.000
Model:                            OLS   Adj. R-squared:                  1.000
Method:                 Least Squares   F-statistic:                 3.284e+31
Date:                Fri, 26 Feb 2016   Prob (F-statistic):               0.00
Time:                        00:59:49   Log-Likelihood:                 3313.3
No. Observations:                 100   AIC:                            -6619.
Df Residuals:                      96   BIC:                            -6608.
Df Model:                           3                                         
Covariance Type:            nonrobust                                         
==============================================================================
                 coef    std err          t      P>|t|      [95.0% Conf. Int.]
------------------------------------------------------------------------------
const               0   1.01e-16          0      1.000        -2e-16     2e-16
x1             0.0445   1.04e-16   4.28e+14      0.000         0.044     0.044
x2             1.0010   1.01e-16   9.87e+15      0.000         1.001     1.001
x3            -0.0121   1.04e-16  -1.16e+14      0.000        -0.012    -0.012
==============================================================================
Omnibus:                        0.222   Durbin-Watson:                   1.996
Prob(Omnibus):                  0.895   Jarque-Bera (JB):                0.370
Skew:                           0.091   Prob(JB):                        0.831
Kurtosis:                       2.764   Cond. No.                         1.31
==============================================================================

Warnings:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.

In [33]:

    

model = OLS(y, sm.add_constant(X_standard))
res = model.fit_regularized(L1_wt=0, alpha=0.1) # Ridge regression
print res.summary()


    

    




                            OLS Regression Results                            
==============================================================================
Dep. Variable:                      y   R-squared:                       0.991
Model:                            OLS   Adj. R-squared:                  0.991
Method:                 Least Squares   F-statistic:                     3635.
Date:                Fri, 26 Feb 2016   Prob (F-statistic):           1.14e-98
Time:                        01:35:26   Log-Likelihood:                -284.87
No. Observations:                 100   AIC:                             577.7
Df Residuals:                      96   BIC:                             588.2
Df Model:                           3                                         
Covariance Type:            nonrobust                                         
==============================================================================
                 coef    std err          t      P>|t|      [95.0% Conf. Int.]
------------------------------------------------------------------------------
const         -8.1647      0.426    -19.149      0.000        -9.011    -7.318
x1             1.5976      0.440      3.629      0.000         0.724     2.472
x2            40.6441      0.429     94.689      0.000        39.792    41.496
x3            -0.7987      0.442     -1.809      0.074        -1.675     0.078
==============================================================================
Omnibus:                        1.441   Durbin-Watson:                   1.827
Prob(Omnibus):                  0.486   Jarque-Bera (JB):                1.442
Skew:                          -0.205   Prob(JB):                        0.486
Kurtosis:                       2.579   Cond. No.                         1.31
==============================================================================

Warnings:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.

In [35]:

    

model = OLS(y, sm.add_constant(X_standard))
res = model.fit_regularized(L1_wt=1, alpha=0.1) # Ridge regression
print res.summary()


    

    




                            OLS Regression Results                            
==============================================================================
Dep. Variable:                      y   R-squared:                       1.000
Model:                            OLS   Adj. R-squared:                  1.000
Method:                 Least Squares   F-statistic:                 1.399e+06
Date:                Fri, 26 Feb 2016   Prob (F-statistic):          1.40e-222
Time:                        01:37:02   Log-Likelihood:                 12.336
No. Observations:                 100   AIC:                            -16.67
Df Residuals:                      96   BIC:                            -6.251
Df Model:                           3                                         
Covariance Type:            nonrobust                                         
==============================================================================
                 coef    std err          t      P>|t|      [95.0% Conf. Int.]
------------------------------------------------------------------------------
const         -8.8812      0.022   -406.836      0.000        -8.925    -8.838
x1             1.8518      0.023     82.148      0.000         1.807     1.897
x2            44.6669      0.022   2032.470      0.000        44.623    44.710
x3            -0.4175      0.023    -18.465      0.000        -0.462    -0.373
==============================================================================
Omnibus:                        0.641   Durbin-Watson:                   1.707
Prob(Omnibus):                  0.726   Jarque-Bera (JB):                0.762
Skew:                           0.099   Prob(JB):                        0.683
Kurtosis:                       2.621   Cond. No.                         1.31
==============================================================================

Warnings:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.

In [ ]: