notebook.community

Edit and run 





In [1]:



    


%matplotlib inline
from __future__ import print_function
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

quiet = True # to disable unwanted output
#quiet = False
def log(*args):
    if not quiet:
        print(args)



    


















    






/home/tg/anaconda2/lib/python2.7/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.
  warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')

















In [ ]:



    






    











In [2]:



    


from sklearn.datasets import load_boston
boston = load_boston()
print('Feature Names', boston.feature_names)
dfX = boston.data
dfY = np.array([boston.target]).transpose()
assert dfX.shape[0] == dfY.shape[0]

n = dfX.shape[0]
test_split_rule = np.array(range(n)) % 7 == 0
train_split_rule = np.invert(test_split_rule)

trainX, trainY = dfX[train_split_rule], dfY[train_split_rule]
testX, testY = dfX[test_split_rule], dfY[test_split_rule]


print('All', dfX.shape, dfY.shape)
print('Test', testX.shape, testY.shape)
print('Train', trainX.shape, trainY.shape)



    


















    






Feature Names ['CRIM' 'ZN' 'INDUS' 'CHAS' 'NOX' 'RM' 'AGE' 'DIS' 'RAD' 'TAX' 'PTRATIO'
 'B' 'LSTAT']
All (506, 13) (506, 1)
Test (73, 13) (73, 1)
Train (433, 13) (433, 1)

















In [ ]:

1. Histograms and pearson correlation





In [61]:



    


n_attrs = trainX.shape[1]
attrs = [trainX[:, i] for i in range(n_attrs)]
stds = [a.std() for a in attrs]

corrs = np.zeros(shape=(n_attrs, n_attrs))
for i, a in enumerate(attrs):
    for j in range(0, i + 1):
        b = attrs[j]
        res = np.cov(a, b) / (stds[i] * stds[j])
        corrs[i][j] = res[0,1]

print("Correlations between the attributes")
print(pd.DataFrame(corrs))

plt.figure(2, figsize=(6,8))
plt.title("Heatmap of correlations between the attributes")
plt.imshow(corrs, cmap='hot', interpolation='nearest')
plt.savefig("correlation-heatmap.png")
plt.show()



def pearson_cor(X, Y, names):
    Y = Y[:, 0]
    assert X.shape[0] == Y.shape[0]
    assert X.shape[1] == len(names)
    y_std = Y.std()
    cor = []
    for i in range(X.shape[1]): # each column    
        attr = trainX[:, i]
        # Correlation between the attribs and target value
        cor_ab = np.cov(attr, Y) / (stds[i] * y_std)
        cor_ab = abs(cor_ab[0,1])
        cor.append((names[i], cor_ab))
    return cor
    
for i, cor_ab in enumerate(pearson_cor(trainX, trainY, boston.feature_names)):
    print(cor_ab)

print("Genrating Histograms")
plt.figure(1, figsize=(16, 16))

for i, attr in enumerate(attrs):
    plt.subplot(5, 3, 1 + i)
    plt.hist(attr, bins=bins)
    plt.title("Histogram of '%s' with %d bins" % (boston.feature_names[i], bins))

plt.subplot(5, 3, len(attrs) + 1)
plt.hist(trainY, bins=bins)
plt.title("Histogram of Target Price with %d bins" % bins)
plt.show()
plt.savefig("Histograms.png")



    


















    






Correlations between the attributes
          0         1         2         3         4         5         6   \
0   1.002315  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000   
1  -0.197276  1.002315  0.000000  0.000000  0.000000  0.000000  0.000000   
2   0.396280 -0.537783  1.002315  0.000000  0.000000  0.000000  0.000000   
3  -0.054158 -0.038522  0.050507  1.002315  0.000000  0.000000  0.000000   
4   0.410524 -0.525903  0.769347  0.065319  1.002315  0.000000  0.000000   
5  -0.206593  0.314229 -0.387829  0.066559 -0.308899  1.002315  0.000000   
6   0.351814 -0.586247  0.654674  0.081180  0.734113 -0.239191  1.002315   
7  -0.370380  0.672366 -0.713743 -0.097458 -0.771612  0.209400 -0.750088   
8   0.607248 -0.317635  0.597033 -0.021546  0.611365 -0.211677  0.470596   
9   0.567033 -0.318496  0.722513 -0.051311  0.668685 -0.288305  0.520563   
10  0.279365 -0.383817  0.376030 -0.145421  0.183605 -0.356928  0.264867   
11 -0.358765  0.180031 -0.360012  0.054895 -0.386504  0.134887 -0.284053   
12  0.456168 -0.422381  0.591894 -0.057371  0.591094 -0.597977  0.608152   

          7         8         9         10        11        12  
0   0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  
1   0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  
2   0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  
3   0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  
4   0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  
5   0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  
6   0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  
7   1.002315  0.000000  0.000000  0.000000  0.000000  0.000000  
8  -0.498835  1.002315  0.000000  0.000000  0.000000  0.000000  
9  -0.535925  0.913672  1.002315  0.000000  0.000000  0.000000  
10 -0.236001  0.463870  0.457994  1.002315  0.000000  0.000000  
11  0.296206 -0.452862 -0.449102 -0.177750  1.002315  0.000000  
12 -0.498400  0.497685  0.545339  0.374000 -0.391823  1.002315  










    

















    






('CRIM', 0.38859443435207547)
('ZN', 0.36382754420139768)
('INDUS', 0.48418563338201681)
('CHAS', 0.20407144132697344)
('NOX', 0.4258130776462164)
('RM', 0.69252269454509585)
('AGE', 0.39108230278590012)
('DIS', 0.25300487309124547)
('RAD', 0.38638415658601943)
('TAX', 0.46993468487617812)
('PTRATIO', 0.5064403651254078)
('B', 0.34422912357951185)
('LSTAT', 0.74168271373323336)
Genrating Histograms










    

















    








<matplotlib.figure.Figure at 0x7f2ad376dfd0>

2 Linear Regression





In [48]:



    






    











In [15]:



    


def predict(X, W):
    return np.matmul(X, W)

def MSECost(Y2, Y1):
    # Cost      = 1/N  SIGMA[(XW-Y)^2]
    return float(np.sum((Y2 - Y1) ** 2) / len(Y2))


def analytical_optimizer(X, Y):
    return np.matmul(
        np.matmul(
            np.linalg.pinv(np.matmul(X.transpose(), X)),
            X.transpose()),
        Y)

def gradient_desc(X, Y, W, alpha,
                  num_iter = 1000, conv_tol=0.01, print_interval = 500):
    c = float('inf')
    log("Learn Rate", alpha)
    for i in range(num_iter):
        # 
        # delta =  2/N SIGMA[(XW - Y)*x]
        predY = predict(X, W)
        diff = predY - Y
        delta = np.sum(np.multiply(X, diff), axis=0) # sum top to bottom for each attribute
        delta  = delta * 2.0 / len(Y)
        delta = np.array([delta]).transpose()        # restore vector shape of (n_attr x 1)
        
        W = (W - alpha * delta)
        if i % print_interval == 0:
            predY = predict(X, W)
            newcost = MSECost(predY, Y)
            log("#%d, cost = %.8g" % (i, newcost))
            if np.isnan(newcost) or np.isinf(newcost) or np.isneginf(newcost):
                raise Exception("ERROR: number overflow, please adjust learning rate")
            diff = abs(newcost - c)
            c = newcost
            if diff < conv_tol:
                log("Converged with tolerance %f " % conv_tol)
                break            
        if i % (print_interval * 10) == 0:
            log(W.flatten())
    return W


# compute means and stds
class LinearRegression(object):
    
    def __init__(self, X, Y, learn_rate=0.001, num_iter=10000, conv_tol=0.01, opt='analytical'):
        
        self.means = X.mean(axis=0)
        self.stds = X.std(axis=0)        
        X = self.normalize(X)
        self.n_attrs = X.shape[1]
        if opt == 'gradient_desc':
            W = np.random.rand(self.n_attrs, 1)
            self.W = gradient_desc(X, Y, W, alpha=learn_rate,
                                    num_iter=num_iter, conv_tol=conv_tol)
        elif opt == 'analytical':
            self.W = analytical_optimizer(X, Y)
        else:
            raise Exception('Unknown Optimizer %s' % opt)
    
    def normalize(self, X):
        X = (X - self.means) / self.stds
        # Bias is added as a weight to simplify the calculations
        X = np.insert(X, 0, 1, axis=1)
        return X
    
    def predict(self, X, normalize=True):
        if normalize:
            X = self.normalize(X)
        return np.matmul(X, self.W)
            
    
alpha = 0.001
conv_tol = 0.0001
num_iter = 50000
for opt_val in ["analytical", "gradient_desc"]:
    print("Using %s optimizer" % opt_val)
    linreg = LinearRegression(trainX, trainY, alpha, num_iter, conv_tol, opt=opt_val)
    print("W=", linreg.W.flatten())

    train_mse_cost = MSECost(linreg.predict(trainX), trainY)
    test_mse_cost = MSECost(linreg.predict(testX), testY)
    print('Train MSE::', train_mse_cost, '\tTest MSE::', test_mse_cost)



    


















    






Using analytical optimizer
W= [ 22.46351039  -0.9673241    1.04542152  -0.17351707   0.92139354
  -1.6205974    2.72573311  -0.26723668  -3.11240878   2.48363548
  -1.91078756  -1.88799963   0.82940209  -3.67532974]
Train MSE:: 20.950144508 	Test MSE:: 28.4179164975
Using gradient_desc optimizer
W= [ 22.46351039  -0.96304402   1.03690774  -0.20102114   0.92552579
  -1.61552751   2.730206    -0.27205352  -3.11536104   2.41554577
  -1.83310388  -1.88512184   0.82866899  -3.67292127]
Train MSE:: 20.950868436 	Test MSE:: 28.4554242596

Ridge Regression





In [26]:



    


def MSECost_ridge(X, W, Y, lambd):
    # Cost  = 1/N  \SIGMA[(XW-Y)^2] + lambd W||_2^2
    predY = predict(X, W)
    cost = float(np.sum((predY - Y) ** 2)) / len(Y)
    cost += lambd * np.sum((W) ** 2)
    return cost

def gradient_desc_ridge(X, Y, W, alpha, lambd,
                  num_iter = 1000, conv_tol=0.01, check_interval = 500):
    c = float('inf')
    log("Learn Rate", alpha)
    for i in range(num_iter):
        # 
        # delta =  2/N SIGMA[(XW - Y)*x] + 2 * \lambd * W
        diff = predict(X, W) - Y
        delta = np.sum(np.multiply(X, diff), axis=0) # sum top to bottom for each attribute
        delta = delta * 2.0 / len(Y)
        delta = np.array([delta]).transpose()        # restore vector shape of (n_attr x 1)
        delta  = delta + (2 * lambd * W)           # Vectors addition

        W = (W - alpha * delta)
        
        if i % check_interval == 0:
            predY = predict(X, W)
            #print(np.concatenate((Y, predY), axis=1))
            newcost = MSECost_ridge(X, W, Y, lambd)
            
            log("#%d, cost = %.8g" % (i, newcost))
            if np.isnan(newcost) or np.isinf(newcost) or np.isneginf(newcost):
                raise Exception("ERROR: number overflow, please adjust learning rate")
            diff = abs(newcost - c)
            c = newcost
            if diff < conv_tol:
                log("Converged with tolerance %f " % conv_tol)
                break            
        if not quiet and i % (check_interval * 10) == 0:
            print(W.flatten())
    return W


# compute means and stds
class RidgeRegression(LinearRegression):
    
    def __init__(self, X, Y, learn_rate=0.001, lambd=0.1, num_iter=1000, conv_tol=0.1):
        
        self.means = X.mean(axis=0)
        self.stds = X.std(axis=0)        
        X = self.normalize(X)
        self.n_attrs = X.shape[1]
        W = np.random.rand(self.n_attrs, 1)
        self.lambd = lambd
        self.W = gradient_desc_ridge(X, Y, W, alpha=learn_rate, lambd=lambd,
                                num_iter=num_iter, conv_tol=conv_tol)
    
    def find_cost(self, X, Y, lambd=None, normalize=True, regularized=False):
        if lambd is None:
            lambd = self.lambd
        if regularized:
            return MSECost_ridge(self.normalize(X), self.W, Y, lambd)
        else:
            return MSECost(self.predict(X, normalize=normalize), Y)
    
alpha = 0.001
conv_tol = 0.0001
num_iter = 100000
lambds = [0.01, 0.1, 1.0]
results = []
quiet = True
print("\n\nLambda\t\tTrain MSE\t\tTest MSE")
for lambd in lambds:
    ridreg = RidgeRegression(trainX, trainY, alpha, lambd, num_iter, conv_tol)
    train_mse = ridreg.find_cost(trainX, trainY, lambd=lambd)
    test_mse = ridreg.find_cost(testX, testY, lambd=lambd)
    t = (lambd, train_mse, test_mse)
    print("\t\t".join(map(lambda x: str(x), t)))

print("regularized")
print("Lambda\t\tTrain MSE\t\tTest MSE")
for lambd in lambds:
    ridreg = RidgeRegression(trainX, trainY, alpha, lambd, num_iter, conv_tol)
    train_mse = ridreg.find_cost(trainX, trainY, lambd=lambd, regularized=True)
    test_mse = ridreg.find_cost(testX, testY, lambd=lambd, regularized=True)
    t = (lambd, train_mse, test_mse)
    print("\t\t".join(map(lambda x: str(x), t)))



    


















    







Lambda		Train MSE		Test MSE
0.01		21.0236794341		28.6544550786
0.1		25.8053257313		33.4886178665
1.0		155.794681564		165.413748579
regularized
Lambda		Train MSE		Test MSE
0.01		26.4291602125		34.0588744312
0.1		70.67655419		78.3574027478
1.0		292.935650459		302.554265439

10 fold cross validation





In [30]:



    


alpha = 0.01
conv_tol = 0.001
num_iter = 100000
k = 10

def k_fold_cv(X, Y, lambd, k, regularized=False):
    n = len(Y)
    # Shuffle 
    shuf_idx = np.random.permutation(n)
    X, Y = X[shuf_idx], Y[shuf_idx]
    assert k > 1 and k <= n 
    ss = n / k # split size
    # split
    accuracy = []
    for i in range(k):        
        start, end = i * ss, (i + 1) * ss
        if i == k-1 and end < n:
            # anything left over shall go to the last split (if n is not multiple of k)
            end = n
        
        # ith split is for testing
        test_X, test_Y =  X[start:end], Y[start:end]
        # everything else for training
        train_X = np.delete(X, np.s_[start, end], axis=0)
        train_Y = np.delete(Y, np.s_[start, end], axis=0)
        ridreg = RidgeRegression(train_X, train_Y, alpha, lambd, num_iter, conv_tol)
        acc = ridreg.find_cost(test_X, test_Y, regularized=regularized)
        accuracy.append(acc)
    return np.array(accuracy).mean()

lambds = [0.0001, 0.001, 0.01, 0.1, 1, 10]

print("\n\nLambda\t\tTrain MSE\t\tTest MSE")
for lambd in lambds:
    train_mse = k_fold_cv(trainX, trainY, lambd, k)
    ridreg = RidgeRegression(trainX, trainY, alpha, lambd, num_iter, conv_tol) 
    test_mse = ridreg.find_cost(testX, testY, lambd=lambd)
    t = (lambd, train_mse, test_mse)
    print("\t\t".join(map(lambda x: str(x), t)))
    
print("Regularized\nLambda\t\tTrain MSE\t\tTest MSE")
for lambd in lambds:
    train_mse = k_fold_cv(trainX, trainY, lambd, k, regularized=True)
    ridreg = RidgeRegression(trainX, trainY, alpha, lambd, num_iter, conv_tol) 
    test_mse = ridreg.find_cost(testX, testY, lambd=lambd, regularized=True)
    t = (lambd, train_mse, test_mse)
    print("\t\t".join(map(lambda x: str(x), t)))



    


















    







Lambda		Train MSE		Test MSE
0.0001		21.1189316799		28.4418403091
0.001		21.0175547458		28.4585931037
0.01		21.1074051471		28.6421598842
0.1		25.8028276762		33.4671436945
1		155.966948824		165.411064702
10		474.536674225		500.398146396
Regularized
Lambda		Train MSE		Test MSE
0.0001		20.9894391669		28.4975967274
0.001		21.4837664007		29.0087370447
0.01		26.4126802108		34.0392762844
0.1		70.6084124031		78.3463496611
1		292.718370978		302.55457715
10		526.789978787		552.172840312

Feature Selection

1. Select top 4 features, fit linear regressor.





In [19]:



    


alpha = 0.001


top4TrainX = np.zeros(shape=(len(trainY), 0))
top4TestX = np.zeros(shape=(len(testY), 0))
tuples = pearson_cor(trainX, trainY, boston.feature_names)
target_cor = dict(tuples)
top4= sorted(target_cor, key=target_cor.get, reverse=True)[:4]
for k in top4:
    column = np.where(boston.feature_names == k)[0][0]
    
    x = np.array([trainX[:, column]]).transpose()
    top4TrainX = np.concatenate((top4TrainX, x), axis=1)
    
    x = np.array([testX[:, column]]).transpose()
    top4TestX = np.concatenate((top4TestX, x), axis=1)

print("Top4 attrs::", top4, '\n from', target_cor)
linreg = LinearRegression(top4TrainX, trainY, alpha)
train_mse_cost = MSECost(linreg.predict(top4TrainX), trainY)
test_mse_cost = MSECost(linreg.predict(top4TestX), testY)

print('Train MSE::', train_mse_cost, " Test MSE::", test_mse_cost)



    


















    






Top4 attrs:: ['LSTAT', 'RM', 'PTRATIO', 'INDUS'] 
 from {'CRIM': 0.38859443435207547, 'ZN': 0.36382754420139768, 'B': 0.34422912357951185, 'LSTAT': 0.74168271373323336, 'AGE': 0.39108230278590012, 'TAX': 0.46993468487617812, 'RAD': 0.38638415658601943, 'CHAS': 0.20407144132697344, 'NOX': 0.4258130776462164, 'RM': 0.69252269454509585, 'INDUS': 0.48418563338201681, 'PTRATIO': 0.5064403651254078, 'DIS': 0.25300487309124547}
Train MSE:: 26.4066042155  Test MSE:: 31.4962025449

2. Select top 4 features iteratively from residual errors





In [20]:



    


alpha = 0.001
Y_train = trainY
Y_test = testY
X_train = trainX
names = boston.feature_names

X_test = testX

Z_train = np.zeros(shape=(X_train.shape[0], 0)) 
Z_test = np.zeros(shape=(X_test.shape[0], 0)) 

pred_Y = None
for i in range(4):
    
    # first time we use the target, then on we use the Y - predY
    
    _Y = Y_train if pred_Y is None else np.subtract(Y_train, pred_Y) 
    corrs = dict(pearson_cor(X_train, _Y, names))
    top1 = sorted(corrs, key=corrs.get)[-1]
    print("Choose : ", top1)
    top1_col = names.tolist().index(top1)
    
    x = np.array([X_train[:, top1_col]]).transpose()
    Z_train = np.concatenate((Z_train, x), axis=1)
    
    x = np.array([X_test[:, top1_col]]).transpose()
    Z_test = np.concatenate((Z_test, x), axis=1)
    
    linreg = LinearRegression(Z_train, Y_train, alpha)
    pred_Y = linreg.predict(Z_train)
    train_mse_cost = MSECost(pred_Y, Y_train)
    test_mse_cost = MSECost(linreg.predict(Z_test), Y_test)

    print('Train MSE::', train_mse_cost, " Test MSE::", test_mse_cost)
    
    # for the next iteration
    X_train = np.delete(X_train, top1_col, axis=1)
    X_test = np.delete(X_test, top1_col, axis=1)
    names = np.delete(names, top1_col)



    


















    






Choose :  LSTAT
Train MSE:: 37.5682291124  Test MSE:: 43.9181708494
Choose :  RM
Train MSE:: 29.5454831972  Test MSE:: 36.2967947962
Choose :  B
Train MSE:: 28.6939844449  Test MSE:: 34.3784509663
Choose :  CHAS
Train MSE:: 26.9461754396  Test MSE:: 38.5301578153

Selection with highest Mutual information





In [ ]:

Selection with brute force





In [21]:



    


import itertools

least_cost = float('inf')
best_combination = None
for cols in itertools.combinations(range(trainX.shape[1]), 4):
    Z_train = np.zeros(shape=(X_train.shape[0], 0)) 
    Z_test = np.zeros(shape=(X_test.shape[0], 0)) 
    
    for col in cols:
        x = np.array([trainX[:, col]]).transpose()
        Z_train = np.concatenate((Z_train, x), axis=1)
    
        x = np.array([testX[:, col]]).transpose()
        Z_test = np.concatenate((Z_test, x), axis=1)
    
    linreg = LinearRegression(Z_train, Y_train, alpha)
    train_mse_cost = MSECost(linreg.predict(Z_train), Y_train)
    test_mse_cost = MSECost(linreg.predict(Z_test), Y_test)
    if test_mse_cost < least_cost:
        best_combination = cols 
        least_cost = test_mse_cost
    print(cols, 'Train MSE::', train_mse_cost, " Test MSE::", test_mse_cost)

print(best_combination, least_cost)



    


















    






(0, 1, 2, 3) Train MSE:: 54.7305210127  Test MSE:: 68.4461303194
(0, 1, 2, 4) Train MSE:: 58.4730836378  Test MSE:: 66.5960116117
(0, 1, 2, 5) Train MSE:: 35.9095646094  Test MSE:: 40.1036741833
(0, 1, 2, 6) Train MSE:: 58.4526510355  Test MSE:: 67.0295679399
(0, 1, 2, 7) Train MSE:: 52.9819649629  Test MSE:: 61.2636219244
(0, 1, 2, 8) Train MSE:: 58.4845270675  Test MSE:: 66.8775129902
(0, 1, 2, 9) Train MSE:: 57.5894674556  Test MSE:: 65.8612209485
(0, 1, 2, 10) Train MSE:: 51.0885005837  Test MSE:: 58.5613690869
(0, 1, 2, 11) Train MSE:: 57.0148737203  Test MSE:: 66.9792322345
(0, 1, 2, 12) Train MSE:: 36.9717042231  Test MSE:: 43.2933913214
(0, 1, 3, 4) Train MSE:: 57.3466185047  Test MSE:: 70.1550812401
(0, 1, 3, 5) Train MSE:: 34.9548344956  Test MSE:: 46.7586303987
(0, 1, 3, 6) Train MSE:: 58.3617103424  Test MSE:: 75.4728580207
(0, 1, 3, 7) Train MSE:: 59.7400311193  Test MSE:: 74.693859929
(0, 1, 3, 8) Train MSE:: 59.0208550445  Test MSE:: 74.3981989669
(0, 1, 3, 9) Train MSE:: 55.881296719  Test MSE:: 69.3411062826
(0, 1, 3, 10) Train MSE:: 52.5149487266  Test MSE:: 63.8340740659
(0, 1, 3, 11) Train MSE:: 57.6018098347  Test MSE:: 74.7669988083
(0, 1, 3, 12) Train MSE:: 34.8005208379  Test MSE:: 45.3619784471
(0, 1, 4, 5) Train MSE:: 36.234111655  Test MSE:: 39.1556716512
(0, 1, 4, 6) Train MSE:: 60.9556982733  Test MSE:: 69.7271000892
(0, 1, 4, 7) Train MSE:: 55.585384627  Test MSE:: 61.2945291624
(0, 1, 4, 8) Train MSE:: 60.8883638085  Test MSE:: 69.3583762202
(0, 1, 4, 9) Train MSE:: 58.635224115  Test MSE:: 66.9857041412
(0, 1, 4, 10) Train MSE:: 50.6568677193  Test MSE:: 56.9441203625
(0, 1, 4, 11) Train MSE:: 59.2901971123  Test MSE:: 69.3407711776
(0, 1, 4, 12) Train MSE:: 36.8158219954  Test MSE:: 42.8310213955
(0, 1, 5, 6) Train MSE:: 35.9078345108  Test MSE:: 41.7186319151
(0, 1, 5, 7) Train MSE:: 36.6153040111  Test MSE:: 41.1681566317
(0, 1, 5, 8) Train MSE:: 36.2298831608  Test MSE:: 40.2687945234
(0, 1, 5, 9) Train MSE:: 35.0540830402  Test MSE:: 38.9168299737
(0, 1, 5, 10) Train MSE:: 33.626230374  Test MSE:: 36.8456161018
(0, 1, 5, 11) Train MSE:: 34.68935778  Test MSE:: 39.8108366346
(0, 1, 5, 12) Train MSE:: 28.8625149865  Test MSE:: 35.5282048173
(0, 1, 6, 7) Train MSE:: 58.642791297  Test MSE:: 69.6664438472
(0, 1, 6, 8) Train MSE:: 61.4011693661  Test MSE:: 72.1101123097
(0, 1, 6, 9) Train MSE:: 58.6109656969  Test MSE:: 67.7442516547
(0, 1, 6, 10) Train MSE:: 52.7787398418  Test MSE:: 61.9072286269
(0, 1, 6, 11) Train MSE:: 59.6802491127  Test MSE:: 72.1115315573
(0, 1, 6, 12) Train MSE:: 35.8503400955  Test MSE:: 39.9280351466
(0, 1, 7, 8) Train MSE:: 60.7057978138  Test MSE:: 70.9520019077
(0, 1, 7, 9) Train MSE:: 55.6994056111  Test MSE:: 64.3843182841
(0, 1, 7, 10) Train MSE:: 53.9578881131  Test MSE:: 62.336404322
(0, 1, 7, 11) Train MSE:: 59.2456126105  Test MSE:: 71.5539726759
(0, 1, 7, 12) Train MSE:: 32.8827271539  Test MSE:: 37.0892860561
(0, 1, 8, 9) Train MSE:: 56.6647577555  Test MSE:: 64.1012396265
(0, 1, 8, 10) Train MSE:: 54.2605753532  Test MSE:: 62.3879927827
(0, 1, 8, 11) Train MSE:: 60.2213736655  Test MSE:: 71.8322216599
(0, 1, 8, 12) Train MSE:: 37.0111857403  Test MSE:: 42.9618235075
(0, 1, 9, 10) Train MSE:: 52.7150126713  Test MSE:: 60.5620311995
(0, 1, 9, 11) Train MSE:: 57.6360477363  Test MSE:: 67.5975448362
(0, 1, 9, 12) Train MSE:: 36.7959503145  Test MSE:: 42.7256205539
(0, 1, 10, 11) Train MSE:: 51.8772759234  Test MSE:: 61.8291008865
(0, 1, 10, 12) Train MSE:: 32.4436928136  Test MSE:: 37.390862937
(0, 1, 11, 12) Train MSE:: 36.8561069166  Test MSE:: 42.4015973601
(0, 2, 3, 4) Train MSE:: 55.8440150585  Test MSE:: 68.515728507
(0, 2, 3, 5) Train MSE:: 33.8952898853  Test MSE:: 45.5231810046
(0, 2, 3, 6) Train MSE:: 55.5034626217  Test MSE:: 70.3086781415
(0, 2, 3, 7) Train MSE:: 54.4848312174  Test MSE:: 66.8653271427
(0, 2, 3, 8) Train MSE:: 56.1202545582  Test MSE:: 69.5589105377
(0, 2, 3, 9) Train MSE:: 55.7506617143  Test MSE:: 68.7504225758
(0, 2, 3, 10) Train MSE:: 49.1836688494  Test MSE:: 59.3922310916
(0, 2, 3, 11) Train MSE:: 54.9354976439  Test MSE:: 69.556799457
(0, 2, 3, 12) Train MSE:: 34.8140518271  Test MSE:: 46.3545787039
(0, 2, 4, 5) Train MSE:: 36.0052094555  Test MSE:: 39.4755884914
(0, 2, 4, 6) Train MSE:: 59.4148460832  Test MSE:: 68.2721806298
(0, 2, 4, 7) Train MSE:: 56.0136708807  Test MSE:: 62.0960511811
(0, 2, 4, 8) Train MSE:: 59.6676374638  Test MSE:: 67.5174466849
(0, 2, 4, 9) Train MSE:: 59.1246785912  Test MSE:: 67.049766885
(0, 2, 4, 10) Train MSE:: 50.1932569869  Test MSE:: 56.5280715451
(0, 2, 4, 11) Train MSE:: 58.3308463618  Test MSE:: 67.8584214378
(0, 2, 4, 12) Train MSE:: 36.6717679928  Test MSE:: 44.4693627859
(0, 2, 5, 6) Train MSE:: 35.538193715  Test MSE:: 40.9420973472
(0, 2, 5, 7) Train MSE:: 35.2972696304  Test MSE:: 39.9347597701
(0, 2, 5, 8) Train MSE:: 35.9282933658  Test MSE:: 40.0055370051
(0, 2, 5, 9) Train MSE:: 35.3714980439  Test MSE:: 39.2996027792
(0, 2, 5, 10) Train MSE:: 32.9019682772  Test MSE:: 36.1377567375
(0, 2, 5, 11) Train MSE:: 34.4320963786  Test MSE:: 39.3302920672
(0, 2, 5, 12) Train MSE:: 28.8781046006  Test MSE:: 35.8272996408
(0, 2, 6, 7) Train MSE:: 54.9380333218  Test MSE:: 64.6680221317
(0, 2, 6, 8) Train MSE:: 59.424567351  Test MSE:: 68.5541603722
(0, 2, 6, 9) Train MSE:: 58.8193047107  Test MSE:: 67.747189585
(0, 2, 6, 10) Train MSE:: 50.928534979  Test MSE:: 58.8852500375
(0, 2, 6, 11) Train MSE:: 58.0340965738  Test MSE:: 68.6028478114
(0, 2, 6, 12) Train MSE:: 36.0231278886  Test MSE:: 41.9659615318
(0, 2, 7, 8) Train MSE:: 57.6272853236  Test MSE:: 65.5267337503
(0, 2, 7, 9) Train MSE:: 57.0820539789  Test MSE:: 64.7342143234
(0, 2, 7, 10) Train MSE:: 49.7192336247  Test MSE:: 57.2198032548
(0, 2, 7, 11) Train MSE:: 56.1374010165  Test MSE:: 65.7012121592
(0, 2, 7, 12) Train MSE:: 33.282173141  Test MSE:: 41.1213609216
(0, 2, 8, 9) Train MSE:: 58.3179230061  Test MSE:: 65.4420093614
(0, 2, 8, 10) Train MSE:: 50.8039634284  Test MSE:: 57.5729411932
(0, 2, 8, 11) Train MSE:: 58.3771347303  Test MSE:: 68.1898708151
(0, 2, 8, 12) Train MSE:: 37.031449141  Test MSE:: 43.9897755322
(0, 2, 9, 10) Train MSE:: 51.3109615598  Test MSE:: 58.5280012288
(0, 2, 9, 11) Train MSE:: 58.0700723461  Test MSE:: 67.5394480324
(0, 2, 9, 12) Train MSE:: 36.9734963436  Test MSE:: 43.3784320798
(0, 2, 10, 11) Train MSE:: 49.9386229534  Test MSE:: 58.5039864157
(0, 2, 10, 12) Train MSE:: 32.4506522905  Test MSE:: 37.4223586416
(0, 2, 11, 12) Train MSE:: 36.9662128539  Test MSE:: 43.2844994818
(0, 3, 4, 5) Train MSE:: 34.306105958  Test MSE:: 43.8370647392
(0, 3, 4, 6) Train MSE:: 58.7728739261  Test MSE:: 73.9521289252
(0, 3, 4, 7) Train MSE:: 58.5252454561  Test MSE:: 68.5632393197
(0, 3, 4, 8) Train MSE:: 59.6158077764  Test MSE:: 72.0958232959
(0, 3, 4, 9) Train MSE:: 58.0490890141  Test MSE:: 69.996776881
(0, 3, 4, 10) Train MSE:: 48.6968615159  Test MSE:: 56.6234989348
(0, 3, 4, 11) Train MSE:: 58.3832222524  Test MSE:: 72.0691274039
(0, 3, 4, 12) Train MSE:: 35.174435286  Test MSE:: 46.0192914745
(0, 3, 5, 6) Train MSE:: 33.6840363117  Test MSE:: 48.2281008317
(0, 3, 5, 7) Train MSE:: 36.1786012714  Test MSE:: 47.9686332069
(0, 3, 5, 8) Train MSE:: 35.0889828954  Test MSE:: 46.1359041749
(0, 3, 5, 9) Train MSE:: 33.9001844823  Test MSE:: 44.2852508595
(0, 3, 5, 10) Train MSE:: 32.6951310554  Test MSE:: 40.7695265865
(0, 3, 5, 11) Train MSE:: 33.8767829342  Test MSE:: 45.9347456657
(0, 3, 5, 12) Train MSE:: 27.1572935429  Test MSE:: 39.9687702341
(0, 3, 6, 7) Train MSE:: 59.9598686653  Test MSE:: 79.6236888739
(0, 3, 6, 8) Train MSE:: 59.8875704632  Test MSE:: 78.2847076012
(0, 3, 6, 9) Train MSE:: 57.5758184366  Test MSE:: 73.2870360236
(0, 3, 6, 10) Train MSE:: 51.055997012  Test MSE:: 63.9505810563
(0, 3, 6, 11) Train MSE:: 58.517830096  Test MSE:: 78.5884173361
(0, 3, 6, 12) Train MSE:: 34.9055136423  Test MSE:: 44.7354294489
(0, 3, 7, 8) Train MSE:: 64.2393101925  Test MSE:: 80.1420363052
(0, 3, 7, 9) Train MSE:: 60.3784889212  Test MSE:: 73.7121319962
(0, 3, 7, 10) Train MSE:: 54.4994210163  Test MSE:: 64.6555591382
(0, 3, 7, 11) Train MSE:: 63.1598972129  Test MSE:: 81.0819099466
(0, 3, 7, 12) Train MSE:: 33.9162199084  Test MSE:: 43.6614974838
(0, 3, 8, 9) Train MSE:: 58.8964134041  Test MSE:: 70.6680316554
(0, 3, 8, 10) Train MSE:: 54.8407679575  Test MSE:: 64.989237645
(0, 3, 8, 11) Train MSE:: 62.5968785738  Test MSE:: 79.5814804063
(0, 3, 8, 12) Train MSE:: 35.1751988889  Test MSE:: 46.0127813893
(0, 3, 9, 10) Train MSE:: 52.8158088756  Test MSE:: 62.5462161346
(0, 3, 9, 11) Train MSE:: 59.1202207695  Test MSE:: 73.4953204393
(0, 3, 9, 12) Train MSE:: 34.8924904186  Test MSE:: 45.4373457004
(0, 3, 10, 11) Train MSE:: 52.2750981868  Test MSE:: 64.1567441071
(0, 3, 10, 12) Train MSE:: 31.0620626964  Test MSE:: 38.9932318171
(0, 3, 11, 12) Train MSE:: 35.0243242774  Test MSE:: 45.3788469568
(0, 4, 5, 6) Train MSE:: 35.8875533613  Test MSE:: 40.9630829092
(0, 4, 5, 7) Train MSE:: 35.5964252269  Test MSE:: 37.2281058866
(0, 4, 5, 8) Train MSE:: 36.2596426322  Test MSE:: 39.1163437005
(0, 4, 5, 9) Train MSE:: 35.438828247  Test MSE:: 38.6381550841
(0, 4, 5, 10) Train MSE:: 32.2801581666  Test MSE:: 33.8063379089
(0, 4, 5, 11) Train MSE:: 34.804715588  Test MSE:: 38.5226750803
(0, 4, 5, 12) Train MSE:: 28.9230234495  Test MSE:: 35.9261963102
(0, 4, 6, 7) Train MSE:: 59.4346656809  Test MSE:: 69.2160437232
(0, 4, 6, 8) Train MSE:: 62.4963587191  Test MSE:: 72.745341281
(0, 4, 6, 9) Train MSE:: 60.5445007238  Test MSE:: 70.1626352766
(0, 4, 6, 10) Train MSE:: 50.6834729617  Test MSE:: 57.2679524369
(0, 4, 6, 11) Train MSE:: 60.9636063453  Test MSE:: 72.7184886562
(0, 4, 6, 12) Train MSE:: 36.7071564152  Test MSE:: 41.5296102317
(0, 4, 7, 8) Train MSE:: 61.652873943  Test MSE:: 68.7711110262
(0, 4, 7, 9) Train MSE:: 59.6258263728  Test MSE:: 66.4941453948
(0, 4, 7, 10) Train MSE:: 47.638441906  Test MSE:: 53.0465806409
(0, 4, 7, 11) Train MSE:: 60.2603896023  Test MSE:: 68.7151011955
(0, 4, 7, 12) Train MSE:: 34.7482501006  Test MSE:: 39.6544187098
(0, 4, 8, 9) Train MSE:: 59.4990438461  Test MSE:: 66.0543796243
(0, 4, 8, 10) Train MSE:: 49.6331062312  Test MSE:: 54.5732482075
(0, 4, 8, 11) Train MSE:: 61.7551942381  Test MSE:: 71.5572977362
(0, 4, 8, 12) Train MSE:: 37.2397114085  Test MSE:: 43.5459738968
(0, 4, 9, 10) Train MSE:: 50.7409767133  Test MSE:: 56.840292979
(0, 4, 9, 11) Train MSE:: 60.2040918553  Test MSE:: 69.4493051127
(0, 4, 9, 12) Train MSE:: 36.6657614109  Test MSE:: 43.0667382882
(0, 4, 10, 11) Train MSE:: 49.60581458  Test MSE:: 56.8914814334
(0, 4, 10, 12) Train MSE:: 32.4465800117  Test MSE:: 37.4072418764
(0, 4, 11, 12) Train MSE:: 37.0093199956  Test MSE:: 42.8584652719
(0, 5, 6, 7) Train MSE:: 34.8036742419  Test MSE:: 42.896135148
(0, 5, 6, 8) Train MSE:: 35.6092515075  Test MSE:: 41.1839308116
(0, 5, 6, 9) Train MSE:: 34.8210891726  Test MSE:: 39.8502804352
(0, 5, 6, 10) Train MSE:: 32.478886454  Test MSE:: 37.0208566428
(0, 5, 6, 11) Train MSE:: 34.0970081114  Test MSE:: 40.5223975009
(0, 5, 6, 12) Train MSE:: 28.8912363085  Test MSE:: 35.5562025182
(0, 5, 7, 8) Train MSE:: 36.9387173516  Test MSE:: 41.0259656354
(0, 5, 7, 9) Train MSE:: 35.4621032612  Test MSE:: 39.4603564013
(0, 5, 7, 10) Train MSE:: 33.9020305013  Test MSE:: 36.8598492722
(0, 5, 7, 11) Train MSE:: 35.5437229472  Test MSE:: 40.8252689802
(0, 5, 7, 12) Train MSE:: 27.8967850117  Test MSE:: 35.0682705184
(0, 5, 8, 9) Train MSE:: 35.34448014  Test MSE:: 39.3806904889
(0, 5, 8, 10) Train MSE:: 33.7729458471  Test MSE:: 36.7524542317
(0, 5, 8, 11) Train MSE:: 35.1535808648  Test MSE:: 39.8390119635
(0, 5, 8, 12) Train MSE:: 28.8983673521  Test MSE:: 35.6304903859
(0, 5, 9, 10) Train MSE:: 33.001205875  Test MSE:: 35.9523236896
(0, 5, 9, 11) Train MSE:: 34.2531824953  Test MSE:: 38.5881404985
(0, 5, 9, 12) Train MSE:: 28.5700196687  Test MSE:: 35.1112868964
(0, 5, 10, 11) Train MSE:: 31.7032230322  Test MSE:: 35.5051205991
(0, 5, 10, 12) Train MSE:: 26.1347160179  Test MSE:: 31.4844258986
(0, 5, 11, 12) Train MSE:: 28.3301377204  Test MSE:: 34.5013301986
(0, 6, 7, 8) Train MSE:: 62.1926847674  Test MSE:: 75.4101263571
(0, 6, 7, 9) Train MSE:: 58.8830782479  Test MSE:: 69.7325204559
(0, 6, 7, 10) Train MSE:: 51.9609486953  Test MSE:: 62.4592867087
(0, 6, 7, 11) Train MSE:: 60.7003094093  Test MSE:: 75.959165399
(0, 6, 7, 12) Train MSE:: 35.4934833923  Test MSE:: 41.321052841
(0, 6, 8, 9) Train MSE:: 59.1638676991  Test MSE:: 68.267095068
(0, 6, 8, 10) Train MSE:: 53.0185323679  Test MSE:: 62.3663703734
(0, 6, 8, 11) Train MSE:: 61.6862277836  Test MSE:: 75.3666707843
(0, 6, 8, 12) Train MSE:: 36.7511870961  Test MSE:: 41.8650703184
(0, 6, 9, 10) Train MSE:: 52.4214245763  Test MSE:: 61.1203553657
(0, 6, 9, 11) Train MSE:: 59.6112618845  Test MSE:: 71.0987264773
(0, 6, 9, 12) Train MSE:: 36.1628252737  Test MSE:: 41.1480438703
(0, 6, 10, 11) Train MSE:: 51.1424664583  Test MSE:: 61.6877914299
(0, 6, 10, 12) Train MSE:: 31.7831154194  Test MSE:: 35.604737772
(0, 6, 11, 12) Train MSE:: 36.5447118157  Test MSE:: 41.1856321341
(0, 7, 8, 9) Train MSE:: 61.1528684738  Test MSE:: 68.6053942311
(0, 7, 8, 10) Train MSE:: 56.0652945797  Test MSE:: 63.3576022248
(0, 7, 8, 11) Train MSE:: 65.1186533438  Test MSE:: 77.3176912859
(0, 7, 8, 12) Train MSE:: 35.4213479795  Test MSE:: 40.9442070767
(0, 7, 9, 10) Train MSE:: 54.2706871708  Test MSE:: 61.5052162802
(0, 7, 9, 11) Train MSE:: 61.5249313579  Test MSE:: 71.7212707794
(0, 7, 9, 12) Train MSE:: 34.4596175355  Test MSE:: 39.9450829184
(0, 7, 10, 11) Train MSE:: 53.485531407  Test MSE:: 62.8494023263
(0, 7, 10, 12) Train MSE:: 30.4374200959  Test MSE:: 35.0092460597
(0, 7, 11, 12) Train MSE:: 35.2125184314  Test MSE:: 40.2123794947
(0, 8, 9, 10) Train MSE:: 51.3381986976  Test MSE:: 56.9311054951
(0, 8, 9, 11) Train MSE:: 59.4831761753  Test MSE:: 68.6452760931
(0, 8, 9, 12) Train MSE:: 35.9107282426  Test MSE:: 42.5967800481
(0, 8, 10, 11) Train MSE:: 53.482096013  Test MSE:: 62.9632303717
(0, 8, 10, 12) Train MSE:: 31.7541468635  Test MSE:: 36.7064263323
(0, 8, 11, 12) Train MSE:: 37.0729667983  Test MSE:: 42.886145185
(0, 9, 10, 11) Train MSE:: 52.5995194705  Test MSE:: 61.2994145539
(0, 9, 10, 12) Train MSE:: 32.437114209  Test MSE:: 37.3643685609
(0, 9, 11, 12) Train MSE:: 36.8982922693  Test MSE:: 42.7589014299
(0, 10, 11, 12) Train MSE:: 32.2794295526  Test MSE:: 36.8492676611
(1, 2, 3, 4) Train MSE:: 57.6156010685  Test MSE:: 69.3343664377
(1, 2, 3, 5) Train MSE:: 35.9788110799  Test MSE:: 46.0610816324
(1, 2, 3, 6) Train MSE:: 57.580234855  Test MSE:: 71.4524929186
(1, 2, 3, 7) Train MSE:: 54.7710208815  Test MSE:: 66.7341690262
(1, 2, 3, 8) Train MSE:: 57.0496077762  Test MSE:: 69.8330448564
(1, 2, 3, 9) Train MSE:: 55.8255506032  Test MSE:: 67.6415402564
(1, 2, 3, 10) Train MSE:: 50.9085842566  Test MSE:: 60.2497063152
(1, 2, 3, 11) Train MSE:: 55.7037138829  Test MSE:: 69.6086849499
(1, 2, 3, 12) Train MSE:: 34.7953451335  Test MSE:: 46.0878136737
(1, 2, 4, 5) Train MSE:: 38.2980141665  Test MSE:: 39.2770500259
(1, 2, 4, 6) Train MSE:: 62.002882011  Test MSE:: 69.3993660182
(1, 2, 4, 7) Train MSE:: 55.730225733  Test MSE:: 60.7243310746
(1, 2, 4, 8) Train MSE:: 61.1140418563  Test MSE:: 68.6119480255
(1, 2, 4, 9) Train MSE:: 59.4049576483  Test MSE:: 66.5069300861
(1, 2, 4, 10) Train MSE:: 51.6830600005  Test MSE:: 56.7754476586
(1, 2, 4, 11) Train MSE:: 59.5328817512  Test MSE:: 68.16124008
(1, 2, 4, 12) Train MSE:: 36.7248254421  Test MSE:: 44.2758973247
(1, 2, 5, 6) Train MSE:: 37.8462382432  Test MSE:: 41.6018310053
(1, 2, 5, 7) Train MSE:: 37.5407372394  Test MSE:: 40.2575290135
(1, 2, 5, 8) Train MSE:: 37.1152063818  Test MSE:: 39.2680184825
(1, 2, 5, 9) Train MSE:: 36.1422006997  Test MSE:: 38.3003127014
(1, 2, 5, 10) Train MSE:: 34.7747625344  Test MSE:: 36.0539665929
(1, 2, 5, 11) Train MSE:: 35.7879173779  Test MSE:: 38.6293200904
(1, 2, 5, 12) Train MSE:: 29.3892457774  Test MSE:: 36.1067295728
(1, 2, 6, 7) Train MSE:: 56.0715769611  Test MSE:: 64.9145114637
(1, 2, 6, 8) Train MSE:: 61.0265656757  Test MSE:: 69.1871198053
(1, 2, 6, 9) Train MSE:: 59.3330799684  Test MSE:: 66.7664450738
(1, 2, 6, 10) Train MSE:: 52.8423894832  Test MSE:: 59.9600084137
(1, 2, 6, 11) Train MSE:: 59.4338786836  Test MSE:: 68.8184416352
(1, 2, 6, 12) Train MSE:: 35.7115771028  Test MSE:: 40.9330971784
(1, 2, 7, 8) Train MSE:: 56.4069895337  Test MSE:: 64.4704986632
(1, 2, 7, 9) Train MSE:: 54.5489171945  Test MSE:: 61.5358702709
(1, 2, 7, 10) Train MSE:: 51.2837538536  Test MSE:: 58.5646794477
(1, 2, 7, 11) Train MSE:: 55.068104967  Test MSE:: 64.4270429245
(1, 2, 7, 12) Train MSE:: 31.6843000553  Test MSE:: 38.834263298
(1, 2, 8, 9) Train MSE:: 58.7573490985  Test MSE:: 65.2769209535
(1, 2, 8, 10) Train MSE:: 53.341689528  Test MSE:: 59.6696797939
(1, 2, 8, 11) Train MSE:: 59.2102109228  Test MSE:: 68.1053057874
(1, 2, 8, 12) Train MSE:: 37.1834188028  Test MSE:: 43.7245676711
(1, 2, 9, 10) Train MSE:: 52.7410394078  Test MSE:: 59.0416903751
(1, 2, 9, 11) Train MSE:: 57.9055961786  Test MSE:: 66.173076136
(1, 2, 9, 12) Train MSE:: 36.8589897929  Test MSE:: 42.6605493639
(1, 2, 10, 11) Train MSE:: 51.0584608955  Test MSE:: 58.7975549822
(1, 2, 10, 12) Train MSE:: 32.4845117204  Test MSE:: 37.5345524705
(1, 2, 11, 12) Train MSE:: 36.9608299267  Test MSE:: 42.8167330343
(1, 3, 4, 5) Train MSE:: 36.3273521646  Test MSE:: 43.7563346642
(1, 3, 4, 6) Train MSE:: 60.84226865  Test MSE:: 74.2525493056
(1, 3, 4, 7) Train MSE:: 57.6919755034  Test MSE:: 65.5146885729
(1, 3, 4, 8) Train MSE:: 59.6174244556  Test MSE:: 71.7492186304
(1, 3, 4, 9) Train MSE:: 56.9184877988  Test MSE:: 68.1769299547
(1, 3, 4, 10) Train MSE:: 50.0468791068  Test MSE:: 56.6733210016
(1, 3, 4, 11) Train MSE:: 58.6158696665  Test MSE:: 71.5634394224
(1, 3, 4, 12) Train MSE:: 34.9840279861  Test MSE:: 45.7443903904
(1, 3, 5, 6) Train MSE:: 36.0826722922  Test MSE:: 49.8499579745
(1, 3, 5, 7) Train MSE:: 39.0051521431  Test MSE:: 49.1200821684
(1, 3, 5, 8) Train MSE:: 35.5165993948  Test MSE:: 44.9026896757
(1, 3, 5, 9) Train MSE:: 34.1662643413  Test MSE:: 43.0321452718
(1, 3, 5, 10) Train MSE:: 35.125722607  Test MSE:: 41.4769475756
(1, 3, 5, 11) Train MSE:: 34.9248241734  Test MSE:: 45.062824779
(1, 3, 5, 12) Train MSE:: 27.5597554464  Test MSE:: 40.0569106518
(1, 3, 6, 7) Train MSE:: 62.1286327072  Test MSE:: 81.082379824
(1, 3, 6, 8) Train MSE:: 60.0771811007  Test MSE:: 76.0610926469
(1, 3, 6, 9) Train MSE:: 56.8347587725  Test MSE:: 69.9379537535
(1, 3, 6, 10) Train MSE:: 53.4220961189  Test MSE:: 65.8677527133
(1, 3, 6, 11) Train MSE:: 59.4693150782  Test MSE:: 77.8388232206
(1, 3, 6, 12) Train MSE:: 34.3258746008  Test MSE:: 43.2658860669
(1, 3, 7, 8) Train MSE:: 61.1053489367  Test MSE:: 75.3981219622
(1, 3, 7, 9) Train MSE:: 55.5321473613  Test MSE:: 66.7111365572
(1, 3, 7, 10) Train MSE:: 57.0138792821  Test MSE:: 67.0316016919
(1, 3, 7, 11) Train MSE:: 62.0176806769  Test MSE:: 78.9269244405
(1, 3, 7, 12) Train MSE:: 32.1706176781  Test MSE:: 40.9140055921
(1, 3, 8, 9) Train MSE:: 56.5012796113  Test MSE:: 67.6807784852
(1, 3, 8, 10) Train MSE:: 55.3365379144  Test MSE:: 65.8975308701
(1, 3, 8, 11) Train MSE:: 59.408319861  Test MSE:: 74.7169542037
(1, 3, 8, 12) Train MSE:: 35.0073070203  Test MSE:: 45.5728317401
(1, 3, 9, 10) Train MSE:: 52.3873885283  Test MSE:: 61.8733488118
(1, 3, 9, 11) Train MSE:: 56.0035315039  Test MSE:: 69.034907248
(1, 3, 9, 12) Train MSE:: 34.6421232457  Test MSE:: 44.9259767112
(1, 3, 10, 11) Train MSE:: 52.7116855381  Test MSE:: 64.383074218
(1, 3, 10, 12) Train MSE:: 31.1046976764  Test MSE:: 38.9873244429
(1, 3, 11, 12) Train MSE:: 34.7983033352  Test MSE:: 44.8240362119
(1, 4, 5, 6) Train MSE:: 38.3384390609  Test MSE:: 41.0110611821
(1, 4, 5, 7) Train MSE:: 37.4700933626  Test MSE:: 36.3588746066
(1, 4, 5, 8) Train MSE:: 37.3487523116  Test MSE:: 38.5225281966
(1, 4, 5, 9) Train MSE:: 36.1310814532  Test MSE:: 37.9610926702
(1, 4, 5, 10) Train MSE:: 33.6154582689  Test MSE:: 32.6568840937
(1, 4, 5, 11) Train MSE:: 36.1485540771  Test MSE:: 37.5666319825
(1, 4, 5, 12) Train MSE:: 29.4683855151  Test MSE:: 36.0484423381
(1, 4, 6, 7) Train MSE:: 59.8746967261  Test MSE:: 67.2487559529
(1, 4, 6, 8) Train MSE:: 63.4110968137  Test MSE:: 72.2260006203
(1, 4, 6, 9) Train MSE:: 60.1442198136  Test MSE:: 67.9000726357
(1, 4, 6, 10) Train MSE:: 52.2595644176  Test MSE:: 57.7055942413
(1, 4, 6, 11) Train MSE:: 62.1898086887  Test MSE:: 72.124065048
(1, 4, 6, 12) Train MSE:: 36.2264384969  Test MSE:: 40.2693294285
(1, 4, 7, 8) Train MSE:: 58.9561540037  Test MSE:: 65.3120622005
(1, 4, 7, 9) Train MSE:: 55.1659917336  Test MSE:: 60.5491268821
(1, 4, 7, 10) Train MSE:: 48.3699194278  Test MSE:: 53.1546390168
(1, 4, 7, 11) Train MSE:: 58.109882309  Test MSE:: 64.9742938693
(1, 4, 7, 12) Train MSE:: 32.8001503815  Test MSE:: 36.5491914408
(1, 4, 8, 9) Train MSE:: 59.1628336062  Test MSE:: 65.4726296266
(1, 4, 8, 10) Train MSE:: 52.2044183793  Test MSE:: 56.6968253538
(1, 4, 8, 11) Train MSE:: 61.5635126487  Test MSE:: 70.5724478221
(1, 4, 8, 12) Train MSE:: 37.0890478916  Test MSE:: 43.1068813941
(1, 4, 9, 10) Train MSE:: 52.0028183645  Test MSE:: 57.2262222501
(1, 4, 9, 11) Train MSE:: 58.8094107278  Test MSE:: 67.1247383666
(1, 4, 9, 12) Train MSE:: 36.2261347167  Test MSE:: 42.33039692
(1, 4, 10, 11) Train MSE:: 50.5533091178  Test MSE:: 56.8607829603
(1, 4, 10, 12) Train MSE:: 32.4933390167  Test MSE:: 37.3864337868
(1, 4, 11, 12) Train MSE:: 36.799086073  Test MSE:: 42.2120802733
(1, 5, 6, 7) Train MSE:: 37.7779436516  Test MSE:: 43.9377203086
(1, 5, 6, 8) Train MSE:: 36.7729180971  Test MSE:: 40.3662480382
(1, 5, 6, 9) Train MSE:: 35.6986630798  Test MSE:: 38.8700724517
(1, 5, 6, 10) Train MSE:: 34.2334992076  Test MSE:: 37.581743791
(1, 5, 6, 11) Train MSE:: 35.557825133  Test MSE:: 40.1516158005
(1, 5, 6, 12) Train MSE:: 29.3816917372  Test MSE:: 35.4548177938
(1, 5, 7, 8) Train MSE:: 37.1810076333  Test MSE:: 39.5309730224
(1, 5, 7, 9) Train MSE:: 35.0123278573  Test MSE:: 37.5477272465
(1, 5, 7, 10) Train MSE:: 36.5284902553  Test MSE:: 37.140602387
(1, 5, 7, 11) Train MSE:: 36.7631177057  Test MSE:: 39.7642002616
(1, 5, 7, 12) Train MSE:: 27.8054704534  Test MSE:: 34.2486033342
(1, 5, 8, 9) Train MSE:: 36.0631980355  Test MSE:: 38.2446287574
(1, 5, 8, 10) Train MSE:: 35.149507886  Test MSE:: 36.3500128176
(1, 5, 8, 11) Train MSE:: 35.6144946446  Test MSE:: 38.1443944634
(1, 5, 8, 12) Train MSE:: 29.2162709335  Test MSE:: 35.38663203
(1, 5, 9, 10) Train MSE:: 33.9139268559  Test MSE:: 35.3363513534
(1, 5, 9, 11) Train MSE:: 34.5053663419  Test MSE:: 37.0771403033
(1, 5, 9, 12) Train MSE:: 28.7494113519  Test MSE:: 34.8775214869
(1, 5, 10, 11) Train MSE:: 32.9247606101  Test MSE:: 34.7936426891
(1, 5, 10, 12) Train MSE:: 26.3706262228  Test MSE:: 31.5036719815
(1, 5, 11, 12) Train MSE:: 28.633371417  Test MSE:: 34.146281529
(1, 6, 7, 8) Train MSE:: 60.5214368646  Test MSE:: 71.3322486481
(1, 6, 7, 9) Train MSE:: 55.4068620643  Test MSE:: 63.2972112191
(1, 6, 7, 10) Train MSE:: 54.6539349734  Test MSE:: 64.8031071553
(1, 6, 7, 11) Train MSE:: 60.6122826657  Test MSE:: 74.122406928
(1, 6, 7, 12) Train MSE:: 33.8062200467  Test MSE:: 38.4702542278
(1, 6, 8, 9) Train MSE:: 59.1219213148  Test MSE:: 66.3403985858
(1, 6, 8, 10) Train MSE:: 55.3440376108  Test MSE:: 63.9598196585
(1, 6, 8, 11) Train MSE:: 61.7007881242  Test MSE:: 72.8430737351
(1, 6, 8, 12) Train MSE:: 36.146257131  Test MSE:: 40.2694225241
(1, 6, 9, 10) Train MSE:: 53.4230087573  Test MSE:: 60.8767844845
(1, 6, 9, 11) Train MSE:: 58.6692623362  Test MSE:: 67.7445479061
(1, 6, 9, 12) Train MSE:: 35.3306858968  Test MSE:: 39.3934380457
(1, 6, 10, 11) Train MSE:: 52.5016906937  Test MSE:: 62.2210376234
(1, 6, 10, 12) Train MSE:: 31.7842831992  Test MSE:: 35.4061225263
(1, 6, 11, 12) Train MSE:: 35.9106448984  Test MSE:: 39.4466672631
(1, 7, 8, 9) Train MSE:: 56.6963747207  Test MSE:: 63.202783127
(1, 7, 8, 10) Train MSE:: 57.2024549617  Test MSE:: 65.0086001678
(1, 7, 8, 11) Train MSE:: 61.3113335022  Test MSE:: 72.1207928604
(1, 7, 8, 12) Train MSE:: 33.2490471295  Test MSE:: 37.5738116418
(1, 7, 9, 10) Train MSE:: 53.2287656477  Test MSE:: 60.3784434918
(1, 7, 9, 11) Train MSE:: 56.0401360881  Test MSE:: 64.6655624611
(1, 7, 9, 12) Train MSE:: 31.7063817192  Test MSE:: 35.912250691
(1, 7, 10, 11) Train MSE:: 54.2683337134  Test MSE:: 63.4805911575
(1, 7, 10, 12) Train MSE:: 29.8882491976  Test MSE:: 34.3329756429
(1, 7, 11, 12) Train MSE:: 33.1620541106  Test MSE:: 36.8320468184
(1, 8, 9, 10) Train MSE:: 52.4283065551  Test MSE:: 58.0861898926
(1, 8, 9, 11) Train MSE:: 57.426495858  Test MSE:: 65.3400121073
(1, 8, 9, 12) Train MSE:: 35.9191149728  Test MSE:: 42.3389839872
(1, 8, 10, 11) Train MSE:: 54.1742867117  Test MSE:: 62.9448134577
(1, 8, 10, 12) Train MSE:: 32.1089100611  Test MSE:: 37.0696552015
(1, 8, 11, 12) Train MSE:: 37.0232902736  Test MSE:: 42.4399424027
(1, 9, 10, 11) Train MSE:: 52.3416201225  Test MSE:: 60.2188058321
(1, 9, 10, 12) Train MSE:: 32.4935736707  Test MSE:: 37.3901272161
(1, 9, 11, 12) Train MSE:: 36.7465773185  Test MSE:: 42.2862821674
(1, 10, 11, 12) Train MSE:: 32.2872650714  Test MSE:: 36.7782013409
(2, 3, 4, 5) Train MSE:: 35.6641761053  Test MSE:: 44.1368958938
(2, 3, 4, 6) Train MSE:: 58.0940826074  Test MSE:: 71.4415030028
(2, 3, 4, 7) Train MSE:: 55.880236783  Test MSE:: 64.19139949
(2, 3, 4, 8) Train MSE:: 58.0303478959  Test MSE:: 69.7176444498
(2, 3, 4, 9) Train MSE:: 57.281308842  Test MSE:: 68.2817551613
(2, 3, 4, 10) Train MSE:: 49.2697084954  Test MSE:: 56.3737924285
(2, 3, 4, 11) Train MSE:: 56.7382706472  Test MSE:: 69.3937893641
(2, 3, 4, 12) Train MSE:: 34.6953823963  Test MSE:: 47.3845176129
(2, 3, 5, 6) Train MSE:: 35.0054379485  Test MSE:: 47.4480134058
(2, 3, 5, 7) Train MSE:: 35.9128297587  Test MSE:: 45.8035558127
(2, 3, 5, 8) Train MSE:: 34.9357465315  Test MSE:: 44.5508056998
(2, 3, 5, 9) Train MSE:: 34.4644543207  Test MSE:: 43.5216865735
(2, 3, 5, 10) Train MSE:: 33.0421005119  Test MSE:: 40.0313691534
(2, 3, 5, 11) Train MSE:: 33.7815517649  Test MSE:: 43.9378334647
(2, 3, 5, 12) Train MSE:: 27.4310790452  Test MSE:: 40.4073392931
(2, 3, 6, 7) Train MSE:: 54.8884045268  Test MSE:: 69.3931165986
(2, 3, 6, 8) Train MSE:: 57.5535506975  Test MSE:: 71.6594408659
(2, 3, 6, 9) Train MSE:: 56.7398853655  Test MSE:: 69.893721315
(2, 3, 6, 10) Train MSE:: 50.2915827496  Test MSE:: 60.7211684758
(2, 3, 6, 11) Train MSE:: 56.2258617347  Test MSE:: 71.4116024206
(2, 3, 6, 12) Train MSE:: 34.1965921228  Test MSE:: 44.8818928894
(2, 3, 7, 8) Train MSE:: 57.0431565294  Test MSE:: 68.7774829411
(2, 3, 7, 9) Train MSE:: 56.3734300238  Test MSE:: 67.0497322324
(2, 3, 7, 10) Train MSE:: 50.3042209081  Test MSE:: 59.1490130687
(2, 3, 7, 11) Train MSE:: 55.8357083177  Test MSE:: 68.7602594599
(2, 3, 7, 12) Train MSE:: 31.8741281444  Test MSE:: 43.8344309185
(2, 3, 8, 9) Train MSE:: 57.4252364158  Test MSE:: 68.585003345
(2, 3, 8, 10) Train MSE:: 51.1335116113  Test MSE:: 60.1275108022
(2, 3, 8, 11) Train MSE:: 56.7946889803  Test MSE:: 70.2594350261
(2, 3, 8, 12) Train MSE:: 34.9321175705  Test MSE:: 46.7456301578
(2, 3, 9, 10) Train MSE:: 50.8144066288  Test MSE:: 59.6198453012
(2, 3, 9, 11) Train MSE:: 56.1873499663  Test MSE:: 68.8551125966
(2, 3, 9, 12) Train MSE:: 34.8206953753  Test MSE:: 45.9290063419
(2, 3, 10, 11) Train MSE:: 49.1828327562  Test MSE:: 59.3023197584
(2, 3, 10, 12) Train MSE:: 31.0579470709  Test MSE:: 39.2840006696
(2, 3, 11, 12) Train MSE:: 34.8055635205  Test MSE:: 45.9130760817
(2, 4, 5, 6) Train MSE:: 37.801991474  Test MSE:: 41.0006375286
(2, 4, 5, 7) Train MSE:: 37.0155431208  Test MSE:: 37.2156766745
(2, 4, 5, 8) Train MSE:: 37.2131578146  Test MSE:: 38.8328653242
(2, 4, 5, 9) Train MSE:: 36.4589528884  Test MSE:: 38.1396893989
(2, 4, 5, 10) Train MSE:: 33.6304542208  Test MSE:: 33.1508877563
(2, 4, 5, 11) Train MSE:: 35.8755483751  Test MSE:: 37.9948885228
(2, 4, 5, 12) Train MSE:: 29.3532121455  Test MSE:: 36.7853394275
(2, 4, 6, 7) Train MSE:: 57.7847867399  Test MSE:: 65.5879618895
(2, 4, 6, 8) Train MSE:: 61.837299256  Test MSE:: 70.3048081736
(2, 4, 6, 9) Train MSE:: 60.5717072443  Test MSE:: 68.5555937886
(2, 4, 6, 10) Train MSE:: 51.6547454332  Test MSE:: 57.0819390736
(2, 4, 6, 11) Train MSE:: 60.2745523131  Test MSE:: 69.9339536118
(2, 4, 6, 12) Train MSE:: 36.2037198207  Test MSE:: 42.8178698143
(2, 4, 7, 8) Train MSE:: 59.0142933892  Test MSE:: 64.8115924031
(2, 4, 7, 9) Train MSE:: 58.2013667491  Test MSE:: 63.4528542207
(2, 4, 7, 10) Train MSE:: 47.8318334068  Test MSE:: 52.4122186822
(2, 4, 7, 11) Train MSE:: 57.6071275493  Test MSE:: 64.2954163825
(2, 4, 7, 12) Train MSE:: 33.5585014468  Test MSE:: 41.1286893167
(2, 4, 8, 9) Train MSE:: 60.6484261957  Test MSE:: 66.7565198591
(2, 4, 8, 10) Train MSE:: 51.4762802658  Test MSE:: 56.1012051289
(2, 4, 8, 11) Train MSE:: 60.4218565588  Test MSE:: 68.8600915207
(2, 4, 8, 12) Train MSE:: 36.9139739342  Test MSE:: 44.7680925755
(2, 4, 9, 10) Train MSE:: 51.6147925332  Test MSE:: 56.7666250974
(2, 4, 9, 11) Train MSE:: 59.5273881165  Test MSE:: 67.5276148114
(2, 4, 9, 12) Train MSE:: 36.4991610252  Test MSE:: 43.8693696139
(2, 4, 10, 11) Train MSE:: 50.0867723092  Test MSE:: 56.4678463342
(2, 4, 10, 12) Train MSE:: 32.493292055  Test MSE:: 37.5941077355
(2, 4, 11, 12) Train MSE:: 36.640569214  Test MSE:: 43.9354774992
(2, 5, 6, 7) Train MSE:: 35.796249494  Test MSE:: 41.6586265176
(2, 5, 6, 8) Train MSE:: 36.6047380699  Test MSE:: 40.0992830489
(2, 5, 6, 9) Train MSE:: 35.8030334564  Test MSE:: 39.2823190178
(2, 5, 6, 10) Train MSE:: 33.906575746  Test MSE:: 36.6425537733
(2, 5, 6, 11) Train MSE:: 35.2295740146  Test MSE:: 39.4693339335
(2, 5, 6, 12) Train MSE:: 29.2872252917  Test MSE:: 35.893739534
(2, 5, 7, 8) Train MSE:: 36.6004989736  Test MSE:: 39.2048139516
(2, 5, 7, 9) Train MSE:: 35.9842758009  Test MSE:: 38.4034168996
(2, 5, 7, 10) Train MSE:: 34.3775838377  Test MSE:: 36.1070379549
(2, 5, 7, 11) Train MSE:: 35.4000778811  Test MSE:: 38.675258486
(2, 5, 7, 12) Train MSE:: 27.5204484176  Test MSE:: 35.7505436269
(2, 5, 8, 9) Train MSE:: 36.5222769836  Test MSE:: 38.700814836
(2, 5, 8, 10) Train MSE:: 34.4267557675  Test MSE:: 35.8162457746
(2, 5, 8, 11) Train MSE:: 35.4610060132  Test MSE:: 38.1462112788
(2, 5, 8, 12) Train MSE:: 29.2330734945  Test MSE:: 35.6533697486
(2, 5, 9, 10) Train MSE:: 33.8471025686  Test MSE:: 35.2489742054
(2, 5, 9, 11) Train MSE:: 34.8631653383  Test MSE:: 37.4960594974
(2, 5, 9, 12) Train MSE:: 28.7421085386  Test MSE:: 34.8636568539
(2, 5, 10, 11) Train MSE:: 32.2742193934  Test MSE:: 34.3234361043
(2, 5, 10, 12) Train MSE:: 26.4066042155  Test MSE:: 31.4962025449
(2, 5, 11, 12) Train MSE:: 28.65186443  Test MSE:: 34.4622635727
(2, 6, 7, 8) Train MSE:: 57.664244262  Test MSE:: 67.1746611593
(2, 6, 7, 9) Train MSE:: 56.7286549669  Test MSE:: 65.1689302268
(2, 6, 7, 10) Train MSE:: 49.8301917153  Test MSE:: 58.5789872554
(2, 6, 7, 11) Train MSE:: 56.2227755324  Test MSE:: 67.0581996438
(2, 6, 7, 12) Train MSE:: 33.6807572959  Test MSE:: 41.8571231038
(2, 6, 8, 9) Train MSE:: 60.1974770238  Test MSE:: 67.7597420479
(2, 6, 8, 10) Train MSE:: 52.8517227134  Test MSE:: 59.9788234146
(2, 6, 8, 11) Train MSE:: 60.0106426095  Test MSE:: 69.7393105248
(2, 6, 8, 12) Train MSE:: 36.2395434725  Test MSE:: 42.3735694738
(2, 6, 9, 10) Train MSE:: 52.4653919936  Test MSE:: 59.4224689023
(2, 6, 9, 11) Train MSE:: 59.0889862295  Test MSE:: 68.1702256133
(2, 6, 9, 12) Train MSE:: 35.9603103277  Test MSE:: 41.5824107564
(2, 6, 10, 11) Train MSE:: 50.7423731074  Test MSE:: 59.0432599491
(2, 6, 10, 12) Train MSE:: 31.6541535967  Test MSE:: 36.0254082376
(2, 6, 11, 12) Train MSE:: 36.0514724541  Test MSE:: 41.5679690118
(2, 7, 8, 9) Train MSE:: 59.4650320872  Test MSE:: 65.6568920605
(2, 7, 8, 10) Train MSE:: 52.3764848164  Test MSE:: 58.8348397107
(2, 7, 8, 11) Train MSE:: 58.6422124991  Test MSE:: 67.270534664
(2, 7, 8, 12) Train MSE:: 33.656879439  Test MSE:: 41.7007857874
(2, 7, 9, 10) Train MSE:: 51.8697878408  Test MSE:: 58.0783266253
(2, 7, 9, 11) Train MSE:: 57.8278616453  Test MSE:: 65.7001778921
(2, 7, 9, 12) Train MSE:: 33.3897576935  Test MSE:: 40.8760374395
(2, 7, 10, 11) Train MSE:: 49.9669285159  Test MSE:: 57.8842288621
(2, 7, 10, 12) Train MSE:: 29.4897478983  Test MSE:: 36.1389787594
(2, 7, 11, 12) Train MSE:: 33.4224202262  Test MSE:: 40.8736603307
(2, 8, 9, 10) Train MSE:: 52.1176981991  Test MSE:: 57.25124261
(2, 8, 9, 11) Train MSE:: 59.0849378015  Test MSE:: 66.7541595885
(2, 8, 9, 12) Train MSE:: 36.1623565863  Test MSE:: 42.7592873448
(2, 8, 10, 11) Train MSE:: 51.0791134009  Test MSE:: 58.5770079472
(2, 8, 10, 12) Train MSE:: 32.0055236195  Test MSE:: 37.4974550679
(2, 8, 11, 12) Train MSE:: 37.0623676257  Test MSE:: 43.4697895376
(2, 9, 10, 11) Train MSE:: 51.1780323579  Test MSE:: 58.5608501586
(2, 9, 10, 12) Train MSE:: 32.4959312468  Test MSE:: 37.5135685669
(2, 9, 11, 12) Train MSE:: 36.9249724069  Test MSE:: 42.907305489
(2, 10, 11, 12) Train MSE:: 32.2962173763  Test MSE:: 36.749197066
(3, 4, 5, 6) Train MSE:: 35.5468635715  Test MSE:: 46.7691301611
(3, 4, 5, 7) Train MSE:: 36.1237324498  Test MSE:: 41.7617910145
(3, 4, 5, 8) Train MSE:: 35.250839683  Test MSE:: 43.0922403945
(3, 4, 5, 9) Train MSE:: 34.439391124  Test MSE:: 42.248928807
(3, 4, 5, 10) Train MSE:: 31.9398426129  Test MSE:: 35.9961275403
(3, 4, 5, 11) Train MSE:: 34.2340607912  Test MSE:: 42.1231576502
(3, 4, 5, 12) Train MSE:: 27.6482262981  Test MSE:: 40.0363373798
(3, 4, 6, 7) Train MSE:: 59.8388153958  Test MSE:: 73.5650455371
(3, 4, 6, 8) Train MSE:: 60.6957900445  Test MSE:: 75.460954512
(3, 4, 6, 9) Train MSE:: 58.4658595517  Test MSE:: 71.7369537561
(3, 4, 6, 10) Train MSE:: 49.9332364753  Test MSE:: 57.5069275977
(3, 4, 6, 11) Train MSE:: 59.7081490157  Test MSE:: 75.5733108432
(3, 4, 6, 12) Train MSE:: 35.0351445446  Test MSE:: 44.3184589159
(3, 4, 7, 8) Train MSE:: 60.9272695682  Test MSE:: 70.9118740971
(3, 4, 7, 9) Train MSE:: 58.4777968625  Test MSE:: 67.2867670395
(3, 4, 7, 10) Train MSE:: 47.7417449961  Test MSE:: 52.4585727605
(3, 4, 7, 11) Train MSE:: 60.1942662467  Test MSE:: 70.7786221275
(3, 4, 7, 12) Train MSE:: 33.3972765898  Test MSE:: 42.0491116034
(3, 4, 8, 9) Train MSE:: 58.8422072691  Test MSE:: 68.4703122597
(3, 4, 8, 10) Train MSE:: 49.9628352438  Test MSE:: 55.8482721011
(3, 4, 8, 11) Train MSE:: 60.2888003844  Test MSE:: 72.8209154307
(3, 4, 8, 12) Train MSE:: 35.3595412652  Test MSE:: 46.1754130393
(3, 4, 9, 10) Train MSE:: 49.9298088878  Test MSE:: 56.5558261652
(3, 4, 9, 11) Train MSE:: 58.3287527682  Test MSE:: 69.9029998722
(3, 4, 9, 12) Train MSE:: 34.8075095407  Test MSE:: 45.45339105
(3, 4, 10, 11) Train MSE:: 48.629423604  Test MSE:: 56.288474933
(3, 4, 10, 12) Train MSE:: 31.0917074443  Test MSE:: 38.903450107
(3, 4, 11, 12) Train MSE:: 35.1643502737  Test MSE:: 45.4430625305
(3, 5, 6, 7) Train MSE:: 35.6659686946  Test MSE:: 50.9889727251
(3, 5, 6, 8) Train MSE:: 34.3351133067  Test MSE:: 46.3979827479
(3, 5, 6, 9) Train MSE:: 33.5543440824  Test MSE:: 44.6594374321
(3, 5, 6, 10) Train MSE:: 32.4834998779  Test MSE:: 42.0741671002
(3, 5, 6, 11) Train MSE:: 33.3024268431  Test MSE:: 46.3886111904
(3, 5, 6, 12) Train MSE:: 27.695174412  Test MSE:: 40.5351946886
(3, 5, 7, 8) Train MSE:: 36.3642054383  Test MSE:: 45.6174003959
(3, 5, 7, 9) Train MSE:: 34.9362401082  Test MSE:: 43.4875567086
(3, 5, 7, 10) Train MSE:: 35.2095089767  Test MSE:: 41.1226126311
(3, 5, 7, 11) Train MSE:: 35.9866391736  Test MSE:: 46.06720866
(3, 5, 7, 12) Train MSE:: 27.2392550206  Test MSE:: 39.7889382032
(3, 5, 8, 9) Train MSE:: 34.9260660334  Test MSE:: 43.5266246066
(3, 5, 8, 10) Train MSE:: 33.9452288367  Test MSE:: 40.3379585043
(3, 5, 8, 11) Train MSE:: 34.3967703398  Test MSE:: 43.7046669748
(3, 5, 8, 12) Train MSE:: 27.4035158364  Test MSE:: 39.6072896088
(3, 5, 9, 10) Train MSE:: 32.7535216849  Test MSE:: 39.0168356882
(3, 5, 9, 11) Train MSE:: 33.3245036954  Test MSE:: 42.2114453417
(3, 5, 9, 12) Train MSE:: 26.9919994446  Test MSE:: 38.9344087146
(3, 5, 10, 11) Train MSE:: 31.9942630153  Test MSE:: 38.3837683212
(3, 5, 10, 12) Train MSE:: 25.1060222464  Test MSE:: 34.6000723135
(3, 5, 11, 12) Train MSE:: 26.9461754396  Test MSE:: 38.5301578153
(3, 6, 7, 8) Train MSE:: 61.07221896  Test MSE:: 79.2354452213
(3, 6, 7, 9) Train MSE:: 57.5583254346  Test MSE:: 72.0948636485
(3, 6, 7, 10) Train MSE:: 53.2115698544  Test MSE:: 66.6046952989
(3, 6, 7, 11) Train MSE:: 61.0841304706  Test MSE:: 82.3330177797
(3, 6, 7, 12) Train MSE:: 34.3248008145  Test MSE:: 44.9056673944
(3, 6, 8, 9) Train MSE:: 58.2772378068  Test MSE:: 72.0265625361
(3, 6, 8, 10) Train MSE:: 53.2622040136  Test MSE:: 65.5703999929
(3, 6, 8, 11) Train MSE:: 59.9601050834  Test MSE:: 78.21452822
(3, 6, 8, 12) Train MSE:: 35.0807950821  Test MSE:: 44.8217373237
(3, 6, 9, 10) Train MSE:: 51.6669702936  Test MSE:: 62.5322638374
(3, 6, 9, 11) Train MSE:: 57.5246207007  Test MSE:: 72.8895455141
(3, 6, 9, 12) Train MSE:: 34.4535306506  Test MSE:: 43.6780735158
(3, 6, 10, 11) Train MSE:: 50.7351510992  Test MSE:: 63.7635276748
(3, 6, 10, 12) Train MSE:: 30.7270248455  Test MSE:: 37.5190609524
(3, 6, 11, 12) Train MSE:: 34.9183115809  Test MSE:: 44.1916390876
(3, 7, 8, 9) Train MSE:: 61.3289799021  Test MSE:: 72.525793561
(3, 7, 8, 10) Train MSE:: 57.1462997877  Test MSE:: 66.5600143319
(3, 7, 8, 11) Train MSE:: 64.3336069803  Test MSE:: 80.1983098407
(3, 7, 8, 12) Train MSE:: 34.0521097474  Test MSE:: 43.6541215107
(3, 7, 9, 10) Train MSE:: 54.2923613551  Test MSE:: 62.7694657855
(3, 7, 9, 11) Train MSE:: 60.3761553734  Test MSE:: 73.3992268116
(3, 7, 9, 12) Train MSE:: 33.082142029  Test MSE:: 42.1868453172
(3, 7, 10, 11) Train MSE:: 54.3453290141  Test MSE:: 64.8421880629
(3, 7, 10, 12) Train MSE:: 29.6979361041  Test MSE:: 36.8056079478
(3, 7, 11, 12) Train MSE:: 33.9906415365  Test MSE:: 43.1300914826
(3, 8, 9, 10) Train MSE:: 52.8594485148  Test MSE:: 59.9086368005
(3, 8, 9, 11) Train MSE:: 59.4142760518  Test MSE:: 71.385092821
(3, 8, 9, 12) Train MSE:: 34.212154086  Test MSE:: 45.024574548
(3, 8, 10, 11) Train MSE:: 54.5385797318  Test MSE:: 65.0219175606
(3, 8, 10, 12) Train MSE:: 30.7963970728  Test MSE:: 38.7053836062
(3, 8, 11, 12) Train MSE:: 35.1650956716  Test MSE:: 45.4148601741
(3, 9, 10, 11) Train MSE:: 52.3520320676  Test MSE:: 61.9072258522
(3, 9, 10, 12) Train MSE:: 31.1059814268  Test MSE:: 38.9804613656
(3, 9, 11, 12) Train MSE:: 34.8495689456  Test MSE:: 45.0014532701
(3, 10, 11, 12) Train MSE:: 30.9316645785  Test MSE:: 38.3735642792
(4, 5, 6, 7) Train MSE:: 36.5498201871  Test MSE:: 39.9872296585
(4, 5, 6, 8) Train MSE:: 36.8092318244  Test MSE:: 40.3115640747
(4, 5, 6, 9) Train MSE:: 35.7945670678  Test MSE:: 39.5631649616
(4, 5, 6, 10) Train MSE:: 33.426407528  Test MSE:: 34.1423993957
(4, 5, 6, 11) Train MSE:: 35.5599797937  Test MSE:: 39.6386574236
(4, 5, 6, 12) Train MSE:: 29.5010608736  Test MSE:: 35.6991866581
(4, 5, 7, 8) Train MSE:: 36.8216419645  Test MSE:: 36.9968307122
(4, 5, 7, 9) Train MSE:: 35.7149277899  Test MSE:: 36.3001422994
(4, 5, 7, 10) Train MSE:: 32.1061012525  Test MSE:: 30.6067025321
(4, 5, 7, 11) Train MSE:: 35.7983796098  Test MSE:: 36.02053599
(4, 5, 7, 12) Train MSE:: 28.0255767916  Test MSE:: 33.7258561105
(4, 5, 8, 9) Train MSE:: 36.4582598249  Test MSE:: 37.901666809
(4, 5, 8, 10) Train MSE:: 33.6229112093  Test MSE:: 33.0917670738
(4, 5, 8, 11) Train MSE:: 35.7728333139  Test MSE:: 37.3690213566
(4, 5, 8, 12) Train MSE:: 29.2131769048  Test MSE:: 35.7859362167
(4, 5, 9, 10) Train MSE:: 33.215527336  Test MSE:: 33.270718761
(4, 5, 9, 11) Train MSE:: 34.918683562  Test MSE:: 36.98564632
(4, 5, 9, 12) Train MSE:: 28.5767909794  Test MSE:: 35.485857759
(4, 5, 10, 11) Train MSE:: 31.6844225729  Test MSE:: 32.1285838168
(4, 5, 10, 12) Train MSE:: 26.3905165737  Test MSE:: 31.3028658165
(4, 5, 11, 12) Train MSE:: 28.6774931144  Test MSE:: 34.5229777136
(4, 6, 7, 8) Train MSE:: 61.9384803366  Test MSE:: 72.0858925517
(4, 6, 7, 9) Train MSE:: 59.098956388  Test MSE:: 67.5152467636
(4, 6, 7, 10) Train MSE:: 48.3759410813  Test MSE:: 54.9728277828
(4, 6, 7, 11) Train MSE:: 61.0090243799  Test MSE:: 72.200469321
(4, 6, 7, 12) Train MSE:: 35.1873071688  Test MSE:: 40.1515824289
(4, 6, 8, 9) Train MSE:: 61.2073864524  Test MSE:: 68.9927407811
(4, 6, 8, 10) Train MSE:: 52.1358533974  Test MSE:: 57.2167924266
(4, 6, 8, 11) Train MSE:: 62.9309793367  Test MSE:: 73.9096610895
(4, 6, 8, 12) Train MSE:: 36.9423060661  Test MSE:: 41.8219196512
(4, 6, 9, 10) Train MSE:: 51.9893367766  Test MSE:: 57.6454883018
(4, 6, 9, 11) Train MSE:: 60.5990099223  Test MSE:: 70.3612025431
(4, 6, 9, 12) Train MSE:: 36.2000659351  Test MSE:: 41.3381980608
(4, 6, 10, 11) Train MSE:: 50.5116316776  Test MSE:: 57.3188713872
(4, 6, 10, 12) Train MSE:: 31.5751285877  Test MSE:: 34.6549594038
(4, 6, 11, 12) Train MSE:: 36.7353204227  Test MSE:: 41.0685034189
(4, 7, 8, 9) Train MSE:: 60.7353333123  Test MSE:: 65.5647914689
(4, 7, 8, 10) Train MSE:: 49.3676604362  Test MSE:: 53.1901159224
(4, 7, 8, 11) Train MSE:: 62.7023249966  Test MSE:: 70.6293041301
(4, 7, 8, 12) Train MSE:: 35.0826228699  Test MSE:: 40.1646010029
(4, 7, 9, 10) Train MSE:: 49.305567416  Test MSE:: 53.7032614594
(4, 7, 9, 11) Train MSE:: 60.1636133074  Test MSE:: 67.1047320289
(4, 7, 9, 12) Train MSE:: 34.290926019  Test MSE:: 39.4250798973
(4, 7, 10, 11) Train MSE:: 47.8780828209  Test MSE:: 53.3555082007
(4, 7, 10, 12) Train MSE:: 29.105127917  Test MSE:: 33.0711835047
(4, 7, 11, 12) Train MSE:: 34.9016589632  Test MSE:: 39.3768246818
(4, 8, 9, 10) Train MSE:: 50.1942912591  Test MSE:: 53.8588650874
(4, 8, 9, 11) Train MSE:: 60.3072240122  Test MSE:: 67.4158748771
(4, 8, 9, 12) Train MSE:: 35.8426867137  Test MSE:: 42.8736693665
(4, 8, 10, 11) Train MSE:: 50.0331900977  Test MSE:: 55.6859307707
(4, 8, 10, 12) Train MSE:: 31.9984114747  Test MSE:: 36.8420741525
(4, 8, 11, 12) Train MSE:: 37.201707448  Test MSE:: 42.930545654
(4, 9, 10, 11) Train MSE:: 50.5806176445  Test MSE:: 56.7233860187
(4, 9, 10, 12) Train MSE:: 32.5019829235  Test MSE:: 37.3947796821
(4, 9, 11, 12) Train MSE:: 36.577254271  Test MSE:: 42.5319990515
(4, 10, 11, 12) Train MSE:: 32.2827403924  Test MSE:: 36.7744798942
(5, 6, 7, 8) Train MSE:: 35.3806269616  Test MSE:: 41.0067948792
(5, 6, 7, 9) Train MSE:: 34.1169546963  Test MSE:: 39.243888938
(5, 6, 7, 10) Train MSE:: 33.4643262596  Test MSE:: 38.3041605007
(5, 6, 7, 11) Train MSE:: 34.5330380331  Test MSE:: 41.3105850836
(5, 6, 7, 12) Train MSE:: 28.4952933212  Test MSE:: 36.5386881911
(5, 6, 8, 9) Train MSE:: 35.7431877289  Test MSE:: 39.2768314203
(5, 6, 8, 10) Train MSE:: 33.836147434  Test MSE:: 36.7005161803
(5, 6, 8, 11) Train MSE:: 34.9390269335  Test MSE:: 39.0619394287
(5, 6, 8, 12) Train MSE:: 29.1881126121  Test MSE:: 35.2288083808
(5, 6, 9, 10) Train MSE:: 33.123278646  Test MSE:: 35.7533079198
(5, 6, 9, 11) Train MSE:: 34.2138256155  Test MSE:: 37.9972297519
(5, 6, 9, 12) Train MSE:: 28.6267402833  Test MSE:: 34.5332789326
(5, 6, 10, 11) Train MSE:: 31.6663050533  Test MSE:: 35.0171653892
(5, 6, 10, 12) Train MSE:: 26.3239973819  Test MSE:: 31.1111016311
(5, 6, 11, 12) Train MSE:: 28.6669775671  Test MSE:: 34.1703611073
(5, 7, 8, 9) Train MSE:: 36.6197896801  Test MSE:: 38.8090855745
(5, 7, 8, 10) Train MSE:: 35.362046335  Test MSE:: 36.3490872435
(5, 7, 8, 11) Train MSE:: 36.195426736  Test MSE:: 38.7304741096
(5, 7, 8, 12) Train MSE:: 28.0532720284  Test MSE:: 34.6001653146
(5, 7, 9, 10) Train MSE:: 34.0017085858  Test MSE:: 35.3259084147
(5, 7, 9, 11) Train MSE:: 34.8427961009  Test MSE:: 37.4812788402
(5, 7, 9, 12) Train MSE:: 27.1836521006  Test MSE:: 33.7773025881
(5, 7, 10, 11) Train MSE:: 33.1234984798  Test MSE:: 34.7330193192
(5, 7, 10, 12) Train MSE:: 25.3648750089  Test MSE:: 30.8370533574
(5, 7, 11, 12) Train MSE:: 27.748231157  Test MSE:: 33.6320456988
(5, 8, 9, 10) Train MSE:: 33.8070451898  Test MSE:: 35.0580077397
(5, 8, 9, 11) Train MSE:: 34.872048995  Test MSE:: 37.5186935313
(5, 8, 9, 12) Train MSE:: 28.5328370323  Test MSE:: 35.1403284907
(5, 8, 10, 11) Train MSE:: 32.8792003321  Test MSE:: 34.5495222394
(5, 8, 10, 12) Train MSE:: 26.3884904791  Test MSE:: 31.517170514
(5, 8, 11, 12) Train MSE:: 28.6235470295  Test MSE:: 34.1935123234
(5, 9, 10, 11) Train MSE:: 32.1635014963  Test MSE:: 33.9571778676
(5, 9, 10, 12) Train MSE:: 26.3241200196  Test MSE:: 31.3678859676
(5, 9, 11, 12) Train MSE:: 28.2997117257  Test MSE:: 33.872532276
(5, 10, 11, 12) Train MSE:: 25.7444167198  Test MSE:: 30.1004061101
(6, 7, 8, 9) Train MSE:: 59.8752699345  Test MSE:: 68.896667626
(6, 7, 8, 10) Train MSE:: 54.6506766259  Test MSE:: 64.6346343641
(6, 7, 8, 11) Train MSE:: 62.3055343997  Test MSE:: 75.9440359111
(6, 7, 8, 12) Train MSE:: 35.6845263355  Test MSE:: 41.4727196797
(6, 7, 9, 10) Train MSE:: 52.3563282089  Test MSE:: 60.9579745045
(6, 7, 9, 11) Train MSE:: 58.8700081139  Test MSE:: 69.738070549
(6, 7, 9, 12) Train MSE:: 34.5431965224  Test MSE:: 40.1391274368
(6, 7, 10, 11) Train MSE:: 51.6983006947  Test MSE:: 62.9297328449
(6, 7, 10, 12) Train MSE:: 30.6522306979  Test MSE:: 35.433472316
(6, 7, 11, 12) Train MSE:: 35.5852541813  Test MSE:: 40.8305530004
(6, 8, 9, 10) Train MSE:: 52.0898733272  Test MSE:: 58.7258493095
(6, 8, 9, 11) Train MSE:: 59.6103489787  Test MSE:: 69.2104052475
(6, 8, 9, 12) Train MSE:: 35.3103264513  Test MSE:: 40.9165865572
(6, 8, 10, 11) Train MSE:: 52.7040213779  Test MSE:: 62.6240430143
(6, 8, 10, 12) Train MSE:: 31.6778887025  Test MSE:: 35.6392086097
(6, 8, 11, 12) Train MSE:: 36.7411422853  Test MSE:: 41.2847516025
(6, 9, 10, 11) Train MSE:: 51.8932440651  Test MSE:: 60.6790302946
(6, 9, 10, 12) Train MSE:: 31.8303392052  Test MSE:: 35.6254908169
(6, 9, 11, 12) Train MSE:: 36.1161127023  Test MSE:: 40.7095440599
(6, 10, 11, 12) Train MSE:: 31.6242897601  Test MSE:: 34.9002880058
(7, 8, 9, 10) Train MSE:: 54.1507087658  Test MSE:: 59.0563124729
(7, 8, 9, 11) Train MSE:: 61.7278725448  Test MSE:: 69.7893452346
(7, 8, 9, 12) Train MSE:: 33.7558068457  Test MSE:: 39.7891974696
(7, 8, 10, 11) Train MSE:: 55.7279055307  Test MSE:: 63.7317273796
(7, 8, 10, 12) Train MSE:: 30.6311285569  Test MSE:: 35.2703716141
(7, 8, 11, 12) Train MSE:: 35.4172908284  Test MSE:: 40.3358579844
(7, 9, 10, 11) Train MSE:: 53.7493187954  Test MSE:: 61.1076103852
(7, 9, 10, 12) Train MSE:: 30.4031061326  Test MSE:: 35.0133610014
(7, 9, 11, 12) Train MSE:: 34.4163027755  Test MSE:: 39.4562216648
(7, 10, 11, 12) Train MSE:: 30.2711120719  Test MSE:: 34.190254353
(8, 9, 10, 11) Train MSE:: 51.4771507043  Test MSE:: 57.8608283317
(8, 9, 10, 12) Train MSE:: 31.0129705094  Test MSE:: 36.5078065869
(8, 9, 11, 12) Train MSE:: 35.9710592641  Test MSE:: 42.2056372907
(8, 10, 11, 12) Train MSE:: 31.6242250851  Test MSE:: 35.9381544222
(9, 10, 11, 12) Train MSE:: 32.2696755873  Test MSE:: 36.6837064306
(5, 10, 11, 12) 30.1004061101

3.4 Feature expansion





In [22]:



    


Z_train = trainX
Z_test = testX

for i in range(n_attrs):
    xi_train = np.array([trainX[:, i]]).transpose()
    xi_test = np.array([testX[:, i]]).transpose()
    
    for j in range(0, i+1):
        xj_train = np.array([trainX[:, j]]).transpose()
        xj_test = np.array([testX[:, j]]).transpose()

        Z_train = np.concatenate((Z_train, xi_train * xj_train), axis=1)
        Z_test = np.concatenate((Z_test, xi_test * xj_test), axis=1)

print('Train Shape:', Z_train.shape)
print('Test Shape:', Z_test.shape)
linreg = LinearRegression(Z_train, trainY, alpha)
pred_Y = linreg.predict(Z_train)
train_mse_cost = MSECost(pred_Y, trainY)
test_mse_cost = MSECost(linreg.predict(Z_test), testY)

print('Train MSE::', train_mse_cost, " Test MSE::", test_mse_cost)



    


















    






Train Shape: (433, 104)
Test Shape: (73, 104)
Train MSE:: 6.68639815515  Test MSE:: 21.0152571898

















In [17]:



    


np.arange(1, 10) *  np.arange(11, 20)



    


















    
Out[17]:








array([ 11,  24,  39,  56,  75,  96, 119, 144, 171])

















In [ ]:

Content source: thammegowda/notes

Similar notebooks:

notebook.community | gallery | about