# Ch `03`: Concept `03`

## Regularization

Import the relevant libraries and initialize the hyper-parameters

``````

In [1]:

import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt

np.random.seed(100)

learning_rate = 0.001
training_epochs = 1000
reg_lambda = 0.

``````

create a helper method to split the dataset

``````

In [2]:

def split_dataset(x_dataset, y_dataset, ratio):
arr = np.arange(x_dataset.size)
np.random.shuffle(arr)
num_train = int(ratio * x_dataset.size)
x_train = x_dataset[arr[0:num_train]]
y_train = y_dataset[arr[0:num_train]]
x_test = x_dataset[arr[num_train:x_dataset.size]]
y_test = y_dataset[arr[num_train:x_dataset.size]]
return x_train, x_test, y_train, y_test

``````

Create a fake dataset. y = x^2

``````

In [3]:

x_dataset = np.linspace(-1, 1, 100)

num_coeffs = 9
y_dataset_params = [0.] * num_coeffs
y_dataset_params[2] = 1
y_dataset = 0
for i in range(num_coeffs):
y_dataset += y_dataset_params[i] * np.power(x_dataset, i)
y_dataset += np.random.randn(*x_dataset.shape) * 0.3

``````

Split the dataset into 70% training and testing 30%

``````

In [4]:

(x_train, x_test, y_train, y_test) = split_dataset(x_dataset, y_dataset, 0.7)

``````

Set up the input/output placeholders

``````

In [5]:

X = tf.placeholder("float")
Y = tf.placeholder("float")

``````

Define our model

``````

In [6]:

def model(X, w):
terms = []
for i in range(num_coeffs):
term = tf.multiply(w[i], tf.pow(X, i))
terms.append(term)

``````

Define the regularized cost function

``````

In [7]:

w = tf.Variable([0.] * num_coeffs, name="parameters")
y_model = model(X, w)
tf.multiply(reg_lambda, tf.reduce_sum(tf.square(w)))),
2*x_train.size)

``````

Set up the session

``````

In [8]:

sess = tf.Session()
init = tf.global_variables_initializer()
sess.run(init)

``````

Try out various regularization parameters

``````

In [9]:

for reg_lambda in np.linspace(0,1,100):
for epoch in range(training_epochs):
sess.run(train_op, feed_dict={X: x_train, Y: y_train})
final_cost = sess.run(cost, feed_dict={X: x_test, Y:y_test})
print('reg lambda', reg_lambda)
print('final cost', final_cost)

``````
``````

reg lambda 0.0
final cost 0.032031
reg lambda 0.010101010101
final cost 0.0289025
reg lambda 0.020202020202
final cost 0.0271288
reg lambda 0.030303030303
final cost 0.0258003
reg lambda 0.040404040404
final cost 0.024813
reg lambda 0.0505050505051
final cost 0.0240771
reg lambda 0.0606060606061
final cost 0.0235204
reg lambda 0.0707070707071
final cost 0.0230916
reg lambda 0.0808080808081
final cost 0.022755
reg lambda 0.0909090909091
final cost 0.0224858
reg lambda 0.10101010101
final cost 0.0222666
reg lambda 0.111111111111
final cost 0.0220851
reg lambda 0.121212121212
final cost 0.0219322
reg lambda 0.131313131313
final cost 0.0218015
reg lambda 0.141414141414
final cost 0.0216881
reg lambda 0.151515151515
final cost 0.0215885
reg lambda 0.161616161616
final cost 0.0214999
reg lambda 0.171717171717
final cost 0.0214203
reg lambda 0.181818181818
final cost 0.0213481
reg lambda 0.191919191919
final cost 0.0212821
reg lambda 0.20202020202
final cost 0.0212215
reg lambda 0.212121212121
final cost 0.0211654
reg lambda 0.222222222222
final cost 0.0211133
reg lambda 0.232323232323
final cost 0.0210647
reg lambda 0.242424242424
final cost 0.0210192
reg lambda 0.252525252525
final cost 0.0209767
reg lambda 0.262626262626
final cost 0.0209367
reg lambda 0.272727272727
final cost 0.0208992
reg lambda 0.282828282828
final cost 0.0208639
reg lambda 0.292929292929
final cost 0.0208308
reg lambda 0.30303030303
final cost 0.0207997
reg lambda 0.313131313131
final cost 0.0207705
reg lambda 0.323232323232
final cost 0.0207431
reg lambda 0.333333333333
final cost 0.0207173
reg lambda 0.343434343434
final cost 0.0206934
reg lambda 0.353535353535
final cost 0.0206709
reg lambda 0.363636363636
final cost 0.0206499
reg lambda 0.373737373737
final cost 0.0206305
reg lambda 0.383838383838
final cost 0.0206125
reg lambda 0.393939393939
final cost 0.0205957
reg lambda 0.40404040404
final cost 0.0205804
reg lambda 0.414141414141
final cost 0.0205663
reg lambda 0.424242424242
final cost 0.0205534
reg lambda 0.434343434343
final cost 0.0205416
reg lambda 0.444444444444
final cost 0.0205311
reg lambda 0.454545454545
final cost 0.0205216
reg lambda 0.464646464646
final cost 0.0205132
reg lambda 0.474747474747
final cost 0.0205057
reg lambda 0.484848484848
final cost 0.0204994
reg lambda 0.494949494949
final cost 0.020494
reg lambda 0.505050505051
final cost 0.0204894
reg lambda 0.515151515152
final cost 0.0204858
reg lambda 0.525252525253
final cost 0.020483
reg lambda 0.535353535354
final cost 0.0204811
reg lambda 0.545454545455
final cost 0.02048
reg lambda 0.555555555556
final cost 0.0204795
reg lambda 0.565656565657
final cost 0.0204799
reg lambda 0.575757575758
final cost 0.020481
reg lambda 0.585858585859
final cost 0.0204828
reg lambda 0.59595959596
final cost 0.0204852
reg lambda 0.606060606061
final cost 0.0204882
reg lambda 0.616161616162
final cost 0.0204919
reg lambda 0.626262626263
final cost 0.0204963
reg lambda 0.636363636364
final cost 0.0205011
reg lambda 0.646464646465
final cost 0.0205065
reg lambda 0.656565656566
final cost 0.0205124
reg lambda 0.666666666667
final cost 0.0205188
reg lambda 0.676767676768
final cost 0.0205258
reg lambda 0.686868686869
final cost 0.0205331
reg lambda 0.69696969697
final cost 0.020541
reg lambda 0.707070707071
final cost 0.0205492
reg lambda 0.717171717172
final cost 0.0205578
reg lambda 0.727272727273
final cost 0.0205668
reg lambda 0.737373737374
final cost 0.0205764
reg lambda 0.747474747475
final cost 0.0205861
reg lambda 0.757575757576
final cost 0.0205963
reg lambda 0.767676767677
final cost 0.0206067
reg lambda 0.777777777778
final cost 0.0206174
reg lambda 0.787878787879
final cost 0.0206286
reg lambda 0.79797979798
final cost 0.0206399
reg lambda 0.808080808081
final cost 0.0206516
reg lambda 0.818181818182
final cost 0.0206635
reg lambda 0.828282828283
final cost 0.0206756
reg lambda 0.838383838384
final cost 0.020688
reg lambda 0.848484848485
final cost 0.0207005
reg lambda 0.858585858586
final cost 0.0207134
reg lambda 0.868686868687
final cost 0.0207264
reg lambda 0.878787878788
final cost 0.0207396
reg lambda 0.888888888889
final cost 0.0207529
reg lambda 0.89898989899
final cost 0.0207665
reg lambda 0.909090909091
final cost 0.0207801
reg lambda 0.919191919192
final cost 0.020794
reg lambda 0.929292929293
final cost 0.020808
reg lambda 0.939393939394
final cost 0.0208222
reg lambda 0.949494949495
final cost 0.0208364
reg lambda 0.959595959596
final cost 0.0208508
reg lambda 0.969696969697
final cost 0.0208652
reg lambda 0.979797979798
final cost 0.0208798
reg lambda 0.989898989899
final cost 0.0208943
reg lambda 1.0
final cost 0.0209091

``````

Close the session

``````

In [10]:

sess.close()

``````