In [235]:
%matplotlib inline
import numpy as np
from matplotlib import pyplot as plt
In [236]:
# Create a random X axis points.
np.random.seed(1)
X = np.random.random(4)
print X
In [237]:
def lin(a, b, x): return a*x + b
In [238]:
# We define the slope "a"
a = 10.
# We define the y-intercept var "b"
b = 3.
# We define the len of the X.
n = len(X)
In [239]:
# Create y axis points which follows a linear form.
y = lin(a, b, X)
In [240]:
plt.scatter(X, y)
Out[240]:
In [241]:
# We choose the Least Square Errors (lse) as cost function.
# But we return the sum of all those errors for measuring the performance.
def sse(y, y_pred): return ((y-y_pred)**2).sum()
In [242]:
# The loss function is the comparison of a perfect regression line with a line created using
# our guessed "a" and "b" using the cost function.
def loss(y, a, b, X): return sse(y, lin(a, b, X))
In [243]:
# This function represents the average loss per point.
def avg_loss(y, a, b, X): return np.sqrt(loss(y, a, b, X)/n)
In [244]:
# We define the random numbers that will conform the initial approximation line.
a_guess = -1
b_guess = 2
In [245]:
# As we can see we get a really bad approximation at the beginning.
avg_loss(y, a_guess, b_guess, X)
Out[245]:
In [246]:
lr=0.6 # This is the learning rate
In [247]:
# The partial derivatives of the cost function (lse) are:
# d[(y-(a*x+b))**2,b] = 2 (b + a x - y) = 2 (y_pred - y)
# d[(y-(a*x+b))**2,a] = 2 x (b + a x - y) = x * dy/db
# This tells us how much is changing df with regards of "b" and "a" respectively.
In [251]:
def update():
print '-----------------------------------------------'
global a_guess, b_guess
# We calculate the current "guessed" line based on the iterative process.
y_pred = lin(a_guess, b_guess, X)
# Now, in order to iterate again we need to calculate how much the
# derivative of the cost function changes depending on "a" and "b".
# d[(y-(a*x+b))**2,a] = 2 x (b + a x - y) = x * dy/db
# d[(y-(a*x+b))**2,b] = 2 (b + a x - y) = 2 (y_pred - y)
dydb = 2 * (y_pred - y)
dyda = X * dydb
print 'dy/da: {}'.format(dyda)
print 'dy/db: {}'.format(dydb)
print 'Current a_guess = {}'.format(a_guess)
print 'Current b_guess = {}'.format(b_guess)
# Because these changes always tell us how to go higher instead than
# lower, we need to substract it from the previous
# guesses in order to approach the minimum. Please notice how the speed is
# controlled by the defined learning rate.
print 'Formula a: a_guess -= lr * dyda.mean()'
print 'Formula b: b_guess -= lr * dydb.mean()'
print '* Where:'
print 'Learning rate = {}'.format(lr)
print 'dyda.mean() = {}'.format(dyda.mean())
print 'dydb.mean() = {}'.format(dydb.mean())
print '* Result:'
a_guess -= lr * dyda.mean()
b_guess -= lr * dydb.mean()
print 'New a_guess = {}'.format(a_guess)
print 'New b_guess = {}'.format(b_guess)
In [252]:
plt.plot(X, y, 'ro')
line, = plt.plot(X, lin(a_guess, b_guess, X))
In [253]:
# We can see how why are getting better.
for i in range(3):
update()
avg_loss(y, a_guess, b_guess, X)
Out[253]:
In [230]:
plt.plot(X, y, 'ro')
line, = plt.plot(X, lin(a_guess, b_guess, X))
In [231]:
# And even better!
for i in range(3):
update()
avg_loss(y, a_guess, b_guess, X)
Out[231]:
In [232]:
plt.plot(X, y, 'ro')
line, = plt.plot(X, lin(a_guess, b_guess, X))
In [233]:
# And even better!
for i in range(15):
update()
avg_loss(y, a_guess, b_guess, X)
Out[233]:
In [234]:
plt.plot(X, y, 'ro')
line, = plt.plot(X, lin(a_guess, b_guess, X))