Try not to peek at the solutions when you go through the exercises. ;-)
First let's make sure this notebook works well in both Python 2 and Python 3:
In [ ]:
from __future__ import absolute_import, division, print_function, unicode_literals
In [ ]:
import tensorflow as tf
tf.__version__
From notebook 4 linear regression
In [ ]:
import numpy as np
data = np.loadtxt("data/life_satisfaction.csv",
dtype=np.float32,
delimiter=",",
skiprows=1,
usecols=[1, 2])
X_train = data[:, 0:1] / 10000 # feature scaling
y_train = data[:, 1:2]
learning_rate = 0.01
In [ ]:
%matplotlib inline
import matplotlib.pyplot as plt
plt.rcParams['axes.labelsize'] = 14
plt.rcParams['xtick.labelsize'] = 12
plt.rcParams['ytick.labelsize'] = 12
In [ ]:
def plot_life_satisfaction(X_train, y_train):
plt.plot(X_train * 10000, y_train, "bo")
plt.axis([0, 60000, 0, 10])
plt.xlabel("GDP per capita ($)")
plt.ylabel("Life Satisfaction")
plt.grid()
def plot_life_satisfaction_with_linear_model(X_train, y_train, w, b):
plot_life_satisfaction(X_train, y_train)
plt.plot([0, 60000], [b, w[0][0] * (60000 / 10000) + b])
In [ ]:
graph = tf.Graph()
with graph.as_default():
X = tf.constant(X_train, dtype=tf.float32, name="X")
y = tf.constant(y_train, dtype=tf.float32, name="y")
b = tf.Variable(0.0, name="b")
w = tf.Variable(tf.zeros([1, 1]), name="w")
y_pred = tf.add(tf.matmul(X, w), b, name="y_pred") # X @ w + b
mse = tf.reduce_mean(tf.square(y_pred - y), name="mse")
gradients_w, gradients_b = tf.gradients(mse, [w, b]) # <= IT'S AUTODIFF MAGIC!
tweak_w_op = tf.assign(w, w - learning_rate * gradients_w)
tweak_b_op = tf.assign(b, b - learning_rate * gradients_b)
training_op = tf.group(tweak_w_op, tweak_b_op)
init = tf.global_variables_initializer()
In [ ]:
n_iterations = 2000
with tf.Session(graph=graph) as sess:
init.run()
for iteration in range(n_iterations):
if iteration % 100 == 0:
print("Iteration {:5}, MSE: {:.4f}".format(iteration, mse.eval()))
training_op.run()
w_val, b_val = sess.run([w, b])
In [ ]:
plt.figure(figsize=(10, 5))
plot_life_satisfaction_with_linear_model(X_train, y_train, w_val, b_val)
plt.show()
In [ ]:
graph = tf.Graph()
with graph.as_default():
X = tf.constant(X_train, dtype=tf.float32, name="X")
y = tf.constant(y_train, dtype=tf.float32, name="y")
b = tf.Variable(0.0, name="b")
w = tf.Variable(tf.zeros([1, 1]), name="w")
y_pred = tf.add(tf.matmul(X, w), b, name="y_pred") # X @ w + b
mse = tf.reduce_mean(tf.square(y_pred - y), name="mse")
optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)
training_op = optimizer.minimize(mse) # <= MOAR AUTODIFF MAGIC!
init = tf.global_variables_initializer()
In [ ]:
n_iterations = 2000
with tf.Session(graph=graph) as sess:
init.run()
for iteration in range(n_iterations):
if iteration % 100 == 0:
print("Iteration {:5}, MSE: {:.4f}".format(iteration, mse.eval()))
training_op.run()
w_val, b_val = sess.run([w, b])
In [ ]:
plt.figure(figsize=(10, 5))
plot_life_satisfaction_with_linear_model(X_train, y_train, w_val, b_val)
plt.show()
In [ ]:
learning_rate = 0.01
momentum = 0.8
graph = tf.Graph()
with graph.as_default():
X = tf.constant(X_train, dtype=tf.float32, name="X")
y = tf.constant(y_train, dtype=tf.float32, name="y")
b = tf.Variable(0.0, name="b")
w = tf.Variable(tf.zeros([1, 1]), name="w")
y_pred = tf.add(tf.matmul(X, w), b, name="y_pred") # X @ w + b
mse = tf.reduce_mean(tf.square(y_pred - y), name="mse")
optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)
training_op = optimizer.minimize(mse)
init = tf.global_variables_initializer()
In [ ]:
n_iterations = 500
with tf.Session(graph=graph) as sess:
init.run()
for iteration in range(n_iterations):
if iteration % 100 == 0:
print("Iteration {:5}, MSE: {:.4f}".format(iteration, mse.eval()))
training_op.run()
w_val, b_val = sess.run([w, b])
In [ ]:
plt.figure(figsize=(10, 5))
plot_life_satisfaction_with_linear_model(X_train, y_train, w_val, b_val)
plt.show()
How does the optimizer know which variables to tweak? Answer: the TRAINABLE_VARIABLES collection.
In [ ]:
coll = graph.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
[var.op.name for var in coll]
In [ ]:
cyprus_gdp_per_capita = 22000
cyprus_life_satisfaction = w_val[0][0] * cyprus_gdp_per_capita / 10000 + b_val
cyprus_life_satisfaction
In [ ]:
graph = tf.Graph()
with graph.as_default():
X = tf.placeholder(tf.float32, shape=[None, 1], name="X") # <= None allows for any
y = tf.placeholder(tf.float32, shape=[None, 1], name="y") # training batch size
b = tf.Variable(0.0, name="b")
w = tf.Variable(tf.zeros([1, 1]), name="w")
y_pred = tf.add(tf.matmul(X, w), b, name="y_pred") # X @ w + b
mse = tf.reduce_mean(tf.square(y_pred - y), name="mse")
optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)
training_op = optimizer.minimize(mse)
init = tf.global_variables_initializer()
In [ ]:
n_iterations = 500
X_test = np.array([[22000]], dtype=np.float32) / 10000
with tf.Session(graph=graph) as sess:
init.run()
for iteration in range(n_iterations):
feed_dict = {X: X_train, y: y_train}
if iteration % 100 == 0:
print("Iteration {:5}, MSE: {:.4f}".format(
iteration,
mse.eval(feed_dict))) # <= FEED TRAINING DATA
training_op.run(feed_dict) # <= FEED TRAINING DATA
# make the prediction:
y_pred_val = y_pred.eval(feed_dict={X: X_test}) # <= FEED TEST DATA
In [ ]:
y_pred_val
5.1) Create a simple graph that computes the function $f(x) = x^2 - 3x + 1$. Define $x$ as a placeholder for a simple scalar value of type float32 value (i.e., shape=[], dtype=tf.float32). Create a session and evaluate $f(5)$. You should find 11.0.
In [ ]:
In [ ]:
In [ ]:
5.2) Add an operation that computes the derivative of $f(x)$ with regards to $x$, noted $f'(x)$. Create a session and evaluate $f'(5)$. You should find 7.0.
Hint: use tf.gradients().
In [ ]:
In [ ]:
In [ ]:
5.3) Using a MomentumOptimizer, find the value of $x$ that minimizes $f(x)$. You should find $\hat{x}=1.5$.
Hint: you need to change x into a Variable. Moreover, the MomentumOptimizer has its own variables that need to be initialized, so don't forget to create an init operation using a tf.global_variables_initializer(), and call it at the start of the session.
In [ ]:
In [ ]:
In [ ]:
Try not to peek at the solution below before you have done the exercise! :)
5.1)
In [ ]:
graph = tf.Graph()
with graph.as_default():
x = tf.placeholder(tf.float32, shape=[], name="x")
f = tf.square(x) - 3 * x + 1
In [ ]:
with tf.Session(graph=graph):
print(f.eval(feed_dict={x: 5.0}))
5.2)
In [ ]:
with graph.as_default():
[fp] = tf.gradients(f, [x])
In [ ]:
with tf.Session(graph=graph):
print(fp.eval(feed_dict={x: 5.0}))
5.3)
In [ ]:
learning_rate = 0.01
momentum = 0.8
graph = tf.Graph()
with graph.as_default():
x = tf.Variable(0.0, name="x")
f = tf.square(x) - 3 * x + 1
optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)
training_op = optimizer.minimize(f)
init = tf.global_variables_initializer()
In [ ]:
n_iterations = 70
with tf.Session(graph=graph):
init.run()
for iteration in range(n_iterations):
training_op.run()
if iteration % 10 == 0:
print("x={:.2f}, f(x)={:.2f}".format(x.eval(), f.eval()))
Note that it's possible to replace the output value of any operation, not just placeholders. So, for example, even though x is now a Variable, you can use a feed_dict to use any value you want, for example to compute f(5.0). Important: this does not affect the variable!
In [ ]:
with tf.Session(graph=graph):
init.run()
print(x.eval()) # x == 0.0
print(f.eval()) # f(0) == 1.0
print(f.eval(feed_dict={x: 5.0})) # use 5.0 instead of the value of x, to compute f(5)
print(x.eval()) # x is still 0.0
print(f.eval()) # f(0) is still 1.0
In [ ]:
graph = tf.Graph()
with graph.as_default():
X = tf.placeholder(tf.float32, shape=[None, 1], name="X")
y = tf.placeholder(tf.float32, shape=[None, 1], name="y")
b = tf.Variable(0.0, name="b")
w = tf.Variable(tf.zeros([1, 1]), name="w")
y_pred = tf.add(tf.matmul(X, w), b, name="y_pred") # X @ w + b
mse = tf.reduce_mean(tf.square(y_pred - y), name="mse")
optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)
training_op = optimizer.minimize(mse)
init = tf.global_variables_initializer()
saver = tf.train.Saver() # <= At the very end of the construction phase
In [ ]:
n_iterations = 500
with tf.Session(graph=graph) as sess:
init.run()
for iteration in range(n_iterations):
if iteration % 100 == 0:
print("Iteration {:5}, MSE: {:.4f}".format(
iteration,
mse.eval(feed_dict={X: X_train, y: y_train})))
training_op.run(feed_dict={X: X_train, y: y_train}) # <= FEED THE DICT
saver.save(sess, "./my_life_satisfaction_model")
In [ ]:
with tf.Session(graph=graph) as sess:
saver.restore(sess, "./my_life_satisfaction_model")
# make the prediction:
y_pred_val = y_pred.eval(feed_dict={X: X_test})
In [ ]:
y_pred_val
In [ ]:
model_path = "./my_life_satisfaction_model"
graph = tf.Graph()
with tf.Session(graph=graph) as sess:
# restore the graph
saver = tf.train.import_meta_graph(model_path + ".meta")
saver.restore(sess, model_path)
# get references to the tensors we need
X = graph.get_tensor_by_name("X:0")
y_pred = graph.get_tensor_by_name("y_pred:0")
# make the prediction:
y_pred_val = y_pred.eval(feed_dict={X: X_test})
In [ ]:
y_pred_val
In [ ]: