In [1]:
# Import python and tensorflow libraries
from __future__ import print_function
import os
import numpy as np
from functools import partial
from IPython.display import clear_output, Image, display, HTML
import tensorflow as tf
# Helper functions for TF Graph visualization
def strip_consts(graph_def, max_const_size=32):
"""Strip large constant values from graph_def."""
strip_def = tf.GraphDef()
for n0 in graph_def.node:
n = strip_def.node.add()
n.MergeFrom(n0)
if n.op == 'Const':
tensor = n.attr['value'].tensor
size = len(tensor.tensor_content)
if size > max_const_size:
tensor.tensor_content = bytes("<stripped %d bytes>"%size, 'utf-8')
return strip_def
def rename_nodes(graph_def, rename_func):
res_def = tf.GraphDef()
for n0 in graph_def.node:
n = res_def.node.add()
n.MergeFrom(n0)
n.name = rename_func(n.name)
for i, s in enumerate(n.input):
n.input[i] = rename_func(s) if s[0]!='^' else '^'+rename_func(s[1:])
return res_def
def show_graph(graph_def, max_const_size=32):
"""Visualize TensorFlow graph."""
if hasattr(graph_def, 'as_graph_def'):
graph_def = graph_def.as_graph_def()
strip_def = strip_consts(graph_def, max_const_size=max_const_size)
code = """
<script>
function load() {{
document.getElementById("{id}").pbtxt = {data};
}}
</script>
<link rel="import" href="https://tensorboard.appspot.com/tf-graph-basic.build.html" onload=load()>
<div style="height:600px">
<tf-graph-basic id="{id}"></tf-graph-basic>
</div>
""".format(data=repr(str(strip_def)), id='graph'+str(np.random.rand()))
iframe = """
<iframe seamless style="width:800px;height:620px;border:0" srcdoc="{}"></iframe>
""".format(code.replace('"', '"'))
display(HTML(iframe))
Element-wise product of two vectors: Given $\mathbf{a} = \begin{bmatrix}1\\2\\3\\4\end{bmatrix}$, $\mathbf{b} = \begin{bmatrix}5\\6\\7\\8\end{bmatrix}$,
compute the element-wise product of the two vectors $\mathbf{c} = \mathbf{a} \odot \mathbf{b} = \begin{bmatrix}a_1b_1\\a_2b_2\\a_3b_3\\a_4b_4\end{bmatrix} = \begin{bmatrix}5\\12\\21\\32\end{bmatrix}$
We will illustrate this example using two constant 1-D tensors in TensorFlow.
In [2]:
# reset the TensorFlow graph to start the new example
tf.reset_default_graph()
In [3]:
a = tf.constant([1,2,3,4], name='a')
b = tf.constant([1,2,3,4], name='b')
c = a*b
In [4]:
# Visualizing the computation graph.
show_graph(tf.get_default_graph().as_graph_def())
In [5]:
sess = tf.InteractiveSession()
init = tf.global_variables_initializer()
sess.run(init)
In [6]:
sess.run([a,b,c])
Out[6]:
In [7]:
# reset the TensorFlow graph to start the new example
tf.reset_default_graph()
In [8]:
x = tf.placeholder(tf.float32, name='x')
c = tf.constant(2.0, name='c')
cx_squared = c*x*x
In [9]:
# Visualizing the computation graph.
show_graph(tf.get_default_graph().as_graph_def())
In [10]:
sess = tf.InteractiveSession()
init = tf.global_variables_initializer()
sess.run(init)
In [11]:
sess.run(cx_squared, feed_dict={x:np.array([1,2,3])})
Out[11]:
In [12]:
sess.run(cx_squared, feed_dict={x:np.array([[1,2],[6,7]])})
Out[12]:
Learn your first parametric model using squared error loss and gradient descent.
We have the following linear regression model that makes predictions by a weighted sum of its 5-dimensional input vector $\mathbf{x} \in \mathbb{R}^5$. The goal of learning is to find the best set of weights $W \in \mathbb{R}^5$ and a bias scalar $b \in \mathbb{R}$, such that squared loss over the training data is minimized.
$\hat{\mathbf{y}} = W^T \mathbf{x} + b = \sum_{i=1}^{5} W_i x_i + b$
Squared error loss function was introduced in the lecture to measure the prediction error of each train examples. For a set of data points, one can compute the expected squared loss using the mean squared error (MSE) loss:
$\text{min}_{W,b} \quad \frac{1}{70} \sum_{m=1}^{70} \|\hat{\mathbf{y}}^{(m)} - {\mathbf{y}}^{(m)} \|^2_2$
We optimize mean squared error over the 70 data points to find the best weights and bias parameters.
Make sure you download the data file "x.npy", "t2.npy" and place them in the same directory as this notebook.
In [2]:
# reset the TensorFlow graph to start the new example
tf.reset_default_graph()
In [3]:
def readMat(path):
return np.load(path)
def writeMat(path, m):
np.save(path, m)
def loadData(dataPath, targetPath):
# Loading my data
inputData = readMat(dataPath)
target = readMat(targetPath)
trainData = inputData[:,0:70].T
testData = inputData[:,70:].T
trainTarget = np.expand_dims(target[0:70], 1)
testTarget = np.expand_dims(target[70:],1)
return trainData, trainTarget, testData, testTarget
def buildGraph():
# Variable creation
W = tf.Variable(tf.truncated_normal(shape=[5,1], stddev=0.5), name='weights')
b = tf.Variable(0.0, name='biases')
X = tf.placeholder(tf.float32, [None, 5], name='input_x')
y_target = tf.placeholder(tf.float32, [None,1], name='target_y')
# Graph definition
y_predicted = tf.matmul(X,W) + b
# Error definition
meanSquaredError = tf.reduce_mean(tf.reduce_sum(tf.square(y_predicted - y_target),
reduction_indices=1,
name='squared_error'),
name='mean_squared_error')
# Training mechanism
optimizer = tf.train.GradientDescentOptimizer(learning_rate = 0.01)
train = optimizer.minimize(loss=meanSquaredError)
return W, b, X, y_target, y_predicted, meanSquaredError, train
In [4]:
W, b, X, y_target, y_predicted, meanSquaredError, train = buildGraph()
In [5]:
# Visualizing the computation graph.
show_graph(tf.get_default_graph().as_graph_def())
In [7]:
# Load training data
trainData, trainTarget, testData, testTarget = loadData('./x.npy', './t2.npy')
In [8]:
# Initialize session
init = tf.global_variables_initializer()
sess = tf.InteractiveSession()
sess.run(init)
initialW = sess.run(W)
initialb = sess.run(b)
print("Initial weights: %s, initial bias: %.2f"%(initialW, initialb))
In [9]:
# Training model
wList = []
for step in xrange(0,201):
_, err, currentW, currentb, yhat = sess.run([train, meanSquaredError, W, b, y_predicted], feed_dict={X: trainData, y_target: trainTarget})
wList.append(currentW)
if not (step % 50) or step < 10:
print("Iter: %3d, MSE-train: %4.2f, weights: %s, bias: %.2f"%(step, err, currentW.T, currentb))
# Testing model
errTest = sess.run(meanSquaredError, feed_dict={X: testData, y_target: testTarget})
print("Final testing MSE: %.2f"%(errTest))
In [ ]: