In [1]:
%matplotlib inline
import os
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.colors as colors
import matplotlib.cm as cmx
plt.style.use('ggplot')
In [2]:
fig = plt.figure(figsize=(10, 6))
ax = fig.gca()
x = np.linspace(-1, 1 , 200)
hz =10
cost = np.sin(hz*x)*np.exp(-x)
ax.plot(x, cost)
ax.set_ylabel('Cost')
ax.set_xlabel('Some parameter')
Out[2]:
In [3]:
gradient = np.diff(cost)
In [4]:
fig = plt.figure(figsize=(10,6))
ax = fig.gca()
x = np.linspace(-1, 1, 200)
hz = 10
cost = np.sin(hz*x) * np.exp(-x)
ax.plot(x, cost)
ax.set_ylabel('Cost')
ax.set_xlabel('Some Parameter')
n_iterations = 500
cmap = plt.get_cmap('coolwarm')
c_norm = colors.Normalize(vmin=0, vmax=n_iterations)
scalar_map = cmx.ScalarMappable(norm=c_norm, cmap=cmap)
init_p =120 #np.radom.randint(len(x)*0.2, len(x)*0.8)
learning_rate = 1.0
for iter_i in range(n_iterations):
init_p -= learning_rate *gradient[int(init_p)]
ax.plot(x[int(init_p)], cost[int(init_p)],'ro',alpha=(iter_i+1)/n_iterations, color=scalar_map.to_rgba(iter_i))
In [5]:
from mpl_toolkits.mplot3d import axes3d
from matplotlib import cm
In [6]:
fig = plt.figure(figsize=(10,6))
ax = fig.gca(projection='3d')
x, y = np.mgrid[-1:1:0.02, -1:1:0.02]
X, Y, Z = x, y, np.sin(hz*x)*np.exp(-x)*np.cos(hz*y)*np.exp(-y)
ax.plot_surface(X, Y, Z, rstride=2, cstride=2, alpha=0.75, cmap='jet', shade=False)
ax.set_xlabel('Some parameter 1')
ax.set_ylabel('Some parameter 2')
ax.set_zlabel('Cost')
Out[6]:
In [7]:
np.mgrid[0:2,0:3]
Out[7]:
In [8]:
fig, axs = plt.subplots(1, 3, figsize=(20, 6))
for rate_i, learning_rate in enumerate([0.01, 1.0, 500.]):
ax = axs[rate_i]
x = np.linspace(-1,1, 200)
hz = 10
cost = np.sin(hz*x)*np.exp(-x)
ax.plot(x, cost)
ax.set_ylabel('Cost')
ax.set_xlabel('Some Parameter')
ax.set_title(str(learning_rate))
n_iterations = 500
cmap = plt.get_cmap('coolwarm')
c_norm = colors.Normalize(vmin=0, vmax=n_iterations)
scalar_map = cmx.ScalarMappable(norm=c_norm, cmap=cmap)
init_p = 120
for iter_i in range(n_iterations):
init_p -= learning_rate * gradient[int(init_p)]
ax.plot(x[int(init_p)], cost[int(init_p)], 'ro', alpha = (iter_i+1)/ n_iterations, color = scalar_map.to_rgba(iter_i))
In [9]:
n_observations = 1000
xs = np.linspace(-3, 3, n_observations)
ys = np.sin(xs) + np.random.uniform(-0.5, 0.5, n_observations)
plt.scatter(xs, ys, alpha=0.15, marker='+')
Out[9]:
In [10]:
X = tf.placeholder(tf.float32, name='X')
Y = tf.placeholder(tf.float32, name='Y')
In [11]:
sess = tf.InteractiveSession()
n = tf.random_normal([1000],mean = 0, stddev=1.0, dtype=tf.float32).eval()
plt.hist(n)
Out[11]:
In [12]:
n = tf.random_normal([1000], stddev=0.1).eval()
plt.hist(n)
Out[12]:
In [13]:
W = tf.Variable(tf.random_normal([1], dtype=tf.float32, stddev=0.1, name='weight'))
B = tf.Variable(tf.constant([0],dtype=tf.float32), name='bias')
Y_pred = X*W + B
In [14]:
def distance(p1, p2):
return tf.abs(p1-p2)
In [15]:
cost = tf.reduce_mean(distance(Y_pred, Y))
In [16]:
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01).minimize(cost)
In [17]:
# We create a session to use the graph
n_iterations = 5000
# Plot the true data distribution
fig, ax = plt.subplots(1, 1)
ax.scatter(xs, ys, alpha=0.15, marker='+')
with tf.Session() as sess:
# Herexx we tell tensorflow thaXt we want to initialize all
# the variables in the graph so we can use them
# This will set `W` and `b` to their initial random normal value.
sess.run(tf.global_variables_initializer())
# We now run a loop over epochs
prev_training_cost = 0.0
for it_i in range(n_iterations):
sess.run(optimizer, feed_dict={X: xs, Y: ys})
training_cost = sess.run(cost, feed_dict={X: xs, Y: ys})
# every 10 iterations
if it_i % 10 == 0:
# let's plot the x versus the predicted y
ys_pred = Y_pred.eval(feed_dict={X: xs}, session=sess)
# We'll draw points as a scatter plot just like before
# Except we'll also scale the alpha value so that it gets
# darker as the iterations get closer to the end
ax.plot(xs, ys_pred, 'k', alpha=it_i / n_iterations)
fig.show()
plt.draw()
# And let's print our training cost: mean of absolute differences
print(training_cost)
# Allow the training to quit if we've reached a minimum
if np.abs(prev_training_cost - training_cost) < 0.00000001:
break
# Keep track of the training cost
prev_training_cost = training_cost
In [18]:
plt.scatter(xs, ys)
Out[18]:
In [19]:
idxs = np.arange(100)
batch_size = 10
n_batches = len(idxs) // batch_size
for batch_i in range(n_batches):
print(idxs[batch_i * batch_size : (batch_i + 1) * batch_size])
In [20]:
rand_idxs = np.random.permutation(idxs)
batch_size = 10
n_batches = len(rand_idxs)//batch_size
for batch_i in range(n_batches):
print(rand_idxs[batch_i * batch_size : (batch_i + 1) * batch_size] )
In [21]:
n_iterations = 500
batch_size = 1000
fig, ax = plt.subplots(1, 1)
ax.scatter(xs, ys, alpha =0.15, marker='+')
ax.set_xlim([-4, 4])
ax.set_ylim([-2, 2])
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
prev_training_cost = 0.
for it_i in range(n_iterations):
rand_idxs = np.random.permutation(range(len(xs)))
n_batches = len(rand_idxs) // batch_size
for batch_i in range(n_batches):
rand_idx = rand_idxs[batch_i*batch_size : (batch_i+1)*batch_size]
batch_x = xs[rand_idx]
batch_y = ys[rand_idx]
sess.run(optimizer, feed_dict={X:batch_x, Y: batch_y})
if it_i %10 == 0:
ys_pred = Y_pred.eval(feed_dict={X:xs },session=sess)
training_cost = sess.run(cost, feed_dict={X:xs,Y:ys})
ax.plot(xs, ys_pred, 'k', alpha=it_i/n_iterations)
print(training_cost)
fig.show()
plt.draw()
In [22]:
from tensorflow.python.framework import ops
ops.reset_default_graph()
In [23]:
import matplotlib.colors as colors
import matplotlib.cm as cmx
In [24]:
def train(X, Y, Y_pred, n_iterations=100, batch_size=200, learning_rate=0.02):
cost = tf.reduce_mean(distance(Y_pred, Y))
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)
fig, ax =plt.subplots(1, 1)
ax.scatter(xs, ys, alpha=0.15, marker='+')
ax.set_xlim([-4, 4])
ax.set_ylim([-2, 2])
cmap = plt.get_cmap('coolwarm')
c_norm = colors.Normalize(vmin=0, vmax=n_iterations)
scalar_map = cmx.ScalarMappable(norm=c_norm, cmap=cmap)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
prev_trainig_cost = 0.0
for it_i in range(n_iterations):
idxs = np.random.permutation(range(len(xs)))
n_batches = len(idxs)// batch_size
for batch_i in range(n_batches):
idxs_i = idxs[batch_i * batch_size: (batch_i + 1) * batch_size]
sess.run(optimizer, feed_dict={X: xs[idxs_i], Y: ys[idxs_i]})
if it_i%10 :
ys_pred = Y_pred.eval(feed_dict={X:xs},session = sess)
training_cost = sess.run(cost, feed_dict={X:xs, Y:ys})
ax.plot(xs,ys_pred, alpha=it_i/n_iterations, color=scalar_map.to_rgba(it_i))
print(training_cost)
fig.show()
plt.draw()
In [25]:
X = tf.placeholder(tf.float32, name='X')
Y = tf.placeholder(tf.float32, name='Y')
n_neurons = 100
W = tf.Variable(tf.random_normal([1, n_neurons], stddev=0.1))
b= tf.Variable(tf.constant(0, dtype=tf.float32, shape=[n_neurons]))
h = tf.matmul(tf.expand_dims(X,1),W) + b
Y_pred = tf.reduce_sum(h, 1)
cost = tf.reduce_mean(distance(Y_pred, Y))
In [26]:
train(X,Y,Y_pred,n_iterations=500,learning_rate=0.002)
In [27]:
Y_pred = tf.Variable(tf.random_normal([1]), name='bias')
for pow_i in range(0, 5):
W = tf.Variable(
tf.random_normal([1], stddev=0.1), name='weight_%d'%pow_i)
Y_pred = tf.add(tf.mul(tf.pow(X, pow_i), W), Y_pred)
train(X, Y, Y_pred, n_iterations=500, learning_rate=0.002)
polynomial expansion is not useful
In [28]:
sess = tf.InteractiveSession()
x = np.linspace(-6, 6, 1000)
plt.plot(x, tf.nn.tanh(x).eval(), label='tanh')
plt.plot(x, tf.nn.sigmoid(x).eval(),label='sigmoid')
plt.plot(x, tf.nn.relu(x).eval(), label='relu')
plt.legend(loc='lower right')
plt.xlim([-6, 6])
plt.ylim([-2, 2])
plt.xlabel('Input')
plt.ylabel('Output')
plt.grid('on')
In [29]:
n_neurons = 10
W = tf.Variable(tf.random_normal([1, n_neurons]), name='W')
b = tf.Variable(tf.constant(0, dtype=tf.float32, shape=[n_neurons]), name='b')
h = tf.nn.tanh(tf.matmul(tf.expand_dims(X,1), W) +b , name= 'h')
Y_pred = tf.reduce_sum(h, 1)
train(X, Y, Y_pred)
In [30]:
def linear(X, n_input, n_output, activation=None):
W = tf.Variable(tf.random_normal([n_input, n_output], stddev=0.1), name= 'weight')
b = tf.Variable(tf.constant(0, dtype=tf.float32, shape = [n_output]), name='b')
h = tf.nn.tanh(tf.matmul(X, W) + b, name='h')
return h
In [31]:
from pprint import pprint
In [32]:
from tensorflow.python.framework import ops
ops.reset_default_graph()
g = tf.get_default_graph()
print([op.name for op in tf.get_default_graph().get_operations()])
X = tf.placeholder(tf.float32, name='X')
h = linear(X, 2, 10)
print([op.name for op in tf.get_default_graph().get_operations()])
In [33]:
def linear(X, n_input, n_output, activation=None, scope=None):
with tf.variable_scope(scope or "linear"):
W = tf.get_variable(
name='W',
shape=[n_input, n_output],
initializer=tf.random_normal_initializer(mean=0.0, stddev=0.1))
b = tf.get_variable(
name='b',
shape=[n_output],
initializer=tf.constant_initializer())
h = tf.add(tf.matmul(X,W), b)
if activation is not None:
h = activation(h)
return h
In [34]:
from tensorflow.python.framework import ops
ops.reset_default_graph()
g = tf.get_default_graph()
print([op.name for op in tf.get_default_graph().get_operations()])
X = tf.placeholder(tf.float32, name='X')
h = linear(X, 2, 10, scope='layer1')
pprint([op.name for op in tf.get_default_graph().get_operations()])
In [35]:
h2 = linear(h, 10, 10, scope='layer2')
h3 = linear(h2, 10, 3, scope='layer3')
pprint([op.name for op in tf.get_default_graph().get_operations()])
In [36]:
from skimage.data import astronaut
from scipy.misc import imresize
img = imresize(astronaut(), (64,64))
plt.imshow(img)
Out[36]:
In [37]:
xs =[]
ys =[]
for row_i in range(img.shape[0]):
for col_i in range(img.shape[1]):
xs.append([row_i, col_i])
ys.append(img[row_i, col_i])
xs = np.array(xs)
ys = np.array(ys)
xs = (xs - np.mean(xs)) / np.std(xs)
print(xs.shape, ys.shape)
In [ ]:
plt.imshow(ys.reshape(img.shape))
In [39]:
X = tf.placeholder(tf.float32, shape=[None, 2], name='X')
Y = tf.placeholder(tf.float32, shape=[None, 3], name='Y')
In [40]:
n_neurons = [2, 64, 64, 64, 64, 64, 64 ,3]
current_input = X
for layer_i in range(1, len(n_neurons)):
current_input = linear(current_input,
n_input=n_neurons[layer_i-1],
n_output=n_neurons[layer_i],
activation=tf.nn.relu if (layer_i+1) <len(n_neurons) else None,
scope='layer_'+str(layer_i)
)
Y_pred = current_input
In [42]:
cost = tf.reduce_mean(tf.reduce_sum(distance(Y_pred, Y), 1))
In [43]:
optimizer = tf.train.AdadeltaOptimizer(0.001).minimize(cost)
In [44]:
n_iterations = 500
batch_size = 50
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
prev_training_cost = 0.0
for it_i in range(n_iterations):
idxs = np.random.permutation(range(len(xs)))
n_batches = len(idxs) // batch_size
for batch_i in range(n_batches):
idxs_i = idxs[batch_i * batch_size: (batch_i +1)* batch_size]
sess.run(optimizer, feed_dict={X :xs[idxs_i], Y:ys[idxs_i]})
training_cost = sess.run(cost, feed_dict={X: xs, Y: ys})
print(it_i, training_cost)
if (it_i+1)%20 == 0:
ys_pred = Y_pred.eval(feed_dict={X:xs}, session=sess)
fig, ax = plt.subplots(1,1)
img = np.clip(ys_pred.reshape(img.shape), 0, 255).astype(np.uint8)
plt.imshow(img)
plt.show()
In [ ]: