In [1]:
%matplotlib inline
import os
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.colors as colors
import matplotlib.cm as cmx
plt.style.use('ggplot')
In [2]:
K = 10
x = np.linspace(-1,1,200)
cost = np.sin(K*x) * np.exp(-x)
fig = plt.figure()
ax = fig.gca()
ax.set_ylabel('Cost')
ax.set_xlabel('Some param')
ax.plot(x,cost)
Out[2]:
In [3]:
simplified_gradient = np.diff(cost)
print (simplified_gradient)
In [4]:
K = 10
x = np.linspace(-1,1,200)
cost = np.sin(K*x) * np.exp(-x)
In [5]:
n_iterations = 400
learning_rate = 1.0
init_p = np.random.randint(len(x)*0.2, len(x)*0.8)
print(init_p)
In [6]:
cmap = plt.get_cmap('coolwarm')
c_norm = colors.Normalize(vmin=0, vmax=n_iterations)
scalar_map = cmx.ScalarMappable(norm=c_norm, cmap=cmap)
In [7]:
fig = plt.figure()
ax = fig.gca()
ax.set_ylabel('Cost')
ax.set_xlabel('Some Parm')
ax.plot(x,cost)
for i in range(n_iterations):
init_p -= learning_rate * simplified_gradient[int(init_p)]
int_init_p = int(init_p)
ax.plot(x[init_p], cost[init_p], 'ro', alpha=(i+1)/n_iterations)
# ax.plot(x[int_init_p], cost[int_init_p], 'ro', alpha=(i + 1) / n_iterations, color=scalar_map.to_rgba(iter_i))
In [9]:
nb_obs = 1000
x = np.linspace(-3,3, nb_obs)
y = np.sin(x) + np.random.uniform(-0.5,0.5, nb_obs)
plt.scatter(x, y, alpha=0.15, marker='o')
Out[9]:
In [10]:
X = tf.placeholder(tf.float32, name='X')
Y = tf.placeholder(tf.float32, name='Y')
In [15]:
sess = tf.InteractiveSession()
n = tf.random_normal([1000]).eval()
plt.hist(n)
Out[15]:
In [16]:
print(n.mean())
print(n.std())
In [20]:
W = tf.Variable(tf.random_normal([1], dtype=tf.float32, stddev=0.1), name='Weight')
B = tf.Variable(tf.constant([0], dtype=tf.float32), name='Bias')
Y_pred = X * W + B
Let's make sure we know what we're working with
In [32]:
tf.random_normal([1], dtype=tf.float32, stddev=0.1).eval()
Out[32]:
In [25]:
tf.constant([0], dtype=tf.float32)
Out[25]:
In [33]:
def distance(a,b):
return tf.abs(a -b)
In [38]:
cost = tf.reduce_mean(distance(Y_pred, Y))
In [104]:
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001).minimize(cost)
In [56]:
nb_iterations = 500
fig, ax = plt.subplots(1, 1)
ax.scatter(x, y, alpha=0.15, marker='+')
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
prev_training_cost = 0.0
for i in range(nb_iterations):
sess.run(optimizer, feed_dict={X: x, Y: y})
training_cost = sess.run(cost, feed_dict={X: x, Y: y})
# Plot
if(i%10 == 0):
y_pred = Y_pred.eval(feed_dict={X: x}, session=sess)
ax.plot(x, y_pred, 'k', alpha=i / nb_iterations)
fig.show()
plt.draw()
print(training_cost)
# Delta too small?
if np.abs(prev_training_cost - training_cost) < 0.000001:
break
prev_training_cost = training_cost
In [102]:
idxs = np.arange(100)
rand_idxs = np.random.permutation(idxs)
batch_size = 10
nb_batches = len(rand_idxs) // batch_size
for b in range(nb_batches):
print(rand_idxs[b * batch_size : (b + 1) * batch_size])
In [109]:
def train(X, Y, Y_pred, n_iterations=100, batch_size=200, learning_rate=0.02):
cost = tf.reduce_mean(distance(Y_pred, Y))
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)
# plot
fig, ax = plt.subplots(1, 1)
ax.scatter(x, y, alpha=0.15, marker='+')
ax.set_xlim([-4, 4])
ax.set_ylim([-2, 2])
nb_iterations = 1000
batch_size = 1000 # 10
nb_batches = len(x) // batch_size
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for iter_i in range(nb_iterations):
# shuffle all data
idxs = np.random.permutation(range(len(x)))
for b in range(nb_batches):
idxs_i = idxs[b * batch_size: (b + 1) * batch_size]
sess.run(optimizer, feed_dict={X:x[idxs_i], Y:y[idxs_i]})
if(iter_i%100 == 0):
# metrics
training_cost = sess.run(cost, feed_dict={X:x, Y:y})
print("{} - Iteration {}, Batch {}".format(training_cost, iter_i, b))
# and plot
y_pred = Y_pred.eval(feed_dict={X: x}, session=sess)
ax.plot(x, y_pred, 'k', alpha= iter_i / nb_iterations)
plt.draw()
In [114]:
nb_neurons = 100
W = tf.Variable(tf.random_normal([1, nb_neurons], stddev=0.1))
b = tf.Variable(tf.constant([0], dtype=tf.float32, shape=[nb_neurons]))
h = tf.matmul(tf.expand_dims(X,1), W) + b
Y_pred = tf.reduce_sum(h, 1)
train(X, Y, Y_pred)
In [121]:
Y_pred = tf.Variable(tf.random_normal([1], name='yolo'))
for pow_i in range(0,2):
W = tf.Variable(tf.random_normal([1], stddev=0.1), name='Weight_{}'.format(pow_i))
Y_pred = Y_pred + tf.multiply(tf.pow(X,pow_i), W)
train(X, Y, Y_pred)
In [118]:
Y_pred = tf.Variable(tf.random_normal([1], name='yolo'))
for pow_i in range(0,4):
W = tf.Variable(tf.random_normal([1], stddev=0.1), name='Weight_{}'.format(pow_i))
Y_pred = Y_pred + tf.multiply(tf.pow(X,pow_i), W)
train(X, Y, Y_pred)
In [119]:
Y_pred = tf.Variable(tf.random_normal([1], name='yolo'))
for pow_i in range(0,5):
W = tf.Variable(tf.random_normal([1], stddev=0.1), name='Weight_{}'.format(pow_i))
Y_pred = Y_pred + tf.multiply(tf.pow(X,pow_i), W)
train(X, Y, Y_pred)
In [120]:
Y_pred = tf.Variable(tf.random_normal([1], name='yolo'))
for pow_i in range(0,6):
W = tf.Variable(tf.random_normal([1], stddev=0.1), name='Weight_{}'.format(pow_i))
Y_pred = Y_pred + tf.multiply(tf.pow(X,pow_i), W)
train(X, Y, Y_pred)
In [124]:
sess = tf.InteractiveSession()
x = np.linspace(-6,6,1000)
plt.plot(x, tf.nn.tanh(x).eval(), label='tanh')
plt.plot(x, tf.nn.sigmoid(x).eval(), label='sigmoid')
plt.plot(x, tf.nn.relu(x).eval(), label='relu')
plt.legend(loc='lower right')
plt.ylim([-2, 2])
plt.xlabel('Input')
plt.ylabel('Output')
plt.grid('on')
In [131]:
nb_neurons = 10
W = tf.Variable(tf.random_normal([1, nb_neurons]), name='W')
b = tf.Variable(tf.constant([0], dtype=tf.float32, shape=[nb_neurons]), name='b')
# h = tf.nn.relu(tf.matmul(tf.expand_dims(X,1), W) + b, name='h')
h = tf.nn.tanh(tf.matmul(tf.expand_dims(X,1), W) + b, name='h')
Y_pred = tf.reduce_mean(h, 1)
train(X, Y, Y_pred, learning_rate=1)
In [132]:
def linear(X, n_input, n_output):
W = tf.Variable(tf.random_normal([n_input, n_output], stddev=0.1), name='W')
b = tf.Variable(
tf.constant(0, dtype=tf.float32, shape=[n_output]), name='b')
h = tf.nn.tanh(tf.matmul(X, W) + b, name='h')
return h
In [137]:
from tensorflow.python.framework import ops
ops.reset_default_graph()
In [138]:
g = tf.get_default_graph()
In [139]:
[op.name for op in tf.get_default_graph().get_operations()]
Out[139]:
In [140]:
# New network
X = tf.placeholder(tf.float32, name='X')
h = linear(X, 2, 10)
In [141]:
[op.name for op in tf.get_default_graph().get_operations()]
Out[141]:
In [142]:
def linear(X, n_input, n_output, activation=None, scope=None):
with tf.variable_scope(scope or "linear"):
W = tf.get_variable(
name='W',
shape=[n_input, n_output],
initializer=tf.random_normal_initializer(mean=0.0, stddev=0.1))
b = tf.get_variable(
name='b',
shape=[n_output],
initializer=tf.constant_initializer())
h = tf.matmul(X, W) + b
if activation is not None:
h = activation(h)
return h
In [143]:
ops.reset_default_graph()
In [144]:
[op.name for op in tf.get_default_graph().get_operations()]
Out[144]:
In [145]:
# New network with scope
X = tf.placeholder(tf.float32, name='X')
h = linear(X, 2, 10, scope='layer1')
# See the names of any operations in the graph
[op.name for op in tf.get_default_graph().get_operations()]
Out[145]:
In [147]:
h2 = linear(h, 10, 10, scope='layer2')
h3 = linear(h2, 10, 3, scope='layer3')
In [148]:
[op.name for op in tf.get_default_graph().get_operations()]
Out[148]:
In [187]:
from scipy.misc import imread, imresize
# Of course we'll test it with Lena ;)
comic = imread("/home/may/training/CADL/data/random/lena.jpg")
img = imresize(comic, (64, 64))
plt.imshow(comic)
Out[187]:
In [175]:
x = []
y = []
for row in range(img.shape[0]):
for col in range(img.shape[1]):
x.append([row, col])
y.append(img[row, col])
x = np.array(x)
y = np.array(y)
x = (x - np.mean(x)) / np.std(x)
print (x.shape, y.shape)
In [176]:
x[0], y[0]
Out[176]:
In [177]:
plt.imshow(y.reshape(img.shape))
Out[177]:
In [182]:
ops.reset_default_graph()
X = tf.placeholder(tf.float32, shape=[None, 2], name='X')
Y = tf.placeholder(tf.float32, shape=[None, 3], name='Y')
In [183]:
n_neurons = [2, 64, 64, 64, 64, 64, 64, 3]
current_input = X
for layer_i in range(1, len(n_neurons)):
current_input = linear(
X=current_input,
n_input=n_neurons[layer_i - 1],
n_output=n_neurons[layer_i],
activation=tf.nn.relu if (layer_i+1) < len(n_neurons) else None,
scope='layer_' + str(layer_i))
Y_pred = current_input
In [184]:
cost = tf.reduce_mean(tf.reduce_sum(distance(Y_pred, Y), 1))
In [185]:
optimizer = tf.train.AdamOptimizer(0.001).minimize(cost)
In [186]:
n_iterations = 500
batch_size = 50
with tf.Session() as sess:
# Here we tell tensorflow that we want to initialize all
# the variables in the graph so we can use them
# This will set W and b to their initial random normal value.
sess.run(tf.global_variables_initializer())
# We now run a loop over epochs
prev_training_cost = 0.0
for it_i in range(n_iterations):
idxs = np.random.permutation(range(len(x)))
n_batches = len(idxs) // batch_size
for batch_i in range(n_batches):
idxs_i = idxs[batch_i * batch_size: (batch_i + 1) * batch_size]
sess.run(optimizer, feed_dict={X: x[idxs_i], Y: y[idxs_i]})
training_cost = sess.run(cost, feed_dict={X: x, Y: y})
print(it_i, training_cost)
if (it_i + 1) % 20 == 0:
ys_pred = Y_pred.eval(feed_dict={X: x}, session=sess)
fig, ax = plt.subplots(1, 1)
img = np.clip(ys_pred.reshape(img.shape), 0, 255).astype(np.uint8)
plt.imshow(img)
plt.show()
In [ ]: