Tensorflow to learn simple functions

Import needed for Jupiter


In [1]:
%matplotlib notebook
import matplotlib
import matplotlib.pyplot as plt

from IPython.display import Image

Import needed for the code


In [2]:
import numpy as np
import tensorflow as tf

import fnmatch, os
import time

A simple model to learn simples functions


In [4]:
import math
import random
batch_size = 100

# define the data
xx = [i*math.pi/100 for i in range(100)]
yy = [math.sin(i*5) for i in xx]
y_ = [ [yy[i]+random.random()-0.5 ] for i in range(batch_size) ]

plt.figure(figsize=(12,5))
plt.plot(range(len(yy)), yy, label='sin')
plt.plot(range(len(y_)), y_, label='sin+noise')
plt.legend()
plt.show()


Define the model


In [23]:
#model
tf.reset_default_graph()
hidden_size = 1
vector_size = 100

# define place holder to for the input data and the target.
x = tf.placeholder(tf.float32, [vector_size, 1], name='x')
y = tf.placeholder(tf.float32, [vector_size, 1], name='y')
lr = tf.Variable(0.0, trainable=False, name='learning_rate')

# model parameters
Wxh = tf.Variable(tf.random_uniform((hidden_size, vector_size))*0.01, name='Wxh') # input to hidden
Why = tf.Variable(tf.random_uniform((vector_size, hidden_size))*0.01, name='Why') # hidden to output
bh = tf.Variable(tf.zeros((hidden_size, 1)), name='bh') # hidden bias
by = tf.Variable(tf.zeros((vector_size, 1)), name='by') # output bias


hidden = tf.tanh(tf.matmul(Wxh, x)+bh)
pred = tf.tanh(tf.matmul(Why, hidden)+by)

#hidden = tf.tanh(tf.matmul(Wxh, x))
#pred = tf.tanh(tf.matmul(Why, hidden))

# Mean squared error
cost = tf.reduce_sum(tf.pow(pred-y, 2))

# Gradient descent
optimizer = tf.train.GradientDescentOptimizer(lr).minimize(cost)

Training


In [24]:
epoch_count = 100
learning_rate = 0.02
decay_rate = 0.98
batch = 100

sess =  tf.Session()

cost_optimisation = []
with sess.as_default():
    tf.initialize_all_variables().run()
    print ("variable initialized")
    
    for e in range(epoch_count):
        sess.run(tf.assign(lr, learning_rate * (decay_rate ** e)))
        start = time.time()
        # Get learning data
        x_ = [ [xx[i]] for i in range(batch) ]
        y_ = [ [yy[i]+random.random()-0.5 ] for i in range(batch) ] # We add noise to the training
        feed = {x: x_, y: y_}
        # Run a session using train_op
        train_loss, _ = sess.run([cost, optimizer], feed)
        end = time.time()
        cost_optimisation.append(train_loss)
        print("{}/{} (epoch {}), train_loss = {:.6f}, time/batch = {:.3f}" \
            .format(e, epoch_count,
                    e, train_loss, end - start))


variable initialized
0/100 (epoch 0), train_loss = 59.218170, time/batch = 0.009
1/100 (epoch 1), train_loss = 52.831665, time/batch = 0.005
2/100 (epoch 2), train_loss = 42.627319, time/batch = 0.004
3/100 (epoch 3), train_loss = 40.978222, time/batch = 0.005
4/100 (epoch 4), train_loss = 33.692562, time/batch = 0.003
5/100 (epoch 5), train_loss = 32.270683, time/batch = 0.003
6/100 (epoch 6), train_loss = 29.360413, time/batch = 0.003
7/100 (epoch 7), train_loss = 23.717667, time/batch = 0.003
8/100 (epoch 8), train_loss = 25.691589, time/batch = 0.003
9/100 (epoch 9), train_loss = 26.314137, time/batch = 0.003
10/100 (epoch 10), train_loss = 24.088791, time/batch = 0.003
11/100 (epoch 11), train_loss = 22.199383, time/batch = 0.004
12/100 (epoch 12), train_loss = 19.793179, time/batch = 0.003
13/100 (epoch 13), train_loss = 20.785755, time/batch = 0.003
14/100 (epoch 14), train_loss = 18.275990, time/batch = 0.003
15/100 (epoch 15), train_loss = 16.146816, time/batch = 0.003
16/100 (epoch 16), train_loss = 17.781689, time/batch = 0.003
17/100 (epoch 17), train_loss = 15.842779, time/batch = 0.003
18/100 (epoch 18), train_loss = 16.490122, time/batch = 0.003
19/100 (epoch 19), train_loss = 14.187423, time/batch = 0.005
20/100 (epoch 20), train_loss = 18.460510, time/batch = 0.003
21/100 (epoch 21), train_loss = 12.207088, time/batch = 0.003
22/100 (epoch 22), train_loss = 15.661929, time/batch = 0.003
23/100 (epoch 23), train_loss = 14.500246, time/batch = 0.003
24/100 (epoch 24), train_loss = 12.415720, time/batch = 0.003
25/100 (epoch 25), train_loss = 14.261065, time/batch = 0.003
26/100 (epoch 26), train_loss = 13.003960, time/batch = 0.003
27/100 (epoch 27), train_loss = 12.263586, time/batch = 0.003
28/100 (epoch 28), train_loss = 15.224291, time/batch = 0.004
29/100 (epoch 29), train_loss = 13.408282, time/batch = 0.005
30/100 (epoch 30), train_loss = 14.158794, time/batch = 0.004
31/100 (epoch 31), train_loss = 11.362571, time/batch = 0.004
32/100 (epoch 32), train_loss = 11.191925, time/batch = 0.005
33/100 (epoch 33), train_loss = 12.004545, time/batch = 0.005
34/100 (epoch 34), train_loss = 12.155859, time/batch = 0.005
35/100 (epoch 35), train_loss = 11.814886, time/batch = 0.006
36/100 (epoch 36), train_loss = 11.184128, time/batch = 0.004
37/100 (epoch 37), train_loss = 13.014248, time/batch = 0.004
38/100 (epoch 38), train_loss = 11.263016, time/batch = 0.004
39/100 (epoch 39), train_loss = 11.720228, time/batch = 0.003
40/100 (epoch 40), train_loss = 11.469646, time/batch = 0.003
41/100 (epoch 41), train_loss = 11.863186, time/batch = 0.003
42/100 (epoch 42), train_loss = 12.836370, time/batch = 0.003
43/100 (epoch 43), train_loss = 11.836532, time/batch = 0.004
44/100 (epoch 44), train_loss = 11.129698, time/batch = 0.004
45/100 (epoch 45), train_loss = 11.640477, time/batch = 0.004
46/100 (epoch 46), train_loss = 11.219101, time/batch = 0.004
47/100 (epoch 47), train_loss = 10.796835, time/batch = 0.004
48/100 (epoch 48), train_loss = 12.222923, time/batch = 0.004
49/100 (epoch 49), train_loss = 11.628985, time/batch = 0.004
50/100 (epoch 50), train_loss = 11.847551, time/batch = 0.003
51/100 (epoch 51), train_loss = 11.762800, time/batch = 0.004
52/100 (epoch 52), train_loss = 10.936792, time/batch = 0.003
53/100 (epoch 53), train_loss = 9.992686, time/batch = 0.004
54/100 (epoch 54), train_loss = 9.979464, time/batch = 0.004
55/100 (epoch 55), train_loss = 10.796082, time/batch = 0.004
56/100 (epoch 56), train_loss = 11.092273, time/batch = 0.005
57/100 (epoch 57), train_loss = 11.155396, time/batch = 0.005
58/100 (epoch 58), train_loss = 10.433061, time/batch = 0.004
59/100 (epoch 59), train_loss = 9.824587, time/batch = 0.006
60/100 (epoch 60), train_loss = 10.055858, time/batch = 0.003
61/100 (epoch 61), train_loss = 9.898058, time/batch = 0.005
62/100 (epoch 62), train_loss = 11.533484, time/batch = 0.003
63/100 (epoch 63), train_loss = 8.525514, time/batch = 0.004
64/100 (epoch 64), train_loss = 10.799514, time/batch = 0.003
65/100 (epoch 65), train_loss = 8.706631, time/batch = 0.003
66/100 (epoch 66), train_loss = 10.438148, time/batch = 0.004
67/100 (epoch 67), train_loss = 10.097369, time/batch = 0.005
68/100 (epoch 68), train_loss = 11.195714, time/batch = 0.005
69/100 (epoch 69), train_loss = 9.447927, time/batch = 0.004
70/100 (epoch 70), train_loss = 10.198162, time/batch = 0.005
71/100 (epoch 71), train_loss = 9.307793, time/batch = 0.005
72/100 (epoch 72), train_loss = 10.854749, time/batch = 0.004
73/100 (epoch 73), train_loss = 10.428018, time/batch = 0.005
74/100 (epoch 74), train_loss = 8.488185, time/batch = 0.005
75/100 (epoch 75), train_loss = 8.085508, time/batch = 0.004
76/100 (epoch 76), train_loss = 10.553905, time/batch = 0.004
77/100 (epoch 77), train_loss = 12.255890, time/batch = 0.004
78/100 (epoch 78), train_loss = 9.149986, time/batch = 0.005
79/100 (epoch 79), train_loss = 9.229828, time/batch = 0.005
80/100 (epoch 80), train_loss = 10.565948, time/batch = 0.008
81/100 (epoch 81), train_loss = 10.386895, time/batch = 0.005
82/100 (epoch 82), train_loss = 10.007664, time/batch = 0.005
83/100 (epoch 83), train_loss = 9.739275, time/batch = 0.005
84/100 (epoch 84), train_loss = 11.818852, time/batch = 0.006
85/100 (epoch 85), train_loss = 11.200649, time/batch = 0.004
86/100 (epoch 86), train_loss = 10.338413, time/batch = 0.004
87/100 (epoch 87), train_loss = 10.309578, time/batch = 0.003
88/100 (epoch 88), train_loss = 9.559597, time/batch = 0.003
89/100 (epoch 89), train_loss = 9.074076, time/batch = 0.003
90/100 (epoch 90), train_loss = 10.043921, time/batch = 0.004
91/100 (epoch 91), train_loss = 9.096073, time/batch = 0.004
92/100 (epoch 92), train_loss = 11.605042, time/batch = 0.003
93/100 (epoch 93), train_loss = 12.427410, time/batch = 0.003
94/100 (epoch 94), train_loss = 10.214948, time/batch = 0.003
95/100 (epoch 95), train_loss = 10.150050, time/batch = 0.003
96/100 (epoch 96), train_loss = 10.088663, time/batch = 0.003
97/100 (epoch 97), train_loss = 11.131211, time/batch = 0.003
98/100 (epoch 98), train_loss = 12.051834, time/batch = 0.003
99/100 (epoch 99), train_loss = 9.969053, time/batch = 0.004

cost


In [25]:
# Check the cost
plt.figure(figsize=(12,5))
plt.plot(range(len(cost_optimisation)), cost_optimisation, label='cost')
plt.legend()
plt.show()


use the model to make a prediction


In [26]:
with sess.as_default():
    x_ = [ [xx[i]] for i in range(batch) ]
    feed = {x: x_}
    [prediction] = sess.run([pred], feed)

In [27]:
plt.figure(figsize=(12,5))
plt.plot(range(len(prediction)), prediction, label='prediction')
plt.legend()
plt.show()


inspect the model


In [28]:
with sess.as_default():
    for var in tf.all_variables():
        if var in tf.trainable_variables():
            print 't', var.name, var.eval().shape, ':'
            plt.figure(figsize=(1,1))
            plt.figimage(var.eval(), label=var.name)
            plt.show()
        else:
            print 'nt', var.name, var.eval().shape


nt learning_rate:0 ()
t Wxh:0 (1, 100) :
t Why:0 (100, 1) :
t bh:0 (1, 1) :
t by:0 (100, 1) :

Feedback wellcome @dh7net