Chapter 10 – Introduction to Artificial Neural Networks

This notebook contains all the sample code and solutions to the exercices in chapter 10.

Setup

First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:



In [1]:

    
# To support both python 2 and python 3
from __future__ import division, print_function, unicode_literals

# Common imports
import numpy as np
import os

# to make this notebook's output stable across runs
def reset_graph(seed=42):
    tf.reset_default_graph()
    tf.set_random_seed(seed)
    np.random.seed(seed)

# To plot pretty figures
%matplotlib inline
import matplotlib
import matplotlib.pyplot as plt
plt.rcParams['axes.labelsize'] = 14
plt.rcParams['xtick.labelsize'] = 12
plt.rcParams['ytick.labelsize'] = 12

# Where to save the figures
PROJECT_ROOT_DIR = "."
CHAPTER_ID = "ann"

def save_fig(fig_id, tight_layout=True):
    path = os.path.join(PROJECT_ROOT_DIR, "images", CHAPTER_ID, fig_id + ".png")
    print("Saving figure", fig_id)
    if tight_layout:
        plt.tight_layout()
    plt.savefig(path, format='png', dpi=300)

Perceptrons



In [2]:

    
import numpy as np
from sklearn.datasets import load_iris
from sklearn.linear_model import Perceptron

iris = load_iris()
X = iris.data[:, (2, 3)]  # petal length, petal width
y = (iris.target == 0).astype(np.int)

per_clf = Perceptron(random_state=42)
per_clf.fit(X, y)

y_pred = per_clf.predict([[2, 0.5]])



In [3]:

    
y_pred









    Out[3]:





array([1])



In [4]:

    
a = -per_clf.coef_[0][0] / per_clf.coef_[0][1]
b = -per_clf.intercept_ / per_clf.coef_[0][1]

axes = [0, 5, 0, 2]

x0, x1 = np.meshgrid(
        np.linspace(axes[0], axes[1], 500).reshape(-1, 1),
        np.linspace(axes[2], axes[3], 200).reshape(-1, 1),
    )
X_new = np.c_[x0.ravel(), x1.ravel()]
y_predict = per_clf.predict(X_new)
zz = y_predict.reshape(x0.shape)

plt.figure(figsize=(10, 4))
plt.plot(X[y==0, 0], X[y==0, 1], "bs", label="Not Iris-Setosa")
plt.plot(X[y==1, 0], X[y==1, 1], "yo", label="Iris-Setosa")

plt.plot([axes[0], axes[1]], [a * axes[0] + b, a * axes[1] + b], "k-", linewidth=3)
from matplotlib.colors import ListedColormap
custom_cmap = ListedColormap(['#9898ff', '#fafab0'])

plt.contourf(x0, x1, zz, cmap=custom_cmap, linewidth=5)
plt.xlabel("Petal length", fontsize=14)
plt.ylabel("Petal width", fontsize=14)
plt.legend(loc="lower right", fontsize=14)
plt.axis(axes)

save_fig("perceptron_iris_plot")
plt.show()









    



Saving figure perceptron_iris_plot

Activation functions



In [5]:

    
def logit(z):
    return 1 / (1 + np.exp(-z))

def relu(z):
    return np.maximum(0, z)

def derivative(f, z, eps=0.000001):
    return (f(z + eps) - f(z - eps))/(2 * eps)



In [6]:

    
z = np.linspace(-5, 5, 200)

plt.figure(figsize=(11,4))

plt.subplot(121)
plt.plot(z, np.sign(z), "r-", linewidth=2, label="Step")
plt.plot(z, logit(z), "g--", linewidth=2, label="Logit")
plt.plot(z, np.tanh(z), "b-", linewidth=2, label="Tanh")
plt.plot(z, relu(z), "m-.", linewidth=2, label="ReLU")
plt.grid(True)
plt.legend(loc="center right", fontsize=14)
plt.title("Activation functions", fontsize=14)
plt.axis([-5, 5, -1.2, 1.2])

plt.subplot(122)
plt.plot(z, derivative(np.sign, z), "r-", linewidth=2, label="Step")
plt.plot(0, 0, "ro", markersize=5)
plt.plot(0, 0, "rx", markersize=10)
plt.plot(z, derivative(logit, z), "g--", linewidth=2, label="Logit")
plt.plot(z, derivative(np.tanh, z), "b-", linewidth=2, label="Tanh")
plt.plot(z, derivative(relu, z), "m-.", linewidth=2, label="ReLU")
plt.grid(True)
#plt.legend(loc="center right", fontsize=14)
plt.title("Derivatives", fontsize=14)
plt.axis([-5, 5, -0.2, 1.2])

save_fig("activation_functions_plot")
plt.show()









    



Saving figure activation_functions_plot



In [7]:

    
def heaviside(z):
    return (z >= 0).astype(z.dtype)

def sigmoid(z):
    return 1/(1+np.exp(-z))

def mlp_xor(x1, x2, activation=heaviside):
    return activation(-activation(x1 + x2 - 1.5) + activation(x1 + x2 - 0.5) - 0.5)



In [8]:

    
x1s = np.linspace(-0.2, 1.2, 100)
x2s = np.linspace(-0.2, 1.2, 100)
x1, x2 = np.meshgrid(x1s, x2s)

z1 = mlp_xor(x1, x2, activation=heaviside)
z2 = mlp_xor(x1, x2, activation=sigmoid)

plt.figure(figsize=(10,4))

plt.subplot(121)
plt.contourf(x1, x2, z1)
plt.plot([0, 1], [0, 1], "gs", markersize=20)
plt.plot([0, 1], [1, 0], "y^", markersize=20)
plt.title("Activation function: heaviside", fontsize=14)
plt.grid(True)

plt.subplot(122)
plt.contourf(x1, x2, z2)
plt.plot([0, 1], [0, 1], "gs", markersize=20)
plt.plot([0, 1], [1, 0], "y^", markersize=20)
plt.title("Activation function: sigmoid", fontsize=14)
plt.grid(True)

FNN for MNIST

using tf.learn



In [9]:

    
from tensorflow.examples.tutorials.mnist import input_data

mnist = input_data.read_data_sets("/tmp/data/")









    



Extracting /tmp/data/train-images-idx3-ubyte.gz
Extracting /tmp/data/train-labels-idx1-ubyte.gz
Extracting /tmp/data/t10k-images-idx3-ubyte.gz
Extracting /tmp/data/t10k-labels-idx1-ubyte.gz



In [10]:

    
X_train = mnist.train.images
X_test = mnist.test.images
y_train = mnist.train.labels.astype("int")
y_test = mnist.test.labels.astype("int")



In [11]:

    
import tensorflow as tf

config = tf.contrib.learn.RunConfig(tf_random_seed=42) # not shown in the config

feature_cols = tf.contrib.learn.infer_real_valued_columns_from_input(X_train)
dnn_clf = tf.contrib.learn.DNNClassifier(hidden_units=[300,100], n_classes=10,
                                         feature_columns=feature_cols, config=config)
dnn_clf = tf.contrib.learn.SKCompat(dnn_clf) # if TensorFlow >= 1.1
dnn_clf.fit(X_train, y_train, batch_size=50, steps=40000)









    



INFO:tensorflow:Using config: {'_master': '', '_num_ps_replicas': 0, '_task_type': None, '_save_checkpoints_steps': None, '_tf_config': gpu_options {
  per_process_gpu_memory_fraction: 1.0
}
, '_evaluation_master': '', '_save_summary_steps': 100, '_save_checkpoints_secs': 600, '_num_worker_replicas': 0, '_model_dir': None, '_environment': 'local', '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f67f1f57d30>, '_is_chief': True, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_tf_random_seed': 42, '_task_id': 0}
WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmpecewk7kb
WARNING:tensorflow:From /home/ageron/dev/py/envs/ml/lib/python3.5/site-packages/tensorflow/contrib/learn/python/learn/estimators/head.py:615: scalar_summary (from tensorflow.python.ops.logging_ops) is deprecated and will be removed after 2016-11-30.
Instructions for updating:
Please switch to tf.summary.scalar. Note that tf.summary.scalar uses the node name instead of the tag. This means that TensorFlow will automatically de-duplicate summary names based on the scope they are created in. Also, passing a tensor or list of tags to a scalar summary op is no longer supported.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Saving checkpoints for 1 into /tmp/tmpecewk7kb/model.ckpt.
INFO:tensorflow:loss = 2.36404, step = 1
INFO:tensorflow:global_step/sec: 260.817
INFO:tensorflow:loss = 0.311432, step = 101 (0.383 sec)
INFO:tensorflow:global_step/sec: 260.391
INFO:tensorflow:loss = 0.265409, step = 201 (0.384 sec)
INFO:tensorflow:global_step/sec: 262.476
INFO:tensorflow:loss = 0.408733, step = 301 (0.381 sec)
INFO:tensorflow:global_step/sec: 261.314
INFO:tensorflow:loss = 0.244357, step = 401 (0.383 sec)
INFO:tensorflow:global_step/sec: 258.889
INFO:tensorflow:loss = 0.238858, step = 501 (0.386 sec)
INFO:tensorflow:global_step/sec: 258.784
INFO:tensorflow:loss = 0.0918271, step = 601 (0.386 sec)
INFO:tensorflow:global_step/sec: 164.338
INFO:tensorflow:loss = 0.123374, step = 701 (0.609 sec)
INFO:tensorflow:global_step/sec: 184.472
INFO:tensorflow:loss = 0.196473, step = 801 (0.542 sec)
INFO:tensorflow:global_step/sec: 187.728
INFO:tensorflow:loss = 0.0932024, step = 901 (0.533 sec)
INFO:tensorflow:global_step/sec: 189.618
INFO:tensorflow:loss = 0.196834, step = 1001 (0.527 sec)
INFO:tensorflow:global_step/sec: 264.9
INFO:tensorflow:loss = 0.194408, step = 1101 (0.377 sec)
INFO:tensorflow:global_step/sec: 262.289
INFO:tensorflow:loss = 0.152817, step = 1201 (0.381 sec)
INFO:tensorflow:global_step/sec: 262.723
INFO:tensorflow:loss = 0.149922, step = 1301 (0.381 sec)
INFO:tensorflow:global_step/sec: 164.842
INFO:tensorflow:loss = 0.0659786, step = 1401 (0.607 sec)
INFO:tensorflow:global_step/sec: 228.232
INFO:tensorflow:loss = 0.0721019, step = 1501 (0.438 sec)
INFO:tensorflow:global_step/sec: 202.979
INFO:tensorflow:loss = 0.120699, step = 1601 (0.493 sec)
INFO:tensorflow:global_step/sec: 228.385
INFO:tensorflow:loss = 0.0398833, step = 1701 (0.437 sec)
INFO:tensorflow:global_step/sec: 262.751
INFO:tensorflow:loss = 0.151045, step = 1801 (0.381 sec)
INFO:tensorflow:global_step/sec: 259.859
INFO:tensorflow:loss = 0.0709309, step = 1901 (0.385 sec)
INFO:tensorflow:global_step/sec: 260.716
INFO:tensorflow:loss = 0.0622928, step = 2001 (0.384 sec)
INFO:tensorflow:global_step/sec: 259.09
INFO:tensorflow:loss = 0.0229075, step = 2101 (0.386 sec)
INFO:tensorflow:global_step/sec: 259.745
INFO:tensorflow:loss = 0.0303926, step = 2201 (0.385 sec)
INFO:tensorflow:global_step/sec: 260.825
INFO:tensorflow:loss = 0.0458983, step = 2301 (0.383 sec)
INFO:tensorflow:global_step/sec: 259.816
INFO:tensorflow:loss = 0.0586828, step = 2401 (0.385 sec)
INFO:tensorflow:global_step/sec: 243.524
INFO:tensorflow:loss = 0.0920379, step = 2501 (0.411 sec)
INFO:tensorflow:global_step/sec: 224.828
INFO:tensorflow:loss = 0.0315541, step = 2601 (0.445 sec)
INFO:tensorflow:global_step/sec: 233.629
INFO:tensorflow:loss = 0.0114352, step = 2701 (0.428 sec)
INFO:tensorflow:global_step/sec: 185.194
INFO:tensorflow:loss = 0.0563824, step = 2801 (0.540 sec)
INFO:tensorflow:global_step/sec: 178.371
INFO:tensorflow:loss = 0.0811181, step = 2901 (0.561 sec)
INFO:tensorflow:global_step/sec: 173.41
INFO:tensorflow:loss = 0.0140813, step = 3001 (0.576 sec)
INFO:tensorflow:global_step/sec: 194.442
INFO:tensorflow:loss = 0.0322334, step = 3101 (0.514 sec)
INFO:tensorflow:global_step/sec: 186.116
INFO:tensorflow:loss = 0.0132405, step = 3201 (0.538 sec)
INFO:tensorflow:global_step/sec: 192.012
INFO:tensorflow:loss = 0.0339669, step = 3301 (0.520 sec)
INFO:tensorflow:global_step/sec: 182.959
INFO:tensorflow:loss = 0.157921, step = 3401 (0.547 sec)
INFO:tensorflow:global_step/sec: 182.487
INFO:tensorflow:loss = 0.0848148, step = 3501 (0.548 sec)
INFO:tensorflow:global_step/sec: 142.784
INFO:tensorflow:loss = 0.155988, step = 3601 (0.700 sec)
INFO:tensorflow:global_step/sec: 184.358
INFO:tensorflow:loss = 0.0346186, step = 3701 (0.542 sec)
INFO:tensorflow:global_step/sec: 169.954
INFO:tensorflow:loss = 0.00991458, step = 3801 (0.589 sec)
INFO:tensorflow:global_step/sec: 160.217
INFO:tensorflow:loss = 0.150861, step = 3901 (0.623 sec)
INFO:tensorflow:global_step/sec: 211.631
INFO:tensorflow:loss = 0.110915, step = 4001 (0.473 sec)
INFO:tensorflow:global_step/sec: 177.961
INFO:tensorflow:loss = 0.0493516, step = 4101 (0.562 sec)
INFO:tensorflow:global_step/sec: 178.605
INFO:tensorflow:loss = 0.0546287, step = 4201 (0.560 sec)
INFO:tensorflow:global_step/sec: 183.171
INFO:tensorflow:loss = 0.159896, step = 4301 (0.546 sec)
INFO:tensorflow:global_step/sec: 207.876
INFO:tensorflow:loss = 0.112916, step = 4401 (0.481 sec)
INFO:tensorflow:global_step/sec: 158.187
INFO:tensorflow:loss = 0.0161132, step = 4501 (0.632 sec)
INFO:tensorflow:global_step/sec: 241.573
INFO:tensorflow:loss = 0.0171333, step = 4601 (0.414 sec)
INFO:tensorflow:global_step/sec: 233.973
INFO:tensorflow:loss = 0.00835642, step = 4701 (0.427 sec)
INFO:tensorflow:global_step/sec: 236.421
INFO:tensorflow:loss = 0.01708, step = 4801 (0.423 sec)
INFO:tensorflow:global_step/sec: 237.868
INFO:tensorflow:loss = 0.0864718, step = 4901 (0.420 sec)
INFO:tensorflow:global_step/sec: 239.187
INFO:tensorflow:loss = 0.0439425, step = 5001 (0.418 sec)
INFO:tensorflow:global_step/sec: 238.716
INFO:tensorflow:loss = 0.00764107, step = 5101 (0.419 sec)
INFO:tensorflow:global_step/sec: 237.782
INFO:tensorflow:loss = 0.0232963, step = 5201 (0.421 sec)
INFO:tensorflow:global_step/sec: 238.707
INFO:tensorflow:loss = 0.046633, step = 5301 (0.419 sec)
INFO:tensorflow:global_step/sec: 239.04
INFO:tensorflow:loss = 0.066787, step = 5401 (0.418 sec)
INFO:tensorflow:global_step/sec: 252.49
INFO:tensorflow:loss = 0.0494149, step = 5501 (0.396 sec)
INFO:tensorflow:global_step/sec: 258.641
INFO:tensorflow:loss = 0.0707151, step = 5601 (0.387 sec)
INFO:tensorflow:global_step/sec: 241.49
INFO:tensorflow:loss = 0.0192079, step = 5701 (0.414 sec)
INFO:tensorflow:global_step/sec: 229.017
INFO:tensorflow:loss = 0.00933775, step = 5801 (0.437 sec)
INFO:tensorflow:global_step/sec: 235.112
INFO:tensorflow:loss = 0.106693, step = 5901 (0.425 sec)
INFO:tensorflow:global_step/sec: 233.677
INFO:tensorflow:loss = 0.0908673, step = 6001 (0.428 sec)
INFO:tensorflow:global_step/sec: 236.366
INFO:tensorflow:loss = 0.01711, step = 6101 (0.423 sec)
INFO:tensorflow:global_step/sec: 186.503
INFO:tensorflow:loss = 0.0224653, step = 6201 (0.536 sec)
INFO:tensorflow:global_step/sec: 167.806
INFO:tensorflow:loss = 0.0684235, step = 6301 (0.596 sec)
INFO:tensorflow:global_step/sec: 210.251
INFO:tensorflow:loss = 0.0247326, step = 6401 (0.476 sec)
INFO:tensorflow:global_step/sec: 197.723
INFO:tensorflow:loss = 0.00986267, step = 6501 (0.506 sec)
INFO:tensorflow:global_step/sec: 266.76
INFO:tensorflow:loss = 0.0220192, step = 6601 (0.375 sec)
INFO:tensorflow:global_step/sec: 264.179
INFO:tensorflow:loss = 0.0224969, step = 6701 (0.379 sec)
INFO:tensorflow:global_step/sec: 257.535
INFO:tensorflow:loss = 0.0130047, step = 6801 (0.388 sec)
INFO:tensorflow:global_step/sec: 260.336
INFO:tensorflow:loss = 0.0126607, step = 6901 (0.384 sec)
INFO:tensorflow:global_step/sec: 268.145
INFO:tensorflow:loss = 0.015101, step = 7001 (0.373 sec)
INFO:tensorflow:global_step/sec: 251.59
INFO:tensorflow:loss = 0.00366965, step = 7101 (0.397 sec)
INFO:tensorflow:global_step/sec: 237.158
INFO:tensorflow:loss = 0.044044, step = 7201 (0.422 sec)
INFO:tensorflow:global_step/sec: 232.033
INFO:tensorflow:loss = 0.00571019, step = 7301 (0.431 sec)
INFO:tensorflow:global_step/sec: 236.205
INFO:tensorflow:loss = 0.013417, step = 7401 (0.423 sec)
INFO:tensorflow:global_step/sec: 225.404
INFO:tensorflow:loss = 0.0043693, step = 7501 (0.444 sec)
INFO:tensorflow:global_step/sec: 238.452
INFO:tensorflow:loss = 0.0141966, step = 7601 (0.419 sec)
INFO:tensorflow:global_step/sec: 240.771
INFO:tensorflow:loss = 0.00819222, step = 7701 (0.415 sec)
INFO:tensorflow:global_step/sec: 229.499
INFO:tensorflow:loss = 0.00547122, step = 7801 (0.436 sec)
INFO:tensorflow:global_step/sec: 222.969
INFO:tensorflow:loss = 0.012388, step = 7901 (0.448 sec)
INFO:tensorflow:global_step/sec: 234.443
INFO:tensorflow:loss = 0.0043136, step = 8001 (0.427 sec)
INFO:tensorflow:global_step/sec: 181.467
INFO:tensorflow:loss = 0.0393611, step = 8101 (0.552 sec)
INFO:tensorflow:global_step/sec: 178.792
INFO:tensorflow:loss = 0.0268929, step = 8201 (0.559 sec)
INFO:tensorflow:global_step/sec: 185.438
INFO:tensorflow:loss = 0.0534401, step = 8301 (0.539 sec)
INFO:tensorflow:global_step/sec: 151.617
INFO:tensorflow:loss = 0.017818, step = 8401 (0.660 sec)
INFO:tensorflow:global_step/sec: 155.867
INFO:tensorflow:loss = 0.0116427, step = 8501 (0.642 sec)
INFO:tensorflow:global_step/sec: 181.382
INFO:tensorflow:loss = 0.00533878, step = 8601 (0.551 sec)
INFO:tensorflow:global_step/sec: 187.288
INFO:tensorflow:loss = 0.00569211, step = 8701 (0.534 sec)
INFO:tensorflow:global_step/sec: 205.664
INFO:tensorflow:loss = 0.00665956, step = 8801 (0.486 sec)
INFO:tensorflow:global_step/sec: 204.953
INFO:tensorflow:loss = 0.00315022, step = 8901 (0.488 sec)
INFO:tensorflow:global_step/sec: 194.252
INFO:tensorflow:loss = 0.0133965, step = 9001 (0.515 sec)
INFO:tensorflow:global_step/sec: 188.951
INFO:tensorflow:loss = 0.00906787, step = 9101 (0.529 sec)
INFO:tensorflow:global_step/sec: 173.39
INFO:tensorflow:loss = 0.00360787, step = 9201 (0.577 sec)
INFO:tensorflow:global_step/sec: 198.042
INFO:tensorflow:loss = 0.01566, step = 9301 (0.505 sec)
INFO:tensorflow:global_step/sec: 220.507
INFO:tensorflow:loss = 0.0403218, step = 9401 (0.454 sec)
INFO:tensorflow:global_step/sec: 220.935
INFO:tensorflow:loss = 0.00779154, step = 9501 (0.452 sec)
INFO:tensorflow:global_step/sec: 239.463
INFO:tensorflow:loss = 0.017734, step = 9601 (0.418 sec)
INFO:tensorflow:global_step/sec: 238.71
INFO:tensorflow:loss = 0.00776719, step = 9701 (0.419 sec)
INFO:tensorflow:global_step/sec: 236.017
INFO:tensorflow:loss = 0.00336712, step = 9801 (0.424 sec)
INFO:tensorflow:global_step/sec: 230.414
INFO:tensorflow:loss = 0.0176963, step = 9901 (0.434 sec)
INFO:tensorflow:global_step/sec: 229.642
INFO:tensorflow:loss = 0.0169004, step = 10001 (0.435 sec)
INFO:tensorflow:global_step/sec: 218.107
INFO:tensorflow:loss = 0.00473883, step = 10101 (0.458 sec)
INFO:tensorflow:global_step/sec: 202.282
INFO:tensorflow:loss = 0.00767333, step = 10201 (0.494 sec)
INFO:tensorflow:global_step/sec: 221.607
INFO:tensorflow:loss = 0.00469218, step = 10301 (0.451 sec)
INFO:tensorflow:global_step/sec: 211.051
INFO:tensorflow:loss = 0.00941437, step = 10401 (0.474 sec)
INFO:tensorflow:global_step/sec: 213.276
INFO:tensorflow:loss = 0.00360837, step = 10501 (0.469 sec)
INFO:tensorflow:global_step/sec: 204.939
INFO:tensorflow:loss = 0.00824566, step = 10601 (0.488 sec)
INFO:tensorflow:global_step/sec: 199.485
INFO:tensorflow:loss = 0.034342, step = 10701 (0.501 sec)
INFO:tensorflow:global_step/sec: 220.138
INFO:tensorflow:loss = 0.0160151, step = 10801 (0.454 sec)
INFO:tensorflow:global_step/sec: 232.804
INFO:tensorflow:loss = 0.00446511, step = 10901 (0.429 sec)
INFO:tensorflow:global_step/sec: 238.091
INFO:tensorflow:loss = 0.0278223, step = 11001 (0.420 sec)
INFO:tensorflow:global_step/sec: 236.775
INFO:tensorflow:loss = 0.00451434, step = 11101 (0.422 sec)
INFO:tensorflow:global_step/sec: 213.7
INFO:tensorflow:loss = 0.000942479, step = 11201 (0.468 sec)
INFO:tensorflow:global_step/sec: 215.302
INFO:tensorflow:loss = 0.0130732, step = 11301 (0.464 sec)
INFO:tensorflow:global_step/sec: 234.367
INFO:tensorflow:loss = 0.0126658, step = 11401 (0.427 sec)
INFO:tensorflow:global_step/sec: 201.051
INFO:tensorflow:loss = 0.0185032, step = 11501 (0.497 sec)
INFO:tensorflow:global_step/sec: 211.212
INFO:tensorflow:loss = 0.000567014, step = 11601 (0.473 sec)
INFO:tensorflow:global_step/sec: 170.486
INFO:tensorflow:loss = 0.00216249, step = 11701 (0.587 sec)
INFO:tensorflow:global_step/sec: 194.085
INFO:tensorflow:loss = 0.000577293, step = 11801 (0.515 sec)
INFO:tensorflow:global_step/sec: 215.625
INFO:tensorflow:loss = 0.00549651, step = 11901 (0.464 sec)
INFO:tensorflow:global_step/sec: 191.043
INFO:tensorflow:loss = 0.000167048, step = 12001 (0.524 sec)
INFO:tensorflow:global_step/sec: 154.278
INFO:tensorflow:loss = 0.00373405, step = 12101 (0.648 sec)
INFO:tensorflow:global_step/sec: 158.362
INFO:tensorflow:loss = 0.00398317, step = 12201 (0.631 sec)
INFO:tensorflow:global_step/sec: 195.115
INFO:tensorflow:loss = 0.00490612, step = 12301 (0.513 sec)
INFO:tensorflow:global_step/sec: 180.842
INFO:tensorflow:loss = 0.000254854, step = 12401 (0.553 sec)
INFO:tensorflow:global_step/sec: 239.513
INFO:tensorflow:loss = 0.00370345, step = 12501 (0.417 sec)
INFO:tensorflow:global_step/sec: 229.337
INFO:tensorflow:loss = 0.00152101, step = 12601 (0.436 sec)
INFO:tensorflow:global_step/sec: 235.719
INFO:tensorflow:loss = 0.00526951, step = 12701 (0.424 sec)
INFO:tensorflow:global_step/sec: 238.127
INFO:tensorflow:loss = 0.00592783, step = 12801 (0.420 sec)
INFO:tensorflow:global_step/sec: 224.962
INFO:tensorflow:loss = 0.00324891, step = 12901 (0.445 sec)
INFO:tensorflow:global_step/sec: 133.41
INFO:tensorflow:loss = 0.00343657, step = 13001 (0.749 sec)
INFO:tensorflow:global_step/sec: 161.258
INFO:tensorflow:loss = 0.00340272, step = 13101 (0.620 sec)
INFO:tensorflow:global_step/sec: 237.132
INFO:tensorflow:loss = 0.00556199, step = 13201 (0.422 sec)
INFO:tensorflow:global_step/sec: 228.502
INFO:tensorflow:loss = 0.0042778, step = 13301 (0.438 sec)
INFO:tensorflow:global_step/sec: 232.921
INFO:tensorflow:loss = 0.00246822, step = 13401 (0.429 sec)
INFO:tensorflow:global_step/sec: 238.2
INFO:tensorflow:loss = 0.00610874, step = 13501 (0.420 sec)
INFO:tensorflow:global_step/sec: 203.273
INFO:tensorflow:loss = 0.00441995, step = 13601 (0.492 sec)
INFO:tensorflow:global_step/sec: 238.859
INFO:tensorflow:loss = 0.0023879, step = 13701 (0.419 sec)
INFO:tensorflow:global_step/sec: 250.859
INFO:tensorflow:loss = 0.0074444, step = 13801 (0.399 sec)
INFO:tensorflow:global_step/sec: 260.437
INFO:tensorflow:loss = 0.00315427, step = 13901 (0.384 sec)
INFO:tensorflow:global_step/sec: 248.155
INFO:tensorflow:loss = 0.00219044, step = 14001 (0.403 sec)
INFO:tensorflow:global_step/sec: 236.323
INFO:tensorflow:loss = 0.00482443, step = 14101 (0.423 sec)
INFO:tensorflow:global_step/sec: 247.825
INFO:tensorflow:loss = 0.00988796, step = 14201 (0.404 sec)
INFO:tensorflow:global_step/sec: 254.179
INFO:tensorflow:loss = 0.00231401, step = 14301 (0.393 sec)
INFO:tensorflow:global_step/sec: 248.802
INFO:tensorflow:loss = 0.000743411, step = 14401 (0.402 sec)
INFO:tensorflow:global_step/sec: 255.943
INFO:tensorflow:loss = 0.00100101, step = 14501 (0.391 sec)
INFO:tensorflow:global_step/sec: 231.248
INFO:tensorflow:loss = 0.00342037, step = 14601 (0.432 sec)
INFO:tensorflow:global_step/sec: 225.419
INFO:tensorflow:loss = 0.00153701, step = 14701 (0.444 sec)
INFO:tensorflow:global_step/sec: 236.466
INFO:tensorflow:loss = 0.00118163, step = 14801 (0.423 sec)
INFO:tensorflow:global_step/sec: 233.832
INFO:tensorflow:loss = 0.00225318, step = 14901 (0.428 sec)
INFO:tensorflow:global_step/sec: 227.359
INFO:tensorflow:loss = 0.00111509, step = 15001 (0.440 sec)
INFO:tensorflow:global_step/sec: 233.517
INFO:tensorflow:loss = 0.00252178, step = 15101 (0.428 sec)
INFO:tensorflow:global_step/sec: 232.674
INFO:tensorflow:loss = 0.00225656, step = 15201 (0.430 sec)
INFO:tensorflow:global_step/sec: 221.965
INFO:tensorflow:loss = 0.00173806, step = 15301 (0.451 sec)
INFO:tensorflow:global_step/sec: 228.725
INFO:tensorflow:loss = 0.00382904, step = 15401 (0.437 sec)
INFO:tensorflow:global_step/sec: 220.263
INFO:tensorflow:loss = 0.00451967, step = 15501 (0.454 sec)
INFO:tensorflow:global_step/sec: 185.685
INFO:tensorflow:loss = 0.0047358, step = 15601 (0.538 sec)
INFO:tensorflow:global_step/sec: 214.146
INFO:tensorflow:loss = 0.0145779, step = 15701 (0.467 sec)
INFO:tensorflow:global_step/sec: 211.497
INFO:tensorflow:loss = 0.00161293, step = 15801 (0.473 sec)
INFO:tensorflow:global_step/sec: 200.982
INFO:tensorflow:loss = 0.000548923, step = 15901 (0.498 sec)
INFO:tensorflow:global_step/sec: 215.662
INFO:tensorflow:loss = 0.00568894, step = 16001 (0.464 sec)
INFO:tensorflow:global_step/sec: 208.946
INFO:tensorflow:loss = 0.00496605, step = 16101 (0.479 sec)
INFO:tensorflow:global_step/sec: 219.495
INFO:tensorflow:loss = 7.30671e-05, step = 16201 (0.456 sec)
INFO:tensorflow:global_step/sec: 207.948
INFO:tensorflow:loss = 0.00304547, step = 16301 (0.481 sec)
INFO:tensorflow:global_step/sec: 204.766
INFO:tensorflow:loss = 0.00170099, step = 16401 (0.488 sec)
INFO:tensorflow:global_step/sec: 235.146
INFO:tensorflow:loss = 0.00151268, step = 16501 (0.425 sec)
INFO:tensorflow:global_step/sec: 234.786
INFO:tensorflow:loss = 0.00499173, step = 16601 (0.426 sec)
INFO:tensorflow:global_step/sec: 233.865
INFO:tensorflow:loss = 0.00237819, step = 16701 (0.428 sec)
INFO:tensorflow:global_step/sec: 224.686
INFO:tensorflow:loss = 0.00212857, step = 16801 (0.445 sec)
INFO:tensorflow:global_step/sec: 237.371
INFO:tensorflow:loss = 0.0025386, step = 16901 (0.421 sec)
INFO:tensorflow:global_step/sec: 228.135
INFO:tensorflow:loss = 0.00303927, step = 17001 (0.438 sec)
INFO:tensorflow:global_step/sec: 234.993
INFO:tensorflow:loss = 0.000547433, step = 17101 (0.425 sec)
INFO:tensorflow:global_step/sec: 222.667
INFO:tensorflow:loss = 0.0022543, step = 17201 (0.449 sec)
INFO:tensorflow:global_step/sec: 225.896
INFO:tensorflow:loss = 0.000823148, step = 17301 (0.442 sec)
INFO:tensorflow:global_step/sec: 210.336
INFO:tensorflow:loss = 0.00171766, step = 17401 (0.475 sec)
INFO:tensorflow:global_step/sec: 203.11
INFO:tensorflow:loss = 0.00192135, step = 17501 (0.492 sec)
INFO:tensorflow:global_step/sec: 212.637
INFO:tensorflow:loss = 0.000309051, step = 17601 (0.470 sec)
INFO:tensorflow:global_step/sec: 208.182
INFO:tensorflow:loss = 0.000621968, step = 17701 (0.480 sec)
INFO:tensorflow:global_step/sec: 164.634
INFO:tensorflow:loss = 0.000520324, step = 17801 (0.607 sec)
INFO:tensorflow:global_step/sec: 210.295
INFO:tensorflow:loss = 0.00209451, step = 17901 (0.475 sec)
INFO:tensorflow:global_step/sec: 174.574
INFO:tensorflow:loss = 0.00160342, step = 18001 (0.573 sec)
INFO:tensorflow:global_step/sec: 132.402
INFO:tensorflow:loss = 0.00103904, step = 18101 (0.756 sec)
INFO:tensorflow:global_step/sec: 155.235
INFO:tensorflow:loss = 0.00412339, step = 18201 (0.645 sec)
INFO:tensorflow:global_step/sec: 147.223
INFO:tensorflow:loss = 0.014485, step = 18301 (0.678 sec)
INFO:tensorflow:global_step/sec: 147.963
INFO:tensorflow:loss = 0.0034526, step = 18401 (0.676 sec)
INFO:tensorflow:global_step/sec: 152.085
INFO:tensorflow:loss = 0.00173879, step = 18501 (0.658 sec)
INFO:tensorflow:global_step/sec: 143.207
INFO:tensorflow:loss = 0.00422458, step = 18601 (0.698 sec)
INFO:tensorflow:global_step/sec: 164.083
INFO:tensorflow:loss = 0.000858988, step = 18701 (0.609 sec)
INFO:tensorflow:global_step/sec: 132.525
INFO:tensorflow:loss = 0.00143733, step = 18801 (0.755 sec)
INFO:tensorflow:global_step/sec: 176.179
INFO:tensorflow:loss = 0.00293665, step = 18901 (0.568 sec)
INFO:tensorflow:global_step/sec: 224.261
INFO:tensorflow:loss = 0.001538, step = 19001 (0.446 sec)
INFO:tensorflow:global_step/sec: 228.922
INFO:tensorflow:loss = 0.000944429, step = 19101 (0.437 sec)
INFO:tensorflow:global_step/sec: 231.406
INFO:tensorflow:loss = 0.00041447, step = 19201 (0.432 sec)
INFO:tensorflow:global_step/sec: 230.819
INFO:tensorflow:loss = 0.00526123, step = 19301 (0.433 sec)
INFO:tensorflow:global_step/sec: 231.187
INFO:tensorflow:loss = 0.000478011, step = 19401 (0.433 sec)
INFO:tensorflow:global_step/sec: 235.88
INFO:tensorflow:loss = 0.00260038, step = 19501 (0.424 sec)
INFO:tensorflow:global_step/sec: 230.394
INFO:tensorflow:loss = 0.000504177, step = 19601 (0.434 sec)
INFO:tensorflow:global_step/sec: 232.283
INFO:tensorflow:loss = 0.000130909, step = 19701 (0.430 sec)
INFO:tensorflow:global_step/sec: 229.826
INFO:tensorflow:loss = 0.000504942, step = 19801 (0.435 sec)
INFO:tensorflow:global_step/sec: 227.287
INFO:tensorflow:loss = 0.00158035, step = 19901 (0.440 sec)
INFO:tensorflow:global_step/sec: 233.08
INFO:tensorflow:loss = 0.0020706, step = 20001 (0.429 sec)
INFO:tensorflow:global_step/sec: 229.328
INFO:tensorflow:loss = 0.000324922, step = 20101 (0.436 sec)
INFO:tensorflow:global_step/sec: 205.141
INFO:tensorflow:loss = 0.00199532, step = 20201 (0.487 sec)
INFO:tensorflow:global_step/sec: 216.646
INFO:tensorflow:loss = 0.00110691, step = 20301 (0.461 sec)
INFO:tensorflow:global_step/sec: 234.478
INFO:tensorflow:loss = 0.0001444, step = 20401 (0.426 sec)
INFO:tensorflow:global_step/sec: 251.37
INFO:tensorflow:loss = 0.000286913, step = 20501 (0.398 sec)
INFO:tensorflow:global_step/sec: 223.94
INFO:tensorflow:loss = 0.00139893, step = 20601 (0.447 sec)
INFO:tensorflow:global_step/sec: 241.293
INFO:tensorflow:loss = 0.00105777, step = 20701 (0.414 sec)
INFO:tensorflow:global_step/sec: 262.75
INFO:tensorflow:loss = 0.00169489, step = 20801 (0.381 sec)
INFO:tensorflow:global_step/sec: 249.916
INFO:tensorflow:loss = 0.00156369, step = 20901 (0.400 sec)
INFO:tensorflow:global_step/sec: 226.122
INFO:tensorflow:loss = 0.00274591, step = 21001 (0.443 sec)
INFO:tensorflow:global_step/sec: 224.531
INFO:tensorflow:loss = 0.00134583, step = 21101 (0.445 sec)
INFO:tensorflow:global_step/sec: 257.902
INFO:tensorflow:loss = 0.00222729, step = 21201 (0.387 sec)
INFO:tensorflow:global_step/sec: 217.25
INFO:tensorflow:loss = 0.000948768, step = 21301 (0.460 sec)
INFO:tensorflow:global_step/sec: 205.733
INFO:tensorflow:loss = 0.00258286, step = 21401 (0.488 sec)
INFO:tensorflow:global_step/sec: 208.844
INFO:tensorflow:loss = 0.000207113, step = 21501 (0.477 sec)
INFO:tensorflow:global_step/sec: 147.089
INFO:tensorflow:loss = 0.00130515, step = 21601 (0.680 sec)
INFO:tensorflow:global_step/sec: 217.076
INFO:tensorflow:loss = 0.000956831, step = 21701 (0.460 sec)
INFO:tensorflow:global_step/sec: 231.529
INFO:tensorflow:loss = 0.000395721, step = 21801 (0.432 sec)
INFO:tensorflow:global_step/sec: 147.527
INFO:tensorflow:loss = 0.000597151, step = 21901 (0.678 sec)
INFO:tensorflow:global_step/sec: 227.008
INFO:tensorflow:loss = 0.000139627, step = 22001 (0.440 sec)
INFO:tensorflow:global_step/sec: 152.091
INFO:tensorflow:loss = 0.000522552, step = 22101 (0.658 sec)
INFO:tensorflow:global_step/sec: 122.338
INFO:tensorflow:loss = 0.00138648, step = 22201 (0.819 sec)
INFO:tensorflow:global_step/sec: 166.695
INFO:tensorflow:loss = 0.00254255, step = 22301 (0.598 sec)
INFO:tensorflow:global_step/sec: 182.221
INFO:tensorflow:loss = 0.00194729, step = 22401 (0.549 sec)
INFO:tensorflow:global_step/sec: 200.809
INFO:tensorflow:loss = 0.00153244, step = 22501 (0.498 sec)
INFO:tensorflow:global_step/sec: 195.102
INFO:tensorflow:loss = 0.00344794, step = 22601 (0.513 sec)
INFO:tensorflow:global_step/sec: 190.696
INFO:tensorflow:loss = 0.00100069, step = 22701 (0.525 sec)
INFO:tensorflow:global_step/sec: 174.991
INFO:tensorflow:loss = 0.00124635, step = 22801 (0.571 sec)
INFO:tensorflow:global_step/sec: 175.633
INFO:tensorflow:loss = 0.00231376, step = 22901 (0.569 sec)
INFO:tensorflow:global_step/sec: 195.749
INFO:tensorflow:loss = 0.000602279, step = 23001 (0.511 sec)
INFO:tensorflow:global_step/sec: 214.824
INFO:tensorflow:loss = 0.00305156, step = 23101 (0.466 sec)
INFO:tensorflow:global_step/sec: 204.676
INFO:tensorflow:loss = 0.00235002, step = 23201 (0.489 sec)
INFO:tensorflow:global_step/sec: 228.647
INFO:tensorflow:loss = 0.00154166, step = 23301 (0.437 sec)
INFO:tensorflow:global_step/sec: 226.05
INFO:tensorflow:loss = 0.000704969, step = 23401 (0.442 sec)
INFO:tensorflow:global_step/sec: 231.15
INFO:tensorflow:loss = 0.000767915, step = 23501 (0.433 sec)
INFO:tensorflow:global_step/sec: 199.699
INFO:tensorflow:loss = 0.00063945, step = 23601 (0.501 sec)
INFO:tensorflow:global_step/sec: 223.919
INFO:tensorflow:loss = 0.000301056, step = 23701 (0.447 sec)
INFO:tensorflow:global_step/sec: 245.216
INFO:tensorflow:loss = 0.00175696, step = 23801 (0.408 sec)
INFO:tensorflow:global_step/sec: 240.635
INFO:tensorflow:loss = 0.00106804, step = 23901 (0.416 sec)
INFO:tensorflow:global_step/sec: 235.93
INFO:tensorflow:loss = 0.000982589, step = 24001 (0.424 sec)
INFO:tensorflow:global_step/sec: 236.769
INFO:tensorflow:loss = 0.000534493, step = 24101 (0.422 sec)
INFO:tensorflow:global_step/sec: 213.622
INFO:tensorflow:loss = 0.000924729, step = 24201 (0.468 sec)
INFO:tensorflow:global_step/sec: 221.864
INFO:tensorflow:loss = 0.000206531, step = 24301 (0.451 sec)
INFO:tensorflow:global_step/sec: 199.399
INFO:tensorflow:loss = 0.00184635, step = 24401 (0.501 sec)
INFO:tensorflow:global_step/sec: 219.656
INFO:tensorflow:loss = 0.000563592, step = 24501 (0.455 sec)
INFO:tensorflow:global_step/sec: 215.674
INFO:tensorflow:loss = 0.000240078, step = 24601 (0.464 sec)
INFO:tensorflow:global_step/sec: 202.946
INFO:tensorflow:loss = 0.000911916, step = 24701 (0.493 sec)
INFO:tensorflow:global_step/sec: 203.138
INFO:tensorflow:loss = 0.00135369, step = 24801 (0.492 sec)
INFO:tensorflow:global_step/sec: 228.396
INFO:tensorflow:loss = 0.00251554, step = 24901 (0.438 sec)
INFO:tensorflow:global_step/sec: 241.946
INFO:tensorflow:loss = 0.000153224, step = 25001 (0.413 sec)
INFO:tensorflow:global_step/sec: 235.858
INFO:tensorflow:loss = 0.00190746, step = 25101 (0.424 sec)
INFO:tensorflow:global_step/sec: 184.528
INFO:tensorflow:loss = 0.00279237, step = 25201 (0.542 sec)
INFO:tensorflow:global_step/sec: 186.822
INFO:tensorflow:loss = 0.000147187, step = 25301 (0.536 sec)
INFO:tensorflow:global_step/sec: 187.126
INFO:tensorflow:loss = 0.000728114, step = 25401 (0.534 sec)
INFO:tensorflow:global_step/sec: 197.134
INFO:tensorflow:loss = 0.00135055, step = 25501 (0.507 sec)
INFO:tensorflow:global_step/sec: 200.474
INFO:tensorflow:loss = 0.00101404, step = 25601 (0.499 sec)
INFO:tensorflow:global_step/sec: 217.685
INFO:tensorflow:loss = 0.000952221, step = 25701 (0.459 sec)
INFO:tensorflow:global_step/sec: 220.52
INFO:tensorflow:loss = 0.00164703, step = 25801 (0.453 sec)
INFO:tensorflow:global_step/sec: 221.992
INFO:tensorflow:loss = 0.00165784, step = 25901 (0.450 sec)
INFO:tensorflow:global_step/sec: 221.295
INFO:tensorflow:loss = 5.12721e-05, step = 26001 (0.452 sec)
INFO:tensorflow:global_step/sec: 223.473
INFO:tensorflow:loss = 0.00105168, step = 26101 (0.447 sec)
INFO:tensorflow:global_step/sec: 242.589
INFO:tensorflow:loss = 0.000811974, step = 26201 (0.412 sec)
INFO:tensorflow:global_step/sec: 243.138
INFO:tensorflow:loss = 0.000871437, step = 26301 (0.411 sec)
INFO:tensorflow:global_step/sec: 243.198
INFO:tensorflow:loss = 0.00176584, step = 26401 (0.411 sec)
INFO:tensorflow:global_step/sec: 247.072
INFO:tensorflow:loss = 0.000849402, step = 26501 (0.405 sec)
INFO:tensorflow:global_step/sec: 240.846
INFO:tensorflow:loss = 0.000518591, step = 26601 (0.415 sec)
INFO:tensorflow:global_step/sec: 241.065
INFO:tensorflow:loss = 1.50678e-05, step = 26701 (0.415 sec)
INFO:tensorflow:global_step/sec: 242.412
INFO:tensorflow:loss = 0.000841992, step = 26801 (0.413 sec)
INFO:tensorflow:global_step/sec: 243.623
INFO:tensorflow:loss = 0.000838306, step = 26901 (0.411 sec)
INFO:tensorflow:global_step/sec: 243.662
INFO:tensorflow:loss = 0.000587201, step = 27001 (0.410 sec)
INFO:tensorflow:global_step/sec: 232.392
INFO:tensorflow:loss = 0.00046221, step = 27101 (0.430 sec)
INFO:tensorflow:global_step/sec: 217.556
INFO:tensorflow:loss = 0.000655521, step = 27201 (0.460 sec)
INFO:tensorflow:global_step/sec: 218.291
INFO:tensorflow:loss = 0.00101177, step = 27301 (0.458 sec)
INFO:tensorflow:global_step/sec: 219.019
INFO:tensorflow:loss = 0.000149555, step = 27401 (0.457 sec)
INFO:tensorflow:global_step/sec: 215.76
INFO:tensorflow:loss = 0.00147278, step = 27501 (0.463 sec)
INFO:tensorflow:global_step/sec: 217.106
INFO:tensorflow:loss = 0.000980559, step = 27601 (0.461 sec)
INFO:tensorflow:global_step/sec: 217.779
INFO:tensorflow:loss = 0.000261994, step = 27701 (0.459 sec)
INFO:tensorflow:global_step/sec: 216.324
INFO:tensorflow:loss = 0.000522326, step = 27801 (0.462 sec)
INFO:tensorflow:global_step/sec: 219.423
INFO:tensorflow:loss = 0.000382858, step = 27901 (0.456 sec)
INFO:tensorflow:global_step/sec: 228.358
INFO:tensorflow:loss = 0.00150058, step = 28001 (0.438 sec)
INFO:tensorflow:global_step/sec: 237.938
INFO:tensorflow:loss = 0.000560936, step = 28101 (0.420 sec)
INFO:tensorflow:global_step/sec: 236.436
INFO:tensorflow:loss = 0.00132494, step = 28201 (0.423 sec)
INFO:tensorflow:global_step/sec: 238.36
INFO:tensorflow:loss = 0.000744157, step = 28301 (0.420 sec)
INFO:tensorflow:global_step/sec: 230.558
INFO:tensorflow:loss = 0.000808092, step = 28401 (0.434 sec)
INFO:tensorflow:global_step/sec: 236.566
INFO:tensorflow:loss = 0.000633052, step = 28501 (0.423 sec)
INFO:tensorflow:global_step/sec: 236.807
INFO:tensorflow:loss = 0.00040288, step = 28601 (0.422 sec)
INFO:tensorflow:global_step/sec: 236.69
INFO:tensorflow:loss = 0.000790712, step = 28701 (0.423 sec)
INFO:tensorflow:global_step/sec: 239.467
INFO:tensorflow:loss = 0.00100996, step = 28801 (0.418 sec)
INFO:tensorflow:global_step/sec: 227.861
INFO:tensorflow:loss = 0.000233286, step = 28901 (0.439 sec)
INFO:tensorflow:global_step/sec: 217.61
INFO:tensorflow:loss = 0.00175366, step = 29001 (0.460 sec)
INFO:tensorflow:global_step/sec: 216.918
INFO:tensorflow:loss = 0.00104872, step = 29101 (0.461 sec)
INFO:tensorflow:global_step/sec: 219.206
INFO:tensorflow:loss = 0.00168296, step = 29201 (0.456 sec)
INFO:tensorflow:global_step/sec: 219.362
INFO:tensorflow:loss = 0.00105643, step = 29301 (0.456 sec)
INFO:tensorflow:global_step/sec: 222.068
INFO:tensorflow:loss = 0.000981568, step = 29401 (0.450 sec)
INFO:tensorflow:global_step/sec: 219.518
INFO:tensorflow:loss = 0.000960704, step = 29501 (0.456 sec)
INFO:tensorflow:global_step/sec: 218.85
INFO:tensorflow:loss = 0.000378366, step = 29601 (0.457 sec)
INFO:tensorflow:global_step/sec: 222.507
INFO:tensorflow:loss = 0.000236591, step = 29701 (0.449 sec)
INFO:tensorflow:global_step/sec: 230.935
INFO:tensorflow:loss = 0.000619082, step = 29801 (0.433 sec)
INFO:tensorflow:global_step/sec: 243.681
INFO:tensorflow:loss = 0.000708774, step = 29901 (0.410 sec)
INFO:tensorflow:global_step/sec: 242.447
INFO:tensorflow:loss = 9.85982e-05, step = 30001 (0.412 sec)
INFO:tensorflow:global_step/sec: 241.814
INFO:tensorflow:loss = 0.000597217, step = 30101 (0.414 sec)
INFO:tensorflow:global_step/sec: 237.015
INFO:tensorflow:loss = 0.000242105, step = 30201 (0.422 sec)
INFO:tensorflow:global_step/sec: 235.772
INFO:tensorflow:loss = 0.000666237, step = 30301 (0.424 sec)
INFO:tensorflow:global_step/sec: 240.741
INFO:tensorflow:loss = 0.00107709, step = 30401 (0.415 sec)
INFO:tensorflow:global_step/sec: 239.014
INFO:tensorflow:loss = 0.000853609, step = 30501 (0.418 sec)
INFO:tensorflow:global_step/sec: 238.176
INFO:tensorflow:loss = 0.00153748, step = 30601 (0.420 sec)
INFO:tensorflow:global_step/sec: 240.268
INFO:tensorflow:loss = 0.00100649, step = 30701 (0.416 sec)
INFO:tensorflow:global_step/sec: 257.441
INFO:tensorflow:loss = 0.00119904, step = 30801 (0.388 sec)
INFO:tensorflow:global_step/sec: 258.798
INFO:tensorflow:loss = 0.00121068, step = 30901 (0.386 sec)
INFO:tensorflow:global_step/sec: 267.911
INFO:tensorflow:loss = 0.000979147, step = 31001 (0.373 sec)
INFO:tensorflow:global_step/sec: 264.721
INFO:tensorflow:loss = 0.00142901, step = 31101 (0.378 sec)
INFO:tensorflow:global_step/sec: 266.221
INFO:tensorflow:loss = 0.000594258, step = 31201 (0.376 sec)
INFO:tensorflow:global_step/sec: 265.146
INFO:tensorflow:loss = 0.000413715, step = 31301 (0.377 sec)
INFO:tensorflow:global_step/sec: 267.588
INFO:tensorflow:loss = 0.000751181, step = 31401 (0.374 sec)
INFO:tensorflow:global_step/sec: 265.785
INFO:tensorflow:loss = 0.000252517, step = 31501 (0.376 sec)
INFO:tensorflow:global_step/sec: 260.745
INFO:tensorflow:loss = 8.00835e-05, step = 31601 (0.384 sec)
INFO:tensorflow:global_step/sec: 271.475
INFO:tensorflow:loss = 0.000599239, step = 31701 (0.368 sec)
INFO:tensorflow:global_step/sec: 257.765
INFO:tensorflow:loss = 0.00010314, step = 31801 (0.388 sec)
INFO:tensorflow:global_step/sec: 244.79
INFO:tensorflow:loss = 0.000702046, step = 31901 (0.408 sec)
INFO:tensorflow:global_step/sec: 236.558
INFO:tensorflow:loss = 0.000189627, step = 32001 (0.423 sec)
INFO:tensorflow:global_step/sec: 236.523
INFO:tensorflow:loss = 0.000453534, step = 32101 (0.423 sec)
INFO:tensorflow:global_step/sec: 238.548
INFO:tensorflow:loss = 0.00113025, step = 32201 (0.419 sec)
INFO:tensorflow:global_step/sec: 237.276
INFO:tensorflow:loss = 0.000369316, step = 32301 (0.421 sec)
INFO:tensorflow:global_step/sec: 239.846
INFO:tensorflow:loss = 9.61032e-05, step = 32401 (0.417 sec)
INFO:tensorflow:global_step/sec: 240.832
INFO:tensorflow:loss = 0.000561119, step = 32501 (0.415 sec)
INFO:tensorflow:global_step/sec: 245.837
INFO:tensorflow:loss = 0.00012762, step = 32601 (0.407 sec)
INFO:tensorflow:global_step/sec: 244.065
INFO:tensorflow:loss = 0.00137255, step = 32701 (0.410 sec)
INFO:tensorflow:global_step/sec: 241.211
INFO:tensorflow:loss = 0.000887949, step = 32801 (0.415 sec)
INFO:tensorflow:global_step/sec: 246.688
INFO:tensorflow:loss = 0.000337199, step = 32901 (0.405 sec)
INFO:tensorflow:global_step/sec: 237.944
INFO:tensorflow:loss = 0.000573135, step = 33001 (0.420 sec)
INFO:tensorflow:global_step/sec: 247.035
INFO:tensorflow:loss = 0.00195268, step = 33101 (0.405 sec)
INFO:tensorflow:global_step/sec: 241.959
INFO:tensorflow:loss = 0.000947158, step = 33201 (0.413 sec)
INFO:tensorflow:global_step/sec: 239.726
INFO:tensorflow:loss = 0.00028633, step = 33301 (0.417 sec)
INFO:tensorflow:global_step/sec: 214.33
INFO:tensorflow:loss = 0.00124017, step = 33401 (0.467 sec)
INFO:tensorflow:global_step/sec: 193.288
INFO:tensorflow:loss = 0.000209001, step = 33501 (0.517 sec)
INFO:tensorflow:global_step/sec: 191.715
INFO:tensorflow:loss = 0.0010908, step = 33601 (0.522 sec)
INFO:tensorflow:global_step/sec: 193.058
INFO:tensorflow:loss = 0.000751158, step = 33701 (0.518 sec)
INFO:tensorflow:global_step/sec: 193.318
INFO:tensorflow:loss = 0.00117691, step = 33801 (0.517 sec)
INFO:tensorflow:global_step/sec: 192.555
INFO:tensorflow:loss = 0.00109041, step = 33901 (0.519 sec)
INFO:tensorflow:global_step/sec: 194.089
INFO:tensorflow:loss = 0.000558409, step = 34001 (0.515 sec)
INFO:tensorflow:global_step/sec: 190.68
INFO:tensorflow:loss = 0.000955204, step = 34101 (0.524 sec)
INFO:tensorflow:global_step/sec: 212.115
INFO:tensorflow:loss = 0.00114629, step = 34201 (0.471 sec)
INFO:tensorflow:global_step/sec: 220.762
INFO:tensorflow:loss = 0.000116086, step = 34301 (0.453 sec)
INFO:tensorflow:global_step/sec: 220.499
INFO:tensorflow:loss = 0.000376291, step = 34401 (0.454 sec)
INFO:tensorflow:global_step/sec: 220.746
INFO:tensorflow:loss = 0.000596384, step = 34501 (0.453 sec)
INFO:tensorflow:global_step/sec: 219.763
INFO:tensorflow:loss = 0.000287235, step = 34601 (0.455 sec)
INFO:tensorflow:global_step/sec: 166.882
INFO:tensorflow:loss = 0.0017038, step = 34701 (0.599 sec)
INFO:tensorflow:global_step/sec: 193.247
INFO:tensorflow:loss = 0.000414888, step = 34801 (0.518 sec)
INFO:tensorflow:global_step/sec: 168.67
INFO:tensorflow:loss = 0.000323204, step = 34901 (0.593 sec)
INFO:tensorflow:global_step/sec: 215.312
INFO:tensorflow:loss = 0.000240949, step = 35001 (0.464 sec)
INFO:tensorflow:global_step/sec: 209.156
INFO:tensorflow:loss = 0.00105711, step = 35101 (0.478 sec)
INFO:tensorflow:global_step/sec: 214.048
INFO:tensorflow:loss = 0.000331689, step = 35201 (0.467 sec)
INFO:tensorflow:global_step/sec: 172.105
INFO:tensorflow:loss = 0.000143141, step = 35301 (0.581 sec)
INFO:tensorflow:global_step/sec: 218.348
INFO:tensorflow:loss = 0.00099728, step = 35401 (0.458 sec)
INFO:tensorflow:global_step/sec: 216.362
INFO:tensorflow:loss = 0.000150676, step = 35501 (0.462 sec)
INFO:tensorflow:global_step/sec: 229.565
INFO:tensorflow:loss = 0.000298942, step = 35601 (0.435 sec)
INFO:tensorflow:global_step/sec: 172.278
INFO:tensorflow:loss = 0.000777814, step = 35701 (0.580 sec)
INFO:tensorflow:global_step/sec: 189.537
INFO:tensorflow:loss = 0.000844667, step = 35801 (0.528 sec)
INFO:tensorflow:global_step/sec: 236.462
INFO:tensorflow:loss = 0.000575924, step = 35901 (0.423 sec)
INFO:tensorflow:global_step/sec: 208.91
INFO:tensorflow:loss = 0.000289464, step = 36001 (0.478 sec)
INFO:tensorflow:global_step/sec: 180.32
INFO:tensorflow:loss = 0.00115901, step = 36101 (0.555 sec)
INFO:tensorflow:global_step/sec: 169.349
INFO:tensorflow:loss = 0.000844626, step = 36201 (0.591 sec)
INFO:tensorflow:global_step/sec: 156.569
INFO:tensorflow:loss = 8.0961e-05, step = 36301 (0.639 sec)
INFO:tensorflow:global_step/sec: 133.603
INFO:tensorflow:loss = 0.000652663, step = 36401 (0.748 sec)
INFO:tensorflow:global_step/sec: 163.481
INFO:tensorflow:loss = 0.000233111, step = 36501 (0.612 sec)
INFO:tensorflow:global_step/sec: 171.56
INFO:tensorflow:loss = 0.000246441, step = 36601 (0.583 sec)
INFO:tensorflow:global_step/sec: 161.011
INFO:tensorflow:loss = 0.000453849, step = 36701 (0.621 sec)
INFO:tensorflow:global_step/sec: 198.293
INFO:tensorflow:loss = 0.000899995, step = 36801 (0.504 sec)
INFO:tensorflow:global_step/sec: 181.652
INFO:tensorflow:loss = 0.000713761, step = 36901 (0.550 sec)
INFO:tensorflow:global_step/sec: 190.299
INFO:tensorflow:loss = 0.000382187, step = 37001 (0.525 sec)
INFO:tensorflow:global_step/sec: 200.515
INFO:tensorflow:loss = 0.000168937, step = 37101 (0.499 sec)
INFO:tensorflow:global_step/sec: 205.368
INFO:tensorflow:loss = 0.000345245, step = 37201 (0.487 sec)
INFO:tensorflow:global_step/sec: 195.98
INFO:tensorflow:loss = 0.000750656, step = 37301 (0.510 sec)
INFO:tensorflow:global_step/sec: 212.979
INFO:tensorflow:loss = 0.000352557, step = 37401 (0.470 sec)
INFO:tensorflow:global_step/sec: 210.174
INFO:tensorflow:loss = 0.000393667, step = 37501 (0.476 sec)
INFO:tensorflow:global_step/sec: 156.95
INFO:tensorflow:loss = 0.000323634, step = 37601 (0.637 sec)
INFO:tensorflow:global_step/sec: 172.972
INFO:tensorflow:loss = 0.00011668, step = 37701 (0.578 sec)
INFO:tensorflow:global_step/sec: 209.986
INFO:tensorflow:loss = 0.000419136, step = 37801 (0.476 sec)
INFO:tensorflow:global_step/sec: 242.398
INFO:tensorflow:loss = 0.00133944, step = 37901 (0.413 sec)
INFO:tensorflow:global_step/sec: 232.131
INFO:tensorflow:loss = 0.000302842, step = 38001 (0.431 sec)
INFO:tensorflow:global_step/sec: 192.622
INFO:tensorflow:loss = 0.00128635, step = 38101 (0.519 sec)
INFO:tensorflow:global_step/sec: 228.307
INFO:tensorflow:loss = 0.000813133, step = 38201 (0.438 sec)
INFO:tensorflow:global_step/sec: 235.862
INFO:tensorflow:loss = 0.000126317, step = 38301 (0.424 sec)
INFO:tensorflow:global_step/sec: 239.058
INFO:tensorflow:loss = 0.000201125, step = 38401 (0.418 sec)
INFO:tensorflow:global_step/sec: 214.99
INFO:tensorflow:loss = 0.000416319, step = 38501 (0.465 sec)
INFO:tensorflow:global_step/sec: 216.723
INFO:tensorflow:loss = 0.000711198, step = 38601 (0.461 sec)
INFO:tensorflow:global_step/sec: 240.79
INFO:tensorflow:loss = 0.000432415, step = 38701 (0.415 sec)
INFO:tensorflow:global_step/sec: 239.449
INFO:tensorflow:loss = 0.000230968, step = 38801 (0.418 sec)
INFO:tensorflow:global_step/sec: 224.749
INFO:tensorflow:loss = 0.00131089, step = 38901 (0.445 sec)
INFO:tensorflow:global_step/sec: 185.815
INFO:tensorflow:loss = 0.000195044, step = 39001 (0.538 sec)
INFO:tensorflow:global_step/sec: 186.065
INFO:tensorflow:loss = 0.000720432, step = 39101 (0.537 sec)
INFO:tensorflow:global_step/sec: 189.428
INFO:tensorflow:loss = 0.000604851, step = 39201 (0.528 sec)
INFO:tensorflow:global_step/sec: 190.859
INFO:tensorflow:loss = 0.000287668, step = 39301 (0.524 sec)
INFO:tensorflow:global_step/sec: 188.808
INFO:tensorflow:loss = 0.000637942, step = 39401 (0.530 sec)
INFO:tensorflow:global_step/sec: 180.328
INFO:tensorflow:loss = 0.000344775, step = 39501 (0.554 sec)
INFO:tensorflow:global_step/sec: 191.627
INFO:tensorflow:loss = 0.000685066, step = 39601 (0.522 sec)
INFO:tensorflow:global_step/sec: 212.621
INFO:tensorflow:loss = 0.000204588, step = 39701 (0.470 sec)
INFO:tensorflow:global_step/sec: 216.98
INFO:tensorflow:loss = 0.00110618, step = 39801 (0.461 sec)
INFO:tensorflow:global_step/sec: 215.497
INFO:tensorflow:loss = 0.0012423, step = 39901 (0.464 sec)
INFO:tensorflow:Saving checkpoints for 40000 into /tmp/tmpecewk7kb/model.ckpt.
INFO:tensorflow:Loss for final step: 0.000481317.






    Out[11]:





SKCompat()



In [12]:

    
from sklearn.metrics import accuracy_score

y_pred = dnn_clf.predict(X_test)
accuracy_score(y_test, y_pred['classes'])









    



INFO:tensorflow:Restoring parameters from /tmp/tmpecewk7kb/model.ckpt-40000






    Out[12]:





0.98250000000000004



In [13]:

    
from sklearn.metrics import log_loss

y_pred_proba = y_pred['probabilities']
log_loss(y_test, y_pred_proba)









    Out[13]:





0.071487383848950564

Using plain TensorFlow



In [14]:

    
import tensorflow as tf

n_inputs = 28*28  # MNIST
n_hidden1 = 300
n_hidden2 = 100
n_outputs = 10



In [15]:

    
reset_graph()

X = tf.placeholder(tf.float32, shape=(None, n_inputs), name="X")
y = tf.placeholder(tf.int64, shape=(None), name="y")



In [16]:

    
def neuron_layer(X, n_neurons, name, activation=None):
    with tf.name_scope(name):
        n_inputs = int(X.get_shape()[1])
        stddev = 2 / np.sqrt(n_inputs)
        init = tf.truncated_normal((n_inputs, n_neurons), stddev=stddev)
        W = tf.Variable(init, name="kernel")
        b = tf.Variable(tf.zeros([n_neurons]), name="bias")
        Z = tf.matmul(X, W) + b
        if activation is not None:
            return activation(Z)
        else:
            return Z



In [17]:

    
with tf.name_scope("dnn"):
    hidden1 = neuron_layer(X, n_hidden1, name="hidden1",
                           activation=tf.nn.relu)
    hidden2 = neuron_layer(hidden1, n_hidden2, name="hidden2",
                           activation=tf.nn.relu)
    logits = neuron_layer(hidden2, n_outputs, name="outputs")



In [18]:

    
with tf.name_scope("loss"):
    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y,
                                                              logits=logits)
    loss = tf.reduce_mean(xentropy, name="loss")



In [19]:

    
learning_rate = 0.01

with tf.name_scope("train"):
    optimizer = tf.train.GradientDescentOptimizer(learning_rate)
    training_op = optimizer.minimize(loss)



In [20]:

    
with tf.name_scope("eval"):
    correct = tf.nn.in_top_k(logits, y, 1)
    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))



In [21]:

    
init = tf.global_variables_initializer()
saver = tf.train.Saver()



In [22]:

    
n_epochs = 40
batch_size = 50



In [23]:

    
with tf.Session() as sess:
    init.run()
    for epoch in range(n_epochs):
        for iteration in range(mnist.train.num_examples // batch_size):
            X_batch, y_batch = mnist.train.next_batch(batch_size)
            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})
        acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})
        acc_test = accuracy.eval(feed_dict={X: mnist.test.images,
                                            y: mnist.test.labels})
        print(epoch, "Train accuracy:", acc_train, "Test accuracy:", acc_test)

    save_path = saver.save(sess, "./my_model_final.ckpt")









    



0 Train accuracy: 0.9 Test accuracy: 0.9128
1 Train accuracy: 0.94 Test accuracy: 0.9291
2 Train accuracy: 0.92 Test accuracy: 0.9398
3 Train accuracy: 0.96 Test accuracy: 0.9449
4 Train accuracy: 0.92 Test accuracy: 0.9511
5 Train accuracy: 0.94 Test accuracy: 0.954
6 Train accuracy: 0.98 Test accuracy: 0.9559
7 Train accuracy: 0.96 Test accuracy: 0.9594
8 Train accuracy: 0.92 Test accuracy: 0.9625
9 Train accuracy: 0.96 Test accuracy: 0.9646
10 Train accuracy: 0.98 Test accuracy: 0.965
11 Train accuracy: 0.94 Test accuracy: 0.9669
12 Train accuracy: 1.0 Test accuracy: 0.9693
13 Train accuracy: 0.94 Test accuracy: 0.9683
14 Train accuracy: 1.0 Test accuracy: 0.9695
15 Train accuracy: 1.0 Test accuracy: 0.9713
16 Train accuracy: 1.0 Test accuracy: 0.9711
17 Train accuracy: 0.98 Test accuracy: 0.9714
18 Train accuracy: 1.0 Test accuracy: 0.9728
19 Train accuracy: 1.0 Test accuracy: 0.973
20 Train accuracy: 0.98 Test accuracy: 0.9736
21 Train accuracy: 1.0 Test accuracy: 0.9736
22 Train accuracy: 1.0 Test accuracy: 0.9743
23 Train accuracy: 1.0 Test accuracy: 0.975
24 Train accuracy: 1.0 Test accuracy: 0.975
25 Train accuracy: 1.0 Test accuracy: 0.9756
26 Train accuracy: 1.0 Test accuracy: 0.975
27 Train accuracy: 1.0 Test accuracy: 0.9759
28 Train accuracy: 0.98 Test accuracy: 0.9761
29 Train accuracy: 1.0 Test accuracy: 0.9765
30 Train accuracy: 1.0 Test accuracy: 0.9764
31 Train accuracy: 1.0 Test accuracy: 0.9769
32 Train accuracy: 0.98 Test accuracy: 0.9766
33 Train accuracy: 0.98 Test accuracy: 0.9763
34 Train accuracy: 0.98 Test accuracy: 0.9786
35 Train accuracy: 0.98 Test accuracy: 0.9774
36 Train accuracy: 0.98 Test accuracy: 0.9774
37 Train accuracy: 1.0 Test accuracy: 0.9785
38 Train accuracy: 1.0 Test accuracy: 0.9785
39 Train accuracy: 1.0 Test accuracy: 0.9781



In [24]:

    
with tf.Session() as sess:
    saver.restore(sess, "./my_model_final.ckpt") # or better, use save_path
    X_new_scaled = mnist.test.images[:20]
    Z = logits.eval(feed_dict={X: X_new_scaled})
    y_pred = np.argmax(Z, axis=1)









    



INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt



In [25]:

    
print("Predicted classes:", y_pred)
print("Actual classes:   ", mnist.test.labels[:20])









    



Predicted classes: [7 2 1 0 4 1 4 9 6 9 0 6 9 0 1 5 9 7 3 4]
Actual classes:    [7 2 1 0 4 1 4 9 5 9 0 6 9 0 1 5 9 7 3 4]



In [26]:

    
from IPython.display import clear_output, Image, display, HTML

def strip_consts(graph_def, max_const_size=32):
    """Strip large constant values from graph_def."""
    strip_def = tf.GraphDef()
    for n0 in graph_def.node:
        n = strip_def.node.add() 
        n.MergeFrom(n0)
        if n.op == 'Const':
            tensor = n.attr['value'].tensor
            size = len(tensor.tensor_content)
            if size > max_const_size:
                tensor.tensor_content = b"<stripped %d bytes>"%size
    return strip_def

def show_graph(graph_def, max_const_size=32):
    """Visualize TensorFlow graph."""
    if hasattr(graph_def, 'as_graph_def'):
        graph_def = graph_def.as_graph_def()
    strip_def = strip_consts(graph_def, max_const_size=max_const_size)
    code = """
        <script>
          function load() {{
            document.getElementById("{id}").pbtxt = {data};
          }}
        </script>
        <link rel="import" href="https://tensorboard.appspot.com/tf-graph-basic.build.html" onload=load()>
        <div style="height:600px">
          <tf-graph-basic id="{id}"></tf-graph-basic>
        </div>
    """.format(data=repr(str(strip_def)), id='graph'+str(np.random.rand()))

    iframe = """
        <iframe seamless style="width:1200px;height:620px;border:0" srcdoc="{}"></iframe>
    """.format(code.replace('"', '&quot;'))
    display(HTML(iframe))



In [27]:

    
show_graph(tf.get_default_graph())

Using `dense()` instead of `neuron_layer()`

Note: the book uses tensorflow.contrib.layers.fully_connected() rather than tf.layers.dense() (which did not exist when this chapter was written). It is now preferable to use tf.layers.dense(), because anything in the contrib module may change or be deleted without notice. The dense() function is almost identical to the fully_connected() function, except for a few minor differences:

several parameters are renamed: scope becomes name, activation_fn becomes activation (and similarly the _fn suffix is removed from other parameters such as normalizer_fn), weights_initializer becomes kernel_initializer, etc.
the default activation is now None rather than tf.nn.relu.
a few more differences are presented in chapter 11.



In [28]:

    
n_inputs = 28*28  # MNIST
n_hidden1 = 300
n_hidden2 = 100
n_outputs = 10



In [29]:

    
reset_graph()

X = tf.placeholder(tf.float32, shape=(None, n_inputs), name="X")
y = tf.placeholder(tf.int64, shape=(None), name="y")



In [30]:

    
with tf.name_scope("dnn"):
    hidden1 = tf.layers.dense(X, n_hidden1, name="hidden1",
                              activation=tf.nn.relu)
    hidden2 = tf.layers.dense(hidden1, n_hidden2, name="hidden2",
                              activation=tf.nn.relu)
    logits = tf.layers.dense(hidden2, n_outputs, name="outputs")



In [31]:

    
with tf.name_scope("loss"):
    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)
    loss = tf.reduce_mean(xentropy, name="loss")



In [32]:

    
learning_rate = 0.01

with tf.name_scope("train"):
    optimizer = tf.train.GradientDescentOptimizer(learning_rate)
    training_op = optimizer.minimize(loss)



In [33]:

    
with tf.name_scope("eval"):
    correct = tf.nn.in_top_k(logits, y, 1)
    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))



In [34]:

    
init = tf.global_variables_initializer()
saver = tf.train.Saver()



In [35]:

    
n_epochs = 20
n_batches = 50

with tf.Session() as sess:
    init.run()
    for epoch in range(n_epochs):
        for iteration in range(mnist.train.num_examples // batch_size):
            X_batch, y_batch = mnist.train.next_batch(batch_size)
            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})
        acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})
        acc_test = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})
        print(epoch, "Train accuracy:", acc_train, "Test accuracy:", acc_test)

    save_path = saver.save(sess, "./my_model_final.ckpt")









    



0 Train accuracy: 0.9 Test accuracy: 0.9053
1 Train accuracy: 0.88 Test accuracy: 0.9206
2 Train accuracy: 0.94 Test accuracy: 0.9301
3 Train accuracy: 0.94 Test accuracy: 0.9397
4 Train accuracy: 0.92 Test accuracy: 0.9451
5 Train accuracy: 0.94 Test accuracy: 0.9476
6 Train accuracy: 0.92 Test accuracy: 0.9515
7 Train accuracy: 0.98 Test accuracy: 0.9546
8 Train accuracy: 0.96 Test accuracy: 0.9569
9 Train accuracy: 0.94 Test accuracy: 0.9605
10 Train accuracy: 0.92 Test accuracy: 0.9619
11 Train accuracy: 0.96 Test accuracy: 0.9631
12 Train accuracy: 1.0 Test accuracy: 0.9661
13 Train accuracy: 0.94 Test accuracy: 0.9657
14 Train accuracy: 1.0 Test accuracy: 0.9669
15 Train accuracy: 0.94 Test accuracy: 0.9682
16 Train accuracy: 0.96 Test accuracy: 0.9701
17 Train accuracy: 0.98 Test accuracy: 0.9696
18 Train accuracy: 1.0 Test accuracy: 0.97
19 Train accuracy: 1.0 Test accuracy: 0.971



In [36]:

    
show_graph(tf.get_default_graph())

Exercise solutions

1. to 8.

See appendix A.

9.

Train a deep MLP on the MNIST dataset and see if you can get over 98% precision. Just like in the last exercise of chapter 9, try adding all the bells and whistles (i.e., save checkpoints, restore the last checkpoint in case of an interruption, add summaries, plot learning curves using TensorBoard, and so on).

First let's create the deep net. It's exactly the same as earlier, with just one addition: we add a tf.summary.scalar() to track the loss and the accuracy during training, so we can view nice learning curves using TensorBoard.



In [37]:

    
n_inputs = 28*28  # MNIST
n_hidden1 = 300
n_hidden2 = 100
n_outputs = 10



In [38]:

    
reset_graph()

X = tf.placeholder(tf.float32, shape=(None, n_inputs), name="X")
y = tf.placeholder(tf.int64, shape=(None), name="y")



In [39]:

    
with tf.name_scope("dnn"):
    hidden1 = tf.layers.dense(X, n_hidden1, name="hidden1",
                              activation=tf.nn.relu)
    hidden2 = tf.layers.dense(hidden1, n_hidden2, name="hidden2",
                              activation=tf.nn.relu)
    logits = tf.layers.dense(hidden2, n_outputs, name="outputs")



In [40]:

    
with tf.name_scope("loss"):
    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)
    loss = tf.reduce_mean(xentropy, name="loss")
    loss_summary = tf.summary.scalar('log_loss', loss)



In [41]:

    
learning_rate = 0.01

with tf.name_scope("train"):
    optimizer = tf.train.GradientDescentOptimizer(learning_rate)
    training_op = optimizer.minimize(loss)



In [42]:

    
with tf.name_scope("eval"):
    correct = tf.nn.in_top_k(logits, y, 1)
    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))
    accuracy_summary = tf.summary.scalar('accuracy', accuracy)



In [43]:

    
init = tf.global_variables_initializer()
saver = tf.train.Saver()

Now we need to define the directory to write the TensorBoard logs to:



In [44]:

    
from datetime import datetime

def log_dir(prefix=""):
    now = datetime.utcnow().strftime("%Y%m%d%H%M%S")
    root_logdir = "tf_logs"
    if prefix:
        prefix += "-"
    name = prefix + "run-" + now
    return "{}/{}/".format(root_logdir, name)



In [45]:

    
logdir = log_dir("mnist_dnn")

Now we can create the FileWriter that we will use to write the TensorBoard logs:



In [46]:

    
file_writer = tf.summary.FileWriter(logdir, tf.get_default_graph())

Hey! Why don't we implement early stopping? For this, we are going to need a validation set. Luckily, the dataset returned by TensorFlow's input_data() function (see above) is already split into a training set (60,000 instances, already shuffled for us), a validation set (5,000 instances) and a test set (5,000 instances). So we can easily define X_valid and y_valid:



In [47]:

    
X_valid = mnist.validation.images
y_valid = mnist.validation.labels



In [48]:

    
m, n = X_train.shape



In [49]:

    
n_epochs = 10001
batch_size = 50
n_batches = int(np.ceil(m / batch_size))

checkpoint_path = "/tmp/my_deep_mnist_model.ckpt"
checkpoint_epoch_path = checkpoint_path + ".epoch"
final_model_path = "./my_deep_mnist_model"

best_loss = np.infty
epochs_without_progress = 0
max_epochs_without_progress = 50

with tf.Session() as sess:
    if os.path.isfile(checkpoint_epoch_path):
        # if the checkpoint file exists, restore the model and load the epoch number
        with open(checkpoint_epoch_path, "rb") as f:
            start_epoch = int(f.read())
        print("Training was interrupted. Continuing at epoch", start_epoch)
        saver.restore(sess, checkpoint_path)
    else:
        start_epoch = 0
        sess.run(init)

    for epoch in range(start_epoch, n_epochs):
        for iteration in range(mnist.train.num_examples // batch_size):
            X_batch, y_batch = mnist.train.next_batch(batch_size)
            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})
        accuracy_val, loss_val, accuracy_summary_str, loss_summary_str = sess.run([accuracy, loss, accuracy_summary, loss_summary], feed_dict={X: X_valid, y: y_valid})
        file_writer.add_summary(accuracy_summary_str, epoch)
        file_writer.add_summary(loss_summary_str, epoch)
        if epoch % 5 == 0:
            print("Epoch:", epoch,
                  "\tValidation accuracy: {:.3f}%".format(accuracy_val * 100),
                  "\tLoss: {:.5f}".format(loss_val))
            saver.save(sess, checkpoint_path)
            with open(checkpoint_epoch_path, "wb") as f:
                f.write(b"%d" % (epoch + 1))
            if loss_val < best_loss:
                saver.save(sess, final_model_path)
                best_loss = loss_val
            else:
                epochs_without_progress += 5
                if epochs_without_progress > max_epochs_without_progress:
                    print("Early stopping")
                    break









    



Epoch: 0 	Validation accuracy: 90.440% 	Loss: 0.35228
Epoch: 5 	Validation accuracy: 95.060% 	Loss: 0.17539
Epoch: 10 	Validation accuracy: 96.680% 	Loss: 0.12546
Epoch: 15 	Validation accuracy: 97.220% 	Loss: 0.10438
Epoch: 20 	Validation accuracy: 97.600% 	Loss: 0.08914
Epoch: 25 	Validation accuracy: 97.740% 	Loss: 0.08115
Epoch: 30 	Validation accuracy: 97.780% 	Loss: 0.07788
Epoch: 35 	Validation accuracy: 97.920% 	Loss: 0.07094
Epoch: 40 	Validation accuracy: 97.920% 	Loss: 0.06983
Epoch: 45 	Validation accuracy: 97.880% 	Loss: 0.06778
Epoch: 50 	Validation accuracy: 98.100% 	Loss: 0.06649
Epoch: 55 	Validation accuracy: 98.080% 	Loss: 0.06642
Epoch: 60 	Validation accuracy: 98.220% 	Loss: 0.06510
Epoch: 65 	Validation accuracy: 98.060% 	Loss: 0.06588
Epoch: 70 	Validation accuracy: 98.080% 	Loss: 0.06762
Epoch: 75 	Validation accuracy: 98.160% 	Loss: 0.06705
Epoch: 80 	Validation accuracy: 98.160% 	Loss: 0.06705
Epoch: 85 	Validation accuracy: 98.200% 	Loss: 0.06709
Epoch: 90 	Validation accuracy: 98.180% 	Loss: 0.06698
Epoch: 95 	Validation accuracy: 98.180% 	Loss: 0.06890
Epoch: 100 	Validation accuracy: 98.220% 	Loss: 0.06838
Epoch: 105 	Validation accuracy: 98.120% 	Loss: 0.06893
Epoch: 110 	Validation accuracy: 98.180% 	Loss: 0.06980
Epoch: 115 	Validation accuracy: 98.240% 	Loss: 0.07049
Early stopping



In [50]:

    
os.remove(checkpoint_epoch_path)



In [51]:

    
with tf.Session() as sess:
    saver.restore(sess, final_model_path)
    accuracy_val = accuracy.eval(feed_dict={X: X_test, y: y_test})









    



INFO:tensorflow:Restoring parameters from ./my_deep_mnist_model



In [52]:

    
accuracy_val









    Out[52]:





0.97839999



In [ ]: