In [11]:

    

import argparse
import sys

import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data

log_dir = "mnist_with_summaries"
data_dir = "mnist"
fake_data = False
learning_rate = 0.001
dropout = 0.9
max_steps=1000

def train():
    # Import data
    mnist = input_data.read_data_sets(data_dir, 
                                     one_hot = True,
                                     fake_data = fake_data)
    sess = tf.InteractiveSession()
    
    # Input placeholders
    with tf.name_scope('input'):
        x = tf.placeholder(tf.float32, [None, 784], name='x-input')
        y_ = tf.placeholder(tf.float32, [None, 10], name='y-input')
        
    with tf.name_scope('input_reshape'):
        image_shaped_input = tf.reshape(x, [-1, 28, 28, 1])
        tf.summary.image('input', image_shaped_input, 10)
        
    # we can't initialize these variables to 0 - the network will get stuck.
    def weight_variable(shape):
        initial = tf.truncated_normal(shape, stddev=0.1)
        return tf.Variable(initial)
    
    def bias_variable(shape):
        initial = tf.constant(0.1, shape=shape)
        return tf.Variable(initial)
    
    def variable_summaries(var):
        with tf.name_scope('summaries'):
            mean = tf.reduce_mean(var)
            tf.summary.scalar('mean', mean)
            with tf.name_scope('stddev'):
                stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
            tf.summary.scalar('stddev', stddev)
            tf.summary.scalar('max', tf.reduce_max(var))
            tf.summary.scalar('min', tf.reduce_min(var))
            tf.summary.histogram('histogram', var)
            
    def nn_layer(input_tensor, input_dim, output_dim, layer_name, act=tf.nn.relu):
        with tf.name_scope(layer_name):
            # this Variable will hold the state of the weights for the layer
            with tf.name_scope('weights'):
                weights = weight_variable([input_dim, output_dim])
                variable_summaries(weights)
            with tf.name_scope('biases'):
                biases = bias_variable([output_dim])
                variable_summaries(biases)
            with tf.name_scope('Wx_plus_b'):
                preactivate = tf.matmul(input_tensor, weights) + biases
                tf.summary.histogram('pre_activate', preactivate)
            activations = act(preactivate, name='activation')
            tf.summary.histogram('activation', activations)
            return activations
    
    hidden1 = nn_layer(x, 784, 500, 'layer1')
    
    with tf.name_scope('dropout'):
        keep_prob = tf.placeholder(tf.float32)
        tf.summary.scalar('dropout_keep_probability', keep_prob)
        dropped = tf.nn.dropout(hidden1, keep_prob)
        
    # Do not apply softmax activation yet.
    y = nn_layer(dropped, 500, 10, 'layer2', act=tf.identity)
    
    with tf.name_scope('cross_entropy'):
        # The raw formulation of cross-entropy
        diff = tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)
        with tf.name_scope('total'):
            cross_entropy = tf.reduce_mean(diff)
    tf.summary.scalar('cross_entropy', cross_entropy)
    
    with tf.name_scope('train'):
        train_step = tf.train.AdamOptimizer(learning_rate).minimize(cross_entropy)
        
    with tf.name_scope('accuracy'):
        with tf.name_scope('correct_prediction'):
            correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
        with tf.name_scope('accuracy'):
            accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
    tf.summary.scalar('accuracy', accuracy)
    
    merged = tf.summary.merge_all()
    train_writer = tf.summary.FileWriter(log_dir + '/train', sess.graph)
    test_writer = tf.summary.FileWriter(log_dir + '/test')
    tf.global_variables_initializer().run()
    
    def feed_dict(train):
        if train or fake_data:
            xs, ys = mnist.train.next_batch(100, fake_data=fake_data)
            k = dropout
        else:
            xs, ys = mnist.test.images, mnist.test.labels
            k = 1.0
        return {x: xs, y_:ys, keep_prob : k}
    
    for i in range(max_steps):
        if i % 10 == 0:
            summary, acc = sess.run([merged, accuracy], feed_dict = feed_dict(False))
            test_writer.add_summary(summary, i)
            print('Accuracy at step %s: %s' % (i, acc))
        else: # Record train set summaries, and train
            if i % 100 == 99: # Record execution stats
                run_options = tf.RunOptions(trace_level = tf.RunOptions.FULL_TRACE)
                run_metadata = tf.RunMetadata()
                summary, _ = sess.run([merged, train_step],
                                     feed_dict=feed_dict(True),
                                     options = run_options,
                                     run_metadata=run_metadata)
                train_writer.add_run_metadata(run_metadata, 'step%03d' % i)
                train_writer.add_summary(summary, i)
                print('Add run metadata for', i)
            else:
                summary, _ = sess.run([merged, train_step], feed_dict = feed_dict(True))
                train_writer.add_summary(summary, i)
    train_writer.close()
    test_writer.close()
    
if tf.gfile.Exists(log_dir):
    tf.gfile.DeleteRecursively(log_dir)
tf.gfile.MakeDirs(log_dir)
train()


    

    




Extracting mnist\train-images-idx3-ubyte.gz
Extracting mnist\train-labels-idx1-ubyte.gz
Extracting mnist\t10k-images-idx3-ubyte.gz
Extracting mnist\t10k-labels-idx1-ubyte.gz
Accuracy at step 0: 0.1436
Accuracy at step 10: 0.6761
Accuracy at step 20: 0.8029
Accuracy at step 30: 0.8447
Accuracy at step 40: 0.8759
Accuracy at step 50: 0.8757
Accuracy at step 60: 0.8831
Accuracy at step 70: 0.8858
Accuracy at step 80: 0.8853
Accuracy at step 90: 0.8997
Add run metadata for 99
Accuracy at step 100: 0.9031
Accuracy at step 110: 0.9143
Accuracy at step 120: 0.9181
Accuracy at step 130: 0.9189
Accuracy at step 140: 0.927
Accuracy at step 150: 0.917
Accuracy at step 160: 0.9286
Accuracy at step 170: 0.9278
Accuracy at step 180: 0.9197
Accuracy at step 190: 0.9252
Add run metadata for 199
Accuracy at step 200: 0.932
Accuracy at step 210: 0.9335
Accuracy at step 220: 0.9307
Accuracy at step 230: 0.9326
Accuracy at step 240: 0.9331
Accuracy at step 250: 0.9276
Accuracy at step 260: 0.9345
Accuracy at step 270: 0.9379
Accuracy at step 280: 0.9329
Accuracy at step 290: 0.9383
Add run metadata for 299
Accuracy at step 300: 0.9418
Accuracy at step 310: 0.9395
Accuracy at step 320: 0.9387
Accuracy at step 330: 0.9404
Accuracy at step 340: 0.9456
Accuracy at step 350: 0.9431
Accuracy at step 360: 0.941
Accuracy at step 370: 0.942
Accuracy at step 380: 0.9455
Accuracy at step 390: 0.9424
Add run metadata for 399
Accuracy at step 400: 0.9419
Accuracy at step 410: 0.9435
Accuracy at step 420: 0.9427
Accuracy at step 430: 0.9448
Accuracy at step 440: 0.9492
Accuracy at step 450: 0.9455
Accuracy at step 460: 0.9482
Accuracy at step 470: 0.946
Accuracy at step 480: 0.95
Accuracy at step 490: 0.9512
Add run metadata for 499
Accuracy at step 500: 0.9516
Accuracy at step 510: 0.9526
Accuracy at step 520: 0.9521
Accuracy at step 530: 0.9535
Accuracy at step 540: 0.9544
Accuracy at step 550: 0.9552
Accuracy at step 560: 0.9562
Accuracy at step 570: 0.9546
Accuracy at step 580: 0.9543
Accuracy at step 590: 0.9552
Add run metadata for 599
Accuracy at step 600: 0.9557
Accuracy at step 610: 0.9547
Accuracy at step 620: 0.9569
Accuracy at step 630: 0.9588
Accuracy at step 640: 0.9589
Accuracy at step 650: 0.9605
Accuracy at step 660: 0.9601
Accuracy at step 670: 0.9587
Accuracy at step 680: 0.9595
Accuracy at step 690: 0.9604
Add run metadata for 699
Accuracy at step 700: 0.9612
Accuracy at step 710: 0.9602
Accuracy at step 720: 0.9602
Accuracy at step 730: 0.9588
Accuracy at step 740: 0.962
Accuracy at step 750: 0.9617
Accuracy at step 760: 0.9619
Accuracy at step 770: 0.9625
Accuracy at step 780: 0.9606
Accuracy at step 790: 0.9631
Add run metadata for 799
Accuracy at step 800: 0.9631
Accuracy at step 810: 0.9623
Accuracy at step 820: 0.9628
Accuracy at step 830: 0.9636
Accuracy at step 840: 0.9618
Accuracy at step 850: 0.9621
Accuracy at step 860: 0.9638
Accuracy at step 870: 0.9635
Accuracy at step 880: 0.9656
Accuracy at step 890: 0.9667
Add run metadata for 899
Accuracy at step 900: 0.9655
Accuracy at step 910: 0.9666
Accuracy at step 920: 0.9651
Accuracy at step 930: 0.9665
Accuracy at step 940: 0.9667
Accuracy at step 950: 0.9669
Accuracy at step 960: 0.9662
Accuracy at step 970: 0.9666
Accuracy at step 980: 0.9655
Accuracy at step 990: 0.9678
Add run metadata for 999

In [ ]: