In [1]:

    

import tensorflow as tf

%pylab inline


    

    




Populating the interactive namespace from numpy and matplotlib

In [2]:

    

from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets('MNIST_data', one_hot=True)


    

    




Extracting MNIST_data/train-images-idx3-ubyte.gz
Extracting MNIST_data/train-labels-idx1-ubyte.gz
Extracting MNIST_data/t10k-images-idx3-ubyte.gz
Extracting MNIST_data/t10k-labels-idx1-ubyte.gz

In [3]:

    

with tf.Graph().as_default() as g: 

  learning_rate = tf.placeholder(tf.float32, shape=[])

  with tf.name_scope('in') as scope:
    x = tf.placeholder(tf.float32, shape=[None, 784])
    tf.histogram_summary(scope + '/image', x)
    tf.image_summary('input', tf.reshape(x, [-1,28,28,1]), 3)

  with tf.name_scope('hidden_1') as scope:
    W = tf.Variable(tf.truncated_normal([784,100], stddev=0.1))
    b = tf.Variable(tf.truncated_normal([100], stddev=0.1))
    tf.histogram_summary(scope + '/weights', W)
    tf.histogram_summary(scope + '/biases', b)
    a = tf.nn.relu(tf.matmul(x, W) + b)
    tf.histogram_summary(scope + '/activations', a)
  with tf.name_scope('hidden_2') as scope:
    W_2 = tf.Variable(tf.truncated_normal([100,100], stddev=0.1))
    b_2 = tf.Variable(tf.truncated_normal([100], stddev=0.1))
    tf.histogram_summary(scope + '/weights', W_2)
    tf.histogram_summary(scope + '/biases', b_2)
    a_2 = tf.nn.relu(tf.matmul(a, W_2) + b_2)
    tf.histogram_summary(scope + '/activations', a_2)
  with tf.name_scope('hidden_3') as scope:
    W_3 = tf.Variable(tf.truncated_normal([100,100], stddev=0.1))
    b_3 = tf.Variable(tf.truncated_normal([100], stddev=0.1))
    tf.histogram_summary(scope + '/weights', W_3)
    tf.histogram_summary(scope + '/biases', b_3)
    a_3 = tf.nn.relu(tf.matmul(a_2, W_3) + b_3)
    tf.histogram_summary(scope + '/activations', a_3)
  with tf.name_scope('out') as scope:
    y_ = tf.placeholder(tf.float32, shape=[None, 10])
    W_4 = tf.Variable(tf.truncated_normal([100,10], stddev=0.1))
    b_4 = tf.Variable(tf.truncated_normal([10], stddev=0.1))
    tf.histogram_summary(scope + '/weights', W_4)
    tf.histogram_summary(scope + '/biases', b_4)
    y = tf.nn.sigmoid(tf.matmul(a_3, W_4) + b_4)
    tf.histogram_summary(scope + '/activations', y)

  #     with tf.name_scope('hidden_2') as scope:
  #       a_2_r = tf.nn.relu(tf.matmul(a_3 - b_3, tf.transpose(W_3)))
  #       tf.histogram_summary(scope + '/recons_activations', a_2_r)
  #     with tf.name_scope('hidden_1') as scope:
  #       a_r = tf.nn.relu(tf.matmul(a_2_r - b_2, tf.transpose(W_2)))
  #       tf.histogram_summary(scope + '/recons_activations', a_r)
  #     with tf.name_scope('in') as scope:
  #       x_r = tf.matmul(a_r - b, tf.transpose(W))
  #       tf.histogram_summary(scope + '/reconstruction', x_r)
  #       tf.image_summary('reconstruction', tf.reshape(x_r, [-1,28,28,1]), 3)

  global_step = tf.Variable(0, name='global_step', trainable=False)

  # losses
  cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y) + (1 - y_) * tf.log(1-y), 
                                                reduction_indices=[1]),
                                 name='xentropy_mean')
  tf.scalar_summary(cross_entropy.op.name, cross_entropy)

  # training
  # train_step = tf.train.GradientDescentOptimizer(1e-6).minimize(cross_entropy)
  train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(cross_entropy, 
                                                            global_step=global_step)
  
  #     back_train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(reconstruction_error)

  # evaluation
  correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
  accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

  ########## INIT AND SUMMARY HAVE TO APPEAR HERE AT THE END TO BE PROPERLY DEFINED
  # misc
  init = tf.initialize_all_variables()
  summary_op = tf.merge_all_summaries()

  with tf.Session() as sess:

    sess.run(init)
    summary_writer = tf.train.SummaryWriter('./mnist_own_descent/tf_grad', sess.graph)

    num_epochs = 10
    steps_per_epoch = 10000
    for epoch in range(num_epochs):
      l_r = 10**np.linspace(-4,-7,num_epochs)[epoch]
      print("epoch %d, learning rate %g"%(epoch,l_r))

      for step in range(steps_per_epoch):
        batch = mnist.train.next_batch(50)
        feed_dict = {x: batch[0], y_: batch[1], learning_rate: l_r}

  #           if step%1000 == 0:
  # #             reconstruction = x_r.eval(feed_dict=feed_dict)
  # #             print("Reconstruction of a %d"%(np.argmax(batch[1][0])))
  # #             plt.imshow(np.reshape(reconstruction[0],(28,28)),
  # #                        cmap=plt.get_cmap('gray'),
  # #                        interpolation='none',
  # #                        aspect="auto")
  # #             plt.show()

        if step%100 == 0:
          train_accuracy = accuracy.eval(feed_dict=feed_dict)
          print("step %d, train accuracy %g"%(step + epoch*steps_per_epoch, train_accuracy))
          summary_str = sess.run(summary_op, feed_dict=feed_dict)
          summary_writer.add_summary(summary_str, step + epoch*steps_per_epoch)
          summary_writer.flush()

        train_step.run(feed_dict=feed_dict)
  #           back_train_step.run(feed_dict=feed_dict)

    print("test accuracy %g"%accuracy.eval(feed_dict={
        x: mnist.test.images, y_: mnist.test.labels}))


    

    




epoch 0, learning rate 0.0001
step 0, train accuracy 0.12
step 100, train accuracy 0.12
step 200, train accuracy 0.06
step 300, train accuracy 0.06
step 400, train accuracy 0.06
step 500, train accuracy 0.1
step 600, train accuracy 0.12
step 700, train accuracy 0.08
step 800, train accuracy 0.12
step 900, train accuracy 0.08
step 1000, train accuracy 0.04
step 1100, train accuracy 0.02
step 1200, train accuracy 0.06
step 1300, train accuracy 0.1
step 1400, train accuracy 0.14
step 1500, train accuracy 0.08
step 1600, train accuracy 0.06
step 1700, train accuracy 0.08
step 1800, train accuracy 0.2
step 1900, train accuracy 0.14
step 2000, train accuracy 0.14
step 2100, train accuracy 0.18
step 2200, train accuracy 0.1
step 2300, train accuracy 0.04
step 2400, train accuracy 0.14
step 2500, train accuracy 0.12
step 2600, train accuracy 0.16
step 2700, train accuracy 0.06
step 2800, train accuracy 0.2
step 2900, train accuracy 0.22
step 3000, train accuracy 0.16
step 3100, train accuracy 0.1
step 3200, train accuracy 0.1
step 3300, train accuracy 0.14
step 3400, train accuracy 0.1
step 3500, train accuracy 0.18
step 3600, train accuracy 0.24
step 3700, train accuracy 0.16
step 3800, train accuracy 0.1
step 3900, train accuracy 0.24
step 4000, train accuracy 0.14
step 4100, train accuracy 0.22
step 4200, train accuracy 0.14
step 4300, train accuracy 0.22
step 4400, train accuracy 0.16
step 4500, train accuracy 0.12
step 4600, train accuracy 0.12
step 4700, train accuracy 0.1
step 4800, train accuracy 0.14
step 4900, train accuracy 0.12
step 5000, train accuracy 0.26
step 5100, train accuracy 0.24
step 5200, train accuracy 0.2
step 5300, train accuracy 0.3
step 5400, train accuracy 0.22
step 5500, train accuracy 0.22
step 5600, train accuracy 0.24
step 5700, train accuracy 0.16
step 5800, train accuracy 0.3
step 5900, train accuracy 0.18
step 6000, train accuracy 0.22
step 6100, train accuracy 0.18
step 6200, train accuracy 0.26
step 6300, train accuracy 0.34
step 6400, train accuracy 0.28
step 6500, train accuracy 0.14
step 6600, train accuracy 0.28
step 6700, train accuracy 0.2
step 6800, train accuracy 0.18
step 6900, train accuracy 0.32
step 7000, train accuracy 0.14
step 7100, train accuracy 0.28
step 7200, train accuracy 0.2
step 7300, train accuracy 0.22
step 7400, train accuracy 0.14
step 7500, train accuracy 0.32
step 7600, train accuracy 0.2
step 7700, train accuracy 0.28
step 7800, train accuracy 0.28
step 7900, train accuracy 0.08
step 8000, train accuracy 0.3
step 8100, train accuracy 0.28
step 8200, train accuracy 0.18
step 8300, train accuracy 0.34
step 8400, train accuracy 0.34
step 8500, train accuracy 0.3
step 8600, train accuracy 0.36
step 8700, train accuracy 0.4
step 8800, train accuracy 0.24
step 8900, train accuracy 0.32
step 9000, train accuracy 0.3
step 9100, train accuracy 0.34
step 9200, train accuracy 0.26
step 9300, train accuracy 0.34
step 9400, train accuracy 0.52
step 9500, train accuracy 0.4
step 9600, train accuracy 0.3
step 9700, train accuracy 0.28
step 9800, train accuracy 0.4
step 9900, train accuracy 0.32
epoch 1, learning rate 4.64159e-05
step 10000, train accuracy 0.34
step 10100, train accuracy 0.34
step 10200, train accuracy 0.34
step 10300, train accuracy 0.48
step 10400, train accuracy 0.28
step 10500, train accuracy 0.48
step 10600, train accuracy 0.24
step 10700, train accuracy 0.3
step 10800, train accuracy 0.34
step 10900, train accuracy 0.32
step 11000, train accuracy 0.28
step 11100, train accuracy 0.42
step 11200, train accuracy 0.38
step 11300, train accuracy 0.4
step 11400, train accuracy 0.3
step 11500, train accuracy 0.3
step 11600, train accuracy 0.26
step 11700, train accuracy 0.34
step 11800, train accuracy 0.28
step 11900, train accuracy 0.3
step 12000, train accuracy 0.48
step 12100, train accuracy 0.42
step 12200, train accuracy 0.38
step 12300, train accuracy 0.38
step 12400, train accuracy 0.52
step 12500, train accuracy 0.32
step 12600, train accuracy 0.34
step 12700, train accuracy 0.48
step 12800, train accuracy 0.38
step 12900, train accuracy 0.34
step 13000, train accuracy 0.44
step 13100, train accuracy 0.42
step 13200, train accuracy 0.38
step 13300, train accuracy 0.42
step 13400, train accuracy 0.36
step 13500, train accuracy 0.32
step 13600, train accuracy 0.38
step 13700, train accuracy 0.56
step 13800, train accuracy 0.4
step 13900, train accuracy 0.36
step 14000, train accuracy 0.46
step 14100, train accuracy 0.36
step 14200, train accuracy 0.46
step 14300, train accuracy 0.6
step 14400, train accuracy 0.5
step 14500, train accuracy 0.38
step 14600, train accuracy 0.46
step 14700, train accuracy 0.32
step 14800, train accuracy 0.38
step 14900, train accuracy 0.4
step 15000, train accuracy 0.4
step 15100, train accuracy 0.4
step 15200, train accuracy 0.6
step 15300, train accuracy 0.44
step 15400, train accuracy 0.4
step 15500, train accuracy 0.5
step 15600, train accuracy 0.42
step 15700, train accuracy 0.36
step 15800, train accuracy 0.36
step 15900, train accuracy 0.48
step 16000, train accuracy 0.38
step 16100, train accuracy 0.38
step 16200, train accuracy 0.42
step 16300, train accuracy 0.38
step 16400, train accuracy 0.46
step 16500, train accuracy 0.48
step 16600, train accuracy 0.52
step 16700, train accuracy 0.38
step 16800, train accuracy 0.4
step 16900, train accuracy 0.54
step 17000, train accuracy 0.36
step 17100, train accuracy 0.36
step 17200, train accuracy 0.38
step 17300, train accuracy 0.38
step 17400, train accuracy 0.42
step 17500, train accuracy 0.48
step 17600, train accuracy 0.44
step 17700, train accuracy 0.46
step 17800, train accuracy 0.4
step 17900, train accuracy 0.44
step 18000, train accuracy 0.44
step 18100, train accuracy 0.54
step 18200, train accuracy 0.48
step 18300, train accuracy 0.46
step 18400, train accuracy 0.44
step 18500, train accuracy 0.46
step 18600, train accuracy 0.52
step 18700, train accuracy 0.44
step 18800, train accuracy 0.54
step 18900, train accuracy 0.56
step 19000, train accuracy 0.48
step 19100, train accuracy 0.46
step 19200, train accuracy 0.46
step 19300, train accuracy 0.52
step 19400, train accuracy 0.44
step 19500, train accuracy 0.56
step 19600, train accuracy 0.44
step 19700, train accuracy 0.44
step 19800, train accuracy 0.4
step 19900, train accuracy 0.46
epoch 2, learning rate 2.15443e-05
step 20000, train accuracy 0.46
step 20100, train accuracy 0.44
step 20200, train accuracy 0.36
step 20300, train accuracy 0.44
step 20400, train accuracy 0.54
step 20500, train accuracy 0.4
step 20600, train accuracy 0.46
step 20700, train accuracy 0.48
step 20800, train accuracy 0.58
step 20900, train accuracy 0.4
step 21000, train accuracy 0.56
step 21100, train accuracy 0.44
step 21200, train accuracy 0.36
step 21300, train accuracy 0.52
step 21400, train accuracy 0.38
step 21500, train accuracy 0.58
step 21600, train accuracy 0.48
step 21700, train accuracy 0.52
step 21800, train accuracy 0.54
step 21900, train accuracy 0.46
step 22000, train accuracy 0.38
step 22100, train accuracy 0.46
step 22200, train accuracy 0.52
step 22300, train accuracy 0.52
step 22400, train accuracy 0.48
step 22500, train accuracy 0.44
step 22600, train accuracy 0.44
step 22700, train accuracy 0.52
step 22800, train accuracy 0.52
step 22900, train accuracy 0.54
step 23000, train accuracy 0.42
step 23100, train accuracy 0.5
step 23200, train accuracy 0.54
step 23300, train accuracy 0.58
step 23400, train accuracy 0.54
step 23500, train accuracy 0.44
step 23600, train accuracy 0.42
step 23700, train accuracy 0.54
step 23800, train accuracy 0.48
step 23900, train accuracy 0.62
step 24000, train accuracy 0.56
step 24100, train accuracy 0.42
step 24200, train accuracy 0.54
step 24300, train accuracy 0.46
step 24400, train accuracy 0.58
step 24500, train accuracy 0.5
step 24600, train accuracy 0.46
step 24700, train accuracy 0.5
step 24800, train accuracy 0.56
step 24900, train accuracy 0.54
step 25000, train accuracy 0.46
step 25100, train accuracy 0.62
step 25200, train accuracy 0.56
step 25300, train accuracy 0.46
step 25400, train accuracy 0.52
step 25500, train accuracy 0.58
step 25600, train accuracy 0.62
step 25700, train accuracy 0.5
step 25800, train accuracy 0.52
step 25900, train accuracy 0.38
step 26000, train accuracy 0.4
step 26100, train accuracy 0.54
step 26200, train accuracy 0.4
step 26300, train accuracy 0.54
step 26400, train accuracy 0.6
step 26500, train accuracy 0.42
step 26600, train accuracy 0.68
step 26700, train accuracy 0.62
step 26800, train accuracy 0.46
step 26900, train accuracy 0.36
step 27000, train accuracy 0.46
step 27100, train accuracy 0.42
step 27200, train accuracy 0.6
step 27300, train accuracy 0.44
step 27400, train accuracy 0.44
step 27500, train accuracy 0.42
step 27600, train accuracy 0.54
step 27700, train accuracy 0.5
step 27800, train accuracy 0.46
step 27900, train accuracy 0.46
step 28000, train accuracy 0.38
step 28100, train accuracy 0.54
step 28200, train accuracy 0.46
step 28300, train accuracy 0.52
step 28400, train accuracy 0.52
step 28500, train accuracy 0.4
step 28600, train accuracy 0.46
step 28700, train accuracy 0.58
step 28800, train accuracy 0.42
step 28900, train accuracy 0.44
step 29000, train accuracy 0.56
step 29100, train accuracy 0.48
step 29200, train accuracy 0.66
step 29300, train accuracy 0.5
step 29400, train accuracy 0.48
step 29500, train accuracy 0.66
step 29600, train accuracy 0.54
step 29700, train accuracy 0.46
step 29800, train accuracy 0.58
step 29900, train accuracy 0.52
epoch 3, learning rate 1e-05
step 30000, train accuracy 0.5
step 30100, train accuracy 0.46
step 30200, train accuracy 0.52
step 30300, train accuracy 0.48
step 30400, train accuracy 0.54
step 30500, train accuracy 0.52
step 30600, train accuracy 0.46
step 30700, train accuracy 0.48
step 30800, train accuracy 0.34
step 30900, train accuracy 0.5
step 31000, train accuracy 0.44
step 31100, train accuracy 0.48
step 31200, train accuracy 0.4
step 31300, train accuracy 0.48
step 31400, train accuracy 0.32
step 31500, train accuracy 0.48
step 31600, train accuracy 0.5
step 31700, train accuracy 0.36
step 31800, train accuracy 0.46
step 31900, train accuracy 0.66
step 32000, train accuracy 0.52
step 32100, train accuracy 0.5
step 32200, train accuracy 0.44
step 32300, train accuracy 0.46
step 32400, train accuracy 0.56
step 32500, train accuracy 0.44
step 32600, train accuracy 0.58
step 32700, train accuracy 0.68
step 32800, train accuracy 0.54
step 32900, train accuracy 0.44
step 33000, train accuracy 0.48
step 33100, train accuracy 0.62
step 33200, train accuracy 0.58
step 33300, train accuracy 0.48
step 33400, train accuracy 0.4
step 33500, train accuracy 0.54
step 33600, train accuracy 0.48
step 33700, train accuracy 0.54
step 33800, train accuracy 0.46
step 33900, train accuracy 0.56
step 34000, train accuracy 0.56
step 34100, train accuracy 0.48
step 34200, train accuracy 0.46
step 34300, train accuracy 0.52
step 34400, train accuracy 0.52
step 34500, train accuracy 0.44
step 34600, train accuracy 0.54
step 34700, train accuracy 0.58
step 34800, train accuracy 0.62
step 34900, train accuracy 0.36
step 35000, train accuracy 0.48
step 35100, train accuracy 0.62
step 35200, train accuracy 0.44
step 35300, train accuracy 0.52
step 35400, train accuracy 0.56
step 35500, train accuracy 0.46
step 35600, train accuracy 0.52
step 35700, train accuracy 0.56
step 35800, train accuracy 0.66
step 35900, train accuracy 0.46
step 36000, train accuracy 0.48
step 36100, train accuracy 0.54
step 36200, train accuracy 0.44
step 36300, train accuracy 0.62
step 36400, train accuracy 0.52
step 36500, train accuracy 0.48
step 36600, train accuracy 0.5
step 36700, train accuracy 0.46
step 36800, train accuracy 0.5
step 36900, train accuracy 0.4
step 37000, train accuracy 0.64
step 37100, train accuracy 0.52
step 37200, train accuracy 0.52
step 37300, train accuracy 0.44
step 37400, train accuracy 0.46
step 37500, train accuracy 0.38
step 37600, train accuracy 0.44
step 37700, train accuracy 0.56
step 37800, train accuracy 0.62
step 37900, train accuracy 0.52
step 38000, train accuracy 0.52
step 38100, train accuracy 0.6
step 38200, train accuracy 0.46
step 38300, train accuracy 0.56
step 38400, train accuracy 0.5
step 38500, train accuracy 0.48
step 38600, train accuracy 0.54
step 38700, train accuracy 0.48
step 38800, train accuracy 0.44
step 38900, train accuracy 0.5
step 39000, train accuracy 0.6
step 39100, train accuracy 0.48
step 39200, train accuracy 0.52
step 39300, train accuracy 0.62
step 39400, train accuracy 0.48
step 39500, train accuracy 0.4
step 39600, train accuracy 0.5
step 39700, train accuracy 0.5
step 39800, train accuracy 0.42
step 39900, train accuracy 0.56
epoch 4, learning rate 4.64159e-06
step 40000, train accuracy 0.48
step 40100, train accuracy 0.6
step 40200, train accuracy 0.56
step 40300, train accuracy 0.62
step 40400, train accuracy 0.48
step 40500, train accuracy 0.52
step 40600, train accuracy 0.44
step 40700, train accuracy 0.44
step 40800, train accuracy 0.5
step 40900, train accuracy 0.42
step 41000, train accuracy 0.54
step 41100, train accuracy 0.54
step 41200, train accuracy 0.4
step 41300, train accuracy 0.46
step 41400, train accuracy 0.44
step 41500, train accuracy 0.6
step 41600, train accuracy 0.66
step 41700, train accuracy 0.44
step 41800, train accuracy 0.64
step 41900, train accuracy 0.62
step 42000, train accuracy 0.46
step 42100, train accuracy 0.58
step 42200, train accuracy 0.48
step 42300, train accuracy 0.38
step 42400, train accuracy 0.58
step 42500, train accuracy 0.62
step 42600, train accuracy 0.48
step 42700, train accuracy 0.62
step 42800, train accuracy 0.5
step 42900, train accuracy 0.5
step 43000, train accuracy 0.44
step 43100, train accuracy 0.36
step 43200, train accuracy 0.48
step 43300, train accuracy 0.42
step 43400, train accuracy 0.44
step 43500, train accuracy 0.46
step 43600, train accuracy 0.56
step 43700, train accuracy 0.6
step 43800, train accuracy 0.56
step 43900, train accuracy 0.42
step 44000, train accuracy 0.68
step 44100, train accuracy 0.5
step 44200, train accuracy 0.46
step 44300, train accuracy 0.48
step 44400, train accuracy 0.5
step 44500, train accuracy 0.58
step 44600, train accuracy 0.5
step 44700, train accuracy 0.54
step 44800, train accuracy 0.46
step 44900, train accuracy 0.52
step 45000, train accuracy 0.4
step 45100, train accuracy 0.52
step 45200, train accuracy 0.6
step 45300, train accuracy 0.56
step 45400, train accuracy 0.44
step 45500, train accuracy 0.68
step 45600, train accuracy 0.52
step 45700, train accuracy 0.56
step 45800, train accuracy 0.3
step 45900, train accuracy 0.6
step 46000, train accuracy 0.4
step 46100, train accuracy 0.4
step 46200, train accuracy 0.58
step 46300, train accuracy 0.46
step 46400, train accuracy 0.44
step 46500, train accuracy 0.52
step 46600, train accuracy 0.42
step 46700, train accuracy 0.62
step 46800, train accuracy 0.58
step 46900, train accuracy 0.48
step 47000, train accuracy 0.52
step 47100, train accuracy 0.62
step 47200, train accuracy 0.64
step 47300, train accuracy 0.56
step 47400, train accuracy 0.56
step 47500, train accuracy 0.48
step 47600, train accuracy 0.42
step 47700, train accuracy 0.6
step 47800, train accuracy 0.64
step 47900, train accuracy 0.58
step 48000, train accuracy 0.62
step 48100, train accuracy 0.64
step 48200, train accuracy 0.5
step 48300, train accuracy 0.56
step 48400, train accuracy 0.5
step 48500, train accuracy 0.4
step 48600, train accuracy 0.36
step 48700, train accuracy 0.46
step 48800, train accuracy 0.58
step 48900, train accuracy 0.66
step 49000, train accuracy 0.56
step 49100, train accuracy 0.48
step 49200, train accuracy 0.56
step 49300, train accuracy 0.46
step 49400, train accuracy 0.38
step 49500, train accuracy 0.48
step 49600, train accuracy 0.48
step 49700, train accuracy 0.64
step 49800, train accuracy 0.64
step 49900, train accuracy 0.66
epoch 5, learning rate 2.15443e-06
step 50000, train accuracy 0.38
step 50100, train accuracy 0.52
step 50200, train accuracy 0.54
step 50300, train accuracy 0.44
step 50400, train accuracy 0.5
step 50500, train accuracy 0.46
step 50600, train accuracy 0.46
step 50700, train accuracy 0.7
step 50800, train accuracy 0.48
step 50900, train accuracy 0.46
step 51000, train accuracy 0.44
step 51100, train accuracy 0.56
step 51200, train accuracy 0.58
step 51300, train accuracy 0.5
step 51400, train accuracy 0.52
step 51500, train accuracy 0.52
step 51600, train accuracy 0.44
step 51700, train accuracy 0.36
step 51800, train accuracy 0.44
step 51900, train accuracy 0.52
step 52000, train accuracy 0.5
step 52100, train accuracy 0.48
step 52200, train accuracy 0.52
step 52300, train accuracy 0.6
step 52400, train accuracy 0.4
step 52500, train accuracy 0.46
step 52600, train accuracy 0.42
step 52700, train accuracy 0.64
step 52800, train accuracy 0.52
step 52900, train accuracy 0.64
step 53000, train accuracy 0.5
step 53100, train accuracy 0.56
step 53200, train accuracy 0.46
step 53300, train accuracy 0.44
step 53400, train accuracy 0.46
step 53500, train accuracy 0.62
step 53600, train accuracy 0.56
step 53700, train accuracy 0.56
step 53800, train accuracy 0.48
step 53900, train accuracy 0.54
step 54000, train accuracy 0.48
step 54100, train accuracy 0.52
step 54200, train accuracy 0.56
step 54300, train accuracy 0.54
step 54400, train accuracy 0.42
step 54500, train accuracy 0.6
step 54600, train accuracy 0.46
step 54700, train accuracy 0.62
step 54800, train accuracy 0.62
step 54900, train accuracy 0.5
step 55000, train accuracy 0.5
step 55100, train accuracy 0.42
step 55200, train accuracy 0.48
step 55300, train accuracy 0.52
step 55400, train accuracy 0.52
step 55500, train accuracy 0.4
step 55600, train accuracy 0.6
step 55700, train accuracy 0.44
step 55800, train accuracy 0.44
step 55900, train accuracy 0.58
step 56000, train accuracy 0.48
step 56100, train accuracy 0.44
step 56200, train accuracy 0.48
step 56300, train accuracy 0.58
step 56400, train accuracy 0.56
step 56500, train accuracy 0.56
step 56600, train accuracy 0.54
step 56700, train accuracy 0.48
step 56800, train accuracy 0.52
step 56900, train accuracy 0.58
step 57000, train accuracy 0.4
step 57100, train accuracy 0.42
step 57200, train accuracy 0.62
step 57300, train accuracy 0.44
step 57400, train accuracy 0.56
step 57500, train accuracy 0.46
step 57600, train accuracy 0.34
step 57700, train accuracy 0.4
step 57800, train accuracy 0.52
step 57900, train accuracy 0.56
step 58000, train accuracy 0.58
step 58100, train accuracy 0.58
step 58200, train accuracy 0.5
step 58300, train accuracy 0.4
step 58400, train accuracy 0.58
step 58500, train accuracy 0.48
step 58600, train accuracy 0.44
step 58700, train accuracy 0.52
step 58800, train accuracy 0.52
step 58900, train accuracy 0.44
step 59000, train accuracy 0.6
step 59100, train accuracy 0.48
step 59200, train accuracy 0.48
step 59300, train accuracy 0.5
step 59400, train accuracy 0.58
step 59500, train accuracy 0.52
step 59600, train accuracy 0.5
step 59700, train accuracy 0.58
step 59800, train accuracy 0.52
step 59900, train accuracy 0.52
epoch 6, learning rate 1e-06
step 60000, train accuracy 0.48
step 60100, train accuracy 0.52
step 60200, train accuracy 0.64
step 60300, train accuracy 0.52
step 60400, train accuracy 0.52
step 60500, train accuracy 0.52
step 60600, train accuracy 0.5
step 60700, train accuracy 0.56
step 60800, train accuracy 0.46
step 60900, train accuracy 0.62
step 61000, train accuracy 0.48
step 61100, train accuracy 0.62
step 61200, train accuracy 0.58
step 61300, train accuracy 0.4
step 61400, train accuracy 0.52
step 61500, train accuracy 0.54
step 61600, train accuracy 0.52
step 61700, train accuracy 0.56
step 61800, train accuracy 0.46
step 61900, train accuracy 0.46
step 62000, train accuracy 0.46
step 62100, train accuracy 0.56
step 62200, train accuracy 0.64
step 62300, train accuracy 0.48
step 62400, train accuracy 0.48
step 62500, train accuracy 0.52
step 62600, train accuracy 0.48
step 62700, train accuracy 0.54
step 62800, train accuracy 0.52
step 62900, train accuracy 0.44
step 63000, train accuracy 0.48
step 63100, train accuracy 0.56
step 63200, train accuracy 0.52
step 63300, train accuracy 0.56
step 63400, train accuracy 0.54
step 63500, train accuracy 0.56
step 63600, train accuracy 0.58
step 63700, train accuracy 0.48
step 63800, train accuracy 0.44
step 63900, train accuracy 0.52
step 64000, train accuracy 0.58
step 64100, train accuracy 0.48
step 64200, train accuracy 0.44
step 64300, train accuracy 0.54
step 64400, train accuracy 0.5
step 64500, train accuracy 0.56
step 64600, train accuracy 0.5
step 64700, train accuracy 0.48
step 64800, train accuracy 0.56
step 64900, train accuracy 0.54
step 65000, train accuracy 0.56
step 65100, train accuracy 0.48
step 65200, train accuracy 0.6
step 65300, train accuracy 0.54
step 65400, train accuracy 0.48
step 65500, train accuracy 0.5
step 65600, train accuracy 0.5
step 65700, train accuracy 0.5
step 65800, train accuracy 0.48
step 65900, train accuracy 0.52
step 66000, train accuracy 0.5
step 66100, train accuracy 0.62
step 66200, train accuracy 0.6
step 66300, train accuracy 0.42
step 66400, train accuracy 0.48
step 66500, train accuracy 0.52
step 66600, train accuracy 0.42
step 66700, train accuracy 0.52
step 66800, train accuracy 0.52
step 66900, train accuracy 0.52
step 67000, train accuracy 0.6
step 67100, train accuracy 0.48
step 67200, train accuracy 0.46
step 67300, train accuracy 0.52
step 67400, train accuracy 0.52
step 67500, train accuracy 0.68
step 67600, train accuracy 0.46
step 67700, train accuracy 0.64
step 67800, train accuracy 0.6
step 67900, train accuracy 0.36
step 68000, train accuracy 0.5
step 68100, train accuracy 0.48
step 68200, train accuracy 0.54
step 68300, train accuracy 0.62
step 68400, train accuracy 0.4
step 68500, train accuracy 0.52
step 68600, train accuracy 0.5
step 68700, train accuracy 0.56
step 68800, train accuracy 0.6
step 68900, train accuracy 0.58
step 69000, train accuracy 0.44
step 69100, train accuracy 0.5
step 69200, train accuracy 0.62
step 69300, train accuracy 0.48
step 69400, train accuracy 0.52
step 69500, train accuracy 0.46
step 69600, train accuracy 0.54
step 69700, train accuracy 0.6
step 69800, train accuracy 0.5
step 69900, train accuracy 0.42
epoch 7, learning rate 4.64159e-07
step 70000, train accuracy 0.5
step 70100, train accuracy 0.68
step 70200, train accuracy 0.36
step 70300, train accuracy 0.44
step 70400, train accuracy 0.6
step 70500, train accuracy 0.48
step 70600, train accuracy 0.42
step 70700, train accuracy 0.58
step 70800, train accuracy 0.52
step 70900, train accuracy 0.48
step 71000, train accuracy 0.58
step 71100, train accuracy 0.36
step 71200, train accuracy 0.56
step 71300, train accuracy 0.54
step 71400, train accuracy 0.56
step 71500, train accuracy 0.46
step 71600, train accuracy 0.52
step 71700, train accuracy 0.58
step 71800, train accuracy 0.5
step 71900, train accuracy 0.4
step 72000, train accuracy 0.44
step 72100, train accuracy 0.54
step 72200, train accuracy 0.5
step 72300, train accuracy 0.46
step 72400, train accuracy 0.56
step 72500, train accuracy 0.48
step 72600, train accuracy 0.46
step 72700, train accuracy 0.46
step 72800, train accuracy 0.58
step 72900, train accuracy 0.56
step 73000, train accuracy 0.46
step 73100, train accuracy 0.52
step 73200, train accuracy 0.44
step 73300, train accuracy 0.66
step 73400, train accuracy 0.42
step 73500, train accuracy 0.56
step 73600, train accuracy 0.62
step 73700, train accuracy 0.52
step 73800, train accuracy 0.54
step 73900, train accuracy 0.68
step 74000, train accuracy 0.64
step 74100, train accuracy 0.52
step 74200, train accuracy 0.48
step 74300, train accuracy 0.6
step 74400, train accuracy 0.48
step 74500, train accuracy 0.48
step 74600, train accuracy 0.6
step 74700, train accuracy 0.46
step 74800, train accuracy 0.58
step 74900, train accuracy 0.56
step 75000, train accuracy 0.54
step 75100, train accuracy 0.52
step 75200, train accuracy 0.48
step 75300, train accuracy 0.52
step 75400, train accuracy 0.5
step 75500, train accuracy 0.46
step 75600, train accuracy 0.46
step 75700, train accuracy 0.6
step 75800, train accuracy 0.48
step 75900, train accuracy 0.64
step 76000, train accuracy 0.66
step 76100, train accuracy 0.54
step 76200, train accuracy 0.54
step 76300, train accuracy 0.56
step 76400, train accuracy 0.5
step 76500, train accuracy 0.48
step 76600, train accuracy 0.52
step 76700, train accuracy 0.58
step 76800, train accuracy 0.62
step 76900, train accuracy 0.52
step 77000, train accuracy 0.44
step 77100, train accuracy 0.62
step 77200, train accuracy 0.52
step 77300, train accuracy 0.7
step 77400, train accuracy 0.54
step 77500, train accuracy 0.46
step 77600, train accuracy 0.4
step 77700, train accuracy 0.44
step 77800, train accuracy 0.54
step 77900, train accuracy 0.64
step 78000, train accuracy 0.52
step 78100, train accuracy 0.5
step 78200, train accuracy 0.38
step 78300, train accuracy 0.56
step 78400, train accuracy 0.58
step 78500, train accuracy 0.72
step 78600, train accuracy 0.56
step 78700, train accuracy 0.5
step 78800, train accuracy 0.48
step 78900, train accuracy 0.54
step 79000, train accuracy 0.52
step 79100, train accuracy 0.6
step 79200, train accuracy 0.46
step 79300, train accuracy 0.5
step 79400, train accuracy 0.6
step 79500, train accuracy 0.44
step 79600, train accuracy 0.38
step 79700, train accuracy 0.56
step 79800, train accuracy 0.64
step 79900, train accuracy 0.48
epoch 8, learning rate 2.15443e-07
step 80000, train accuracy 0.48
step 80100, train accuracy 0.54
step 80200, train accuracy 0.58
step 80300, train accuracy 0.5
step 80400, train accuracy 0.58
step 80500, train accuracy 0.4
step 80600, train accuracy 0.72
step 80700, train accuracy 0.42
step 80800, train accuracy 0.42
step 80900, train accuracy 0.6
step 81000, train accuracy 0.52
step 81100, train accuracy 0.42
step 81200, train accuracy 0.58
step 81300, train accuracy 0.58
step 81400, train accuracy 0.7
step 81500, train accuracy 0.54
step 81600, train accuracy 0.68
step 81700, train accuracy 0.62
step 81800, train accuracy 0.44
step 81900, train accuracy 0.7
step 82000, train accuracy 0.48
step 82100, train accuracy 0.66
step 82200, train accuracy 0.52
step 82300, train accuracy 0.5
step 82400, train accuracy 0.5
step 82500, train accuracy 0.6
step 82600, train accuracy 0.54
step 82700, train accuracy 0.46
step 82800, train accuracy 0.48
step 82900, train accuracy 0.56
step 83000, train accuracy 0.54
step 83100, train accuracy 0.58
step 83200, train accuracy 0.64
step 83300, train accuracy 0.46
step 83400, train accuracy 0.6
step 83500, train accuracy 0.6
step 83600, train accuracy 0.4
step 83700, train accuracy 0.72
step 83800, train accuracy 0.46
step 83900, train accuracy 0.48
step 84000, train accuracy 0.46
step 84100, train accuracy 0.48
step 84200, train accuracy 0.64
step 84300, train accuracy 0.52
step 84400, train accuracy 0.52
step 84500, train accuracy 0.46
step 84600, train accuracy 0.54
step 84700, train accuracy 0.56
step 84800, train accuracy 0.44
step 84900, train accuracy 0.66
step 85000, train accuracy 0.52
step 85100, train accuracy 0.64
step 85200, train accuracy 0.42
step 85300, train accuracy 0.62
step 85400, train accuracy 0.56
step 85500, train accuracy 0.6
step 85600, train accuracy 0.5
step 85700, train accuracy 0.44
step 85800, train accuracy 0.62
step 85900, train accuracy 0.54
step 86000, train accuracy 0.36
step 86100, train accuracy 0.44
step 86200, train accuracy 0.64
step 86300, train accuracy 0.58
step 86400, train accuracy 0.64
step 86500, train accuracy 0.42
step 86600, train accuracy 0.68
step 86700, train accuracy 0.52
step 86800, train accuracy 0.58
step 86900, train accuracy 0.4
step 87000, train accuracy 0.4
step 87100, train accuracy 0.36
step 87200, train accuracy 0.58
step 87300, train accuracy 0.3
step 87400, train accuracy 0.58
step 87500, train accuracy 0.64
step 87600, train accuracy 0.68
step 87700, train accuracy 0.6
step 87800, train accuracy 0.46
step 87900, train accuracy 0.52
step 88000, train accuracy 0.58
step 88100, train accuracy 0.44
step 88200, train accuracy 0.42
step 88300, train accuracy 0.58
step 88400, train accuracy 0.5
step 88500, train accuracy 0.52
step 88600, train accuracy 0.54
step 88700, train accuracy 0.6
step 88800, train accuracy 0.3
step 88900, train accuracy 0.46
step 89000, train accuracy 0.46
step 89100, train accuracy 0.54
step 89200, train accuracy 0.44
step 89300, train accuracy 0.46
step 89400, train accuracy 0.68
step 89500, train accuracy 0.58
step 89600, train accuracy 0.4
step 89700, train accuracy 0.58
step 89800, train accuracy 0.5
step 89900, train accuracy 0.54
epoch 9, learning rate 1e-07
step 90000, train accuracy 0.42
step 90100, train accuracy 0.5
step 90200, train accuracy 0.7
step 90300, train accuracy 0.52
step 90400, train accuracy 0.48
step 90500, train accuracy 0.52
step 90600, train accuracy 0.42
step 90700, train accuracy 0.6
step 90800, train accuracy 0.4
step 90900, train accuracy 0.56
step 91000, train accuracy 0.6
step 91100, train accuracy 0.48
step 91200, train accuracy 0.54
step 91300, train accuracy 0.56
step 91400, train accuracy 0.54
step 91500, train accuracy 0.58
step 91600, train accuracy 0.46
step 91700, train accuracy 0.56
step 91800, train accuracy 0.44
step 91900, train accuracy 0.58
step 92000, train accuracy 0.42
step 92100, train accuracy 0.66
step 92200, train accuracy 0.42
step 92300, train accuracy 0.6
step 92400, train accuracy 0.52
step 92500, train accuracy 0.62
step 92600, train accuracy 0.56
step 92700, train accuracy 0.66
step 92800, train accuracy 0.4
step 92900, train accuracy 0.48
step 93000, train accuracy 0.42
step 93100, train accuracy 0.56
step 93200, train accuracy 0.4
step 93300, train accuracy 0.4
step 93400, train accuracy 0.42
step 93500, train accuracy 0.56
step 93600, train accuracy 0.58
step 93700, train accuracy 0.56
step 93800, train accuracy 0.54
step 93900, train accuracy 0.48
step 94000, train accuracy 0.48
step 94100, train accuracy 0.38
step 94200, train accuracy 0.48
step 94300, train accuracy 0.5
step 94400, train accuracy 0.52
step 94500, train accuracy 0.58
step 94600, train accuracy 0.54
step 94700, train accuracy 0.64
step 94800, train accuracy 0.52
step 94900, train accuracy 0.62
step 95000, train accuracy 0.46
step 95100, train accuracy 0.54
step 95200, train accuracy 0.48
step 95300, train accuracy 0.52
step 95400, train accuracy 0.48
step 95500, train accuracy 0.44
step 95600, train accuracy 0.52
step 95700, train accuracy 0.56
step 95800, train accuracy 0.52
step 95900, train accuracy 0.42
step 96000, train accuracy 0.54
step 96100, train accuracy 0.5
step 96200, train accuracy 0.54
step 96300, train accuracy 0.48
step 96400, train accuracy 0.54
step 96500, train accuracy 0.58
step 96600, train accuracy 0.6
step 96700, train accuracy 0.62
step 96800, train accuracy 0.54
step 96900, train accuracy 0.52
step 97000, train accuracy 0.56
step 97100, train accuracy 0.4
step 97200, train accuracy 0.52
step 97300, train accuracy 0.5
step 97400, train accuracy 0.58
step 97500, train accuracy 0.44
step 97600, train accuracy 0.62
step 97700, train accuracy 0.46
step 97800, train accuracy 0.48
step 97900, train accuracy 0.48
step 98000, train accuracy 0.56
step 98100, train accuracy 0.42
step 98200, train accuracy 0.52
step 98300, train accuracy 0.66
step 98400, train accuracy 0.58
step 98500, train accuracy 0.6
step 98600, train accuracy 0.54
step 98700, train accuracy 0.64
step 98800, train accuracy 0.54
step 98900, train accuracy 0.54
step 99000, train accuracy 0.36
step 99100, train accuracy 0.6
step 99200, train accuracy 0.46
step 99300, train accuracy 0.58
step 99400, train accuracy 0.56
step 99500, train accuracy 0.54
step 99600, train accuracy 0.42
step 99700, train accuracy 0.6
step 99800, train accuracy 0.5
step 99900, train accuracy 0.46
test accuracy 0.5377

In [3]:

    

tf.get_default_graph()


    

    
Out[3]:





<tensorflow.python.framework.ops.Graph at 0x7fc6a0cdec10>

In [4]:

    

learning_rate = tf.placeholder(tf.float32, shape=[])

with tf.name_scope('in') as scope:
    x = tf.placeholder(tf.float32, shape=[None, 784])
    tf.histogram_summary(scope + '/image', x)
    tf.image_summary('input', tf.reshape(x, [-1,28,28,1]), 3)

with tf.name_scope('hidden_1') as scope:
    W = tf.Variable(tf.truncated_normal([784,100], stddev=0.1))
    b = tf.Variable(tf.truncated_normal([100], stddev=0.1))
    tf.histogram_summary(scope + '/weights', W)
    tf.histogram_summary(scope + '/biases', b)
    a = tf.nn.relu(tf.matmul(x, W) + b)
    tf.histogram_summary(scope + '/activations', a)
with tf.name_scope('hidden_2') as scope:
    W_2 = tf.Variable(tf.truncated_normal([100,100], stddev=0.1))
    b_2 = tf.Variable(tf.truncated_normal([100], stddev=0.1))
    tf.histogram_summary(scope + '/weights', W_2)
    tf.histogram_summary(scope + '/biases', b_2)
    a_2 = tf.nn.relu(tf.matmul(a, W_2) + b_2)
    tf.histogram_summary(scope + '/activations', a_2)
with tf.name_scope('hidden_3') as scope:
    W_3 = tf.Variable(tf.truncated_normal([100,100], stddev=0.1))
    b_3 = tf.Variable(tf.truncated_normal([100], stddev=0.1))
    tf.histogram_summary(scope + '/weights', W_3)
    tf.histogram_summary(scope + '/biases', b_3)
    a_3 = tf.nn.relu(tf.matmul(a_2, W_3) + b_3)
    tf.histogram_summary(scope + '/activations', a_3)
with tf.name_scope('out') as scope:
    y_ = tf.placeholder(tf.float32, shape=[None, 10])
    W_4 = tf.Variable(tf.truncated_normal([100,10], stddev=0.1))
    b_4 = tf.Variable(tf.truncated_normal([10], stddev=0.1))
    tf.histogram_summary(scope + '/weights', W_4)
    tf.histogram_summary(scope + '/biases', b_4)
    y = tf.nn.sigmoid(tf.matmul(a_3, W_4) + b_4)
    tf.histogram_summary(scope + '/activations', y)

#     with tf.name_scope('hidden_2') as scope:
#       a_2_r = tf.nn.relu(tf.matmul(a_3 - b_3, tf.transpose(W_3)))
#       tf.histogram_summary(scope + '/recons_activations', a_2_r)
#     with tf.name_scope('hidden_1') as scope:
#       a_r = tf.nn.relu(tf.matmul(a_2_r - b_2, tf.transpose(W_2)))
#       tf.histogram_summary(scope + '/recons_activations', a_r)
#     with tf.name_scope('in') as scope:
#       x_r = tf.matmul(a_r - b, tf.transpose(W))
#       tf.histogram_summary(scope + '/reconstruction', x_r)
#       tf.image_summary('reconstruction', tf.reshape(x_r, [-1,28,28,1]), 3)

global_step = tf.Variable(0, name='global_step', trainable=False)

# losses
cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y) + (1 - y_) * tf.log(1-y), 
                                            reduction_indices=[1]),
                             name='xentropy_mean')
tf.scalar_summary(cross_entropy.op.name, cross_entropy)

# training
# train_step = tf.train.GradientDescentOptimizer(1e-6).minimize(cross_entropy)
train_step = tf.train.AdamOptimizer(learning_rate).minimize(cross_entropy, 
                                                        global_step=global_step)

#     back_train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(reconstruction_error)

# evaluation
correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

########## INIT AND SUMMARY HAVE TO APPEAR HERE AT THE END TO BE PROPERLY DEFINED
# misc
init = tf.initialize_all_variables()
summary_op = tf.merge_all_summaries()

with tf.Session() as sess:

    sess.run(init)
    summary_writer = tf.train.SummaryWriter('./mnist_own_descent/tf_adam', sess.graph)

    num_epochs = 10
    steps_per_epoch = 10000
    for epoch in range(num_epochs):
        l_r = 10**np.linspace(-5,-8,num_epochs)[epoch]
        print("epoch %d, learning rate %g"%(epoch,l_r))

        for step in range(steps_per_epoch):
            batch = mnist.train.next_batch(50)
            feed_dict = {x: batch[0], y_: batch[1], learning_rate: l_r}

            #           if step%1000 == 0:
            # #             reconstruction = x_r.eval(feed_dict=feed_dict)
            # #             print("Reconstruction of a %d"%(np.argmax(batch[1][0])))
            # #             plt.imshow(np.reshape(reconstruction[0],(28,28)),
            # #                        cmap=plt.get_cmap('gray'),
            # #                        interpolation='none',
            # #                        aspect="auto")
            # #             plt.show()

            if step%100 == 0:
                train_accuracy = accuracy.eval(feed_dict=feed_dict)
                print("step %d, train accuracy %g"%(step + epoch*steps_per_epoch, train_accuracy))
                summary_str = sess.run(summary_op, feed_dict=feed_dict)
                summary_writer.add_summary(summary_str, step + epoch*steps_per_epoch)
                summary_writer.flush()

            train_step.run(feed_dict=feed_dict)
            #           back_train_step.run(feed_dict=feed_dict)

    print("test accuracy %g"%accuracy.eval(feed_dict={
    x: mnist.test.images, y_: mnist.test.labels}))


    

    




epoch 0, learning rate 1e-05
step 0, train accuracy 0.08
step 100, train accuracy 0.12
step 200, train accuracy 0.1
step 300, train accuracy 0.2
step 400, train accuracy 0.22
step 500, train accuracy 0.18
step 600, train accuracy 0.12
step 700, train accuracy 0.2
step 800, train accuracy 0.2
step 900, train accuracy 0.16
step 1000, train accuracy 0.24
step 1100, train accuracy 0.22
step 1200, train accuracy 0.26
step 1300, train accuracy 0.28
step 1400, train accuracy 0.32
step 1500, train accuracy 0.48
step 1600, train accuracy 0.34
step 1700, train accuracy 0.36
step 1800, train accuracy 0.42
step 1900, train accuracy 0.46
step 2000, train accuracy 0.4
step 2100, train accuracy 0.58
step 2200, train accuracy 0.5
step 2300, train accuracy 0.48
step 2400, train accuracy 0.52
step 2500, train accuracy 0.66
step 2600, train accuracy 0.6
step 2700, train accuracy 0.68
step 2800, train accuracy 0.6
step 2900, train accuracy 0.7
step 3000, train accuracy 0.78
step 3100, train accuracy 0.7
step 3200, train accuracy 0.56
step 3300, train accuracy 0.72
step 3400, train accuracy 0.58
step 3500, train accuracy 0.74
step 3600, train accuracy 0.74
step 3700, train accuracy 0.64
step 3800, train accuracy 0.78
step 3900, train accuracy 0.78
step 4000, train accuracy 0.72
step 4100, train accuracy 0.76
step 4200, train accuracy 0.86
step 4300, train accuracy 0.72
step 4400, train accuracy 0.78
step 4500, train accuracy 0.68
step 4600, train accuracy 0.82
step 4700, train accuracy 0.76
step 4800, train accuracy 0.8
step 4900, train accuracy 0.86
step 5000, train accuracy 0.78
step 5100, train accuracy 0.82
step 5200, train accuracy 0.8
step 5300, train accuracy 0.86
step 5400, train accuracy 0.84
step 5500, train accuracy 0.82
step 5600, train accuracy 0.8
step 5700, train accuracy 0.76
step 5800, train accuracy 0.78
step 5900, train accuracy 0.78
step 6000, train accuracy 0.82
step 6100, train accuracy 0.74
step 6200, train accuracy 0.78
step 6300, train accuracy 0.9
step 6400, train accuracy 0.88
step 6500, train accuracy 0.84
step 6600, train accuracy 0.82
step 6700, train accuracy 0.84
step 6800, train accuracy 0.88
step 6900, train accuracy 0.76
step 7000, train accuracy 0.82
step 7100, train accuracy 0.92
step 7200, train accuracy 0.86
step 7300, train accuracy 0.82
step 7400, train accuracy 0.86
step 7500, train accuracy 0.9
step 7600, train accuracy 0.9
step 7700, train accuracy 0.86
step 7800, train accuracy 0.86
step 7900, train accuracy 0.86
step 8000, train accuracy 0.86
step 8100, train accuracy 0.9
step 8200, train accuracy 0.86
step 8300, train accuracy 0.78
step 8400, train accuracy 0.82
step 8500, train accuracy 0.86
step 8600, train accuracy 0.84
step 8700, train accuracy 0.82
step 8800, train accuracy 0.82
step 8900, train accuracy 0.84
step 9000, train accuracy 0.92
step 9100, train accuracy 0.9
step 9200, train accuracy 0.82
step 9300, train accuracy 0.9
step 9400, train accuracy 0.94
step 9500, train accuracy 0.88
step 9600, train accuracy 0.76
step 9700, train accuracy 0.88
step 9800, train accuracy 0.92
step 9900, train accuracy 0.88
epoch 1, learning rate 4.64159e-06
step 10000, train accuracy 0.86
step 10100, train accuracy 0.9
step 10200, train accuracy 0.94
step 10300, train accuracy 0.94
step 10400, train accuracy 0.84
step 10500, train accuracy 0.88
step 10600, train accuracy 0.92
step 10700, train accuracy 0.92
step 10800, train accuracy 0.9
step 10900, train accuracy 0.9
step 11000, train accuracy 0.96
step 11100, train accuracy 0.9
step 11200, train accuracy 0.8
step 11300, train accuracy 0.86
step 11400, train accuracy 0.94
step 11500, train accuracy 0.88
step 11600, train accuracy 0.94
step 11700, train accuracy 0.94
step 11800, train accuracy 0.9
step 11900, train accuracy 0.98
step 12000, train accuracy 0.9
step 12100, train accuracy 0.9
step 12200, train accuracy 0.88
step 12300, train accuracy 0.92
step 12400, train accuracy 0.94
step 12500, train accuracy 0.88
step 12600, train accuracy 0.88
step 12700, train accuracy 0.88
step 12800, train accuracy 0.96
step 12900, train accuracy 0.8
step 13000, train accuracy 0.78
step 13100, train accuracy 0.86
step 13200, train accuracy 0.9
step 13300, train accuracy 0.88
step 13400, train accuracy 0.92
step 13500, train accuracy 0.84






    




---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-4-3787df88daf8> in <module>()
    104                 summary_writer.flush()
    105 
--> 106             train_step.run(feed_dict=feed_dict)
    107             #           back_train_step.run(feed_dict=feed_dict)
    108 

/home/frederik/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.pyc in run(self, feed_dict, session)
   1549         none, the default session will be used.
   1550     """
-> 1551     _run_using_default_session(self, feed_dict, self.graph, session)
   1552 
   1553 

/home/frederik/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.pyc in _run_using_default_session(operation, feed_dict, graph, session)
   3531                        "the operation's graph is different from the session's "
   3532                        "graph.")
-> 3533   session.run(operation, feed_dict)
   3534 
   3535 

/home/frederik/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
    370     try:
    371       result = self._run(None, fetches, feed_dict, options_ptr,
--> 372                          run_metadata_ptr)
    373       if run_metadata:
    374         proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)

/home/frederik/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
    634     try:
    635       results = self._do_run(handle, target_list, unique_fetches,
--> 636                              feed_dict_string, options, run_metadata)
    637     finally:
    638       # The movers are no longer used. Delete them.

/home/frederik/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
    706     if handle is None:
    707       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 708                            target_list, options, run_metadata)
    709     else:
    710       return self._do_call(_prun_fn, self._session, handle, feed_dict,

/home/frederik/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
    713   def _do_call(self, fn, *args):
    714     try:
--> 715       return fn(*args)
    716     except errors.OpError as e:
    717       message = compat.as_text(e.message)

/home/frederik/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run_fn(session, feed_dict, fetch_list, target_list, options, run_metadata)
    695         return tf_session.TF_Run(session, options,
    696                                  feed_dict, fetch_list, target_list,
--> 697                                  status, run_metadata)
    698 
    699     def _prun_fn(session, handle, feed_dict, fetch_list):

KeyboardInterrupt: 

In [5]:

    

tf.get_default_graph()


    

    
Out[5]:





<tensorflow.python.framework.ops.Graph at 0x7f3519eeb150>

In [6]:

    

with tf.Graph().as_default() as g: 

  learning_rate = tf.placeholder(tf.float32, shape=[])

  with tf.name_scope('in') as scope:
    x = tf.placeholder(tf.float32, shape=[None, 784])
    tf.histogram_summary(scope + '/image', x)
    tf.image_summary('input', tf.reshape(x, [-1,28,28,1]), 3)

  with tf.name_scope('hidden_1') as scope:
    W = tf.Variable(tf.truncated_normal([784,100], stddev=0.1))
    b = tf.Variable(tf.truncated_normal([100], stddev=0.1))
    tf.histogram_summary(scope + '/weights', W)
    tf.histogram_summary(scope + '/biases', b)
    a = tf.nn.relu(tf.matmul(x, W) + b)
    tf.histogram_summary(scope + '/activations', a)
  with tf.name_scope('hidden_2') as scope:
    W_2 = tf.Variable(tf.truncated_normal([100,100], stddev=0.1))
    b_2 = tf.Variable(tf.truncated_normal([100], stddev=0.1))
    tf.histogram_summary(scope + '/weights', W_2)
    tf.histogram_summary(scope + '/biases', b_2)
    a_2 = tf.nn.relu(tf.matmul(a, W_2) + b_2)
    tf.histogram_summary(scope + '/activations', a_2)
  with tf.name_scope('hidden_3') as scope:
    W_3 = tf.Variable(tf.truncated_normal([100,100], stddev=0.1))
    b_3 = tf.Variable(tf.truncated_normal([100], stddev=0.1))
    tf.histogram_summary(scope + '/weights', W_3)
    tf.histogram_summary(scope + '/biases', b_3)
    a_3 = tf.nn.relu(tf.matmul(a_2, W_3) + b_3)
    tf.histogram_summary(scope + '/activations', a_3)
  with tf.name_scope('out') as scope:
    y_ = tf.placeholder(tf.float32, shape=[None, 10])
    W_4 = tf.Variable(tf.truncated_normal([100,10], stddev=0.1))
    b_4 = tf.Variable(tf.truncated_normal([10], stddev=0.1))
    tf.histogram_summary(scope + '/weights', W_4)
    tf.histogram_summary(scope + '/biases', b_4)
    y = tf.nn.sigmoid(tf.matmul(a_3, W_4) + b_4)
    tf.histogram_summary(scope + '/activations', y)

  #     with tf.name_scope('hidden_2') as scope:
  #       a_2_r = tf.nn.relu(tf.matmul(a_3 - b_3, tf.transpose(W_3)))
  #       tf.histogram_summary(scope + '/recons_activations', a_2_r)
  #     with tf.name_scope('hidden_1') as scope:
  #       a_r = tf.nn.relu(tf.matmul(a_2_r - b_2, tf.transpose(W_2)))
  #       tf.histogram_summary(scope + '/recons_activations', a_r)
  #     with tf.name_scope('in') as scope:
  #       x_r = tf.matmul(a_r - b, tf.transpose(W))
  #       tf.histogram_summary(scope + '/reconstruction', x_r)
  #       tf.image_summary('reconstruction', tf.reshape(x_r, [-1,28,28,1]), 3)

  global_step = tf.Variable(0, name='global_step', trainable=False)

  # losses
  cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y) + (1 - y_) * tf.log(1-y), 
                                                reduction_indices=[1]),
                                 name='xentropy_mean')
  tf.scalar_summary(cross_entropy.op.name, cross_entropy)

  # training
  # train_step = tf.train.GradientDescentOptimizer(1e-6).minimize(cross_entropy)
#   train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(cross_entropy, 
#                                                             global_step=global_step)
  opt = tf.train.GradientDescentOptimizer(learning_rate)
  var_list = [W,b, W_2,b_2, W_3,b_3, W_4,b_4]
  var_grad = tf.gradients(cross_entropy, var_list)
  grads_and_vars = zip(var_grad, var_list)
  train_step = opt.apply_gradients(grads_and_vars, 
                                   global_step=global_step)
  
  #     back_train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(reconstruction_error)

  # evaluation
  correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
  accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

  ########## INIT AND SUMMARY HAVE TO APPEAR HERE AT THE END TO BE PROPERLY DEFINED
  # misc
  init = tf.initialize_all_variables()
  summary_op = tf.merge_all_summaries()

  with tf.Session() as sess:

    sess.run(init)
    summary_writer = tf.train.SummaryWriter('./mnist_own_descent/own_grad', sess.graph)

    num_epochs = 10
    steps_per_epoch = 10000
    for epoch in range(num_epochs):
      l_r = 10**np.linspace(-4,-7,num_epochs)[epoch]
      print("epoch %d, learning rate %g"%(epoch,l_r))

      for step in range(steps_per_epoch):
        batch = mnist.train.next_batch(50)
        feed_dict = {x: batch[0], y_: batch[1], learning_rate: l_r}

  #           if step%1000 == 0:
  # #             reconstruction = x_r.eval(feed_dict=feed_dict)
  # #             print("Reconstruction of a %d"%(np.argmax(batch[1][0])))
  # #             plt.imshow(np.reshape(reconstruction[0],(28,28)),
  # #                        cmap=plt.get_cmap('gray'),
  # #                        interpolation='none',
  # #                        aspect="auto")
  # #             plt.show()

        if step%100 == 0:
          train_accuracy = accuracy.eval(feed_dict=feed_dict)
          print("step %d, train accuracy %g"%(step + epoch*steps_per_epoch, train_accuracy))
          summary_str = sess.run(summary_op, feed_dict=feed_dict)
          summary_writer.add_summary(summary_str, step + epoch*steps_per_epoch)
          summary_writer.flush()

        train_step.run(feed_dict=feed_dict)
  #           back_train_step.run(feed_dict=feed_dict)

    print("test accuracy %g"%accuracy.eval(feed_dict={
        x: mnist.test.images, y_: mnist.test.labels}))


    

    




epoch 0, learning rate 0.0001
step 0, train accuracy 0.06
step 100, train accuracy 0.1
step 200, train accuracy 0.1
step 300, train accuracy 0.1
step 400, train accuracy 0.06
step 500, train accuracy 0.06
step 600, train accuracy 0.06
step 700, train accuracy 0.16
step 800, train accuracy 0.04
step 900, train accuracy 0.08
step 1000, train accuracy 0.02
step 1100, train accuracy 0.1
step 1200, train accuracy 0.1
step 1300, train accuracy 0.14
step 1400, train accuracy 0.08
step 1500, train accuracy 0.12
step 1600, train accuracy 0.14
step 1700, train accuracy 0.12
step 1800, train accuracy 0.12
step 1900, train accuracy 0.1
step 2000, train accuracy 0.18
step 2100, train accuracy 0.16
step 2200, train accuracy 0.12
step 2300, train accuracy 0.1
step 2400, train accuracy 0.04
step 2500, train accuracy 0.16
step 2600, train accuracy 0.02
step 2700, train accuracy 0.14
step 2800, train accuracy 0.06
step 2900, train accuracy 0.14
step 3000, train accuracy 0.08
step 3100, train accuracy 0.16
step 3200, train accuracy 0.16
step 3300, train accuracy 0.28
step 3400, train accuracy 0.1
step 3500, train accuracy 0.12
step 3600, train accuracy 0.1
step 3700, train accuracy 0.04
step 3800, train accuracy 0.1
step 3900, train accuracy 0.14
step 4000, train accuracy 0.24
step 4100, train accuracy 0.18
step 4200, train accuracy 0.18
step 4300, train accuracy 0.2
step 4400, train accuracy 0.24
step 4500, train accuracy 0.2
step 4600, train accuracy 0.18
step 4700, train accuracy 0.18
step 4800, train accuracy 0.22
step 4900, train accuracy 0.24
step 5000, train accuracy 0.28
step 5100, train accuracy 0.24
step 5200, train accuracy 0.26
step 5300, train accuracy 0.3
step 5400, train accuracy 0.18
step 5500, train accuracy 0.16
step 5600, train accuracy 0.18
step 5700, train accuracy 0.3
step 5800, train accuracy 0.22
step 5900, train accuracy 0.28
step 6000, train accuracy 0.4
step 6100, train accuracy 0.28
step 6200, train accuracy 0.22
step 6300, train accuracy 0.2
step 6400, train accuracy 0.3
step 6500, train accuracy 0.28
step 6600, train accuracy 0.28
step 6700, train accuracy 0.34
step 6800, train accuracy 0.2
step 6900, train accuracy 0.22
step 7000, train accuracy 0.28
step 7100, train accuracy 0.22
step 7200, train accuracy 0.34
step 7300, train accuracy 0.34
step 7400, train accuracy 0.28
step 7500, train accuracy 0.32
step 7600, train accuracy 0.22
step 7700, train accuracy 0.36
step 7800, train accuracy 0.38
step 7900, train accuracy 0.26
step 8000, train accuracy 0.28
step 8100, train accuracy 0.28
step 8200, train accuracy 0.34
step 8300, train accuracy 0.2
step 8400, train accuracy 0.48
step 8500, train accuracy 0.44
step 8600, train accuracy 0.36
step 8700, train accuracy 0.26
step 8800, train accuracy 0.4
step 8900, train accuracy 0.42
step 9000, train accuracy 0.26
step 9100, train accuracy 0.38
step 9200, train accuracy 0.34
step 9300, train accuracy 0.26
step 9400, train accuracy 0.36
step 9500, train accuracy 0.3
step 9600, train accuracy 0.38
step 9700, train accuracy 0.32
step 9800, train accuracy 0.38
step 9900, train accuracy 0.36
epoch 1, learning rate 4.64159e-05
step 10000, train accuracy 0.42
step 10100, train accuracy 0.4
step 10200, train accuracy 0.3
step 10300, train accuracy 0.4
step 10400, train accuracy 0.38
step 10500, train accuracy 0.44
step 10600, train accuracy 0.52
step 10700, train accuracy 0.5
step 10800, train accuracy 0.32
step 10900, train accuracy 0.38
step 11000, train accuracy 0.34
step 11100, train accuracy 0.34
step 11200, train accuracy 0.4
step 11300, train accuracy 0.48
step 11400, train accuracy 0.46
step 11500, train accuracy 0.5
step 11600, train accuracy 0.46
step 11700, train accuracy 0.4
step 11800, train accuracy 0.52
step 11900, train accuracy 0.26
step 12000, train accuracy 0.52
step 12100, train accuracy 0.52
step 12200, train accuracy 0.28
step 12300, train accuracy 0.46
step 12400, train accuracy 0.44
step 12500, train accuracy 0.34
step 12600, train accuracy 0.5
step 12700, train accuracy 0.46
step 12800, train accuracy 0.56
step 12900, train accuracy 0.52
step 13000, train accuracy 0.46
step 13100, train accuracy 0.42
step 13200, train accuracy 0.36
step 13300, train accuracy 0.44
step 13400, train accuracy 0.48
step 13500, train accuracy 0.28
step 13600, train accuracy 0.34
step 13700, train accuracy 0.46
step 13800, train accuracy 0.38
step 13900, train accuracy 0.48
step 14000, train accuracy 0.46
step 14100, train accuracy 0.32
step 14200, train accuracy 0.54
step 14300, train accuracy 0.44
step 14400, train accuracy 0.62
step 14500, train accuracy 0.38
step 14600, train accuracy 0.48
step 14700, train accuracy 0.52
step 14800, train accuracy 0.44
step 14900, train accuracy 0.4
step 15000, train accuracy 0.52
step 15100, train accuracy 0.36
step 15200, train accuracy 0.4
step 15300, train accuracy 0.56
step 15400, train accuracy 0.36
step 15500, train accuracy 0.5
step 15600, train accuracy 0.52
step 15700, train accuracy 0.44
step 15800, train accuracy 0.52
step 15900, train accuracy 0.42
step 16000, train accuracy 0.5
step 16100, train accuracy 0.64
step 16200, train accuracy 0.46
step 16300, train accuracy 0.42
step 16400, train accuracy 0.54
step 16500, train accuracy 0.44
step 16600, train accuracy 0.48
step 16700, train accuracy 0.46
step 16800, train accuracy 0.52
step 16900, train accuracy 0.36
step 17000, train accuracy 0.48
step 17100, train accuracy 0.4
step 17200, train accuracy 0.56
step 17300, train accuracy 0.48
step 17400, train accuracy 0.6
step 17500, train accuracy 0.56
step 17600, train accuracy 0.38
step 17700, train accuracy 0.58
step 17800, train accuracy 0.5
step 17900, train accuracy 0.6
step 18000, train accuracy 0.52
step 18100, train accuracy 0.44
step 18200, train accuracy 0.48
step 18300, train accuracy 0.44
step 18400, train accuracy 0.58
step 18500, train accuracy 0.4
step 18600, train accuracy 0.52
step 18700, train accuracy 0.44
step 18800, train accuracy 0.44
step 18900, train accuracy 0.58
step 19000, train accuracy 0.52
step 19100, train accuracy 0.54
step 19200, train accuracy 0.68
step 19300, train accuracy 0.58
step 19400, train accuracy 0.5
step 19500, train accuracy 0.48
step 19600, train accuracy 0.44
step 19700, train accuracy 0.44
step 19800, train accuracy 0.56
step 19900, train accuracy 0.56
epoch 2, learning rate 2.15443e-05
step 20000, train accuracy 0.46
step 20100, train accuracy 0.4
step 20200, train accuracy 0.4
step 20300, train accuracy 0.5
step 20400, train accuracy 0.58
step 20500, train accuracy 0.44
step 20600, train accuracy 0.56
step 20700, train accuracy 0.46
step 20800, train accuracy 0.6
step 20900, train accuracy 0.56
step 21000, train accuracy 0.54
step 21100, train accuracy 0.52
step 21200, train accuracy 0.54
step 21300, train accuracy 0.46
step 21400, train accuracy 0.48
step 21500, train accuracy 0.48
step 21600, train accuracy 0.6
step 21700, train accuracy 0.6
step 21800, train accuracy 0.5
step 21900, train accuracy 0.56
step 22000, train accuracy 0.6
step 22100, train accuracy 0.62
step 22200, train accuracy 0.56
step 22300, train accuracy 0.44
step 22400, train accuracy 0.54
step 22500, train accuracy 0.38
step 22600, train accuracy 0.52
step 22700, train accuracy 0.46
step 22800, train accuracy 0.58
step 22900, train accuracy 0.54
step 23000, train accuracy 0.58
step 23100, train accuracy 0.6
step 23200, train accuracy 0.5
step 23300, train accuracy 0.74
step 23400, train accuracy 0.58
step 23500, train accuracy 0.72
step 23600, train accuracy 0.5
step 23700, train accuracy 0.56
step 23800, train accuracy 0.44
step 23900, train accuracy 0.44
step 24000, train accuracy 0.5
step 24100, train accuracy 0.52
step 24200, train accuracy 0.56
step 24300, train accuracy 0.56
step 24400, train accuracy 0.52
step 24500, train accuracy 0.66
step 24600, train accuracy 0.5
step 24700, train accuracy 0.64
step 24800, train accuracy 0.62
step 24900, train accuracy 0.58
step 25000, train accuracy 0.62
step 25100, train accuracy 0.62
step 25200, train accuracy 0.48
step 25300, train accuracy 0.54
step 25400, train accuracy 0.66
step 25500, train accuracy 0.6
step 25600, train accuracy 0.6
step 25700, train accuracy 0.54
step 25800, train accuracy 0.58
step 25900, train accuracy 0.5
step 26000, train accuracy 0.56
step 26100, train accuracy 0.66
step 26200, train accuracy 0.52
step 26300, train accuracy 0.56
step 26400, train accuracy 0.6
step 26500, train accuracy 0.48
step 26600, train accuracy 0.52
step 26700, train accuracy 0.62
step 26800, train accuracy 0.7
step 26900, train accuracy 0.48
step 27000, train accuracy 0.56
step 27100, train accuracy 0.4
step 27200, train accuracy 0.6
step 27300, train accuracy 0.56
step 27400, train accuracy 0.5
step 27500, train accuracy 0.56
step 27600, train accuracy 0.66
step 27700, train accuracy 0.64
step 27800, train accuracy 0.6
step 27900, train accuracy 0.66
step 28000, train accuracy 0.48
step 28100, train accuracy 0.42
step 28200, train accuracy 0.54
step 28300, train accuracy 0.54
step 28400, train accuracy 0.72
step 28500, train accuracy 0.54
step 28600, train accuracy 0.46
step 28700, train accuracy 0.54
step 28800, train accuracy 0.64
step 28900, train accuracy 0.5
step 29000, train accuracy 0.5
step 29100, train accuracy 0.46
step 29200, train accuracy 0.46
step 29300, train accuracy 0.48
step 29400, train accuracy 0.5
step 29500, train accuracy 0.6
step 29600, train accuracy 0.58
step 29700, train accuracy 0.58
step 29800, train accuracy 0.6
step 29900, train accuracy 0.82
epoch 3, learning rate 1e-05
step 30000, train accuracy 0.5
step 30100, train accuracy 0.52
step 30200, train accuracy 0.64
step 30300, train accuracy 0.54
step 30400, train accuracy 0.58
step 30500, train accuracy 0.56
step 30600, train accuracy 0.56
step 30700, train accuracy 0.66
step 30800, train accuracy 0.54
step 30900, train accuracy 0.62
step 31000, train accuracy 0.56
step 31100, train accuracy 0.48
step 31200, train accuracy 0.56
step 31300, train accuracy 0.64
step 31400, train accuracy 0.5
step 31500, train accuracy 0.58
step 31600, train accuracy 0.64
step 31700, train accuracy 0.64
step 31800, train accuracy 0.68
step 31900, train accuracy 0.56
step 32000, train accuracy 0.54
step 32100, train accuracy 0.6
step 32200, train accuracy 0.5
step 32300, train accuracy 0.52
step 32400, train accuracy 0.56
step 32500, train accuracy 0.52
step 32600, train accuracy 0.54
step 32700, train accuracy 0.46
step 32800, train accuracy 0.56
step 32900, train accuracy 0.52
step 33000, train accuracy 0.58
step 33100, train accuracy 0.56
step 33200, train accuracy 0.58
step 33300, train accuracy 0.58
step 33400, train accuracy 0.52
step 33500, train accuracy 0.5
step 33600, train accuracy 0.52
step 33700, train accuracy 0.7
step 33800, train accuracy 0.64
step 33900, train accuracy 0.52
step 34000, train accuracy 0.52
step 34100, train accuracy 0.52
step 34200, train accuracy 0.64
step 34300, train accuracy 0.48
step 34400, train accuracy 0.52
step 34500, train accuracy 0.5
step 34600, train accuracy 0.56
step 34700, train accuracy 0.66
step 34800, train accuracy 0.56
step 34900, train accuracy 0.62
step 35000, train accuracy 0.52
step 35100, train accuracy 0.54
step 35200, train accuracy 0.64
step 35300, train accuracy 0.68
step 35400, train accuracy 0.66
step 35500, train accuracy 0.58
step 35600, train accuracy 0.58
step 35700, train accuracy 0.54
step 35800, train accuracy 0.6
step 35900, train accuracy 0.5
step 36000, train accuracy 0.52
step 36100, train accuracy 0.58
step 36200, train accuracy 0.72
step 36300, train accuracy 0.52
step 36400, train accuracy 0.62
step 36500, train accuracy 0.58
step 36600, train accuracy 0.58
step 36700, train accuracy 0.54
step 36800, train accuracy 0.54
step 36900, train accuracy 0.58
step 37000, train accuracy 0.6
step 37100, train accuracy 0.48
step 37200, train accuracy 0.54
step 37300, train accuracy 0.58
step 37400, train accuracy 0.66
step 37500, train accuracy 0.56
step 37600, train accuracy 0.64
step 37700, train accuracy 0.66
step 37800, train accuracy 0.48
step 37900, train accuracy 0.66
step 38000, train accuracy 0.52
step 38100, train accuracy 0.54
step 38200, train accuracy 0.62
step 38300, train accuracy 0.64
step 38400, train accuracy 0.6
step 38500, train accuracy 0.58
step 38600, train accuracy 0.58
step 38700, train accuracy 0.54
step 38800, train accuracy 0.54
step 38900, train accuracy 0.62
step 39000, train accuracy 0.68
step 39100, train accuracy 0.62
step 39200, train accuracy 0.52
step 39300, train accuracy 0.48
step 39400, train accuracy 0.52
step 39500, train accuracy 0.48
step 39600, train accuracy 0.48
step 39700, train accuracy 0.58
step 39800, train accuracy 0.54
step 39900, train accuracy 0.44
epoch 4, learning rate 4.64159e-06
step 40000, train accuracy 0.56
step 40100, train accuracy 0.62
step 40200, train accuracy 0.58
step 40300, train accuracy 0.56
step 40400, train accuracy 0.62
step 40500, train accuracy 0.62
step 40600, train accuracy 0.64
step 40700, train accuracy 0.46
step 40800, train accuracy 0.62
step 40900, train accuracy 0.68
step 41000, train accuracy 0.5
step 41100, train accuracy 0.6
step 41200, train accuracy 0.58
step 41300, train accuracy 0.48
step 41400, train accuracy 0.66
step 41500, train accuracy 0.4
step 41600, train accuracy 0.74
step 41700, train accuracy 0.5
step 41800, train accuracy 0.6
step 41900, train accuracy 0.66
step 42000, train accuracy 0.58
step 42100, train accuracy 0.5
step 42200, train accuracy 0.56
step 42300, train accuracy 0.62
step 42400, train accuracy 0.66
step 42500, train accuracy 0.48
step 42600, train accuracy 0.54
step 42700, train accuracy 0.6
step 42800, train accuracy 0.48
step 42900, train accuracy 0.58
step 43000, train accuracy 0.58
step 43100, train accuracy 0.52
step 43200, train accuracy 0.54
step 43300, train accuracy 0.46
step 43400, train accuracy 0.52
step 43500, train accuracy 0.64
step 43600, train accuracy 0.58
step 43700, train accuracy 0.48
step 43800, train accuracy 0.52
step 43900, train accuracy 0.64
step 44000, train accuracy 0.72
step 44100, train accuracy 0.64
step 44200, train accuracy 0.54
step 44300, train accuracy 0.54
step 44400, train accuracy 0.6
step 44500, train accuracy 0.56
step 44600, train accuracy 0.62
step 44700, train accuracy 0.56
step 44800, train accuracy 0.58
step 44900, train accuracy 0.44
step 45000, train accuracy 0.62
step 45100, train accuracy 0.66
step 45200, train accuracy 0.68
step 45300, train accuracy 0.54
step 45400, train accuracy 0.46
step 45500, train accuracy 0.44
step 45600, train accuracy 0.56
step 45700, train accuracy 0.48
step 45800, train accuracy 0.5
step 45900, train accuracy 0.58
step 46000, train accuracy 0.54
step 46100, train accuracy 0.64
step 46200, train accuracy 0.5
step 46300, train accuracy 0.52
step 46400, train accuracy 0.62
step 46500, train accuracy 0.62
step 46600, train accuracy 0.56
step 46700, train accuracy 0.6
step 46800, train accuracy 0.62
step 46900, train accuracy 0.62
step 47000, train accuracy 0.66
step 47100, train accuracy 0.6
step 47200, train accuracy 0.66
step 47300, train accuracy 0.5
step 47400, train accuracy 0.68
step 47500, train accuracy 0.66
step 47600, train accuracy 0.64
step 47700, train accuracy 0.7
step 47800, train accuracy 0.64
step 47900, train accuracy 0.56
step 48000, train accuracy 0.48
step 48100, train accuracy 0.56
step 48200, train accuracy 0.6
step 48300, train accuracy 0.64
step 48400, train accuracy 0.66
step 48500, train accuracy 0.62
step 48600, train accuracy 0.68
step 48700, train accuracy 0.5
step 48800, train accuracy 0.58
step 48900, train accuracy 0.66
step 49000, train accuracy 0.56
step 49100, train accuracy 0.58
step 49200, train accuracy 0.54
step 49300, train accuracy 0.4
step 49400, train accuracy 0.46
step 49500, train accuracy 0.62
step 49600, train accuracy 0.56
step 49700, train accuracy 0.56
step 49800, train accuracy 0.58
step 49900, train accuracy 0.52
epoch 5, learning rate 2.15443e-06
step 50000, train accuracy 0.38
step 50100, train accuracy 0.5
step 50200, train accuracy 0.58
step 50300, train accuracy 0.66
step 50400, train accuracy 0.62
step 50500, train accuracy 0.6
step 50600, train accuracy 0.58
step 50700, train accuracy 0.58
step 50800, train accuracy 0.62
step 50900, train accuracy 0.62
step 51000, train accuracy 0.64
step 51100, train accuracy 0.5
step 51200, train accuracy 0.6
step 51300, train accuracy 0.62
step 51400, train accuracy 0.46
step 51500, train accuracy 0.56
step 51600, train accuracy 0.6
step 51700, train accuracy 0.64
step 51800, train accuracy 0.62
step 51900, train accuracy 0.68
step 52000, train accuracy 0.6
step 52100, train accuracy 0.66
step 52200, train accuracy 0.58
step 52300, train accuracy 0.68
step 52400, train accuracy 0.52
step 52500, train accuracy 0.6
step 52600, train accuracy 0.64
step 52700, train accuracy 0.56
step 52800, train accuracy 0.52
step 52900, train accuracy 0.64
step 53000, train accuracy 0.54
step 53100, train accuracy 0.6
step 53200, train accuracy 0.6
step 53300, train accuracy 0.56
step 53400, train accuracy 0.52
step 53500, train accuracy 0.54
step 53600, train accuracy 0.48
step 53700, train accuracy 0.5
step 53800, train accuracy 0.64
step 53900, train accuracy 0.58
step 54000, train accuracy 0.56
step 54100, train accuracy 0.54
step 54200, train accuracy 0.58
step 54300, train accuracy 0.62
step 54400, train accuracy 0.52
step 54500, train accuracy 0.62
step 54600, train accuracy 0.64
step 54700, train accuracy 0.44
step 54800, train accuracy 0.56
step 54900, train accuracy 0.68
step 55000, train accuracy 0.34
step 55100, train accuracy 0.48
step 55200, train accuracy 0.56
step 55300, train accuracy 0.58
step 55400, train accuracy 0.56
step 55500, train accuracy 0.62
step 55600, train accuracy 0.56
step 55700, train accuracy 0.56
step 55800, train accuracy 0.62
step 55900, train accuracy 0.56
step 56000, train accuracy 0.62
step 56100, train accuracy 0.56
step 56200, train accuracy 0.58
step 56300, train accuracy 0.58
step 56400, train accuracy 0.66
step 56500, train accuracy 0.6
step 56600, train accuracy 0.58
step 56700, train accuracy 0.68
step 56800, train accuracy 0.68
step 56900, train accuracy 0.48
step 57000, train accuracy 0.4
step 57100, train accuracy 0.56
step 57200, train accuracy 0.58
step 57300, train accuracy 0.64
step 57400, train accuracy 0.46
step 57500, train accuracy 0.68
step 57600, train accuracy 0.54
step 57700, train accuracy 0.6
step 57800, train accuracy 0.56
step 57900, train accuracy 0.54
step 58000, train accuracy 0.64
step 58100, train accuracy 0.52
step 58200, train accuracy 0.7
step 58300, train accuracy 0.7
step 58400, train accuracy 0.74
step 58500, train accuracy 0.6
step 58600, train accuracy 0.6
step 58700, train accuracy 0.6
step 58800, train accuracy 0.64
step 58900, train accuracy 0.52
step 59000, train accuracy 0.6
step 59100, train accuracy 0.64
step 59200, train accuracy 0.5
step 59300, train accuracy 0.62
step 59400, train accuracy 0.62
step 59500, train accuracy 0.5
step 59600, train accuracy 0.56
step 59700, train accuracy 0.5
step 59800, train accuracy 0.6
step 59900, train accuracy 0.6
epoch 6, learning rate 1e-06
step 60000, train accuracy 0.64
step 60100, train accuracy 0.56
step 60200, train accuracy 0.48
step 60300, train accuracy 0.58
step 60400, train accuracy 0.42
step 60500, train accuracy 0.64
step 60600, train accuracy 0.66
step 60700, train accuracy 0.48
step 60800, train accuracy 0.64
step 60900, train accuracy 0.66
step 61000, train accuracy 0.64
step 61100, train accuracy 0.46
step 61200, train accuracy 0.5
step 61300, train accuracy 0.56
step 61400, train accuracy 0.64
step 61500, train accuracy 0.4
step 61600, train accuracy 0.66
step 61700, train accuracy 0.62
step 61800, train accuracy 0.64
step 61900, train accuracy 0.5
step 62000, train accuracy 0.6
step 62100, train accuracy 0.64
step 62200, train accuracy 0.5
step 62300, train accuracy 0.76
step 62400, train accuracy 0.66
step 62500, train accuracy 0.48
step 62600, train accuracy 0.64
step 62700, train accuracy 0.66
step 62800, train accuracy 0.58
step 62900, train accuracy 0.62
step 63000, train accuracy 0.52
step 63100, train accuracy 0.58
step 63200, train accuracy 0.58
step 63300, train accuracy 0.62
step 63400, train accuracy 0.62
step 63500, train accuracy 0.52
step 63600, train accuracy 0.52
step 63700, train accuracy 0.48
step 63800, train accuracy 0.56
step 63900, train accuracy 0.56
step 64000, train accuracy 0.5
step 64100, train accuracy 0.44
step 64200, train accuracy 0.58
step 64300, train accuracy 0.58
step 64400, train accuracy 0.66
step 64500, train accuracy 0.52
step 64600, train accuracy 0.7
step 64700, train accuracy 0.64
step 64800, train accuracy 0.56
step 64900, train accuracy 0.58
step 65000, train accuracy 0.62
step 65100, train accuracy 0.54
step 65200, train accuracy 0.6
step 65300, train accuracy 0.5
step 65400, train accuracy 0.48
step 65500, train accuracy 0.62
step 65600, train accuracy 0.66
step 65700, train accuracy 0.58
step 65800, train accuracy 0.58
step 65900, train accuracy 0.58
step 66000, train accuracy 0.56
step 66100, train accuracy 0.56
step 66200, train accuracy 0.52
step 66300, train accuracy 0.72
step 66400, train accuracy 0.58
step 66500, train accuracy 0.62
step 66600, train accuracy 0.66
step 66700, train accuracy 0.54
step 66800, train accuracy 0.62
step 66900, train accuracy 0.54
step 67000, train accuracy 0.54
step 67100, train accuracy 0.64
step 67200, train accuracy 0.6
step 67300, train accuracy 0.6
step 67400, train accuracy 0.66
step 67500, train accuracy 0.58
step 67600, train accuracy 0.64
step 67700, train accuracy 0.58
step 67800, train accuracy 0.54
step 67900, train accuracy 0.74
step 68000, train accuracy 0.5
step 68100, train accuracy 0.66
step 68200, train accuracy 0.64
step 68300, train accuracy 0.5
step 68400, train accuracy 0.58
step 68500, train accuracy 0.56
step 68600, train accuracy 0.58
step 68700, train accuracy 0.54
step 68800, train accuracy 0.52
step 68900, train accuracy 0.58
step 69000, train accuracy 0.52
step 69100, train accuracy 0.54
step 69200, train accuracy 0.54
step 69300, train accuracy 0.6
step 69400, train accuracy 0.66
step 69500, train accuracy 0.6
step 69600, train accuracy 0.54
step 69700, train accuracy 0.64
step 69800, train accuracy 0.56
step 69900, train accuracy 0.46
epoch 7, learning rate 4.64159e-07
step 70000, train accuracy 0.66
step 70100, train accuracy 0.54
step 70200, train accuracy 0.66
step 70300, train accuracy 0.6
step 70400, train accuracy 0.54
step 70500, train accuracy 0.56
step 70600, train accuracy 0.58
step 70700, train accuracy 0.54
step 70800, train accuracy 0.56
step 70900, train accuracy 0.66
step 71000, train accuracy 0.6
step 71100, train accuracy 0.56
step 71200, train accuracy 0.6
step 71300, train accuracy 0.56
step 71400, train accuracy 0.62
step 71500, train accuracy 0.7
step 71600, train accuracy 0.58
step 71700, train accuracy 0.56
step 71800, train accuracy 0.64
step 71900, train accuracy 0.62
step 72000, train accuracy 0.56
step 72100, train accuracy 0.46
step 72200, train accuracy 0.64
step 72300, train accuracy 0.58
step 72400, train accuracy 0.48
step 72500, train accuracy 0.58
step 72600, train accuracy 0.62
step 72700, train accuracy 0.72
step 72800, train accuracy 0.5
step 72900, train accuracy 0.64
step 73000, train accuracy 0.58
step 73100, train accuracy 0.68
step 73200, train accuracy 0.6
step 73300, train accuracy 0.6
step 73400, train accuracy 0.8
step 73500, train accuracy 0.54
step 73600, train accuracy 0.58
step 73700, train accuracy 0.64
step 73800, train accuracy 0.54
step 73900, train accuracy 0.56
step 74000, train accuracy 0.7
step 74100, train accuracy 0.58
step 74200, train accuracy 0.54
step 74300, train accuracy 0.6
step 74400, train accuracy 0.56
step 74500, train accuracy 0.66
step 74600, train accuracy 0.58
step 74700, train accuracy 0.66
step 74800, train accuracy 0.54
step 74900, train accuracy 0.58
step 75000, train accuracy 0.5
step 75100, train accuracy 0.5
step 75200, train accuracy 0.7
step 75300, train accuracy 0.56
step 75400, train accuracy 0.64
step 75500, train accuracy 0.5
step 75600, train accuracy 0.56
step 75700, train accuracy 0.68
step 75800, train accuracy 0.6
step 75900, train accuracy 0.5
step 76000, train accuracy 0.54
step 76100, train accuracy 0.66
step 76200, train accuracy 0.6
step 76300, train accuracy 0.64
step 76400, train accuracy 0.56
step 76500, train accuracy 0.58
step 76600, train accuracy 0.6
step 76700, train accuracy 0.58
step 76800, train accuracy 0.54
step 76900, train accuracy 0.7
step 77000, train accuracy 0.66
step 77100, train accuracy 0.62
step 77200, train accuracy 0.58
step 77300, train accuracy 0.58
step 77400, train accuracy 0.56
step 77500, train accuracy 0.64
step 77600, train accuracy 0.52
step 77700, train accuracy 0.54
step 77800, train accuracy 0.6
step 77900, train accuracy 0.52
step 78000, train accuracy 0.58
step 78100, train accuracy 0.56
step 78200, train accuracy 0.6
step 78300, train accuracy 0.72
step 78400, train accuracy 0.62
step 78500, train accuracy 0.58
step 78600, train accuracy 0.62
step 78700, train accuracy 0.58
step 78800, train accuracy 0.52
step 78900, train accuracy 0.58
step 79000, train accuracy 0.64
step 79100, train accuracy 0.46
step 79200, train accuracy 0.64
step 79300, train accuracy 0.58
step 79400, train accuracy 0.56
step 79500, train accuracy 0.54
step 79600, train accuracy 0.58
step 79700, train accuracy 0.56
step 79800, train accuracy 0.64
step 79900, train accuracy 0.62
epoch 8, learning rate 2.15443e-07
step 80000, train accuracy 0.74
step 80100, train accuracy 0.64
step 80200, train accuracy 0.64
step 80300, train accuracy 0.6
step 80400, train accuracy 0.7
step 80500, train accuracy 0.5
step 80600, train accuracy 0.7
step 80700, train accuracy 0.64
step 80800, train accuracy 0.58
step 80900, train accuracy 0.56
step 81000, train accuracy 0.64
step 81100, train accuracy 0.62
step 81200, train accuracy 0.6
step 81300, train accuracy 0.74
step 81400, train accuracy 0.58
step 81500, train accuracy 0.6
step 81600, train accuracy 0.72
step 81700, train accuracy 0.48
step 81800, train accuracy 0.56
step 81900, train accuracy 0.64
step 82000, train accuracy 0.54
step 82100, train accuracy 0.5
step 82200, train accuracy 0.56
step 82300, train accuracy 0.66
step 82400, train accuracy 0.6
step 82500, train accuracy 0.64
step 82600, train accuracy 0.58
step 82700, train accuracy 0.48
step 82800, train accuracy 0.62
step 82900, train accuracy 0.62
step 83000, train accuracy 0.56
step 83100, train accuracy 0.68
step 83200, train accuracy 0.5
step 83300, train accuracy 0.52
step 83400, train accuracy 0.62
step 83500, train accuracy 0.56
step 83600, train accuracy 0.62
step 83700, train accuracy 0.62
step 83800, train accuracy 0.54
step 83900, train accuracy 0.6
step 84000, train accuracy 0.64
step 84100, train accuracy 0.62
step 84200, train accuracy 0.56
step 84300, train accuracy 0.48
step 84400, train accuracy 0.52
step 84500, train accuracy 0.56
step 84600, train accuracy 0.6
step 84700, train accuracy 0.5
step 84800, train accuracy 0.46
step 84900, train accuracy 0.62
step 85000, train accuracy 0.62
step 85100, train accuracy 0.44
step 85200, train accuracy 0.66
step 85300, train accuracy 0.66
step 85400, train accuracy 0.6
step 85500, train accuracy 0.62
step 85600, train accuracy 0.64
step 85700, train accuracy 0.72
step 85800, train accuracy 0.54
step 85900, train accuracy 0.44
step 86000, train accuracy 0.64
step 86100, train accuracy 0.6
step 86200, train accuracy 0.52
step 86300, train accuracy 0.52
step 86400, train accuracy 0.68
step 86500, train accuracy 0.54
step 86600, train accuracy 0.6
step 86700, train accuracy 0.64
step 86800, train accuracy 0.6
step 86900, train accuracy 0.56
step 87000, train accuracy 0.54
step 87100, train accuracy 0.56
step 87200, train accuracy 0.48
step 87300, train accuracy 0.5
step 87400, train accuracy 0.64
step 87500, train accuracy 0.46
step 87600, train accuracy 0.68
step 87700, train accuracy 0.56
step 87800, train accuracy 0.68
step 87900, train accuracy 0.68
step 88000, train accuracy 0.58
step 88100, train accuracy 0.6
step 88200, train accuracy 0.6
step 88300, train accuracy 0.58
step 88400, train accuracy 0.58
step 88500, train accuracy 0.54
step 88600, train accuracy 0.6
step 88700, train accuracy 0.56
step 88800, train accuracy 0.62
step 88900, train accuracy 0.52
step 89000, train accuracy 0.62
step 89100, train accuracy 0.66
step 89200, train accuracy 0.48
step 89300, train accuracy 0.66
step 89400, train accuracy 0.5
step 89500, train accuracy 0.56
step 89600, train accuracy 0.62
step 89700, train accuracy 0.66
step 89800, train accuracy 0.62
step 89900, train accuracy 0.4
epoch 9, learning rate 1e-07
step 90000, train accuracy 0.64
step 90100, train accuracy 0.6
step 90200, train accuracy 0.62
step 90300, train accuracy 0.7
step 90400, train accuracy 0.64
step 90500, train accuracy 0.56
step 90600, train accuracy 0.52
step 90700, train accuracy 0.62
step 90800, train accuracy 0.54
step 90900, train accuracy 0.5
step 91000, train accuracy 0.46
step 91100, train accuracy 0.6
step 91200, train accuracy 0.66
step 91300, train accuracy 0.62
step 91400, train accuracy 0.58
step 91500, train accuracy 0.58
step 91600, train accuracy 0.6
step 91700, train accuracy 0.5
step 91800, train accuracy 0.66
step 91900, train accuracy 0.54
step 92000, train accuracy 0.46
step 92100, train accuracy 0.58
step 92200, train accuracy 0.6
step 92300, train accuracy 0.6
step 92400, train accuracy 0.68
step 92500, train accuracy 0.68
step 92600, train accuracy 0.64
step 92700, train accuracy 0.64
step 92800, train accuracy 0.52
step 92900, train accuracy 0.5
step 93000, train accuracy 0.54
step 93100, train accuracy 0.58
step 93200, train accuracy 0.64
step 93300, train accuracy 0.72
step 93400, train accuracy 0.48
step 93500, train accuracy 0.48
step 93600, train accuracy 0.54
step 93700, train accuracy 0.6
step 93800, train accuracy 0.54
step 93900, train accuracy 0.58
step 94000, train accuracy 0.56
step 94100, train accuracy 0.5
step 94200, train accuracy 0.6
step 94300, train accuracy 0.54
step 94400, train accuracy 0.58
step 94500, train accuracy 0.62
step 94600, train accuracy 0.5
step 94700, train accuracy 0.6
step 94800, train accuracy 0.54
step 94900, train accuracy 0.5
step 95000, train accuracy 0.7
step 95100, train accuracy 0.5
step 95200, train accuracy 0.62
step 95300, train accuracy 0.4
step 95400, train accuracy 0.62
step 95500, train accuracy 0.68
step 95600, train accuracy 0.7
step 95700, train accuracy 0.68
step 95800, train accuracy 0.56
step 95900, train accuracy 0.58
step 96000, train accuracy 0.64
step 96100, train accuracy 0.56
step 96200, train accuracy 0.48
step 96300, train accuracy 0.66
step 96400, train accuracy 0.72
step 96500, train accuracy 0.58
step 96600, train accuracy 0.48
step 96700, train accuracy 0.64
step 96800, train accuracy 0.58
step 96900, train accuracy 0.56
step 97000, train accuracy 0.64
step 97100, train accuracy 0.68
step 97200, train accuracy 0.56
step 97300, train accuracy 0.46
step 97400, train accuracy 0.54
step 97500, train accuracy 0.56
step 97600, train accuracy 0.66
step 97700, train accuracy 0.66
step 97800, train accuracy 0.6
step 97900, train accuracy 0.66
step 98000, train accuracy 0.54
step 98100, train accuracy 0.7
step 98200, train accuracy 0.72
step 98300, train accuracy 0.46
step 98400, train accuracy 0.62
step 98500, train accuracy 0.58
step 98600, train accuracy 0.46
step 98700, train accuracy 0.54
step 98800, train accuracy 0.76
step 98900, train accuracy 0.56
step 99000, train accuracy 0.7
step 99100, train accuracy 0.58
step 99200, train accuracy 0.48
step 99300, train accuracy 0.64
step 99400, train accuracy 0.58
step 99500, train accuracy 0.52
step 99600, train accuracy 0.58
step 99700, train accuracy 0.66
step 99800, train accuracy 0.66
step 99900, train accuracy 0.58
test accuracy 0.5943

In [5]:

    

with tf.Graph().as_default() as g: 

  learning_rate = tf.placeholder(tf.float32, shape=[])

  with tf.name_scope('in') as scope:
    x = tf.placeholder(tf.float32, shape=[None, 784])
    tf.histogram_summary(scope + '/image', x)
    tf.image_summary('input', tf.reshape(x, [-1,28,28,1]), 3)

  with tf.name_scope('hidden_1') as scope:
    W = tf.Variable(tf.truncated_normal([784,100], stddev=0.1))
    b = tf.Variable(tf.truncated_normal([100], stddev=0.1))
    tf.histogram_summary(scope + '/weights', W)
    tf.histogram_summary(scope + '/biases', b)
    a = tf.nn.relu(tf.matmul(x, W) + b)
    tf.histogram_summary(scope + '/activations', a)
  with tf.name_scope('hidden_2') as scope:
    W_2 = tf.Variable(tf.truncated_normal([100,100], stddev=0.1))
    b_2 = tf.Variable(tf.truncated_normal([100], stddev=0.1))
    tf.histogram_summary(scope + '/weights', W_2)
    tf.histogram_summary(scope + '/biases', b_2)
    a_2 = tf.nn.relu(tf.matmul(a, W_2) + b_2)
    tf.histogram_summary(scope + '/activations', a_2)
  with tf.name_scope('hidden_3') as scope:
    W_3 = tf.Variable(tf.truncated_normal([100,100], stddev=0.1))
    b_3 = tf.Variable(tf.truncated_normal([100], stddev=0.1))
    tf.histogram_summary(scope + '/weights', W_3)
    tf.histogram_summary(scope + '/biases', b_3)
    a_3 = tf.nn.relu(tf.matmul(a_2, W_3) + b_3)
    tf.histogram_summary(scope + '/activations', a_3)
  with tf.name_scope('out') as scope:
    y_ = tf.placeholder(tf.float32, shape=[None, 10])
    W_4 = tf.Variable(tf.truncated_normal([100,10], stddev=0.1))
    b_4 = tf.Variable(tf.truncated_normal([10], stddev=0.1))
    tf.histogram_summary(scope + '/weights', W_4)
    tf.histogram_summary(scope + '/biases', b_4)
    y = tf.nn.sigmoid(tf.matmul(a_3, W_4) + b_4)
    tf.histogram_summary(scope + '/activations', y)

  #     with tf.name_scope('hidden_2') as scope:
  #       a_2_r = tf.nn.relu(tf.matmul(a_3 - b_3, tf.transpose(W_3)))
  #       tf.histogram_summary(scope + '/recons_activations', a_2_r)
  #     with tf.name_scope('hidden_1') as scope:
  #       a_r = tf.nn.relu(tf.matmul(a_2_r - b_2, tf.transpose(W_2)))
  #       tf.histogram_summary(scope + '/recons_activations', a_r)
  #     with tf.name_scope('in') as scope:
  #       x_r = tf.matmul(a_r - b, tf.transpose(W))
  #       tf.histogram_summary(scope + '/reconstruction', x_r)
  #       tf.image_summary('reconstruction', tf.reshape(x_r, [-1,28,28,1]), 3)

  global_step = tf.Variable(0, name='global_step', trainable=False)

  # losses
  cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y) + (1 - y_) * tf.log(1-y), 
                                                reduction_indices=[1]),
                                 name='xentropy_mean')
  tf.scalar_summary(cross_entropy.op.name, cross_entropy)

  # training
  # train_step = tf.train.GradientDescentOptimizer(1e-6).minimize(cross_entropy)
#   train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(cross_entropy, 
#                                                             global_step=global_step)
  opt = tf.train.AdamOptimizer(learning_rate)
  var_list = [W,b, W_2,b_2, W_3,b_3, W_4,b_4]
  var_grad = tf.gradients(cross_entropy, var_list)
  grads_and_vars = zip(var_grad, var_list)
  train_step = opt.apply_gradients(grads_and_vars, 
                                   global_step=global_step)
  
  #     back_train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(reconstruction_error)

  # evaluation
  correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
  accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

  ########## INIT AND SUMMARY HAVE TO APPEAR HERE AT THE END TO BE PROPERLY DEFINED
  # misc
  init = tf.initialize_all_variables()
  summary_op = tf.merge_all_summaries()

  with tf.Session() as sess:

    sess.run(init)
    summary_writer = tf.train.SummaryWriter('./mnist_own_descent/own_adam', sess.graph)

    num_epochs = 10
    steps_per_epoch = 10000
    for epoch in range(num_epochs):
      l_r = 10**np.linspace(-5,-8,num_epochs)[epoch]
      print("epoch %d, learning rate %g"%(epoch,l_r))

      for step in range(steps_per_epoch):
        batch = mnist.train.next_batch(50)
        feed_dict = {x: batch[0], y_: batch[1], learning_rate: l_r}

  #           if step%1000 == 0:
  # #             reconstruction = x_r.eval(feed_dict=feed_dict)
  # #             print("Reconstruction of a %d"%(np.argmax(batch[1][0])))
  # #             plt.imshow(np.reshape(reconstruction[0],(28,28)),
  # #                        cmap=plt.get_cmap('gray'),
  # #                        interpolation='none',
  # #                        aspect="auto")
  # #             plt.show()

        if step%100 == 0:
          train_accuracy = accuracy.eval(feed_dict=feed_dict)
          print("step %d, train accuracy %g"%(step + epoch*steps_per_epoch, train_accuracy))
          summary_str = sess.run(summary_op, feed_dict=feed_dict)
          summary_writer.add_summary(summary_str, step + epoch*steps_per_epoch)
          summary_writer.flush()

        train_step.run(feed_dict=feed_dict)
  #           back_train_step.run(feed_dict=feed_dict)

    print("test accuracy %g"%accuracy.eval(feed_dict={
        x: mnist.test.images, y_: mnist.test.labels}))


    

    




epoch 0, learning rate 1e-05
step 0, train accuracy 0.08
step 100, train accuracy 0.1
step 200, train accuracy 0.08
step 300, train accuracy 0
step 400, train accuracy 0.06
step 500, train accuracy 0.08
step 600, train accuracy 0.06
step 700, train accuracy 0.08
step 800, train accuracy 0.12
step 900, train accuracy 0.18
step 1000, train accuracy 0.1
step 1100, train accuracy 0.14
step 1200, train accuracy 0.26
step 1300, train accuracy 0.22
step 1400, train accuracy 0.4
step 1500, train accuracy 0.4
step 1600, train accuracy 0.44
step 1700, train accuracy 0.6
step 1800, train accuracy 0.46
step 1900, train accuracy 0.64
step 2000, train accuracy 0.56
step 2100, train accuracy 0.7
step 2200, train accuracy 0.62
step 2300, train accuracy 0.74
step 2400, train accuracy 0.7
step 2500, train accuracy 0.62
step 2600, train accuracy 0.76
step 2700, train accuracy 0.82
step 2800, train accuracy 0.84
step 2900, train accuracy 0.8
step 3000, train accuracy 0.8
step 3100, train accuracy 0.78
step 3200, train accuracy 0.72
step 3300, train accuracy 0.74
step 3400, train accuracy 0.7
step 3500, train accuracy 0.68
step 3600, train accuracy 0.88
step 3700, train accuracy 0.88
step 3800, train accuracy 0.74
step 3900, train accuracy 0.8
step 4000, train accuracy 0.74
step 4100, train accuracy 0.68
step 4200, train accuracy 0.82
step 4300, train accuracy 0.84
step 4400, train accuracy 0.78
step 4500, train accuracy 0.86
step 4600, train accuracy 0.84
step 4700, train accuracy 0.78
step 4800, train accuracy 0.82
step 4900, train accuracy 0.82
step 5000, train accuracy 0.88
step 5100, train accuracy 0.84
step 5200, train accuracy 0.86
step 5300, train accuracy 0.9
step 5400, train accuracy 0.86
step 5500, train accuracy 0.88
step 5600, train accuracy 0.72
step 5700, train accuracy 0.82
step 5800, train accuracy 0.88
step 5900, train accuracy 0.88
step 6000, train accuracy 0.76
step 6100, train accuracy 0.92
step 6200, train accuracy 0.76
step 6300, train accuracy 0.86
step 6400, train accuracy 0.84
step 6500, train accuracy 0.9
step 6600, train accuracy 0.82
step 6700, train accuracy 0.9
step 6800, train accuracy 0.82
step 6900, train accuracy 0.82
step 7000, train accuracy 0.92
step 7100, train accuracy 0.88
step 7200, train accuracy 0.86
step 7300, train accuracy 0.94
step 7400, train accuracy 0.8
step 7500, train accuracy 0.84
step 7600, train accuracy 0.92
step 7700, train accuracy 0.86
step 7800, train accuracy 0.82
step 7900, train accuracy 0.82
step 8000, train accuracy 0.92
step 8100, train accuracy 0.92
step 8200, train accuracy 0.88
step 8300, train accuracy 0.82
step 8400, train accuracy 0.86
step 8500, train accuracy 0.9
step 8600, train accuracy 0.9
step 8700, train accuracy 0.84
step 8800, train accuracy 0.82
step 8900, train accuracy 0.96
step 9000, train accuracy 0.92
step 9100, train accuracy 0.84
step 9200, train accuracy 0.82
step 9300, train accuracy 0.86
step 9400, train accuracy 0.9
step 9500, train accuracy 0.82
step 9600, train accuracy 0.9
step 9700, train accuracy 0.88
step 9800, train accuracy 0.9
step 9900, train accuracy 0.92
epoch 1, learning rate 4.64159e-06
step 10000, train accuracy 0.94
step 10100, train accuracy 0.96
step 10200, train accuracy 0.84
step 10300, train accuracy 0.92
step 10400, train accuracy 0.9
step 10500, train accuracy 0.94
step 10600, train accuracy 0.86
step 10700, train accuracy 0.88
step 10800, train accuracy 0.9
step 10900, train accuracy 0.92
step 11000, train accuracy 0.94
step 11100, train accuracy 0.88
step 11200, train accuracy 0.84
step 11300, train accuracy 0.8
step 11400, train accuracy 0.92
step 11500, train accuracy 0.88
step 11600, train accuracy 0.9
step 11700, train accuracy 0.9
step 11800, train accuracy 0.92
step 11900, train accuracy 0.88
step 12000, train accuracy 0.92
step 12100, train accuracy 0.86
step 12200, train accuracy 0.84
step 12300, train accuracy 0.9
step 12400, train accuracy 0.94
step 12500, train accuracy 0.9
step 12600, train accuracy 0.98
step 12700, train accuracy 0.82
step 12800, train accuracy 0.84
step 12900, train accuracy 0.94
step 13000, train accuracy 0.8
step 13100, train accuracy 0.88
step 13200, train accuracy 0.86
step 13300, train accuracy 0.84
step 13400, train accuracy 0.9
step 13500, train accuracy 0.92
step 13600, train accuracy 0.9
step 13700, train accuracy 0.98
step 13800, train accuracy 0.86
step 13900, train accuracy 0.96
step 14000, train accuracy 0.82
step 14100, train accuracy 0.92
step 14200, train accuracy 0.82
step 14300, train accuracy 0.9
step 14400, train accuracy 0.94
step 14500, train accuracy 0.8
step 14600, train accuracy 0.82
step 14700, train accuracy 0.94
step 14800, train accuracy 0.9
step 14900, train accuracy 0.96
step 15000, train accuracy 0.92
step 15100, train accuracy 0.84
step 15200, train accuracy 0.9
step 15300, train accuracy 0.88
step 15400, train accuracy 0.8
step 15500, train accuracy 0.92
step 15600, train accuracy 0.9
step 15700, train accuracy 0.86
step 15800, train accuracy 0.84
step 15900, train accuracy 0.9
step 16000, train accuracy 0.9
step 16100, train accuracy 0.84
step 16200, train accuracy 0.9
step 16300, train accuracy 0.86
step 16400, train accuracy 0.86
step 16500, train accuracy 0.88
step 16600, train accuracy 0.84
step 16700, train accuracy 0.96
step 16800, train accuracy 0.92
step 16900, train accuracy 0.9
step 17000, train accuracy 0.94
step 17100, train accuracy 0.84
step 17200, train accuracy 0.94
step 17300, train accuracy 0.86
step 17400, train accuracy 0.96
step 17500, train accuracy 0.88
step 17600, train accuracy 0.84
step 17700, train accuracy 0.94
step 17800, train accuracy 0.94
step 17900, train accuracy 0.9
step 18000, train accuracy 0.88
step 18100, train accuracy 0.86
step 18200, train accuracy 0.88
step 18300, train accuracy 0.9
step 18400, train accuracy 0.88
step 18500, train accuracy 0.96
step 18600, train accuracy 0.9
step 18700, train accuracy 0.92
step 18800, train accuracy 0.9
step 18900, train accuracy 0.92
step 19000, train accuracy 0.96
step 19100, train accuracy 0.9
step 19200, train accuracy 0.86
step 19300, train accuracy 0.84
step 19400, train accuracy 0.84
step 19500, train accuracy 0.9
step 19600, train accuracy 0.82
step 19700, train accuracy 0.9
step 19800, train accuracy 0.82
step 19900, train accuracy 0.92
epoch 2, learning rate 2.15443e-06
step 20000, train accuracy 0.92
step 20100, train accuracy 0.9
step 20200, train accuracy 0.92
step 20300, train accuracy 0.9
step 20400, train accuracy 0.9
step 20500, train accuracy 0.92
step 20600, train accuracy 0.94
step 20700, train accuracy 0.76
step 20800, train accuracy 0.98
step 20900, train accuracy 0.88
step 21000, train accuracy 0.96
step 21100, train accuracy 0.92
step 21200, train accuracy 0.92
step 21300, train accuracy 0.92
step 21400, train accuracy 0.86
step 21500, train accuracy 0.92
step 21600, train accuracy 0.84
step 21700, train accuracy 0.92
step 21800, train accuracy 0.98
step 21900, train accuracy 0.86
step 22000, train accuracy 0.98
step 22100, train accuracy 0.88
step 22200, train accuracy 0.92
step 22300, train accuracy 0.9
step 22400, train accuracy 0.94
step 22500, train accuracy 0.92
step 22600, train accuracy 0.88
step 22700, train accuracy 0.86
step 22800, train accuracy 0.9
step 22900, train accuracy 0.94
step 23000, train accuracy 0.78
step 23100, train accuracy 0.82
step 23200, train accuracy 0.92
step 23300, train accuracy 0.96
step 23400, train accuracy 0.92
step 23500, train accuracy 0.92
step 23600, train accuracy 0.88
step 23700, train accuracy 0.88
step 23800, train accuracy 0.88
step 23900, train accuracy 0.92
step 24000, train accuracy 0.9
step 24100, train accuracy 0.94
step 24200, train accuracy 0.88
step 24300, train accuracy 0.82
step 24400, train accuracy 0.9
step 24500, train accuracy 0.9
step 24600, train accuracy 0.94
step 24700, train accuracy 0.94
step 24800, train accuracy 0.84
step 24900, train accuracy 0.88
step 25000, train accuracy 0.86
step 25100, train accuracy 0.92
step 25200, train accuracy 0.96
step 25300, train accuracy 0.94
step 25400, train accuracy 0.96
step 25500, train accuracy 0.86
step 25600, train accuracy 0.86
step 25700, train accuracy 0.86
step 25800, train accuracy 0.92
step 25900, train accuracy 0.9
step 26000, train accuracy 0.94
step 26100, train accuracy 0.92
step 26200, train accuracy 0.96
step 26300, train accuracy 0.98
step 26400, train accuracy 0.9
step 26500, train accuracy 0.92
step 26600, train accuracy 0.92
step 26700, train accuracy 0.98
step 26800, train accuracy 0.88
step 26900, train accuracy 0.98
step 27000, train accuracy 0.9
step 27100, train accuracy 0.92
step 27200, train accuracy 0.82
step 27300, train accuracy 0.9
step 27400, train accuracy 0.94
step 27500, train accuracy 0.86
step 27600, train accuracy 0.88
step 27700, train accuracy 0.88
step 27800, train accuracy 0.92
step 27900, train accuracy 1
step 28000, train accuracy 0.98
step 28100, train accuracy 0.92
step 28200, train accuracy 0.86
step 28300, train accuracy 0.9
step 28400, train accuracy 0.96
step 28500, train accuracy 0.96
step 28600, train accuracy 0.88
step 28700, train accuracy 0.9
step 28800, train accuracy 0.88
step 28900, train accuracy 0.9
step 29000, train accuracy 0.94
step 29100, train accuracy 0.88
step 29200, train accuracy 0.88
step 29300, train accuracy 0.92
step 29400, train accuracy 0.88
step 29500, train accuracy 0.94
step 29600, train accuracy 0.9
step 29700, train accuracy 0.96
step 29800, train accuracy 0.92
step 29900, train accuracy 0.82
epoch 3, learning rate 1e-06
step 30000, train accuracy 1
step 30100, train accuracy 0.92
step 30200, train accuracy 0.9
step 30300, train accuracy 0.86
step 30400, train accuracy 0.92
step 30500, train accuracy 0.9
step 30600, train accuracy 0.94
step 30700, train accuracy 0.88
step 30800, train accuracy 0.9
step 30900, train accuracy 0.94
step 31000, train accuracy 0.96
step 31100, train accuracy 0.94
step 31200, train accuracy 0.86
step 31300, train accuracy 0.88
step 31400, train accuracy 0.92
step 31500, train accuracy 0.9
step 31600, train accuracy 0.94
step 31700, train accuracy 0.88
step 31800, train accuracy 0.84
step 31900, train accuracy 0.92
step 32000, train accuracy 0.94
step 32100, train accuracy 0.98
step 32200, train accuracy 0.92
step 32300, train accuracy 0.98
step 32400, train accuracy 0.92
step 32500, train accuracy 0.94
step 32600, train accuracy 0.96
step 32700, train accuracy 0.9
step 32800, train accuracy 0.88
step 32900, train accuracy 0.9
step 33000, train accuracy 0.9
step 33100, train accuracy 0.94
step 33200, train accuracy 0.92
step 33300, train accuracy 0.92
step 33400, train accuracy 0.94
step 33500, train accuracy 0.92
step 33600, train accuracy 0.94
step 33700, train accuracy 0.88
step 33800, train accuracy 0.92
step 33900, train accuracy 0.88
step 34000, train accuracy 0.86
step 34100, train accuracy 0.92
step 34200, train accuracy 0.92
step 34300, train accuracy 0.92
step 34400, train accuracy 0.88
step 34500, train accuracy 0.9
step 34600, train accuracy 0.94
step 34700, train accuracy 0.82
step 34800, train accuracy 0.9
step 34900, train accuracy 0.88
step 35000, train accuracy 0.88
step 35100, train accuracy 0.92
step 35200, train accuracy 0.9
step 35300, train accuracy 0.86
step 35400, train accuracy 0.92
step 35500, train accuracy 0.86
step 35600, train accuracy 0.84
step 35700, train accuracy 0.88
step 35800, train accuracy 0.88
step 35900, train accuracy 0.86
step 36000, train accuracy 0.94
step 36100, train accuracy 0.86
step 36200, train accuracy 0.9
step 36300, train accuracy 0.86
step 36400, train accuracy 0.9
step 36500, train accuracy 0.92
step 36600, train accuracy 0.9
step 36700, train accuracy 0.96
step 36800, train accuracy 0.94
step 36900, train accuracy 0.88
step 37000, train accuracy 0.82
step 37100, train accuracy 0.98
step 37200, train accuracy 0.88
step 37300, train accuracy 0.92
step 37400, train accuracy 0.94
step 37500, train accuracy 0.92
step 37600, train accuracy 0.92
step 37700, train accuracy 0.92
step 37800, train accuracy 0.94
step 37900, train accuracy 0.94
step 38000, train accuracy 0.86
step 38100, train accuracy 0.94
step 38200, train accuracy 0.9
step 38300, train accuracy 0.92
step 38400, train accuracy 0.88
step 38500, train accuracy 0.92
step 38600, train accuracy 0.92
step 38700, train accuracy 0.94
step 38800, train accuracy 0.92
step 38900, train accuracy 0.94
step 39000, train accuracy 0.92
step 39100, train accuracy 0.94
step 39200, train accuracy 0.92
step 39300, train accuracy 0.88
step 39400, train accuracy 0.98
step 39500, train accuracy 0.86
step 39600, train accuracy 0.86
step 39700, train accuracy 0.9
step 39800, train accuracy 0.94
step 39900, train accuracy 0.92
epoch 4, learning rate 4.64159e-07
step 40000, train accuracy 0.94
step 40100, train accuracy 0.86
step 40200, train accuracy 0.9
step 40300, train accuracy 0.9
step 40400, train accuracy 0.88
step 40500, train accuracy 0.96
step 40600, train accuracy 0.96
step 40700, train accuracy 0.9
step 40800, train accuracy 0.92
step 40900, train accuracy 0.88
step 41000, train accuracy 0.88
step 41100, train accuracy 0.9
step 41200, train accuracy 0.94
step 41300, train accuracy 0.94
step 41400, train accuracy 0.84
step 41500, train accuracy 0.94
step 41600, train accuracy 0.86
step 41700, train accuracy 0.88
step 41800, train accuracy 0.82
step 41900, train accuracy 0.92
step 42000, train accuracy 0.96
step 42100, train accuracy 0.96
step 42200, train accuracy 0.94
step 42300, train accuracy 0.92
step 42400, train accuracy 0.88
step 42500, train accuracy 0.94
step 42600, train accuracy 0.94
step 42700, train accuracy 0.88
step 42800, train accuracy 0.98
step 42900, train accuracy 0.88
step 43000, train accuracy 0.92
step 43100, train accuracy 0.94
step 43200, train accuracy 0.9
step 43300, train accuracy 0.92
step 43400, train accuracy 0.94
step 43500, train accuracy 0.94
step 43600, train accuracy 0.82
step 43700, train accuracy 0.92
step 43800, train accuracy 0.9
step 43900, train accuracy 0.84
step 44000, train accuracy 0.88
step 44100, train accuracy 0.88
step 44200, train accuracy 0.84
step 44300, train accuracy 0.98
step 44400, train accuracy 0.92
step 44500, train accuracy 0.94
step 44600, train accuracy 0.9
step 44700, train accuracy 0.88
step 44800, train accuracy 0.92
step 44900, train accuracy 0.9
step 45000, train accuracy 0.9
step 45100, train accuracy 0.94
step 45200, train accuracy 0.96
step 45300, train accuracy 0.82
step 45400, train accuracy 0.92
step 45500, train accuracy 0.94
step 45600, train accuracy 0.88
step 45700, train accuracy 0.88
step 45800, train accuracy 0.92
step 45900, train accuracy 0.96
step 46000, train accuracy 0.88
step 46100, train accuracy 0.86
step 46200, train accuracy 0.9
step 46300, train accuracy 0.9
step 46400, train accuracy 0.84
step 46500, train accuracy 0.92
step 46600, train accuracy 0.88
step 46700, train accuracy 0.88
step 46800, train accuracy 0.86
step 46900, train accuracy 0.92
step 47000, train accuracy 0.86
step 47100, train accuracy 0.9
step 47200, train accuracy 0.96
step 47300, train accuracy 0.86
step 47400, train accuracy 0.92
step 47500, train accuracy 0.94
step 47600, train accuracy 0.88
step 47700, train accuracy 0.94
step 47800, train accuracy 0.96
step 47900, train accuracy 0.9
step 48000, train accuracy 0.86
step 48100, train accuracy 0.88
step 48200, train accuracy 0.82
step 48300, train accuracy 0.94
step 48400, train accuracy 0.9
step 48500, train accuracy 0.88
step 48600, train accuracy 0.88
step 48700, train accuracy 0.94
step 48800, train accuracy 0.86
step 48900, train accuracy 0.92
step 49000, train accuracy 0.92
step 49100, train accuracy 0.86
step 49200, train accuracy 0.96
step 49300, train accuracy 0.84
step 49400, train accuracy 0.96
step 49500, train accuracy 0.9
step 49600, train accuracy 0.88
step 49700, train accuracy 0.9
step 49800, train accuracy 0.9
step 49900, train accuracy 0.82
epoch 5, learning rate 2.15443e-07
step 50000, train accuracy 0.9
step 50100, train accuracy 0.9
step 50200, train accuracy 0.96
step 50300, train accuracy 0.86
step 50400, train accuracy 0.92
step 50500, train accuracy 0.86
step 50600, train accuracy 0.94
step 50700, train accuracy 0.9
step 50800, train accuracy 0.9
step 50900, train accuracy 0.88
step 51000, train accuracy 0.92
step 51100, train accuracy 0.92
step 51200, train accuracy 0.98
step 51300, train accuracy 0.84
step 51400, train accuracy 0.96
step 51500, train accuracy 0.84
step 51600, train accuracy 0.92
step 51700, train accuracy 0.86
step 51800, train accuracy 0.98
step 51900, train accuracy 0.88
step 52000, train accuracy 0.94
step 52100, train accuracy 0.94
step 52200, train accuracy 0.92
step 52300, train accuracy 0.98
step 52400, train accuracy 0.92
step 52500, train accuracy 0.78
step 52600, train accuracy 0.96
step 52700, train accuracy 0.88
step 52800, train accuracy 0.94
step 52900, train accuracy 0.88
step 53000, train accuracy 0.88
step 53100, train accuracy 0.92
step 53200, train accuracy 0.92
step 53300, train accuracy 0.86
step 53400, train accuracy 0.88
step 53500, train accuracy 0.96
step 53600, train accuracy 0.9
step 53700, train accuracy 0.96
step 53800, train accuracy 0.94
step 53900, train accuracy 0.96
step 54000, train accuracy 0.98
step 54100, train accuracy 0.88
step 54200, train accuracy 0.86
step 54300, train accuracy 0.92
step 54400, train accuracy 0.92
step 54500, train accuracy 0.9
step 54600, train accuracy 0.98
step 54700, train accuracy 0.94
step 54800, train accuracy 0.94
step 54900, train accuracy 0.9
step 55000, train accuracy 0.9
step 55100, train accuracy 0.9
step 55200, train accuracy 0.9
step 55300, train accuracy 0.86
step 55400, train accuracy 0.96
step 55500, train accuracy 0.9
step 55600, train accuracy 0.94
step 55700, train accuracy 0.86
step 55800, train accuracy 0.82
step 55900, train accuracy 0.84
step 56000, train accuracy 0.92
step 56100, train accuracy 0.92
step 56200, train accuracy 0.94
step 56300, train accuracy 0.9
step 56400, train accuracy 0.96
step 56500, train accuracy 0.92
step 56600, train accuracy 0.88
step 56700, train accuracy 0.88
step 56800, train accuracy 0.94
step 56900, train accuracy 0.92
step 57000, train accuracy 0.88
step 57100, train accuracy 0.94
step 57200, train accuracy 0.96
step 57300, train accuracy 0.94
step 57400, train accuracy 0.92
step 57500, train accuracy 0.9
step 57600, train accuracy 0.92
step 57700, train accuracy 0.86
step 57800, train accuracy 0.9
step 57900, train accuracy 0.96
step 58000, train accuracy 0.84
step 58100, train accuracy 0.92
step 58200, train accuracy 0.92
step 58300, train accuracy 0.88
step 58400, train accuracy 0.86
step 58500, train accuracy 0.88
step 58600, train accuracy 0.92
step 58700, train accuracy 0.92
step 58800, train accuracy 0.96
step 58900, train accuracy 0.84
step 59000, train accuracy 0.92
step 59100, train accuracy 0.88
step 59200, train accuracy 0.96
step 59300, train accuracy 0.98
step 59400, train accuracy 0.86
step 59500, train accuracy 0.84
step 59600, train accuracy 0.88
step 59700, train accuracy 1
step 59800, train accuracy 0.92
step 59900, train accuracy 0.86
epoch 6, learning rate 1e-07
step 60000, train accuracy 0.88
step 60100, train accuracy 0.94
step 60200, train accuracy 0.86
step 60300, train accuracy 0.86
step 60400, train accuracy 0.9
step 60500, train accuracy 1
step 60600, train accuracy 0.84
step 60700, train accuracy 0.88
step 60800, train accuracy 0.94
step 60900, train accuracy 0.88
step 61000, train accuracy 0.9
step 61100, train accuracy 0.86
step 61200, train accuracy 0.96
step 61300, train accuracy 0.9
step 61400, train accuracy 0.96
step 61500, train accuracy 0.96
step 61600, train accuracy 0.96
step 61700, train accuracy 0.9
step 61800, train accuracy 0.94
step 61900, train accuracy 0.98
step 62000, train accuracy 0.96
step 62100, train accuracy 0.78
step 62200, train accuracy 0.92
step 62300, train accuracy 0.94
step 62400, train accuracy 0.96
step 62500, train accuracy 0.96
step 62600, train accuracy 0.88
step 62700, train accuracy 0.88
step 62800, train accuracy 0.86
step 62900, train accuracy 0.94
step 63000, train accuracy 0.88
step 63100, train accuracy 0.84
step 63200, train accuracy 0.88
step 63300, train accuracy 0.92
step 63400, train accuracy 0.92
step 63500, train accuracy 0.9
step 63600, train accuracy 0.86
step 63700, train accuracy 0.9
step 63800, train accuracy 0.94
step 63900, train accuracy 0.92
step 64000, train accuracy 0.9
step 64100, train accuracy 0.98
step 64200, train accuracy 0.92
step 64300, train accuracy 0.88
step 64400, train accuracy 0.88
step 64500, train accuracy 0.92
step 64600, train accuracy 0.92
step 64700, train accuracy 0.92
step 64800, train accuracy 0.92
step 64900, train accuracy 0.88
step 65000, train accuracy 0.94
step 65100, train accuracy 0.96
step 65200, train accuracy 0.9
step 65300, train accuracy 0.94
step 65400, train accuracy 0.84
step 65500, train accuracy 0.84
step 65600, train accuracy 0.88
step 65700, train accuracy 0.9
step 65800, train accuracy 0.96
step 65900, train accuracy 0.86
step 66000, train accuracy 0.92
step 66100, train accuracy 0.96
step 66200, train accuracy 0.92
step 66300, train accuracy 0.96
step 66400, train accuracy 0.9
step 66500, train accuracy 0.92
step 66600, train accuracy 0.88
step 66700, train accuracy 0.88
step 66800, train accuracy 0.92
step 66900, train accuracy 0.94
step 67000, train accuracy 0.92
step 67100, train accuracy 0.84
step 67200, train accuracy 0.86
step 67300, train accuracy 0.92
step 67400, train accuracy 0.94
step 67500, train accuracy 0.94
step 67600, train accuracy 0.9
step 67700, train accuracy 0.82
step 67800, train accuracy 0.94
step 67900, train accuracy 0.86
step 68000, train accuracy 0.92
step 68100, train accuracy 0.86
step 68200, train accuracy 0.9
step 68300, train accuracy 0.94
step 68400, train accuracy 0.92
step 68500, train accuracy 0.88
step 68600, train accuracy 0.86
step 68700, train accuracy 0.9
step 68800, train accuracy 0.86
step 68900, train accuracy 0.88
step 69000, train accuracy 0.94
step 69100, train accuracy 0.88
step 69200, train accuracy 0.94
step 69300, train accuracy 0.9
step 69400, train accuracy 0.94
step 69500, train accuracy 0.96
step 69600, train accuracy 0.88
step 69700, train accuracy 0.86
step 69800, train accuracy 0.98
step 69900, train accuracy 0.94
epoch 7, learning rate 4.64159e-08
step 70000, train accuracy 0.9
step 70100, train accuracy 0.9
step 70200, train accuracy 0.88
step 70300, train accuracy 0.94
step 70400, train accuracy 0.92
step 70500, train accuracy 0.92
step 70600, train accuracy 0.98
step 70700, train accuracy 0.98
step 70800, train accuracy 0.9
step 70900, train accuracy 0.92
step 71000, train accuracy 0.86
step 71100, train accuracy 0.96
step 71200, train accuracy 0.92
step 71300, train accuracy 0.92
step 71400, train accuracy 0.92
step 71500, train accuracy 0.92
step 71600, train accuracy 0.92
step 71700, train accuracy 0.92
step 71800, train accuracy 0.92
step 71900, train accuracy 0.88
step 72000, train accuracy 0.94
step 72100, train accuracy 0.96
step 72200, train accuracy 0.94
step 72300, train accuracy 0.98
step 72400, train accuracy 0.88
step 72500, train accuracy 0.94
step 72600, train accuracy 0.9
step 72700, train accuracy 0.88
step 72800, train accuracy 0.8
step 72900, train accuracy 0.84
step 73000, train accuracy 0.94
step 73100, train accuracy 0.92
step 73200, train accuracy 0.92
step 73300, train accuracy 0.94
step 73400, train accuracy 0.9
step 73500, train accuracy 0.98
step 73600, train accuracy 0.84
step 73700, train accuracy 0.82
step 73800, train accuracy 0.98
step 73900, train accuracy 0.82
step 74000, train accuracy 0.92
step 74100, train accuracy 0.86
step 74200, train accuracy 0.94
step 74300, train accuracy 0.82
step 74400, train accuracy 0.9
step 74500, train accuracy 0.9
step 74600, train accuracy 0.9
step 74700, train accuracy 0.92
step 74800, train accuracy 0.9
step 74900, train accuracy 0.86
step 75000, train accuracy 0.92
step 75100, train accuracy 0.88
step 75200, train accuracy 1
step 75300, train accuracy 0.92
step 75400, train accuracy 0.88
step 75500, train accuracy 0.94
step 75600, train accuracy 0.94
step 75700, train accuracy 0.94
step 75800, train accuracy 0.96
step 75900, train accuracy 0.88
step 76000, train accuracy 0.82
step 76100, train accuracy 0.96
step 76200, train accuracy 0.9
step 76300, train accuracy 0.88
step 76400, train accuracy 0.88
step 76500, train accuracy 0.9
step 76600, train accuracy 0.86
step 76700, train accuracy 0.96
step 76800, train accuracy 0.88
step 76900, train accuracy 0.88
step 77000, train accuracy 0.84
step 77100, train accuracy 0.92
step 77200, train accuracy 0.82
step 77300, train accuracy 0.92
step 77400, train accuracy 0.92
step 77500, train accuracy 0.9
step 77600, train accuracy 0.9
step 77700, train accuracy 0.94
step 77800, train accuracy 0.94
step 77900, train accuracy 0.98
step 78000, train accuracy 0.86
step 78100, train accuracy 0.96
step 78200, train accuracy 0.88
step 78300, train accuracy 0.9
step 78400, train accuracy 0.92






    




---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-5-d8622a81574a> in <module>()
    110           summary_writer.flush()
    111 
--> 112         train_step.run(feed_dict=feed_dict)
    113   #           back_train_step.run(feed_dict=feed_dict)
    114 

/home/frederik/anaconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/framework/ops.pyc in run(self, feed_dict, session)
   1459         none, the default session will be used.
   1460     """
-> 1461     _run_using_default_session(self, feed_dict, self.graph, session)
   1462 
   1463 

/home/frederik/anaconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/framework/ops.pyc in _run_using_default_session(operation, feed_dict, graph, session)
   3367                        "the operation's graph is different from the session's "
   3368                        "graph.")
-> 3369   session.run(operation, feed_dict)
   3370 
   3371 

/home/frederik/anaconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
    338     try:
    339       result = self._run(None, fetches, feed_dict, options_ptr,
--> 340                          run_metadata_ptr)
    341       if run_metadata:
    342         proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)

/home/frederik/anaconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
    562     try:
    563       results = self._do_run(handle, target_list, unique_fetches,
--> 564                              feed_dict_string, options, run_metadata)
    565     finally:
    566       # The movers are no longer used. Delete them.

/home/frederik/anaconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
    635     if handle is None:
    636       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 637                            target_list, options, run_metadata)
    638     else:
    639       return self._do_call(_prun_fn, self._session, handle, feed_dict,

/home/frederik/anaconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
    642   def _do_call(self, fn, *args):
    643     try:
--> 644       return fn(*args)
    645     except tf_session.StatusNotOK as e:
    646       error_message = compat.as_text(e.error_message)

/home/frederik/anaconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run_fn(session, feed_dict, fetch_list, target_list, options, run_metadata)
    626       else:
    627         return tf_session.TF_Run(
--> 628             session, None, feed_dict, fetch_list, target_list, None)
    629 
    630     def _prun_fn(session, handle, feed_dict, fetch_list):

KeyboardInterrupt: 

In [ ]: