Embeddings

事前準備


In [1]:
import os

import tensorflow as tf

sess = tf.Session()

In [2]:
LOG_PATH = "/root/tmp/tensorboard/mnist"
DATA_PATH = os.path.join(os.getcwd(), "mnist/data/")
LABELS_PATH = os.path.join(os.getcwd(), "mnist/labels_1024.tsv")
SPRITE_PATH = os.path.join(os.getcwd(), "mnist/sprite_1024.png")

In [3]:
mnist = tf.contrib.learn.datasets.mnist.read_data_sets(train_dir=DATA_PATH, one_hot=True)


Extracting /root/notebooks/playground_ja/tensorflow/mnist/data/train-images-idx3-ubyte.gz
Extracting /root/notebooks/playground_ja/tensorflow/mnist/data/train-labels-idx1-ubyte.gz
Extracting /root/notebooks/playground_ja/tensorflow/mnist/data/t10k-images-idx3-ubyte.gz
Extracting /root/notebooks/playground_ja/tensorflow/mnist/data/t10k-labels-idx1-ubyte.gz

Layer definition

Convolutional Layer


In [4]:
def conv_layer(input, channels_in, channels_out, name="conv"):
    with tf.name_scope(name):
        w = tf.Variable(tf.truncated_normal([5, 5, channels_in, channels_out], stddev=0.1), name="w")
        b = tf.Variable(tf.constant(0.1, shape=[channels_out]), name="b")
        conv = tf.nn.conv2d(input, w, strides=[1, 1, 1, 1], padding="SAME")
        a = tf.nn.relu(conv + b)
        tf.summary.histogram("weights", w)
        tf.summary.histogram("biases", b)
        tf.summary.histogram("activations", a)
        return a

Fully Connected Layer


In [5]:
def fc_layer(input, channels_in, channels_out, name="fc"):
    with tf.name_scope(name):
        w = tf.Variable(tf.truncated_normal([channels_in, channels_out], stddev=0.1), name="w")
        b = tf.Variable(tf.constant(0.1, shape=[channels_out]), name="b")
        a = tf.matmul(input, w) + b
        tf.summary.histogram("weights", w)
        tf.summary.histogram("biases", b)
        tf.summary.histogram("activations", a)
        return a

Feed-Forward network


In [6]:
# Setup placeholders, and reshape the data
x = tf.placeholder(tf.float32, shape=[None, 784], name="images")
y = tf.placeholder(tf.float32, shape=[None, 10], name="labels")

x_image = tf.reshape(x, [-1, 28, 28, 1])

tf.summary.image('input', x_image, 3)

# Create the network
conv1 = conv_layer(x_image, 1, 32, "conv1")
pool1 = tf.nn.max_pool(conv1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding="SAME")

conv2 = conv_layer(pool1, 32, 64, "conv2")
pool2 = tf.nn.max_pool(conv2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding="SAME")

flattened = tf.reshape(pool2, [-1, 7 * 7 * 64])

fc1 = fc_layer(flattened, 7 * 7 * 64, 1024, "fc1")

fc2 = fc_layer(fc1, 1024, 10, "fc2")

Loss & Training


In [7]:
# Compute cross entropy as loss function
with tf.name_scope("loss"):
    cross_entropy = tf.reduce_mean(
        tf.nn.softmax_cross_entropy_with_logits(logits=fc2, labels=y)
    )
    tf.summary.scalar("cross_entropy", cross_entropy)

# use the AdamOptimizer to train the network
with tf.name_scope("train"):
    train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)

# compute the accuracy
with tf.name_scope("accuracy"):
    prediction = tf.equal(tf.argmax(fc2, 1), tf.argmax(y, 1))
    accuracy = tf.reduce_mean(tf.cast(prediction, tf.float32))
    tf.summary.scalar("accuracy", accuracy)

In [8]:
sess.run(tf.global_variables_initializer())

In [9]:
merged_summary = tf.summary.merge_all()

embedding = tf.Variable(tf.zeros([1024, 7 * 7 * 64]), name="embedding")
assignment = embedding.assign(flattened)

saver = tf.train.Saver()

writer = tf.summary.FileWriter(LOG_PATH)
writer.add_graph(sess.graph)

config = tf.contrib.tensorboard.plugins.projector.ProjectorConfig()
embedding_config = config.embeddings.add()
embedding_config.tensor_name = embedding.name
embedding_config.sprite.image_path = SPRITE_PATH
embedding_config.metadata_path = LABELS_PATH
embedding_config.sprite.single_image_dim.extend([28, 28])

tf.contrib.tensorboard.plugins.projector.visualize_embeddings(writer, config)

In [10]:
for i in range(300):
    batch = mnist.train.next_batch(100)

    if i % 10 == 0:
        [train_accuracy, summary] = sess.run([accuracy, merged_summary], feed_dict={
            x: batch[0],
            y: batch[1]
        })
        writer.add_summary(summary, i)
        print("step %d train accuracy %g" % (i, train_accuracy))

    if i % 100 == 0:
        sess.run(
            assignment,
            feed_dict={
                x: mnist.test.images[:1024],
                y: mnist.test.labels[:1024]
            }
        )

        saver.save(sess, os.path.join(LOG_PATH, "model.ckpt"), i)

    sess.run(train_step, feed_dict={
        x: batch[0],
        y: batch[1]
    })


step 0 train accuracy 0.1
step 10 train accuracy 0.23
step 20 train accuracy 0.3
step 30 train accuracy 0.39
step 40 train accuracy 0.67
step 50 train accuracy 0.71
step 60 train accuracy 0.81
step 70 train accuracy 0.75
step 80 train accuracy 0.77
step 90 train accuracy 0.84
step 100 train accuracy 0.92
step 110 train accuracy 0.87
step 120 train accuracy 0.85
step 130 train accuracy 0.87
step 140 train accuracy 0.94
step 150 train accuracy 0.89
step 160 train accuracy 0.91
step 170 train accuracy 0.85
step 180 train accuracy 0.88
step 190 train accuracy 0.93
step 200 train accuracy 0.84
step 210 train accuracy 0.93
step 220 train accuracy 0.98
step 230 train accuracy 0.89
step 240 train accuracy 0.94
step 250 train accuracy 0.9
step 260 train accuracy 0.93
step 270 train accuracy 0.94
step 280 train accuracy 0.91
step 290 train accuracy 0.94