In [1]:

    

# GPUs or CPU
import tensorflow as tf

# Check TensorFlow Version
print('TensorFlow Version: {}'.format(tf.__version__))

# Check for a GPU
print('Default GPU Device: {}'.format(tf.test.gpu_device_name()))


    

    




TensorFlow Version: 1.4.1
Default GPU Device: /device:GPU:0

In [18]:

    

import numpy as np
import sys
from tensorflow.examples.tutorials.mnist import input_data

def convolution(X, W, b, padding, stride):
    n, h, w, c = map(lambda d: d.value, X.get_shape())
    filter_h, filter_w, filter_c, filter_n = [d.value for d in W.get_shape()]
    
    out_h = (h + 2*padding - filter_h)//stride + 1
    out_w = (w + 2*padding - filter_w)//stride + 1

    X_flat = flatten(X, filter_h, filter_w, filter_c, out_h, out_w, stride, padding)
    W_flat = tf.reshape(W, [filter_h*filter_w*filter_c, filter_n])
    
    z = tf.matmul(X_flat, W_flat) + b     # b: 1 X filter_n
    
    #     TypeError: Failed to convert object of type <class 'list'> to Tensor. 
    #         Contents: [28, 28, None, 10]. Consider casting elements to a supported type.
    print('z.shape, z.dtype', z.shape, z.dtype)
    print('out_h, out_w, n, filter_n', out_h, out_w, n, filter_n)
    
    return tf.transpose(tf.reshape(z, [out_h, out_w, n, filter_n]), [2, 0, 1, 3])

# Question: Is this the same img2col in NumPy and Cython implementation.
# To compute convolution easily, we do a simple trick called flattening. 
# After flattening, input data will be transformed into a 2D matrix, 
# which allows for matrix multiplication with a filter (which is also flattened into 2D).
def flatten(X, window_h, window_w, window_c, out_h, out_w, stride=1, padding=0):
    
    X_padded = tf.pad(X, [[0,0], [padding, padding], [padding, padding], [0,0]])

    windows = []
    for y in range(out_h):
        for x in range(out_w):
            window = tf.slice(X_padded, [0, y*stride, x*stride, 0], [-1, window_h, window_w, -1])
            windows.append(window)
    stacked = tf.stack(windows) # shape : [out_h, out_w, n, filter_h, filter_w, c]

    return tf.reshape(stacked, [-1, window_c*window_w*window_h])

# Max pooling is not neccessary at all.
def max_pool(X, pool_h, pool_w, padding, stride):
    n, h, w, c = [d.value for d in X.get_shape()]
    
    out_h = (h + 2*padding - pool_h)//stride + 1
    out_w = (w + 2*padding - pool_w)//stride + 1

    X_flat = flatten(X, pool_h, pool_w, c, out_h, out_w, stride, padding)

    pool = tf.reduce_max(tf.reshape(X_flat, [out_h, out_w, n, pool_h*pool_w, c]), axis=3)
    return tf.transpose(pool, [2, 0, 1, 3])

# The simplest possible non-linearity
# ReLU or Leaky ReLU
# Gated Linear unit is also another one of these but with more computational complexity.
# Sigmoid is a gate and resemble the soma non-linearity better.
def relu(X):
    return tf.maximum(X, tf.zeros_like(X))

def softmax(X):
    X_centered = X - tf.reduce_max(X) # to avoid overflow
    X_exp = tf.exp(X_centered)
    exp_sum = tf.reduce_sum(X_exp, axis=1)
    return tf.transpose(tf.transpose(X_exp) / exp_sum)

# def cross_entropy_error(y, t):
#     return -tf.reduce_mean(tf.log(tf.reduce_sum(y * t, axis=1)))

# def accuracy(network, t):
#     t_predict = tf.argmax(network, axis=1)
#     t_actual = tf.argmax(t, axis=1)
#     return tf.reduce_mean(tf.cast(tf.equal(t_predict, t_actual), tf.float32))


    

In [21]:

    

def xavier_init(size):
    in_dim = size[0]
    xavier_stddev = 1. / tf.sqrt(in_dim / 2.)
    return tf.random_normal(shape=size, stddev=xavier_stddev)

def feedforward_net(D, H, C):
    X = tf.placeholder(tf.float32, shape=[None, D])
    y = tf.placeholder(tf.float32, shape=[None, C])

    Wxh = tf.Variable(xavier_init([D, H]))
    bxh = tf.Variable(tf.zeros(shape=[H]))

    Whh = tf.Variable(xavier_init([H, H]))
    bhh = tf.Variable(tf.zeros(shape=[H]))

    Why = tf.Variable(xavier_init([H, C]))
    bhy = tf.Variable(tf.zeros(shape=[C]))

    h1 = relu(tf.matmul(X, Wxh) + bxh)
    h2 = relu(tf.matmul(h1, Whh) + bhh)
    prob = softmax(tf.matmul(h2, Why) + bhy)

    loss = -tf.reduce_mean(y * tf.log(prob))

    return X, y, prob, loss

def convnet(D, H, C):
    X = tf.placeholder(tf.float32, shape=[None, *D])
    y = tf.placeholder(tf.float32, shape=[None, C])
    print('X.shape, y.shape, X.dtype, y.dtype', X.shape, y.shape, X.dtype, y.dtype)

    Wconv1 = tf.Variable(xavier_init([3, 3, 1, 10]))
    bconv1 = tf.Variable(tf.zeros(shape=[10]))
    print('Wconv1.shape, bconv1.shape, Wconv1.dtype, bconv1.dtype', 
          Wconv1.shape, bconv1.shape, Wconv1.dtype, bconv1.dtype)
#     n, h, w, c = map(lambda d: d.value, X.get_shape())
#     filter_h, filter_w, filter_c, filter_n = [d.value for d in W.get_shape()]

    Wfc1 = tf.Variable(xavier_init([14 * 14 * 10, H]))
    bfc1 = tf.Variable(tf.zeros(shape=[H]))

    Wfc2 = tf.Variable(xavier_init([H, C]))
    bfc2 = tf.Variable(tf.zeros(shape=[C]))

#     hconv1 = relu(tf.nn.conv2d(X, Wconv1, [1, 1, 1, 1], padding='SAME') + bconv1)
    hconv1 = relu(convolution(X=X, W=Wconv1, b=bconv1, stride=1, padding=1))
#     hpool1 = tf.nn.max_pool(hconv1, [1, 2, 2, 1], [1, 2, 2, 1], padding='SAME')
    hpool1 = max_pool(X=hconv1, pool_h=2, pool_w=2, padding=1, stride=2)
    
    hpool1 = tf.reshape(hpool1, shape=[-1, 14 * 14 * 10])
    h = relu(tf.matmul(hpool1, Wfc1) + bfc1)
    prob = softmax(tf.matmul(h, Wfc2) + bfc2)

    loss = -tf.reduce_mean(y * tf.log(prob))

    return X, y, prob, loss

def accuracy(y_true, y_pred):
    return np.mean(np.argmax(y_true, axis=1) == np.argmax(y_pred, axis=1))


    

In [22]:

    

# if __name__ == '__main__':
alpha = 1e-3

mnist = input_data.read_data_sets('/home/arasdar/datasets/MNIST_data', one_hot=True)

X_train, y_train = mnist.train.images, mnist.train.labels
X_val, y_val = mnist.validation.images, mnist.validation.labels
X_test, y_test = mnist.test.images, mnist.test.labels

D, C = X_train.shape[1], y_train.shape[1]
H = 64
M = 128

#     if net_type == 'cnn':
D = [28, 28, 1]
X, y, forward_step, loss = convnet(D, H, C)
X_val = X_val.reshape([-1, 28, 28, 1])
#     n, h, w, c = map(lambda d: d.value, X.get_shape())
#     filter_h, filter_w, filter_c, filter_n = [d.value for d in W.get_shape()]

#     #     elif net_type == 'ff':
#     X, y, forward_step, loss = feedforward_net(D, H, C)

solver = tf.train.RMSPropOptimizer(alpha)
train_step = solver.minimize(loss)

sess = tf.Session()
#     sess.run(tf.initialize_all_variables())

#     WARNING:tensorflow:From 
#     Instructions for updating:
#     Use `tf.global_variables_initializer` instead.
sess.run(tf.global_variables_initializer())

for i in range(5000):
    X_mb, y_mb = mnist.train.next_batch(M)

    #         if net_type == 'cnn':
    X_mb = X_mb.reshape([-1, 28, 28, 1])

    _, loss_val = sess.run([train_step, loss], feed_dict={X: X_mb, y: y_mb})

    if i % 100 == 0:
        y_pred = sess.run(forward_step, feed_dict={X: X_val})
        acc = accuracy(y_val, y_pred)

        print('Iter: {} Loss: {:.4f} Validation: {}'.format(i, loss_val, acc))


    

    




Extracting /home/arasdar/datasets/MNIST_data/train-images-idx3-ubyte.gz
Extracting /home/arasdar/datasets/MNIST_data/train-labels-idx1-ubyte.gz
Extracting /home/arasdar/datasets/MNIST_data/t10k-images-idx3-ubyte.gz
Extracting /home/arasdar/datasets/MNIST_data/t10k-labels-idx1-ubyte.gz
X.shape, y.shape, X.dtype, y.dtype (?, 28, 28, 1) (?, 10) <dtype: 'float32'> <dtype: 'float32'>
Wconv1.shape, bconv1.shape, Wconv1.dtype, bconv1.dtype (3, 3, 1, 10) (10,) <dtype: 'float32_ref'> <dtype: 'float32_ref'>
z.shape, z.dtype (?, 10) <dtype: 'float32'>
out_h, out_w, n, filter_n 28 28 None 10






    




---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
~/anaconda3/envs/arasdar-DL-env/lib/python3.6/site-packages/tensorflow/python/framework/tensor_util.py in make_tensor_proto(values, dtype, shape, verify_shape)
    467     try:
--> 468       str_values = [compat.as_bytes(x) for x in proto_values]
    469     except TypeError:

~/anaconda3/envs/arasdar-DL-env/lib/python3.6/site-packages/tensorflow/python/framework/tensor_util.py in <listcomp>(.0)
    467     try:
--> 468       str_values = [compat.as_bytes(x) for x in proto_values]
    469     except TypeError:

~/anaconda3/envs/arasdar-DL-env/lib/python3.6/site-packages/tensorflow/python/util/compat.py in as_bytes(bytes_or_text, encoding)
     64     raise TypeError('Expected binary or unicode string, got %r' %
---> 65                     (bytes_or_text,))
     66 

TypeError: Expected binary or unicode string, got 28

During handling of the above exception, another exception occurred:

TypeError                                 Traceback (most recent call last)
<ipython-input-22-525c5a390c51> in <module>()
     14 #     if net_type == 'cnn':
     15 D = [28, 28, 1]
---> 16 X, y, forward_step, loss = convnet(D, H, C)
     17 X_val = X_val.reshape([-1, 28, 28, 1])
     18 #     n, h, w, c = map(lambda d: d.value, X.get_shape())

<ipython-input-21-f8e44dce2c66> in convnet(D, H, C)
     44 
     45 #     hconv1 = relu(tf.nn.conv2d(X, Wconv1, [1, 1, 1, 1], padding='SAME') + bconv1)
---> 46     hconv1 = relu(convolution(X=X, W=Wconv1, b=bconv1, stride=1, padding=1))
     47 #     hpool1 = tf.nn.max_pool(hconv1, [1, 2, 2, 1], [1, 2, 2, 1], padding='SAME')
     48     hpool1 = max_pool(X=hconv1, pool_h=2, pool_w=2, padding=1, stride=2)

<ipython-input-18-41b03376f88e> in convolution(X, W, b, padding, stride)
     20     print('out_h, out_w, n, filter_n', out_h, out_w, n, filter_n)
     21 
---> 22     return tf.transpose(tf.reshape(z, [out_h, out_w, n, filter_n]), [2, 0, 1, 3])
     23 
     24 # Question: Is this the same img2col in NumPy and Cython implementation.

~/anaconda3/envs/arasdar-DL-env/lib/python3.6/site-packages/tensorflow/python/ops/gen_array_ops.py in reshape(tensor, shape, name)
   3936   if _ctx.in_graph_mode():
   3937     _, _, _op = _op_def_lib._apply_op_helper(
-> 3938         "Reshape", tensor=tensor, shape=shape, name=name)
   3939     _result = _op.outputs[:]
   3940     _inputs_flat = _op.inputs

~/anaconda3/envs/arasdar-DL-env/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py in _apply_op_helper(self, op_type_name, name, **keywords)
    511           except TypeError as err:
    512             if dtype is None:
--> 513               raise err
    514             else:
    515               raise TypeError(

~/anaconda3/envs/arasdar-DL-env/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py in _apply_op_helper(self, op_type_name, name, **keywords)
    508                 dtype=dtype,
    509                 as_ref=input_arg.is_ref,
--> 510                 preferred_dtype=default_dtype)
    511           except TypeError as err:
    512             if dtype is None:

~/anaconda3/envs/arasdar-DL-env/lib/python3.6/site-packages/tensorflow/python/framework/ops.py in internal_convert_to_tensor(value, dtype, name, as_ref, preferred_dtype, ctx)
    924 
    925     if ret is None:
--> 926       ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)
    927 
    928     if ret is NotImplemented:

~/anaconda3/envs/arasdar-DL-env/lib/python3.6/site-packages/tensorflow/python/framework/constant_op.py in _constant_tensor_conversion_function(v, dtype, name, as_ref)
    227                                          as_ref=False):
    228   _ = as_ref
--> 229   return constant(v, dtype=dtype, name=name)
    230 
    231 

~/anaconda3/envs/arasdar-DL-env/lib/python3.6/site-packages/tensorflow/python/framework/constant_op.py in constant(value, dtype, shape, name, verify_shape)
    206   tensor_value.tensor.CopyFrom(
    207       tensor_util.make_tensor_proto(
--> 208           value, dtype=dtype, shape=shape, verify_shape=verify_shape))
    209   dtype_value = attr_value_pb2.AttrValue(type=tensor_value.tensor.dtype)
    210   const_tensor = g.create_op(

~/anaconda3/envs/arasdar-DL-env/lib/python3.6/site-packages/tensorflow/python/framework/tensor_util.py in make_tensor_proto(values, dtype, shape, verify_shape)
    470       raise TypeError("Failed to convert object of type %s to Tensor. "
    471                       "Contents: %s. Consider casting elements to a "
--> 472                       "supported type." % (type(values), values))
    473     tensor_proto.string_val.extend(str_values)
    474     return tensor_proto

TypeError: Failed to convert object of type <class 'list'> to Tensor. Contents: [28, 28, None, 10]. Consider casting elements to a supported type.

In [ ]: