In [1]:

    

import tensorflow.contrib.learn.python.learn as learn
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
from sklearn import datasets, metrics
from sklearn.decomposition import PCA

import tensorflow.contrib.slim as slim
import numpy as np

import collections
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import sklearn
import seaborn as sns
import six.moves.cPickle as pickle
import sys
from pandas import *
from sklearn.preprocessing import OneHotEncoder

from sklearn.linear_model import LogisticRegression
from sklearn import svm
%matplotlib inline


    

In [2]:

    

def accuracy_fn(predictions, labels):
    return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1))
          / predictions.shape[0])


    

In [6]:

    

mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)


    

    




Extracting MNIST_data/train-images-idx3-ubyte.gz
Extracting MNIST_data/train-labels-idx1-ubyte.gz
Extracting MNIST_data/t10k-images-idx3-ubyte.gz
Extracting MNIST_data/t10k-labels-idx1-ubyte.gz

In [7]:

    

train_set= mnist.train.images.reshape(-1,28,28, 1)
test_set = mnist.test.images.reshape(-1,28,28, 1)


    

In [5]:

    

def variable_summaries(name, var):
    """Attach a lot of summaries to a Tensor."""
    with tf.name_scope('summaries'):
        mean = tf.reduce_mean(var)
        tf.scalar_summary('mean/' + name, mean)
        with tf.name_scope('stddev'):
            stddev = tf.sqrt(tf.reduce_sum(tf.square(var - mean)))
        tf.scalar_summary('sttdev/' + name, stddev)
        tf.scalar_summary('max/' + name, tf.reduce_max(var))
        tf.scalar_summary('min/' + name, tf.reduce_min(var))
        tf.histogram_summary(name, var)


    

In [7]:

    

def train_model(train_dataset, train_labels, test_dataset, test_labels, train_tensor,
                accuracy, tf_batch_data, tf_batch_labels, log_dir='./logs',
                num_steps=20000, batch_size=10, test_steps=1000, log_steps=100, predictor=None, last_test='np'):
    with tf.Session() as session:
        summaries = tf.merge_all_summaries()

        if tf.gfile.Exists(log_dir):
            tf.gfile.DeleteRecursively(log_dir)
            
        train_writer = tf.train.SummaryWriter(log_dir + '/train', session.graph)
        test_writer = tf.train.SummaryWriter(log_dir + '/test')

        session.run(tf.initialize_all_variables())
        
        shuffle_train = np.random.permutation(train_dataset.shape[0])
        train_dataset = train_dataset[shuffle_train]
        train_labels = train_labels[shuffle_train]

        for step in range(num_steps):
            # Pick an offset within the training data, which has been randomized.
            # Note: we could use better randomization across epochs.
            offset = ((step * batch_size) % (train_labels.shape[0] - batch_size))

            # Generate a minibatch.
            batch_data = train_dataset[offset:(offset + batch_size)]
            batch_labels = train_labels[offset:(offset + batch_size)]

            # Prepare a dictionary telling the session where to feed the minibatch.
            # The key of the dictionary is the placeholder node of the graph to be fed,
            # and the value is the numpy array to feed to it.
            feed_dict = {
                tf_batch_data : batch_data, 
                tf_batch_labels : batch_labels,
                keep_prob: 0.5
            }
    
    
            if step % test_steps == 0:
                run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
                run_metadata = tf.RunMetadata()
                _, acc, summary = session.run([train_tensor, accuracy, summaries], 
                                             feed_dict=feed_dict,
                                             run_metadata=run_metadata,
                                             options=run_options)
                print("Train accuracy at step %s: %.1f%%" % (step, acc))
                train_writer.add_run_metadata(run_metadata, "step%d" % step)
                train_writer.add_summary(summary, step)
                
            elif step % log_steps == 0:
                run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
                run_metadata = tf.RunMetadata()
                _, summary = session.run([train_tensor, summaries], 
                                         feed_dict=feed_dict, 
                                         run_metadata=run_metadata,
                                         options=run_options)
                train_writer.add_run_metadata(run_metadata, "step%d" % step)
                train_writer.add_summary(summary, step)
            else:
                session.run(train_tensor, feed_dict=feed_dict, options=run_options)


        feed_dict = {
            tf_batch_data : test_dataset, 
            tf_batch_labels : test_labels,
            keep_prob: 1
        }
        
        if last_test == 'splitted':
            predictions = np.empty([0,10])
            for batch in np.array_split(test_dataset, test_dataset.shape[0] / 16):
                tmp = session.run(predictor,
                                          feed_dict={
                                                tf_batch_data: batch,
#                                                 batch_labels: np.array([]),
                                                keep_prob: 1.0
                })
                predictions = np.vstack((predictions, tmp))
            acc = accuracy_fn(predictions, test_labels)
        elif accuracy is not None:   
            acc = session.run(accuracy, feed_dict=feed_dict)
        print("Test accuracy: %.3f%%" % acc)


    

In [ ]:

    

def convnet(inputs, keep_prob):
    with slim.arg_scope([slim.conv2d, slim.fully_connected],
                      activation_fn=tf.nn.relu,
                      weights_initializer=tf.truncated_normal_initializer(0.0, 0.01),
                      weights_regularizer=slim.l2_regularizer(0.0005)):
        
        net = slim.conv2d(inputs, 32, [5, 5], scope='conv1')
        net = slim.max_pool2d(net, [2, 2], scope='pool1')
        
        net = slim.conv2d(net, 64, [5, 5], scope='conv2')
        net = slim.max_pool2d(net, [2, 2], scope='pool2')
        
#         net = slim.repeat(net, 2, slim.conv2d, 128, [3, 3], scope='conv2')
#         net = slim.max_pool2d(net, [2, 2], scope='pool2')
        
#         net = slim.repeat(net, 3, slim.conv2d, 256, [3, 3], scope='conv3')
#         net = slim.max_pool2d(net, [2, 2], scope='pool3')
        
#         net = slim.repeat(net, 3, slim.conv2d, 512, [3, 3], scope='conv4')
#         net = slim.max_pool2d(net, [2, 2], scope='pool4')
        
#         net = slim.repeat(net, 3, slim.conv2d, 512, [3, 3], scope='conv5')
#         net = slim.max_pool2d(net, [2, 2], scope='pool5')
        
        net = slim.flatten(net)
        net = slim.fully_connected(net, 1024, scope='fc6')
        net = slim.dropout(net, keep_prob, scope='dropout6')
        
#         net = slim.fully_connected(net, 4096, scope='fc7')
#         net = slim.dropout(net, 0.5, scope='dropout7')
        
        net = slim.fully_connected(net, 10, activation_fn=None, scope='fc8')
        predictor = slim.softmax(net)
    return net, predictor

image_size = 28
num_labels = 10
num_channels = 1
g = tf.Graph()
with g.as_default():
    batch_data = tf.placeholder(tf.float32, shape=(None, image_size, image_size, num_channels))
    batch_labels = tf.placeholder(tf.float32, shape=(None, num_labels))
    keep_prob = tf.placeholder(tf.float32)
    last_layer, predictor = convnet(batch_data, keep_prob)
    
    print(last_layer)
    print(batch_labels)
    slim.losses.softmax_cross_entropy(last_layer, batch_labels)
    total_loss = slim.losses.get_total_loss()
    tf.scalar_summary('losses/total_loss', total_loss)
    tf.scalar_summary('accuracy', accuracy)
    
    optimizer = tf.train.AdamOptimizer()
    
    train_tensor = slim.learning.create_train_op(total_loss, optimizer)
    correct_prediction = tf.equal(tf.argmax(predictor,1), tf.argmax(batch_labels,1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
    
    

    train_model(train_dataset=train_set, 
                train_labels=mnist.train.labels, 
                test_dataset=test_set,
                test_labels=mnist.test.labels, 
                train_tensor=train_tensor,
                accuracy=accuracy,
                last_test='splitted',
                predictor=predictor,
                tf_batch_data=batch_data,
                log_dir='mnist_conv_max_conv_max_flatten_fc_d_sm_autoADAM',
                tf_batch_labels=batch_labels,
                batch_size=16, num_steps=20000, test_steps=100)


    

    




Tensor("fc8/BiasAdd:0", shape=(?, 10), dtype=float32)
Tensor("Placeholder_1:0", shape=(?, 10), dtype=float32)
Train accuracy at step 0: 0.0%
Train accuracy at step 100: 0.9%
Train accuracy at step 200: 1.0%
Train accuracy at step 300: 0.8%

In [3]:

    

def unpickle(file):
    fo = open(file, 'rb')
    dict = pickle.load(fo, encoding='latin-1')
    fo.close()
    return dict

def from_flat_to_3d(image):
#     print(image.shape)
    return np.dstack((image[0:1024].reshape(32,32),
                       image[1024:2048].reshape(32,32),
                       image[2048:3072].reshape(32,32)))

cifar_test = unpickle('cifar-10-batches-py/test_batch')
cifar_test['data_3d'] = np.array([from_flat_to_3d(image) for image in cifar_test['data']])

cifar = unpickle('cifar-10-batches-py/data_batch_1')
for i in range(2, 6):
    tmp = unpickle('cifar-10-batches-py/data_batch_' + str(i))
    cifar['data'] = np.vstack((cifar['data'], tmp['data']))
    cifar['labels'] = np.concatenate((cifar['labels'], tmp['labels']))

cifar['data_3d'] = np.array([from_flat_to_3d(image) for image in cifar['data']])

# cifar['data_bw'] = (cifar['data'][:,0:1024] + cifar['data'][:,1024:2048] + cifar['data'][:, 2048:3072]) / 3 
# cifar_test['data_bw'] = (cifar_test['data'][:,0:1024] + cifar_test['data'][:,1024:2048] + cifar_test['data'][:, 2048:3072]) / 3 

enc = OneHotEncoder()
cifar['labels_oh'] = enc.fit_transform(cifar['labels'].reshape(-1, 1))
cifar['labels_oh'] = cifar['labels_oh'].toarray()

cifar_test['labels'] = np.array(cifar_test['labels'])
cifar_test['labels_oh'] = enc.fit_transform(cifar_test['labels'].reshape(-1, 1))
cifar_test['labels_oh'] = cifar_test['labels_oh'].toarray()

# pca = PCA(whiten=True)
# cifar['data_bw_whitened'] = pca.fit_transform(cifar['data_bw'])
# cifar_test['data_bw_whitened'] = pca.fit_transform(cifar_test['data_bw'])


    

In [34]:

    

cifar['data_3d'].shape


    

    
Out[34]:





(50000, 32, 32, 3)

In [9]:

    

def convnet(inputs, keep_prob, is_training):
    with slim.arg_scope([slim.conv2d, slim.fully_connected],
                      activation_fn=tf.nn.relu,
                      weights_initializer=tf.truncated_normal_initializer(0.0, 0.01),
                      weights_regularizer=slim.l2_regularizer(0.0005)):
        
        net = slim.conv2d(inputs, 32, [5, 5], scope='conv1')
        variable_summaries('conv1', net)
        net = slim.max_pool2d(net, [2, 2], scope='pool1')
        net = tf.nn.lrn(net, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75, name='norm1')
        
        net = slim.conv2d(net, 64, [5, 5], scope='conv2')
        variable_summaries('conv2', net)
        net = tf.nn.lrn(net, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75, name='norm2')
        net = slim.max_pool2d(net, [2, 2], scope='pool2')      
#         net = slim.repeat(net, 2, slim.conv2d, 128, [3, 3], scope='conv2')
#         net = slim.max_pool2d(net, [2, 2], scope='pool2')
        
#         net = slim.repeat(net, 3, slim.conv2d, 256, [3, 3], scope='conv3')
#         net = slim.max_pool2d(net, [2, 2], scope='pool3')
        
#         net = slim.repeat(net, 3, slim.conv2d, 512, [3, 3], scope='conv4')
#         net = slim.max_pool2d(net, [2, 2], scope='pool4')
        
#         net = slim.repeat(net, 3, slim.conv2d, 512, [3, 3], scope='conv5')
#          net = slim.max_pool2d(net, [2, 2], scope='pool5')
        
        net = slim.flatten(net)
        net = slim.fully_connected(net, 1024, scope='fc6')
        variable_summaries('fc1', net)
        net = slim.dropout(net, keep_prob, scope='dropout6')
        
        net = slim.fully_connected(net, 1024, scope='fc7')
        variable_summaries('fc2', net)
        net = slim.dropout(net, keep_prob, scope='dropout7')
        
        net = slim.fully_connected(net, 10, activation_fn=None, scope='fc8')
        predictor = slim.softmax(net)
    return net, predictor

image_size = 32
num_labels = 10
num_channels = 3
g = tf.Graph()
with g.as_default():
    batch_data = tf.placeholder(tf.float32, shape=(None, image_size, image_size, num_channels), name='batch_data')
    batch_labels = tf.placeholder(tf.float32, shape=(None, num_labels), name='batch_labels')
    keep_prob = tf.placeholder(tf.float32)
    is_training = tf.placeholder(tf.bool)
    last_layer, predictor = convnet(batch_data, keep_prob, is_training)
    
    print(last_layer)
    print(batch_labels)
    slim.losses.softmax_cross_entropy(last_layer, batch_labels)
    total_loss = slim.losses.get_total_loss()
    tf.scalar_summary('losses/total_loss', total_loss)
    
    optimizer = tf.train.AdamOptimizer(1e-4)
    
    train_tensor = slim.learning.create_train_op(total_loss, optimizer)
    correct_prediction = tf.equal(tf.argmax(predictor,1), tf.argmax(batch_labels,1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
    tf.scalar_summary('accuracy', accuracy)

    train_model(train_dataset=cifar['data_3d'], 
                train_labels=cifar['labels_oh'], 
                test_dataset=cifar_test['data_3d'],
                test_labels=cifar_test['labels_oh'], 
                train_tensor=train_tensor,
                accuracy=accuracy,
                last_test='splitted',
                predictor=predictor,
                tf_batch_data=batch_data, 
                tf_batch_labels=batch_labels,
                log_dir='cifar_conv_max_lrn_conv_lrn_max_flatten_fc_d_fc_d_sm_autoADAM',
                batch_size=32, num_steps=30000, test_steps=1000)


    

    




Tensor("fc8/BiasAdd:0", shape=(?, 10), dtype=float32)
Tensor("batch_labels:0", shape=(?, 10), dtype=float32)
Train accuracy at step 0: 0.1%
Train accuracy at step 1000: 0.4%
Train accuracy at step 2000: 0.5%
Train accuracy at step 3000: 0.6%
Train accuracy at step 4000: 0.7%
Train accuracy at step 5000: 0.7%
Train accuracy at step 6000: 0.7%
Train accuracy at step 7000: 0.7%
Train accuracy at step 8000: 0.7%
Train accuracy at step 9000: 0.8%
Train accuracy at step 10000: 0.8%
Train accuracy at step 11000: 0.8%
Train accuracy at step 12000: 0.8%
Train accuracy at step 13000: 0.9%
Train accuracy at step 14000: 0.8%
Train accuracy at step 15000: 0.8%
Train accuracy at step 16000: 0.8%
Train accuracy at step 17000: 0.9%
Train accuracy at step 18000: 0.8%
Train accuracy at step 19000: 1.0%
Train accuracy at step 20000: 0.9%
Train accuracy at step 21000: 0.9%
Train accuracy at step 22000: 0.9%
Train accuracy at step 23000: 1.0%
Train accuracy at step 24000: 1.0%
Train accuracy at step 25000: 1.0%
Train accuracy at step 26000: 1.0%
Train accuracy at step 27000: 0.9%
Train accuracy at step 28000: 0.9%
Train accuracy at step 29000: 1.0%
Test accuracy: 74.040%

In [10]:

    

print(1 - 66.77)
print(1 - 66.48)
print(1 - 73.09)


    

    




-65.77
-65.48
-72.09

In [22]:

    

predictions = np.empty([0,10])
for batch in np.array_split(cifar_test['data_3d'], cifar_test['data_3d'].shape[0] / 16):
    tmp = session.run(predictor,
                              feed_dict={
                                    tf_batch_data: batch,
#                                   batch_labels: np.array([]),
                                    keep_prob: 1.0
    })
    predictions = np.vstack((predictions, tmp))
acc = accuracy_fn(predictions, cifar_test['labels'])


    

    




---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-22-3d27bbcb6453> in <module>()
      1 predictions = np.empty([0,10])
      2 for batch in np.array_split(cifar_test['data_3d'], cifar_test['data_3d'].shape[0] / 16):
----> 3     tmp = session.run(predictor,
      4                               feed_dict={
      5                                     tf_batch_data: batch,

NameError: name 'session' is not defined

In [43]:

    

for batch in np.array_split(cifar_test['data_3d'], cifar_test['data_3d'].shape[0] / 16):
    predictions = session.run(predictions,
                              feed_dict={
                                    batch_data: batch,
                                    batch_labels: np.array([]),
                                    keep_prob: 1.0
            })


    

    




ERROR:root:Internal Python error in the inspect module.
Below is the traceback from this internal error.







    




Traceback (most recent call last):
  File "/home/kkari/DevTools/anaconda3/lib/python3.5/site-packages/IPython/core/interactiveshell.py", line 2881, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-43-fe8ef3147aaa>", line 2, in <module>
    predictions = session.run(predictions,
NameError: name 'session' is not defined

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/kkari/DevTools/anaconda3/lib/python3.5/site-packages/IPython/core/interactiveshell.py", line 1821, in showtraceback
    stb = value._render_traceback_()
AttributeError: 'NameError' object has no attribute '_render_traceback_'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/kkari/DevTools/anaconda3/lib/python3.5/site-packages/IPython/core/ultratb.py", line 1132, in get_records
    return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset)
  File "/home/kkari/DevTools/anaconda3/lib/python3.5/site-packages/IPython/core/ultratb.py", line 313, in wrapped
    return f(*args, **kwargs)
  File "/home/kkari/DevTools/anaconda3/lib/python3.5/site-packages/IPython/core/ultratb.py", line 358, in _fixed_getinnerframes
    records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))
  File "/home/kkari/DevTools/anaconda3/lib/python3.5/inspect.py", line 1453, in getinnerframes
    frameinfo = (tb.tb_frame,) + getframeinfo(tb, context)
  File "/home/kkari/DevTools/anaconda3/lib/python3.5/inspect.py", line 1410, in getframeinfo
    filename = getsourcefile(frame) or getfile(frame)
  File "/home/kkari/DevTools/anaconda3/lib/python3.5/inspect.py", line 672, in getsourcefile
    if getattr(getmodule(object, filename), '__loader__', None) is not None:
  File "/home/kkari/DevTools/anaconda3/lib/python3.5/inspect.py", line 709, in getmodule
    if ismodule(module) and hasattr(module, '__file__'):
KeyboardInterrupt






    




---------------------------------------------------------------------------

In [ ]:

    

a = np.array