In [1]:

    

import warnings
warnings.filterwarnings("ignore")


    

In [2]:

    

%load_ext watermark
%watermark -u -n -t -z -v -p numpy,scipy,neupy,scikit-learn,matplotlib


    

    




/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/watermark/watermark.py:155: DeprecationWarning: Importing scikit-learn as `scikit-learn` has been depracated and will not be supported anymore in v1.7.0. Please use the package name `sklearn` instead.
  DeprecationWarning)






    




last updated: Thu Jan 24 2019 11:05:42 CET

CPython 3.5.6
IPython 7.1.1

numpy 1.14.5
scipy 1.2.0
neupy 0.8.0
sklearn 0.20.2
matplotlib 3.0.1

In [3]:

    

%matplotlib inline

import numpy as np
import matplotlib.pyplot as plt

plt.style.use('ggplot')


    

In [4]:

    

from sklearn import datasets, preprocessing, model_selection

dataset = datasets.load_digits()
target_scaler = preprocessing.OneHotEncoder()

transformed_input = (dataset.data / 16) - 0.5
target = dataset.target.reshape(-1, 1)
transformed_target = target_scaler.fit_transform(target)

x_train, x_test, y_train, y_test = model_selection.train_test_split(
    transformed_input,
    transformed_target.todense(),
    train_size=0.5,
    random_state=0,
)


    

In [5]:

    

fig, axes = plt.subplots(4, 4, figsize=(4, 5))

for i in range(16):
    image_coords = (i // 4, i % 4)
    image_axes = axes[image_coords]

    image_matrix = x_train[i, :].reshape((8, 8))
    image_name = y_train[i].argmax()

    image_axes.set_title(image_name)
    image_axes.imshow(
        image_matrix,
        cmap=plt.cm.binary,
        interpolation='nearest'
    )

    image_axes.axis('off')


    

In [6]:

    

import time
from sklearn import metrics
from neupy import algorithms, layers, utils

net = algorithms.RPROP(
    [
        layers.Input(64),
        layers.Tanh(70),
        layers.Softmax(10),
    ],

    step=0.05,
    verbose=True,
    show_epoch=5,
    loss='categorical_crossentropy',
    shuffle_data=True,
)


t0 = time.time()
net.train(x_train, y_train, x_test, y_test, epochs=25)
print("Total train time: {:.2f} sec".format(time.time() - t0))

y_predicted = net.predict(x_test).argmax(axis=1)
accuracy = metrics.accuracy_score(y_predicted, y_test.argmax(axis=1))

print("Validation data sample accuracy: {:.2%}".format(accuracy))


    

    




/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/tensorflow/python/framework/ops.py:923: DeprecationWarning: builtin type EagerTensor has no __module__ attribute
  EagerTensor = c_api.TFE_Py_InitEagerTensor(_EagerTensorBase)
/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/tensorflow/python/util/tf_inspect.py:75: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead
  return _inspect.getargspec(target)
/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/tensorflow/python/util/tf_inspect.py:75: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead
  return _inspect.getargspec(target)
/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/tensorflow/python/util/tf_inspect.py:75: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead
  return _inspect.getargspec(target)
/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/tensorflow/python/util/tf_inspect.py:75: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead
  return _inspect.getargspec(target)
/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/tensorflow/python/util/tf_inspect.py:75: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead
  return _inspect.getargspec(target)
/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/tensorflow/python/util/tf_inspect.py:75: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead
  return _inspect.getargspec(target)
/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/tensorflow/python/util/tf_inspect.py:75: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead
  return _inspect.getargspec(target)
/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/tensorflow/python/util/tf_inspect.py:75: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead
  return _inspect.getargspec(target)
/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/tensorflow/python/util/tf_inspect.py:75: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead
  return _inspect.getargspec(target)
/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/tensorflow/python/util/tf_inspect.py:75: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead
  return _inspect.getargspec(target)
/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/tensorflow/python/util/tf_inspect.py:75: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead
  return _inspect.getargspec(target)
/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/tensorflow/python/util/tf_inspect.py:75: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead
  return _inspect.getargspec(target)
/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/tensorflow/python/util/tf_inspect.py:75: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead
  return _inspect.getargspec(target)
/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/tensorflow/python/util/tf_inspect.py:75: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead
  return _inspect.getargspec(target)
/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/tensorflow/python/util/tf_inspect.py:75: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead
  return _inspect.getargspec(target)
/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/tensorflow/python/util/tf_inspect.py:75: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead
  return _inspect.getargspec(target)
/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/tensorflow/python/util/tf_inspect.py:75: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead
  return _inspect.getargspec(target)
/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/tensorflow/python/util/tf_inspect.py:75: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead
  return _inspect.getargspec(target)
/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/tensorflow/python/util/tf_inspect.py:75: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead
  return _inspect.getargspec(target)
/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/tensorflow/python/util/tf_inspect.py:75: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead
  return _inspect.getargspec(target)
/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/tensorflow/python/util/tf_inspect.py:75: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead
  return _inspect.getargspec(target)
/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/tensorflow/python/util/tf_inspect.py:75: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead
  return _inspect.getargspec(target)
/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/tensorflow/python/util/tf_inspect.py:75: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead
  return _inspect.getargspec(target)
/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/tensorflow/python/util/tf_inspect.py:75: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead
  return _inspect.getargspec(target)
/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/tensorflow/python/util/tf_inspect.py:75: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead
  return _inspect.getargspec(target)
/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/tensorflow/python/util/tf_inspect.py:75: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead
  return _inspect.getargspec(target)
/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/tensorflow/python/util/tf_inspect.py:75: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead
  return _inspect.getargspec(target)
/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/tensorflow/python/util/tf_inspect.py:75: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead
  return _inspect.getargspec(target)
/Users/itdxer/.pyenv/versions/3.5.6/envs/py35/lib/python3.5/site-packages/pandas/__init__.py:7: DeprecationWarning: bad escape \s
  from . import hashtable, tslib, lib






    




Main information

[ALGORITHM] RPROP

[OPTION] decrease_factor = 0.5
[OPTION] increase_factor = 1.2
[OPTION] loss = categorical_crossentropy
[OPTION] maxstep = 10
[OPTION] minstep = 0.001
[OPTION] regularizer = None
[OPTION] show_epoch = 5
[OPTION] shuffle_data = True
[OPTION] signals = None
[OPTION] step = <tf.Variable 'step:0' shape=() dtype=float32_ref>
[OPTION] target = Tensor("placeholder/target/softmax-1:0", shape=(?, 10), dtype=float32)
[OPTION] verbose = True

[TENSORFLOW] Initializing Tensorflow variables and functions.
[TENSORFLOW] Initialization finished successfully. It took 0.81 seconds
#1 : [239 ms] train: 2.333779, valid: 2.232341
#5 : [4 ms] train: 0.642679, valid: 0.564598
#10 : [5 ms] train: 0.171300, valid: 0.227467
#15 : [12 ms] train: 0.065193, valid: 0.150010
#20 : [4 ms] train: 0.022588, valid: 0.152494
#25 : [5 ms] train: 0.005565, valid: 0.182332
Total train time: 0.35 sec
Validation data sample accuracy: 95.22%






    




                                                                               

In [7]:

    

net.plot_errors()


    

In [8]:

    

from sklearn import metrics

y_test_vector = y_test.argmax(axis=1)
y_test_vector = np.asarray(y_test_vector).reshape(y_test_vector.size)
confusion_matrix = metrics.confusion_matrix(y_test_vector, y_predicted)

plt.figure(figsize=(6, 6))
possible_classes = np.arange(10)

plt.imshow(confusion_matrix, cmap=plt.cm.Blues, interpolation='nearest')
plt.colorbar()

plt.xticks(possible_classes, possible_classes)
plt.yticks(possible_classes, possible_classes)

plt.show()


    

In [9]:

    

from neupy.utils import tensorflow_session

layers = list(net.network)
hidden_layer = layers[1]

sess = tensorflow_session()
input_weights = sess.run(hidden_layer.weight)

n_features = input_weights.shape[0]
feature_index = np.arange(n_features)

plt.bar(feature_index, input_weights.sum(axis=1), color='r')
plt.xticks(np.arange(0, 65, 8), np.arange(0, 65, 8))
plt.xlim(0, 64)


    

    
Out[9]:





(0, 64)






    

In [10]:

    

plt.bar(feature_index, np.abs(input_weights.sum(axis=1)), color='r')
plt.xticks(np.arange(0, 65, 8), np.arange(0, 65, 8))
plt.xlim(0, 64)


    

    
Out[10]:





(0, 64)






    

In [11]:

    

def feature_contribution(network):
    colors = [c['color'] for c in plt.rcParams['axes.prop_cycle']]

    layers = list(network.network)
    layer = layers[1]
        
    weights = sess.run(layer.weight)
    n_features = network.network.input_shape[1].value
    feature_index = np.arange(n_features)

    positive_weights = np.where(weights > 0, weights, 0)
    
    plt.bar(feature_index, positive_weights.sum(axis=1),
            color=colors[0])

    negative_weights = np.where(weights < 0, weights, 0)
    plt.bar(feature_index, negative_weights.sum(axis=1),
            color=colors[1])

    plt.xlabel("Input feature position")
    plt.xlim(0, n_features)

plt.figure(figsize=(14, 6))
label_frequency = np.arange(0, n_features + 1, 8)
plt.xticks(label_frequency, label_frequency)

feature_contribution(net)


    

In [13]:

    

from neupy import plots

weights = sess.run(layers[1].weight)
positive_weights = np.where(weights > 0, weights, 0)
negative_weights = np.where(weights < 0, weights, 0)

fig, (axes_left, axes_right) = plt.subplots(1, 2, figsize=(8, 12))

axes_left.set_title("Sum over positive weights")
plots.hinton(
    positive_weights.sum(axis=1).reshape((8, 8)),
    ax=axes_left,
    add_legend=False
)

axes_right.set_title("Sum over negative weights")
plots.hinton(
    negative_weights.sum(axis=1).reshape((8, 8)),
    ax=axes_right,
    add_legend=False
)
plt.show()


    

In [14]:

    

import six

def generate_layer_names(n_layers):
    if n_layers < 2:
        raise ValueError("Number of layers should be greater or "
                         "equal to 2")
        
    if not isinstance(n_layers, six.integer_types):
        raise ValueError("Number of layers should be an integer "
                         "number")

    patterns = {
        2: ['Input layer', 'Output layer'],
        3: ['Input layer', 'Hidden layer', 'Output layer'],
    } 

    if n_layers in patterns:
        return patterns[n_layers]
    
    layers_names = patterns[2]
    for i in range(1, n_layers - 1):
        layers_names.insert(-1, 'Hidden layer #{}'.format(i))

    return layers_names

print(generate_layer_names(n_layers=2))
print(generate_layer_names(n_layers=3))
print(generate_layer_names(n_layers=4))
print(generate_layer_names(n_layers=5))


    

    




['Input layer', 'Output layer']
['Input layer', 'Hidden layer', 'Output layer']
['Input layer', 'Hidden layer #1', 'Hidden layer #2', 'Output layer']
['Input layer', 'Hidden layer #1', 'Hidden layer #2', 'Hidden layer #3', 'Output layer']

In [15]:

    

def connection_threads(optimizer, stretch=False):
    colors = [c['color'] for c in plt.rcParams['axes.prop_cycle']]
    
    layers = list(optimizer.network)
    layers = layers[1:]  # ignore the Input layer

    layer_sizes = np.array([layer.output_shape[1].value for layer in optimizer.network])
    max_n_inputs = float(layer_sizes.max())
    layer_shifts = (max_n_inputs - layer_sizes) / 2
    layers_iter = zip(layers, layer_shifts, layer_shifts[1:])

    for i, (layer, left_shift, right_shift) in enumerate(layers_iter):
        weight = sess.run(layer.weight)
        max_value = np.abs(weight).max()

        layer_input_size, layer_output_size = weight.shape
        left_scaler = max_n_inputs / max(layer_input_size - 2, 2)
        right_scaler = max_n_inputs / max(layer_output_size - 1, 2)

        for (left_index, right_index), value in np.ndenumerate(weight):
            color = (colors[0] if np.sign(value) == 1 else colors[1])

            if stretch:
                y_coords = [(left_index) * left_scaler,
                            (right_index) * right_scaler]
            else:
                left_index += left_shift
                right_index += right_shift
                y_coords = [left_index, right_index]

            line = plt.plot([i, i + 1], y_coords, color,
                            alpha=abs(value) / (max_value))

    plt.ylim(-1, max_n_inputs + 1)

    n_layers = len(optimizer.network.layers)
    plt.yticks([], [])
    plt.xticks(np.arange(n_layers), generate_layer_names(n_layers))


    

In [16]:

    

plt.figure(figsize=(12, 6))
connection_threads(net)


    

In [17]:

    

plt.figure(figsize=(12, 6))
connection_threads(net, stretch=True)