Neural Networks Design


In [ ]:
%matplotlib inline

In [ ]:
import itertools
import matplotlib.pyplot as plt
import h5py
import numpy as np
import sklearn.preprocessing
import sklearn.metrics
import imblearn.over_sampling
import spectraldl.plot as plot
import spectraldl.preprocessing as preprocessing
import tensorflow.contrib.keras as keras

In [ ]:
with h5py.File('data/data.hdf5') as f:
    X_tr_ = f['X_tr'][...]
    y_tr_ = f['y_tr'][...]
    X_val = f['X_val'][...]
    y_val = f['y_val'][...]
    X_te = f['X_te'][...]
    y_te = f['y_te'][...]

In [ ]:
def plot_hist(hist):
    fig, (ax1, ax2) = plt.subplots(2, 1)
    ax1.plot(hist.epoch, hist.history['acc'], label='training data')
    ax1.plot(hist.epoch, hist.history['val_acc'], label='validation data')
    ax1.set_xlabel('epoch')
    ax1.set_ylabel('accuracy')
    ax1.legend()
    ax2.plot(hist.epoch, hist.history['loss'])
    ax2.set_xlabel('epoch')
    ax2.set_ylabel('loss')
    ax1.grid()
    ax2.grid()
    fig.tight_layout()

def plot_cm(model, X, y):
    y_out = model.predict_classes(X, verbose=0)
    cm = sklearn.metrics.confusion_matrix(y, y_out)
    classes = ['emission', 'absorption', 'double-peak']
    plot.plot_confusion_matrix(cm, classes, normalize=True)

def plot_confusion_matrix(
        cm, classes, ax,
        normalize=False, title='confusion matrix', cmap=plt.cm.Greys
    ):
    '''
    This function plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    '''
    if normalize:
        cm = cm.astype(np.float) / cm.sum(axis=1)[:, np.newaxis]

    ax.imshow(cm, interpolation='nearest', cmap=cmap)
    ax.set_title(title)
    #ax.colorbar()
    tick_marks = np.arange(len(classes))
    ax.set_xticks(tick_marks)
    ax.set_xticklabels(classes, rotation=45)
    ax.set_yticks(tick_marks)
    ax.set_yticklabels(classes)

    thresh = cm.max() / 2
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        ax.text(j, i, round(cm[i, j], 2),
                 horizontalalignment='center',
                 color='white' if cm[i, j] > thresh else 'black')

    ax.set_ylabel('true label')
    ax.set_xlabel('predicted label')

classes = ['emission', 'absorption', 'double-peak']

In [ ]:
X_val = preprocessing.scale_samples(X_val)
X_te = preprocessing.scale_samples(X_te)

y_val_oh = keras.utils.to_categorical(y_val, num_classes=3)
y_te_oh = keras.utils.to_categorical(y_te, num_classes=3)

Deep Neural Net


In [ ]:
# variables ending with _ (underscore) hold original data
X_tr, y_tr = np.copy(X_tr_), np.copy(y_tr_)

# preprocessing
X_tr = preprocessing.scale_samples(X_tr)
#X_tr, y_tr = preprocessing.smote_over_sample(X_tr, y_tr, n_classes=3)

# oh stands fo one hot vector
y_tr_oh = keras.utils.to_categorical(y_tr, num_classes=3)

dnn_model = keras.models.Sequential()

units = 512

dnn_model.add(keras.layers.Dense(units=units, activation='relu', input_shape=(140, )))
dnn_model.add(keras.layers.Dropout(0.5))

dnn_model.add(keras.layers.Dense(units=units, activation='relu'))
dnn_model.add(keras.layers.Dropout(0.5))

dnn_model.add(keras.layers.Dense(units=units, activation='relu'))
dnn_model.add(keras.layers.Dropout(0.5))

dnn_model.add(keras.layers.Dense(units=units, activation='relu'))
dnn_model.add(keras.layers.Dropout(0.5))

dnn_model.add(keras.layers.Dense(units=units, activation='relu'))
dnn_model.add(keras.layers.Dropout(0.5))
    
dnn_model.add(keras.layers.Dense(units=3, activation='softmax'))

dnn_model.compile(
    loss='categorical_crossentropy',
    optimizer='adam',
    metrics=['accuracy'],
)

# train the model
dnn_hist = dnn_model.fit(
    X_tr, y_tr_oh,
    epochs=250, batch_size=256, verbose=2,
    validation_data=(X_val, y_val_oh)
)

In [ ]:
plot_hist(dnn_hist)

In [ ]:
fig, ax = plt.subplots(figsize=(4, 4))
tmp_out = dnn_model.predict_classes(X_te, verbose=0)
tmp_cm = sklearn.metrics.confusion_matrix(y_te, tmp_out)
plot_confusion_matrix(tmp_cm, classes, ax, normalize=True)
fig.tight_layout()

ConvNet


In [ ]:
# variables ending with _ (underscore) hold original data
X_tr, y_tr = np.copy(X_tr_), np.copy(y_tr_)

# preprocessing
X_tr = preprocessing.scale_samples(X_tr)
X_tr, y_tr = preprocessing.smote_over_sample(X_tr, y_tr, n_classes=3)

# oh stands fo one hot vector
y_tr_oh = keras.utils.to_categorical(y_tr, num_classes=3)

HEIGHT = 1
WIDTH = 140
DEPTH = 1
X_tr = X_tr.reshape(-1, HEIGHT, WIDTH, DEPTH)
X_val = X_val.reshape(-1, HEIGHT, WIDTH, DEPTH)

input_shape = X_tr.shape[1:]

conv_model = keras.models.Sequential([
    keras.layers.Conv2D(64, (1, 3), activation='relu', input_shape=input_shape),
    keras.layers.Conv2D(64, (1, 3), activation='relu'),
    keras.layers.MaxPooling2D(pool_size=(1, 2)),

    keras.layers.Conv2D(128, (1, 3), activation='relu'),
    keras.layers.Conv2D(128, (1, 3), activation='relu'),
    keras.layers.MaxPooling2D(pool_size=(1, 2)),

    keras.layers.Conv2D(256, (1, 3), activation='relu'),
    keras.layers.Conv2D(256, (1, 3), activation='relu'),
    keras.layers.MaxPooling2D(pool_size=(1, 2)),
    
    keras.layers.Flatten(),
    
    keras.layers.Dense(512, activation='relu'),
    keras.layers.Dropout(0.5),
    keras.layers.Dense(512, activation='relu'),
    keras.layers.Dropout(0.5),
    keras.layers.Dense(3, activation='softmax')
])

conv_model.compile(
    loss='categorical_crossentropy',
    optimizer='adam',
    metrics=['accuracy']
)

# training
conv_hist = conv_model.fit(
    X_tr, y_tr_oh,
    epochs=250, batch_size=256, verbose=2,
    validation_data=(X_val, y_val_oh)
)

In [ ]:
# vizualize traing resutls
plot_hist(conv_hist)

In [ ]:
plot_cm(conv_model, X_te.reshape(-1, HEIGHT, WIDTH, DEPTH), y_te)

In [ ]:
dnn_out = dnn_model.predict_classes(X_te, verbose=0)
dnn_cm = sklearn.metrics.confusion_matrix(y_te, dnn_out)

conv_out = conv_model.predict_classes(X_te.reshape(-1, HEIGHT, WIDTH, DEPTH), verbose=0)
conv_cm = sklearn.metrics.confusion_matrix(y_te, conv_out)

In [ ]:
np.count_nonzero(y_te == dnn_out), np.count_nonzero(y_te == conv_out)

In [ ]:
fig, (ax1, ax2) = plt.subplots(1, 2)

plot_confusion_matrix(dnn_cm, classes, ax1, title='feedforward network', normalize=True)
cax = plot_confusion_matrix(conv_cm, classes, ax2, title='convolutional network', normalize=True)

fig.tight_layout()