Time series classification with fully convolutional neural networks


In [2]:
# Numbers
import numpy as np

# Keras
from keras.models import Sequential
from keras.layers.convolutional import Conv1D
from keras.layers.normalization import BatchNormalization
from keras.layers.advanced_activations import LeakyReLU
from keras.layers.pooling import GlobalAveragePooling1D
from keras.layers.core import Dense, Activation
from keras.optimizers import Adam
from keras.callbacks import EarlyStopping
from keras.utils import np_utils

# Plotting
from matplotlib import pyplot as plt
import matplotlib.gridspec as gridspec
%matplotlib inline

First we check a GPU is available by listing the available devices


In [3]:
from tensorflow.python.client import device_lib
device_lib.list_local_devices()


Out[3]:
[name: "/cpu:0"
 device_type: "CPU"
 memory_limit: 268435456
 locality {
 }
 incarnation: 5802544244545411878, name: "/gpu:0"
 device_type: "GPU"
 memory_limit: 11332668621
 locality {
   bus_id: 1
 }
 incarnation: 7224872843010531636
 physical_device_desc: "device: 0, name: Tesla K80, pci bus id: 0000:00:1e.0"]

Load Adiac dataset with numpy


In [3]:
adiac_train = np.loadtxt('UCR_TS_Archive_2015/StarLightCurves/StarLightCurves_TRAIN', delimiter = ',')
adiac_test = np.loadtxt('UCR_TS_Archive_2015/StarLightCurves/StarLightCurves_TEST', delimiter = ',')
x_train = adiac_train[:,1:]
y_train = adiac_train[:,0]
x_test = adiac_test[:,1:]
y_test = adiac_test[:,0]

Preprocess data, convert labels to one hot encodings and normalize the features


In [6]:
y_train.shape


Out[6]:
(1000,)

In [4]:
nb_classes = len(np.unique(y_test))

# Fix label range and One-hot encode
# save original labels
y_train_orig = y_train
y_test_orig = y_test

# fix range of labels
y_train = (y_train - y_train.min())/(y_train.max()-y_train.min())*(nb_classes-1)
y_test = (y_test - y_test.min())/(y_test.max()-y_test.min())*(nb_classes-1)

# one-hot encode
y_train = np_utils.to_categorical(y_train, nb_classes)
y_test = np_utils.to_categorical(y_test, nb_classes)

# Normalize features
x_train_mean = x_train.mean()
x_train_std = x_train.std()
x_train = (x_train - x_train_mean) / (x_train_std)
x_test_mean = x_test.mean()
x_test_std = x_test.std()
x_test = (x_test - x_test_mean) / (x_test_std)

# Keras expects input in a (batch_size, seq_size, num_features) format
# in this case the number of features is just one
x_train = np.expand_dims(x_train, axis=2)
x_test = np.expand_dims(x_test, axis=2)

Visualize some of the different categories


In [8]:
examples = []
for cls in np.unique(y_train_orig):
    one_class = np.where(y_train_orig==cls)
    examples.append(one_class[0][np.random.randint(len(one_class))])

gs = gridspec.GridSpec(4, 4, top=1., bottom=0., right=1., left=0., hspace=0.5,
        wspace=0.5)
i = 0
for g in gs:
    ax = plt.subplot(g)
    ax.plot(adiac_train[examples[np.random.randint(nb_classes)]][1:])
    i += 1


Initialize hyperparameters


In [14]:
epochs = 45
batch_size = min(x_train.shape[0]/10, 16)
learning_rate = 0.00005
batch_normalization_momentum = 0.2

Create keras model, this is a fully convolutional neural network that uses ReLu activations and batch normalization in every layer. In the last layer we use Global Average Pooling.


In [15]:
model = Sequential([
    Conv1D(128, 8, padding='same', input_shape=(x_train.shape[1], 1)),
    BatchNormalization(momentum=batch_normalization_momentum),
    LeakyReLU(),
    # model.add(Activation('relu'))

    Conv1D(256, 5, padding='same'),
    BatchNormalization(momentum=batch_normalization_momentum),
    LeakyReLU(),

    Conv1D(128, 3, padding='same'),
    BatchNormalization(momentum=batch_normalization_momentum),
    LeakyReLU(),

    GlobalAveragePooling1D(),
    Dense(nb_classes, activation='softmax')
])

optimizer = Adam(lr=learning_rate)
    
model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])

In [16]:
for layer in model.layers:
    print("layer: \"{}\" input_shape:{} , output_shape:{}".format(layer.name, layer.input_shape, layer.output_shape))


layer: "conv1d_4" input_shape:(None, 1024, 1) , output_shape:(None, 1024, 128)
layer: "batch_normalization_4" input_shape:(None, 1024, 128) , output_shape:(None, 1024, 128)
layer: "leaky_re_lu_4" input_shape:(None, 1024, 128) , output_shape:(None, 1024, 128)
layer: "conv1d_5" input_shape:(None, 1024, 128) , output_shape:(None, 1024, 256)
layer: "batch_normalization_5" input_shape:(None, 1024, 256) , output_shape:(None, 1024, 256)
layer: "leaky_re_lu_5" input_shape:(None, 1024, 256) , output_shape:(None, 1024, 256)
layer: "conv1d_6" input_shape:(None, 1024, 256) , output_shape:(None, 1024, 128)
layer: "batch_normalization_6" input_shape:(None, 1024, 128) , output_shape:(None, 1024, 128)
layer: "leaky_re_lu_6" input_shape:(None, 1024, 128) , output_shape:(None, 1024, 128)
layer: "global_average_pooling1d_2" input_shape:(None, 1024, 128) , output_shape:(None, 128)
layer: "dense_2" input_shape:(None, 128) , output_shape:(None, 3)

Train the neural network


In [17]:
early_stopper = EarlyStopping(monitor = 'loss', patience=50) 
hist = model.fit(x_train, y_train, 
                 batch_size, epochs, 1, 
                 validation_data=(x_test, y_test), 
                 callbacks = [early_stopper])


Train on 1000 samples, validate on 8236 samples
Epoch 1/45
1000/1000 [==============================] - 10s - loss: 0.5392 - acc: 0.8130 - val_loss: 0.5130 - val_acc: 0.8312
Epoch 2/45
1000/1000 [==============================] - 10s - loss: 0.4717 - acc: 0.8260 - val_loss: 0.4036 - val_acc: 0.8480
Epoch 3/45
1000/1000 [==============================] - 9s - loss: 0.4318 - acc: 0.8280 - val_loss: 0.3732 - val_acc: 0.8470
Epoch 4/45
1000/1000 [==============================] - 10s - loss: 0.4032 - acc: 0.8410 - val_loss: 0.3614 - val_acc: 0.8715
Epoch 5/45
1000/1000 [==============================] - 10s - loss: 0.3648 - acc: 0.8600 - val_loss: 0.3207 - val_acc: 0.8986
Epoch 6/45
1000/1000 [==============================] - 10s - loss: 0.3656 - acc: 0.8700 - val_loss: 0.3351 - val_acc: 0.8967
Epoch 7/45
1000/1000 [==============================] - 10s - loss: 0.3398 - acc: 0.8770 - val_loss: 0.2901 - val_acc: 0.8700
Epoch 8/45
1000/1000 [==============================] - 10s - loss: 0.3136 - acc: 0.8810 - val_loss: 0.3585 - val_acc: 0.8919
Epoch 9/45
1000/1000 [==============================] - 10s - loss: 0.3268 - acc: 0.8830 - val_loss: 0.3805 - val_acc: 0.8826
Epoch 10/45
1000/1000 [==============================] - 10s - loss: 0.3193 - acc: 0.8830 - val_loss: 0.3935 - val_acc: 0.8398
Epoch 11/45
1000/1000 [==============================] - 10s - loss: 0.2902 - acc: 0.9120 - val_loss: 0.3521 - val_acc: 0.9021
Epoch 12/45
1000/1000 [==============================] - 10s - loss: 0.2910 - acc: 0.8940 - val_loss: 0.2687 - val_acc: 0.9165
Epoch 13/45
1000/1000 [==============================] - 10s - loss: 0.2923 - acc: 0.9000 - val_loss: 0.2570 - val_acc: 0.9025
Epoch 14/45
1000/1000 [==============================] - 9s - loss: 0.2867 - acc: 0.9030 - val_loss: 0.3165 - val_acc: 0.9339
Epoch 15/45
1000/1000 [==============================] - 10s - loss: 0.2987 - acc: 0.8960 - val_loss: 0.3165 - val_acc: 0.8548
Epoch 16/45
1000/1000 [==============================] - 10s - loss: 0.2855 - acc: 0.9110 - val_loss: 0.2420 - val_acc: 0.9506
Epoch 17/45
1000/1000 [==============================] - 10s - loss: 0.2627 - acc: 0.9080 - val_loss: 0.2660 - val_acc: 0.9382
Epoch 18/45
1000/1000 [==============================] - 10s - loss: 0.2656 - acc: 0.9120 - val_loss: 0.3318 - val_acc: 0.8499
Epoch 19/45
1000/1000 [==============================] - 10s - loss: 0.2742 - acc: 0.8970 - val_loss: 0.2625 - val_acc: 0.8600
Epoch 20/45
1000/1000 [==============================] - 10s - loss: 0.2498 - acc: 0.9280 - val_loss: 0.2092 - val_acc: 0.9370
Epoch 21/45
1000/1000 [==============================] - 10s - loss: 0.2628 - acc: 0.9100 - val_loss: 0.2909 - val_acc: 0.8831
Epoch 22/45
1000/1000 [==============================] - 10s - loss: 0.2576 - acc: 0.9180 - val_loss: 0.2155 - val_acc: 0.9205
Epoch 23/45
1000/1000 [==============================] - 10s - loss: 0.2535 - acc: 0.9230 - val_loss: 0.4247 - val_acc: 0.8273
Epoch 24/45
1000/1000 [==============================] - 10s - loss: 0.2544 - acc: 0.9150 - val_loss: 0.2102 - val_acc: 0.9229
Epoch 25/45
1000/1000 [==============================] - 10s - loss: 0.2428 - acc: 0.9300 - val_loss: 0.2090 - val_acc: 0.9540
Epoch 26/45
1000/1000 [==============================] - 10s - loss: 0.2348 - acc: 0.9260 - val_loss: 0.2105 - val_acc: 0.9505
Epoch 27/45
1000/1000 [==============================] - 10s - loss: 0.2245 - acc: 0.9350 - val_loss: 0.2118 - val_acc: 0.9482
Epoch 28/45
1000/1000 [==============================] - 10s - loss: 0.2417 - acc: 0.9210 - val_loss: 0.2797 - val_acc: 0.9023
Epoch 29/45
1000/1000 [==============================] - 10s - loss: 0.2194 - acc: 0.9490 - val_loss: 0.2285 - val_acc: 0.9346
Epoch 30/45
1000/1000 [==============================] - 10s - loss: 0.2243 - acc: 0.9330 - val_loss: 0.2063 - val_acc: 0.9171
Epoch 31/45
1000/1000 [==============================] - 9s - loss: 0.2077 - acc: 0.9480 - val_loss: 0.2329 - val_acc: 0.8998
Epoch 32/45
1000/1000 [==============================] - 10s - loss: 0.1960 - acc: 0.9560 - val_loss: 0.2184 - val_acc: 0.9450
Epoch 33/45
1000/1000 [==============================] - 10s - loss: 0.2035 - acc: 0.9420 - val_loss: 0.2370 - val_acc: 0.9426
Epoch 34/45
1000/1000 [==============================] - 10s - loss: 0.2225 - acc: 0.9390 - val_loss: 0.1804 - val_acc: 0.9605
Epoch 35/45
1000/1000 [==============================] - 10s - loss: 0.2073 - acc: 0.9460 - val_loss: 0.1718 - val_acc: 0.9645
Epoch 36/45
1000/1000 [==============================] - 10s - loss: 0.2210 - acc: 0.9340 - val_loss: 0.2485 - val_acc: 0.9486
Epoch 37/45
1000/1000 [==============================] - 10s - loss: 0.2197 - acc: 0.9330 - val_loss: 0.1746 - val_acc: 0.9622
Epoch 38/45
1000/1000 [==============================] - 10s - loss: 0.1910 - acc: 0.9510 - val_loss: 0.1979 - val_acc: 0.9514
Epoch 39/45
1000/1000 [==============================] - 9s - loss: 0.2060 - acc: 0.9390 - val_loss: 0.2172 - val_acc: 0.9451
Epoch 40/45
1000/1000 [==============================] - 10s - loss: 0.2188 - acc: 0.9320 - val_loss: 0.1841 - val_acc: 0.9523
Epoch 41/45
1000/1000 [==============================] - 10s - loss: 0.1988 - acc: 0.9480 - val_loss: 0.2079 - val_acc: 0.9491
Epoch 42/45
1000/1000 [==============================] - 10s - loss: 0.1823 - acc: 0.9570 - val_loss: 0.2148 - val_acc: 0.9438
Epoch 43/45
1000/1000 [==============================] - 10s - loss: 0.1997 - acc: 0.9490 - val_loss: 0.1824 - val_acc: 0.9575
Epoch 44/45
1000/1000 [==============================] - 10s - loss: 0.1869 - acc: 0.9550 - val_loss: 0.1673 - val_acc: 0.9546
Epoch 45/45
1000/1000 [==============================] - 9s - loss: 0.1798 - acc: 0.9570 - val_loss: 0.1732 - val_acc: 0.9678

In [22]:
def moving_average(a, n=3) :
    ret = np.cumsum(a, dtype=float)
    ret[n:] = ret[n:] - ret[:-n]
    return ret[n - 1:] / n

plt.plot(hist.history['loss'])
# plt.plot(moving_average(hist.history['val_loss']))
plt.plot(hist.history['val_loss'])


Out[22]:
[<matplotlib.lines.Line2D at 0x7fb6492e2908>]

In [23]:
plt.plot(hist.history['acc'])
plt.plot(hist.history['val_acc'])


Out[23]:
[<matplotlib.lines.Line2D at 0x7fb64f81b5f8>]

In [ ]: