In [12]:

    

import numpy
import pandas
from keras.models import Sequential
from keras.layers import Dense
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.cross_validation import cross_val_score
from sklearn.preprocessing import LabelEncoder
from sklearn.cross_validation import StratifiedKFold
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline

# fix random seed for reproducibility
seed = 67
numpy.random.seed(seed)

data = pandas.read_csv("../data/processed/train.csv")
notnull_data = data[data.notnull().all(axis=1)]
#train = notnull_data.values
data2 = pandas.read_csv("../data/processed/test.csv")
notnull_data2 = data2[data2.notnull().all(axis=1)]
test = notnull_data2.values

# Divide the train dataset further into validation and train sets
# keeping in the val set a session from each teacher, 
# with a variety of tags for Activity and Social
#pandas.crosstab(index=data["session"], columns=data["Activity"])
#pandas.crosstab(index=data["session"], columns=data["Social"])
# e.g., case1-day1-session2-teacher1, case2-day2-session2-teacher2
print 'Loaded csvs'


    

    




Loaded csvs

In [13]:

    

val = notnull_data[notnull_data.session.isin(['case1-day1-session2-teacher1','case2-day2-session2-teacher2'])].values
print val.shape
tr = notnull_data[~notnull_data.session.isin(['case1-day1-session2-teacher1','case2-day2-session2-teacher2'])].values
print tr.shape

X_train = tr[:,3:7558].astype(float)
Y_trainA = tr[:,7558] #Activity
Y_trainS = tr[:,7559] #Social
X_val = val[:,3:7558].astype(float)
Y_valA = val[:,7558] #Activity
Y_valS = val[:,7559] #Social
X_test = test[:,3:7558].astype(float)
Y_testA = test[:,7558]
Y_testS = test[:,7559]

# One hot encoding of the response variable (using dummy variables)
from keras.utils.np_utils import to_categorical

# encode class values as integers
encoderA = LabelEncoder()
encoderA.fit(Y_trainA)

encoded_Y_trainA = encoderA.transform(Y_trainA)
# convert integers to dummy variables (i.e. one hot encoded)
dummy_y_trainA = to_categorical(encoded_Y_trainA)

encoded_Y_valA = encoderA.transform(Y_valA)
# convert integers to dummy variables (i.e. one hot encoded)
dummy_y_valA = to_categorical(encoded_Y_valA)

#encoderA.fit(Y_testA)
encoded_Y_testA = encoderA.transform(Y_testA)
# convert integers to dummy variables (i.e. one hot encoded)
dummy_y_testA = to_categorical(encoded_Y_testA)

# encode class values as integers
encoderS = LabelEncoder()
encoderS.fit(Y_trainS)
encoded_Y_trainS = encoderS.transform(Y_trainS)
# convert integers to dummy variables (i.e. one hot encoded)
dummy_y_trainS = to_categorical(encoded_Y_trainS)

encoded_Y_valS = encoderS.transform(Y_valS)
# convert integers to dummy variables (i.e. one hot encoded)
dummy_y_valS = to_categorical(encoded_Y_valS)

#encoderS.fit(Y_testS)
encoded_Y_testS = encoderS.transform(Y_testS)
# convert integers to dummy variables (i.e. one hot encoded)
dummy_y_testS = to_categorical(encoded_Y_testS)

# We standardize on the basis of the training data
scaler = StandardScaler().fit(X_train)
X_train_st = scaler.transform(X_train)
X_val_st = scaler.transform(X_val)
X_test_st = scaler.transform(X_test)
print 'Split into train, val, test sets and scaled'


    

    




(969, 7560)
(3503, 7560)
Split into train, val, test sets and scaled

In [31]:

    

# Reduce the dataset to work with to the AUDIO variables
# Get the column names of the X matrix, to separate the different sources
colnamesX = list(data.columns.values)[3:7558]
# Eyetracking: colnamesX[0:10]
# Acc: colnamesX[10:150]
# Audio: colnamesX[150:6555]
# Video: colnamesX[6555:]
X_train_st_audio = X_train_st[:,150:6555]
X_val_st_audio = X_val_st[:,150:6555]
X_test_st_audio = X_test_st[:,150:6555]
# Number of components to extract from the dataset
n_components = 100

from sklearn import decomposition
print 'Reducing AUDIO dataset with PCA',n_components
pca = decomposition.PCA(n_components=n_components)
X_train_pca = pca.fit_transform(X_train_st_audio)
X_val_pca = pca.transform(X_val_st_audio)
X_test_pca = pca.transform(X_test_st_audio)

#print 'Variance explained:'
#print pca.explained_variance_ratio_
print 'Total variance explained by %d components:',n_components
print sum(pca.explained_variance_ratio_)

trainX = numpy.reshape(X_train_pca, (X_train_pca.shape[0], 1, X_train_pca.shape[1]))
valX = numpy.reshape(X_val_pca, (X_val_pca.shape[0], 1, X_val_pca.shape[1]))
testX = numpy.reshape(X_test_pca, (X_test_pca.shape[0], 1, X_test_pca.shape[1]))


    

    




Reducing AUDIO dataset with PCA 100
Total variance explained by %d components: 100
0.684495816407

In [32]:

    

from keras.layers import Dropout
from keras.layers import LSTM
from keras.constraints import maxnorm
from keras.optimizers import SGD

# This is our winning architecture so far
def create_LSTM3_PCA(n_outputs, batch_size = 1, trainShape1=100):
    # create and fit the LSTM network
    model = Sequential()
    # stateful LSTM!
    model.add(LSTM(200, batch_input_shape=(batch_size, 1, trainShape1), 
                   return_sequences=True, stateful=True))
    model.add(Dropout(0.2))
    model.add(LSTM(100, 
                   return_sequences=True, stateful=True))
    model.add(Dropout(0.2))
    model.add(LSTM(50, 
                   return_sequences=False, stateful=True))
    model.add(Dropout(0.2))
    model.add(Dense(50, activation='tanh'))
    model.add(Dropout(0.2))
    model.add(Dense(20, activation='tanh'))
    model.add(Dropout(0.2))
    model.add(Dense(n_outputs, activation='softmax'))
    # Compile model
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
    return model

def create_LSTM2_PCA(n_outputs, batch_size = 1, trainShape1=100):
    # create and fit the LSTM network
    model = Sequential()
    # stateful LSTM!
    model.add(LSTM(300, batch_input_shape=(batch_size, 1, trainShape1), 
                   return_sequences=True, stateful=True))
    model.add(Dropout(0.2))
    model.add(LSTM(50, 
                   return_sequences=False, stateful=True))
    model.add(Dropout(0.2))
    model.add(Dense(50, activation='tanh'))
    model.add(Dropout(0.2))
    model.add(Dense(20, activation='tanh'))
    model.add(Dropout(0.2))
    model.add(Dense(n_outputs, activation='softmax'))
    # Compile model
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
    return model

def create_LSTM1_PCA(n_outputs, batch_size = 1, trainShape1=100):
    # create and fit the LSTM network
    model = Sequential()
    # stateful LSTM!
    model.add(LSTM(400, batch_input_shape=(batch_size, 1, trainShape1), 
                   return_sequences=False, stateful=True))
    model.add(Dropout(0.2))
    model.add(Dense(50, activation='tanh'))
    model.add(Dropout(0.2))
    model.add(Dense(20, activation='tanh'))
    model.add(Dropout(0.2))
    model.add(Dense(n_outputs, activation='softmax'))
    # Compile model
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
    return model

from keras.callbacks import ModelCheckpoint
import matplotlib.pyplot as plt
from sklearn.metrics import confusion_matrix, roc_auc_score, accuracy_score, cohen_kappa_score

def printValStats(model, testX, dummy_y_test, batch=1):
    # Other performance/accuracy metrics
    Y_pred = model.predict(testX, batch_size=batch)
    model.reset_states()
    print 'Performance of model on test set ----------------------------'
    # Accuracy
    print('Accuracy:')
    print(accuracy_score(numpy.argmax(dummy_y_test, axis=1), numpy.argmax(Y_pred, axis=1)))
    # Kappa
    print('Kappa:')
    kappa = cohen_kappa_score(numpy.argmax(dummy_y_test, axis=1), numpy.argmax(Y_pred, axis=1))
    print(kappa)
    # Confusion matrix
    cm = confusion_matrix(numpy.argmax(dummy_y_test, axis=1), numpy.argmax(Y_pred, axis=1))
    numpy.set_printoptions(precision=2)
    print('Confusion matrix:')
    print(cm)
    # AUC
    roc = roc_auc_score(dummy_y_test, Y_pred, average='macro')
    print('AUC score:')
    print(roc)
    return kappa, roc

def plot_training(accs, val_accs, losss, val_losss, kappas, aucs):
    # summarize history for accuracy
    plt.plot(accs)
    plt.plot(val_accs)
    plt.title('model accuracy')
    plt.ylabel('accuracy')
    plt.xlabel('epoch')
    plt.legend(['train','test'], loc='upper left')
    plt.show()
    # summarize history for loss
    plt.plot(losss)
    plt.plot(val_losss)
    plt.title('model loss')
    plt.ylabel('loss')
    plt.xlabel('epoch')
    plt.legend(['train','test'], loc='upper left')
    plt.show()
    # summarize kappa and auc
    plt.plot(kappas)
    plt.plot(aucs)
    plt.title('Other performance')
    plt.ylabel('metric')
    plt.xlabel('epoch')
    plt.legend(['Kappa','AUC'], loc='upper left')
    plt.show()
    
    
import operator

def get_max_values(list):
    index, value = max(enumerate(list), key=operator.itemgetter(1))
    return index, value

print 'Ready for training!'


    

    




Ready for training!

In [ ]:

    

# Create the model and parameters for training
numpy.random.seed(seed)

batch = 1
epochs = 100

modelA3 = create_LSTM3_PCA(dummy_y_trainA.shape[1], batch_size = batch, trainShape1=n_components)
print modelA3.summary()

# To save the best model
# serialize model to JSON
modelA3_json = modelA3.to_json()
with open("activity.model--3lstmaudio.json", "w") as json_file:
    json_file.write(modelA3_json)
filepathA3="activity.weights--3lstmaudio.best.hdf5"
# Define that the accuracy in cv is monitored, and that weights are stored in a file when max accuracy is achieved
checkpointA3 = ModelCheckpoint(filepathA3, monitor='val_acc', verbose=1, save_best_only=True, mode='max')
callbacks_listA3 = [checkpointA3]

# Fit the model
accs =[]
val_accs =[]
losss =[]
val_losss =[]
kappas = []
aucs = []

# Manually create epochs and reset between sessions
for i in range(epochs):
    # Single epoch. Remember to not shuffle the data!
    print('Epoch', i+1, '/', epochs)
    history = modelA3.fit(trainX, dummy_y_trainA, validation_data=(valX, dummy_y_valA), 
                        nb_epoch=1, batch_size=batch, shuffle=False, 
                        verbose=1, callbacks=callbacks_listA3)
    modelA3.reset_states()
    kappa, auc = printValStats(modelA3, valX, dummy_y_valA, batch=batch)
    accs.append(history.history['acc'][0])
    val_accs.append(history.history['val_acc'][0])
    losss.append(history.history['loss'][0])
    val_losss.append(history.history['val_loss'][0])
    kappas.append(kappa)
    aucs.append(auc)
    
print 'Best validation accuracy: ', get_max_values(val_accs)
plot_training(accs, val_accs, losss, val_losss, kappas, aucs)


    

    




____________________________________________________________________________________________________
Layer (type)                     Output Shape          Param #     Connected to                     
====================================================================================================
lstm_1 (LSTM)                    (1, 1, 200)           240800      lstm_input_1[0][0]               
____________________________________________________________________________________________________
dropout_1 (Dropout)              (1, 1, 200)           0           lstm_1[0][0]                     
____________________________________________________________________________________________________
lstm_2 (LSTM)                    (1, 1, 100)           120400      dropout_1[0][0]                  
____________________________________________________________________________________________________
dropout_2 (Dropout)              (1, 1, 100)           0           lstm_2[0][0]                     
____________________________________________________________________________________________________
lstm_3 (LSTM)                    (1, 50)               30200       dropout_2[0][0]                  
____________________________________________________________________________________________________
dropout_3 (Dropout)              (1, 50)               0           lstm_3[0][0]                     
____________________________________________________________________________________________________
dense_1 (Dense)                  (1, 50)               2550        dropout_3[0][0]                  
____________________________________________________________________________________________________
dropout_4 (Dropout)              (1, 50)               0           dense_1[0][0]                    
____________________________________________________________________________________________________
dense_2 (Dense)                  (1, 20)               1020        dropout_4[0][0]                  
____________________________________________________________________________________________________
dropout_5 (Dropout)              (1, 20)               0           dense_2[0][0]                    
____________________________________________________________________________________________________
dense_3 (Dense)                  (1, 5)                105         dropout_5[0][0]                  
====================================================================================================
Total params: 395075
____________________________________________________________________________________________________
None
('Epoch', 1, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 1.2689 - acc: 0.5023Epoch 00000: val_acc improved from -inf to 0.25077, saving model to activity.weights--3lstmaudio.best.hdf5
3503/3503 [==============================] - 62s - loss: 1.2693 - acc: 0.5021 - val_loss: 1.9646 - val_acc: 0.2508
Performance of model on test set ----------------------------
Accuracy:
0.251805985552
Kappa:
0.0468951594851
Confusion matrix:
[[  0   0   0   0 122]
 [  0 144   0   0 118]
 [  0 125   0   0 118]
 [  0  35   0   0 133]
 [  0  74   0   0 100]]
AUC score:
0.659596465468
('Epoch', 2, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 1.1570 - acc: 0.5485Epoch 00000: val_acc improved from 0.25077 to 0.30857, saving model to activity.weights--3lstmaudio.best.hdf5
3503/3503 [==============================] - 70s - loss: 1.1573 - acc: 0.5484 - val_loss: 1.8879 - val_acc: 0.3086
Performance of model on test set ----------------------------
Accuracy:
0.310629514964
Kappa:
0.13378172582
Confusion matrix:
[[116   1   5   0   0]
 [ 75 143  44   0   0]
 [ 69 132  42   0   0]
 [120  36  12   0   0]
 [ 45  91  38   0   0]]
AUC score:
0.61159059271
('Epoch', 3, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 1.0809 - acc: 0.5914Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 81s - loss: 1.0812 - acc: 0.5912 - val_loss: 2.0865 - val_acc: 0.2797
Performance of model on test set ----------------------------
Accuracy:
0.28689370485
Kappa:
0.0940494391752
Confusion matrix:
[[109   0  13   0   0]
 [ 29 121 112   0   0]
 [ 74 121  48   0   0]
 [106  33  29   0   0]
 [ 28  55  91   0   0]]
AUC score:
0.586630974546
('Epoch', 4, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 1.0058 - acc: 0.6325Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 84s - loss: 1.0058 - acc: 0.6323 - val_loss: 2.0634 - val_acc: 0.3034
Performance of model on test set ----------------------------
Accuracy:
0.301341589267
Kappa:
0.10439465924
Confusion matrix:
[[101   0  21   0   0]
 [ 17  97 145   0   3]
 [ 45 104  94   0   0]
 [ 99  24  44   0   1]
 [ 15  28 131   0   0]]
AUC score:
0.595045966039
('Epoch', 5, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.9295 - acc: 0.6710Epoch 00000: val_acc improved from 0.30857 to 0.32611, saving model to activity.weights--3lstmaudio.best.hdf5
3503/3503 [==============================] - 85s - loss: 0.9295 - acc: 0.6709 - val_loss: 1.8770 - val_acc: 0.3261
Performance of model on test set ----------------------------
Accuracy:
0.334365325077
Kappa:
0.134203002144
Confusion matrix:
[[ 93   0  23   0   6]
 [ 12 124  98   0  28]
 [ 16 131  95   0   1]
 [ 47  31  87   0   3]
 [ 12  74  76   0  12]]
AUC score:
0.671039582073
('Epoch', 6, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.8512 - acc: 0.7013Epoch 00000: val_acc improved from 0.32611 to 0.35913, saving model to activity.weights--3lstmaudio.best.hdf5
3503/3503 [==============================] - 90s - loss: 0.8511 - acc: 0.7014 - val_loss: 2.1873 - val_acc: 0.3591
Performance of model on test set ----------------------------
Accuracy:
0.37048503612
Kappa:
0.195028183342
Confusion matrix:
[[105   3  10   0   4]
 [ 25  41 191   0   5]
 [ 33  11 199   0   0]
 [ 78  17  73   0   0]
 [ 19  46  95   0  14]]
AUC score:
0.636614370532
('Epoch', 7, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.7602 - acc: 0.7479Epoch 00000: val_acc improved from 0.35913 to 0.37771, saving model to activity.weights--3lstmaudio.best.hdf5
3503/3503 [==============================] - 92s - loss: 0.7601 - acc: 0.7479 - val_loss: 1.9019 - val_acc: 0.3777
Performance of model on test set ----------------------------
Accuracy:
0.37048503612
Kappa:
0.186228848269
Confusion matrix:
[[ 84   1  31   6   0]
 [  4  64 146  23  25]
 [ 13  48 172   6   4]
 [ 42  29  83   6   8]
 [  6  59  66  10  33]]
AUC score:
0.702416028237
('Epoch', 8, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.7144 - acc: 0.7676Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 94s - loss: 0.7143 - acc: 0.7676 - val_loss: 1.9181 - val_acc: 0.3560
Performance of model on test set ----------------------------
Accuracy:
0.355005159959
Kappa:
0.161198151831
Confusion matrix:
[[ 83   2  36   0   1]
 [  4  62 178   4  14]
 [ 12  57 169   0   5]
 [ 47  31  82   2   6]
 [  6  73  64   3  28]]
AUC score:
0.675981713055
('Epoch', 9, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.6623 - acc: 0.7864Epoch 00000: val_acc improved from 0.37771 to 0.38803, saving model to activity.weights--3lstmaudio.best.hdf5
3503/3503 [==============================] - 92s - loss: 0.6621 - acc: 0.7865 - val_loss: 2.2118 - val_acc: 0.3880
Performance of model on test set ----------------------------
Accuracy:
0.391124871001
Kappa:
0.217428441195
Confusion matrix:
[[ 94   2  22   0   4]
 [  8  26 197   6  25]
 [ 14   6 217   0   6]
 [ 54   9  89   7   9]
 [  7  18 109   5  35]]
AUC score:
0.679616741276
('Epoch', 10, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.6179 - acc: 0.7975Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 93s - loss: 0.6185 - acc: 0.7973 - val_loss: 2.1570 - val_acc: 0.3860
Performance of model on test set ----------------------------
Accuracy:
0.418988648091
Kappa:
0.253274768474
Confusion matrix:
[[ 85  13  21   2   1]
 [ 11 118  80   4  49]
 [ 14  79 132   3  15]
 [ 46  20  62  15  25]
 [ 10  42  62   4  56]]
AUC score:
0.701809581521
('Epoch', 11, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.5884 - acc: 0.8067Epoch 00000: val_acc improved from 0.38803 to 0.46233, saving model to activity.weights--3lstmaudio.best.hdf5
3503/3503 [==============================] - 89s - loss: 0.5883 - acc: 0.8067 - val_loss: 1.8456 - val_acc: 0.4623
Performance of model on test set ----------------------------
Accuracy:
0.468524251806
Kappa:
0.323859448503
Confusion matrix:
[[ 82  11  18   4   7]
 [ 13 101  71  19  58]
 [ 15  41 161   6  20]
 [ 50  31  35  29  23]
 [  8  37  34  14  81]]
AUC score:
0.727081838647
('Epoch', 12, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.5101 - acc: 0.8452Epoch 00000: val_acc improved from 0.46233 to 0.47472, saving model to activity.weights--3lstmaudio.best.hdf5
3503/3503 [==============================] - 88s - loss: 0.5100 - acc: 0.8453 - val_loss: 1.9867 - val_acc: 0.4747
Performance of model on test set ----------------------------
Accuracy:
0.479876160991
Kappa:
0.338037599794
Confusion matrix:
[[ 97   6  15   1   3]
 [ 20 108  80   6  48]
 [ 20  25 186   1  11]
 [ 62  29  26  18  33]
 [ 16  64  34   4  56]]
AUC score:
0.736110166754
('Epoch', 13, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.4865 - acc: 0.8555Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 87s - loss: 0.4864 - acc: 0.8556 - val_loss: 2.2015 - val_acc: 0.4231
Performance of model on test set ----------------------------
Accuracy:
0.422084623323
Kappa:
0.264623774237
Confusion matrix:
[[ 88   5  25   1   3]
 [ 22  78  99  16  47]
 [ 21  23 177   7  15]
 [ 54  23  54   8  29]
 [ 12  40  52  12  58]]
AUC score:
0.714565790409
('Epoch', 14, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.4324 - acc: 0.8675Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 90s - loss: 0.4323 - acc: 0.8675 - val_loss: 2.2272 - val_acc: 0.4303
Performance of model on test set ----------------------------
Accuracy:
0.43137254902
Kappa:
0.277056605306
Confusion matrix:
[[ 96   7  10   3   6]
 [  8  74 113  11  56]
 [ 24  16 183   9  11]
 [ 66  23  33  17  29]
 [  7  50  61   8  48]]
AUC score:
0.715783232956
('Epoch', 15, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.4402 - acc: 0.8601Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 92s - loss: 0.4401 - acc: 0.8601 - val_loss: 2.3535 - val_acc: 0.3953
Performance of model on test set ----------------------------
Accuracy:
0.389060887513
Kappa:
0.229362382973
Confusion matrix:
[[ 83   1  26   4   8]
 [ 14  31 144  16  57]
 [ 21  12 178  20  12]
 [ 69   8  34  25  32]
 [ 13  19  62  20  60]]
AUC score:
0.711972590191
('Epoch', 16, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.4029 - acc: 0.8789Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 97s - loss: 0.4028 - acc: 0.8790 - val_loss: 2.0671 - val_acc: 0.4685
Performance of model on test set ----------------------------
Accuracy:
0.463364293086
Kappa:
0.323050384301
Confusion matrix:
[[ 92   6  11   4   9]
 [ 12 127  28  16  79]
 [ 22  68 122  11  20]
 [ 54  20   6  29  59]
 [ 18  52  17   8  79]]
AUC score:
0.741876928989
('Epoch', 17, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.3576 - acc: 0.8992Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 88s - loss: 0.3575 - acc: 0.8992 - val_loss: 2.1797 - val_acc: 0.4737
Performance of model on test set ----------------------------
Accuracy:
0.488132094943
Kappa:
0.352124295339
Confusion matrix:
[[ 86   9  12  12   3]
 [  7  98  76   6  75]
 [ 18  18 173  13  21]
 [ 54  21  20  42  31]
 [ 12  43  32  13  74]]
AUC score:
0.749655365107
('Epoch', 18, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.3204 - acc: 0.9118Epoch 00000: val_acc improved from 0.47472 to 0.48400, saving model to activity.weights--3lstmaudio.best.hdf5
3503/3503 [==============================] - 80s - loss: 0.3204 - acc: 0.9118 - val_loss: 2.1297 - val_acc: 0.4840
Performance of model on test set ----------------------------
Accuracy:
0.486068111455
Kappa:
0.351600836294
Confusion matrix:
[[ 95   5   5   8   9]
 [ 10  93  80  13  66]
 [ 19  26 170  16  12]
 [ 55  23   6  39  45]
 [ 16  46  26  12  74]]
AUC score:
0.747603640693
('Epoch', 19, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.3161 - acc: 0.9121Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 76s - loss: 0.3160 - acc: 0.9121 - val_loss: 2.3955 - val_acc: 0.4696
Performance of model on test set ----------------------------
Accuracy:
0.470588235294
Kappa:
0.332975956932
Confusion matrix:
[[ 89   1  11  12   9]
 [  4  62 105  31  60]
 [ 19  17 171  15  21]
 [ 41  13  11  58  45]
 [  8  34  37  19  76]]
AUC score:
0.740531586247
('Epoch', 20, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.3217 - acc: 0.9132Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 86s - loss: 0.3217 - acc: 0.9132 - val_loss: 2.5254 - val_acc: 0.4087
Performance of model on test set ----------------------------
Accuracy:
0.410732714138
Kappa:
0.252878172877
Confusion matrix:
[[ 93   3  18   4   4]
 [ 12  64 120  11  55]
 [ 24  37 163   4  15]
 [ 57  22  38   7  44]
 [ 15  27  52   9  71]]
AUC score:
0.691995854452
('Epoch', 21, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.2961 - acc: 0.9163Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 85s - loss: 0.2961 - acc: 0.9164 - val_loss: 2.4601 - val_acc: 0.4293
Performance of model on test set ----------------------------
Accuracy:
0.434468524252
Kappa:
0.285308784043
Confusion matrix:
[[ 89   7   8  15   3]
 [  5 121  49  29  58]
 [ 18  94  78  43  10]
 [ 46  28  13  60  21]
 [ 17  36  30  18  73]]
AUC score:
0.703658205573
('Epoch', 22, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.2719 - acc: 0.9243Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 90s - loss: 0.2718 - acc: 0.9244 - val_loss: 2.4873 - val_acc: 0.4613
Performance of model on test set ----------------------------
Accuracy:
0.458204334365
Kappa:
0.316805595285
Confusion matrix:
[[100   4  12   3   3]
 [ 15  75  87  24  61]
 [ 18  21 177  14  13]
 [ 57  17  38  27  29]
 [ 25  32  38  14  65]]
AUC score:
0.730885216947
('Epoch', 23, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.3155 - acc: 0.9058Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 84s - loss: 0.3154 - acc: 0.9058 - val_loss: 2.5323 - val_acc: 0.4417
Performance of model on test set ----------------------------
Accuracy:
0.433436532508
Kappa:
0.283551395576
Confusion matrix:
[[ 91   2  11  14   4]
 [  4  79  89  27  63]
 [ 13  26 158  24  22]
 [ 58  17  36  36  21]
 [  8  34  50  26  56]]
AUC score:
0.70685528936
('Epoch', 24, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.2954 - acc: 0.9218Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 75s - loss: 0.2954 - acc: 0.9218 - val_loss: 2.6807 - val_acc: 0.4159
Performance of model on test set ----------------------------
Accuracy:
0.412796697626
Kappa:
0.261869954617
Confusion matrix:
[[ 88   5   9  17   3]
 [ 13  79  72  39  59]
 [ 16  49 116  42  20]
 [ 60  23  18  42  25]
 [  9  21  52  17  75]]
AUC score:
0.691513256187
('Epoch', 25, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.2528 - acc: 0.9249Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 77s - loss: 0.2528 - acc: 0.9249 - val_loss: 2.5991 - val_acc: 0.4644
Performance of model on test set ----------------------------
Accuracy:
0.457172342621
Kappa:
0.305813421371
Confusion matrix:
[[ 73   6  17  21   5]
 [  3  96  98  20  45]
 [  8  18 176  23  18]
 [ 37  30  34  53  14]
 [  4  50  50  25  45]]
AUC score:
0.717191200676
('Epoch', 26, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.2623 - acc: 0.9300Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 79s - loss: 0.2622 - acc: 0.9301 - val_loss: 2.4574 - val_acc: 0.4623
Performance of model on test set ----------------------------
Accuracy:
0.460268317853
Kappa:
0.31829224329
Confusion matrix:
[[ 81   3  15  16   7]
 [  4  78  69  58  53]
 [ 12  29 171  18  13]
 [ 44  17  36  43  28]
 [ 12  18  44  27  73]]
AUC score:
0.734716951012
('Epoch', 27, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.2352 - acc: 0.9372Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 75s - loss: 0.2351 - acc: 0.9372 - val_loss: 2.5326 - val_acc: 0.4716
Performance of model on test set ----------------------------
Accuracy:
0.476780185759
Kappa:
0.342620989756
Confusion matrix:
[[ 96   1  10   9   6]
 [  8  80  77  43  54]
 [ 17  26 164  26  10]
 [ 57  16  19  50  26]
 [ 16  23  37  26  72]]
AUC score:
0.736181016801
('Epoch', 28, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.2137 - acc: 0.9403Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 74s - loss: 0.2137 - acc: 0.9403 - val_loss: 2.7601 - val_acc: 0.4293
Performance of model on test set ----------------------------
Accuracy:
0.427244582043
Kappa:
0.277780463632
Confusion matrix:
[[ 94   4   6   4  14]
 [  8 104  50  45  55]
 [ 18  74 120  19  12]
 [ 60  27  14  28  39]
 [ 12  40  21  33  68]]
AUC score:
0.71564146057
('Epoch', 29, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.1976 - acc: 0.9475Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 79s - loss: 0.1975 - acc: 0.9475 - val_loss: 2.5526 - val_acc: 0.4551
Performance of model on test set ----------------------------
Accuracy:
0.449948400413
Kappa:
0.307021333691
Confusion matrix:
[[ 90   6  11   7   8]
 [  7 114  32  29  80]
 [ 12  64 113  30  24]
 [ 59  25  14  32  38]
 [  8  38  21  20  87]]
AUC score:
0.74238387623
('Epoch', 30, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.2145 - acc: 0.9412Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 76s - loss: 0.2144 - acc: 0.9412 - val_loss: 3.0161 - val_acc: 0.4107
Performance of model on test set ----------------------------
Accuracy:
0.407636738906
Kappa:
0.248012568141
Confusion matrix:
[[ 80   3  18   4  17]
 [  5  97  65  17  78]
 [ 13  64 125  22  19]
 [ 51  29  21  19  48]
 [  5  38  44  13  74]]
AUC score:
0.691911408302
('Epoch', 31, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.1923 - acc: 0.9477Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 76s - loss: 0.1923 - acc: 0.9478 - val_loss: 3.3254 - val_acc: 0.3870
Performance of model on test set ----------------------------
Accuracy:
0.385964912281
Kappa:
0.219209804652
Confusion matrix:
[[ 80   4  14  16   8]
 [  3  91 101  22  45]
 [ 14  75 102  29  23]
 [ 44  20  23  35  46]
 [  7  31  62   8  66]]
AUC score:
0.670595952919
('Epoch', 32, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.1932 - acc: 0.9492Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 80s - loss: 0.1932 - acc: 0.9492 - val_loss: 3.1076 - val_acc: 0.3953
Performance of model on test set ----------------------------
Accuracy:
0.393188854489
Kappa:
0.233533546326
Confusion matrix:
[[82  7 16  9  8]
 [ 5 94 72 36 55]
 [14 77 91 42 19]
 [47 17 20 48 36]
 [ 8 19 51 30 66]]
AUC score:
0.67551089819
('Epoch', 33, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.1770 - acc: 0.9512Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 78s - loss: 0.1770 - acc: 0.9512 - val_loss: 3.0118 - val_acc: 0.4283
Performance of model on test set ----------------------------
Accuracy:
0.424148606811
Kappa:
0.268624838529
Confusion matrix:
[[ 81   2  19   9  11]
 [  4  85  91  26  56]
 [ 10  42 150  30  11]
 [ 44  19  29  31  45]
 [  8  33  51  18  64]]
AUC score:
0.697893063474
('Epoch', 34, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.1919 - acc: 0.9472Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 80s - loss: 0.1918 - acc: 0.9472 - val_loss: 2.8548 - val_acc: 0.4458
Performance of model on test set ----------------------------
Accuracy:
0.443756449948
Kappa:
0.300646742187
Confusion matrix:
[[ 98   4   8   6   6]
 [ 10  99  51  22  80]
 [ 14  55 126  27  21]
 [ 50  15  21  23  59]
 [ 12  29  25  24  84]]
AUC score:
0.728607362262
('Epoch', 35, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.1896 - acc: 0.9472Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 76s - loss: 0.1896 - acc: 0.9472 - val_loss: 2.8639 - val_acc: 0.4469
Performance of model on test set ----------------------------
Accuracy:
0.446852425181
Kappa:
0.301112280296
Confusion matrix:
[[ 93   2  14   5   8]
 [ 16 129  29  15  73]
 [ 16  70 119  26  12]
 [ 60  25  21  20  42]
 [ 12  48  18  24  72]]
AUC score:
0.735153043969
('Epoch', 36, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.1767 - acc: 0.9523Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 75s - loss: 0.1767 - acc: 0.9523 - val_loss: 2.9662 - val_acc: 0.4190
Performance of model on test set ----------------------------
Accuracy:
0.414860681115
Kappa:
0.258340285258
Confusion matrix:
[[ 81   2  27   8   4]
 [ 25 102  83  17  35]
 [ 15  74 109  31  14]
 [ 52  20  18  48  30]
 [ 13  39  40  20  62]]
AUC score:
0.716509742014
('Epoch', 37, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.1676 - acc: 0.9609Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 74s - loss: 0.1675 - acc: 0.9609 - val_loss: 3.0033 - val_acc: 0.4190
Performance of model on test set ----------------------------
Accuracy:
0.410732714138
Kappa:
0.246411493029
Confusion matrix:
[[ 77   2  24  10   9]
 [ 12 115  79   8  48]
 [ 14  87 107  22  13]
 [ 39  29  33  28  39]
 [  8  47  43   5  71]]
AUC score:
0.699252473417
('Epoch', 38, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.1655 - acc: 0.9572Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 75s - loss: 0.1655 - acc: 0.9572 - val_loss: 2.8768 - val_acc: 0.4396
Performance of model on test set ----------------------------
Accuracy:
0.43653250774
Kappa:
0.284735497303
Confusion matrix:
[[ 80   2  17  17   6]
 [ 10  85 103  10  54]
 [ 10  40 146  23  24]
 [ 35  18  33  37  45]
 [ 13  32  40  14  75]]
AUC score:
0.717027330355
('Epoch', 39, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.1443 - acc: 0.9597Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 79s - loss: 0.1443 - acc: 0.9597 - val_loss: 3.3124 - val_acc: 0.4149
Performance of model on test set ----------------------------
Accuracy:
0.410732714138
Kappa:
0.253119208221
Confusion matrix:
[[ 93   3  11   9   6]
 [ 13  81 114  12  42]
 [ 17  52 128  28  18]
 [ 41  23  30  30  44]
 [ 13  34  41  20  66]]
AUC score:
0.697017441994
('Epoch', 40, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.1620 - acc: 0.9572Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 76s - loss: 0.1620 - acc: 0.9572 - val_loss: 3.2233 - val_acc: 0.4045
Performance of model on test set ----------------------------
Accuracy:
0.399380804954
Kappa:
0.242569171075
Confusion matrix:
[[ 84   0  13  17   8]
 [ 13  72 101  20  56]
 [ 11  65 104  43  20]
 [ 39  22  19  48  40]
 [ 14  26  35  20  79]]
AUC score:
0.702387456147
('Epoch', 41, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.1660 - acc: 0.9532Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 73s - loss: 0.1660 - acc: 0.9532 - val_loss: 3.0240 - val_acc: 0.4252
Performance of model on test set ----------------------------
Accuracy:
0.422084623323
Kappa:
0.269892483245
Confusion matrix:
[[ 82   3  13  13  11]
 [ 12  86  89  15  60]
 [ 16  63 120  24  20]
 [ 46  20  17  35  50]
 [ 11  27  35  15  86]]
AUC score:
0.704468215194
('Epoch', 42, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.1678 - acc: 0.9560Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 74s - loss: 0.1677 - acc: 0.9560 - val_loss: 3.2347 - val_acc: 0.4241
Performance of model on test set ----------------------------
Accuracy:
0.420020639835
Kappa:
0.267296786389
Confusion matrix:
[[ 94   5   6  11   6]
 [ 18  71 109   6  58]
 [ 16  53 135  17  22]
 [ 44  22  21  32  49]
 [ 17  31  39  12  75]]
AUC score:
0.701138227437
('Epoch', 43, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.1431 - acc: 0.9657Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 77s - loss: 0.1430 - acc: 0.9657 - val_loss: 3.0899 - val_acc: 0.4469
Performance of model on test set ----------------------------
Accuracy:
0.437564499484
Kappa:
0.288552006208
Confusion matrix:
[[ 86   2  11  14   9]
 [ 16  68 116  15  47]
 [ 15  35 155  21  17]
 [ 43  23  22  43  37]
 [ 16  30  43  13  72]]
AUC score:
0.711970712901
('Epoch', 44, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.1434 - acc: 0.9632Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 74s - loss: 0.1434 - acc: 0.9632 - val_loss: 3.1590 - val_acc: 0.4685
Performance of model on test set ----------------------------
Accuracy:
0.465428276574
Kappa:
0.322269659826
Confusion matrix:
[[ 87   2  14  12   7]
 [ 18  92  91  11  50]
 [ 12  44 163   8  16]
 [ 57  21  35  34  21]
 [ 15  24  44  16  75]]
AUC score:
0.710476513411
('Epoch', 45, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.1470 - acc: 0.9629Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 79s - loss: 0.1469 - acc: 0.9629 - val_loss: 3.3880 - val_acc: 0.4180
Performance of model on test set ----------------------------
Accuracy:
0.41382868937
Kappa:
0.255978684777
Confusion matrix:
[[ 92   0  16   9   5]
 [ 16  71 116  10  49]
 [ 14  49 147  20  13]
 [ 57  27  32  34  18]
 [ 12  28  61  16  57]]
AUC score:
0.672356573874
('Epoch', 46, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.1335 - acc: 0.9683Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 75s - loss: 0.1334 - acc: 0.9683 - val_loss: 3.5998 - val_acc: 0.3983
Performance of model on test set ----------------------------
Accuracy:
0.398348813209
Kappa:
0.236434485723
Confusion matrix:
[[ 87   0  18   8   9]
 [ 12  81  96  19  54]
 [ 16  65 118  25  19]
 [ 46  30  34  33  25]
 [  8  29  51  19  67]]
AUC score:
0.682599329885
('Epoch', 47, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.1346 - acc: 0.9686Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 74s - loss: 0.1345 - acc: 0.9686 - val_loss: 2.9412 - val_acc: 0.4489
Performance of model on test set ----------------------------
Accuracy:
0.452012383901
Kappa:
0.307003248555
Confusion matrix:
[[ 87   4   9  11  11]
 [  9 125  43  21  64]
 [ 17  76 109  25  16]
 [ 49  27  23  44  25]
 [  9  36  19  37  73]]
AUC score:
0.723529399899
('Epoch', 48, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.1388 - acc: 0.9666Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 73s - loss: 0.1387 - acc: 0.9666 - val_loss: 3.3838 - val_acc: 0.4097
Performance of model on test set ----------------------------
Accuracy:
0.410732714138
Kappa:
0.255604168517
Confusion matrix:
[[ 90   1   8  16   7]
 [  8  88  81  30  55]
 [ 20  66 123  18  16]
 [ 52  19  39  33  25]
 [ 16  24  40  30  64]]
AUC score:
0.701055124781
('Epoch', 49, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.0901 - acc: 0.9769Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 75s - loss: 0.0901 - acc: 0.9769 - val_loss: 3.5546 - val_acc: 0.4314
Performance of model on test set ----------------------------
Accuracy:
0.429308565531
Kappa:
0.277630913785
Confusion matrix:
[[ 82   4  16  11   9]
 [  2  84  64  16  96]
 [ 10  67 126  26  14]
 [ 47  17  36  34  34]
 [  6  25  40  13  90]]
AUC score:
0.708887821817
('Epoch', 86, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.0948 - acc: 0.9734Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 78s - loss: 0.0948 - acc: 0.9735 - val_loss: 3.5034 - val_acc: 0.4499
Performance of model on test set ----------------------------
Accuracy:
0.444788441692
Kappa:
0.29603497936
Confusion matrix:
[[ 87   3  13  12   7]
 [  4  89  74  15  80]
 [ 10  58 138  21  16]
 [ 53  21  30  38  26]
 [  4  31  47  13  79]]
AUC score:
0.702348491006
('Epoch', 87, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.0603 - acc: 0.9849Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 76s - loss: 0.0603 - acc: 0.9849 - val_loss: 3.7005 - val_acc: 0.4572
Performance of model on test set ----------------------------
Accuracy:
0.448916408669
Kappa:
0.302762158469
Confusion matrix:
[[ 86   6   9  16   5]
 [  5  83  70  18  86]
 [  9  60 131  25  18]
 [ 44  19  26  50  29]
 [  6  33  40  10  85]]
AUC score:
0.709917452895
('Epoch', 88, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.0893 - acc: 0.9792Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 80s - loss: 0.0892 - acc: 0.9792 - val_loss: 3.4982 - val_acc: 0.4582
Performance of model on test set ----------------------------
Accuracy:
0.450980392157
Kappa:
0.306294255856
Confusion matrix:
[[ 80   1  16  14  11]
 [  6  83  54  20  99]
 [ 11  63 129  24  16]
 [ 41  18  22  53  34]
 [  5  37  34   6  92]]
AUC score:
0.729690534037
('Epoch', 89, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.0950 - acc: 0.9769Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 79s - loss: 0.0950 - acc: 0.9769 - val_loss: 3.6014 - val_acc: 0.4241
Performance of model on test set ----------------------------
Accuracy:
0.422084623323
Kappa:
0.265586195229
Confusion matrix:
[[ 84   5  16   9   8]
 [ 11  95  59  18  79]
 [ 11  82 118  20  12]
 [ 43  25  37  34  29]
 [  6  45  30  15  78]]
AUC score:
0.700645718717
('Epoch', 90, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.1143 - acc: 0.9740Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 74s - loss: 0.1143 - acc: 0.9740 - val_loss: 3.2175 - val_acc: 0.4696
Performance of model on test set ----------------------------
Accuracy:
0.470588235294
Kappa:
0.329044283884
Confusion matrix:
[[ 86   6   9  13   8]
 [  5  81  83   5  88]
 [ 14  33 166  14  16]
 [ 55  22  24  40  27]
 [  4  41  42   4  83]]
AUC score:
0.723035850452
('Epoch', 91, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.0923 - acc: 0.9777Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 78s - loss: 0.0923 - acc: 0.9777 - val_loss: 3.1113 - val_acc: 0.4799
Performance of model on test set ----------------------------
Accuracy:
0.477812177503
Kappa:
0.336182760469
Confusion matrix:
[[ 87   8  14  10   3]
 [  6 107  65  21  63]
 [ 10  59 142  20  12]
 [ 49  22  23  47  27]
 [  3  48  31  12  80]]
AUC score:
0.734920181861
('Epoch', 92, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.0957 - acc: 0.9774Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 79s - loss: 0.0957 - acc: 0.9774 - val_loss: 3.1677 - val_acc: 0.4871
Performance of model on test set ----------------------------
Accuracy:
0.482972136223
Kappa:
0.348723053456
Confusion matrix:
[[ 94   2  11   9   6]
 [ 12  79  69  13  89]
 [ 16  41 158  18  10]
 [ 57  19  15  48  29]
 [  7  38  29  11  89]]
AUC score:
0.740744312834
('Epoch', 93, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.0715 - acc: 0.9843Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 75s - loss: 0.0715 - acc: 0.9843 - val_loss: 3.4666 - val_acc: 0.4634
Performance of model on test set ----------------------------
Accuracy:
0.459236326109
Kappa:
0.317834831325
Confusion matrix:
[[ 88   5   8  12   9]
 [ 14  95  51  14  88]
 [ 16  65 130  18  14]
 [ 52  24  16  49  27]
 [  8  40  24  19  83]]
AUC score:
0.720746407954
('Epoch', 94, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.0793 - acc: 0.9803Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 77s - loss: 0.0793 - acc: 0.9803 - val_loss: 3.4702 - val_acc: 0.4747
Performance of model on test set ----------------------------
Accuracy:
0.46955624355
Kappa:
0.329625676511
Confusion matrix:
[[ 84   4  11  16   7]
 [  8  67  90  19  78]
 [ 10  30 170  19  14]
 [ 46  24  18  58  22]
 [  5  32  35  26  76]]
AUC score:
0.734806406736
('Epoch', 95, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.0771 - acc: 0.9814Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 76s - loss: 0.0771 - acc: 0.9814 - val_loss: 3.3863 - val_acc: 0.4757
Performance of model on test set ----------------------------
Accuracy:
0.471620227038
Kappa:
0.332851030183
Confusion matrix:
[[ 85   4  10  10  13]
 [ 10  80  80  13  79]
 [  9  43 143  25  23]
 [ 42  25  16  54  31]
 [  7  26  36  10  95]]
AUC score:
0.714918326632
('Epoch', 96, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.0836 - acc: 0.9812Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 80s - loss: 0.0835 - acc: 0.9812 - val_loss: 3.4266 - val_acc: 0.4561
Performance of model on test set ----------------------------
Accuracy:
0.450980392157
Kappa:
0.305018617932
Confusion matrix:
[[ 86   3  13  10  10]
 [ 18  83  71  11  79]
 [ 12  40 155  21  15]
 [ 47  26  28  40  27]
 [  6  42  36  17  73]]
AUC score:
0.72127870954
('Epoch', 97, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.0810 - acc: 0.9814Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 79s - loss: 0.0810 - acc: 0.9814 - val_loss: 3.5782 - val_acc: 0.4665
Performance of model on test set ----------------------------
Accuracy:
0.463364293086
Kappa:
0.316727484267
Confusion matrix:
[[ 86   4  10  14   8]
 [ 12  95  85  11  59]
 [ 10  49 151  20  13]
 [ 41  31  18  56  22]
 [  6  51  46  10  61]]
AUC score:
0.708913341916
('Epoch', 98, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.0773 - acc: 0.9832Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 77s - loss: 0.0773 - acc: 0.9832 - val_loss: 3.6385 - val_acc: 0.4499
Performance of model on test set ----------------------------
Accuracy:
0.440660474716
Kappa:
0.287797588375
Confusion matrix:
[[ 77   4  15  19   7]
 [  9  87  95  21  50]
 [  9  41 147  32  14]
 [ 35  25  27  57  24]
 [  4  42  53  16  59]]
AUC score:
0.690753100627
('Epoch', 99, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.0764 - acc: 0.9834Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 79s - loss: 0.0764 - acc: 0.9834 - val_loss: 3.5415 - val_acc: 0.4634
Performance of model on test set ----------------------------
Accuracy:
0.457172342621
Kappa:
0.317764565102
Confusion matrix:
[[ 90   2  11  10   9]
 [ 18  85  57  29  73]
 [ 17  36 142  33  15]
 [ 51  21  20  53  23]
 [  6  37  27  31  73]]
AUC score:
0.719833632243
('Epoch', 100, '/', 100)
Train on 3503 samples, validate on 969 samples
Epoch 1/1
3502/3503 [============================>.] - ETA: 0s - loss: 0.0619 - acc: 0.9869Epoch 00000: val_acc did not improve
3503/3503 [==============================] - 80s - loss: 0.0619 - acc: 0.9869 - val_loss: 3.6337 - val_acc: 0.4685
Performance of model on test set ----------------------------
Accuracy:
0.462332301342
Kappa:
0.321837845244
Confusion matrix:
[[ 86   3  16  13   4]
 [ 13  93  49  26  81]
 [ 13  45 138  28  19]
 [ 42  20  26  54  26]
 [  6  39  25  27  77]]
AUC score:
0.720621435659
Best validation accuracy:  (58, 0.50567595459236325)






    













    













    

In [ ]:

    

from keras.models import Sequential
from keras.layers import Dense
from keras.models import model_from_json
import numpy
import os

# LOAD AND USE MODEL
json_file3 = open('activity.model--3lstmbis.json','r')
loaded_model_json3 = json_file3.read()
json_file3.close()
loaded_model3 = model_from_json(loaded_model_json3)

# load weights into new model
loaded_model3.load_weights("activity.weights--3lstmbis.best.hdf5")
print("Loaded model 3 from disk")
# evaluate loaded model on test data
# IMPORTANT: compile the model again before use!
loaded_model3.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
score3 = loaded_model3.evaluate(testX, dummy_y_testA, batch_size=1, verbose=0)
print "3 Layer LSTM --- %s: %.2f%%" % (loaded_model3.metrics_names[1], score3[1]*100)
printValStats(loaded_model3,  testX, dummy_y_testA, batch=1)