In [1]:
import numpy
import pandas
from keras.models import Sequential
from keras.layers import Dense
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.cross_validation import cross_val_score
from sklearn.preprocessing import LabelEncoder
from sklearn.cross_validation import StratifiedKFold
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
# fix random seed for reproducibility
seed = 67
numpy.random.seed(seed)
data = pandas.read_csv("../data/processed/train.csv")
notnull_data = data[data.notnull().all(axis=1)]
train = notnull_data.values
data2 = pandas.read_csv("../data/processed/test.csv")
notnull_data2 = data2[data2.notnull().all(axis=1)]
test = notnull_data2.values
Using Theano backend.
In [2]:
X_train = train[:,3:7558].astype(float)
Y_trainA = train[:,7558] #Activity
Y_trainS = train[:,7559] #Social
X_test = test[:,3:7558].astype(float)
Y_testA = test[:,7558]
Y_testS = test[:,7559]
# One hot encoding of the response variable (using dummy variables)
from keras.utils.np_utils import to_categorical
# encode class values as integers
encoderA = LabelEncoder()
encoderA.fit(Y_trainA)
encoded_Y_trainA = encoderA.transform(Y_trainA)
# convert integers to dummy variables (i.e. one hot encoded)
dummy_y_trainA = to_categorical(encoded_Y_trainA)
encoderA.fit(Y_testA)
encoded_Y_testA = encoderA.transform(Y_testA)
# convert integers to dummy variables (i.e. one hot encoded)
dummy_y_testA = to_categorical(encoded_Y_testA)
# encode class values as integers
encoderS = LabelEncoder()
encoderS.fit(Y_trainS)
encoded_Y_trainS = encoderS.transform(Y_trainS)
# convert integers to dummy variables (i.e. one hot encoded)
dummy_y_trainS = to_categorical(encoded_Y_trainS)
encoderS.fit(Y_testS)
encoded_Y_testS = encoderS.transform(Y_testS)
# convert integers to dummy variables (i.e. one hot encoded)
dummy_y_testS = to_categorical(encoded_Y_testS)
# We standardize on the basis of the training data
scaler = StandardScaler().fit(X_train)
X_train_st = scaler.transform(X_train)
X_test_st = scaler.transform(X_test)
# Number of components to extract from the dataset
n_components = 100
from sklearn import decomposition
print 'Reducing dataset with PCA',n_components
pca = decomposition.PCA(n_components=n_components)
X_train_pca = pca.fit_transform(X_train_st)
X_test_pca = pca.transform(X_test_st)
#print 'Variance explained:'
#print pca.explained_variance_ratio_
print 'Total variance explained by %d components:',n_components
print sum(pca.explained_variance_ratio_)
trainX = numpy.reshape(X_train_pca, (X_train_pca.shape[0], 1, X_train_pca.shape[1]))
testX = numpy.reshape(X_test_pca, (X_test_pca.shape[0], 1, X_test_pca.shape[1]))
Reducing dataset with PCA 100
Total variance explained by %d components: 100
0.630815671837
In [3]:
from keras.layers import Dropout
from keras.layers import LSTM
from keras.constraints import maxnorm
from keras.optimizers import SGD
# This is our winning architecture so far
def create_LSTM3_PCA(n_outputs, batch_size = 1, trainShape1=100):
# create and fit the LSTM network
model = Sequential()
# stateful LSTM!
model.add(LSTM(200, batch_input_shape=(batch_size, 1, trainShape1),
return_sequences=True, stateful=True))
model.add(Dropout(0.2))
model.add(LSTM(100,
return_sequences=True, stateful=True))
model.add(Dropout(0.2))
model.add(LSTM(50,
return_sequences=False, stateful=True))
model.add(Dropout(0.2))
model.add(Dense(50, activation='tanh'))
model.add(Dropout(0.2))
model.add(Dense(20, activation='tanh'))
model.add(Dropout(0.2))
model.add(Dense(n_outputs, activation='softmax'))
# Compile model
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
return model
def create_LSTM2_PCA(n_outputs, batch_size = 1, trainShape1=100):
# create and fit the LSTM network
model = Sequential()
# stateful LSTM!
model.add(LSTM(300, batch_input_shape=(batch_size, 1, trainShape1),
return_sequences=True, stateful=True))
model.add(Dropout(0.2))
model.add(LSTM(50,
return_sequences=False, stateful=True))
model.add(Dropout(0.2))
model.add(Dense(50, activation='tanh'))
model.add(Dropout(0.2))
model.add(Dense(20, activation='tanh'))
model.add(Dropout(0.2))
model.add(Dense(n_outputs, activation='softmax'))
# Compile model
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
return model
def create_LSTM1_PCA(n_outputs, batch_size = 1, trainShape1=100):
# create and fit the LSTM network
model = Sequential()
# stateful LSTM!
model.add(LSTM(400, batch_input_shape=(batch_size, 1, trainShape1),
return_sequences=False, stateful=True))
model.add(Dropout(0.2))
model.add(Dense(50, activation='tanh'))
model.add(Dropout(0.2))
model.add(Dense(20, activation='tanh'))
model.add(Dropout(0.2))
model.add(Dense(n_outputs, activation='softmax'))
# Compile model
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
return model
from keras.callbacks import ModelCheckpoint
import matplotlib.pyplot as plt
from sklearn.metrics import confusion_matrix, roc_auc_score, accuracy_score, cohen_kappa_score
def printValStats(model, testX, dummy_y_test, batch=1):
# Other performance/accuracy metrics
Y_pred = model.predict(testX, batch_size=batch)
model.reset_states()
print 'Performance of model on test set ----------------------------'
# Accuracy
print('Accuracy:')
print(accuracy_score(numpy.argmax(dummy_y_test, axis=1), numpy.argmax(Y_pred, axis=1)))
# Kappa
print('Kappa:')
kappa = cohen_kappa_score(numpy.argmax(dummy_y_test, axis=1), numpy.argmax(Y_pred, axis=1))
print(kappa)
# Confusion matrix
cm = confusion_matrix(numpy.argmax(dummy_y_test, axis=1), numpy.argmax(Y_pred, axis=1))
numpy.set_printoptions(precision=2)
print('Confusion matrix:')
print(cm)
# AUC
roc = roc_auc_score(dummy_y_test, Y_pred, average='macro')
print('AUC score:')
print(roc)
return kappa, roc
def plot_training(accs, val_accs, losss, val_losss, kappas, aucs):
# summarize history for accuracy
plt.plot(accs)
plt.plot(val_accs)
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train','test'], loc='upper left')
plt.show()
# summarize history for loss
plt.plot(losss)
plt.plot(val_losss)
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train','test'], loc='upper left')
plt.show()
# summarize kappa and auc
plt.plot(kappas)
plt.plot(aucs)
plt.title('Other performance')
plt.ylabel('metric')
plt.xlabel('epoch')
plt.legend(['Kappa','AUC'], loc='upper left')
plt.show()
import operator
def get_max_values(list):
index, value = max(enumerate(list), key=operator.itemgetter(1))
return index, value
print 'Ready for training!'
/usr/local/lib/python2.7/dist-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.
warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')
Ready for training!
In [ ]:
# Create the model and parameters for training
numpy.random.seed(seed)
batch = 1
epochs = 100
modelS3 = create_LSTM3_PCA(dummy_y_trainS.shape[1], batch_size = batch, trainShape1=n_components)
print modelS3.summary()
# To save the best model
# serialize model to JSON
modelS3_json = modelS3.to_json()
with open("social.model--3lstmbis.json", "w") as json_file:
json_file.write(modelS3_json)
filepathS3="social.weights--3lstmbis.best.hdf5"
# Define that the accuracy in cv is monitored, and that weights are stored in a file when max accuracy is achieved
checkpointS3 = ModelCheckpoint(filepathS3, monitor='val_acc', verbose=1, save_best_only=True, mode='max')
callbacks_listS3 = [checkpointS3]
# Fit the model
accs =[]
val_accs =[]
losss =[]
val_losss =[]
kappas = []
aucs = []
# Manually create epochs and reset between sessions
for i in range(epochs):
# Single epoch. Remember to not shuffle the data!
print('Epoch', i+1, '/', epochs)
history = modelS3.fit(trainX, dummy_y_trainS, validation_data=(testX, dummy_y_testS),
nb_epoch=1, batch_size=batch, shuffle=False,
verbose=1, callbacks=callbacks_listS3)
modelS3.reset_states()
kappa, auc = printValStats(modelS3, testX, dummy_y_testS, batch=batch)
accs.append(history.history['acc'][0])
val_accs.append(history.history['val_acc'][0])
losss.append(history.history['loss'][0])
val_losss.append(history.history['val_loss'][0])
kappas.append(kappa)
aucs.append(auc)
print 'Best validation accuracy: ', get_max_values(val_accs)
plot_training(accs, val_accs, losss, val_losss, kappas, aucs)
____________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
====================================================================================================
lstm_1 (LSTM) (1, 1, 200) 240800 lstm_input_1[0][0]
____________________________________________________________________________________________________
dropout_1 (Dropout) (1, 1, 200) 0 lstm_1[0][0]
____________________________________________________________________________________________________
lstm_2 (LSTM) (1, 1, 100) 120400 dropout_1[0][0]
____________________________________________________________________________________________________
dropout_2 (Dropout) (1, 1, 100) 0 lstm_2[0][0]
____________________________________________________________________________________________________
lstm_3 (LSTM) (1, 50) 30200 dropout_2[0][0]
____________________________________________________________________________________________________
dropout_3 (Dropout) (1, 50) 0 lstm_3[0][0]
____________________________________________________________________________________________________
dense_1 (Dense) (1, 50) 2550 dropout_3[0][0]
____________________________________________________________________________________________________
dropout_4 (Dropout) (1, 50) 0 dense_1[0][0]
____________________________________________________________________________________________________
dense_2 (Dense) (1, 20) 1020 dropout_4[0][0]
____________________________________________________________________________________________________
dropout_5 (Dropout) (1, 20) 0 dense_2[0][0]
____________________________________________________________________________________________________
dense_3 (Dense) (1, 4) 84 dropout_5[0][0]
====================================================================================================
Total params: 395054
____________________________________________________________________________________________________
None
('Epoch', 1, '/', 100)
WARNING (theano.tensor.blas): We did not found a dynamic library into the library_dir of the library we use for blas. If you use ATLAS, make sure to compile it with dynamics library.
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.9178 - acc: 0.6442Epoch 00000: val_acc improved from -inf to 0.61590, saving model to social.weights--3lstmbis.best.hdf5
4472/4472 [==============================] - 74s - loss: 0.9179 - acc: 0.6440 - val_loss: 1.2108 - val_acc: 0.6159
Performance of model on test set ----------------------------
Accuracy:
0.614942528736
Kappa:
0.0206493254958
Confusion matrix:
[[642 0 5 0]
[218 0 0 1]
[ 23 0 0 1]
[138 0 16 0]]
AUC score:
0.565920196673
('Epoch', 2, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.7643 - acc: 0.7332Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 106s - loss: 0.7645 - acc: 0.7330 - val_loss: 1.3155 - val_acc: 0.5939
Performance of model on test set ----------------------------
Accuracy:
0.592911877395
Kappa:
0.14246784496
Confusion matrix:
[[612 0 28 7]
[134 0 75 10]
[ 18 0 6 0]
[111 0 42 1]]
AUC score:
0.653253856512
('Epoch', 3, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.6712 - acc: 0.7685Epoch 00000: val_acc improved from 0.61590 to 0.62069, saving model to social.weights--3lstmbis.best.hdf5
4472/4472 [==============================] - 115s - loss: 0.6713 - acc: 0.7683 - val_loss: 1.2572 - val_acc: 0.6207
Performance of model on test set ----------------------------
Accuracy:
0.619731800766
Kappa:
0.211507364272
Confusion matrix:
[[626 0 0 21]
[ 94 0 43 82]
[ 16 0 0 8]
[102 0 31 21]]
AUC score:
0.756358338882
('Epoch', 4, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.5947 - acc: 0.8135Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 116s - loss: 0.5946 - acc: 0.8135 - val_loss: 1.2053 - val_acc: 0.5785
Performance of model on test set ----------------------------
Accuracy:
0.577586206897
Kappa:
0.223698897608
Confusion matrix:
[[563 0 1 83]
[ 42 0 0 177]
[ 9 0 1 14]
[ 78 0 37 39]]
AUC score:
0.75570603511
('Epoch', 5, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.5299 - acc: 0.8369Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 117s - loss: 0.5299 - acc: 0.8370 - val_loss: 1.2391 - val_acc: 0.5776
Performance of model on test set ----------------------------
Accuracy:
0.580459770115
Kappa:
0.301543175462
Confusion matrix:
[[493 0 1 153]
[ 14 0 0 205]
[ 9 0 0 15]
[ 41 0 0 113]]
AUC score:
0.758055431814
('Epoch', 6, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.4749 - acc: 0.8537Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 74s - loss: 0.4748 - acc: 0.8538 - val_loss: 1.1929 - val_acc: 0.5891
Performance of model on test set ----------------------------
Accuracy:
0.58908045977
Kappa:
0.310566492567
Confusion matrix:
[[506 0 0 141]
[ 13 0 0 206]
[ 3 0 0 21]
[ 45 0 0 109]]
AUC score:
0.773285297389
('Epoch', 7, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.4500 - acc: 0.8624Epoch 00000: val_acc improved from 0.62069 to 0.65134, saving model to social.weights--3lstmbis.best.hdf5
4472/4472 [==============================] - 76s - loss: 0.4501 - acc: 0.8625 - val_loss: 1.0722 - val_acc: 0.6513
Performance of model on test set ----------------------------
Accuracy:
0.650383141762
Kappa:
0.269228630221
Confusion matrix:
[[622 0 0 25]
[ 94 0 0 125]
[ 14 0 0 10]
[ 97 0 0 57]]
AUC score:
0.806006921925
('Epoch', 8, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.4061 - acc: 0.8862Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 78s - loss: 0.4060 - acc: 0.8862 - val_loss: 1.2512 - val_acc: 0.6169
Performance of model on test set ----------------------------
Accuracy:
0.616858237548
Kappa:
0.341684191277
Confusion matrix:
[[527 0 0 120]
[ 26 0 0 193]
[ 8 0 0 16]
[ 37 0 0 117]]
AUC score:
0.762742364163
('Epoch', 9, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.3849 - acc: 0.8870Epoch 00000: val_acc improved from 0.65134 to 0.66188, saving model to social.weights--3lstmbis.best.hdf5
4472/4472 [==============================] - 80s - loss: 0.3848 - acc: 0.8871 - val_loss: 0.9987 - val_acc: 0.6619
Performance of model on test set ----------------------------
Accuracy:
0.663793103448
Kappa:
0.42894187719
Confusion matrix:
[[510 3 0 134]
[ 19 65 0 135]
[ 8 3 1 12]
[ 37 0 0 117]]
AUC score:
0.813903053697
('Epoch', 10, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.3564 - acc: 0.9032Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 83s - loss: 0.3563 - acc: 0.9032 - val_loss: 1.1583 - val_acc: 0.6025
Performance of model on test set ----------------------------
Accuracy:
0.603448275862
Kappa:
0.331629221041
Confusion matrix:
[[509 0 0 138]
[ 25 0 0 194]
[ 6 0 0 18]
[ 33 0 0 121]]
AUC score:
0.802900212063
('Epoch', 11, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.3151 - acc: 0.9148Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 84s - loss: 0.3151 - acc: 0.9148 - val_loss: 1.2242 - val_acc: 0.6216
Performance of model on test set ----------------------------
Accuracy:
0.624521072797
Kappa:
0.364211742993
Confusion matrix:
[[519 1 0 127]
[ 22 11 0 186]
[ 7 0 0 17]
[ 30 0 2 122]]
AUC score:
0.769104954466
('Epoch', 12, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.2705 - acc: 0.9300Epoch 00000: val_acc improved from 0.66188 to 0.78640, saving model to social.weights--3lstmbis.best.hdf5
4472/4472 [==============================] - 88s - loss: 0.2705 - acc: 0.9300 - val_loss: 0.8785 - val_acc: 0.7864
Performance of model on test set ----------------------------
Accuracy:
0.787356321839
Kappa:
0.590323367519
Confusion matrix:
[[590 27 1 29]
[ 69 130 0 20]
[ 10 7 1 6]
[ 48 1 4 101]]
AUC score:
0.821636801488
('Epoch', 13, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.2652 - acc: 0.9304Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 88s - loss: 0.2651 - acc: 0.9305 - val_loss: 1.0482 - val_acc: 0.7241
Performance of model on test set ----------------------------
Accuracy:
0.725095785441
Kappa:
0.506690989555
Confusion matrix:
[[546 9 0 92]
[ 38 102 0 79]
[ 9 7 0 8]
[ 44 1 0 109]]
AUC score:
0.797891591575
('Epoch', 14, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.2469 - acc: 0.9394Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 92s - loss: 0.2469 - acc: 0.9394 - val_loss: 0.9155 - val_acc: 0.7385
Performance of model on test set ----------------------------
Accuracy:
0.738505747126
Kappa:
0.542057441253
Confusion matrix:
[[527 28 0 92]
[ 25 137 0 57]
[ 8 15 0 1]
[ 40 1 6 107]]
AUC score:
0.812778758629
('Epoch', 15, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.2395 - acc: 0.9418Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 93s - loss: 0.2395 - acc: 0.9419 - val_loss: 0.8632 - val_acc: 0.7605
Performance of model on test set ----------------------------
Accuracy:
0.760536398467
Kappa:
0.553647011744
Confusion matrix:
[[564 21 0 62]
[ 48 137 0 34]
[ 9 13 0 2]
[ 56 1 4 93]]
AUC score:
0.837869303651
('Epoch', 16, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.2188 - acc: 0.9454Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 113s - loss: 0.2188 - acc: 0.9454 - val_loss: 0.8180 - val_acc: 0.7739
Performance of model on test set ----------------------------
Accuracy:
0.774904214559
Kappa:
0.583561067186
Confusion matrix:
[[568 16 1 62]
[ 54 132 0 33]
[ 8 9 1 6]
[ 39 6 1 108]]
AUC score:
0.754077413133
('Epoch', 17, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.2154 - acc: 0.9477Epoch 00000: val_acc improved from 0.78640 to 0.78736, saving model to social.weights--3lstmbis.best.hdf5
4472/4472 [==============================] - 113s - loss: 0.2153 - acc: 0.9477 - val_loss: 1.0201 - val_acc: 0.7874
Performance of model on test set ----------------------------
Accuracy:
0.788314176245
Kappa:
0.575158861537
Confusion matrix:
[[610 22 1 14]
[ 80 132 1 6]
[ 9 11 0 4]
[ 65 0 8 81]]
AUC score:
0.832339768188
('Epoch', 18, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.2010 - acc: 0.9537Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 119s - loss: 0.2009 - acc: 0.9537 - val_loss: 1.1119 - val_acc: 0.7462
Performance of model on test set ----------------------------
Accuracy:
0.746168582375
Kappa:
0.491148529966
Confusion matrix:
[[598 31 2 16]
[ 74 134 2 9]
[ 7 13 0 4]
[ 86 0 21 47]]
AUC score:
0.810773193842
('Epoch', 19, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.2062 - acc: 0.9486Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 109s - loss: 0.2061 - acc: 0.9486 - val_loss: 0.9504 - val_acc: 0.7519
Performance of model on test set ----------------------------
Accuracy:
0.750957854406
Kappa:
0.526362901917
Confusion matrix:
[[563 61 1 22]
[ 63 149 0 7]
[ 8 13 0 3]
[ 62 6 14 72]]
AUC score:
0.751583139704
('Epoch', 20, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1840 - acc: 0.9586Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 110s - loss: 0.1839 - acc: 0.9586 - val_loss: 1.1552 - val_acc: 0.7500
Performance of model on test set ----------------------------
Accuracy:
0.749042145594
Kappa:
0.517745501476
Confusion matrix:
[[572 46 1 28]
[ 70 139 1 9]
[ 9 10 0 5]
[ 62 4 17 71]]
AUC score:
0.80975782697
('Epoch', 21, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1840 - acc: 0.9566Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 113s - loss: 0.1840 - acc: 0.9566 - val_loss: 1.1988 - val_acc: 0.7433
Performance of model on test set ----------------------------
Accuracy:
0.744252873563
Kappa:
0.513177910082
Confusion matrix:
[[572 27 2 46]
[ 66 118 1 34]
[ 9 8 0 7]
[ 59 0 8 87]]
AUC score:
0.817907147713
('Epoch', 22, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1697 - acc: 0.9557Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 115s - loss: 0.1697 - acc: 0.9557 - val_loss: 1.1400 - val_acc: 0.7672
Performance of model on test set ----------------------------
Accuracy:
0.76724137931
Kappa:
0.533903738993
Confusion matrix:
[[601 24 2 20]
[ 82 128 1 8]
[ 10 12 0 2]
[ 70 0 12 72]]
AUC score:
0.79246733524
('Epoch', 23, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1617 - acc: 0.9593Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 111s - loss: 0.1617 - acc: 0.9593 - val_loss: 1.1012 - val_acc: 0.7529
Performance of model on test set ----------------------------
Accuracy:
0.752873563218
Kappa:
0.519402196798
Confusion matrix:
[[573 38 0 36]
[ 74 139 0 6]
[ 11 10 0 3]
[ 68 1 11 74]]
AUC score:
0.815951527262
('Epoch', 24, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1600 - acc: 0.9597Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 111s - loss: 0.1600 - acc: 0.9597 - val_loss: 1.0082 - val_acc: 0.7730
Performance of model on test set ----------------------------
Accuracy:
0.772988505747
Kappa:
0.56088055315
Confusion matrix:
[[586 24 1 36]
[ 67 127 1 24]
[ 13 8 0 3]
[ 56 1 3 94]]
AUC score:
0.807419188812
('Epoch', 25, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1427 - acc: 0.9653Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 113s - loss: 0.1427 - acc: 0.9653 - val_loss: 1.1444 - val_acc: 0.7605
Performance of model on test set ----------------------------
Accuracy:
0.761494252874
Kappa:
0.537615148869
Confusion matrix:
[[583 25 0 39]
[ 69 136 0 14]
[ 11 8 0 5]
[ 63 0 15 76]]
AUC score:
0.819992021813
('Epoch', 26, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1380 - acc: 0.9662Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 116s - loss: 0.1380 - acc: 0.9662 - val_loss: 1.1724 - val_acc: 0.7490
Performance of model on test set ----------------------------
Accuracy:
0.749042145594
Kappa:
0.514036424882
Confusion matrix:
[[576 31 4 36]
[ 74 132 4 9]
[ 9 8 0 7]
[ 67 0 13 74]]
AUC score:
0.811063716088
('Epoch', 27, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1349 - acc: 0.9698Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 113s - loss: 0.1349 - acc: 0.9698 - val_loss: 1.0772 - val_acc: 0.7625
Performance of model on test set ----------------------------
Accuracy:
0.765325670498
Kappa:
0.544305421194
Confusion matrix:
[[583 37 1 26]
[ 67 141 3 8]
[ 11 8 0 5]
[ 65 1 13 75]]
AUC score:
0.803011749828
('Epoch', 28, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1378 - acc: 0.9696Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 109s - loss: 0.1378 - acc: 0.9696 - val_loss: 1.1846 - val_acc: 0.7510
Performance of model on test set ----------------------------
Accuracy:
0.751915708812
Kappa:
0.527209462945
Confusion matrix:
[[570 28 8 41]
[ 71 135 4 9]
[ 9 8 2 5]
[ 60 0 16 78]]
AUC score:
0.83061394754
('Epoch', 29, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1374 - acc: 0.9694Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 110s - loss: 0.1374 - acc: 0.9694 - val_loss: 1.0562 - val_acc: 0.7701
Performance of model on test set ----------------------------
Accuracy:
0.771072796935
Kappa:
0.555525470587
Confusion matrix:
[[582 27 0 38]
[ 69 134 2 14]
[ 9 9 0 6]
[ 64 1 0 89]]
AUC score:
0.84044752787
('Epoch', 30, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1238 - acc: 0.9716Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 108s - loss: 0.1238 - acc: 0.9716 - val_loss: 1.1390 - val_acc: 0.7625
Performance of model on test set ----------------------------
Accuracy:
0.763409961686
Kappa:
0.558237940404
Confusion matrix:
[[565 45 1 36]
[ 53 147 3 16]
[ 8 8 1 7]
[ 52 6 12 84]]
AUC score:
0.825304932276
('Epoch', 31, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1131 - acc: 0.9741Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 109s - loss: 0.1131 - acc: 0.9741 - val_loss: 1.1632 - val_acc: 0.7682
Performance of model on test set ----------------------------
Accuracy:
0.770114942529
Kappa:
0.547994876698
Confusion matrix:
[[594 39 0 14]
[ 68 139 2 10]
[ 9 8 0 7]
[ 69 0 14 71]]
AUC score:
0.811879755672
('Epoch', 32, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0967 - acc: 0.9796Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 113s - loss: 0.0967 - acc: 0.9797 - val_loss: 1.1842 - val_acc: 0.7663
Performance of model on test set ----------------------------
Accuracy:
0.766283524904
Kappa:
0.554932786353
Confusion matrix:
[[580 42 4 21]
[ 63 142 4 10]
[ 10 7 1 6]
[ 54 0 23 77]]
AUC score:
0.823253369455
('Epoch', 33, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1265 - acc: 0.9732Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 109s - loss: 0.1265 - acc: 0.9732 - val_loss: 1.0611 - val_acc: 0.7634
Performance of model on test set ----------------------------
Accuracy:
0.763409961686
Kappa:
0.563716466798
Confusion matrix:
[[556 46 3 42]
[ 48 155 3 13]
[ 8 12 0 4]
[ 51 2 15 86]]
AUC score:
0.818255073783
('Epoch', 34, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1109 - acc: 0.9734Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 108s - loss: 0.1108 - acc: 0.9734 - val_loss: 1.3336 - val_acc: 0.7596
Performance of model on test set ----------------------------
Accuracy:
0.757662835249
Kappa:
0.529025355729
Confusion matrix:
[[592 32 2 21]
[ 67 144 4 4]
[ 8 8 0 8]
[ 63 10 26 55]]
AUC score:
0.803446930513
('Epoch', 35, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1091 - acc: 0.9749Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 107s - loss: 0.1092 - acc: 0.9750 - val_loss: 1.1925 - val_acc: 0.7557
Performance of model on test set ----------------------------
Accuracy:
0.754789272031
Kappa:
0.531198035432
Confusion matrix:
[[584 33 4 26]
[ 67 139 5 8]
[ 7 8 0 9]
[ 57 1 31 65]]
AUC score:
0.819046206064
('Epoch', 36, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1060 - acc: 0.9738Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 111s - loss: 0.1060 - acc: 0.9738 - val_loss: 1.3015 - val_acc: 0.7538
Performance of model on test set ----------------------------
Accuracy:
0.753831417625
Kappa:
0.510320736742
Confusion matrix:
[[602 39 0 6]
[ 72 146 1 0]
[ 11 9 1 3]
[ 73 4 39 38]]
AUC score:
0.82321005354
('Epoch', 37, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1238 - acc: 0.9700Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 104s - loss: 0.1238 - acc: 0.9700 - val_loss: 1.2258 - val_acc: 0.7538
Performance of model on test set ----------------------------
Accuracy:
0.755747126437
Kappa:
0.517504114137
Confusion matrix:
[[600 30 3 14]
[ 80 134 2 3]
[ 11 9 0 4]
[ 62 2 35 55]]
AUC score:
0.80594120376
('Epoch', 38, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0917 - acc: 0.9772Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 103s - loss: 0.0917 - acc: 0.9772 - val_loss: 1.2920 - val_acc: 0.7605
Performance of model on test set ----------------------------
Accuracy:
0.761494252874
Kappa:
0.539637704387
Confusion matrix:
[[593 21 1 32]
[ 71 133 3 12]
[ 12 7 0 5]
[ 50 6 29 69]]
AUC score:
0.807936662349
('Epoch', 39, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0893 - acc: 0.9796Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 138s - loss: 0.0892 - acc: 0.9797 - val_loss: 1.3153 - val_acc: 0.7768
Performance of model on test set ----------------------------
Accuracy:
0.778735632184
Kappa:
0.557229147924
Confusion matrix:
[[614 30 1 2]
[ 78 137 4 0]
[ 12 9 0 3]
[ 61 3 28 62]]
AUC score:
0.802950794194
('Epoch', 40, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0968 - acc: 0.9770Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 150s - loss: 0.0968 - acc: 0.9770 - val_loss: 1.2846 - val_acc: 0.7519
Performance of model on test set ----------------------------
Accuracy:
0.752873563218
Kappa:
0.513213705458
Confusion matrix:
[[601 30 2 14]
[ 78 132 4 5]
[ 10 8 0 6]
[ 62 2 37 53]]
AUC score:
0.80134910035
('Epoch', 41, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0909 - acc: 0.9803Epoch 00000: val_acc improved from 0.78736 to 0.79023, saving model to social.weights--3lstmbis.best.hdf5
4472/4472 [==============================] - 145s - loss: 0.0909 - acc: 0.9803 - val_loss: 1.1704 - val_acc: 0.7902
Performance of model on test set ----------------------------
Accuracy:
0.79214559387
Kappa:
0.578049829859
Confusion matrix:
[[616 27 1 3]
[ 82 133 3 1]
[ 14 9 0 1]
[ 64 4 8 78]]
AUC score:
0.822886620555
('Epoch', 42, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0813 - acc: 0.9837Epoch 00000: val_acc improved from 0.79023 to 0.80651, saving model to social.weights--3lstmbis.best.hdf5
4472/4472 [==============================] - 154s - loss: 0.0813 - acc: 0.9837 - val_loss: 1.0560 - val_acc: 0.8065
Performance of model on test set ----------------------------
Accuracy:
0.808429118774
Kappa:
0.628774921684
Confusion matrix:
[[601 30 1 15]
[ 71 137 2 9]
[ 11 8 0 5]
[ 38 8 2 106]]
AUC score:
0.829646133854
('Epoch', 43, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0893 - acc: 0.9805Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 145s - loss: 0.0893 - acc: 0.9805 - val_loss: 1.1048 - val_acc: 0.7969
Performance of model on test set ----------------------------
Accuracy:
0.7969348659
Kappa:
0.616435887216
Confusion matrix:
[[580 40 2 25]
[ 63 144 2 10]
[ 7 11 3 3]
[ 40 7 2 105]]
AUC score:
0.78125079156
('Epoch', 44, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0911 - acc: 0.9794Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 148s - loss: 0.0911 - acc: 0.9794 - val_loss: 1.0518 - val_acc: 0.7902
Performance of model on test set ----------------------------
Accuracy:
0.791187739464
Kappa:
0.608505836496
Confusion matrix:
[[582 32 4 29]
[ 55 137 3 24]
[ 9 8 1 6]
[ 40 1 7 106]]
AUC score:
0.794613476629
('Epoch', 45, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0695 - acc: 0.9834Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 147s - loss: 0.0695 - acc: 0.9835 - val_loss: 1.2506 - val_acc: 0.7864
Performance of model on test set ----------------------------
Accuracy:
0.788314176245
Kappa:
0.58279568337
Confusion matrix:
[[609 23 8 7]
[ 73 132 4 10]
[ 11 9 1 3]
[ 54 7 12 81]]
AUC score:
0.815885613831
('Epoch', 46, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0724 - acc: 0.9832Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 153s - loss: 0.0724 - acc: 0.9832 - val_loss: 1.2419 - val_acc: 0.7835
Performance of model on test set ----------------------------
Accuracy:
0.784482758621
Kappa:
0.572627802299
Confusion matrix:
[[606 28 6 7]
[ 73 136 7 3]
[ 11 8 0 5]
[ 64 0 13 77]]
AUC score:
0.827430576762
('Epoch', 47, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0836 - acc: 0.9828Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 152s - loss: 0.0836 - acc: 0.9828 - val_loss: 1.1235 - val_acc: 0.7826
Performance of model on test set ----------------------------
Accuracy:
0.784482758621
Kappa:
0.59020254357
Confusion matrix:
[[587 20 1 39]
[ 64 133 4 18]
[ 10 5 0 9]
[ 44 1 10 99]]
AUC score:
0.823587992044
('Epoch', 48, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0746 - acc: 0.9843Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 153s - loss: 0.0746 - acc: 0.9843 - val_loss: 1.2438 - val_acc: 0.7778
Performance of model on test set ----------------------------
Accuracy:
0.77969348659
Kappa:
0.562488270524
Confusion matrix:
[[607 25 1 14]
[ 75 129 4 11]
[ 9 8 2 5]
[ 62 8 8 76]]
AUC score:
0.826213822967
('Epoch', 49, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0742 - acc: 0.9843Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 160s - loss: 0.0742 - acc: 0.9843 - val_loss: 1.1745 - val_acc: 0.7845
Performance of model on test set ----------------------------
Accuracy:
0.785440613027
Kappa:
0.575936053849
Confusion matrix:
[[604 33 2 8]
[ 74 133 4 8]
[ 8 8 6 2]
[ 61 9 7 77]]
AUC score:
0.848021452977
('Epoch', 50, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0872 - acc: 0.9812Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 158s - loss: 0.0872 - acc: 0.9812 - val_loss: 1.1351 - val_acc: 0.7960
Performance of model on test set ----------------------------
Accuracy:
0.7969348659
Kappa:
0.602272135077
Confusion matrix:
[[598 30 4 15]
[ 68 146 0 5]
[ 8 10 2 4]
[ 59 6 3 86]]
AUC score:
0.806662918312
('Epoch', 51, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0778 - acc: 0.9823Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 154s - loss: 0.0778 - acc: 0.9823 - val_loss: 1.2061 - val_acc: 0.7893
Performance of model on test set ----------------------------
Accuracy:
0.790229885057
Kappa:
0.577761156819
Confusion matrix:
[[610 29 1 7]
[ 83 135 1 0]
[ 8 11 1 4]
[ 64 5 6 79]]
AUC score:
0.761931005276
('Epoch', 52, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0561 - acc: 0.9886Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 152s - loss: 0.0561 - acc: 0.9886 - val_loss: 1.2607 - val_acc: 0.7874
Performance of model on test set ----------------------------
Accuracy:
0.788314176245
Kappa:
0.57211130996
Confusion matrix:
[[614 25 0 8]
[ 77 137 2 3]
[ 10 12 0 2]
[ 69 5 8 72]]
AUC score:
0.821906162135
('Epoch', 53, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0637 - acc: 0.9868Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 147s - loss: 0.0637 - acc: 0.9868 - val_loss: 1.3321 - val_acc: 0.7845
Performance of model on test set ----------------------------
Accuracy:
0.784482758621
Kappa:
0.576671328268
Confusion matrix:
[[600 31 1 15]
[ 68 142 0 9]
[ 9 12 0 3]
[ 62 0 15 77]]
AUC score:
0.818159907613
('Epoch', 54, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0613 - acc: 0.9868Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 147s - loss: 0.0613 - acc: 0.9868 - val_loss: 1.3508 - val_acc: 0.7931
Performance of model on test set ----------------------------
Accuracy:
0.79214559387
Kappa:
0.576742787002
Confusion matrix:
[[621 19 1 6]
[ 85 131 1 2]
[ 12 9 2 1]
[ 64 4 13 73]]
AUC score:
0.801971159281
('Epoch', 55, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0639 - acc: 0.9873Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 147s - loss: 0.0639 - acc: 0.9873 - val_loss: 1.1304 - val_acc: 0.7989
Performance of model on test set ----------------------------
Accuracy:
0.798850574713
Kappa:
0.608019824285
Confusion matrix:
[[601 32 6 8]
[ 66 148 3 2]
[ 9 8 3 4]
[ 56 2 14 82]]
AUC score:
0.798735888083
('Epoch', 56, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0736 - acc: 0.9837Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 156s - loss: 0.0736 - acc: 0.9837 - val_loss: 1.2734 - val_acc: 0.7931
Performance of model on test set ----------------------------
Accuracy:
0.794061302682
Kappa:
0.59372873257
Confusion matrix:
[[608 29 0 10]
[ 73 131 1 14]
[ 11 7 1 5]
[ 52 5 8 89]]
AUC score:
0.799568581921
('Epoch', 57, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0652 - acc: 0.9873Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 161s - loss: 0.0652 - acc: 0.9873 - val_loss: 1.2438 - val_acc: 0.7950
Performance of model on test set ----------------------------
Accuracy:
0.794061302682
Kappa:
0.600563759249
Confusion matrix:
[[594 40 0 13]
[ 58 153 1 7]
[ 10 9 2 3]
[ 59 5 10 80]]
AUC score:
0.809957409647
('Epoch', 58, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0713 - acc: 0.9859Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 152s - loss: 0.0712 - acc: 0.9859 - val_loss: 1.3587 - val_acc: 0.7663
Performance of model on test set ----------------------------
Accuracy:
0.766283524904
Kappa:
0.536913520323
Confusion matrix:
[[592 43 0 12]
[ 62 147 1 9]
[ 10 9 0 5]
[ 80 6 7 61]]
AUC score:
0.800738266451
('Epoch', 59, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0664 - acc: 0.9868Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 154s - loss: 0.0664 - acc: 0.9868 - val_loss: 1.4051 - val_acc: 0.7672
Performance of model on test set ----------------------------
Accuracy:
0.76724137931
Kappa:
0.544818082972
Confusion matrix:
[[594 36 1 16]
[ 66 143 3 7]
[ 10 7 1 6]
[ 66 4 21 63]]
AUC score:
0.795222493898
('Epoch', 60, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0655 - acc: 0.9873Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 154s - loss: 0.0655 - acc: 0.9873 - val_loss: 1.2580 - val_acc: 0.7854
Performance of model on test set ----------------------------
Accuracy:
0.786398467433
Kappa:
0.585924285676
Confusion matrix:
[[593 30 1 23]
[ 70 134 3 12]
[ 8 8 3 5]
[ 54 2 7 91]]
AUC score:
0.831372077127
('Epoch', 61, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0659 - acc: 0.9868Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 153s - loss: 0.0659 - acc: 0.9868 - val_loss: 1.3296 - val_acc: 0.7759
Performance of model on test set ----------------------------
Accuracy:
0.776819923372
Kappa:
0.564561602615
Confusion matrix:
[[594 33 2 18]
[ 67 140 3 9]
[ 8 7 2 7]
[ 62 8 9 75]]
AUC score:
0.802566855521
('Epoch', 62, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0609 - acc: 0.9886Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 152s - loss: 0.0609 - acc: 0.9886 - val_loss: 1.4385 - val_acc: 0.7701
Performance of model on test set ----------------------------
Accuracy:
0.771072796935
Kappa:
0.55211068888
Confusion matrix:
[[594 35 1 17]
[ 72 140 0 7]
[ 8 9 0 7]
[ 60 7 16 71]]
AUC score:
0.783187239577
('Epoch', 63, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0740 - acc: 0.9843Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 152s - loss: 0.0739 - acc: 0.9843 - val_loss: 1.4442 - val_acc: 0.7625
Performance of model on test set ----------------------------
Accuracy:
0.763409961686
Kappa:
0.528222353948
Confusion matrix:
[[602 37 0 8]
[ 77 137 4 1]
[ 10 9 3 2]
[ 70 6 23 55]]
AUC score:
0.81350021445
('Epoch', 64, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0551 - acc: 0.9895Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 152s - loss: 0.0551 - acc: 0.9895 - val_loss: 1.3016 - val_acc: 0.7893
Performance of model on test set ----------------------------
Accuracy:
0.791187739464
Kappa:
0.585360299222
Confusion matrix:
[[602 38 0 7]
[ 72 142 3 2]
[ 9 9 1 5]
[ 65 4 4 81]]
AUC score:
0.806450282049
('Epoch', 65, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0672 - acc: 0.9850Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 154s - loss: 0.0671 - acc: 0.9850 - val_loss: 1.3121 - val_acc: 0.7797
Performance of model on test set ----------------------------
Accuracy:
0.781609195402
Kappa:
0.568058010358
Confusion matrix:
[[600 35 1 11]
[ 74 139 2 4]
[ 9 8 2 5]
[ 64 3 12 75]]
AUC score:
0.801317454971
('Epoch', 66, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0538 - acc: 0.9888Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 155s - loss: 0.0538 - acc: 0.9888 - val_loss: 1.3960 - val_acc: 0.7797
Performance of model on test set ----------------------------
Accuracy:
0.782567049808
Kappa:
0.575943126978
Confusion matrix:
[[599 37 3 8]
[ 68 144 2 5]
[ 9 7 3 5]
[ 56 9 18 71]]
AUC score:
0.80726327669
('Epoch', 67, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0481 - acc: 0.9893Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 155s - loss: 0.0481 - acc: 0.9893 - val_loss: 1.4676 - val_acc: 0.7787
Performance of model on test set ----------------------------
Accuracy:
0.77969348659
Kappa:
0.563260161404
Confusion matrix:
[[600 36 0 11]
[ 73 142 1 3]
[ 9 8 1 6]
[ 66 5 12 71]]
AUC score:
0.745445203966
('Epoch', 68, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0678 - acc: 0.9881Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 121s - loss: 0.0678 - acc: 0.9881 - val_loss: 1.4354 - val_acc: 0.7644
Performance of model on test set ----------------------------
Accuracy:
0.766283524904
Kappa:
0.526280554027
Confusion matrix:
[[608 35 0 4]
[ 73 142 1 3]
[ 12 9 1 2]
[ 80 9 16 49]]
AUC score:
0.696322260343
('Epoch', 69, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0618 - acc: 0.9868Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 109s - loss: 0.0618 - acc: 0.9868 - val_loss: 1.4869 - val_acc: 0.7692
Performance of model on test set ----------------------------
Accuracy:
0.772988505747
Kappa:
0.543074314966
Confusion matrix:
[[608 23 0 16]
[ 87 121 7 4]
[ 14 4 3 3]
[ 64 1 14 75]]
AUC score:
0.782140188085
('Epoch', 70, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0585 - acc: 0.9875Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 108s - loss: 0.0585 - acc: 0.9875 - val_loss: 1.4923 - val_acc: 0.7720
Performance of model on test set ----------------------------
Accuracy:
0.772030651341
Kappa:
0.549409996065
Confusion matrix:
[[593 35 2 17]
[ 75 139 3 2]
[ 12 6 4 2]
[ 66 9 9 70]]
AUC score:
0.805364729926
('Epoch', 71, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0590 - acc: 0.9888Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 112s - loss: 0.0590 - acc: 0.9888 - val_loss: 1.4971 - val_acc: 0.7720
Performance of model on test set ----------------------------
Accuracy:
0.772030651341
Kappa:
0.547774746425
Confusion matrix:
[[602 32 3 10]
[ 67 143 5 4]
[ 12 5 4 3]
[ 73 5 19 57]]
AUC score:
0.772080405041
('Epoch', 72, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0668 - acc: 0.9881Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 110s - loss: 0.0668 - acc: 0.9881 - val_loss: 1.4234 - val_acc: 0.7749
Performance of model on test set ----------------------------
Accuracy:
0.775862068966
Kappa:
0.557825690332
Confusion matrix:
[[596 35 2 14]
[ 71 143 2 3]
[ 9 7 4 4]
[ 68 7 12 67]]
AUC score:
0.796108978415
('Epoch', 73, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0669 - acc: 0.9868Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 110s - loss: 0.0669 - acc: 0.9868 - val_loss: 1.3023 - val_acc: 0.7672
Performance of model on test set ----------------------------
Accuracy:
0.769157088123
Kappa:
0.554583065574
Confusion matrix:
[[577 53 4 13]
[ 60 155 3 1]
[ 10 7 4 3]
[ 67 12 8 67]]
AUC score:
0.804279122983
('Epoch', 74, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0623 - acc: 0.9866Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 112s - loss: 0.0623 - acc: 0.9866 - val_loss: 1.3974 - val_acc: 0.7701
Performance of model on test set ----------------------------
Accuracy:
0.770114942529
Kappa:
0.548638945383
Confusion matrix:
[[596 35 2 14]
[ 67 138 4 10]
[ 9 7 2 6]
[ 66 11 9 68]]
AUC score:
0.804560364242
('Epoch', 75, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0645 - acc: 0.9861Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 109s - loss: 0.0645 - acc: 0.9861 - val_loss: 1.2756 - val_acc: 0.7749
Performance of model on test set ----------------------------
Accuracy:
0.774904214559
Kappa:
0.562934789386
Confusion matrix:
[[592 35 3 17]
[ 59 145 2 13]
[ 11 5 1 7]
[ 63 9 11 71]]
AUC score:
0.795626985297
('Epoch', 76, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0580 - acc: 0.9879Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 111s - loss: 0.0580 - acc: 0.9879 - val_loss: 1.4459 - val_acc: 0.7625
Performance of model on test set ----------------------------
Accuracy:
0.763409961686
Kappa:
0.528629400797
Confusion matrix:
[[595 47 0 5]
[ 68 146 3 2]
[ 12 8 1 3]
[ 75 15 9 55]]
AUC score:
0.791502887849
('Epoch', 77, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0621 - acc: 0.9859Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 110s - loss: 0.0621 - acc: 0.9859 - val_loss: 1.2988 - val_acc: 0.7787
Performance of model on test set ----------------------------
Accuracy:
0.77969348659
Kappa:
0.559580122779
Confusion matrix:
[[602 37 2 6]
[ 72 140 2 5]
[ 13 7 1 3]
[ 69 9 5 71]]
AUC score:
0.782060082859
('Epoch', 78, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0552 - acc: 0.9906Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 113s - loss: 0.0552 - acc: 0.9906 - val_loss: 1.3067 - val_acc: 0.7816
Performance of model on test set ----------------------------
Accuracy:
0.781609195402
Kappa:
0.578621640712
Confusion matrix:
[[592 40 2 13]
[ 62 144 4 9]
[ 8 11 3 2]
[ 55 11 11 77]]
AUC score:
0.815071212004
('Epoch', 79, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0532 - acc: 0.9895Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 110s - loss: 0.0532 - acc: 0.9895 - val_loss: 1.2370 - val_acc: 0.7797
Performance of model on test set ----------------------------
Accuracy:
0.780651340996
Kappa:
0.568471231235
Confusion matrix:
[[603 32 2 10]
[ 69 138 5 7]
[ 10 7 1 6]
[ 61 5 15 73]]
AUC score:
0.788179308491
('Epoch', 92, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0539 - acc: 0.9902Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 115s - loss: 0.0539 - acc: 0.9902 - val_loss: 1.2153 - val_acc: 0.7787
Performance of model on test set ----------------------------
Accuracy:
0.77969348659
Kappa:
0.571957245638
Confusion matrix:
[[597 38 2 10]
[ 57 144 5 13]
[ 9 10 2 3]
[ 63 8 12 71]]
AUC score:
0.829628355179
('Epoch', 93, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0493 - acc: 0.9906Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 116s - loss: 0.0493 - acc: 0.9906 - val_loss: 1.2261 - val_acc: 0.7835
Performance of model on test set ----------------------------
Accuracy:
0.783524904215
Kappa:
0.58739868321
Confusion matrix:
[[588 44 6 9]
[ 57 147 4 11]
[ 9 9 2 4]
[ 51 6 16 81]]
AUC score:
0.793141844555
('Epoch', 94, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0380 - acc: 0.9933Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 116s - loss: 0.0380 - acc: 0.9933 - val_loss: 1.4979 - val_acc: 0.7586
Performance of model on test set ----------------------------
Accuracy:
0.758620689655
Kappa:
0.540514626134
Confusion matrix:
[[583 44 7 13]
[ 56 143 4 16]
[ 9 10 0 5]
[ 55 15 18 66]]
AUC score:
0.76632709991
('Epoch', 95, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0410 - acc: 0.9919Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 112s - loss: 0.0410 - acc: 0.9919 - val_loss: 1.4890 - val_acc: 0.7663
Performance of model on test set ----------------------------
Accuracy:
0.768199233716
Kappa:
0.55453290382
Confusion matrix:
[[595 36 7 9]
[ 63 136 5 15]
[ 13 5 0 6]
[ 48 11 24 71]]
AUC score:
0.79010805742
('Epoch', 96, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0428 - acc: 0.9931Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 114s - loss: 0.0428 - acc: 0.9931 - val_loss: 1.4300 - val_acc: 0.7739
Performance of model on test set ----------------------------
Accuracy:
0.774904214559
Kappa:
0.557744341114
Confusion matrix:
[[601 32 4 10]
[ 68 133 1 17]
[ 9 8 0 7]
[ 63 3 13 75]]
AUC score:
0.808088196174
('Epoch', 97, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0456 - acc: 0.9926Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 119s - loss: 0.0456 - acc: 0.9926 - val_loss: 1.4755 - val_acc: 0.7778
Performance of model on test set ----------------------------
Accuracy:
0.778735632184
Kappa:
0.569709599834
Confusion matrix:
[[594 36 8 9]
[ 63 144 3 9]
[ 10 10 0 4]
[ 62 3 14 75]]
AUC score:
0.784704454788
('Epoch', 98, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0525 - acc: 0.9911Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 118s - loss: 0.0525 - acc: 0.9911 - val_loss: 1.4293 - val_acc: 0.7672
Performance of model on test set ----------------------------
Accuracy:
0.768199233716
Kappa:
0.550964549646
Confusion matrix:
[[592 47 5 3]
[ 59 149 4 7]
[ 10 11 2 1]
[ 62 12 21 59]]
AUC score:
0.795840996717
('Epoch', 99, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0729 - acc: 0.9870Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 127s - loss: 0.0728 - acc: 0.9870 - val_loss: 1.3044 - val_acc: 0.7826
Performance of model on test set ----------------------------
Accuracy:
0.783524904215
Kappa:
0.564778537752
Confusion matrix:
[[611 28 2 6]
[ 71 139 4 5]
[ 16 5 1 2]
[ 69 5 13 67]]
AUC score:
0.789197457837
('Epoch', 100, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0594 - acc: 0.9884Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 114s - loss: 0.0594 - acc: 0.9884 - val_loss: 1.3969 - val_acc: 0.7672
Performance of model on test set ----------------------------
Accuracy:
0.769157088123
Kappa:
0.542859011675
Confusion matrix:
[[600 38 4 5]
[ 67 143 3 6]
[ 12 8 1 3]
[ 69 11 15 59]]
AUC score:
0.73024023288
Best validation accuracy: (41, 0.80651340996168586)
In [6]:
print 'Best validation accuracy: ', get_max_values(val_accs)
print 'Best validation Kappa: ', get_max_values(kappas)
print 'Best validation AUC: ', get_max_values(aucs)
Best validation accuracy: (41, 0.80651340996168586)
Best validation Kappa: (41, 0.62877492168359805)
Best validation AUC: (48, 0.84802145297650755)
In [7]:
# Create the model and parameters for training
numpy.random.seed(seed)
batch = 1
epochs = 100
modelS2 = create_LSTM2_PCA(dummy_y_trainS.shape[1], batch_size = batch, trainShape1=n_components)
print modelS2.summary()
# To save the best model
# serialize model to JSON
modelS2_json = modelS2.to_json()
with open("social.model--2lstmbis.json", "w") as json_file:
json_file.write(modelS2_json)
filepathS2="social.weights--2lstmbis.best.hdf5"
# Define that the accuracy in cv is monitored, and that weights are stored in a file when max accuracy is achieved
checkpointS2 = ModelCheckpoint(filepathS2, monitor='val_acc', verbose=1, save_best_only=True, mode='max')
callbacks_listS2 = [checkpointS2]
# Fit the model
accs =[]
val_accs =[]
losss =[]
val_losss =[]
kappas = []
aucs = []
# Manually create epochs and reset between sessions
for i in range(epochs):
# Single epoch. Remember to not shuffle the data!
print('Epoch', i+1, '/', epochs)
history = modelS2.fit(trainX, dummy_y_trainS, validation_data=(testX, dummy_y_testS),
nb_epoch=1, batch_size=batch, shuffle=False,
verbose=1, callbacks=callbacks_listS2)
modelS2.reset_states()
kappa, auc = printValStats(modelS2, testX, dummy_y_testS, batch=batch)
accs.append(history.history['acc'][0])
val_accs.append(history.history['val_acc'][0])
losss.append(history.history['loss'][0])
val_losss.append(history.history['val_loss'][0])
kappas.append(kappa)
aucs.append(auc)
print 'Best validation accuracy: ', get_max_values(val_accs)
print 'Best validation Kappa: ', get_max_values(kappas)
print 'Best validation AUC: ', get_max_values(aucs)
plot_training(accs, val_accs, losss, val_losss, kappas, aucs)
____________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
====================================================================================================
lstm_4 (LSTM) (1, 1, 300) 481200 lstm_input_2[0][0]
____________________________________________________________________________________________________
dropout_6 (Dropout) (1, 1, 300) 0 lstm_4[0][0]
____________________________________________________________________________________________________
lstm_5 (LSTM) (1, 50) 70200 dropout_6[0][0]
____________________________________________________________________________________________________
dropout_7 (Dropout) (1, 50) 0 lstm_5[0][0]
____________________________________________________________________________________________________
dense_4 (Dense) (1, 50) 2550 dropout_7[0][0]
____________________________________________________________________________________________________
dropout_8 (Dropout) (1, 50) 0 dense_4[0][0]
____________________________________________________________________________________________________
dense_5 (Dense) (1, 20) 1020 dropout_8[0][0]
____________________________________________________________________________________________________
dropout_9 (Dropout) (1, 20) 0 dense_5[0][0]
____________________________________________________________________________________________________
dense_6 (Dense) (1, 4) 84 dropout_9[0][0]
====================================================================================================
Total params: 555054
____________________________________________________________________________________________________
None
('Epoch', 1, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.7696 - acc: 0.7197Epoch 00000: val_acc improved from -inf to 0.65230, saving model to social.weights--2lstmbis.best.hdf5
4472/4472 [==============================] - 100s - loss: 0.7696 - acc: 0.7198 - val_loss: 0.9076 - val_acc: 0.6523
Performance of model on test set ----------------------------
Accuracy:
0.653256704981
Kappa:
0.276692822967
Confusion matrix:
[[598 3 0 46]
[116 48 0 55]
[ 19 0 0 5]
[ 92 0 26 36]]
AUC score:
0.668699549035
('Epoch', 2, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.6181 - acc: 0.7931Epoch 00000: val_acc improved from 0.65230 to 0.65613, saving model to social.weights--2lstmbis.best.hdf5
4472/4472 [==============================] - 78s - loss: 0.6181 - acc: 0.7932 - val_loss: 0.7486 - val_acc: 0.6561
Performance of model on test set ----------------------------
Accuracy:
0.657088122605
Kappa:
0.432613863731
Confusion matrix:
[[480 4 0 163]
[ 29 74 0 116]
[ 7 2 0 15]
[ 22 0 0 132]]
AUC score:
0.802498892414
('Epoch', 3, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.4849 - acc: 0.8446Epoch 00000: val_acc improved from 0.65613 to 0.75766, saving model to social.weights--2lstmbis.best.hdf5
4472/4472 [==============================] - 84s - loss: 0.4849 - acc: 0.8446 - val_loss: 0.6916 - val_acc: 0.7577
Performance of model on test set ----------------------------
Accuracy:
0.757662835249
Kappa:
0.553837863951
Confusion matrix:
[[563 13 0 71]
[ 60 109 0 50]
[ 9 6 0 9]
[ 35 0 0 119]]
AUC score:
0.76446161563
('Epoch', 4, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.3894 - acc: 0.8799Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 82s - loss: 0.3893 - acc: 0.8799 - val_loss: 0.8798 - val_acc: 0.7079
Performance of model on test set ----------------------------
Accuracy:
0.708812260536
Kappa:
0.508310856555
Confusion matrix:
[[494 35 0 118]
[ 16 137 1 65]
[ 6 7 0 11]
[ 37 3 5 109]]
AUC score:
0.735164028876
('Epoch', 5, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.3397 - acc: 0.8980Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 81s - loss: 0.3396 - acc: 0.8980 - val_loss: 0.7779 - val_acc: 0.7538
Performance of model on test set ----------------------------
Accuracy:
0.755747126437
Kappa:
0.544695035291
Confusion matrix:
[[556 52 2 37]
[ 41 152 0 26]
[ 10 7 0 7]
[ 64 7 2 81]]
AUC score:
0.769949255524
('Epoch', 6, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.2999 - acc: 0.9123Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 85s - loss: 0.2998 - acc: 0.9123 - val_loss: 0.9391 - val_acc: 0.7136
Performance of model on test set ----------------------------
Accuracy:
0.716475095785
Kappa:
0.481717277768
Confusion matrix:
[[540 41 0 66]
[ 44 131 0 44]
[ 8 8 0 8]
[ 61 7 9 77]]
AUC score:
0.709804545366
('Epoch', 7, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.2624 - acc: 0.9242Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 89s - loss: 0.2623 - acc: 0.9242 - val_loss: 0.9029 - val_acc: 0.7490
Performance of model on test set ----------------------------
Accuracy:
0.750957854406
Kappa:
0.522707610936
Confusion matrix:
[[578 23 1 45]
[ 69 132 0 18]
[ 8 7 0 9]
[ 59 4 17 74]]
AUC score:
0.757151918966
('Epoch', 8, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.2164 - acc: 0.9349Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 89s - loss: 0.2163 - acc: 0.9349 - val_loss: 0.9915 - val_acc: 0.7241
Performance of model on test set ----------------------------
Accuracy:
0.728927203065
Kappa:
0.528145766756
Confusion matrix:
[[517 31 1 98]
[ 35 139 3 42]
[ 6 9 0 9]
[ 35 7 7 105]]
AUC score:
0.764996135042
('Epoch', 9, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1934 - acc: 0.9423Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 91s - loss: 0.1934 - acc: 0.9423 - val_loss: 0.7880 - val_acc: 0.7577
Performance of model on test set ----------------------------
Accuracy:
0.761494252874
Kappa:
0.557743771213
Confusion matrix:
[[549 53 0 45]
[ 50 157 1 11]
[ 9 10 3 2]
[ 57 6 5 86]]
AUC score:
0.829493440591
('Epoch', 10, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.2046 - acc: 0.9369Epoch 00000: val_acc improved from 0.75766 to 0.77586, saving model to social.weights--2lstmbis.best.hdf5
4472/4472 [==============================] - 94s - loss: 0.2046 - acc: 0.9369 - val_loss: 0.7819 - val_acc: 0.7759
Performance of model on test set ----------------------------
Accuracy:
0.778735632184
Kappa:
0.586712502956
Confusion matrix:
[[559 54 0 34]
[ 48 162 0 9]
[ 10 9 3 2]
[ 56 4 5 89]]
AUC score:
0.829369764395
('Epoch', 11, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1444 - acc: 0.9541Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 97s - loss: 0.1443 - acc: 0.9542 - val_loss: 0.9896 - val_acc: 0.7385
Performance of model on test set ----------------------------
Accuracy:
0.740421455939
Kappa:
0.480941015024
Confusion matrix:
[[587 32 3 25]
[ 85 113 4 17]
[ 10 7 4 3]
[ 81 0 4 69]]
AUC score:
0.830422799155
('Epoch', 12, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1458 - acc: 0.9580Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 118s - loss: 0.1458 - acc: 0.9580 - val_loss: 0.9166 - val_acc: 0.7672
Performance of model on test set ----------------------------
Accuracy:
0.769157088123
Kappa:
0.561003951948
Confusion matrix:
[[569 60 1 17]
[ 46 158 4 11]
[ 9 9 3 3]
[ 71 5 5 73]]
AUC score:
0.800255829152
('Epoch', 13, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1330 - acc: 0.9638Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 119s - loss: 0.1330 - acc: 0.9638 - val_loss: 1.0120 - val_acc: 0.7557
Performance of model on test set ----------------------------
Accuracy:
0.76245210728
Kappa:
0.54022123053
Confusion matrix:
[[580 45 1 21]
[ 58 144 0 17]
[ 11 7 1 5]
[ 70 4 9 71]]
AUC score:
0.794619666326
('Epoch', 14, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1219 - acc: 0.9649Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 118s - loss: 0.1219 - acc: 0.9649 - val_loss: 1.2442 - val_acc: 0.7586
Performance of model on test set ----------------------------
Accuracy:
0.760536398467
Kappa:
0.507677217339
Confusion matrix:
[[614 21 2 10]
[ 96 111 1 11]
[ 10 7 1 6]
[ 75 0 11 68]]
AUC score:
0.803646962605
('Epoch', 15, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1386 - acc: 0.9580Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 117s - loss: 0.1386 - acc: 0.9580 - val_loss: 1.1839 - val_acc: 0.7308
Performance of model on test set ----------------------------
Accuracy:
0.733716475096
Kappa:
0.482831249087
Confusion matrix:
[[572 34 3 38]
[ 60 135 1 23]
[ 9 9 2 4]
[ 86 1 10 57]]
AUC score:
0.784581789077
('Epoch', 16, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1137 - acc: 0.9665Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 117s - loss: 0.1137 - acc: 0.9665 - val_loss: 1.1207 - val_acc: 0.7414
Performance of model on test set ----------------------------
Accuracy:
0.745210727969
Kappa:
0.504963661612
Confusion matrix:
[[583 34 3 27]
[ 70 130 1 18]
[ 8 10 2 4]
[ 70 0 21 63]]
AUC score:
0.790408729845
('Epoch', 17, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1235 - acc: 0.9644Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 117s - loss: 0.1235 - acc: 0.9644 - val_loss: 0.9522 - val_acc: 0.7625
Performance of model on test set ----------------------------
Accuracy:
0.765325670498
Kappa:
0.538393237419
Confusion matrix:
[[585 38 1 23]
[ 75 137 2 5]
[ 10 11 2 1]
[ 69 3 7 75]]
AUC score:
0.830919028021
('Epoch', 18, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1041 - acc: 0.9696Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 117s - loss: 0.1041 - acc: 0.9696 - val_loss: 1.1364 - val_acc: 0.7510
Performance of model on test set ----------------------------
Accuracy:
0.754789272031
Kappa:
0.539418379217
Confusion matrix:
[[558 40 2 47]
[ 55 145 3 16]
[ 11 9 2 2]
[ 61 4 6 83]]
AUC score:
0.816169400948
('Epoch', 19, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0874 - acc: 0.9752Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 105s - loss: 0.0874 - acc: 0.9752 - val_loss: 1.2448 - val_acc: 0.7452
Performance of model on test set ----------------------------
Accuracy:
0.748084291188
Kappa:
0.509110802809
Confusion matrix:
[[576 50 3 18]
[ 58 155 1 5]
[ 10 9 4 1]
[ 84 6 18 46]]
AUC score:
0.832734288901
('Epoch', 20, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0842 - acc: 0.9767Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 115s - loss: 0.0842 - acc: 0.9767 - val_loss: 1.1988 - val_acc: 0.7519
Performance of model on test set ----------------------------
Accuracy:
0.755747126437
Kappa:
0.515769063569
Confusion matrix:
[[590 42 6 9]
[ 70 146 1 2]
[ 14 7 2 1]
[ 78 2 23 51]]
AUC score:
0.813497638482
('Epoch', 21, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0717 - acc: 0.9781Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 118s - loss: 0.0717 - acc: 0.9781 - val_loss: 1.3345 - val_acc: 0.7529
Performance of model on test set ----------------------------
Accuracy:
0.755747126437
Kappa:
0.508391071191
Confusion matrix:
[[605 30 4 8]
[ 64 148 1 6]
[ 14 8 2 0]
[ 86 9 25 34]]
AUC score:
0.798650079108
('Epoch', 22, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0652 - acc: 0.9790Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 113s - loss: 0.0652 - acc: 0.9790 - val_loss: 1.3199 - val_acc: 0.7567
Performance of model on test set ----------------------------
Accuracy:
0.758620689655
Kappa:
0.51872943165
Confusion matrix:
[[595 35 8 9]
[ 75 134 4 6]
[ 14 8 2 0]
[ 75 5 13 61]]
AUC score:
0.786119717366
('Epoch', 23, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0627 - acc: 0.9817Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 119s - loss: 0.0627 - acc: 0.9817 - val_loss: 1.4671 - val_acc: 0.7356
Performance of model on test set ----------------------------
Accuracy:
0.738505747126
Kappa:
0.484180388097
Confusion matrix:
[[581 26 2 38]
[ 61 138 4 16]
[ 13 7 1 3]
[ 91 1 11 51]]
AUC score:
0.763664995271
('Epoch', 24, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0714 - acc: 0.9810Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 111s - loss: 0.0713 - acc: 0.9810 - val_loss: 1.3000 - val_acc: 0.7615
Performance of model on test set ----------------------------
Accuracy:
0.760536398467
Kappa:
0.538321116237
Confusion matrix:
[[575 60 3 9]
[ 61 152 1 5]
[ 11 8 2 3]
[ 67 6 16 65]]
AUC score:
0.768163665531
('Epoch', 25, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0723 - acc: 0.9785Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 118s - loss: 0.0723 - acc: 0.9785 - val_loss: 1.2799 - val_acc: 0.7385
Performance of model on test set ----------------------------
Accuracy:
0.739463601533
Kappa:
0.48679987277
Confusion matrix:
[[583 39 3 22]
[ 66 143 0 10]
[ 11 11 0 2]
[ 79 16 13 46]]
AUC score:
0.80499842281
('Epoch', 26, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0758 - acc: 0.9781Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 122s - loss: 0.0758 - acc: 0.9781 - val_loss: 1.3381 - val_acc: 0.7577
Performance of model on test set ----------------------------
Accuracy:
0.758620689655
Kappa:
0.518806013827
Confusion matrix:
[[595 43 1 8]
[ 72 139 0 8]
[ 13 7 0 4]
[ 75 4 17 58]]
AUC score:
0.800992958438
('Epoch', 27, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0645 - acc: 0.9819Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 121s - loss: 0.0645 - acc: 0.9819 - val_loss: 1.3618 - val_acc: 0.7538
Performance of model on test set ----------------------------
Accuracy:
0.755747126437
Kappa:
0.549206005168
Confusion matrix:
[[558 57 2 30]
[ 34 161 5 19]
[ 9 11 0 4]
[ 58 16 10 70]]
AUC score:
0.826953465649
('Epoch', 28, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0579 - acc: 0.9839Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 117s - loss: 0.0579 - acc: 0.9839 - val_loss: 1.2597 - val_acc: 0.7625
Performance of model on test set ----------------------------
Accuracy:
0.764367816092
Kappa:
0.543032429566
Confusion matrix:
[[580 45 1 21]
[ 58 144 2 15]
[ 14 8 0 2]
[ 67 10 3 74]]
AUC score:
0.789689119281
('Epoch', 29, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0673 - acc: 0.9805Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 118s - loss: 0.0673 - acc: 0.9805 - val_loss: 1.2187 - val_acc: 0.7462
Performance of model on test set ----------------------------
Accuracy:
0.749042145594
Kappa:
0.509932920537
Confusion matrix:
[[583 49 3 12]
[ 64 140 8 7]
[ 16 6 1 1]
[ 69 13 14 58]]
AUC score:
0.78276293548
('Epoch', 30, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0593 - acc: 0.9830Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 120s - loss: 0.0593 - acc: 0.9830 - val_loss: 1.2737 - val_acc: 0.7596
Performance of model on test set ----------------------------
Accuracy:
0.759578544061
Kappa:
0.54452320282
Confusion matrix:
[[575 47 1 24]
[ 57 141 3 18]
[ 8 11 1 4]
[ 54 13 11 76]]
AUC score:
0.777411842274
('Epoch', 31, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0466 - acc: 0.9873Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 124s - loss: 0.0466 - acc: 0.9873 - val_loss: 1.4000 - val_acc: 0.7653
Performance of model on test set ----------------------------
Accuracy:
0.76724137931
Kappa:
0.544592561743
Confusion matrix:
[[595 44 2 6]
[ 70 138 5 6]
[ 9 12 3 0]
[ 59 17 13 65]]
AUC score:
0.781299022757
('Epoch', 32, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0454 - acc: 0.9884Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 119s - loss: 0.0454 - acc: 0.9884 - val_loss: 1.3667 - val_acc: 0.7625
Performance of model on test set ----------------------------
Accuracy:
0.763409961686
Kappa:
0.544118505004
Confusion matrix:
[[591 32 1 23]
[ 68 126 4 21]
[ 12 7 1 4]
[ 50 7 18 79]]
AUC score:
0.796676775555
('Epoch', 33, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0450 - acc: 0.9861Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 120s - loss: 0.0450 - acc: 0.9861 - val_loss: 1.3484 - val_acc: 0.7653
Performance of model on test set ----------------------------
Accuracy:
0.765325670498
Kappa:
0.566756551668
Confusion matrix:
[[567 44 5 31]
[ 51 139 5 24]
[ 7 9 2 6]
[ 42 9 12 91]]
AUC score:
0.850655660064
('Epoch', 34, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0370 - acc: 0.9886Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 121s - loss: 0.0370 - acc: 0.9886 - val_loss: 1.4474 - val_acc: 0.7672
Performance of model on test set ----------------------------
Accuracy:
0.76724137931
Kappa:
0.552265042039
Confusion matrix:
[[582 49 3 13]
[ 58 146 2 13]
[ 9 10 2 3]
[ 62 12 9 71]]
AUC score:
0.806542582585
('Epoch', 35, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0395 - acc: 0.9897Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 120s - loss: 0.0395 - acc: 0.9897 - val_loss: 1.5010 - val_acc: 0.7510
Performance of model on test set ----------------------------
Accuracy:
0.75
Kappa:
0.527835105139
Confusion matrix:
[[565 60 5 17]
[ 47 157 3 12]
[ 10 8 3 3]
[ 68 12 16 58]]
AUC score:
0.795711091819
('Epoch', 36, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0330 - acc: 0.9908Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 119s - loss: 0.0330 - acc: 0.9908 - val_loss: 1.4671 - val_acc: 0.7596
Performance of model on test set ----------------------------
Accuracy:
0.758620689655
Kappa:
0.539573784435
Confusion matrix:
[[575 52 2 18]
[ 59 149 3 8]
[ 9 10 1 4]
[ 60 9 18 67]]
AUC score:
0.797850183875
('Epoch', 37, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0404 - acc: 0.9893Epoch 00000: val_acc improved from 0.77586 to 0.77682, saving model to social.weights--2lstmbis.best.hdf5
4472/4472 [==============================] - 120s - loss: 0.0404 - acc: 0.9893 - val_loss: 1.3846 - val_acc: 0.7768
Performance of model on test set ----------------------------
Accuracy:
0.776819923372
Kappa:
0.579491452467
Confusion matrix:
[[575 58 5 9]
[ 50 154 4 11]
[ 10 9 3 2]
[ 53 8 14 79]]
AUC score:
0.795961426745
('Epoch', 38, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0383 - acc: 0.9886Epoch 00000: val_acc improved from 0.77682 to 0.78640, saving model to social.weights--2lstmbis.best.hdf5
4472/4472 [==============================] - 118s - loss: 0.0383 - acc: 0.9886 - val_loss: 1.2810 - val_acc: 0.7864
Performance of model on test set ----------------------------
Accuracy:
0.787356321839
Kappa:
0.595016512607
Confusion matrix:
[[576 61 0 10]
[ 57 158 0 4]
[ 11 9 3 1]
[ 51 11 7 85]]
AUC score:
0.78304913657
('Epoch', 39, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0414 - acc: 0.9881Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 118s - loss: 0.0414 - acc: 0.9881 - val_loss: 1.3496 - val_acc: 0.7797
Performance of model on test set ----------------------------
Accuracy:
0.77969348659
Kappa:
0.591102059305
Confusion matrix:
[[565 55 1 26]
[ 51 149 1 18]
[ 9 9 4 2]
[ 41 14 3 96]]
AUC score:
0.779463907675
('Epoch', 40, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0373 - acc: 0.9902Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 118s - loss: 0.0373 - acc: 0.9902 - val_loss: 1.3871 - val_acc: 0.7816
Performance of model on test set ----------------------------
Accuracy:
0.782567049808
Kappa:
0.590300876144
Confusion matrix:
[[574 42 1 30]
[ 57 145 1 16]
[ 8 9 3 4]
[ 48 7 4 95]]
AUC score:
0.787663740895
('Epoch', 41, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0335 - acc: 0.9906Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 119s - loss: 0.0335 - acc: 0.9906 - val_loss: 1.4895 - val_acc: 0.7778
Performance of model on test set ----------------------------
Accuracy:
0.778735632184
Kappa:
0.576670575266
Confusion matrix:
[[579 49 1 18]
[ 51 156 0 12]
[ 8 9 3 4]
[ 64 10 5 75]]
AUC score:
0.777995619787
('Epoch', 42, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0379 - acc: 0.9913Epoch 00000: val_acc improved from 0.78640 to 0.78736, saving model to social.weights--2lstmbis.best.hdf5
4472/4472 [==============================] - 122s - loss: 0.0379 - acc: 0.9913 - val_loss: 1.2885 - val_acc: 0.7874
Performance of model on test set ----------------------------
Accuracy:
0.787356321839
Kappa:
0.601738296655
Confusion matrix:
[[579 38 1 29]
[ 53 141 0 25]
[ 9 8 3 4]
[ 41 6 8 99]]
AUC score:
0.794789202693
('Epoch', 43, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0497 - acc: 0.9875Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 126s - loss: 0.0496 - acc: 0.9875 - val_loss: 1.3158 - val_acc: 0.7739
Performance of model on test set ----------------------------
Accuracy:
0.774904214559
Kappa:
0.579782132091
Confusion matrix:
[[565 48 1 33]
[ 52 146 2 19]
[ 12 7 0 5]
[ 46 6 4 98]]
AUC score:
0.777344088121
('Epoch', 44, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0311 - acc: 0.9917Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 120s - loss: 0.0311 - acc: 0.9917 - val_loss: 1.4643 - val_acc: 0.7711
Performance of model on test set ----------------------------
Accuracy:
0.772988505747
Kappa:
0.563211532453
Confusion matrix:
[[584 39 1 23]
[ 65 139 2 13]
[ 11 6 2 5]
[ 53 12 7 82]]
AUC score:
0.769904496925
('Epoch', 45, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0338 - acc: 0.9906Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 121s - loss: 0.0338 - acc: 0.9906 - val_loss: 1.5111 - val_acc: 0.7711
Performance of model on test set ----------------------------
Accuracy:
0.771072796935
Kappa:
0.558003259406
Confusion matrix:
[[593 28 4 22]
[ 65 134 5 15]
[ 10 6 4 4]
[ 57 5 18 74]]
AUC score:
0.786375232164
('Epoch', 46, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0375 - acc: 0.9902Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 123s - loss: 0.0375 - acc: 0.9902 - val_loss: 1.4271 - val_acc: 0.7778
Performance of model on test set ----------------------------
Accuracy:
0.778735632184
Kappa:
0.579262867371
Confusion matrix:
[[585 37 4 21]
[ 57 144 4 14]
[ 9 7 3 5]
[ 51 10 12 81]]
AUC score:
0.810645394033
('Epoch', 47, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0237 - acc: 0.9933Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 121s - loss: 0.0237 - acc: 0.9933 - val_loss: 1.5413 - val_acc: 0.7749
Performance of model on test set ----------------------------
Accuracy:
0.774904214559
Kappa:
0.57525341274
Confusion matrix:
[[576 45 3 23]
[ 55 152 1 11]
[ 9 11 3 1]
[ 50 11 15 78]]
AUC score:
0.791717907754
('Epoch', 48, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0236 - acc: 0.9937Epoch 00000: val_acc improved from 0.78736 to 0.79215, saving model to social.weights--2lstmbis.best.hdf5
4472/4472 [==============================] - 123s - loss: 0.0236 - acc: 0.9937 - val_loss: 1.4971 - val_acc: 0.7921
Performance of model on test set ----------------------------
Accuracy:
0.793103448276
Kappa:
0.600393397305
Confusion matrix:
[[588 39 0 20]
[ 65 150 1 3]
[ 11 10 2 1]
[ 51 9 6 88]]
AUC score:
0.800326221261
('Epoch', 49, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0345 - acc: 0.9922Epoch 00000: val_acc improved from 0.79215 to 0.79693, saving model to social.weights--2lstmbis.best.hdf5
4472/4472 [==============================] - 124s - loss: 0.0345 - acc: 0.9922 - val_loss: 1.3576 - val_acc: 0.7969
Performance of model on test set ----------------------------
Accuracy:
0.797892720307
Kappa:
0.607139417675
Confusion matrix:
[[593 47 0 7]
[ 60 159 0 0]
[ 14 8 2 0]
[ 52 17 6 79]]
AUC score:
0.806719196979
('Epoch', 50, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0364 - acc: 0.9924Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 123s - loss: 0.0364 - acc: 0.9924 - val_loss: 1.3409 - val_acc: 0.7749
Performance of model on test set ----------------------------
Accuracy:
0.775862068966
Kappa:
0.58018755252
Confusion matrix:
[[563 59 1 24]
[ 57 151 1 10]
[ 8 12 1 3]
[ 48 7 4 95]]
AUC score:
0.768393209206
('Epoch', 51, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0300 - acc: 0.9915Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 126s - loss: 0.0300 - acc: 0.9915 - val_loss: 1.3766 - val_acc: 0.7921
Performance of model on test set ----------------------------
Accuracy:
0.793103448276
Kappa:
0.606658259245
Confusion matrix:
[[582 51 0 14]
[ 54 156 1 8]
[ 7 11 3 3]
[ 51 11 5 87]]
AUC score:
0.804168891151
('Epoch', 52, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0221 - acc: 0.9935Epoch 00000: val_acc improved from 0.79693 to 0.80268, saving model to social.weights--2lstmbis.best.hdf5
4472/4472 [==============================] - 127s - loss: 0.0221 - acc: 0.9935 - val_loss: 1.3408 - val_acc: 0.8027
Performance of model on test set ----------------------------
Accuracy:
0.803639846743
Kappa:
0.627238969743
Confusion matrix:
[[588 42 3 14]
[ 60 151 1 7]
[ 8 11 2 3]
[ 38 14 4 98]]
AUC score:
0.798662836696
('Epoch', 53, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0336 - acc: 0.9915Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 125s - loss: 0.0336 - acc: 0.9915 - val_loss: 1.4875 - val_acc: 0.7768
Performance of model on test set ----------------------------
Accuracy:
0.776819923372
Kappa:
0.574308574309
Confusion matrix:
[[585 43 1 18]
[ 65 141 2 11]
[ 9 9 2 4]
[ 45 11 15 83]]
AUC score:
0.828892819887
('Epoch', 54, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0230 - acc: 0.9949Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 123s - loss: 0.0230 - acc: 0.9949 - val_loss: 1.5960 - val_acc: 0.7644
Performance of model on test set ----------------------------
Accuracy:
0.765325670498
Kappa:
0.555829541903
Confusion matrix:
[[568 57 3 19]
[ 61 152 1 5]
[ 10 9 2 3]
[ 51 16 10 77]]
AUC score:
0.799611879347
('Epoch', 55, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0343 - acc: 0.9902Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 127s - loss: 0.0343 - acc: 0.9902 - val_loss: 1.5854 - val_acc: 0.7615
Performance of model on test set ----------------------------
Accuracy:
0.76245210728
Kappa:
0.536259295878
Confusion matrix:
[[585 45 2 15]
[ 73 139 1 6]
[ 11 8 3 2]
[ 62 9 14 69]]
AUC score:
0.784532180075
('Epoch', 56, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0282 - acc: 0.9924Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 127s - loss: 0.0282 - acc: 0.9924 - val_loss: 1.5553 - val_acc: 0.7644
Performance of model on test set ----------------------------
Accuracy:
0.765325670498
Kappa:
0.553396668197
Confusion matrix:
[[572 43 2 30]
[ 60 147 1 11]
[ 9 9 3 3]
[ 58 11 8 77]]
AUC score:
0.778389823154
('Epoch', 57, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0343 - acc: 0.9922Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 122s - loss: 0.0343 - acc: 0.9922 - val_loss: 1.4889 - val_acc: 0.7596
Performance of model on test set ----------------------------
Accuracy:
0.761494252874
Kappa:
0.539333408943
Confusion matrix:
[[579 40 1 27]
[ 74 132 0 13]
[ 9 9 1 5]
[ 55 10 6 83]]
AUC score:
0.813139444218
('Epoch', 58, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0200 - acc: 0.9946Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 127s - loss: 0.0200 - acc: 0.9946 - val_loss: 1.6084 - val_acc: 0.7759
Performance of model on test set ----------------------------
Accuracy:
0.774904214559
Kappa:
0.583070635319
Confusion matrix:
[[565 42 2 38]
[ 54 147 0 18]
[ 9 12 1 2]
[ 37 15 6 96]]
AUC score:
0.799174882401
('Epoch', 59, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0190 - acc: 0.9953Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 130s - loss: 0.0190 - acc: 0.9953 - val_loss: 1.8223 - val_acc: 0.7663
Performance of model on test set ----------------------------
Accuracy:
0.766283524904
Kappa:
0.554857345563
Confusion matrix:
[[577 41 3 26]
[ 64 140 0 15]
[ 11 10 3 0]
[ 49 15 10 80]]
AUC score:
0.778883927846
('Epoch', 60, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0276 - acc: 0.9926Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 127s - loss: 0.0276 - acc: 0.9926 - val_loss: 1.8020 - val_acc: 0.7443
Performance of model on test set ----------------------------
Accuracy:
0.745210727969
Kappa:
0.514987660919
Confusion matrix:
[[569 45 3 30]
[ 63 140 0 16]
[ 10 10 2 2]
[ 61 7 19 67]]
AUC score:
0.727763594211
('Epoch', 61, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0201 - acc: 0.9944Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 128s - loss: 0.0201 - acc: 0.9944 - val_loss: 1.6804 - val_acc: 0.7711
Performance of model on test set ----------------------------
Accuracy:
0.773946360153
Kappa:
0.56774585174
Confusion matrix:
[[581 44 3 19]
[ 59 151 1 8]
[ 10 12 1 1]
[ 54 13 12 75]]
AUC score:
0.800850305615
('Epoch', 62, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0347 - acc: 0.9911Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 122s - loss: 0.0346 - acc: 0.9911 - val_loss: 1.5155 - val_acc: 0.7730
Performance of model on test set ----------------------------
Accuracy:
0.773946360153
Kappa:
0.568823796032
Confusion matrix:
[[574 50 2 21]
[ 64 152 1 2]
[ 12 10 0 2]
[ 49 14 9 82]]
AUC score:
0.754972072626
('Epoch', 63, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0176 - acc: 0.9951Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 121s - loss: 0.0176 - acc: 0.9951 - val_loss: 1.6154 - val_acc: 0.7864
Performance of model on test set ----------------------------
Accuracy:
0.788314176245
Kappa:
0.595265786413
Confusion matrix:
[[581 41 1 24]
[ 53 155 3 8]
[ 10 10 2 2]
[ 60 3 6 85]]
AUC score:
0.773527400049
('Epoch', 64, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0260 - acc: 0.9935Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 130s - loss: 0.0260 - acc: 0.9935 - val_loss: 1.7179 - val_acc: 0.7787
Performance of model on test set ----------------------------
Accuracy:
0.781609195402
Kappa:
0.582493755788
Confusion matrix:
[[581 38 0 28]
[ 62 145 2 10]
[ 11 10 1 2]
[ 50 9 6 89]]
AUC score:
0.773118652465
('Epoch', 65, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0284 - acc: 0.9926Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 153s - loss: 0.0284 - acc: 0.9926 - val_loss: 1.6061 - val_acc: 0.7739
Performance of model on test set ----------------------------
Accuracy:
0.775862068966
Kappa:
0.581830288444
Confusion matrix:
[[569 49 0 29]
[ 54 151 3 11]
[ 9 9 3 3]
[ 42 16 9 87]]
AUC score:
0.791186247684
('Epoch', 66, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0276 - acc: 0.9933Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 139s - loss: 0.0276 - acc: 0.9933 - val_loss: 1.5427 - val_acc: 0.7816
Performance of model on test set ----------------------------
Accuracy:
0.782567049808
Kappa:
0.564515595541
Confusion matrix:
[[602 38 0 7]
[ 82 137 0 0]
[ 15 9 0 0]
[ 58 11 7 78]]
AUC score:
0.818322275306
('Epoch', 67, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0198 - acc: 0.9955Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 137s - loss: 0.0197 - acc: 0.9955 - val_loss: 1.6945 - val_acc: 0.7826
Performance of model on test set ----------------------------
Accuracy:
0.783524904215
Kappa:
0.582870286067
Confusion matrix:
[[586 47 2 12]
[ 54 163 1 1]
[ 14 10 0 0]
[ 57 14 14 69]]
AUC score:
0.778059324898
('Epoch', 68, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0148 - acc: 0.9960Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 137s - loss: 0.0148 - acc: 0.9960 - val_loss: 1.7442 - val_acc: 0.7749
Performance of model on test set ----------------------------
Accuracy:
0.772988505747
Kappa:
0.553077348246
Confusion matrix:
[[596 44 1 6]
[ 73 142 1 3]
[ 13 11 0 0]
[ 56 15 14 69]]
AUC score:
0.755583454433
('Epoch', 69, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0171 - acc: 0.9966Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 136s - loss: 0.0171 - acc: 0.9966 - val_loss: 1.6965 - val_acc: 0.7835
Performance of model on test set ----------------------------
Accuracy:
0.784482758621
Kappa:
0.577241215472
Confusion matrix:
[[597 38 1 11]
[ 68 145 2 4]
[ 13 9 0 2]
[ 55 15 7 77]]
AUC score:
0.788472517246
('Epoch', 70, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0181 - acc: 0.9964Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 135s - loss: 0.0181 - acc: 0.9964 - val_loss: 1.7093 - val_acc: 0.7960
Performance of model on test set ----------------------------
Accuracy:
0.7969348659
Kappa:
0.602233536475
Confusion matrix:
[[603 37 1 6]
[ 63 151 3 2]
[ 12 11 0 1]
[ 53 15 8 78]]
AUC score:
0.761781072703
('Epoch', 71, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0224 - acc: 0.9949Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 138s - loss: 0.0224 - acc: 0.9949 - val_loss: 1.6352 - val_acc: 0.7845
Performance of model on test set ----------------------------
Accuracy:
0.784482758621
Kappa:
0.585720130933
Confusion matrix:
[[582 57 1 7]
[ 66 152 0 1]
[ 11 13 0 0]
[ 45 20 4 85]]
AUC score:
0.748635154688
('Epoch', 72, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0155 - acc: 0.9964Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 127s - loss: 0.0155 - acc: 0.9964 - val_loss: 1.6909 - val_acc: 0.7807
Performance of model on test set ----------------------------
Accuracy:
0.780651340996
Kappa:
0.576924029801
Confusion matrix:
[[585 50 0 12]
[ 64 152 0 3]
[ 10 13 0 1]
[ 51 17 8 78]]
AUC score:
0.747757667832
('Epoch', 73, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0112 - acc: 0.9969Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 150s - loss: 0.0112 - acc: 0.9969 - val_loss: 1.6958 - val_acc: 0.7816
Performance of model on test set ----------------------------
Accuracy:
0.781609195402
Kappa:
0.576244392224
Confusion matrix:
[[589 47 1 10]
[ 72 141 1 5]
[ 11 10 1 2]
[ 48 14 7 85]]
AUC score:
0.780452693182
('Epoch', 74, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0161 - acc: 0.9971Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 176s - loss: 0.0161 - acc: 0.9971 - val_loss: 1.6778 - val_acc: 0.7941
Performance of model on test set ----------------------------
Accuracy:
0.794061302682
Kappa:
0.593935995079
Confusion matrix:
[[603 37 0 7]
[ 75 142 0 2]
[ 14 8 1 1]
[ 48 13 10 83]]
AUC score:
0.789297467923
('Epoch', 75, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0244 - acc: 0.9951Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 171s - loss: 0.0244 - acc: 0.9951 - val_loss: 1.5869 - val_acc: 0.7893
Performance of model on test set ----------------------------
Accuracy:
0.789272030651
Kappa:
0.580105851059
Confusion matrix:
[[607 36 0 4]
[ 79 139 0 1]
[ 11 12 1 0]
[ 53 18 6 77]]
AUC score:
0.781404578731
('Epoch', 76, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0239 - acc: 0.9953Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 170s - loss: 0.0239 - acc: 0.9953 - val_loss: 1.6344 - val_acc: 0.7615
Performance of model on test set ----------------------------
Accuracy:
0.761494252874
Kappa:
0.533075523448
Confusion matrix:
[[588 43 0 16]
[ 79 135 0 5]
[ 11 11 1 1]
[ 55 13 15 71]]
AUC score:
0.817163237982
('Epoch', 77, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0103 - acc: 0.9978Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 165s - loss: 0.0103 - acc: 0.9978 - val_loss: 1.6957 - val_acc: 0.7768
Performance of model on test set ----------------------------
Accuracy:
0.776819923372
Kappa:
0.561727060451
Confusion matrix:
[[591 45 0 11]
[ 74 144 0 1]
[ 11 12 1 0]
[ 56 17 6 75]]
AUC score:
0.805570831463
('Epoch', 78, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0150 - acc: 0.9975Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 123s - loss: 0.0150 - acc: 0.9975 - val_loss: 1.6510 - val_acc: 0.7854
Performance of model on test set ----------------------------
Accuracy:
0.785440613027
Kappa:
0.583148100284
Confusion matrix:
[[587 48 1 11]
[ 69 150 0 0]
[ 11 11 1 1]
[ 53 12 7 82]]
AUC score:
0.764944902034
('Epoch', 79, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0099 - acc: 0.9966Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 122s - loss: 0.0099 - acc: 0.9966 - val_loss: 1.6887 - val_acc: 0.7797
Performance of model on test set ----------------------------
Accuracy:
0.77969348659
Kappa:
0.587828478148
Confusion matrix:
[[564 72 0 11]
[ 51 166 0 2]
[ 11 12 1 0]
[ 43 22 6 83]]
AUC score:
0.771732784917
('Epoch', 80, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0174 - acc: 0.9964Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 124s - loss: 0.0174 - acc: 0.9964 - val_loss: 1.6067 - val_acc: 0.8027
Performance of model on test set ----------------------------
Accuracy:
0.802681992337
Kappa:
0.611725645743
Confusion matrix:
[[602 39 0 6]
[ 69 149 1 0]
[ 12 9 3 0]
[ 53 12 5 84]]
AUC score:
0.769387539044
('Epoch', 81, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0158 - acc: 0.9960Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 124s - loss: 0.0158 - acc: 0.9960 - val_loss: 1.6172 - val_acc: 0.7998
Performance of model on test set ----------------------------
Accuracy:
0.799808429119
Kappa:
0.604636456376
Confusion matrix:
[[604 35 0 8]
[ 70 147 0 2]
[ 12 10 2 0]
[ 55 10 7 82]]
AUC score:
0.763156166847
('Epoch', 82, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0200 - acc: 0.9949Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 123s - loss: 0.0200 - acc: 0.9949 - val_loss: 1.7164 - val_acc: 0.7787
Performance of model on test set ----------------------------
Accuracy:
0.777777777778
Kappa:
0.56275408889
Confusion matrix:
[[599 39 1 8]
[ 72 144 2 1]
[ 9 13 1 1]
[ 60 8 18 68]]
AUC score:
0.749167696155
('Epoch', 83, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0155 - acc: 0.9958Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 123s - loss: 0.0155 - acc: 0.9958 - val_loss: 1.9577 - val_acc: 0.7749
Performance of model on test set ----------------------------
Accuracy:
0.774904214559
Kappa:
0.542906328774
Confusion matrix:
[[612 29 1 5]
[ 80 138 1 0]
[ 9 12 2 1]
[ 76 5 16 57]]
AUC score:
0.73384952928
('Epoch', 84, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0189 - acc: 0.9951Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 170s - loss: 0.0189 - acc: 0.9951 - val_loss: 1.6053 - val_acc: 0.7979
Performance of model on test set ----------------------------
Accuracy:
0.797892720307
Kappa:
0.597438624576
Confusion matrix:
[[611 25 1 10]
[ 72 145 0 2]
[ 10 11 1 2]
[ 60 8 10 76]]
AUC score:
0.83089149427
('Epoch', 85, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0142 - acc: 0.9966Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 175s - loss: 0.0142 - acc: 0.9966 - val_loss: 1.7478 - val_acc: 0.7797
Performance of model on test set ----------------------------
Accuracy:
0.77969348659
Kappa:
0.555784130179
Confusion matrix:
[[608 36 0 3]
[ 76 142 1 0]
[ 10 13 1 0]
[ 70 13 8 63]]
AUC score:
0.756369959361
('Epoch', 86, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0111 - acc: 0.9966Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 174s - loss: 0.0111 - acc: 0.9966 - val_loss: 1.8334 - val_acc: 0.7893
Performance of model on test set ----------------------------
Accuracy:
0.788314176245
Kappa:
0.576329822397
Confusion matrix:
[[607 31 0 9]
[ 73 145 1 0]
[ 11 9 1 3]
[ 66 10 8 70]]
AUC score:
0.786458074881
('Epoch', 87, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0156 - acc: 0.9964Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 176s - loss: 0.0156 - acc: 0.9964 - val_loss: 1.6835 - val_acc: 0.7835
Performance of model on test set ----------------------------
Accuracy:
0.783524904215
Kappa:
0.578938351581
Confusion matrix:
[[589 48 0 10]
[ 59 157 2 1]
[ 9 14 0 1]
[ 63 12 7 72]]
AUC score:
0.753577656413
('Epoch', 88, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0113 - acc: 0.9975Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 173s - loss: 0.0113 - acc: 0.9975 - val_loss: 1.6783 - val_acc: 0.7816
Performance of model on test set ----------------------------
Accuracy:
0.781609195402
Kappa:
0.572513051864
Confusion matrix:
[[595 36 0 16]
[ 59 156 2 2]
[ 10 10 0 4]
[ 67 10 12 65]]
AUC score:
0.777226949468
('Epoch', 89, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0142 - acc: 0.9962Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 176s - loss: 0.0142 - acc: 0.9962 - val_loss: 1.7212 - val_acc: 0.7845
Performance of model on test set ----------------------------
Accuracy:
0.785440613027
Kappa:
0.573485076464
Confusion matrix:
[[605 31 0 11]
[ 72 144 2 1]
[ 11 10 1 2]
[ 62 12 10 70]]
AUC score:
0.745455095793
('Epoch', 90, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0160 - acc: 0.9969Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 175s - loss: 0.0160 - acc: 0.9969 - val_loss: 1.6941 - val_acc: 0.7854
Performance of model on test set ----------------------------
Accuracy:
0.785440613027
Kappa:
0.582591560659
Confusion matrix:
[[597 33 1 16]
[ 65 149 2 3]
[ 12 8 1 3]
[ 52 15 14 73]]
AUC score:
0.788883009667
('Epoch', 91, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0228 - acc: 0.9960Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 176s - loss: 0.0228 - acc: 0.9960 - val_loss: 1.7836 - val_acc: 0.7835
Performance of model on test set ----------------------------
Accuracy:
0.781609195402
Kappa:
0.571516519598
Confusion matrix:
[[602 35 2 8]
[ 61 155 2 1]
[ 11 10 2 1]
[ 62 16 19 57]]
AUC score:
0.802773414292
('Epoch', 92, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0264 - acc: 0.9944Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 174s - loss: 0.0264 - acc: 0.9944 - val_loss: 1.6694 - val_acc: 0.7816
Performance of model on test set ----------------------------
Accuracy:
0.780651340996
Kappa:
0.572045366671
Confusion matrix:
[[600 37 1 9]
[ 69 147 1 2]
[ 9 12 2 1]
[ 51 19 18 66]]
AUC score:
0.765724917333
('Epoch', 93, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0123 - acc: 0.9973Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 176s - loss: 0.0123 - acc: 0.9973 - val_loss: 1.6490 - val_acc: 0.7826
Performance of model on test set ----------------------------
Accuracy:
0.782567049808
Kappa:
0.582368355664
Confusion matrix:
[[591 42 2 12]
[ 63 152 1 3]
[ 8 13 3 0]
[ 49 15 19 71]]
AUC score:
0.748779190802
('Epoch', 94, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0103 - acc: 0.9969Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 175s - loss: 0.0103 - acc: 0.9969 - val_loss: 1.7297 - val_acc: 0.7845
Performance of model on test set ----------------------------
Accuracy:
0.784482758621
Kappa:
0.576977033581
Confusion matrix:
[[602 34 2 9]
[ 71 145 0 3]
[ 10 10 3 1]
[ 54 15 16 69]]
AUC score:
0.801109341194
('Epoch', 95, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0181 - acc: 0.9969Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 177s - loss: 0.0181 - acc: 0.9969 - val_loss: 1.5622 - val_acc: 0.7941
Performance of model on test set ----------------------------
Accuracy:
0.794061302682
Kappa:
0.593716966652
Confusion matrix:
[[606 31 1 9]
[ 79 136 0 4]
[ 11 8 2 3]
[ 45 18 6 85]]
AUC score:
0.75119615187
('Epoch', 96, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0080 - acc: 0.9978Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 175s - loss: 0.0080 - acc: 0.9978 - val_loss: 1.7196 - val_acc: 0.7912
Performance of model on test set ----------------------------
Accuracy:
0.791187739464
Kappa:
0.583502000212
Confusion matrix:
[[612 23 1 11]
[ 82 133 0 4]
[ 10 7 1 6]
[ 52 13 9 80]]
AUC score:
0.768659507553
('Epoch', 97, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0069 - acc: 0.9980Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 175s - loss: 0.0069 - acc: 0.9980 - val_loss: 1.8506 - val_acc: 0.7912
Performance of model on test set ----------------------------
Accuracy:
0.790229885057
Kappa:
0.587226621316
Confusion matrix:
[[602 25 2 18]
[ 76 137 1 5]
[ 10 7 1 6]
[ 53 8 8 85]]
AUC score:
0.79457728748
('Epoch', 98, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0170 - acc: 0.9971Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 179s - loss: 0.0170 - acc: 0.9971 - val_loss: 1.7880 - val_acc: 0.7816
Performance of model on test set ----------------------------
Accuracy:
0.780651340996
Kappa:
0.567833146241
Confusion matrix:
[[602 28 5 12]
[ 76 132 0 11]
[ 13 7 0 4]
[ 53 8 12 81]]
AUC score:
0.747646930954
('Epoch', 99, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0207 - acc: 0.9953Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 179s - loss: 0.0207 - acc: 0.9953 - val_loss: 1.8220 - val_acc: 0.7720
Performance of model on test set ----------------------------
Accuracy:
0.771072796935
Kappa:
0.544775193572
Confusion matrix:
[[601 34 3 9]
[ 83 132 0 4]
[ 11 9 0 4]
[ 58 10 14 72]]
AUC score:
0.744823094863
('Epoch', 100, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0121 - acc: 0.9982Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 176s - loss: 0.0121 - acc: 0.9982 - val_loss: 1.8194 - val_acc: 0.7759
Performance of model on test set ----------------------------
Accuracy:
0.772988505747
Kappa:
0.55602288897
Confusion matrix:
[[595 42 1 9]
[ 69 144 1 5]
[ 11 9 0 4]
[ 57 12 17 68]]
AUC score:
0.73066807528
Best validation accuracy: (51, 0.80268199233716475)
Best validation Kappa: (51, 0.62723896974299309)
Best validation AUC: (32, 0.850655660064494)
In [8]:
# Create the model and parameters for training
numpy.random.seed(seed)
batch = 1
epochs = 100
modelS1 = create_LSTM1_PCA(dummy_y_trainS.shape[1], batch_size = batch, trainShape1=n_components)
print modelS1.summary()
# To save the best model
# serialize model to JSON
modelS1_json = modelS1.to_json()
with open("social.model--1lstmbis.json", "w") as json_file:
json_file.write(modelS1_json)
filepathS1="social.weights--1lstmbis.best.hdf5"
# Define that the accuracy in cv is monitored, and that weights are stored in a file when max accuracy is achieved
checkpointS1 = ModelCheckpoint(filepathS1, monitor='val_acc', verbose=1, save_best_only=True, mode='max')
callbacks_listS1 = [checkpointS1]
# Fit the model
accs =[]
val_accs =[]
losss =[]
val_losss =[]
kappas = []
aucs = []
# Manually create epochs and reset between sessions
for i in range(epochs):
# Single epoch. Remember to not shuffle the data!
print('Epoch', i+1, '/', epochs)
history = modelS1.fit(trainX, dummy_y_trainS, validation_data=(testX, dummy_y_testS),
nb_epoch=1, batch_size=batch, shuffle=False,
verbose=1, callbacks=callbacks_listS1)
modelS1.reset_states()
kappa, auc = printValStats(modelS1, testX, dummy_y_testS, batch=batch)
accs.append(history.history['acc'][0])
val_accs.append(history.history['val_acc'][0])
losss.append(history.history['loss'][0])
val_losss.append(history.history['val_loss'][0])
kappas.append(kappa)
aucs.append(auc)
print 'Best validation accuracy: ', get_max_values(val_accs)
print 'Best validation Kappa: ', get_max_values(kappas)
print 'Best validation AUC: ', get_max_values(aucs)
plot_training(accs, val_accs, losss, val_losss, kappas, aucs)
____________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
====================================================================================================
lstm_6 (LSTM) (1, 400) 801600 lstm_input_3[0][0]
____________________________________________________________________________________________________
dropout_10 (Dropout) (1, 400) 0 lstm_6[0][0]
____________________________________________________________________________________________________
dense_7 (Dense) (1, 50) 20050 dropout_10[0][0]
____________________________________________________________________________________________________
dropout_11 (Dropout) (1, 50) 0 dense_7[0][0]
____________________________________________________________________________________________________
dense_8 (Dense) (1, 20) 1020 dropout_11[0][0]
____________________________________________________________________________________________________
dropout_12 (Dropout) (1, 20) 0 dense_8[0][0]
____________________________________________________________________________________________________
dense_9 (Dense) (1, 4) 84 dropout_12[0][0]
====================================================================================================
Total params: 822754
____________________________________________________________________________________________________
None
('Epoch', 1, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.7003 - acc: 0.7426Epoch 00000: val_acc improved from -inf to 0.67337, saving model to social.weights--1lstmbis.best.hdf5
4472/4472 [==============================] - 116s - loss: 0.7003 - acc: 0.7426 - val_loss: 0.7722 - val_acc: 0.6734
Performance of model on test set ----------------------------
Accuracy:
0.672413793103
Kappa:
0.314252789675
Confusion matrix:
[[589 18 0 40]
[ 83 88 0 48]
[ 19 4 0 1]
[124 1 4 25]]
AUC score:
0.756560948708
('Epoch', 2, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.5451 - acc: 0.8175Epoch 00000: val_acc improved from 0.67337 to 0.69732, saving model to social.weights--1lstmbis.best.hdf5
4472/4472 [==============================] - 135s - loss: 0.5451 - acc: 0.8175 - val_loss: 0.7968 - val_acc: 0.6973
Performance of model on test set ----------------------------
Accuracy:
0.698275862069
Kappa:
0.399993431784
Confusion matrix:
[[572 17 0 58]
[ 71 100 0 48]
[ 18 3 0 3]
[ 96 1 0 57]]
AUC score:
0.744565988963
('Epoch', 3, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.4465 - acc: 0.8524Epoch 00000: val_acc improved from 0.69732 to 0.70785, saving model to social.weights--1lstmbis.best.hdf5
4472/4472 [==============================] - 135s - loss: 0.4465 - acc: 0.8524 - val_loss: 0.7399 - val_acc: 0.7079
Performance of model on test set ----------------------------
Accuracy:
0.70785440613
Kappa:
0.45643100402
Confusion matrix:
[[552 23 0 72]
[ 44 110 0 65]
[ 13 4 0 7]
[ 71 0 6 77]]
AUC score:
0.786520198919
('Epoch', 4, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.3581 - acc: 0.8859Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 159s - loss: 0.3580 - acc: 0.8860 - val_loss: 0.8737 - val_acc: 0.6954
Performance of model on test set ----------------------------
Accuracy:
0.695402298851
Kappa:
0.469413000014
Confusion matrix:
[[498 57 1 91]
[ 26 139 1 53]
[ 8 4 1 11]
[ 59 2 5 88]]
AUC score:
0.762446249082
('Epoch', 5, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.3148 - acc: 0.9034Epoch 00000: val_acc improved from 0.70785 to 0.75096, saving model to social.weights--1lstmbis.best.hdf5
4472/4472 [==============================] - 159s - loss: 0.3147 - acc: 0.9034 - val_loss: 0.8026 - val_acc: 0.7510
Performance of model on test set ----------------------------
Accuracy:
0.750957854406
Kappa:
0.555041194425
Confusion matrix:
[[534 57 1 55]
[ 36 151 1 31]
[ 11 7 0 6]
[ 41 3 11 99]]
AUC score:
0.821551781328
('Epoch', 6, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.2753 - acc: 0.9128Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 160s - loss: 0.2752 - acc: 0.9128 - val_loss: 0.8896 - val_acc: 0.7011
Performance of model on test set ----------------------------
Accuracy:
0.7030651341
Kappa:
0.488237721832
Confusion matrix:
[[495 49 0 103]
[ 38 139 1 41]
[ 8 5 0 11]
[ 38 5 11 100]]
AUC score:
0.775797251853
('Epoch', 7, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.2519 - acc: 0.9208Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 160s - loss: 0.2519 - acc: 0.9208 - val_loss: 0.9241 - val_acc: 0.7414
Performance of model on test set ----------------------------
Accuracy:
0.743295019157
Kappa:
0.518965143747
Confusion matrix:
[[555 73 1 18]
[ 49 158 0 12]
[ 8 11 1 4]
[ 66 6 20 62]]
AUC score:
0.775619390777
('Epoch', 8, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.2048 - acc: 0.9387Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 161s - loss: 0.2048 - acc: 0.9387 - val_loss: 0.9090 - val_acc: 0.7385
Performance of model on test set ----------------------------
Accuracy:
0.738505747126
Kappa:
0.507679020236
Confusion matrix:
[[563 42 0 42]
[ 70 131 1 17]
[ 8 7 0 9]
[ 49 11 17 77]]
AUC score:
0.793998677319
('Epoch', 9, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1780 - acc: 0.9465Epoch 00000: val_acc improved from 0.75096 to 0.76341, saving model to social.weights--1lstmbis.best.hdf5
4472/4472 [==============================] - 155s - loss: 0.1779 - acc: 0.9466 - val_loss: 0.9042 - val_acc: 0.7634
Performance of model on test set ----------------------------
Accuracy:
0.763409961686
Kappa:
0.549098259468
Confusion matrix:
[[575 46 0 26]
[ 59 139 0 21]
[ 11 9 0 4]
[ 57 2 12 83]]
AUC score:
0.788138750602
('Epoch', 10, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1490 - acc: 0.9568Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 163s - loss: 0.1489 - acc: 0.9568 - val_loss: 1.0894 - val_acc: 0.7490
Performance of model on test set ----------------------------
Accuracy:
0.75
Kappa:
0.503935072666
Confusion matrix:
[[596 19 1 31]
[ 93 110 0 16]
[ 12 7 2 3]
[ 57 3 19 75]]
AUC score:
0.815907899773
('Epoch', 11, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1528 - acc: 0.9526Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 155s - loss: 0.1528 - acc: 0.9526 - val_loss: 1.1385 - val_acc: 0.7404
Performance of model on test set ----------------------------
Accuracy:
0.741379310345
Kappa:
0.472976687052
Confusion matrix:
[[602 14 1 30]
[112 92 1 14]
[ 17 3 0 4]
[ 58 3 13 80]]
AUC score:
0.759060674943
('Epoch', 12, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1355 - acc: 0.9580Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 165s - loss: 0.1355 - acc: 0.9580 - val_loss: 1.0679 - val_acc: 0.7605
Performance of model on test set ----------------------------
Accuracy:
0.759578544061
Kappa:
0.541791399715
Confusion matrix:
[[580 35 6 26]
[ 68 134 2 15]
[ 10 7 2 5]
[ 51 5 21 77]]
AUC score:
0.789882086871
('Epoch', 13, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1166 - acc: 0.9638Epoch 00000: val_acc improved from 0.76341 to 0.77490, saving model to social.weights--1lstmbis.best.hdf5
4472/4472 [==============================] - 162s - loss: 0.1165 - acc: 0.9638 - val_loss: 1.0028 - val_acc: 0.7749
Performance of model on test set ----------------------------
Accuracy:
0.774904214559
Kappa:
0.563617347873
Confusion matrix:
[[588 50 1 8]
[ 61 153 0 5]
[ 10 12 2 0]
[ 60 13 15 66]]
AUC score:
0.787330885194
('Epoch', 14, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.1084 - acc: 0.9651Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 161s - loss: 0.1083 - acc: 0.9651 - val_loss: 1.1584 - val_acc: 0.7500
Performance of model on test set ----------------------------
Accuracy:
0.750957854406
Kappa:
0.543690185405
Confusion matrix:
[[555 49 4 39]
[ 57 142 0 20]
[ 8 10 0 6]
[ 37 9 21 87]]
AUC score:
0.76553185908
('Epoch', 15, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0951 - acc: 0.9711Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 161s - loss: 0.0951 - acc: 0.9712 - val_loss: 1.1925 - val_acc: 0.7538
Performance of model on test set ----------------------------
Accuracy:
0.753831417625
Kappa:
0.53967306267
Confusion matrix:
[[570 19 0 58]
[ 68 122 0 29]
[ 13 4 0 7]
[ 36 6 17 95]]
AUC score:
0.755979163242
('Epoch', 16, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0786 - acc: 0.9761Epoch 00000: val_acc improved from 0.77490 to 0.77874, saving model to social.weights--1lstmbis.best.hdf5
4472/4472 [==============================] - 157s - loss: 0.0786 - acc: 0.9761 - val_loss: 1.2029 - val_acc: 0.7787
Performance of model on test set ----------------------------
Accuracy:
0.778735632184
Kappa:
0.554062300065
Confusion matrix:
[[611 27 0 9]
[ 92 123 0 4]
[ 16 6 1 1]
[ 52 7 17 78]]
AUC score:
0.765887226739
('Epoch', 17, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0912 - acc: 0.9734Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 160s - loss: 0.0911 - acc: 0.9734 - val_loss: 1.1323 - val_acc: 0.7471
Performance of model on test set ----------------------------
Accuracy:
0.747126436782
Kappa:
0.534822961129
Confusion matrix:
[[551 38 0 58]
[ 53 145 2 19]
[ 12 6 2 4]
[ 45 13 14 82]]
AUC score:
0.796564159324
('Epoch', 18, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0725 - acc: 0.9783Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 164s - loss: 0.0725 - acc: 0.9783 - val_loss: 1.1995 - val_acc: 0.7596
Performance of model on test set ----------------------------
Accuracy:
0.760536398467
Kappa:
0.554253802078
Confusion matrix:
[[565 36 2 44]
[ 61 131 3 24]
[ 14 4 3 3]
[ 40 4 15 95]]
AUC score:
0.791473584137
('Epoch', 19, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0739 - acc: 0.9792Epoch 00000: val_acc improved from 0.77874 to 0.78161, saving model to social.weights--1lstmbis.best.hdf5
4472/4472 [==============================] - 159s - loss: 0.0739 - acc: 0.9792 - val_loss: 1.0977 - val_acc: 0.7816
Performance of model on test set ----------------------------
Accuracy:
0.782567049808
Kappa:
0.577736123386
Confusion matrix:
[[595 31 0 21]
[ 68 137 2 12]
[ 15 4 2 3]
[ 48 12 11 83]]
AUC score:
0.818198096254
('Epoch', 20, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0678 - acc: 0.9808Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 161s - loss: 0.0678 - acc: 0.9808 - val_loss: 1.2065 - val_acc: 0.7653
Performance of model on test set ----------------------------
Accuracy:
0.765325670498
Kappa:
0.547680309823
Confusion matrix:
[[584 42 1 20]
[ 67 139 0 13]
[ 12 8 2 2]
[ 52 15 13 74]]
AUC score:
0.829106067408
('Epoch', 21, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0517 - acc: 0.9843Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 176s - loss: 0.0517 - acc: 0.9843 - val_loss: 1.1485 - val_acc: 0.7807
Performance of model on test set ----------------------------
Accuracy:
0.781609195402
Kappa:
0.584495745145
Confusion matrix:
[[579 50 1 17]
[ 52 158 0 9]
[ 10 12 2 0]
[ 54 14 9 77]]
AUC score:
0.815023929052
('Epoch', 22, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0578 - acc: 0.9801Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 171s - loss: 0.0578 - acc: 0.9801 - val_loss: 1.2349 - val_acc: 0.7778
Performance of model on test set ----------------------------
Accuracy:
0.778735632184
Kappa:
0.552848513166
Confusion matrix:
[[609 27 1 10]
[ 80 130 0 9]
[ 18 5 0 1]
[ 65 4 11 74]]
AUC score:
0.805724575696
('Epoch', 23, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0518 - acc: 0.9841Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 164s - loss: 0.0518 - acc: 0.9841 - val_loss: 1.2892 - val_acc: 0.7682
Performance of model on test set ----------------------------
Accuracy:
0.768199233716
Kappa:
0.537561843471
Confusion matrix:
[[598 30 1 18]
[ 76 132 0 11]
[ 13 5 2 4]
[ 70 5 9 70]]
AUC score:
0.811694546466
('Epoch', 24, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0438 - acc: 0.9886Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 154s - loss: 0.0438 - acc: 0.9886 - val_loss: 1.3982 - val_acc: 0.7625
Performance of model on test set ----------------------------
Accuracy:
0.761494252874
Kappa:
0.530190377393
Confusion matrix:
[[595 25 0 27]
[ 77 122 0 20]
[ 16 3 2 3]
[ 58 6 14 76]]
AUC score:
0.806548883198
('Epoch', 25, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0357 - acc: 0.9875Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 159s - loss: 0.0357 - acc: 0.9875 - val_loss: 1.4123 - val_acc: 0.7615
Performance of model on test set ----------------------------
Accuracy:
0.761494252874
Kappa:
0.548602863037
Confusion matrix:
[[580 44 0 23]
[ 54 149 0 16]
[ 10 9 1 4]
[ 52 10 27 65]]
AUC score:
0.773331646424
('Epoch', 26, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0466 - acc: 0.9870Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 162s - loss: 0.0466 - acc: 0.9870 - val_loss: 1.3295 - val_acc: 0.7634
Performance of model on test set ----------------------------
Accuracy:
0.763409961686
Kappa:
0.541156876561
Confusion matrix:
[[584 41 0 22]
[ 68 138 1 12]
[ 7 9 3 5]
[ 64 7 11 72]]
AUC score:
0.806644672206
('Epoch', 27, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0376 - acc: 0.9886Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 165s - loss: 0.0376 - acc: 0.9886 - val_loss: 1.4383 - val_acc: 0.7596
Performance of model on test set ----------------------------
Accuracy:
0.759578544061
Kappa:
0.528425583835
Confusion matrix:
[[590 40 1 16]
[ 72 135 0 12]
[ 11 8 2 3]
[ 64 9 15 66]]
AUC score:
0.796339467796
('Epoch', 28, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0387 - acc: 0.9888Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 164s - loss: 0.0387 - acc: 0.9888 - val_loss: 1.4844 - val_acc: 0.7653
Performance of model on test set ----------------------------
Accuracy:
0.765325670498
Kappa:
0.537148627081
Confusion matrix:
[[596 36 3 12]
[ 74 132 1 12]
[ 9 8 3 4]
[ 65 10 11 68]]
AUC score:
0.81090257353
('Epoch', 29, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0374 - acc: 0.9893Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 163s - loss: 0.0374 - acc: 0.9893 - val_loss: 1.5268 - val_acc: 0.7567
Performance of model on test set ----------------------------
Accuracy:
0.756704980843
Kappa:
0.533993276384
Confusion matrix:
[[575 40 0 32]
[ 61 139 0 19]
[ 9 8 1 6]
[ 62 9 8 75]]
AUC score:
0.774397184061
('Epoch', 30, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0259 - acc: 0.9924Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 165s - loss: 0.0259 - acc: 0.9924 - val_loss: 1.5792 - val_acc: 0.7519
Performance of model on test set ----------------------------
Accuracy:
0.751915708812
Kappa:
0.515567809865
Confusion matrix:
[[584 34 0 29]
[ 73 128 0 18]
[ 11 8 2 3]
[ 65 6 12 71]]
AUC score:
0.778765668776
('Epoch', 31, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0320 - acc: 0.9911Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 166s - loss: 0.0320 - acc: 0.9911 - val_loss: 1.5283 - val_acc: 0.7692
Performance of model on test set ----------------------------
Accuracy:
0.769157088123
Kappa:
0.540747933764
Confusion matrix:
[[597 31 0 19]
[ 79 131 1 8]
[ 9 10 2 3]
[ 68 5 8 73]]
AUC score:
0.770087474871
('Epoch', 32, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0212 - acc: 0.9942Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 168s - loss: 0.0212 - acc: 0.9942 - val_loss: 1.4537 - val_acc: 0.7759
Performance of model on test set ----------------------------
Accuracy:
0.775862068966
Kappa:
0.559312494588
Confusion matrix:
[[591 41 1 14]
[ 67 144 1 7]
[ 10 10 3 1]
[ 68 9 5 72]]
AUC score:
0.795659602329
('Epoch', 33, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0244 - acc: 0.9931Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 168s - loss: 0.0244 - acc: 0.9931 - val_loss: 1.4736 - val_acc: 0.7720
Performance of model on test set ----------------------------
Accuracy:
0.772030651341
Kappa:
0.565544590464
Confusion matrix:
[[573 52 1 21]
[ 57 150 0 12]
[ 8 11 2 3]
[ 60 7 6 81]]
AUC score:
0.795522320509
('Epoch', 34, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0237 - acc: 0.9919Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 168s - loss: 0.0237 - acc: 0.9919 - val_loss: 1.4755 - val_acc: 0.7759
Performance of model on test set ----------------------------
Accuracy:
0.775862068966
Kappa:
0.564883008074
Confusion matrix:
[[588 43 1 15]
[ 64 145 0 10]
[ 11 9 2 2]
[ 59 10 10 75]]
AUC score:
0.795129962817
('Epoch', 35, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0155 - acc: 0.9951Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 168s - loss: 0.0155 - acc: 0.9951 - val_loss: 1.5930 - val_acc: 0.7739
Performance of model on test set ----------------------------
Accuracy:
0.773946360153
Kappa:
0.541558282072
Confusion matrix:
[[607 34 0 6]
[ 81 134 1 3]
[ 15 6 2 1]
[ 71 8 10 65]]
AUC score:
0.812868035658
('Epoch', 36, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0234 - acc: 0.9926Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 170s - loss: 0.0234 - acc: 0.9926 - val_loss: 1.5279 - val_acc: 0.7749
Performance of model on test set ----------------------------
Accuracy:
0.774904214559
Kappa:
0.565263616743
Confusion matrix:
[[584 45 0 18]
[ 60 148 0 11]
[ 11 9 3 1]
[ 61 8 11 74]]
AUC score:
0.818930360697
('Epoch', 37, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0315 - acc: 0.9924Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 167s - loss: 0.0315 - acc: 0.9924 - val_loss: 1.4719 - val_acc: 0.7759
Performance of model on test set ----------------------------
Accuracy:
0.775862068966
Kappa:
0.561408327813
Confusion matrix:
[[588 42 0 17]
[ 74 138 0 7]
[ 15 5 3 1]
[ 56 7 10 81]]
AUC score:
0.823194702356
('Epoch', 38, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0257 - acc: 0.9926Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 179s - loss: 0.0257 - acc: 0.9926 - val_loss: 1.5362 - val_acc: 0.7739
Performance of model on test set ----------------------------
Accuracy:
0.773946360153
Kappa:
0.572097708386
Confusion matrix:
[[573 54 0 20]
[ 51 159 0 9]
[ 9 10 2 3]
[ 57 10 13 74]]
AUC score:
0.774296432221
('Epoch', 39, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0294 - acc: 0.9917Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 182s - loss: 0.0294 - acc: 0.9917 - val_loss: 1.5435 - val_acc: 0.7682
Performance of model on test set ----------------------------
Accuracy:
0.768199233716
Kappa:
0.539288859875
Confusion matrix:
[[600 42 0 5]
[ 73 136 0 10]
[ 15 8 1 0]
[ 63 10 16 65]]
AUC score:
0.780648311379
('Epoch', 40, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0249 - acc: 0.9926Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 179s - loss: 0.0249 - acc: 0.9926 - val_loss: 1.4784 - val_acc: 0.7672
Performance of model on test set ----------------------------
Accuracy:
0.76724137931
Kappa:
0.539889584527
Confusion matrix:
[[595 40 0 12]
[ 67 136 1 15]
[ 17 3 1 3]
[ 67 5 13 69]]
AUC score:
0.838466076765
('Epoch', 41, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0192 - acc: 0.9962Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 183s - loss: 0.0192 - acc: 0.9962 - val_loss: 1.6417 - val_acc: 0.7625
Performance of model on test set ----------------------------
Accuracy:
0.763409961686
Kappa:
0.519764005207
Confusion matrix:
[[608 28 0 11]
[ 86 124 0 9]
[ 12 7 1 4]
[ 72 4 14 64]]
AUC score:
0.79438679447
('Epoch', 42, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0122 - acc: 0.9958Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 187s - loss: 0.0122 - acc: 0.9958 - val_loss: 1.7576 - val_acc: 0.7577
Performance of model on test set ----------------------------
Accuracy:
0.757662835249
Kappa:
0.522048951024
Confusion matrix:
[[593 21 0 33]
[ 74 120 0 25]
[ 13 6 2 3]
[ 67 2 9 76]]
AUC score:
0.787221656646
('Epoch', 43, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0155 - acc: 0.9953Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 184s - loss: 0.0155 - acc: 0.9953 - val_loss: 1.7574 - val_acc: 0.7625
Performance of model on test set ----------------------------
Accuracy:
0.763409961686
Kappa:
0.532062844554
Confusion matrix:
[[594 29 0 24]
[ 75 128 0 16]
[ 11 8 2 3]
[ 67 6 8 73]]
AUC score:
0.791987829422
('Epoch', 44, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0273 - acc: 0.9928Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 185s - loss: 0.0273 - acc: 0.9928 - val_loss: 1.5533 - val_acc: 0.7739
Performance of model on test set ----------------------------
Accuracy:
0.773946360153
Kappa:
0.554483481909
Confusion matrix:
[[592 34 0 21]
[ 66 143 0 10]
[ 12 9 2 1]
[ 70 6 7 71]]
AUC score:
0.793431044254
('Epoch', 45, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0147 - acc: 0.9953Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 188s - loss: 0.0147 - acc: 0.9953 - val_loss: 1.6527 - val_acc: 0.7510
Performance of model on test set ----------------------------
Accuracy:
0.750957854406
Kappa:
0.528741818434
Confusion matrix:
[[562 47 0 38]
[ 55 149 1 14]
[ 9 9 2 4]
[ 64 13 6 71]]
AUC score:
0.778150691628
('Epoch', 46, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0185 - acc: 0.9942Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 190s - loss: 0.0185 - acc: 0.9942 - val_loss: 1.7957 - val_acc: 0.7692
Performance of model on test set ----------------------------
Accuracy:
0.769157088123
Kappa:
0.535869041887
Confusion matrix:
[[603 30 0 14]
[ 93 120 0 6]
[ 12 7 1 4]
[ 58 8 9 79]]
AUC score:
0.798454712623
('Epoch', 47, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0177 - acc: 0.9953Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 181s - loss: 0.0177 - acc: 0.9953 - val_loss: 1.6226 - val_acc: 0.7653
Performance of model on test set ----------------------------
Accuracy:
0.765325670498
Kappa:
0.526476267349
Confusion matrix:
[[606 35 0 6]
[ 83 133 0 3]
[ 13 7 2 2]
[ 67 13 16 58]]
AUC score:
0.819793793644
('Epoch', 48, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0161 - acc: 0.9962Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 151s - loss: 0.0161 - acc: 0.9962 - val_loss: 1.6681 - val_acc: 0.7692
Performance of model on test set ----------------------------
Accuracy:
0.769157088123
Kappa:
0.539308314291
Confusion matrix:
[[599 33 1 14]
[ 86 127 0 6]
[ 15 7 1 1]
[ 57 10 11 76]]
AUC score:
0.788989870811
('Epoch', 49, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0082 - acc: 0.9971Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 152s - loss: 0.0082 - acc: 0.9971 - val_loss: 1.7566 - val_acc: 0.7720
Performance of model on test set ----------------------------
Accuracy:
0.772988505747
Kappa:
0.558234494875
Confusion matrix:
[[591 38 0 18]
[ 70 139 0 10]
[ 16 6 1 1]
[ 50 13 15 76]]
AUC score:
0.79457065569
('Epoch', 50, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0109 - acc: 0.9971Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 153s - loss: 0.0109 - acc: 0.9971 - val_loss: 1.9237 - val_acc: 0.7711
Performance of model on test set ----------------------------
Accuracy:
0.771072796935
Kappa:
0.542927276058
Confusion matrix:
[[605 29 0 13]
[ 81 135 0 3]
[ 13 7 1 3]
[ 61 7 22 64]]
AUC score:
0.771586253824
('Epoch', 51, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0094 - acc: 0.9980Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 153s - loss: 0.0094 - acc: 0.9980 - val_loss: 1.9902 - val_acc: 0.7720
Performance of model on test set ----------------------------
Accuracy:
0.772988505747
Kappa:
0.542158642397
Confusion matrix:
[[605 32 1 9]
[ 78 135 0 6]
[ 14 8 1 1]
[ 72 3 13 66]]
AUC score:
0.772853021611
('Epoch', 52, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0224 - acc: 0.9946Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 153s - loss: 0.0223 - acc: 0.9946 - val_loss: 1.8513 - val_acc: 0.7672
Performance of model on test set ----------------------------
Accuracy:
0.768199233716
Kappa:
0.545940933214
Confusion matrix:
[[593 25 1 28]
[ 72 130 0 17]
[ 12 7 1 4]
[ 61 3 12 78]]
AUC score:
0.728702901774
('Epoch', 53, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0108 - acc: 0.9964Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 153s - loss: 0.0108 - acc: 0.9964 - val_loss: 1.8715 - val_acc: 0.7759
Performance of model on test set ----------------------------
Accuracy:
0.776819923372
Kappa:
0.539834057864
Confusion matrix:
[[617 20 1 9]
[ 88 127 0 4]
[ 15 7 1 1]
[ 74 6 8 66]]
AUC score:
0.767265914127
('Epoch', 54, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0138 - acc: 0.9969Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 162s - loss: 0.0138 - acc: 0.9969 - val_loss: 1.7414 - val_acc: 0.7778
Performance of model on test set ----------------------------
Accuracy:
0.778735632184
Kappa:
0.55060040698
Confusion matrix:
[[613 24 0 10]
[ 86 129 0 4]
[ 12 9 1 2]
[ 67 4 13 70]]
AUC score:
0.779714259769
('Epoch', 55, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0040 - acc: 0.9991Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 164s - loss: 0.0040 - acc: 0.9991 - val_loss: 1.9963 - val_acc: 0.7663
Performance of model on test set ----------------------------
Accuracy:
0.76724137931
Kappa:
0.534668048461
Confusion matrix:
[[601 24 0 22]
[ 80 130 0 9]
[ 11 7 1 5]
[ 68 6 11 69]]
AUC score:
0.755331202031
('Epoch', 56, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0087 - acc: 0.9982Epoch 00000: val_acc improved from 0.78161 to 0.78448, saving model to social.weights--1lstmbis.best.hdf5
4472/4472 [==============================] - 164s - loss: 0.0087 - acc: 0.9982 - val_loss: 1.9788 - val_acc: 0.7845
Performance of model on test set ----------------------------
Accuracy:
0.786398467433
Kappa:
0.581158124448
Confusion matrix:
[[604 24 0 19]
[ 60 144 1 14]
[ 9 8 1 6]
[ 63 9 10 72]]
AUC score:
0.759208943535
('Epoch', 57, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0111 - acc: 0.9978Epoch 00000: val_acc improved from 0.78448 to 0.78544, saving model to social.weights--1lstmbis.best.hdf5
4472/4472 [==============================] - 163s - loss: 0.0111 - acc: 0.9978 - val_loss: 1.9483 - val_acc: 0.7854
Performance of model on test set ----------------------------
Accuracy:
0.786398467433
Kappa:
0.577816947382
Confusion matrix:
[[601 33 0 13]
[ 71 139 0 9]
[ 12 8 1 3]
[ 60 6 8 80]]
AUC score:
0.796197920131
('Epoch', 58, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0162 - acc: 0.9962Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 163s - loss: 0.0162 - acc: 0.9962 - val_loss: 1.8883 - val_acc: 0.7797
Performance of model on test set ----------------------------
Accuracy:
0.780651340996
Kappa:
0.566653434766
Confusion matrix:
[[602 31 0 14]
[ 73 135 0 11]
[ 13 5 1 5]
[ 58 6 13 77]]
AUC score:
0.814736327888
('Epoch', 59, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0078 - acc: 0.9982Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 165s - loss: 0.0078 - acc: 0.9982 - val_loss: 1.8318 - val_acc: 0.7854
Performance of model on test set ----------------------------
Accuracy:
0.785440613027
Kappa:
0.57587760343
Confusion matrix:
[[604 30 0 13]
[ 78 135 1 5]
[ 16 5 2 1]
[ 51 5 19 79]]
AUC score:
0.803840013455
('Epoch', 60, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0111 - acc: 0.9973Epoch 00000: val_acc improved from 0.78544 to 0.79502, saving model to social.weights--1lstmbis.best.hdf5
4472/4472 [==============================] - 164s - loss: 0.0111 - acc: 0.9973 - val_loss: 1.8957 - val_acc: 0.7950
Performance of model on test set ----------------------------
Accuracy:
0.795977011494
Kappa:
0.597874833632
Confusion matrix:
[[605 34 0 8]
[ 65 149 0 5]
[ 14 7 1 2]
[ 57 7 14 76]]
AUC score:
0.802856970347
('Epoch', 61, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0191 - acc: 0.9955Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 164s - loss: 0.0191 - acc: 0.9955 - val_loss: 1.7100 - val_acc: 0.7874
Performance of model on test set ----------------------------
Accuracy:
0.787356321839
Kappa:
0.581141916351
Confusion matrix:
[[602 24 0 21]
[ 72 143 0 4]
[ 15 5 1 3]
[ 55 5 18 76]]
AUC score:
0.792269573422
('Epoch', 62, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0128 - acc: 0.9958Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 163s - loss: 0.0128 - acc: 0.9958 - val_loss: 1.7269 - val_acc: 0.7854
Performance of model on test set ----------------------------
Accuracy:
0.786398467433
Kappa:
0.573594243086
Confusion matrix:
[[608 30 0 9]
[ 72 143 0 4]
[ 14 6 1 3]
[ 62 10 13 69]]
AUC score:
0.797373782144
('Epoch', 63, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0109 - acc: 0.9987Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 164s - loss: 0.0109 - acc: 0.9987 - val_loss: 1.8713 - val_acc: 0.7826
Performance of model on test set ----------------------------
Accuracy:
0.783524904215
Kappa:
0.573511681504
Confusion matrix:
[[601 34 1 11]
[ 70 142 0 7]
[ 12 9 1 2]
[ 57 12 11 74]]
AUC score:
0.790058558864
('Epoch', 64, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0195 - acc: 0.9955Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 163s - loss: 0.0195 - acc: 0.9955 - val_loss: 1.8353 - val_acc: 0.7787
Performance of model on test set ----------------------------
Accuracy:
0.77969348659
Kappa:
0.569437446319
Confusion matrix:
[[600 33 3 11]
[ 62 147 0 10]
[ 11 10 1 2]
[ 59 13 16 66]]
AUC score:
0.77856995443
('Epoch', 65, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0090 - acc: 0.9973Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 164s - loss: 0.0090 - acc: 0.9973 - val_loss: 1.7135 - val_acc: 0.7807
Performance of model on test set ----------------------------
Accuracy:
0.782567049808
Kappa:
0.572269379254
Confusion matrix:
[[600 34 1 12]
[ 73 137 3 6]
[ 15 6 1 2]
[ 54 6 15 79]]
AUC score:
0.775969706549
('Epoch', 66, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0186 - acc: 0.9958Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 164s - loss: 0.0186 - acc: 0.9958 - val_loss: 1.7041 - val_acc: 0.7759
Performance of model on test set ----------------------------
Accuracy:
0.776819923372
Kappa:
0.561555640069
Confusion matrix:
[[596 42 1 8]
[ 68 144 3 4]
[ 12 10 1 1]
[ 60 13 11 70]]
AUC score:
0.783759807349
('Epoch', 67, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0052 - acc: 0.9980Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 166s - loss: 0.0052 - acc: 0.9980 - val_loss: 1.6760 - val_acc: 0.7778
Performance of model on test set ----------------------------
Accuracy:
0.77969348659
Kappa:
0.574933616569
Confusion matrix:
[[590 46 0 11]
[ 56 149 3 11]
[ 12 10 1 1]
[ 57 10 13 74]]
AUC score:
0.782325112698
('Epoch', 68, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0088 - acc: 0.9987Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 163s - loss: 0.0088 - acc: 0.9987 - val_loss: 1.7081 - val_acc: 0.7874
Performance of model on test set ----------------------------
Accuracy:
0.789272030651
Kappa:
0.588233606792
Confusion matrix:
[[602 30 1 14]
[ 60 145 1 13]
[ 14 9 0 1]
[ 55 11 11 77]]
AUC score:
0.798717542738
('Epoch', 69, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0128 - acc: 0.9969Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 155s - loss: 0.0128 - acc: 0.9969 - val_loss: 1.7655 - val_acc: 0.7845
Performance of model on test set ----------------------------
Accuracy:
0.786398467433
Kappa:
0.574998493951
Confusion matrix:
[[607 30 1 9]
[ 77 137 2 3]
[ 14 8 2 0]
[ 55 13 11 75]]
AUC score:
0.789730961238
('Epoch', 70, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0095 - acc: 0.9971Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 156s - loss: 0.0095 - acc: 0.9971 - val_loss: 1.9179 - val_acc: 0.7701
Performance of model on test set ----------------------------
Accuracy:
0.772030651341
Kappa:
0.558331674298
Confusion matrix:
[[588 48 1 10]
[ 58 151 3 7]
[ 15 9 0 0]
[ 57 18 12 67]]
AUC score:
0.761921226143
('Epoch', 71, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0064 - acc: 0.9982Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 158s - loss: 0.0064 - acc: 0.9982 - val_loss: 1.9085 - val_acc: 0.7797
Performance of model on test set ----------------------------
Accuracy:
0.781609195402
Kappa:
0.572882779262
Confusion matrix:
[[598 32 1 16]
[ 58 151 1 9]
[ 15 9 0 0]
[ 61 11 15 67]]
AUC score:
0.771315163656
('Epoch', 72, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0030 - acc: 0.9989Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 157s - loss: 0.0030 - acc: 0.9989 - val_loss: 1.9739 - val_acc: 0.7797
Performance of model on test set ----------------------------
Accuracy:
0.780651340996
Kappa:
0.576208573819
Confusion matrix:
[[589 36 0 22]
[ 61 144 1 13]
[ 15 7 1 1]
[ 53 9 11 81]]
AUC score:
0.786227992113
('Epoch', 73, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0128 - acc: 0.9973Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 158s - loss: 0.0128 - acc: 0.9973 - val_loss: 1.8304 - val_acc: 0.7778
Performance of model on test set ----------------------------
Accuracy:
0.777777777778
Kappa:
0.571031597739
Confusion matrix:
[[589 38 0 20]
[ 61 145 2 11]
[ 11 11 1 1]
[ 56 8 13 77]]
AUC score:
0.785706532623
('Epoch', 74, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0067 - acc: 0.9980Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 158s - loss: 0.0067 - acc: 0.9980 - val_loss: 1.9941 - val_acc: 0.7701
Performance of model on test set ----------------------------
Accuracy:
0.770114942529
Kappa:
0.543173029414
Confusion matrix:
[[600 27 0 20]
[ 70 137 1 11]
[ 13 9 0 2]
[ 69 6 12 67]]
AUC score:
0.782964317305
('Epoch', 75, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0070 - acc: 0.9978Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 159s - loss: 0.0070 - acc: 0.9978 - val_loss: 1.9822 - val_acc: 0.7711
Performance of model on test set ----------------------------
Accuracy:
0.771072796935
Kappa:
0.557783137996
Confusion matrix:
[[587 37 0 23]
[ 56 152 0 11]
[ 13 9 1 1]
[ 61 12 16 65]]
AUC score:
0.769204508345
('Epoch', 76, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0054 - acc: 0.9991Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 159s - loss: 0.0054 - acc: 0.9991 - val_loss: 1.9531 - val_acc: 0.7711
Performance of model on test set ----------------------------
Accuracy:
0.771072796935
Kappa:
0.551430297277
Confusion matrix:
[[591 38 0 18]
[ 65 144 0 10]
[ 12 10 1 1]
[ 66 6 13 69]]
AUC score:
0.769914339641
('Epoch', 77, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0156 - acc: 0.9960Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 159s - loss: 0.0156 - acc: 0.9960 - val_loss: 1.9212 - val_acc: 0.7711
Performance of model on test set ----------------------------
Accuracy:
0.771072796935
Kappa:
0.544243867779
Confusion matrix:
[[599 35 0 13]
[ 73 138 1 7]
[ 12 11 1 0]
[ 67 12 8 67]]
AUC score:
0.799611120805
('Epoch', 78, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0091 - acc: 0.9978Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 155s - loss: 0.0091 - acc: 0.9978 - val_loss: 1.9635 - val_acc: 0.7682
Performance of model on test set ----------------------------
Accuracy:
0.768199233716
Kappa:
0.546867685752
Confusion matrix:
[[592 35 1 19]
[ 64 141 2 12]
[ 13 10 1 0]
[ 63 11 12 68]]
AUC score:
0.784751734631
('Epoch', 79, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0036 - acc: 0.9989Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 158s - loss: 0.0036 - acc: 0.9989 - val_loss: 1.8824 - val_acc: 0.7874
Performance of model on test set ----------------------------
Accuracy:
0.787356321839
Kappa:
0.594517000971
Confusion matrix:
[[590 34 0 23]
[ 56 144 3 16]
[ 12 8 3 1]
[ 47 8 14 85]]
AUC score:
0.81044236686
('Epoch', 80, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0070 - acc: 0.9984Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 161s - loss: 0.0070 - acc: 0.9984 - val_loss: 1.9998 - val_acc: 0.7787
Performance of model on test set ----------------------------
Accuracy:
0.77969348659
Kappa:
0.576752031897
Confusion matrix:
[[588 37 1 21]
[ 57 148 1 13]
[ 13 10 1 0]
[ 53 11 13 77]]
AUC score:
0.804572657837
('Epoch', 81, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0053 - acc: 0.9980Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 160s - loss: 0.0053 - acc: 0.9980 - val_loss: 2.0328 - val_acc: 0.7749
Performance of model on test set ----------------------------
Accuracy:
0.775862068966
Kappa:
0.566347442509
Confusion matrix:
[[594 34 1 18]
[ 61 145 1 12]
[ 11 9 1 3]
[ 54 15 15 70]]
AUC score:
0.790392343119
('Epoch', 82, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0059 - acc: 0.9989Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 162s - loss: 0.0059 - acc: 0.9989 - val_loss: 2.0441 - val_acc: 0.7835
Performance of model on test set ----------------------------
Accuracy:
0.785440613027
Kappa:
0.579624017171
Confusion matrix:
[[604 29 0 14]
[ 65 143 0 11]
[ 15 7 0 2]
[ 52 11 18 73]]
AUC score:
0.798688160029
('Epoch', 83, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0082 - acc: 0.9982Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 162s - loss: 0.0082 - acc: 0.9982 - val_loss: 2.0054 - val_acc: 0.7739
Performance of model on test set ----------------------------
Accuracy:
0.774904214559
Kappa:
0.572913964812
Confusion matrix:
[[583 41 1 22]
[ 57 149 1 12]
[ 13 8 0 3]
[ 45 11 21 77]]
AUC score:
0.787288199133
('Epoch', 84, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0122 - acc: 0.9973Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 161s - loss: 0.0122 - acc: 0.9973 - val_loss: 1.8862 - val_acc: 0.7768
Performance of model on test set ----------------------------
Accuracy:
0.776819923372
Kappa:
0.572622783191
Confusion matrix:
[[590 33 0 24]
[ 65 140 3 11]
[ 11 8 1 4]
[ 46 7 21 80]]
AUC score:
0.792332525841
('Epoch', 85, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0065 - acc: 0.9984Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 162s - loss: 0.0065 - acc: 0.9984 - val_loss: 1.9416 - val_acc: 0.7692
Performance of model on test set ----------------------------
Accuracy:
0.769157088123
Kappa:
0.550208356052
Confusion matrix:
[[597 35 0 15]
[ 66 142 3 8]
[ 11 10 1 2]
[ 58 8 25 63]]
AUC score:
0.78582428456
('Epoch', 86, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0084 - acc: 0.9980Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 163s - loss: 0.0084 - acc: 0.9980 - val_loss: 2.0884 - val_acc: 0.7596
Performance of model on test set ----------------------------
Accuracy:
0.760536398467
Kappa:
0.53846235462
Confusion matrix:
[[585 41 0 21]
[ 65 143 4 7]
[ 11 10 2 1]
[ 57 11 22 64]]
AUC score:
0.758500857184
('Epoch', 87, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0102 - acc: 0.9975Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 162s - loss: 0.0102 - acc: 0.9975 - val_loss: 1.9097 - val_acc: 0.7701
Performance of model on test set ----------------------------
Accuracy:
0.772030651341
Kappa:
0.554049542872
Confusion matrix:
[[597 32 1 17]
[ 66 139 4 10]
[ 12 9 2 1]
[ 62 5 19 68]]
AUC score:
0.765812261547
('Epoch', 88, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0054 - acc: 0.9991Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 164s - loss: 0.0054 - acc: 0.9991 - val_loss: 2.1373 - val_acc: 0.7586
Performance of model on test set ----------------------------
Accuracy:
0.760536398467
Kappa:
0.533465369188
Confusion matrix:
[[591 29 0 27]
[ 63 137 3 16]
[ 12 7 2 3]
[ 66 7 17 64]]
AUC score:
0.793914277571
('Epoch', 89, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0068 - acc: 0.9989Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 163s - loss: 0.0067 - acc: 0.9989 - val_loss: 2.1181 - val_acc: 0.7682
Performance of model on test set ----------------------------
Accuracy:
0.770114942529
Kappa:
0.554962709299
Confusion matrix:
[[592 34 0 21]
[ 54 148 5 12]
[ 12 11 1 0]
[ 64 10 17 63]]
AUC score:
0.780373929057
('Epoch', 90, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0079 - acc: 0.9978Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 164s - loss: 0.0079 - acc: 0.9978 - val_loss: 2.0787 - val_acc: 0.7711
Performance of model on test set ----------------------------
Accuracy:
0.772030651341
Kappa:
0.56686539065
Confusion matrix:
[[582 40 1 24]
[ 49 153 6 11]
[ 12 10 1 1]
[ 57 9 18 70]]
AUC score:
0.775827852236
('Epoch', 91, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0122 - acc: 0.9971Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 165s - loss: 0.0122 - acc: 0.9971 - val_loss: 2.0246 - val_acc: 0.7692
Performance of model on test set ----------------------------
Accuracy:
0.771072796935
Kappa:
0.557663977365
Confusion matrix:
[[594 38 1 14]
[ 56 146 5 12]
[ 10 12 1 1]
[ 61 8 21 64]]
AUC score:
0.772573094916
('Epoch', 92, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0018 - acc: 0.9993Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 162s - loss: 0.0018 - acc: 0.9993 - val_loss: 2.2245 - val_acc: 0.7567
Performance of model on test set ----------------------------
Accuracy:
0.757662835249
Kappa:
0.5340132634
Confusion matrix:
[[585 39 0 23]
[ 60 141 2 16]
[ 9 12 1 2]
[ 60 11 19 64]]
AUC score:
0.775660370006
('Epoch', 93, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0028 - acc: 0.9993Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 162s - loss: 0.0028 - acc: 0.9993 - val_loss: 2.2711 - val_acc: 0.7653
Performance of model on test set ----------------------------
Accuracy:
0.766283524904
Kappa:
0.560458218373
Confusion matrix:
[[577 46 1 23]
[ 54 150 2 13]
[ 8 13 2 1]
[ 48 13 22 71]]
AUC score:
0.751099342303
('Epoch', 94, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0015 - acc: 0.9998Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 164s - loss: 0.0015 - acc: 0.9998 - val_loss: 2.3002 - val_acc: 0.7749
Performance of model on test set ----------------------------
Accuracy:
0.775862068966
Kappa:
0.572364079395
Confusion matrix:
[[590 38 0 19]
[ 51 153 2 13]
[ 10 12 0 2]
[ 52 15 20 67]]
AUC score:
0.793055787974
('Epoch', 95, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0031 - acc: 0.9989Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 165s - loss: 0.0031 - acc: 0.9989 - val_loss: 2.5763 - val_acc: 0.7682
Performance of model on test set ----------------------------
Accuracy:
0.768199233716
Kappa:
0.549937473502
Confusion matrix:
[[594 36 0 17]
[ 66 142 2 9]
[ 12 11 1 0]
[ 54 13 22 65]]
AUC score:
0.763012717702
('Epoch', 96, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0081 - acc: 0.9989Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 167s - loss: 0.0081 - acc: 0.9989 - val_loss: 2.6266 - val_acc: 0.7644
Performance of model on test set ----------------------------
Accuracy:
0.764367816092
Kappa:
0.539413416738
Confusion matrix:
[[599 30 0 18]
[ 70 131 3 15]
[ 10 10 3 1]
[ 57 13 19 65]]
AUC score:
0.757554057019
('Epoch', 97, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0096 - acc: 0.9984Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 164s - loss: 0.0096 - acc: 0.9984 - val_loss: 2.3792 - val_acc: 0.7653
Performance of model on test set ----------------------------
Accuracy:
0.766283524904
Kappa:
0.546449828986
Confusion matrix:
[[595 36 0 16]
[ 71 132 2 14]
[ 10 10 1 3]
[ 51 10 21 72]]
AUC score:
0.762154101764
('Epoch', 98, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0026 - acc: 0.9993Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 165s - loss: 0.0026 - acc: 0.9993 - val_loss: 2.2996 - val_acc: 0.7701
Performance of model on test set ----------------------------
Accuracy:
0.771072796935
Kappa:
0.566417307442
Confusion matrix:
[[580 39 0 28]
[ 57 145 1 16]
[ 11 10 1 2]
[ 47 12 16 79]]
AUC score:
0.769044011597
('Epoch', 99, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0040 - acc: 0.9989Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 165s - loss: 0.0040 - acc: 0.9989 - val_loss: 2.3816 - val_acc: 0.7739
Performance of model on test set ----------------------------
Accuracy:
0.773946360153
Kappa:
0.554308599834
Confusion matrix:
[[600 29 0 18]
[ 77 131 1 10]
[ 10 9 1 4]
[ 57 8 13 76]]
AUC score:
0.778217009025
('Epoch', 100, '/', 100)
Train on 4472 samples, validate on 1044 samples
Epoch 1/1
4471/4472 [============================>.] - ETA: 0s - loss: 0.0092 - acc: 0.9982Epoch 00000: val_acc did not improve
4472/4472 [==============================] - 163s - loss: 0.0092 - acc: 0.9982 - val_loss: 2.2968 - val_acc: 0.7749
Performance of model on test set ----------------------------
Accuracy:
0.775862068966
Kappa:
0.564028846308
Confusion matrix:
[[597 37 0 13]
[ 65 141 1 12]
[ 11 9 2 2]
[ 54 13 17 70]]
AUC score:
0.779056860083
Best validation accuracy: (59, 0.79501915708812265)
Best validation Kappa: (59, 0.59787483363231297)
Best validation AUC: (39, 0.83846607676470419)
So, apparently, the 3-layer LSTM gets slightly better results than shallower RNNs
Content source: chili-epfl/paper-JLA-deep-teaching-analytics
Similar notebooks: