In [1]:
import numpy as np
import pandas as pd

from gensim import corpora
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize 
from nltk.stem import SnowballStemmer
from w2v import train_word2vec 
import pickle, datetime
import difflib

import keras
from keras.datasets import mnist
from keras.models import Model, Sequential
from keras.utils import np_utils
from keras.layers import Input, Dense, Dropout, Flatten, Embedding, regularizers
from keras.layers import Conv1D, MaxPooling1D
from keras.layers.merge import Concatenate
from keras import optimizers
from keras.preprocessing import sequence
from keras import backend as K

np.random.seed(0)


Using TensorFlow backend.

In [2]:
df = pd.read_csv('../../Datasets/SST1_dataset/Processed_SST1.tsv', sep='\t')
train_df = pd.read_csv('../../Datasets/Kaggle_dataset/train.tsv', sep='\t', header=0)

raw_docs_train      = df[df.split_ind == 1]['Phrases'].values
sentiment_train     = df[df.split_ind == 1]['Label'].values
# raw_docs_train      = train_df['Phrase'].values
# sentiment_train     = train_df['Sentiment'].values
raw_docs_test       = df[df.split_ind == 2]['Phrases'].values
sentiment_test      = df[df.split_ind == 2]['Label'].values
num_labels          = len(np.unique(sentiment_train))

N_TRAIN = len(raw_docs_train)
N_TEST = len(raw_docs_test)

In [3]:
#text pre-processing
stop_words = set(stopwords.words('english'))
stop_words.update(['.', ',', '"', "'", ':', ';', '(', ')', '[', ']', '{', '}'])
stemmer = SnowballStemmer('english')

print ("pre-processing train docs...")
processed_docs_train = []
for doc in raw_docs_train:
   tokens = word_tokenize(doc)
   filtered = [word for word in tokens if word not in stop_words]
   stemmed = [stemmer.stem(word) for word in filtered]
   processed_docs_train.append(stemmed)

print ("pre-processing test docs...")
processed_docs_test = []
for doc in raw_docs_test:
   tokens = word_tokenize(doc)
   filtered = [word for word in tokens if word not in stop_words]
   stemmed = [stemmer.stem(word) for word in filtered]
   processed_docs_test.append(stemmed)
    
processed_docs_all = np.concatenate((processed_docs_train, processed_docs_test), axis=0)

dictionary = corpora.Dictionary(processed_docs_all)
dictionary_size = len(dictionary.keys())

print ("converting to token ids...")
word_id_train, word_id_len = [], []
for doc in processed_docs_train:
    word_ids = [dictionary.token2id[word] for word in doc]
    word_id_train.append(word_ids)
    word_id_len.append(len(word_ids))

word_id_test, word_ids = [], []
for doc in processed_docs_test:
    word_ids = [dictionary.token2id[word] for word in doc]
    word_id_test.append(word_ids)
    word_id_len.append(len(word_ids))
        
seq_len = np.round((np.mean(word_id_len) + 2*np.std(word_id_len))).astype(int)

#pad sequences
x_train = sequence.pad_sequences(np.array(word_id_train), maxlen=seq_len)
x_test  = sequence.pad_sequences(np.array(word_id_test), maxlen=seq_len)
y_train = np_utils.to_categorical(sentiment_train, num_labels)
y_test  = np_utils.to_categorical(sentiment_test, num_labels)


pre-processing train docs...
pre-processing test docs...
converting to token ids...

In [4]:
print(x_train.shape, x_test.shape, y_train.shape, y_test.shape)


(9645, 21) (2210, 21) (9645, 5) (2210, 5)

-------------------------------- Training model -----------------------------------

Model Paremeters


In [5]:
model_type    = 'CNN-non-static'  # CNN-rand|CNN-non-static|CNN-static
embedding_dim = 100         # word2vec dim
vocab_size    = dictionary_size
max_sent_len  = seq_len
N_CATEGORY    = 5

Generate word2vec


In [6]:
if model_type in ['CNN-non-static', 'CNN-static']:
    embedding_wts = train_word2vec( np.vstack((x_train, x_test)), 
                                    dictionary.token2id, num_features = embedding_dim)
    if model_type == 'CNN-static':
        x_train = embedding_wts[0][x_train]
        x_test  = embedding_wts[0][x_test]
        
elif model_type == 'CNN-rand':
    embedding_wts = None
    
else:
    raise ValueError("Unknown model type")


Load existing Word2Vec model '100features_1minwords_10context'

Create model


In [7]:
filter_sizes = [3,4,5]
num_filters  = 50
dropout_prob = (0.5, 0.8)
hidden_dims  = 50
batch_size   = 64
l2_reg = 0.3

# Deciding dimension of input based on the model
input_shape = (max_sent_len, embedding_dim) if model_type == "CNN-static" else (max_sent_len,)
model_input = Input(shape = input_shape)

# Static model do not have embedding layer
if model_type == "CNN-static":
    z = Dropout(dropout_prob[0])(model_input)
else:
    z = Embedding(vocab_size, embedding_dim, input_length = max_sent_len, name="embedding", trainable=True)(model_input)
    z = Dropout(dropout_prob[0])(z)

# Convolution layers
z1 = Conv1D(    filters=num_filters, kernel_size=filter_sizes[0], 
                padding="valid", activation="relu", 
                strides=1)(z)
z1 = MaxPooling1D(pool_size = int(max_sent_len - filter_sizes[0] + 1))(z1)
z1 = Flatten()(z1)

z2 = Conv1D(    filters=num_filters, kernel_size=filter_sizes[1], 
                padding="valid", activation="relu", 
                strides=1)(z)
z2 = MaxPooling1D(pool_size= int(max_sent_len - filter_sizes[1] + 1))(z2)
z2 = Flatten()(z2)

z3 = Conv1D(    filters=num_filters, kernel_size=filter_sizes[2], 
                padding="valid", activation="relu",
                strides=1)(z)
z3 = MaxPooling1D(pool_size= int(max_sent_len - filter_sizes[2] + 1))(z3)
z3 = Flatten()(z3)

# Concatenate the output of all convolution layers
z = Concatenate()([z1, z2, z3])
z = Dropout(dropout_prob[1])(z)

z = Dense(hidden_dims, activation="relu")(z)
model_output = Dense( N_CATEGORY, activation="softmax")(z)
    
model = Model(model_input, model_output)
model.summary()


____________________________________________________________________________________________________
Layer (type)                     Output Shape          Param #     Connected to                     
====================================================================================================
input_1 (InputLayer)             (None, 21)            0                                            
____________________________________________________________________________________________________
embedding (Embedding)            (None, 21, 100)       1380100                                      
____________________________________________________________________________________________________
dropout_1 (Dropout)              (None, 21, 100)       0                                            
____________________________________________________________________________________________________
conv1d_1 (Conv1D)                (None, 19, 50)        15050                                        
____________________________________________________________________________________________________
conv1d_2 (Conv1D)                (None, 18, 50)        20050                                        
____________________________________________________________________________________________________
conv1d_3 (Conv1D)                (None, 17, 50)        25050                                        
____________________________________________________________________________________________________
max_pooling1d_1 (MaxPooling1D)   (None, 1, 50)         0                                            
____________________________________________________________________________________________________
max_pooling1d_2 (MaxPooling1D)   (None, 1, 50)         0                                            
____________________________________________________________________________________________________
max_pooling1d_3 (MaxPooling1D)   (None, 1, 50)         0                                            
____________________________________________________________________________________________________
flatten_1 (Flatten)              (None, 50)            0                                            
____________________________________________________________________________________________________
flatten_2 (Flatten)              (None, 50)            0                                            
____________________________________________________________________________________________________
flatten_3 (Flatten)              (None, 50)            0                                            
____________________________________________________________________________________________________
concatenate_1 (Concatenate)      (None, 150)           0                                            
____________________________________________________________________________________________________
dropout_2 (Dropout)              (None, 150)           0                                            
____________________________________________________________________________________________________
dense_1 (Dense)                  (None, 50)            7550                                         
____________________________________________________________________________________________________
dense_2 (Dense)                  (None, 5)             255                                          
====================================================================================================
Total params: 1,448,055.0
Trainable params: 1,448,055.0
Non-trainable params: 0.0
____________________________________________________________________________________________________

Train model


In [8]:
from sklearn.metrics import roc_auc_score
import time
import pylab as pl
from IPython import display
pl.style.use('ggplot')
%matplotlib inline

class Histories(keras.callbacks.Callback):

    def on_train_begin(self, logs={}):
        self.acc = []
        self.loss = []
        self.val_loss = []
        self.val_acc = []

    def on_train_end(self, logs={}):
        return

    def on_epoch_begin(self, epoch, logs={}):
        return

    def on_epoch_end(self, epoch, logs={}):
        self.acc.append(logs['acc'])
        self.loss.append(logs['loss'])
        self.val_acc.append(logs['val_acc'])
        self.val_loss.append(logs['val_loss'])
        
        pl.hold(True)
        pl.plot(self.acc)
        pl.plot(self.loss)
        pl.plot(self.val_acc)
        pl.plot(self.val_loss)
        pl.legend(['Train acc','Train loss','Valid acc', 'Valid loss'], loc=2)
        display.clear_output(wait=True)
        display.display(pl.gcf())
        return

    def on_batch_begin(self, batch, logs={}):
        return

    def on_batch_end(self, batch, logs={}):
        return

In [ ]:
if model_type == "CNN-non-static":
    embedding_layer = model.get_layer("embedding")
    embedding_layer.set_weights(embedding_wts)

model.compile(loss="categorical_crossentropy", optimizer=optimizers.SGD(), metrics=["accuracy"])

histories = Histories()
res = model.fit(x_train, y_train, 
          batch_size = batch_size,
          epochs=200,
          validation_data=(x_test, y_test), verbose=2, callbacks=[histories])


4s - loss: 1.5501 - acc: 0.3097 - val_loss: 1.5639 - val_acc: 0.3000
Epoch 39/200

In [27]:
import os
os.system('say done')


Out[27]:
256

Computing Accuracy


In [15]:
scores = model.evaluate(x_train, y_train, verbose=0)
print("Train Accuracy: %.2f%%" % (scores[1]*100))
scores = model.evaluate(x_test, y_test, verbose=0)
print("Test Accuracy: %.2f%%" % (scores[1]*100))


Train Accuracy: 38.80%
Test Accuracy: 33.62%

Save the model


In [17]:
date = str(datetime.date.today() )
time = str(datetime.datetime.now().time())[:-7]

filename = './newtrain_' + model_type + '_' + date + '_' +time

with open( filename + '_history', 'wb') as output:
    pickle.dump([model.history.history], output, pickle.HIGHEST_PROTOCOL)
    
model.save(filename + '.h5')

Plot the results


In [20]:
import matplotlib.pyplot as plt
plt.style.use('ggplot')

plt.rcParams["figure.figsize"] = (10,6)
plt.plot(model.history.history['acc'])
plt.plot(model.history.history['loss'])
plt.plot(model.history.history['val_acc'])
plt.plot(model.history.history['val_loss'])
plt.legend(['Train acc','Train loss','Valid acc', 'Valid loss'], loc=2)
plt.xlabel('Epochs')
plt.ylabel('Loss/Accuracy')
plt.title('Using '+ model_type)
# imgName = 'Images/' + model_type + '_' + date + '_' + time + '.jpg'
# plt.savefig( imgName, dpi= 200, bbox_inches='tight', transparent=False)
plt.show()


---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-20-e99e705ccc69> in <module>()
      3 
      4 plt.rcParams["figure.figsize"] = (10,6)
----> 5 plt.plot(model.history.history['acc'])
      6 plt.plot(model.history.history['loss'])
      7 plt.plot(model.history.history['val_acc'])

AttributeError: 'Model' object has no attribute 'history'

Continue from a pretrained model


In [19]:
import pickle

# filename = './newtrain_CNN-rand_2017-04-08_16:29:11'
model = keras.models.load_model(filename)

In [38]:
model.compile(loss="categorical_crossentropy", optimizer=optimizers.SGD(0.005), metrics=["accuracy"])

histories = Histories()
res = model.fit(x_train, y_train,
          batch_size=batch_size,
          epochs=110,
          validation_data=(x_test, y_test), verbose=2, callbacks=[histories])


Train on 156060 samples, validate on 2210 samples
Epoch 1/300
27s - loss: 0.8415 - acc: 0.6568 - val_loss: 1.4342 - val_acc: 0.4005
Epoch 2/300
27s - loss: 0.8416 - acc: 0.6565 - val_loss: 1.4266 - val_acc: 0.4005
Epoch 3/300
24s - loss: 0.8389 - acc: 0.6578 - val_loss: 1.4288 - val_acc: 0.4014
Epoch 4/300
26s - loss: 0.8403 - acc: 0.6564 - val_loss: 1.4244 - val_acc: 0.4041
Epoch 5/300
25s - loss: 0.8391 - acc: 0.6589 - val_loss: 1.4318 - val_acc: 0.4009
Epoch 6/300
25s - loss: 0.8393 - acc: 0.6585 - val_loss: 1.4242 - val_acc: 0.4023
Epoch 7/300
25s - loss: 0.8373 - acc: 0.6582 - val_loss: 1.4232 - val_acc: 0.4023
Epoch 8/300
25s - loss: 0.8398 - acc: 0.6571 - val_loss: 1.4261 - val_acc: 0.3986
Epoch 9/300
25s - loss: 0.8365 - acc: 0.6595 - val_loss: 1.4225 - val_acc: 0.4014
Epoch 10/300
26s - loss: 0.8355 - acc: 0.6593 - val_loss: 1.4259 - val_acc: 0.4063
Epoch 11/300
30s - loss: 0.8350 - acc: 0.6591 - val_loss: 1.4273 - val_acc: 0.4072
Epoch 12/300
26s - loss: 0.8342 - acc: 0.6587 - val_loss: 1.4296 - val_acc: 0.4050
Epoch 13/300
28s - loss: 0.8342 - acc: 0.6591 - val_loss: 1.4270 - val_acc: 0.4045
Epoch 14/300
25s - loss: 0.8343 - acc: 0.6595 - val_loss: 1.4412 - val_acc: 0.4045
Epoch 15/300
26s - loss: 0.8325 - acc: 0.6605 - val_loss: 1.4350 - val_acc: 0.4023
Epoch 16/300
27s - loss: 0.8315 - acc: 0.6603 - val_loss: 1.4337 - val_acc: 0.4045
Epoch 17/300
27s - loss: 0.8312 - acc: 0.6593 - val_loss: 1.4144 - val_acc: 0.4086
Epoch 18/300
25s - loss: 0.8316 - acc: 0.6600 - val_loss: 1.4393 - val_acc: 0.4009
Epoch 19/300
28s - loss: 0.8295 - acc: 0.6611 - val_loss: 1.4405 - val_acc: 0.4059
Epoch 20/300
25s - loss: 0.8276 - acc: 0.6612 - val_loss: 1.4303 - val_acc: 0.4063
Epoch 21/300
25s - loss: 0.8287 - acc: 0.6620 - val_loss: 1.4468 - val_acc: 0.3937
Epoch 22/300
26s - loss: 0.8290 - acc: 0.6592 - val_loss: 1.4310 - val_acc: 0.4077
Epoch 23/300
25s - loss: 0.8298 - acc: 0.6605 - val_loss: 1.4380 - val_acc: 0.4036
Epoch 24/300
26s - loss: 0.8296 - acc: 0.6608 - val_loss: 1.4438 - val_acc: 0.3986
Epoch 25/300
24s - loss: 0.8277 - acc: 0.6621 - val_loss: 1.4290 - val_acc: 0.4086
Epoch 26/300
26s - loss: 0.8269 - acc: 0.6619 - val_loss: 1.4403 - val_acc: 0.4063
Epoch 27/300
24s - loss: 0.8272 - acc: 0.6614 - val_loss: 1.4415 - val_acc: 0.4109
Epoch 28/300
25s - loss: 0.8257 - acc: 0.6620 - val_loss: 1.4396 - val_acc: 0.4054
Epoch 29/300
26s - loss: 0.8249 - acc: 0.6626 - val_loss: 1.4366 - val_acc: 0.4036
Epoch 30/300
25s - loss: 0.8269 - acc: 0.6618 - val_loss: 1.4400 - val_acc: 0.4068
Epoch 31/300
26s - loss: 0.8230 - acc: 0.6623 - val_loss: 1.4387 - val_acc: 0.4113
Epoch 32/300
24s - loss: 0.8256 - acc: 0.6622 - val_loss: 1.4474 - val_acc: 0.4063
Epoch 33/300
25s - loss: 0.8233 - acc: 0.6626 - val_loss: 1.4353 - val_acc: 0.4104
Epoch 34/300
25s - loss: 0.8241 - acc: 0.6626 - val_loss: 1.4275 - val_acc: 0.4095
Epoch 35/300
24s - loss: 0.8241 - acc: 0.6624 - val_loss: 1.4368 - val_acc: 0.4127
Epoch 36/300
25s - loss: 0.8218 - acc: 0.6637 - val_loss: 1.4433 - val_acc: 0.4118
Epoch 37/300
24s - loss: 0.8205 - acc: 0.6650 - val_loss: 1.4562 - val_acc: 0.4059
Epoch 38/300
26s - loss: 0.8235 - acc: 0.6627 - val_loss: 1.4282 - val_acc: 0.4154
Epoch 39/300
25s - loss: 0.8208 - acc: 0.6643 - val_loss: 1.4388 - val_acc: 0.4136
Epoch 40/300
24s - loss: 0.8192 - acc: 0.6646 - val_loss: 1.4317 - val_acc: 0.4149
Epoch 41/300
25s - loss: 0.8194 - acc: 0.6651 - val_loss: 1.4424 - val_acc: 0.4090
Epoch 42/300
24s - loss: 0.8191 - acc: 0.6639 - val_loss: 1.4440 - val_acc: 0.4127
Epoch 43/300
25s - loss: 0.8198 - acc: 0.6639 - val_loss: 1.4493 - val_acc: 0.4113
Epoch 44/300
25s - loss: 0.8170 - acc: 0.6658 - val_loss: 1.4472 - val_acc: 0.4054
Epoch 45/300
28s - loss: 0.8185 - acc: 0.6664 - val_loss: 1.4462 - val_acc: 0.4063
Epoch 46/300
24s - loss: 0.8176 - acc: 0.6648 - val_loss: 1.4263 - val_acc: 0.4113
Epoch 47/300
24s - loss: 0.8181 - acc: 0.6667 - val_loss: 1.4500 - val_acc: 0.4086
Epoch 48/300
26s - loss: 0.8155 - acc: 0.6653 - val_loss: 1.4371 - val_acc: 0.4158
Epoch 49/300
24s - loss: 0.8176 - acc: 0.6645 - val_loss: 1.4470 - val_acc: 0.4122
Epoch 50/300
25s - loss: 0.8156 - acc: 0.6656 - val_loss: 1.4473 - val_acc: 0.4104
Epoch 51/300
24s - loss: 0.8144 - acc: 0.6667 - val_loss: 1.4421 - val_acc: 0.4090
Epoch 52/300
25s - loss: 0.8133 - acc: 0.6673 - val_loss: 1.4534 - val_acc: 0.4072
Epoch 53/300
25s - loss: 0.8140 - acc: 0.6675 - val_loss: 1.4375 - val_acc: 0.4145
Epoch 54/300
24s - loss: 0.8124 - acc: 0.6677 - val_loss: 1.4605 - val_acc: 0.4072
Epoch 55/300
26s - loss: 0.8141 - acc: 0.6664 - val_loss: 1.4503 - val_acc: 0.4122
Epoch 56/300
24s - loss: 0.8111 - acc: 0.6671 - val_loss: 1.4561 - val_acc: 0.4100
Epoch 57/300
26s - loss: 0.8117 - acc: 0.6669 - val_loss: 1.4462 - val_acc: 0.4158
Epoch 58/300
24s - loss: 0.8113 - acc: 0.6668 - val_loss: 1.4386 - val_acc: 0.4113
Epoch 59/300
26s - loss: 0.8129 - acc: 0.6669 - val_loss: 1.4504 - val_acc: 0.4086
Epoch 60/300
24s - loss: 0.8113 - acc: 0.6670 - val_loss: 1.4421 - val_acc: 0.4140
Epoch 61/300
25s - loss: 0.8105 - acc: 0.6670 - val_loss: 1.4441 - val_acc: 0.4140
Epoch 62/300
27s - loss: 0.8103 - acc: 0.6666 - val_loss: 1.4585 - val_acc: 0.4109
Epoch 63/300
24s - loss: 0.8088 - acc: 0.6695 - val_loss: 1.4680 - val_acc: 0.4095
Epoch 64/300
25s - loss: 0.8097 - acc: 0.6677 - val_loss: 1.4450 - val_acc: 0.4140
Epoch 65/300
24s - loss: 0.8100 - acc: 0.6673 - val_loss: 1.4525 - val_acc: 0.4140
Epoch 66/300
25s - loss: 0.8102 - acc: 0.6678 - val_loss: 1.4504 - val_acc: 0.4131
Epoch 67/300
25s - loss: 0.8057 - acc: 0.6704 - val_loss: 1.4565 - val_acc: 0.4100
Epoch 68/300
24s - loss: 0.8074 - acc: 0.6677 - val_loss: 1.4599 - val_acc: 0.4127
Epoch 69/300
27s - loss: 0.8067 - acc: 0.6700 - val_loss: 1.4599 - val_acc: 0.4136
Epoch 70/300
27s - loss: 0.8067 - acc: 0.6696 - val_loss: 1.4458 - val_acc: 0.4176
Epoch 71/300
27s - loss: 0.8077 - acc: 0.6680 - val_loss: 1.4610 - val_acc: 0.4122
Epoch 72/300
25s - loss: 0.8082 - acc: 0.6677 - val_loss: 1.4740 - val_acc: 0.4104
Epoch 73/300
26s - loss: 0.8052 - acc: 0.6691 - val_loss: 1.4529 - val_acc: 0.4081
Epoch 74/300
24s - loss: 0.8067 - acc: 0.6690 - val_loss: 1.4639 - val_acc: 0.4072
Epoch 75/300
24s - loss: 0.8063 - acc: 0.6689 - val_loss: 1.4492 - val_acc: 0.4059
Epoch 76/300
26s - loss: 0.8056 - acc: 0.6698 - val_loss: 1.4587 - val_acc: 0.4095
Epoch 77/300
26s - loss: 0.8035 - acc: 0.6708 - val_loss: 1.4702 - val_acc: 0.4072
Epoch 78/300
32s - loss: 0.8042 - acc: 0.6704 - val_loss: 1.4583 - val_acc: 0.4109
Epoch 79/300
25s - loss: 0.8030 - acc: 0.6697 - val_loss: 1.4699 - val_acc: 0.4086
Epoch 80/300
26s - loss: 0.8019 - acc: 0.6702 - val_loss: 1.4667 - val_acc: 0.4077
Epoch 81/300
25s - loss: 0.8044 - acc: 0.6694 - val_loss: 1.4795 - val_acc: 0.4063
Epoch 82/300
26s - loss: 0.8031 - acc: 0.6700 - val_loss: 1.4533 - val_acc: 0.4095
Epoch 83/300
25s - loss: 0.8013 - acc: 0.6716 - val_loss: 1.4742 - val_acc: 0.4109
Epoch 84/300
25s - loss: 0.8031 - acc: 0.6703 - val_loss: 1.4859 - val_acc: 0.4000
Epoch 85/300
26s - loss: 0.8017 - acc: 0.6704 - val_loss: 1.4837 - val_acc: 0.4045
Epoch 86/300
25s - loss: 0.7997 - acc: 0.6707 - val_loss: 1.4660 - val_acc: 0.4077
Epoch 87/300
26s - loss: 0.8004 - acc: 0.6713 - val_loss: 1.4734 - val_acc: 0.4014
Epoch 88/300
25s - loss: 0.8006 - acc: 0.6713 - val_loss: 1.4504 - val_acc: 0.4131
Epoch 89/300
26s - loss: 0.8003 - acc: 0.6711 - val_loss: 1.4679 - val_acc: 0.4118
Epoch 90/300
25s - loss: 0.8011 - acc: 0.6703 - val_loss: 1.4684 - val_acc: 0.4081
Epoch 91/300
25s - loss: 0.8004 - acc: 0.6711 - val_loss: 1.4751 - val_acc: 0.4090
Epoch 92/300
26s - loss: 0.7996 - acc: 0.6711 - val_loss: 1.4769 - val_acc: 0.4113
Epoch 93/300
25s - loss: 0.7990 - acc: 0.6727 - val_loss: 1.4701 - val_acc: 0.4113
Epoch 94/300
26s - loss: 0.7993 - acc: 0.6714 - val_loss: 1.5000 - val_acc: 0.4050
Epoch 95/300
25s - loss: 0.8001 - acc: 0.6707 - val_loss: 1.4621 - val_acc: 0.4072
Epoch 96/300
27s - loss: 0.7981 - acc: 0.6726 - val_loss: 1.4394 - val_acc: 0.4023
Epoch 97/300
25s - loss: 0.7983 - acc: 0.6716 - val_loss: 1.4752 - val_acc: 0.4131
Epoch 98/300
26s - loss: 0.7957 - acc: 0.6725 - val_loss: 1.4784 - val_acc: 0.4068
Epoch 99/300
25s - loss: 0.7978 - acc: 0.6724 - val_loss: 1.4792 - val_acc: 0.4077
Epoch 100/300
25s - loss: 0.7973 - acc: 0.6723 - val_loss: 1.4827 - val_acc: 0.4059
Epoch 101/300
26s - loss: 0.7952 - acc: 0.6723 - val_loss: 1.5040 - val_acc: 0.3977
Epoch 102/300
25s - loss: 0.7958 - acc: 0.6730 - val_loss: 1.4860 - val_acc: 0.4081
Epoch 103/300
26s - loss: 0.7962 - acc: 0.6732 - val_loss: 1.4796 - val_acc: 0.4050
Epoch 104/300
25s - loss: 0.7955 - acc: 0.6733 - val_loss: 1.4791 - val_acc: 0.4036
Epoch 105/300
26s - loss: 0.7950 - acc: 0.6719 - val_loss: 1.4881 - val_acc: 0.4041
Epoch 106/300
25s - loss: 0.7953 - acc: 0.6737 - val_loss: 1.4834 - val_acc: 0.4036
Epoch 107/300
25s - loss: 0.7961 - acc: 0.6726 - val_loss: 1.5021 - val_acc: 0.4009
Epoch 108/300
26s - loss: 0.7970 - acc: 0.6728 - val_loss: 1.4832 - val_acc: 0.4050
Epoch 109/300
25s - loss: 0.7938 - acc: 0.6725 - val_loss: 1.4773 - val_acc: 0.4032
Epoch 110/300
26s - loss: 0.7922 - acc: 0.6729 - val_loss: 1.4872 - val_acc: 0.4072
Epoch 111/300
25s - loss: 0.7940 - acc: 0.6734 - val_loss: 1.4862 - val_acc: 0.4041
Epoch 112/300
26s - loss: 0.7932 - acc: 0.6740 - val_loss: 1.4787 - val_acc: 0.4041
Epoch 113/300
25s - loss: 0.7918 - acc: 0.6735 - val_loss: 1.4720 - val_acc: 0.4140
Epoch 114/300
26s - loss: 0.7940 - acc: 0.6729 - val_loss: 1.4909 - val_acc: 0.4027
Epoch 115/300
25s - loss: 0.7922 - acc: 0.6746 - val_loss: 1.4849 - val_acc: 0.4068
Epoch 116/300
25s - loss: 0.7922 - acc: 0.6747 - val_loss: 1.5020 - val_acc: 0.4009
Epoch 117/300
27s - loss: 0.7905 - acc: 0.6760 - val_loss: 1.4797 - val_acc: 0.4136
Epoch 118/300
25s - loss: 0.7925 - acc: 0.6737 - val_loss: 1.4716 - val_acc: 0.4086
Epoch 119/300
26s - loss: 0.7919 - acc: 0.6738 - val_loss: 1.4783 - val_acc: 0.4081
Epoch 120/300
25s - loss: 0.7903 - acc: 0.6733 - val_loss: 1.4775 - val_acc: 0.4154
Epoch 121/300
26s - loss: 0.7895 - acc: 0.6753 - val_loss: 1.4937 - val_acc: 0.4095
Epoch 122/300
25s - loss: 0.7904 - acc: 0.6748 - val_loss: 1.4796 - val_acc: 0.4090
Epoch 123/300
25s - loss: 0.7900 - acc: 0.6740 - val_loss: 1.4773 - val_acc: 0.4104
Epoch 124/300
26s - loss: 0.7906 - acc: 0.6744 - val_loss: 1.4815 - val_acc: 0.4090
Epoch 125/300
25s - loss: 0.7890 - acc: 0.6759 - val_loss: 1.4764 - val_acc: 0.4113
Epoch 126/300
26s - loss: 0.7903 - acc: 0.6753 - val_loss: 1.5078 - val_acc: 0.4032
Epoch 127/300
25s - loss: 0.7898 - acc: 0.6749 - val_loss: 1.4861 - val_acc: 0.4081
Epoch 128/300
26s - loss: 0.7884 - acc: 0.6749 - val_loss: 1.4971 - val_acc: 0.4018
Epoch 129/300
25s - loss: 0.7881 - acc: 0.6753 - val_loss: 1.4776 - val_acc: 0.4086
Epoch 130/300
26s - loss: 0.7872 - acc: 0.6753 - val_loss: 1.4769 - val_acc: 0.4068
Epoch 131/300
27s - loss: 0.7877 - acc: 0.6750 - val_loss: 1.4843 - val_acc: 0.4113
Epoch 132/300
25s - loss: 0.7865 - acc: 0.6764 - val_loss: 1.4460 - val_acc: 0.4077
Epoch 133/300
26s - loss: 0.7865 - acc: 0.6754 - val_loss: 1.4891 - val_acc: 0.4081
Epoch 134/300
25s - loss: 0.7856 - acc: 0.6762 - val_loss: 1.5006 - val_acc: 0.4095
Epoch 135/300
26s - loss: 0.7876 - acc: 0.6758 - val_loss: 1.4771 - val_acc: 0.4104
Epoch 136/300
25s - loss: 0.7857 - acc: 0.6756 - val_loss: 1.4945 - val_acc: 0.4050
Epoch 137/300
26s - loss: 0.7842 - acc: 0.6782 - val_loss: 1.4768 - val_acc: 0.4158
Epoch 138/300
25s - loss: 0.7875 - acc: 0.6749 - val_loss: 1.4852 - val_acc: 0.4140
Epoch 139/300
25s - loss: 0.7853 - acc: 0.6764 - val_loss: 1.4824 - val_acc: 0.4113
Epoch 140/300
26s - loss: 0.7848 - acc: 0.6767 - val_loss: 1.4848 - val_acc: 0.4045
Epoch 141/300
25s - loss: 0.7839 - acc: 0.6778 - val_loss: 1.5136 - val_acc: 0.3964
Epoch 142/300
26s - loss: 0.7855 - acc: 0.6764 - val_loss: 1.4870 - val_acc: 0.4113
Epoch 143/300
25s - loss: 0.7822 - acc: 0.6783 - val_loss: 1.4920 - val_acc: 0.4109
Epoch 144/300
26s - loss: 0.7838 - acc: 0.6767 - val_loss: 1.5031 - val_acc: 0.4027
Epoch 145/300
25s - loss: 0.7822 - acc: 0.6774 - val_loss: 1.5067 - val_acc: 0.4036
Epoch 146/300
25s - loss: 0.7815 - acc: 0.6774 - val_loss: 1.4895 - val_acc: 0.4045
Epoch 147/300
26s - loss: 0.7818 - acc: 0.6770 - val_loss: 1.4995 - val_acc: 0.4041
Epoch 148/300
25s - loss: 0.7810 - acc: 0.6769 - val_loss: 1.4907 - val_acc: 0.4072
Epoch 149/300
26s - loss: 0.7829 - acc: 0.6780 - val_loss: 1.5228 - val_acc: 0.3991
Epoch 150/300
25s - loss: 0.7812 - acc: 0.6779 - val_loss: 1.4958 - val_acc: 0.4068
Epoch 151/300
26s - loss: 0.7819 - acc: 0.6781 - val_loss: 1.5048 - val_acc: 0.4027
Epoch 152/300
25s - loss: 0.7809 - acc: 0.6790 - val_loss: 1.5146 - val_acc: 0.4000
Epoch 153/300
25s - loss: 0.7796 - acc: 0.6792 - val_loss: 1.4922 - val_acc: 0.4059
Epoch 154/300
26s - loss: 0.7789 - acc: 0.6790 - val_loss: 1.5115 - val_acc: 0.4027
Epoch 155/300
25s - loss: 0.7817 - acc: 0.6769 - val_loss: 1.5089 - val_acc: 0.4032
Epoch 156/300
26s - loss: 0.7807 - acc: 0.6787 - val_loss: 1.4910 - val_acc: 0.4104
Epoch 157/300
25s - loss: 0.7804 - acc: 0.6793 - val_loss: 1.5061 - val_acc: 0.4063
Epoch 158/300
26s - loss: 0.7793 - acc: 0.6772 - val_loss: 1.4865 - val_acc: 0.4140
Epoch 159/300
25s - loss: 0.7782 - acc: 0.6790 - val_loss: 1.4985 - val_acc: 0.4090
Epoch 160/300
26s - loss: 0.7792 - acc: 0.6785 - val_loss: 1.4947 - val_acc: 0.4077
Epoch 161/300
25s - loss: 0.7784 - acc: 0.6788 - val_loss: 1.5068 - val_acc: 0.4036
Epoch 162/300
25s - loss: 0.7807 - acc: 0.6785 - val_loss: 1.5141 - val_acc: 0.4072
Epoch 163/300
27s - loss: 0.7774 - acc: 0.6797 - val_loss: 1.5032 - val_acc: 0.4077
Epoch 164/300
25s - loss: 0.7790 - acc: 0.6783 - val_loss: 1.5067 - val_acc: 0.4081
Epoch 165/300
27s - loss: 0.7773 - acc: 0.6791 - val_loss: 1.4949 - val_acc: 0.4104
Epoch 166/300
25s - loss: 0.7779 - acc: 0.6785 - val_loss: 1.5039 - val_acc: 0.4113
Epoch 167/300
26s - loss: 0.7778 - acc: 0.6794 - val_loss: 1.5037 - val_acc: 0.4122
Epoch 168/300
25s - loss: 0.7781 - acc: 0.6793 - val_loss: 1.5121 - val_acc: 0.4077
Epoch 169/300
25s - loss: 0.7771 - acc: 0.6797 - val_loss: 1.4994 - val_acc: 0.4068
Epoch 170/300
26s - loss: 0.7767 - acc: 0.6798 - val_loss: 1.5098 - val_acc: 0.4104
Epoch 171/300
25s - loss: 0.7773 - acc: 0.6778 - val_loss: 1.5307 - val_acc: 0.4045
Epoch 172/300
27s - loss: 0.7764 - acc: 0.6808 - val_loss: 1.5147 - val_acc: 0.4045
Epoch 173/300
25s - loss: 0.7761 - acc: 0.6793 - val_loss: 1.5147 - val_acc: 0.4068
Epoch 174/300
26s - loss: 0.7754 - acc: 0.6809 - val_loss: 1.5055 - val_acc: 0.4109
Epoch 175/300
25s - loss: 0.7760 - acc: 0.6797 - val_loss: 1.5168 - val_acc: 0.4072
Epoch 176/300
25s - loss: 0.7761 - acc: 0.6802 - val_loss: 1.5143 - val_acc: 0.4041
Epoch 177/300
26s - loss: 0.7757 - acc: 0.6801 - val_loss: 1.5248 - val_acc: 0.4090
Epoch 178/300
25s - loss: 0.7755 - acc: 0.6800 - val_loss: 1.5135 - val_acc: 0.4081
Epoch 179/300
26s - loss: 0.7738 - acc: 0.6818 - val_loss: 1.5131 - val_acc: 0.4063
Epoch 180/300
25s - loss: 0.7770 - acc: 0.6799 - val_loss: 1.5160 - val_acc: 0.4054
Epoch 181/300
26s - loss: 0.7744 - acc: 0.6797 - val_loss: 1.5204 - val_acc: 0.4086
Epoch 182/300
25s - loss: 0.7737 - acc: 0.6810 - val_loss: 1.5231 - val_acc: 0.4090
Epoch 183/300
26s - loss: 0.7729 - acc: 0.6825 - val_loss: 1.5153 - val_acc: 0.4090
Epoch 184/300
25s - loss: 0.7739 - acc: 0.6799 - val_loss: 1.4834 - val_acc: 0.4027
Epoch 185/300
25s - loss: 0.7731 - acc: 0.6800 - val_loss: 1.5257 - val_acc: 0.4063
Epoch 186/300
26s - loss: 0.7735 - acc: 0.6823 - val_loss: 1.5222 - val_acc: 0.4077
Epoch 187/300
25s - loss: 0.7733 - acc: 0.6810 - val_loss: 1.5228 - val_acc: 0.4072
Epoch 188/300
26s - loss: 0.7726 - acc: 0.6815 - val_loss: 1.5198 - val_acc: 0.4090
Epoch 189/300
25s - loss: 0.7714 - acc: 0.6809 - val_loss: 1.5078 - val_acc: 0.4081
Epoch 190/300
26s - loss: 0.7713 - acc: 0.6811 - val_loss: 1.5172 - val_acc: 0.4068
Epoch 191/300
25s - loss: 0.7740 - acc: 0.6805 - val_loss: 1.5129 - val_acc: 0.4063
Epoch 192/300
25s - loss: 0.7720 - acc: 0.6819 - val_loss: 1.5271 - val_acc: 0.4009
Epoch 193/300
26s - loss: 0.7721 - acc: 0.6817 - val_loss: 1.5113 - val_acc: 0.4136
Epoch 194/300
25s - loss: 0.7715 - acc: 0.6808 - val_loss: 1.5179 - val_acc: 0.4054
Epoch 195/300
26s - loss: 0.7707 - acc: 0.6821 - val_loss: 1.4968 - val_acc: 0.4081
Epoch 196/300
25s - loss: 0.7723 - acc: 0.6804 - val_loss: 1.5382 - val_acc: 0.4032
Epoch 197/300
26s - loss: 0.7710 - acc: 0.6821 - val_loss: 1.5256 - val_acc: 0.4036
Epoch 198/300
25s - loss: 0.7700 - acc: 0.6823 - val_loss: 1.5298 - val_acc: 0.4068
Epoch 199/300
25s - loss: 0.7699 - acc: 0.6818 - val_loss: 1.5220 - val_acc: 0.4059
Epoch 200/300
28s - loss: 0.7706 - acc: 0.6818 - val_loss: 1.5198 - val_acc: 0.4041
Epoch 201/300
25s - loss: 0.7707 - acc: 0.6814 - val_loss: 1.5121 - val_acc: 0.4095
Epoch 202/300
26s - loss: 0.7696 - acc: 0.6811 - val_loss: 1.5066 - val_acc: 0.4041
Epoch 203/300
25s - loss: 0.7701 - acc: 0.6824 - val_loss: 1.5280 - val_acc: 0.4023
Epoch 204/300
26s - loss: 0.7692 - acc: 0.6821 - val_loss: 1.5235 - val_acc: 0.4063
Epoch 205/300
25s - loss: 0.7692 - acc: 0.6818 - val_loss: 1.5161 - val_acc: 0.4018
Epoch 206/300
26s - loss: 0.7695 - acc: 0.6826 - val_loss: 1.5344 - val_acc: 0.4000
Epoch 207/300
25s - loss: 0.7675 - acc: 0.6835 - val_loss: 1.5290 - val_acc: 0.4041
Epoch 208/300
25s - loss: 0.7690 - acc: 0.6813 - val_loss: 1.4886 - val_acc: 0.4131
Epoch 209/300
26s - loss: 0.7689 - acc: 0.6828 - val_loss: 1.5284 - val_acc: 0.4027
Epoch 210/300
25s - loss: 0.7681 - acc: 0.6827 - val_loss: 1.5290 - val_acc: 0.3995
Epoch 211/300
26s - loss: 0.7686 - acc: 0.6831 - val_loss: 1.5426 - val_acc: 0.3982
Epoch 212/300
25s - loss: 0.7675 - acc: 0.6832 - val_loss: 1.5371 - val_acc: 0.3982
Epoch 213/300
26s - loss: 0.7677 - acc: 0.6826 - val_loss: 1.5213 - val_acc: 0.4077
Epoch 214/300
26s - loss: 0.7676 - acc: 0.6833 - val_loss: 1.5359 - val_acc: 0.3950
Epoch 215/300
25s - loss: 0.7676 - acc: 0.6830 - val_loss: 1.5402 - val_acc: 0.4032
Epoch 216/300
26s - loss: 0.7674 - acc: 0.6840 - val_loss: 1.5367 - val_acc: 0.4005
Epoch 217/300
25s - loss: 0.7668 - acc: 0.6835 - val_loss: 1.5298 - val_acc: 0.4036
Epoch 218/300
26s - loss: 0.7680 - acc: 0.6826 - val_loss: 1.5348 - val_acc: 0.4059
Epoch 219/300
25s - loss: 0.7684 - acc: 0.6819 - val_loss: 1.5295 - val_acc: 0.3991
Epoch 220/300
26s - loss: 0.7666 - acc: 0.6826 - val_loss: 1.5015 - val_acc: 0.4109
Epoch 221/300
25s - loss: 0.7671 - acc: 0.6835 - val_loss: 1.5283 - val_acc: 0.3991
Epoch 222/300
27s - loss: 0.7652 - acc: 0.6836 - val_loss: 1.5382 - val_acc: 0.4054
Epoch 223/300
27s - loss: 0.7654 - acc: 0.6838 - val_loss: 1.5340 - val_acc: 0.4027
Epoch 224/300
25s - loss: 0.7645 - acc: 0.6841 - val_loss: 1.5369 - val_acc: 0.4009
Epoch 225/300
26s - loss: 0.7649 - acc: 0.6836 - val_loss: 1.5249 - val_acc: 0.4023
Epoch 226/300
25s - loss: 0.7652 - acc: 0.6848 - val_loss: 1.5123 - val_acc: 0.4104
Epoch 227/300
30s - loss: 0.7663 - acc: 0.6833 - val_loss: 1.5327 - val_acc: 0.4100
Epoch 228/300
25s - loss: 0.7670 - acc: 0.6838 - val_loss: 1.5376 - val_acc: 0.4036
Epoch 229/300
26s - loss: 0.7631 - acc: 0.6838 - val_loss: 1.5314 - val_acc: 0.4068
Epoch 230/300
30s - loss: 0.7655 - acc: 0.6842 - val_loss: 1.5352 - val_acc: 0.4023
Epoch 231/300
---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-38-e7e41dadf671> in <module>()
      2           batch_size = batch_size,
      3           epochs=300,
----> 4           validation_data=(x_test, y_test), verbose=2)

/usr/local/lib/python3.5/dist-packages/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, **kwargs)
   1483                               val_f=val_f, val_ins=val_ins, shuffle=shuffle,
   1484                               callback_metrics=callback_metrics,
-> 1485                               initial_epoch=initial_epoch)
   1486 
   1487     def evaluate(self, x, y, batch_size=32, verbose=1, sample_weight=None):

/usr/local/lib/python3.5/dist-packages/keras/engine/training.py in _fit_loop(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch)
   1137                 batch_logs['batch'] = batch_index
   1138                 batch_logs['size'] = len(batch_ids)
-> 1139                 callbacks.on_batch_begin(batch_index, batch_logs)
   1140                 outs = f(ins_batch)
   1141                 if not isinstance(outs, list):

/usr/local/lib/python3.5/dist-packages/keras/callbacks.py in on_batch_begin(self, batch, logs)
     87             callback.on_batch_begin(batch, logs)
     88         self._delta_ts_batch_begin.append(time.time() - t_before_callbacks)
---> 89         delta_t_median = np.median(self._delta_ts_batch_begin)
     90         if (self._delta_t_batch > 0. and
     91            delta_t_median > 0.95 * self._delta_t_batch and

/usr/local/lib/python3.5/dist-packages/numpy/lib/function_base.py in median(a, axis, out, overwrite_input, keepdims)
   3942     """
   3943     r, k = _ureduce(a, func=_median, axis=axis, out=out,
-> 3944                     overwrite_input=overwrite_input)
   3945     if keepdims:
   3946         return r.reshape(k)

/usr/local/lib/python3.5/dist-packages/numpy/lib/function_base.py in _ureduce(a, func, **kwargs)
   3856         keepdim = [1] * a.ndim
   3857 
-> 3858     r = func(a, **kwargs)
   3859     return r, keepdim
   3860 

/usr/local/lib/python3.5/dist-packages/numpy/lib/function_base.py in _median(a, axis, out, overwrite_input)
   3992 
   3993     # Check if the array contains any nan's
-> 3994     if np.issubdtype(a.dtype, np.inexact) and sz > 0:
   3995         # warn and return nans like mean would
   3996         rout = mean(part[indexer], axis=axis, out=out)

KeyboardInterrupt: 

Save the model (marking as continued)


In [24]:
date = str(datetime.date.today() )
time = str(datetime.datetime.now().time())[:-7]

filename = './newtrain_continued_' + model_type + '_' + date + '_' +time

with open( filename + '_history', 'wb') as output:
    pickle.dump([model.history.history], output, pickle.HIGHEST_PROTOCOL)
    
model.save(filename + '.h5';)

In [23]:
import os
os.system('say done')


Out[23]:
256

In [24]:
files = [
    'CNN-rand_continued_2017-04-02_19:26:35',
    'CNN-rand_continued_2017-04-03_16:50:47',
    'CNN-rand_continued_2017-04-03_17:18:47'
]

In [25]:
train_acc, train_loss, val_acc, val_loss = [],[],[],[]

for file in files:
    hist = out[0][0]
    train_acc += hist.history['acc']
    train_loss += hist.history['loss']
    val_acc += hist.history['val_acc']
    val_loss += hist.history['val_loss']

In [35]:
import matplotlib.pyplot as plt
plt.style.use('ggplot')
plt.rcParams["figure.figsize"] = (10,6)
plt.plot(train_acc)
plt.plot(train_loss)
plt.plot(val_acc)
# plt.plot(val_loss)
plt.legend(['Train acc','Train loss','Valid acc', 'Valid loss'], loc=2)
plt.xlabel('Epochs')
plt.ylabel('Loss/Accuracy')
plt.title('Using '+ model_type)
date = str(datetime.date.today() )
time = str(datetime.datetime.now().time())[:-7]
imgName = 'Images/' + model_type + '_' + date + '_' + time + '.jpg'
plt.savefig( imgName, dpi= 200, bbox_inches='tight', transparent=False)
plt.show()