In [1]:

    

from __future__ import print_function
import os
import sys
import numpy as np
import time
from IPython import get_ipython

np.random.seed(1337)

from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.utils.np_utils import to_categorical
from keras.layers import Dense, Flatten, Activation
from keras.layers import Convolution1D, MaxPooling1D, Embedding, LSTM
from keras.models import Model, load_model
from keras.layers import Input, Dropout
from keras.optimizers import SGD, Adadelta
from keras.models import Sequential
from keras.callbacks import ModelCheckpoint
from sklearn import metrics
from sklearn import *
from sklearn.model_selection import train_test_split, KFold
import theano
import csv
import h5py

K_FOLD = 5
MAX_SEQUENCE_LENGTH = 1000
MAX_NB_WORDS = 20000
EMBEDDING_DIM = 100
VALIDATION_SPLIT = 0.1
DROP_OUT = 0.3
Nb_EPOCH = 30
BATCH_SIZE = 50
Classes = 20


GLOVE_DIR = './glove.twitter.27B/'
FILENAME = 'glove.twitter.27B.' + str(EMBEDDING_DIM) + 'd.txt'
TEXT_DATA_DIR = './20_newsgroups_' + str(Classes) 
weights_path = "20News_LSTM_model_" + str(Classes) +".h5"


    

    




Using Theano backend.
Using gpu device 0: GeForce GTX 1050 (CNMeM is enabled with initial size: 70.0% of memory, cuDNN 5110)
/home/bsft14/chuiyiliu3/anaconda2/lib/python2.7/site-packages/theano/sandbox/cuda/__init__.py:600: UserWarning: Your cuDNN version is more recent than the one Theano officially supports. If you see any problems, try updating Theano or downgrading cuDNN to version 5.
  warnings.warn(warn)
/home/bsft14/chuiyiliu3/anaconda2/lib/python2.7/site-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.
  "This module will be removed in 0.20.", DeprecationWarning)
/home/bsft14/chuiyiliu3/anaconda2/lib/python2.7/site-packages/sklearn/grid_search.py:43: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.
  DeprecationWarning)
/home/bsft14/chuiyiliu3/anaconda2/lib/python2.7/site-packages/sklearn/lda.py:6: DeprecationWarning: lda.LDA has been moved to discriminant_analysis.LinearDiscriminantAnalysis in 0.17 and will be removed in 0.19
  "in 0.17 and will be removed in 0.19", DeprecationWarning)
/home/bsft14/chuiyiliu3/anaconda2/lib/python2.7/site-packages/sklearn/learning_curve.py:23: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the functions are moved. This module will be removed in 0.20
  DeprecationWarning)
/home/bsft14/chuiyiliu3/anaconda2/lib/python2.7/site-packages/sklearn/qda.py:6: DeprecationWarning: qda.QDA has been moved to discriminant_analysis.QuadraticDiscriminantAnalysis in 0.17 and will be removed in 0.19.
  "in 0.17 and will be removed in 0.19.", DeprecationWarning)

In [2]:

    

def reset_parameter():
    global MAX_SEQUENCE_LENGTH, MAX_NB_WORDS , EMBEDDING_DIM, VALIDATION_SPLIT, DROP_OUT, Nb_EPOCH, BATCH_SIZE, Classes 

    MAX_SEQUENCE_LENGTH = 1000
    MAX_NB_WORDS = 20000
    EMBEDDING_DIM = 100
    VALIDATION_SPLIT = 0.1
    DROP_OUT = 0.3
    Nb_EPOCH = 30
    BATCH_SIZE = 50 
    Classes = 2
    
def write_csv_result(fname, train_accuracy, valid_accuracy, test_accuracy, time): 
    global header, items
    global MAX_SEQUENCE_LENGTH, MAX_NB_WORDS , EMBEDDING_DIM, VALIDATION_SPLIT, DROP_OUT, Nb_EPOCH, BATCH_SIZE, Classes 
    header = [['Classes', 'Dropout', 'Iterations', 'Batch Size','Embedding Dimension',
              'Training Accuracy', 'Validation Accuracy', 'Test Accuracy', 'Time']]
    
    items = [Classes, DROP_OUT, Nb_EPOCH, BATCH_SIZE, EMBEDDING_DIM, 
             train_accuracy, valid_accuracy, test_accuracy, time]
    header.append(items)

    f = open(fname, 'wb')
    writer = csv.writer(f)
    writer.writerows(header)
    header = [] # reset header after each loop 
    f.close()


    

In [3]:

    

def embedding_index(GLOVE_DIR, FILENAME):
    global embeddings_index 
    embeddings_index = {}
    fname = os.path.join(GLOVE_DIR, FILENAME)
    f = open(fname)
    for line in f:
        values = line.split()
        word = values[0]
        coefs = np.asarray(values[1:], dtype='float32')
        embeddings_index[word] = coefs
    f.close()

    print('Found %s word vectors.' % len(embeddings_index))


    

In [4]:

    

def embeddingMatrix():
    global nb_words, embedding_matrix
    print('Preparing embedding matrix.')
    nb_words = min(MAX_NB_WORDS, len(word_index))
    embedding_matrix = np.zeros((nb_words + 1, EMBEDDING_DIM))
    for word, i in word_index.items():
        if i > MAX_NB_WORDS:
            continue
        embedding_vector = embeddings_index.get(word)
        if embedding_vector is not None:
            embedding_matrix[i] = embedding_vector

    return embedding_matrix


    

In [5]:

    

def load_data(TEXT_DATA_DIR):
    print('Processing text dataset')
    texts = []  # list of text samples
    labels_index = {}  # dictionary mapping label name to numeric id
    labels = []  # list of label ids
    for name in sorted(os.listdir(TEXT_DATA_DIR)):
        path = os.path.join(TEXT_DATA_DIR, name)
        if os.path.isdir(path):
            label_id = len(labels_index)
            labels_index[name] = label_id
            for fname in sorted(os.listdir(path)):
                if fname.isdigit():
                    fpath = os.path.join(path, fname)
                    if sys.version_info < (3,):
                        f = open(fpath)
                    else:
                        f = open(fpath, encoding='latin-1')
                    texts.append(f.read())
                    f.close()
                    labels.append(label_id)

    print('Found %s texts.' % len(texts))
    
    global word_index, tokenizer

    tokenizer = Tokenizer(nb_words=MAX_NB_WORDS)
    tokenizer.fit_on_texts(texts)
    sequences = tokenizer.texts_to_sequences(texts)

    word_index = tokenizer.word_index
    print('Found %s unique tokens.' % len(word_index))

    data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)

    labels = to_categorical(np.asarray(labels))
    print('Shape of data tensor:', data.shape)
    print('Shape of label tensor:', labels.shape)

    return (data, labels, labels_index)


    

In [6]:

    

def create_model():
    print('Number of class: ||%d||' % (Classes))
    model = Sequential()
    model.add(Embedding(                          # Layer 0, Start
        input_dim=nb_words + 1,                   # Size to dictionary, has to be input + 1
        output_dim= EMBEDDING_DIM,                # Dimensions to generate
        weights=[embedding_matrix],               # Initialize word weights
        input_length=MAX_SEQUENCE_LENGTH,
        name = "embedding_layer",
        trainable=False))
    model.add(LSTM(128, dropout_W=DROP_OUT, dropout_U=DROP_OUT, name = "lstm_layer"))  
    model.add(Dense(Classes, activation = 'sigmoid', name = "dense_one"))
    
    model.compile(loss='categorical_crossentropy',
                  optimizer='adam',
                  metrics=['accuracy'])
    
    return model


    

In [7]:

    

def train_and_evaluate_model(model, train_X, train_Y, test_X, test_Y):
    global best_weight_path
    start = time.time()

    best_weight_path="weights-improvement.hdf5"
    checkpoint = ModelCheckpoint(best_weight_path, monitor='acc', verbose=1, save_best_only=True, mode='max')
    print("Saved best weights to disk") 
    callbacks_list = [checkpoint]   

    history = model.fit(train_X, train_Y, validation_split=VALIDATION_SPLIT, nb_epoch=Nb_EPOCH, batch_size=BATCH_SIZE, callbacks=callbacks_list)
    
    model.load_weights(best_weight_path)
   
    model.compile(loss='categorical_crossentropy',
                  optimizer='adam',
                  metrics=['accuracy'])
    
    model.save(weights_path)

    training_loss, training_accuracy = model.evaluate(train_X, train_Y)
    print ("Training Loss: ", training_loss)
    print ("Training Accuracy: ", training_accuracy)

    eval_loss, eval_accuracy = model.evaluate(test_X, test_Y)
    print ("Testing Loss: ", eval_loss)
    print ("Testing Accuracy: ", eval_accuracy)

    model_history = history.history

    model.pop()
   
    model.compile(loss='categorical_crossentropy',
                  optimizer='adam',
                  metrics=['accuracy'])
	
    print("Saved model to disk")
    
    #print ("Intermediate output for training: ", intermediate_output_train)
    intermediate_output_train = model.predict(train_X)
    print ("Intermediate training output shape : ", intermediate_output_train.shape)
    #print ("Intermediate output for testing: ", intermediate_output_test)
    intermediate_output_test = model.predict(test_X)
    print ("Intermediate testing output shape : ", intermediate_output_test.shape)

    trainTime = time.time() - start
    print ("Training Time : ", trainTime)
    return (model, training_accuracy, eval_accuracy)


    

In [8]:

    

def transform_labels(train_Y, test_Y):
    trainY = [] 
    testY = []
    for label in train_Y:
        for idx, bit in enumerate(label):
            if bit ==1:
                trainY.append(idx)
    for label in test_Y:
        for idx, bit in enumerate(label):
            if bit ==1:
                testY.append(idx)
    trainY = np.array(trainY)
    testY = np.array(testY)
    print (trainY.shape , testY.shape)
    
    return trainY, testY


    

In [9]:

    

def evaluate_with_SVM(data, labels, train_X, train_Y,test_X, test_Y):
    print ("Starting SVM")
    clf = svm.SVC(kernel='linear')
    clf.fit(train_X, train_Y)
    predict_Y = clf.predict(test_X) 
    s=metrics.accuracy_score(test_Y, predict_Y) 
    print ("SVM Testing Acc: ", s)
    return s


    

In [10]:

    

def evaluate_with_KNN(data, labels, train_X, train_Y,test_X, test_Y):
    print ("Starting KNN")
    kNN = neighbors.KNeighborsClassifier()
    kNN.fit(train_X, train_Y)
    predict_Y = kNN.predict(test_X)  
    s=metrics.accuracy_score(test_Y, predict_Y) 
    print ("KNN Testing Acc: ", s)
    return s


    

In [ ]:

    

def main():
    global Classes, DROP_OUT, EMBEDDING_DIM, Nb_EPOCH, FILENAME, TEXT_DATA_DIR, weights_path
    
    embedding_index(GLOVE_DIR, FILENAME)
    data, labels, labels_index = load_data(TEXT_DATA_DIR)
    embedding_matrix = embeddingMatrix()

    
    i=1
    skf = KFold(n_splits=K_FOLD, shuffle=True, random_state=1337)
    cvscores_train = []
    cvscores_test = []
    svm_score = []
    knn_score = []
    global_time = time.time()
    for train, test in skf.split(data):
        print ("*****Running fold", i, "/", K_FOLD, " *****")
        model = None 
        model = create_model()
        trainX, trainY, testX, testY = data[train], labels[train], data[test], labels[test]
        model, training_accuracy, eval_accuracy = train_and_evaluate_model(model, trainX, trainY, testX, testY)  
        print("Training Acc: %.5f , Testing Acc: %.5f" % (training_accuracy, eval_accuracy))
        cvscores_train.append(training_accuracy)
        cvscores_test.append(eval_accuracy)  
        trainY, testY = transform_labels(trainY, testY)
        svm_test_accuracy = evaluate_with_SVM(data, labels, model.predict(trainX), trainY, model.predict(testX), testY)
        knn_test_accuracy = evaluate_with_KNN(data, labels, model.predict(trainX), trainY, model.predict(testX), testY)
        svm_score.append(svm_test_accuracy)
        knn_score.append(knn_test_accuracy)
        i=i+1   
        
    total_time = time.time()-global_time
    print("Average Training Accuracy: %f , Average Testing Accuracy: %f" % (np.mean(cvscores_train), np.mean(cvscores_test)))
    print ("SVM Testing Accuracy: %f " % (np.mean(svm_score)))
    print ("KNN Testing Accuracy: %f " % (np.mean(knn_score)))
    print ("Total Running Time : ", total_time)
    write_csv_result("_20news_LSTM_CV.csv", np.mean(cvscores_train), 0 , np.mean(cvscores_test), total_time)

main()


    

    




Found 1193514 word vectors.
Processing text dataset
Found 19997 texts.
Found 214909 unique tokens.
Shape of data tensor: (19997, 1000)
Shape of label tensor: (19997, 20)
Preparing embedding matrix.
*****Running fold 1 / 5  *****
Number of class: ||20||
Saved best weights to disk
Train on 14397 samples, validate on 1600 samples
Epoch 1/30
14350/14397 [============================>.] - ETA: 0s - loss: 2.8035 - acc: 0.1101  Epoch 00000: acc improved from -inf to 0.11044, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 149s - loss: 2.8025 - acc: 0.1104 - val_loss: 6.8931 - val_acc: 0.0037
Epoch 2/30
14350/14397 [============================>.] - ETA: 0s - loss: 2.5516 - acc: 0.1739  Epoch 00001: acc improved from 0.11044 to 0.17379, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 150s - loss: 2.5514 - acc: 0.1738 - val_loss: 9.4670 - val_acc: 0.0012
Epoch 3/30
14350/14397 [============================>.] - ETA: 0s - loss: 2.3608 - acc: 0.2289  Epoch 00002: acc improved from 0.17379 to 0.22894, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 150s - loss: 2.3609 - acc: 0.2289 - val_loss: 9.4444 - val_acc: 0.0031
Epoch 4/30
14350/14397 [============================>.] - ETA: 0s - loss: 2.2035 - acc: 0.2754  Epoch 00003: acc improved from 0.22894 to 0.27561, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 149s - loss: 2.2028 - acc: 0.2756 - val_loss: 10.4616 - val_acc: 0.0037
Epoch 5/30
14350/14397 [============================>.] - ETA: 0s - loss: 2.0099 - acc: 0.3265  Epoch 00004: acc improved from 0.27561 to 0.32653, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 149s - loss: 2.0097 - acc: 0.3265 - val_loss: 11.3619 - val_acc: 0.0037
Epoch 6/30
14350/14397 [============================>.] - ETA: 0s - loss: 1.9041 - acc: 0.3561  Epoch 00005: acc improved from 0.32653 to 0.35612, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 149s - loss: 1.9037 - acc: 0.3561 - val_loss: 11.4082 - val_acc: 0.0019
Epoch 7/30
14350/14397 [============================>.] - ETA: 0s - loss: 1.7970 - acc: 0.3845  Epoch 00006: acc improved from 0.35612 to 0.38466, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 149s - loss: 1.7969 - acc: 0.3847 - val_loss: 12.1653 - val_acc: 0.0019
Epoch 8/30
14350/14397 [============================>.] - ETA: 0s - loss: 1.7281 - acc: 0.4125  Epoch 00007: acc improved from 0.38466 to 0.41279, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 149s - loss: 1.7272 - acc: 0.4128 - val_loss: 12.2153 - val_acc: 0.0025
Epoch 9/30
14350/14397 [============================>.] - ETA: 0s - loss: 1.6507 - acc: 0.4347  Epoch 00008: acc improved from 0.41279 to 0.43481, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 149s - loss: 1.6507 - acc: 0.4348 - val_loss: 12.3753 - val_acc: 0.0025
Epoch 10/30
14350/14397 [============================>.] - ETA: 0s - loss: 1.5762 - acc: 0.4607  Epoch 00009: acc improved from 0.43481 to 0.46100, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 150s - loss: 1.5762 - acc: 0.4610 - val_loss: 12.5081 - val_acc: 0.0025
Epoch 11/30
14350/14397 [============================>.] - ETA: 0s - loss: 1.5213 - acc: 0.4818  Epoch 00010: acc improved from 0.46100 to 0.48191, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 149s - loss: 1.5210 - acc: 0.4819 - val_loss: 12.8146 - val_acc: 0.0025
Epoch 12/30
14350/14397 [============================>.] - ETA: 0s - loss: 1.4582 - acc: 0.4981  Epoch 00011: acc improved from 0.48191 to 0.49851, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 149s - loss: 1.4571 - acc: 0.4985 - val_loss: 13.1789 - val_acc: 0.0031
Epoch 13/30
14350/14397 [============================>.] - ETA: 0s - loss: 1.4060 - acc: 0.5207  Epoch 00012: acc improved from 0.49851 to 0.52094, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 150s - loss: 1.4058 - acc: 0.5209 - val_loss: 13.4692 - val_acc: 0.0031
Epoch 14/30
14350/14397 [============================>.] - ETA: 0s - loss: 1.3619 - acc: 0.5371  Epoch 00013: acc improved from 0.52094 to 0.53720, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 149s - loss: 1.3613 - acc: 0.5372 - val_loss: 13.6160 - val_acc: 0.0044
Epoch 15/30
14350/14397 [============================>.] - ETA: 0s - loss: 1.3411 - acc: 0.5393  Epoch 00014: acc improved from 0.53720 to 0.53921, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 157s - loss: 1.3403 - acc: 0.5392 - val_loss: 13.9273 - val_acc: 0.0037
Epoch 16/30
14350/14397 [============================>.] - ETA: 0s - loss: 1.3277 - acc: 0.5480  Epoch 00015: acc improved from 0.53921 to 0.54803, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 157s - loss: 1.3275 - acc: 0.5480 - val_loss: 13.7045 - val_acc: 0.0044
Epoch 17/30
14350/14397 [============================>.] - ETA: 0s - loss: 1.3499 - acc: 0.5395  Epoch 00016: acc did not improve
14397/14397 [==============================] - 158s - loss: 1.3507 - acc: 0.5389 - val_loss: 14.2932 - val_acc: 0.0044
Epoch 18/30
14350/14397 [============================>.] - ETA: 0s - loss: 1.3142 - acc: 0.5479  Epoch 00017: acc improved from 0.54803 to 0.54810, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 157s - loss: 1.3142 - acc: 0.5481 - val_loss: 14.3202 - val_acc: 0.0044
Epoch 19/30
14350/14397 [============================>.] - ETA: 0s - loss: 1.3014 - acc: 0.5538  Epoch 00018: acc improved from 0.54810 to 0.55373, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 151s - loss: 1.3010 - acc: 0.5537 - val_loss: 14.6529 - val_acc: 0.0037
Epoch 20/30
14350/14397 [============================>.] - ETA: 0s - loss: 1.3066 - acc: 0.5500  Epoch 00019: acc did not improve
14397/14397 [==============================] - 151s - loss: 1.3061 - acc: 0.5500 - val_loss: 14.8254 - val_acc: 0.0037
Epoch 21/30
14350/14397 [============================>.] - ETA: 0s - loss: 1.2710 - acc: 0.5614  Epoch 00020: acc improved from 0.55373 to 0.56158, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 151s - loss: 1.2707 - acc: 0.5616 - val_loss: 14.9057 - val_acc: 0.0037
Epoch 22/30
14350/14397 [============================>.] - ETA: 0s - loss: 1.2487 - acc: 0.5732  Epoch 00021: acc improved from 0.56158 to 0.57331, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 156s - loss: 1.2481 - acc: 0.5733 - val_loss: 14.9418 - val_acc: 0.0037
Epoch 23/30
14350/14397 [============================>.] - ETA: 0s - loss: 1.2198 - acc: 0.5816  Epoch 00022: acc improved from 0.57331 to 0.58179, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 179s - loss: 1.2194 - acc: 0.5818 - val_loss: 15.0687 - val_acc: 0.0037
Epoch 24/30
14350/14397 [============================>.] - ETA: 0s - loss: 1.2101 - acc: 0.5774  Epoch 00023: acc did not improve
14397/14397 [==============================] - 179s - loss: 1.2099 - acc: 0.5776 - val_loss: 15.1386 - val_acc: 0.0044
Epoch 25/30
14350/14397 [============================>.] - ETA: 0s - loss: 1.2439 - acc: 0.5739  Epoch 00024: acc did not improve
14397/14397 [==============================] - 179s - loss: 1.2439 - acc: 0.5741 - val_loss: 15.2770 - val_acc: 0.0044
Epoch 26/30
14350/14397 [============================>.] - ETA: 0s - loss: 1.1971 - acc: 0.5866  Epoch 00025: acc improved from 0.58179 to 0.58658, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 153s - loss: 1.1971 - acc: 0.5866 - val_loss: 15.2719 - val_acc: 0.0037
Epoch 27/30
14350/14397 [============================>.] - ETA: 0s - loss: 1.1757 - acc: 0.5960  Epoch 00026: acc improved from 0.58658 to 0.59617, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 148s - loss: 1.1760 - acc: 0.5962 - val_loss: 15.4153 - val_acc: 0.0044
Epoch 28/30
14350/14397 [============================>.] - ETA: 0s - loss: 1.1635 - acc: 0.5962  Epoch 00027: acc did not improve
14397/14397 [==============================] - 149s - loss: 1.1640 - acc: 0.5959 - val_loss: 15.5382 - val_acc: 0.0044
Epoch 29/30
14350/14397 [============================>.] - ETA: 0s - loss: 1.1441 - acc: 0.6076  Epoch 00028: acc improved from 0.59617 to 0.60749, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 149s - loss: 1.1438 - acc: 0.6075 - val_loss: 15.6181 - val_acc: 0.0044
Epoch 30/30
14350/14397 [============================>.] - ETA: 0s - loss: 1.1235 - acc: 0.6130  Epoch 00029: acc improved from 0.60749 to 0.61277, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 149s - loss: 1.1243 - acc: 0.6128 - val_loss: 15.6355 - val_acc: 0.0044
15997/15997 [==============================] - 67s     
Training Loss:  2.31225640972
Training Accuracy:  0.63524410827
4000/4000 [==============================] - 17s     
Testing Loss:  2.4477275207
Testing Accuracy:  0.598
Saved model to disk
Intermediate training output shape :  (15997, 128)
Intermediate testing output shape :  (4000, 128)
Training Time :  4806.83186316
Training Acc: 0.63524 , Testing Acc: 0.59800
(15997,) (4000,)
Starting SVM
SVM Testing Acc:  0.635
Starting KNN
KNN Testing Acc:  0.61175
*****Running fold 2 / 5  *****
Number of class: ||20||
Saved best weights to disk
Train on 14397 samples, validate on 1600 samples
Epoch 1/30
14350/14397 [============================>.] - ETA: 0s - loss: 2.7649 - acc: 0.1099  Epoch 00000: acc improved from -inf to 0.10981, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 158s - loss: 2.7647 - acc: 0.1098 - val_loss: 7.0671 - val_acc: 0.0000e+00
Epoch 2/30
14350/14397 [============================>.] - ETA: 0s - loss: 2.4451 - acc: 0.1957  Epoch 00001: acc improved from 0.10981 to 0.19532, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 182s - loss: 2.4455 - acc: 0.1953 - val_loss: 8.6488 - val_acc: 0.0000e+00
Epoch 3/30
14350/14397 [============================>.] - ETA: 0s - loss: 2.2603 - acc: 0.2451  Epoch 00002: acc improved from 0.19532 to 0.24512, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 250s - loss: 2.2599 - acc: 0.2451 - val_loss: 9.1665 - val_acc: 0.0000e+00
Epoch 4/30
14350/14397 [============================>.] - ETA: 0s - loss: 1.9861 - acc: 0.3186  Epoch 00003: acc improved from 0.24512 to 0.31854, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 251s - loss: 1.9861 - acc: 0.3185 - val_loss: 9.6747 - val_acc: 0.0000e+00
Epoch 5/30
14350/14397 [============================>.] - ETA: 0s - loss: 1.8146 - acc: 0.3734  Epoch 00004: acc improved from 0.31854 to 0.37320, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 188s - loss: 1.8147 - acc: 0.3732 - val_loss: 9.6443 - val_acc: 0.0000e+00
Epoch 6/30
14350/14397 [============================>.] - ETA: 0s - loss: 1.6251 - acc: 0.4292  Epoch 00005: acc improved from 0.37320 to 0.42995, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 179s - loss: 1.6236 - acc: 0.4300 - val_loss: 10.0511 - val_acc: 0.0000e+00
Epoch 7/30
14350/14397 [============================>.] - ETA: 0s - loss: 1.2962 - acc: 0.5360  Epoch 00006: acc improved from 0.42995 to 0.53650, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 180s - loss: 1.2951 - acc: 0.5365 - val_loss: 10.6044 - val_acc: 0.0000e+00
Epoch 8/30
14350/14397 [============================>.] - ETA: 0s - loss: 0.9060 - acc: 0.6883  Epoch 00007: acc improved from 0.53650 to 0.68834, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 179s - loss: 0.9061 - acc: 0.6883 - val_loss: 8.9809 - val_acc: 0.0000e+00
Epoch 9/30
14350/14397 [============================>.] - ETA: 0s - loss: 0.6715 - acc: 0.7942  Epoch 00008: acc improved from 0.68834 to 0.79426, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 179s - loss: 0.6712 - acc: 0.7943 - val_loss: 10.6013 - val_acc: 0.0000e+00
Epoch 10/30
14350/14397 [============================>.] - ETA: 0s - loss: 0.4456 - acc: 0.8739  Epoch 00009: acc improved from 0.79426 to 0.87386, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 179s - loss: 0.4459 - acc: 0.8739 - val_loss: 10.6286 - val_acc: 0.0000e+00
Epoch 11/30
14350/14397 [============================>.] - ETA: 0s - loss: 0.3141 - acc: 0.9146  Epoch 00010: acc improved from 0.87386 to 0.91463, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 173s - loss: 0.3141 - acc: 0.9146 - val_loss: 10.4570 - val_acc: 0.0000e+00
Epoch 12/30
14350/14397 [============================>.] - ETA: 0s - loss: 0.2324 - acc: 0.9391  Epoch 00011: acc improved from 0.91463 to 0.93922, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 151s - loss: 0.2321 - acc: 0.9392 - val_loss: 10.2809 - val_acc: 0.0000e+00
Epoch 13/30
14350/14397 [============================>.] - ETA: 0s - loss: 0.1713 - acc: 0.9597  Epoch 00012: acc improved from 0.93922 to 0.95978, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 150s - loss: 0.1712 - acc: 0.9598 - val_loss: 10.2244 - val_acc: 0.0000e+00
Epoch 14/30
14350/14397 [============================>.] - ETA: 0s - loss: 0.1320 - acc: 0.9724  Epoch 00013: acc improved from 0.95978 to 0.97249, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 150s - loss: 0.1317 - acc: 0.9725 - val_loss: 10.3614 - val_acc: 0.0000e+00
Epoch 15/30
14350/14397 [============================>.] - ETA: 0s - loss: 0.1044 - acc: 0.9776  Epoch 00014: acc improved from 0.97249 to 0.97756, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 151s - loss: 0.1044 - acc: 0.9776 - val_loss: 10.5884 - val_acc: 0.0000e+00
Epoch 16/30
14350/14397 [============================>.] - ETA: 0s - loss: 0.0911 - acc: 0.9804  Epoch 00015: acc improved from 0.97756 to 0.98041, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 151s - loss: 0.0913 - acc: 0.9804 - val_loss: 10.4535 - val_acc: 0.0000e+00
Epoch 17/30
14350/14397 [============================>.] - ETA: 0s - loss: 0.0840 - acc: 0.9804  Epoch 00016: acc improved from 0.98041 to 0.98048, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 151s - loss: 0.0838 - acc: 0.9805 - val_loss: 10.4786 - val_acc: 0.0000e+00
Epoch 18/30
14350/14397 [============================>.] - ETA: 0s - loss: 0.0765 - acc: 0.9806  Epoch 00017: acc improved from 0.98048 to 0.98048, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 151s - loss: 0.0770 - acc: 0.9805 - val_loss: 10.4276 - val_acc: 0.0000e+00
Epoch 19/30
14350/14397 [============================>.] - ETA: 0s - loss: 0.0677 - acc: 0.9830  Epoch 00018: acc improved from 0.98048 to 0.98298, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 151s - loss: 0.0679 - acc: 0.9830 - val_loss: 10.4425 - val_acc: 0.0000e+00
Epoch 20/30
14350/14397 [============================>.] - ETA: 0s - loss: 0.0630 - acc: 0.9853  Epoch 00019: acc improved from 0.98298 to 0.98534, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 150s - loss: 0.0628 - acc: 0.9853 - val_loss: 10.3641 - val_acc: 0.0000e+00
Epoch 21/30
14350/14397 [============================>.] - ETA: 0s - loss: 0.0601 - acc: 0.9848  Epoch 00020: acc did not improve
14397/14397 [==============================] - 150s - loss: 0.0600 - acc: 0.9849 - val_loss: 10.6390 - val_acc: 0.0000e+00
Epoch 22/30
14350/14397 [============================>.] - ETA: 0s - loss: 0.0602 - acc: 0.9843  Epoch 00021: acc did not improve
14397/14397 [==============================] - 150s - loss: 0.0601 - acc: 0.9844 - val_loss: 10.6361 - val_acc: 0.0000e+00
Epoch 23/30
14350/14397 [============================>.] - ETA: 0s - loss: 0.0511 - acc: 0.9859  Epoch 00022: acc improved from 0.98534 to 0.98590, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 171s - loss: 0.0509 - acc: 0.9859 - val_loss: 10.8351 - val_acc: 0.0000e+00
Epoch 24/30
14350/14397 [============================>.] - ETA: 0s - loss: 0.0490 - acc: 0.9868  Epoch 00023: acc improved from 0.98590 to 0.98673, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 224s - loss: 0.0492 - acc: 0.9867 - val_loss: 10.7051 - val_acc: 0.0000e+00
Epoch 25/30
14350/14397 [============================>.] - ETA: 0s - loss: 0.0455 - acc: 0.9885  Epoch 00024: acc improved from 0.98673 to 0.98854, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 224s - loss: 0.0454 - acc: 0.9885 - val_loss: 10.5245 - val_acc: 0.0000e+00
Epoch 26/30
14350/14397 [============================>.] - ETA: 0s - loss: 0.0430 - acc: 0.9877  Epoch 00025: acc did not improve
14397/14397 [==============================] - 179s - loss: 0.0429 - acc: 0.9877 - val_loss: 10.6593 - val_acc: 0.0000e+00
Epoch 27/30
14350/14397 [============================>.] - ETA: 0s - loss: 0.0422 - acc: 0.9882  Epoch 00026: acc did not improve
14397/14397 [==============================] - 150s - loss: 0.0420 - acc: 0.9882 - val_loss: 10.7976 - val_acc: 0.0000e+00
Epoch 28/30
14350/14397 [============================>.] - ETA: 0s - loss: 0.0416 - acc: 0.9891  Epoch 00027: acc improved from 0.98854 to 0.98916, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 150s - loss: 0.0416 - acc: 0.9892 - val_loss: 10.9635 - val_acc: 0.0000e+00
Epoch 29/30
14350/14397 [============================>.] - ETA: 0s - loss: 0.0358 - acc: 0.9900  Epoch 00028: acc improved from 0.98916 to 0.99000, saving model to weights-improvement.hdf5
14397/14397 [==============================] - 151s - loss: 0.0357 - acc: 0.9900 - val_loss: 10.7692 - val_acc: 0.0000e+00
Epoch 30/30
14350/14397 [============================>.] - ETA: 0s - loss: 0.0356 - acc: 0.9899  Epoch 00029: acc did not improve
14397/14397 [==============================] - 150s - loss: 0.0355 - acc: 0.9899 - val_loss: 10.7467 - val_acc: 0.0000e+00
15997/15997 [==============================] - 65s     
Training Loss:  1.09411160611
Training Accuracy:  0.894105144715
4000/4000 [==============================] - 16s     
Testing Loss:  1.14108364189
Testing Accuracy:  0.88875
Saved model to disk
Intermediate training output shape :  (15997, 128)
Intermediate testing output shape :  (4000, 128)
Training Time :  5347.39314318
Training Acc: 0.89411 , Testing Acc: 0.88875
(15997,) (4000,)
Starting SVM
SVM Testing Acc:  0.95825
Starting KNN
KNN Testing Acc:  0.94925
*****Running fold 3 / 5  *****
Number of class: ||20||
Saved best weights to disk
Train on 14398 samples, validate on 1600 samples
Epoch 1/30
14350/14398 [============================>.] - ETA: 0s - loss: 2.8197 - acc: 0.1059  Epoch 00000: acc improved from -inf to 0.10606, saving model to weights-improvement.hdf5
14398/14398 [==============================] - 150s - loss: 2.8189 - acc: 0.1061 - val_loss: 7.2519 - val_acc: 0.0000e+00
Epoch 2/30
14350/14398 [============================>.] - ETA: 0s - loss: 2.5976 - acc: 0.1654  Epoch 00001: acc improved from 0.10606 to 0.16523, saving model to weights-improvement.hdf5
14398/14398 [==============================] - 152s - loss: 2.5974 - acc: 0.1652 - val_loss: 7.7960 - val_acc: 0.0000e+00
Epoch 3/30
14350/14398 [============================>.] - ETA: 0s - loss: 2.4151 - acc: 0.2174  Epoch 00002: acc improved from 0.16523 to 0.21691, saving model to weights-improvement.hdf5
14398/14398 [==============================] - 152s - loss: 2.4155 - acc: 0.2169 - val_loss: 8.4183 - val_acc: 0.0000e+00
Epoch 4/30
14350/14398 [============================>.] - ETA: 0s - loss: 2.2315 - acc: 0.2652  Epoch 00003: acc improved from 0.21691 to 0.26490, saving model to weights-improvement.hdf5
14398/14398 [==============================] - 152s - loss: 2.2312 - acc: 0.2649 - val_loss: 8.4545 - val_acc: 0.0000e+00
Epoch 5/30
 5050/14398 [=========>....................] - ETA: 96s - loss: 2.3637 - acc: 0.2438 

In [ ]: