In [1]:

    

import time
import pickle
from math import ceil

import matplotlib.pyplot as plt
import pandas as pd
import numpy as np

from keras.layers.core import Dense, Activation, Dropout
from keras.layers.recurrent import LSTM
from keras.models import Sequential

from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report, precision_recall_curve, roc_auc_score, roc_curve

random = np.random.RandomState(12)


    

    




Using TensorFlow backend.

In [2]:

    

SIZE = 1000000
TIME_GAP = 5
SEQUENCE = (3, 1)
VALUES = range(1, 6)
WINDOW = 10


    

In [3]:

    

def takespread(sequence, num):
    length = float(len(sequence))
    
    if num > length:
        num = int(length)
        
    return [sequence[int(ceil(i * length / num))] for i in range(num)]


    

In [4]:

    

def create_data(values, window, size, sequence, time_gap, dummy=False):
    data = random.randint(min(values), max(values), size=(size, ))
    df = pd.DataFrame({"x": data})
    
    def match(array):
        if (array == sequence).all():
            return 1
        else:
            return 0
        
    matches = df["x"].rolling(len(sequence)).apply(match)
    df["y"] = matches.shift(time_gap)
    
    if dummy:
        dummies = pd.get_dummies(df["x"])
        dummies["y"] = df["y"]
        df = dummies
    
    x_val = df.iloc[:, :-1].values
    y_val = df.iloc[:, -1].values
    
    X = []
    y = []
    
    for idx in range(df.shape[0]-window):
        X.append(x_val[idx:idx+window, :].flatten())
        y.append(y_val[idx+window-1])
        
    return train_test_split(np.array(X), np.array(y))


    

In [5]:

    

def L15_L10_L5(input_dim=1):
    model = Sequential()

    model.add(LSTM(units=15, return_sequences=True, input_shape=(None, input_dim)))
    model.add(Dropout(0.2))
    
    model.add(LSTM(units=10, return_sequences=True))
    model.add(Dropout(0.2))

    model.add(LSTM(units=5, return_sequences=False))
    model.add(Dropout(0.2))

    model.add(Dense(units=1))
    model.add(Activation("sigmoid"))

    model.compile(loss='binary_crossentropy', optimizer='adam')
    
    return model


def L30_L20_L10(input_dim=1):
    model = Sequential()

    model.add(LSTM(units=30, return_sequences=True, input_shape=(None, input_dim)))
    model.add(Dropout(0.2))
    
    model.add(LSTM(units=20, return_sequences=True))
    model.add(Dropout(0.2))

    model.add(LSTM(units=10, return_sequences=False))
    model.add(Dropout(0.2))

    model.add(Dense(units=1))
    model.add(Activation("sigmoid"))

    model.compile(loss='binary_crossentropy', optimizer='adam')
    
    return model


def L30_L15(input_dim=1):
    model = Sequential()

    model.add(LSTM(units=30, return_sequences=True, input_shape=(None, input_dim)))
    model.add(Dropout(0.2))

    model.add(LSTM(units=15, return_sequences=False))
    model.add(Dropout(0.2))

    model.add(Dense(units=1))
    model.add(Activation("sigmoid"))

    model.compile(loss='binary_crossentropy', optimizer='adam')
    
    return model

def L10_L5(input_dim=1):
    model = Sequential()

    model.add(LSTM(units=10, return_sequences=True, input_shape=(None, input_dim)))
    model.add(Dropout(0.2))

    model.add(LSTM(units=5, return_sequences=False))
    model.add(Dropout(0.2))

    model.add(Dense(units=1))
    model.add(Activation("sigmoid"))

    model.compile(loss='binary_crossentropy', optimizer='adam')
    
    return model

def L5(input_dim=1):
    model = Sequential()

    model.add(LSTM(units=5, return_sequences=False, input_shape=(None, input_dim)))
    model.add(Dropout(0.2))

    model.add(Dense(units=1))
    model.add(Activation("sigmoid"))

    model.compile(loss='binary_crossentropy', optimizer='adam')
    
    return model


    

In [6]:

    

def fit_model(X_train, y_train, model_builder=None, model=None, epochs=1):
    if not model:
        model = model_builder()
        
    X_reshaped = np.reshape(X_train, (X_train.shape[0], WINDOW, int(X_train.shape[1] / WINDOW)))
    
    try:
        model.fit(X_reshaped, y_train, batch_size=512, epochs=epochs, validation_split=0.05)
    except KeyboardInterrupt:
        return model

    return model


    

In [7]:

    

def prediction_results(X, y, model):
    predicted = model.predict(np.reshape(X, (X.shape[0], WINDOW, int(X.shape[1] / WINDOW))))
    predicted = np.reshape(predicted, (predicted.size,))

    precision, recall, thresholds = precision_recall_curve(y, predicted)
    f1 = 2 * (precision * recall) / (precision + recall)
    
    fpr, tpr, thresholds_roc = roc_curve(y, predicted) 
    auc = roc_auc_score(y, predicted)
    
    return {"Precision": takespread(precision, 100), 
            "Recall": takespread(recall, 100), 
            "F1": takespread(f1, 100),
            "F1 Max": np.nanmax(f1),
            "PR thresholds": takespread(thresholds, 100), 
            "FPR": takespread(fpr, 100), 
            "TPR": takespread(tpr, 100), 
            "ROC thresholds": takespread(thresholds_roc, 100), 
            "AUC": auc}


    

In [8]:

    

results = []

verbosity_string = "Finished in {:.02f} seconds with Test/Train AUC {:.04f}/{:.04f} and max F1 {:.04f}"
start_string = "\nTrain {} with {} samples and sequence {} for {} epochs"
SIZES = ((10**5, "10^5"), (5*10**5, "5*10^5"), (10**6, "10^6"))

try:
    for size, size_label in SIZES:
        for sequence in ((3, 1), (4, 1, 3)):
            X_train, X_test, y_train, y_test = create_data(VALUES, WINDOW, size, sequence, TIME_GAP, dummy=True)

            for model_builder in (L5, L10_L5, L30_L15, L15_L10_L5, L30_L20_L10):
                for epochs in (1, 2, 3, 4, 5):
                    print(start_string.format(model_builder.__name__,
                                              size_label, sequence, epochs))
                    start_time = time.time()
                    
                    if epochs == 1:
                        input_dim = int(X_train.shape[1] / WINDOW)
                        model = model_builder(input_dim)
                        
                    model = fit_model(X_train, y_train, model=model, epochs=1)
                    
                    train_res = prediction_results(X_train, y_train, model)
                    test_res = prediction_results(X_test, y_test, model)
                    
                    result = {"model": model_builder.__name__,
                              "size": size_label,
                              "sequence": sequence,
                              "epochs": epochs,
                              "train_res": train_res,
                              "test_res": test_res,
                              "duration": time.time() - start_time}
                    
                    print(verbosity_string.format(result["duration"],
                                                  result["test_res"]["AUC"],
                                                  result["train_res"]["AUC"],
                                                  result["test_res"]["F1 Max"]))
                    
                    results.append(result)                 
                    
                    
except Exception as e:
    print("Error Raised- Save Results")
    print(e)
finally:
    pickle.dump(results, open('BinaryF.p', 'wb'))


    

    




Train L5 with 10^5 samples and sequence (3, 1) for 1 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 2s - loss: 0.5678 - val_loss: 0.3700






    




/linux-home/python/envs/ipython/lib/python3.5/site-packages/ipykernel/__main__.py:6: RuntimeWarning: invalid value encountered in true_divide






    




Finished in 9.34 seconds with Test/Train AUC 0.4540/0.4490 and max F1 0.1171

Train L5 with 10^5 samples and sequence (3, 1) for 2 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 2s - loss: 0.2772 - val_loss: 0.2185
Finished in 7.73 seconds with Test/Train AUC 0.5136/0.5168 and max F1 0.1190

Train L5 with 10^5 samples and sequence (3, 1) for 3 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 2s - loss: 0.2428 - val_loss: 0.2156
Finished in 7.88 seconds with Test/Train AUC 0.5713/0.5784 and max F1 0.1386

Train L5 with 10^5 samples and sequence (3, 1) for 4 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 2s - loss: 0.2385 - val_loss: 0.2094
Finished in 7.75 seconds with Test/Train AUC 0.7183/0.7250 and max F1 0.2126

Train L5 with 10^5 samples and sequence (3, 1) for 5 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 2s - loss: 0.2290 - val_loss: 0.2000
Finished in 8.12 seconds with Test/Train AUC 0.7463/0.7529 and max F1 0.2339

Train L10_L5 with 10^5 samples and sequence (3, 1) for 1 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 5s - loss: 0.3826 - val_loss: 0.2172
Finished in 18.10 seconds with Test/Train AUC 0.4823/0.4731 and max F1 0.1200

Train L10_L5 with 10^5 samples and sequence (3, 1) for 2 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 5s - loss: 0.2455 - val_loss: 0.2166
Finished in 15.28 seconds with Test/Train AUC 0.5576/0.5521 and max F1 0.1322

Train L10_L5 with 10^5 samples and sequence (3, 1) for 3 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 5s - loss: 0.2451 - val_loss: 0.2161
Finished in 15.27 seconds with Test/Train AUC 0.6296/0.6283 and max F1 0.1591

Train L10_L5 with 10^5 samples and sequence (3, 1) for 4 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 5s - loss: 0.2418 - val_loss: 0.2076
Finished in 15.27 seconds with Test/Train AUC 0.7169/0.7220 and max F1 0.2123

Train L10_L5 with 10^5 samples and sequence (3, 1) for 5 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 5s - loss: 0.2256 - val_loss: 0.1973
Finished in 15.31 seconds with Test/Train AUC 0.7616/0.7674 and max F1 0.2440

Train L30_L15 with 10^5 samples and sequence (3, 1) for 1 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 13s - loss: 0.2849 - val_loss: 0.2030
Finished in 29.40 seconds with Test/Train AUC 0.7215/0.7305 and max F1 0.2166

Train L30_L15 with 10^5 samples and sequence (3, 1) for 2 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 12s - loss: 0.2052 - val_loss: 0.1730
Finished in 25.67 seconds with Test/Train AUC 0.8470/0.8480 and max F1 0.3349

Train L30_L15 with 10^5 samples and sequence (3, 1) for 3 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 12s - loss: 0.1502 - val_loss: 0.0871
Finished in 25.70 seconds with Test/Train AUC 0.9826/0.9810 and max F1 0.7146

Train L30_L15 with 10^5 samples and sequence (3, 1) for 4 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 12s - loss: 0.0591 - val_loss: 0.0230
Finished in 25.81 seconds with Test/Train AUC 0.9990/0.9988 and max F1 0.9637

Train L30_L15 with 10^5 samples and sequence (3, 1) for 5 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 12s - loss: 0.0168 - val_loss: 0.0046
Finished in 25.73 seconds with Test/Train AUC 1.0000/1.0000 and max F1 0.9981

Train L15_L10_L5 with 10^5 samples and sequence (3, 1) for 1 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 10s - loss: 0.3527 - val_loss: 0.2198
Finished in 30.03 seconds with Test/Train AUC 0.5578/0.5658 and max F1 0.1304

Train L15_L10_L5 with 10^5 samples and sequence (3, 1) for 2 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 9s - loss: 0.2446 - val_loss: 0.2166
Finished in 24.96 seconds with Test/Train AUC 0.6228/0.6322 and max F1 0.1568

Train L15_L10_L5 with 10^5 samples and sequence (3, 1) for 3 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 9s - loss: 0.2431 - val_loss: 0.2131
Finished in 25.03 seconds with Test/Train AUC 0.7256/0.7309 and max F1 0.2146

Train L15_L10_L5 with 10^5 samples and sequence (3, 1) for 4 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 9s - loss: 0.2304 - val_loss: 0.1942
Finished in 25.32 seconds with Test/Train AUC 0.7921/0.7969 and max F1 0.2683

Train L15_L10_L5 with 10^5 samples and sequence (3, 1) for 5 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 10s - loss: 0.2055 - val_loss: 0.1633
Finished in 25.09 seconds with Test/Train AUC 0.8994/0.8986 and max F1 0.4035

Train L30_L20_L10 with 10^5 samples and sequence (3, 1) for 1 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 17s - loss: 0.3110 - val_loss: 0.2163
Finished in 40.78 seconds with Test/Train AUC 0.6370/0.6396 and max F1 0.1573

Train L30_L20_L10 with 10^5 samples and sequence (3, 1) for 2 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 17s - loss: 0.2278 - val_loss: 0.1905
Finished in 35.30 seconds with Test/Train AUC 0.7903/0.7954 and max F1 0.2720

Train L30_L20_L10 with 10^5 samples and sequence (3, 1) for 3 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 17s - loss: 0.1823 - val_loss: 0.1272
Finished in 35.11 seconds with Test/Train AUC 0.9490/0.9468 and max F1 0.5399

Train L30_L20_L10 with 10^5 samples and sequence (3, 1) for 4 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 17s - loss: 0.1011 - val_loss: 0.0435
Finished in 35.13 seconds with Test/Train AUC 0.9978/0.9974 and max F1 0.9198

Train L30_L20_L10 with 10^5 samples and sequence (3, 1) for 5 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 16s - loss: 0.0375 - val_loss: 0.0107
Finished in 35.03 seconds with Test/Train AUC 1.0000/1.0000 and max F1 0.9906

Train L5 with 10^5 samples and sequence (4, 1, 3) for 1 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 3s - loss: 0.4625 - val_loss: 0.2073
Finished in 11.38 seconds with Test/Train AUC 0.4692/0.4760 and max F1 0.0302

Train L5 with 10^5 samples and sequence (4, 1, 3) for 2 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 2s - loss: 0.1400 - val_loss: 0.0935
Finished in 8.10 seconds with Test/Train AUC 0.4691/0.4488 and max F1 0.0297

Train L5 with 10^5 samples and sequence (4, 1, 3) for 3 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 2s - loss: 0.0982 - val_loss: 0.0809
Finished in 8.12 seconds with Test/Train AUC 0.4732/0.4538 and max F1 0.0299

Train L5 with 10^5 samples and sequence (4, 1, 3) for 4 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 2s - loss: 0.0904 - val_loss: 0.0783
Finished in 8.06 seconds with Test/Train AUC 0.4761/0.4597 and max F1 0.0306

Train L5 with 10^5 samples and sequence (4, 1, 3) for 5 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 2s - loss: 0.0876 - val_loss: 0.0777
Finished in 8.13 seconds with Test/Train AUC 0.4782/0.4518 and max F1 0.0301

Train L10_L5 with 10^5 samples and sequence (4, 1, 3) for 1 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 6s - loss: 0.3152 - val_loss: 0.0831
Finished in 20.57 seconds with Test/Train AUC 0.5565/0.5718 and max F1 0.0360

Train L10_L5 with 10^5 samples and sequence (4, 1, 3) for 2 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 5s - loss: 0.0916 - val_loss: 0.0782
Finished in 15.49 seconds with Test/Train AUC 0.6113/0.6132 and max F1 0.0396

Train L10_L5 with 10^5 samples and sequence (4, 1, 3) for 3 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 5s - loss: 0.0885 - val_loss: 0.0776
Finished in 15.52 seconds with Test/Train AUC 0.6397/0.6384 and max F1 0.0495

Train L10_L5 with 10^5 samples and sequence (4, 1, 3) for 4 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 5s - loss: 0.0871 - val_loss: 0.0776
Finished in 15.59 seconds with Test/Train AUC 0.6416/0.6403 and max F1 0.0499

Train L10_L5 with 10^5 samples and sequence (4, 1, 3) for 5 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 5s - loss: 0.0860 - val_loss: 0.0778
Finished in 15.60 seconds with Test/Train AUC 0.6435/0.6423 and max F1 0.0504

Train L30_L15 with 10^5 samples and sequence (4, 1, 3) for 1 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 13s - loss: 0.2003 - val_loss: 0.0777
Finished in 31.47 seconds with Test/Train AUC 0.4663/0.4896 and max F1 0.0311

Train L30_L15 with 10^5 samples and sequence (4, 1, 3) for 2 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 12s - loss: 0.0800 - val_loss: 0.0777
Finished in 26.07 seconds with Test/Train AUC 0.5367/0.5615 and max F1 0.0344

Train L30_L15 with 10^5 samples and sequence (4, 1, 3) for 3 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 12s - loss: 0.0795 - val_loss: 0.0772
Finished in 25.97 seconds with Test/Train AUC 0.6108/0.6324 and max F1 0.0409

Train L30_L15 with 10^5 samples and sequence (4, 1, 3) for 4 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 12s - loss: 0.0790 - val_loss: 0.0747
Finished in 25.93 seconds with Test/Train AUC 0.6942/0.7045 and max F1 0.0581

Train L30_L15 with 10^5 samples and sequence (4, 1, 3) for 5 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 12s - loss: 0.0749 - val_loss: 0.0684
Finished in 26.02 seconds with Test/Train AUC 0.8054/0.8018 and max F1 0.1086

Train L15_L10_L5 with 10^5 samples and sequence (4, 1, 3) for 1 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 11s - loss: 0.2930 - val_loss: 0.1116
Finished in 32.53 seconds with Test/Train AUC 0.4362/0.4157 and max F1 0.0298

Train L15_L10_L5 with 10^5 samples and sequence (4, 1, 3) for 2 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 9s - loss: 0.1141 - val_loss: 0.0914
Finished in 25.12 seconds with Test/Train AUC 0.4328/0.4117 and max F1 0.0298

Train L15_L10_L5 with 10^5 samples and sequence (4, 1, 3) for 3 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 9s - loss: 0.0990 - val_loss: 0.0833
Finished in 25.11 seconds with Test/Train AUC 0.4320/0.4109 and max F1 0.0297

Train L15_L10_L5 with 10^5 samples and sequence (4, 1, 3) for 4 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 9s - loss: 0.0923 - val_loss: 0.0798
Finished in 25.10 seconds with Test/Train AUC 0.4321/0.4111 and max F1 0.0296

Train L15_L10_L5 with 10^5 samples and sequence (4, 1, 3) for 5 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 9s - loss: 0.0891 - val_loss: 0.0784
Finished in 25.09 seconds with Test/Train AUC 0.4322/0.4107 and max F1 0.0297

Train L30_L20_L10 with 10^5 samples and sequence (4, 1, 3) for 1 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 18s - loss: 0.2358 - val_loss: 0.0828
Finished in 43.81 seconds with Test/Train AUC 0.5414/0.5291 and max F1 0.0391

Train L30_L20_L10 with 10^5 samples and sequence (4, 1, 3) for 2 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 16s - loss: 0.0845 - val_loss: 0.0777
Finished in 35.26 seconds with Test/Train AUC 0.6038/0.5955 and max F1 0.0473

Train L30_L20_L10 with 10^5 samples and sequence (4, 1, 3) for 3 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 17s - loss: 0.0814 - val_loss: 0.0779
Finished in 35.34 seconds with Test/Train AUC 0.6108/0.6023 and max F1 0.0486

Train L30_L20_L10 with 10^5 samples and sequence (4, 1, 3) for 4 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 17s - loss: 0.0819 - val_loss: 0.0779
Finished in 35.34 seconds with Test/Train AUC 0.6219/0.6144 and max F1 0.0499

Train L30_L20_L10 with 10^5 samples and sequence (4, 1, 3) for 5 epochs
Train on 71242 samples, validate on 3750 samples
Epoch 1/1
71242/71242 [==============================] - 16s - loss: 0.0818 - val_loss: 0.0779
Finished in 35.22 seconds with Test/Train AUC 0.6312/0.6248 and max F1 0.0535

Train L5 with 5*10^5 samples and sequence (3, 1) for 1 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 12s - loss: 0.2936 - val_loss: 0.2147
Finished in 46.35 seconds with Test/Train AUC 0.7419/0.7414 and max F1 0.2288

Train L5 with 5*10^5 samples and sequence (3, 1) for 2 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 10s - loss: 0.2074 - val_loss: 0.1609
Finished in 41.62 seconds with Test/Train AUC 0.9106/0.9106 and max F1 0.4274

Train L5 with 5*10^5 samples and sequence (3, 1) for 3 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 10s - loss: 0.0996 - val_loss: 0.0311
Finished in 41.59 seconds with Test/Train AUC 0.9998/0.9998 and max F1 0.9713

Train L5 with 5*10^5 samples and sequence (3, 1) for 4 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 10s - loss: 0.0287 - val_loss: 0.0076
Finished in 41.61 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L5 with 5*10^5 samples and sequence (3, 1) for 5 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 10s - loss: 0.0139 - val_loss: 0.0029
Finished in 41.62 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L10_L5 with 5*10^5 samples and sequence (3, 1) for 1 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 27s - loss: 0.2707 - val_loss: 0.1762
Finished in 84.81 seconds with Test/Train AUC 0.8797/0.8808 and max F1 0.3823

Train L10_L5 with 5*10^5 samples and sequence (3, 1) for 2 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 26s - loss: 0.0754 - val_loss: 0.0092
Finished in 76.97 seconds with Test/Train AUC 1.0000/1.0000 and max F1 0.9999

Train L10_L5 with 5*10^5 samples and sequence (3, 1) for 3 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 26s - loss: 0.0115 - val_loss: 0.0020
Finished in 76.92 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L10_L5 with 5*10^5 samples and sequence (3, 1) for 4 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 26s - loss: 0.0057 - val_loss: 9.1290e-04
Finished in 76.65 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L10_L5 with 5*10^5 samples and sequence (3, 1) for 5 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 26s - loss: 0.0039 - val_loss: 4.9082e-04
Finished in 76.51 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L30_L15 with 5*10^5 samples and sequence (3, 1) for 1 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 63s - loss: 0.1644 - val_loss: 0.0180
Finished in 135.69 seconds with Test/Train AUC 0.9979/0.9979 and max F1 0.9643

Train L30_L15 with 5*10^5 samples and sequence (3, 1) for 2 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 62s - loss: 0.0036 - val_loss: 2.5380e-04
Finished in 128.90 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L30_L15 with 5*10^5 samples and sequence (3, 1) for 3 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 61s - loss: 4.0728e-04 - val_loss: 9.3375e-05
Finished in 128.41 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L30_L15 with 5*10^5 samples and sequence (3, 1) for 4 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 61s - loss: 2.0794e-04 - val_loss: 4.8462e-05
Finished in 128.35 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L30_L15 with 5*10^5 samples and sequence (3, 1) for 5 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 62s - loss: 1.2553e-04 - val_loss: 2.8247e-05
Finished in 130.68 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L15_L10_L5 with 5*10^5 samples and sequence (3, 1) for 1 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 52s - loss: 0.2755 - val_loss: 0.2325
Finished in 135.66 seconds with Test/Train AUC 0.7306/0.7321 and max F1 0.2131

Train L15_L10_L5 with 5*10^5 samples and sequence (3, 1) for 2 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 50s - loss: 0.1387 - val_loss: 0.0243
Finished in 125.43 seconds with Test/Train AUC 0.9994/0.9994 and max F1 0.9616

Train L15_L10_L5 with 5*10^5 samples and sequence (3, 1) for 3 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 49s - loss: 0.0177 - val_loss: 0.0022
Finished in 125.22 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L15_L10_L5 with 5*10^5 samples and sequence (3, 1) for 4 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 50s - loss: 0.0057 - val_loss: 8.3961e-04
Finished in 125.63 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L15_L10_L5 with 5*10^5 samples and sequence (3, 1) for 5 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 50s - loss: 0.0037 - val_loss: 4.0961e-04
Finished in 125.57 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L30_L20_L10 with 5*10^5 samples and sequence (3, 1) for 1 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 86s - loss: 0.1594 - val_loss: 0.0064
Finished in 186.79 seconds with Test/Train AUC 1.0000/1.0000 and max F1 0.9995

Train L30_L20_L10 with 5*10^5 samples and sequence (3, 1) for 2 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 85s - loss: 0.0037 - val_loss: 4.6367e-04
Finished in 176.13 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L30_L20_L10 with 5*10^5 samples and sequence (3, 1) for 3 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 85s - loss: 0.0785 - val_loss: 0.2325
Finished in 176.06 seconds with Test/Train AUC 0.5397/0.5448 and max F1 0.1287

Train L30_L20_L10 with 5*10^5 samples and sequence (3, 1) for 4 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 85s - loss: 0.2430 - val_loss: 0.2324
Finished in 176.20 seconds with Test/Train AUC 0.5613/0.5674 and max F1 0.1388

Train L30_L20_L10 with 5*10^5 samples and sequence (3, 1) for 5 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 85s - loss: 0.1258 - val_loss: 0.0096
Finished in 176.87 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L5 with 5*10^5 samples and sequence (4, 1, 3) for 1 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 12s - loss: 0.1877 - val_loss: 0.0821
Finished in 47.76 seconds with Test/Train AUC 0.4906/0.4874 and max F1 0.0307

Train L5 with 5*10^5 samples and sequence (4, 1, 3) for 2 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 10s - loss: 0.0901 - val_loss: 0.0821
Finished in 41.68 seconds with Test/Train AUC 0.5105/0.5124 and max F1 0.0317

Train L5 with 5*10^5 samples and sequence (4, 1, 3) for 3 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 10s - loss: 0.0892 - val_loss: 0.0810
Finished in 41.57 seconds with Test/Train AUC 0.6060/0.6077 and max F1 0.0483

Train L5 with 5*10^5 samples and sequence (4, 1, 3) for 4 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 12s - loss: 0.0862 - val_loss: 0.0753
Finished in 42.94 seconds with Test/Train AUC 0.7572/0.7600 and max F1 0.0821

Train L5 with 5*10^5 samples and sequence (4, 1, 3) for 5 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 10s - loss: 0.0798 - val_loss: 0.0665
Finished in 41.46 seconds with Test/Train AUC 0.8708/0.8749 and max F1 0.1562

Train L10_L5 with 5*10^5 samples and sequence (4, 1, 3) for 1 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 28s - loss: 0.1465 - val_loss: 0.0820
Finished in 87.35 seconds with Test/Train AUC 0.4905/0.4993 and max F1 0.0349

Train L10_L5 with 5*10^5 samples and sequence (4, 1, 3) for 2 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 26s - loss: 0.0897 - val_loss: 0.0822
Finished in 78.26 seconds with Test/Train AUC 0.5038/0.5123 and max F1 0.0365

Train L10_L5 with 5*10^5 samples and sequence (4, 1, 3) for 3 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 26s - loss: 0.0890 - val_loss: 0.0823
Finished in 78.22 seconds with Test/Train AUC 0.5188/0.5272 and max F1 0.0382

Train L10_L5 with 5*10^5 samples and sequence (4, 1, 3) for 4 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 26s - loss: 0.0891 - val_loss: 0.0821
Finished in 78.27 seconds with Test/Train AUC 0.5407/0.5489 and max F1 0.0410

Train L10_L5 with 5*10^5 samples and sequence (4, 1, 3) for 5 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 26s - loss: 0.0885 - val_loss: 0.0819
Finished in 78.26 seconds with Test/Train AUC 0.6208/0.6277 and max F1 0.0553

Train L30_L15 with 5*10^5 samples and sequence (4, 1, 3) for 1 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 64s - loss: 0.1034 - val_loss: 0.0722
Finished in 139.62 seconds with Test/Train AUC 0.8144/0.8135 and max F1 0.1194

Train L30_L15 with 5*10^5 samples and sequence (4, 1, 3) for 2 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 62s - loss: 0.0326 - val_loss: 0.0020
Finished in 130.64 seconds with Test/Train AUC 1.0000/1.0000 and max F1 0.9948

Train L30_L15 with 5*10^5 samples and sequence (4, 1, 3) for 3 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 62s - loss: 0.0028 - val_loss: 2.3118e-04
Finished in 130.04 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L30_L15 with 5*10^5 samples and sequence (4, 1, 3) for 4 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 61s - loss: 3.1557e-04 - val_loss: 7.2363e-05
Finished in 129.83 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L30_L15 with 5*10^5 samples and sequence (4, 1, 3) for 5 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 62s - loss: 2.1420e-04 - val_loss: 3.7920e-05
Finished in 130.57 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L15_L10_L5 with 5*10^5 samples and sequence (4, 1, 3) for 1 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 53s - loss: 0.1640 - val_loss: 0.0844
Finished in 140.19 seconds with Test/Train AUC 0.5883/0.5824 and max F1 0.0531

Train L15_L10_L5 with 5*10^5 samples and sequence (4, 1, 3) for 2 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 50s - loss: 0.0915 - val_loss: 0.0818
Finished in 126.50 seconds with Test/Train AUC 0.5894/0.5833 and max F1 0.0550

Train L15_L10_L5 with 5*10^5 samples and sequence (4, 1, 3) for 3 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 50s - loss: 0.0892 - val_loss: 0.0821
Finished in 126.53 seconds with Test/Train AUC 0.5915/0.5854 and max F1 0.0552

Train L15_L10_L5 with 5*10^5 samples and sequence (4, 1, 3) for 4 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 50s - loss: 0.0887 - val_loss: 0.0821
Finished in 126.47 seconds with Test/Train AUC 0.5952/0.5890 and max F1 0.0551

Train L15_L10_L5 with 5*10^5 samples and sequence (4, 1, 3) for 5 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 50s - loss: 0.0881 - val_loss: 0.0821
Finished in 126.65 seconds with Test/Train AUC 0.6028/0.5964 and max F1 0.0579

Train L30_L20_L10 with 5*10^5 samples and sequence (4, 1, 3) for 1 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 87s - loss: 0.1121 - val_loss: 0.0821
Finished in 191.47 seconds with Test/Train AUC 0.3825/0.3825 and max F1 0.0305

Train L30_L20_L10 with 5*10^5 samples and sequence (4, 1, 3) for 2 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 84s - loss: 0.0864 - val_loss: 0.0823
Finished in 176.87 seconds with Test/Train AUC 0.4057/0.4057 and max F1 0.0305

Train L30_L20_L10 with 5*10^5 samples and sequence (4, 1, 3) for 3 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 84s - loss: 0.0817 - val_loss: 0.0536
Finished in 177.36 seconds with Test/Train AUC 0.9635/0.9659 and max F1 0.3582

Train L30_L20_L10 with 5*10^5 samples and sequence (4, 1, 3) for 4 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 86s - loss: 0.0208 - val_loss: 0.0011
Finished in 178.12 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L30_L20_L10 with 5*10^5 samples and sequence (4, 1, 3) for 5 epochs
Train on 356242 samples, validate on 18750 samples
Epoch 1/1
356242/356242 [==============================] - 85s - loss: 0.0015 - val_loss: 2.7295e-04
Finished in 177.65 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L5 with 10^6 samples and sequence (3, 1) for 1 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 24s - loss: 0.2630 - val_loss: 0.2094
Finished in 94.42 seconds with Test/Train AUC 0.7790/0.7817 and max F1 0.2478

Train L5 with 10^6 samples and sequence (3, 1) for 2 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 22s - loss: 0.1392 - val_loss: 0.0183
Finished in 86.58 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L5 with 10^6 samples and sequence (3, 1) for 3 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 22s - loss: 0.0099 - val_loss: 9.1980e-04
Finished in 86.28 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L5 with 10^6 samples and sequence (3, 1) for 4 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 22s - loss: 0.0036 - val_loss: 2.0410e-04
Finished in 85.91 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L5 with 10^6 samples and sequence (3, 1) for 5 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 22s - loss: 0.0021 - val_loss: 6.4897e-05
Finished in 86.27 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L10_L5 with 10^6 samples and sequence (3, 1) for 1 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 56s - loss: 0.1889 - val_loss: 0.0214
Finished in 171.88 seconds with Test/Train AUC 0.9999/0.9999 and max F1 0.9848

Train L10_L5 with 10^6 samples and sequence (3, 1) for 2 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 53s - loss: 0.0102 - val_loss: 6.6716e-04
Finished in 159.97 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L10_L5 with 10^6 samples and sequence (3, 1) for 3 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 53s - loss: 0.0028 - val_loss: 4.0869e-04
Finished in 159.93 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L10_L5 with 10^6 samples and sequence (3, 1) for 4 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 53s - loss: 0.0017 - val_loss: 5.6701e-05
Finished in 161.42 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L10_L5 with 10^6 samples and sequence (3, 1) for 5 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 53s - loss: 0.0012 - val_loss: 1.8797e-05
Finished in 159.97 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L30_L15 with 10^6 samples and sequence (3, 1) for 1 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 127s - loss: 0.0899 - val_loss: 4.3649e-04
Finished in 274.68 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L30_L15 with 10^6 samples and sequence (3, 1) for 2 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 125s - loss: 4.3254e-04 - val_loss: 7.6056e-05
Finished in 264.20 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L30_L15 with 10^6 samples and sequence (3, 1) for 3 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 126s - loss: 1.3512e-04 - val_loss: 2.5700e-05
Finished in 264.63 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L30_L15 with 10^6 samples and sequence (3, 1) for 4 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 125s - loss: 0.0957 - val_loss: 0.0098
Finished in 264.07 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L30_L15 with 10^6 samples and sequence (3, 1) for 5 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 126s - loss: 0.0174 - val_loss: 0.3948
Finished in 265.11 seconds with Test/Train AUC 0.8878/0.8927 and max F1 0.5879

Train L15_L10_L5 with 10^6 samples and sequence (3, 1) for 1 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 104s - loss: 0.1543 - val_loss: 0.0156
Finished in 273.87 seconds with Test/Train AUC 0.9998/0.9998 and max F1 0.9720

Train L15_L10_L5 with 10^6 samples and sequence (3, 1) for 2 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 101s - loss: 0.0158 - val_loss: 0.0031
Finished in 254.94 seconds with Test/Train AUC 1.0000/1.0000 and max F1 0.9973

Train L15_L10_L5 with 10^6 samples and sequence (3, 1) for 3 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 101s - loss: 0.0074 - val_loss: 0.0029
Finished in 254.73 seconds with Test/Train AUC 1.0000/1.0000 and max F1 0.9964

Train L15_L10_L5 with 10^6 samples and sequence (3, 1) for 4 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 101s - loss: 0.0062 - val_loss: 0.0018
Finished in 254.81 seconds with Test/Train AUC 1.0000/1.0000 and max F1 0.9983

Train L15_L10_L5 with 10^6 samples and sequence (3, 1) for 5 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 101s - loss: 0.0043 - val_loss: 0.0015
Finished in 254.83 seconds with Test/Train AUC 1.0000/1.0000 and max F1 0.9983

Train L30_L20_L10 with 10^6 samples and sequence (3, 1) for 1 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 175s - loss: 0.1021 - val_loss: 8.9132e-04
Finished in 375.07 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L30_L20_L10 with 10^6 samples and sequence (3, 1) for 2 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 171s - loss: 0.0013 - val_loss: 1.6047e-04
Finished in 358.16 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L30_L20_L10 with 10^6 samples and sequence (3, 1) for 3 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 172s - loss: 4.0539e-04 - val_loss: 4.9375e-05
Finished in 359.54 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L30_L20_L10 with 10^6 samples and sequence (3, 1) for 4 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 171s - loss: 1.6311e-04 - val_loss: 1.7282e-05
Finished in 358.06 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L30_L20_L10 with 10^6 samples and sequence (3, 1) for 5 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 170s - loss: 0.0011 - val_loss: 1.4885e-05
Finished in 357.42 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L5 with 10^6 samples and sequence (4, 1, 3) for 1 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 26s - loss: 0.1426 - val_loss: 0.0782
Finished in 98.20 seconds with Test/Train AUC 0.4850/0.4925 and max F1 0.0312

Train L5 with 10^6 samples and sequence (4, 1, 3) for 2 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 22s - loss: 0.0878 - val_loss: 0.0782
Finished in 88.57 seconds with Test/Train AUC 0.4814/0.4882 and max F1 0.0312

Train L5 with 10^6 samples and sequence (4, 1, 3) for 3 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 22s - loss: 0.0863 - val_loss: 0.0783
Finished in 88.58 seconds with Test/Train AUC 0.4862/0.4918 and max F1 0.0312

Train L5 with 10^6 samples and sequence (4, 1, 3) for 4 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 22s - loss: 0.0852 - val_loss: 0.0781
Finished in 88.41 seconds with Test/Train AUC 0.4805/0.4851 and max F1 0.0312

Train L5 with 10^6 samples and sequence (4, 1, 3) for 5 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 22s - loss: 0.0839 - val_loss: 0.0781
Finished in 88.49 seconds with Test/Train AUC 0.4893/0.4932 and max F1 0.0312

Train L10_L5 with 10^6 samples and sequence (4, 1, 3) for 1 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 57s - loss: 0.1218 - val_loss: 0.0784
Finished in 176.97 seconds with Test/Train AUC 0.6103/0.6115 and max F1 0.0535

Train L10_L5 with 10^6 samples and sequence (4, 1, 3) for 2 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 54s - loss: 0.0886 - val_loss: 0.0778
Finished in 163.58 seconds with Test/Train AUC 0.7167/0.7124 and max F1 0.0661

Train L10_L5 with 10^6 samples and sequence (4, 1, 3) for 3 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 54s - loss: 0.0545 - val_loss: 0.0028
Finished in 163.59 seconds with Test/Train AUC 1.0000/1.0000 and max F1 0.9942

Train L10_L5 with 10^6 samples and sequence (4, 1, 3) for 4 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 53s - loss: 0.0026 - val_loss: 1.6923e-04
Finished in 162.99 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L10_L5 with 10^6 samples and sequence (4, 1, 3) for 5 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 54s - loss: 0.0014 - val_loss: 5.2911e-05
Finished in 163.08 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L30_L15 with 10^6 samples and sequence (4, 1, 3) for 1 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 129s - loss: 0.0589 - val_loss: 0.0016
Finished in 280.60 seconds with Test/Train AUC 1.0000/1.0000 and max F1 0.9994

Train L30_L15 with 10^6 samples and sequence (4, 1, 3) for 2 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 127s - loss: 0.0011 - val_loss: 8.8606e-05
Finished in 267.82 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L30_L15 with 10^6 samples and sequence (4, 1, 3) for 3 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 126s - loss: 6.1329e-04 - val_loss: 1.2083e-04
Finished in 267.57 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L30_L15 with 10^6 samples and sequence (4, 1, 3) for 4 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 126s - loss: 9.7544e-05 - val_loss: 1.2896e-05
Finished in 267.27 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L30_L15 with 10^6 samples and sequence (4, 1, 3) for 5 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 127s - loss: 4.0804e-04 - val_loss: 7.8350e-06
Finished in 268.16 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L15_L10_L5 with 10^6 samples and sequence (4, 1, 3) for 1 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 106s - loss: 0.1168 - val_loss: 0.0784
Finished in 278.77 seconds with Test/Train AUC 0.4877/0.4846 and max F1 0.0338

Train L15_L10_L5 with 10^6 samples and sequence (4, 1, 3) for 2 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 101s - loss: 0.0879 - val_loss: 0.0785
Finished in 257.86 seconds with Test/Train AUC 0.6395/0.6342 and max F1 0.0560

Train L15_L10_L5 with 10^6 samples and sequence (4, 1, 3) for 3 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 101s - loss: 0.0685 - val_loss: 0.0124
Finished in 257.79 seconds with Test/Train AUC 0.9993/0.9993 and max F1 0.9458

Train L15_L10_L5 with 10^6 samples and sequence (4, 1, 3) for 4 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 101s - loss: 0.0054 - val_loss: 2.6983e-04
Finished in 257.99 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L15_L10_L5 with 10^6 samples and sequence (4, 1, 3) for 5 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 101s - loss: 0.0012 - val_loss: 6.7167e-05
Finished in 257.62 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L30_L20_L10 with 10^6 samples and sequence (4, 1, 3) for 1 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 177s - loss: 0.0959 - val_loss: 0.0782
Finished in 382.10 seconds with Test/Train AUC 0.6905/0.6868 and max F1 0.0669

Train L30_L20_L10 with 10^6 samples and sequence (4, 1, 3) for 2 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 173s - loss: 0.0310 - val_loss: 3.5274e-04
Finished in 362.84 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L30_L20_L10 with 10^6 samples and sequence (4, 1, 3) for 3 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 173s - loss: 6.4778e-04 - val_loss: 7.6886e-05
Finished in 363.46 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L30_L20_L10 with 10^6 samples and sequence (4, 1, 3) for 4 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 172s - loss: 5.6381e-04 - val_loss: 3.0536e-05
Finished in 362.76 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

Train L30_L20_L10 with 10^6 samples and sequence (4, 1, 3) for 5 epochs
Train on 712492 samples, validate on 37500 samples
Epoch 1/1
712492/712492 [==============================] - 172s - loss: 1.1090e-04 - val_loss: 9.7543e-06
Finished in 362.26 seconds with Test/Train AUC 1.0000/1.0000 and max F1 1.0000

In [3]:

    

results = pickle.load(open("BinaryF.p", "rb"))


    

In [11]:

    

results[0].keys()


    

    
Out[11]:





dict_keys(['epochs', 'test_res', 'size', 'sequence', 'model', 'duration', 'train_res'])

In [18]:

    

def extract_scores(result):
    extracted = {}
    for kind in ("train_", "test_"):
        for score in ("AUC", "F1 Max"):
            extracted[kind + score] = result[kind + "res"][score]

    extracted.update({k:v for k, v in result.items() if not k.endswith("res")})
    return extracted


    

In [19]:

    

parsed = [extract_scores(x) for x in results]


    

In [23]:

    

df = pd.DataFrame(parsed)
df.head()


    

    
Out[23]:






  

    

      

      
duration
      
epochs
      
model
      
sequence
      
size
      
test_AUC
      
test_F1 Max
      
train_AUC
      
train_F1 Max
    
  
  

    

      
0
      
9.335835
      
1
      
L5
      
(3, 1)
      
10^5
      
0.453976
      
0.117137
      
0.449026
      
0.115465
    
    

      
1
      
7.733341
      
2
      
L5
      
(3, 1)
      
10^5
      
0.513589
      
0.118988
      
0.516810
      
0.118044
    
    

      
2
      
7.882221
      
3
      
L5
      
(3, 1)
      
10^5
      
0.571297
      
0.138604
      
0.578429
      
0.135557
    
    

      
3
      
7.754082
      
4
      
L5
      
(3, 1)
      
10^5
      
0.718298
      
0.212551
      
0.724985
      
0.211710
    
    

      
4
      
8.123513
      
5
      
L5
      
(3, 1)
      
10^5
      
0.746254
      
0.233888
      
0.752928
      
0.231837
    
  

In [24]:

    

df.groupby("size").mean()


    

    
Out[24]:






  

    

      

      
duration
      
epochs
      
test_AUC
      
test_F1 Max
      
train_AUC
      
train_F1 Max
    
    

      
size
      

      

      

      

      

      

    
  
  

    

      
10^5
      
22.881739
      
3
      
0.648394
      
0.200117
      
0.646781
      
0.198101
    
    

      
10^6
      
229.281549
      
3
      
0.904657
      
0.783093
      
0.905073
      
0.782867
    
    

      
5*10^5
      
112.210241
      
3
      
0.816076
      
0.555605
      
0.816722
      
0.557018
    
  

In [ ]: