Recurrent Neural Network in Keras

In [ ]:

    

import keras
from keras.datasets import imdb
from keras.preprocessing.sequence import pad_sequences
from keras.models import Sequential
from keras.layers import Dense, Dropout, Embedding, SpatialDropout1D
from keras.layers import SimpleRNN # new! 
from keras.callbacks import ModelCheckpoint
import os
from sklearn.metrics import roc_auc_score 
import matplotlib.pyplot as plt 
%matplotlib inline


    

In [ ]:

    

# output directory name:
output_dir = 'model_output/rnn'

# training:
epochs = 16 # way more!
batch_size = 128

# vector-space embedding: 
n_dim = 64 
n_unique_words = 10000 
max_review_length = 100 # lowered due to vanishing gradient over time
pad_type = trunc_type = 'pre'
drop_embed = 0.2 

# RNN layer architecture:
n_rnn = 256 
drop_rnn = 0.2

# dense layer architecture: 
# n_dense = 256
# dropout = 0.2


    

In [ ]:

    

(x_train, y_train), (x_valid, y_valid) = imdb.load_data(num_words=n_unique_words) # removed n_words_to_skip


    

In [ ]:

    

x_train = pad_sequences(x_train, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)
x_valid = pad_sequences(x_valid, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)


    

In [ ]:

    

model = Sequential()

model.add(Embedding(n_unique_words, n_dim, input_length=max_review_length))
model.add(SpatialDropout1D(drop_embed))

# CODE HERE

model.add(Dense(1, activation='sigmoid'))


    

In [ ]:

    

model.summary()


    

In [ ]:

    

model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])


    

In [ ]:

    

modelcheckpoint = ModelCheckpoint(filepath=output_dir+"/weights.{epoch:02d}.hdf5")
if not os.path.exists(output_dir):
    os.makedirs(output_dir)


    

In [ ]:

    

# 80.6% validation accuracy in epoch 4
model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_valid, y_valid), callbacks=[modelcheckpoint])


    

In [ ]:

    

model.load_weights(output_dir+"/weights.03.hdf5") # zero-indexed


    

In [ ]:

    

y_hat = model.predict_proba(x_valid)


    

In [ ]:

    

plt.hist(y_hat)
_ = plt.axvline(x=0.5, color='orange')


    

In [ ]:

    

"{:0.2f}".format(roc_auc_score(y_valid, y_hat)*100.0)


    

In [ ]: