In [0]:
# Based on
# https://github.com/fchollet/deep-learning-with-python-notebooks/blob/master/6.2-understanding-recurrent-neural-networks.ipynb
In [0]:
import warnings
warnings.filterwarnings('ignore')
%matplotlib inline
%pylab inline
import matplotlib.pyplot as plt
import pandas as pd
import tensorflow as tf
from tensorflow import keras
In [0]:
# https://keras.io/datasets/#imdb-movie-reviews-sentiment-classification
max_features = 10000 # number of words to consider as features
maxlen = 500 # cut texts after this number of words (among top max_features most common words)
# each review is encoded as a sequence of word indexes
# indexed by overall frequency in the dataset
# output is 0 (negative) or 1 (positive)
imdb = tf.keras.datasets.imdb.load_data(num_words=max_features)
(raw_input_train, y_train), (raw_input_test, y_test) = imdb
# https://www.tensorflow.org/api_docs/python/tf/keras/preprocessing/sequence/pad_sequences
input_train = tf.keras.preprocessing.sequence.pad_sequences(raw_input_train, maxlen=maxlen)
input_test = tf.keras.preprocessing.sequence.pad_sequences(raw_input_test, maxlen=maxlen)
In [0]:
# Batch Normalization:
# https://towardsdatascience.com/batch-normalization-in-neural-networks-1ac91516821c
# https://www.quora.com/Why-does-batch-normalization-help
from tensorflow.keras.layers import GRU, Embedding, Bidirectional, BatchNormalization, Dropout
embedding_dim = 32
dropout = 0.6
recurrent_dropout = 0.4
model = keras.Sequential()
# encoder
model.add(Embedding(input_dim=max_features, output_dim=embedding_dim, input_length=maxlen))
# https://arxiv.org/ftp/arxiv/papers/1701/1701.05923.pdf
# n = output dimension
# m = input dimension
# Total number of parameters for
# RNN = n**2 + nm + n
# GRU = 3 × (n**2 + nm + n)
# LSTM = 4 × (n**2 + nm + n)
# return_sequences passes all outputs of all timesteps (not only the last one) to the next layer
model.add(GRU(name='gru1', units=32, dropout=dropout, recurrent_dropout=recurrent_dropout, return_sequences=True))
# for embedding: 32*2 (“standard deviation” parameter (gamma), “mean” parameter (beta)) trainable parameters
# and 32*2 (moving_mean and moving_variance) non-trainable parameters
model.add(BatchNormalization())
model.add(Dropout(dropout))
# stack recurrent layers like with fc
model.add(GRU(name='gru2', units=32))
model.add(BatchNormalization())
model.add(Dropout(dropout))
# latent space
model.add(tf.keras.layers.Dense(name='fc', units=32, activation='relu'))
# binary classifier as decoder
model.add(tf.keras.layers.Dense(name='classifier', units=1, activation='sigmoid'))
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
model.summary()
In [0]:
batch_size = 1000
%time history = model.fit(input_train, y_train, epochs=15, batch_size=batch_size, validation_split=0.2)
In [0]:
train_loss, train_accuracy = model.evaluate(input_train, y_train, batch_size=batch_size)
train_accuracy
Out[0]:
In [0]:
test_loss, test_accuracy = model.evaluate(input_test, y_test, batch_size=batch_size)
test_accuracy
Out[0]:
In [0]:
def plot_history(history, samples=10, init_phase_samples=None):
epochs = history.params['epochs']
acc = history.history['acc']
val_acc = history.history['val_acc']
every_sample = int(epochs / samples)
acc = pd.DataFrame(acc).iloc[::every_sample, :]
val_acc = pd.DataFrame(val_acc).iloc[::every_sample, :]
fig, ax = plt.subplots(figsize=(20,5))
ax.plot(acc, 'bo', label='Training acc')
ax.plot(val_acc, 'b', label='Validation acc')
ax.set_title('Training and validation accuracy')
ax.legend()
plot_history(history)
In [0]:
# precition
model.predict(input_test[0:5])
Out[0]:
In [0]:
# ground truth
y_test[0:5]
Out[0]:
In [0]:
word_to_id = keras.datasets.imdb.get_word_index()
def encode_text(text):
input_words = text.lower().split()
input_tokens = np.array([word_to_id[word] for word in input_words])
padded_input_tokens = keras.preprocessing.sequence.pad_sequences([input_tokens], maxlen=maxlen)
return padded_input_tokens
def predict_text(model, text):
input_sequence = encode_text(text)
embeddings = model.predict(input_sequence)
return embeddings
In [0]:
predict_text(model, "don't watch this movie")
Out[0]:
In [0]:
predict_text(model, "lovely")
Out[0]:
In [0]:
predict_text(model, "pathetic shit")
Out[0]:
In [0]:
predict_text(model, "this is not a shit movie")
Out[0]:
In [0]:
predict_text(model, "such a bad movie")
Out[0]:
In [0]: