In [2]:
import tarfile
import re
import urllib.request
import os
import random
class ImdbMovieReviews:
DEFAULT_URL = \
'http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz'
TOKEN_REGEX = re.compile(r'[A-Za-z]+|[!?.:,()]')
def __init__(self):
self._cache_dir = './imdb'
self._url = 'http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz'
if not os.path.isfile(self._cache_dir):
urllib.request.urlretrieve(self._url, self._cache_dir)
self.filepath = self._cache_dir
def __iter__(self):
with tarfile.open(self.filepath) as archive:
items = archive.getnames()
for filename in archive.getnames():
if filename.startswith('aclImdb/train/pos/'):
yield self._read(archive, filename), True
elif filename.startswith('aclImdb/train/neg/'):
yield self._read(archive, filename), False
def _read(self, archive, filename):
with archive.extractfile(filename) as file_:
data = file_.read().decode('utf-8')
data = type(self).TOKEN_REGEX.findall(data)
data = [x.lower() for x in data]
return data
In [3]:
import numpy as np
# Spacy is my favourite nlp framework, which havu builtin word embeddings trains on wikipesia
from spacy.en import English
class Embedding:
def __init__(self, length):
# spaCy makes using word vectors very easy.
# The Lexeme , Token , Span and Doc classes all have a .vector property,
# which is a 1-dimensional numpy array of 32-bit floats:
self.parser = English()
self._length = length
self.dimensions = 300
def __call__(self, sequence):
data = np.zeros((self._length, self.dimensions))
# you can access known words from the parser's vocabulary
embedded = [self.parser.vocab[w].vector for w in sequence]
data[:len(sequence)] = embedded
return data
In [18]:
from lazy import lazy
class SequenceClassificationModel:
def __init__(self, data, params):
self.params = params
self.cpnt_path = 'att_checkpoints'
self.attention_size = 500
self._create_placeholders()
self.prediction
self.cost
self.error
self.optimize
self.global_step = 0
self._create_summaries()
self.sess = tf.Session()
self.sess.run(tf.global_variables_initializer())
def _create_placeholders(self):
with tf.name_scope("data"):
self.data = tf.placeholder(tf.float32, [None, self.params.seq_length, self.params.embed_length])
self.target = tf.placeholder(tf.float32, [None, 2])
def _create_summaries(self):
with tf.name_scope("summaries"):
tf.summary.scalar('loss', self.cost)
tf.summary.scalar('erroe', self.error)
self.summary = tf.summary.merge_all()
saver = tf.train.Saver()
@lazy
def length(self):
with tf.name_scope("seq_length"):
used = tf.sign(tf.reduce_max(tf.abs(self.data), reduction_indices=2))
length = tf.reduce_sum(used, reduction_indices=1)
length = tf.cast(length, tf.int32)
return length
@lazy
def prediction(self):
with tf.name_scope("recurrent_layer"):
rnn_output, _ = tf.nn.dynamic_rnn(
self.params.rnn_cell(self.params.rnn_hidden),
self.data,
dtype=tf.float32,
sequence_length=self.length
)
with tf.name_scope("attention"):
hidden_size = rnn_output.shape[2].value # D value - hidden size of the RNN layer
# Trainable parameters
W_omega = tf.Variable(tf.random_normal([self.params.rnn_hidden, self.attention_size], stddev=0.1))
b_omega = tf.Variable(tf.random_normal([self.attention_size], stddev=0.1))
u_omega = tf.Variable(tf.random_normal([self.attention_size], stddev=0.1))
# Applying fully connected layer with non-linear activation to each of the B*T timestamps;
# the shape of `v` is (B,T,D)*(D,A)=(B,T,A), where A=attention_size
v = tf.tanh(tf.tensordot(rnn_output, W_omega, axes=1) + b_omega)
# For each of the timestamps its vector of size A from `v` is reduced with `u` vector
vu = tf.tensordot(v, u_omega, axes=1) # (B,T) shape
alphas = tf.nn.softmax(vu) # (B,T) shape also
# Output of (Bi-)RNN is reduced with attention vector; the result has (B,D) shape
# output = tf.reduce_sum(rnn_output * tf.expand_dims(alphas, -1), 1)
last = tf.reduce_sum(rnn_output * tf.expand_dims(alphas, -1), 1)
# last = self._last_relevant(output, self.length)
with tf.name_scope("softmax_layer"):
num_classes = int(self.target.get_shape()[1])
weight = tf.Variable(tf.truncated_normal(
[self.params.rnn_hidden, num_classes], stddev=0.01))
bias = tf.Variable(tf.constant(0.1, shape=[num_classes]))
prediction = tf.nn.softmax(tf.matmul(last, weight) + bias)
return prediction
@lazy
def cost(self):
cross_entropy = -tf.reduce_sum(self.target * tf.log(self.prediction))
return cross_entropy
@lazy
def error(self):
self.mistakes = tf.not_equal(
tf.argmax(self.target, 1), tf.argmax(self.prediction, 1))
return tf.reduce_mean(tf.cast(self.mistakes, tf.float32))
@lazy
def optimize(self):
with tf.name_scope("optimization"):
gradient = self.params.optimizer.compute_gradients(self.cost)
if self.params.gradient_clipping:
limit = self.params.gradient_clipping
gradient = [
(tf.clip_by_value(g, -limit, limit), v)
if g is not None else (None, v)
for g, v in gradient]
optimize = self.params.optimizer.apply_gradients(gradient)
return optimize
def train(self, batches, save_prefix, save_every=10):
saver = tf.train.Saver()
if os.path.isdir(self.cpnt_path):
saver.restore(self.sess, tf.train.latest_checkpoint(self.cpnt_path))
else:
os.makedirs(self.cpnt_path)
summary_path = os.path.join('att_graphs', 'run{}'.format(self.global_step))
summary_writer = tf.summary.FileWriter(summary_path, self.sess.graph)
self.global_step += 1
for index, batch in enumerate(batches):
feed = {model.data: batch[0], model.target: batch[1]}
error, _, summary_str = self.sess.run([model.error, model.optimize, model.summary], feed)
print('{}: {:3.1f}%'.format(index + 1, 100 * error))
if index % save_every == 0:
summary_writer.add_summary(summary_str, index)
summary_writer.flush()
if index % save_every == 0:
save_path = os.path.join(self.cpnt_path, save_prefix)
print('saving...', save_path)
saver.save(self.sess, save_path, global_step=index)
saver.save(self.sess, os.path.join(self.cpnt_path, save_prefix + '_final'))
def predict_proba(self, data):
feed = {model.data: data, }
prediction = self.sess.run([model.prediction], feed)
return prediction
def close(self):
tf.reset_default_graph()
self.session.close()
In [5]:
def preprocess_batched(iterator, length, embedding, batch_size):
iterator = iter(iterator)
while True:
data = np.zeros((batch_size, length, embedding.dimensions))
target = np.zeros((batch_size, 2))
for index in range(batch_size):
text, label = next(iterator)
data[index] = embedding(text)
target[index] = [1, 0] if label else [0, 1]
yield data, target
In [6]:
reviews = list(ImdbMovieReviews())
In [7]:
random.shuffle(reviews)
In [8]:
length = max(len(x[0]) for x in reviews)
embedding = Embedding(length)
In [9]:
from attrdict import AttrDict
params = AttrDict(
rnn_cell=tf.contrib.rnn.GRUCell,
rnn_hidden=300,
optimizer=tf.train.RMSPropOptimizer(0.002),
batch_size=20,
gradient_clipping=100,
seq_length=length,
embed_length=embedding.dimensions
)
In [19]:
batches = preprocess_batched(reviews, length, embedding, params.batch_size)
In [20]:
tf.reset_default_graph()
model = SequenceClassificationModel(data, params)
In [21]:
model.train(batches, save_prefix='simple-rnn-attention')