In [3]:
from __future__ import division, print_function, absolute_import
import tensorflow as tf
import tflearn
import pickle
import numpy as np
from tensorflow.contrib import learn
from tflearn.layers.core import input_data, dropout, fully_connected
from tflearn.layers.conv import conv_1d, global_max_pool
from tflearn.layers.merge_ops import merge
from tflearn.layers.estimator import regression
In [4]:
# Load preprocessed datasets
with open('preprocess_x_2.pickle', 'rb') as handle:
x_shuffled = pickle.load(handle)
with open('preprocess_y_2.pickle', 'rb') as handle:
y_shuffled = pickle.load(handle)
print ("Files loaded.")
print ("x_shuffled size: {:d}".format(len(x_shuffled)))
print ("y_shuffled size: {:d}".format(len(y_shuffled)))
In [5]:
# Split train/test set
from sklearn.model_selection import StratifiedShuffleSplit
sss = StratifiedShuffleSplit(n_splits=10, test_size=0.1, random_state=42)
for train_ind, test_ind in sss.split(x_shuffled, y_shuffled):
print ("TRAIN:", train_ind, "TEST:", test_ind)
X_train, X_val = x_shuffled[train_ind], x_shuffled[test_ind]
y_train, y_val = y_shuffled[train_ind], y_shuffled[test_ind]
print("Train features dimensions: {:d}, {:d}".format(*X_train.shape))
print("Train labels dimensions: {:d}, {:d}".format(*y_train.shape))
print("Test features dimensions: {:d}, {:d}".format(*X_val.shape))
print("Test labels dimensions: {:d}, {:d}".format(*y_val.shape))
In [14]:
'''# Building convolutional network
network = input_data(shape=[None, 407], name='input')
# Converts all words in vocabulary to lower dimensional representation
network = tflearn.embedding(network, input_dim=3800, output_dim=128)
branch1 = conv_1d(network, 128, 5, padding='valid', activation='relu', regularizer="L2")
branch2 = conv_1d(network, 128, 6, padding='valid', activation='relu', regularizer="L2")
branch3 = conv_1d(network, 128, 7, padding='valid', activation='relu', regularizer="L2")
network = merge([branch1, branch2, branch3], mode='concat', axis=1)
# Change the shape by adding 2 to dimensions
network = tf.expand_dims(network, 2)
network = global_max_pool(network)
network = dropout(network, 0.5)
network = fully_connected(network, 2, activation='softmax')
network = regression(network, optimizer='adam', learning_rate=0.001,
loss='categorical_crossentropy', name='target')
# Training
model = tflearn.DNN(network, tensorboard_verbose=0)
model.fit(X_train, y_train,
n_epoch = 5, shuffle=True,
validation_set=(X_val, y_val),
show_metric=True, batch_size=32,
run_id='oc_1')'''
In [ ]:
'''# TFLearn bi-directional RNN
from __future__ import division, print_function, absolute_import
import tflearn
from tflearn.data_utils import to_categorical, pad_sequences
from tflearn.datasets import imdb
from tflearn.layers.core import input_data, dropout, fully_connected
from tflearn.layers.embedding_ops import embedding
from tflearn.layers.recurrent import bidirectional_rnn, BasicLSTMCell
from tflearn.layers.estimator import regression
net = input_data(shape=[None, 8405])
net = embedding(net, input_dim=15299, output_dim=256)
net = bidirectional_rnn(net, BasicLSTMCell(256), BasicLSTMCell(256))
net = dropout(net, 0.5)
net = fully_connected(net, 2, activation='softmax')
net = regression(net, optimizer='adam', learning_rate=0.0001, loss='categorical_crossentropy')
# Training
model = tflearn.DNN(net, clip_gradients=0., tensorboard_verbose=2)
model.fit(X_train,
y_train,
n_epoch=5,
shuffle=True,
validation_set=(X_val, y_val),
show_metric=True,
batch_size=16,
run_id='os_1'
)'''
In [ ]:
'''#TFLearn LSTM
from __future__ import division, print_function, absolute_import
import tflearn
net = tflearn.input_data([None, 8405])
net = tflearn.embedding(net, input_dim=15299, output_dim=256)
net = tflearn.lstm(net, 256, dropout=0.8)
net = tflearn.fully_connected(net, 2, activation='softmax')
net = tflearn.regression(net, optimizer='adam', learning_rate=0.001,
loss='categorical_crossentropy')
# Training
model = tflearn.DNN(net, tensorboard_verbose=0)
model.fit(X_train, y_train, validation_set=(X_val, y_val), show_metric=True,
batch_size=16)'''
In [23]:
# Build TensorFlow model
sequence_length = X_train.shape[1]
num_classes = y_train.shape[1]
vocab_size = 15299
#embedding_size = 128
embedding_size = 300 # for word2vec
#filter_sizes = [5, 6, 7]
filter_sizes = [3, 5, 7]
num_filters = 256
l2_reg_lambda = 0.0
graph = tf.Graph()
with graph.as_default():
# Placeholders for input, output and dropout
input_x = tf.placeholder(tf.int32, [None, sequence_length], name="input_x")
input_y = tf.placeholder(tf.float32, [None, num_classes], name="input_y")
dropout_keep_prob = tf.placeholder(tf.float32, name="dropout_keep_prob")
# Keep track of L2 regularization loss
l2_loss = tf.constant(0.0)
# Build model
# Embedding layer
with tf.device('/cpu:0'), tf.name_scope("embedding"):
W_em = tf.Variable(tf.random_uniform([vocab_size, embedding_size], -1.0, 1.0), name="W_em")
embedded_chars = tf.nn.embedding_lookup(W_em, input_x)
embedded_chars_expanded = tf.expand_dims(embedded_chars, -1)
# Create a convolution + maxpool layer for each filter size
pooled_outputs = []
for i, filter_size in enumerate(filter_sizes):
with tf.name_scope("conv-maxpool-%s" % filter_size):
# Convolution layer
filter_shape = [filter_size, embedding_size, 1, num_filters]
W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name="W-%s" % filter_size)
b = tf.Variable(tf.constant(0.1, shape=[num_filters]), name="b-%s" % filter_size)
conv = tf.nn.conv2d(
embedded_chars_expanded,
W,
strides=[1, 1, 1, 1],
padding="VALID",
name="conv")
# Apply nonlinearity
h = tf.nn.relu(tf.nn.bias_add(conv, b), name="relu")
# Maxpooling over the outputs
pooled = tf.nn.max_pool(
h,
ksize=[1, sequence_length - filter_size + 1, 1, 1],
strides=[1, 1, 1, 1],
padding="VALID",
name="pool")
pooled_outputs.append(pooled)
# Combine all pooled features
num_filters_total = num_filters * len(filter_sizes)
h_pool = tf.concat(3, pooled_outputs)
h_pool_flat = tf.reshape(h_pool, [-1, num_filters_total])
# Add dropout
with tf.name_scope("dropout"):
h_drop = tf.nn.dropout(h_pool_flat, dropout_keep_prob)
# Final (unnormalized) scores and predictions
with tf.name_scope("output"):
W = tf.get_variable(
"W",
shape=[num_filters_total, num_classes],
initializer=tf.contrib.layers.xavier_initializer())
b = tf.Variable(tf.constant(0.1, shape=[num_classes]), name="b")
l2_loss += tf.nn.l2_loss(W)
l2_loss += tf.nn.l2_loss(b)
scores = tf.nn.xw_plus_b(h_drop, W, b, name="scores")
predictions = tf.argmax(scores, 1, name="predictions")
# Calculate mean cross-entropy loss
with tf.name_scope("loss"):
losses = tf.nn.softmax_cross_entropy_with_logits(scores, input_y)
loss = tf.reduce_mean(losses) + l2_reg_lambda * l2_loss
# Accuracy
with tf.name_scope("accuracy"):
correct_predictions = tf.equal(predictions, tf.argmax(input_y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_predictions, tf.float32), name="accuracy")
# AUC
with tf.name_scope("auc"):
a = tf.cast(tf.argmax(predictions, 1),tf.float32)
b = tf.cast(tf.argmax(input_y, 1),tf.float32)
auc = tf.contrib.metrics.streaming_auc(a, b)
# Optimizer
global_step = tf.Variable(0, name="global_step", trainable=False)
starter_learning_rate = 0.0005
# Decay factor of 0.95 after every 10000 steps.
with tf.name_scope('learning_rate'):
learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step, 10000, 0.95)
with tf.name_scope('optimizer'):
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(loss, global_step=global_step)
In [24]:
# ======== Training =========
with tf.Session(graph=graph) as sess:
tf.initialize_all_variables().run()
#sess.run(tf.initialize_all_variables())
#sess.run(tf.initialize_local_variables())
saver = tf.train.Saver(tf.all_variables())
print('Initialized')
def train_step(x_batch, y_batch):
feed_dict = {
input_x: x_batch,
input_y: y_batch,
dropout_keep_prob: 0.5
}
_, step, l, accuracy_train = sess.run(
[optimizer, global_step, loss, accuracy], feed_dict=feed_dict)
return step, l, accuracy_train
def val_step(x_val, y_val):
feed_dict = {
input_x: x_val,
input_y: y_val,
dropout_keep_prob: 1.0
}
step, loss_val, accuracy_val = sess.run(
[global_step, loss, accuracy], feed_dict=feed_dict)
return accuracy_val
def load_word2vec(filepath, vocab_size, embedding_size, max_document_length):
'''
Loads pretrained word2vec weights to be fed into the graph,
instead of training the word embeddings from scratch.
'''
# Initialize matrix
initW = np.random.uniform(-0.25, 0.25, (vocab_size, embedding_size))
# Load vectors from word2vec
print("Load word2vec file {}\n".format(filepath))
with open(filepath, 'rb') as f:
header = f.readline()
vocab_size, layer1_size = map(int, header.split())
binary_len = np.dtype('float32').itemsize * layer1_size
for line in xrange(vocab_size):
word = []
while True:
ch = f.read(1)
if ch == ' ':
word = ''.join(word)
break
if ch != '\n':
word.append(ch)
vocab_processor = learn.preprocessing.VocabularyProcessor(max_document_length)
idx = vocab_processor.vocabulary_.get(word)
if idx != None:
initW[idx] = np.fromstring(f.read(binary_len), dtype='float32')
else:
f.read(binary_len)
sess.run(W_em.assign(initW))
def batch_iter(data, batch_size, num_epochs, shuffle=False):
'''
Generates a batch iterator for a dataset.
'''
data = np.array(data)
data_size = len(data)
num_batches_per_epoch = int((len(data)-1)/batch_size) + 1
for epoch in range(num_epochs):
# Shuffle the data at each epoch
if shuffle:
shuffle_indices = np.random.permutation(np.arange(data_size))
shuffled_data = data[shuffle_indices]
else:
shuffled_data = data
for batch_num in range(num_batches_per_epoch):
start_index = batch_num * batch_size
end_index = min((batch_num + 1) * batch_size, data_size)
yield shuffled_data[start_index:end_index]
batch_size = 16
num_epochs = 5
evaluate_every = 10
checkpoint_every = 10
checkpoint = '/home/ubuntu/pynb/oscars/cp'
filepath = '/home/ubuntu/pynb/rt-movie-reviews/GoogleNews-vectors-negative300.bin'
# Load word2vec weights
load_word2vec(filepath, vocab_size, embedding_size, sequence_length)
# Generate batches
batches = batch_iter(
list(zip(X_train, y_train)), batch_size, num_epochs)
# Training loop. For each batch...
for batch in batches:
x_batch, y_batch = zip(*batch)
step, l, accuracy_train = train_step(x_batch, y_batch)
if (step % evaluate_every == 0):
accuracy_val = val_step(X_val, y_val)
print('Minibatch loss at step %d: %f' % (step, l))
print('Minibatch accuracy: {:.4f}'.format(accuracy_train))
print('Validation accuracy: {:.4f}'.format(accuracy_val))
#print('Minibatch AUC: {:.4f}'.format(auc_train))
#print('Validation AUC: {:.4f}'.format(auc_val))
if (step % checkpoint_every == 0):
path = saver.save(sess, checkpoint, global_step=step)
print("Saved model checkpoint to {}\n".format(path))
In [25]:
checkpoint_file = 'cp-30'
# Load preprocessed test dataset
with open('preprocess_test_2.pickle', 'rb') as handle:
X_test = pickle.load(handle)
print("Test set loaded.")
# Restore model and run predictions
with tf.Session(graph=graph) as sess:
print("Loading variables from '%s'." % checkpoint_file)
saver.restore(sess, checkpoint_file)
print("Model restored.")
pred = sess.run(predictions, feed_dict={
input_x: X_test,
dropout_keep_prob: 1.0
})
print(pred)
In [ ]: