In [1]:

    

from utils import *
import tensorflow as tf
from sklearn.cross_validation import train_test_split
import time
import dnc


    

    




/usr/local/lib/python3.5/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
  from ._conv import register_converters as _register_converters
/usr/local/lib/python3.5/dist-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.
  "This module will be removed in 0.20.", DeprecationWarning)

In [2]:

    

trainset = sklearn.datasets.load_files(container_path = 'data', encoding = 'UTF-8')
trainset.data, trainset.target = separate_dataset(trainset,1.0)
print (trainset.target_names)
print (len(trainset.data))
print (len(trainset.target))


    

    




['negative', 'positive']
10662
10662

In [3]:

    

ONEHOT = np.zeros((len(trainset.data),len(trainset.target_names)))
ONEHOT[np.arange(len(trainset.data)),trainset.target] = 1.0
train_X, test_X, train_Y, test_Y, train_onehot, test_onehot = train_test_split(trainset.data, 
                                                                               trainset.target, 
                                                                               ONEHOT, test_size = 0.2)


    

In [4]:

    

concat = ' '.join(trainset.data).split()
vocabulary_size = len(list(set(concat)))
data, count, dictionary, rev_dictionary = build_dataset(concat, vocabulary_size)
print('vocab from size: %d'%(vocabulary_size))
print('Most common words', count[4:10])
print('Sample data', data[:10], [rev_dictionary[i] for i in data[:10]])


    

    




vocab from size: 20332
Most common words [('film', 1453), ('movie', 1270), ('one', 727), ('like', 721), ('story', 477), ('much', 386)]
Sample data [546, 2598, 3171, 17661, 36, 8306, 219, 150, 19, 3738] ['rock', 'destined', '21st', 'centurys', 'new', 'conan', 'hes', 'going', 'make', 'splash']

In [5]:

    

GO = dictionary['GO']
PAD = dictionary['PAD']
EOS = dictionary['EOS']
UNK = dictionary['UNK']


    

In [6]:

    

num_write_heads = 1
num_read_heads = 4
hidden_size = 64
memory_size = 16
word_size = 16
clip_value = 20
max_grad_norm = 5
learning_rate = 1e-4
optimizer_epsilon = 1e-10
batch_size = 32

size_layer = 128
embedded_size = 128
dimension_output = len(trainset.target_names)
maxlen = 50


    

In [13]:

    

def run_model(input_sequence, output_size):
    access_config = {
      "memory_size": memory_size,
      "word_size": word_size,
      "num_reads": num_read_heads,
      "num_writes": num_write_heads,
    }
    controller_config = {
      "hidden_size": hidden_size,
    }

    dnc_core = dnc.DNC(access_config, controller_config, output_size, clip_value)
    initial_state = dnc_core.initial_state(batch_size)
    output_sequence, _ = tf.nn.dynamic_rnn(
      cell=dnc_core,
      inputs=input_sequence,
      time_major=False,
      initial_state=initial_state)
    return output_sequence


    

In [14]:

    

class Model:
    def __init__(self, dict_size):
        
        self.X = tf.placeholder(tf.int32, [None, None])
        self.Y = tf.placeholder(tf.float32, [None, dimension_output])
        encoder_embeddings = tf.Variable(tf.random_uniform([dict_size, embedded_size], -1, 1))
        encoder_embedded = tf.nn.embedding_lookup(encoder_embeddings, self.X)
        outputs = run_model(encoder_embedded, size_layer)
        W = tf.get_variable('w',shape=(size_layer, dimension_output),initializer=tf.orthogonal_initializer())
        b = tf.get_variable('b',shape=(dimension_output),initializer=tf.zeros_initializer())
        self.logits = tf.matmul(outputs[:, -1], W) + b
        self.cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = self.logits, labels = self.Y))
        self.optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(self.cost)
        correct_pred = tf.equal(tf.argmax(self.logits, 1), tf.argmax(self.Y, 1))
        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))


    

In [15]:

    

tf.reset_default_graph()
sess = tf.InteractiveSession()
model = Model(vocabulary_size+4)
sess.run(tf.global_variables_initializer())


    

    




WARNING:tensorflow:Sonnet nest is deprecated. Please use tf.contrib.framework.nest instead. In addition, `map` is renamed to `map_structure`.

In [16]:

    

EARLY_STOPPING, CURRENT_CHECKPOINT, CURRENT_ACC, EPOCH = 5, 0, 0, 0
while True:
    lasttime = time.time()
    if CURRENT_CHECKPOINT == EARLY_STOPPING:
        print('break epoch:%d\n'%(EPOCH))
        break
        
    train_acc, train_loss, test_acc, test_loss = 0, 0, 0, 0
    for i in range(0, (len(train_X) // batch_size) * batch_size, batch_size):
        batch_x = str_idx(train_X[i:i+batch_size],dictionary,maxlen)
        acc, loss, _ = sess.run([model.accuracy, model.cost, model.optimizer], 
                           feed_dict = {model.X : batch_x, model.Y : train_onehot[i:i+batch_size]})
        train_loss += loss
        train_acc += acc
    
    for i in range(0, (len(test_X) // batch_size) * batch_size, batch_size):
        batch_x = str_idx(test_X[i:i+batch_size],dictionary,maxlen)
        acc, loss = sess.run([model.accuracy, model.cost], 
                           feed_dict = {model.X : batch_x, model.Y : test_onehot[i:i+batch_size]})
        test_loss += loss
        test_acc += acc
    
    train_loss /= (len(train_X) // batch_size)
    train_acc /= (len(train_X) // batch_size)
    test_loss /= (len(test_X) // batch_size)
    test_acc /= (len(test_X) // batch_size)
    
    if test_acc > CURRENT_ACC:
        print('epoch: %d, pass acc: %f, current acc: %f'%(EPOCH,CURRENT_ACC, test_acc))
        CURRENT_ACC = test_acc
        CURRENT_CHECKPOINT = 0
    else:
        CURRENT_CHECKPOINT += 1
        
    print('time taken:', time.time()-lasttime)
    print('epoch: %d, training loss: %f, training acc: %f, valid loss: %f, valid acc: %f\n'%(EPOCH,train_loss,
                                                                                          train_acc,test_loss,
                                                                                          test_acc))
    EPOCH += 1


    

    




epoch: 0, pass acc: 0.000000, current acc: 0.538826
time taken: 88.87894034385681
epoch: 0, training loss: 0.691036, training acc: 0.521499, valid loss: 0.689125, valid acc: 0.538826

epoch: 1, pass acc: 0.538826, current acc: 0.554924
time taken: 88.79464864730835
epoch: 1, training loss: 0.680774, training acc: 0.569666, valid loss: 0.682365, valid acc: 0.554924

epoch: 2, pass acc: 0.554924, current acc: 0.594697
time taken: 87.6223452091217
epoch: 2, training loss: 0.653498, training acc: 0.623355, valid loss: 0.663631, valid acc: 0.594697

epoch: 3, pass acc: 0.594697, current acc: 0.621212
time taken: 87.09494757652283
epoch: 3, training loss: 0.606839, training acc: 0.663181, valid loss: 0.657538, valid acc: 0.621212

epoch: 4, pass acc: 0.621212, current acc: 0.626894
time taken: 86.82913708686829
epoch: 4, training loss: 0.560962, training acc: 0.706180, valid loss: 0.657206, valid acc: 0.626894

epoch: 5, pass acc: 0.626894, current acc: 0.643466
time taken: 85.80507326126099
epoch: 5, training loss: 0.518225, training acc: 0.742716, valid loss: 0.660810, valid acc: 0.643466

epoch: 6, pass acc: 0.643466, current acc: 0.652462
time taken: 86.04094171524048
epoch: 6, training loss: 0.477542, training acc: 0.770677, valid loss: 0.668553, valid acc: 0.652462

epoch: 7, pass acc: 0.652462, current acc: 0.661932
time taken: 85.84241056442261
epoch: 7, training loss: 0.438366, training acc: 0.799930, valid loss: 0.680427, valid acc: 0.661932

epoch: 8, pass acc: 0.661932, current acc: 0.666667
time taken: 86.04874634742737
epoch: 8, training loss: 0.400434, training acc: 0.821781, valid loss: 0.696460, valid acc: 0.666667

epoch: 9, pass acc: 0.666667, current acc: 0.668561
time taken: 86.3272213935852
epoch: 9, training loss: 0.363637, training acc: 0.841165, valid loss: 0.716862, valid acc: 0.668561

epoch: 10, pass acc: 0.668561, current acc: 0.671875
time taken: 86.15707421302795
epoch: 10, training loss: 0.327963, training acc: 0.858553, valid loss: 0.742070, valid acc: 0.671875

epoch: 11, pass acc: 0.671875, current acc: 0.673769
time taken: 86.16099977493286
epoch: 11, training loss: 0.293413, training acc: 0.877115, valid loss: 0.772785, valid acc: 0.673769

epoch: 12, pass acc: 0.673769, current acc: 0.676136
time taken: 86.08387613296509
epoch: 12, training loss: 0.260007, training acc: 0.892740, valid loss: 0.809994, valid acc: 0.676136

epoch: 13, pass acc: 0.676136, current acc: 0.678977
time taken: 86.1032600402832
epoch: 13, training loss: 0.227831, training acc: 0.907777, valid loss: 0.854677, valid acc: 0.678977

epoch: 14, pass acc: 0.678977, current acc: 0.680871
time taken: 86.21879816055298
epoch: 14, training loss: 0.197080, training acc: 0.924342, valid loss: 0.907651, valid acc: 0.680871

epoch: 15, pass acc: 0.680871, current acc: 0.686553
time taken: 85.98529934883118
epoch: 15, training loss: 0.168010, training acc: 0.937030, valid loss: 0.970093, valid acc: 0.686553

time taken: 86.02332663536072
epoch: 16, training loss: 0.140883, training acc: 0.947956, valid loss: 1.043842, valid acc: 0.683712

time taken: 86.18879580497742
epoch: 17, training loss: 0.116061, training acc: 0.958294, valid loss: 1.130550, valid acc: 0.681818

time taken: 86.10147905349731
epoch: 18, training loss: 0.094075, training acc: 0.968633, valid loss: 1.229595, valid acc: 0.681345

time taken: 86.30896782875061
epoch: 19, training loss: 0.075330, training acc: 0.976974, valid loss: 1.341129, valid acc: 0.679924

time taken: 86.06641054153442
epoch: 20, training loss: 0.060109, training acc: 0.982848, valid loss: 1.463995, valid acc: 0.680871

break epoch:21

In [ ]: