In [0]:
# Adapted from
# https://github.com/keras-team/keras/blob/master/examples/addition_rnn.py

In [0]:
import warnings
warnings.filterwarnings('ignore')

In [22]:
%matplotlib inline
%pylab inline


Populating the interactive namespace from numpy and matplotlib

In [23]:
import tensorflow as tf
tf.logging.set_verbosity(tf.logging.ERROR)
print(tf.__version__)


1.11.0-rc2

In [24]:
# let's see what compute devices we have available, hopefully a GPU 
sess = tf.Session()
devices = sess.list_devices()
for d in devices:
    print(d.name)


/job:localhost/replica:0/task:0/device:CPU:0
/job:localhost/replica:0/task:0/device:GPU:0

In [25]:
# a small sanity check, does tf seem to work ok?
hello = tf.constant('Hello TF!')
print(sess.run(hello))


b'Hello TF!'

In [26]:
from tensorflow import keras
print(keras.__version__)


2.1.6-tf

Step 1: Generate sample equations


In [0]:
class CharacterTable(object):
    """Given a set of characters:
    + Encode them to a one hot integer representation
    + Decode the one hot integer representation to their character output
    + Decode a vector of probabilities to their character output
    """
    def __init__(self, chars):
        """Initialize character table.

        # Arguments
            chars: Characters that can appear in the input.
        """
        self.chars = sorted(set(chars))
        self.char_indices = dict((c, i) for i, c in enumerate(self.chars))
        self.indices_char = dict((i, c) for i, c in enumerate(self.chars))

    def encode(self, C, num_rows):
        """One hot encode given string C.

        # Arguments
            num_rows: Number of rows in the returned one hot encoding. This is
                used to keep the # of rows for each data the same.
        """
        x = np.zeros((num_rows, len(self.chars)))
        for i, c in enumerate(C):
            x[i, self.char_indices[c]] = 1
        return x

    def decode(self, x, calc_argmax=True):
        if calc_argmax:
            x = x.argmax(axis=-1)
        return ''.join(self.indices_char[x] for x in x)

In [0]:
class colors:
    ok = '\033[92m'
    fail = '\033[91m'
    close = '\033[0m'

In [29]:
# Parameters for the model and dataset.
TRAINING_SIZE = 50000
DIGITS = 3
# REVERSE = True
REVERSE = False

# Maximum length of input is 'int + int' (e.g., '345+678'). Maximum length of
# int is DIGITS.
MAXLEN = DIGITS + 1 + DIGITS

# All the numbers, plus sign and space for padding.
chars = '0123456789+ '
ctable = CharacterTable(chars)

questions = []
expected = []
seen = set()
print('Generating data...')
while len(questions) < TRAINING_SIZE:
    f = lambda: int(''.join(np.random.choice(list('0123456789'))
                    for i in range(np.random.randint(1, DIGITS + 1))))
    a, b = f(), f()
    # Skip any addition questions we've already seen
    # Also skip any such that x+Y == Y+x (hence the sorting).
    key = tuple(sorted((a, b)))
    if key in seen:
        continue
    seen.add(key)
    # Pad the data with spaces such that it is always MAXLEN.
    q = '{}+{}'.format(a, b)
    query = q + ' ' * (MAXLEN - len(q))
    ans = str(a + b)
    # Answers can be of maximum size DIGITS + 1.
    ans += ' ' * (DIGITS + 1 - len(ans))
    if REVERSE:
        # Reverse the query, e.g., '12+345  ' becomes '  543+21'. (Note the
        # space used for padding.)
        query = query[::-1]
    questions.append(query)
    expected.append(ans)
print('Total addition questions:', len(questions))


Generating data...
Total addition questions: 50000

In [30]:
questions[0]


Out[30]:
'6+110  '

In [31]:
print('Vectorization...')
x = np.zeros((len(questions), MAXLEN, len(chars)), dtype=np.bool)
y = np.zeros((len(questions), DIGITS + 1, len(chars)), dtype=np.bool)
for i, sentence in enumerate(questions):
    x[i] = ctable.encode(sentence, MAXLEN)
for i, sentence in enumerate(expected):
    y[i] = ctable.encode(sentence, DIGITS + 1)


Vectorization...

In [32]:
len(x[0])


Out[32]:
7

In [33]:
len(questions[0])


Out[33]:
7

In [34]:
questions[0]


Out[34]:
'6+110  '

In [35]:
x[0]


Out[35]:
array([[False, False, False, False, False, False, False, False,  True,
        False, False, False],
       [False,  True, False, False, False, False, False, False, False,
        False, False, False],
       [False, False, False,  True, False, False, False, False, False,
        False, False, False],
       [False, False, False,  True, False, False, False, False, False,
        False, False, False],
       [False, False,  True, False, False, False, False, False, False,
        False, False, False],
       [ True, False, False, False, False, False, False, False, False,
        False, False, False],
       [ True, False, False, False, False, False, False, False, False,
        False, False, False]])

In [36]:
y[0]


Out[36]:
array([[False, False, False,  True, False, False, False, False, False,
        False, False, False],
       [False, False, False,  True, False, False, False, False, False,
        False, False, False],
       [False, False, False, False, False, False, False, False,  True,
        False, False, False],
       [ True, False, False, False, False, False, False, False, False,
        False, False, False]])

In [37]:
expected[0]


Out[37]:
'116 '

In [0]:
# Shuffle (x, y) in unison as the later parts of x will almost all be larger
# digits.
indices = np.arange(len(y))
np.random.shuffle(indices)
x = x[indices]
y = y[indices]

Step 2: Training/Validation Split


In [39]:
# Explicitly set apart 10% for validation data that we never train over.
split_at = len(x) - len(x) // 10
(x_train, x_val) = x[:split_at], x[split_at:]
(y_train, y_val) = y[:split_at], y[split_at:]

print('Training Data:')
print(x_train.shape)
print(y_train.shape)

print('Validation Data:')
print(x_val.shape)
print(y_val.shape)


Training Data:
(45000, 7, 12)
(45000, 4, 12)
Validation Data:
(5000, 7, 12)
(5000, 4, 12)

Step 3: Create Model


In [40]:
# input shape: 7 digits, each being 0-9, + or space (12 possibilities)
MAXLEN, len(chars)


Out[40]:
(7, 12)

In [41]:
from keras.models import Sequential
from keras import layers

# Try replacing LSTM, GRU, or SimpleRNN.
# RNN = layers.LSTM
RNN = layers.SimpleRNN
# RNN = layers.GRU
HIDDEN_SIZE = 128
BATCH_SIZE = 128

print('Build model...')
model = Sequential()
# encoder: simply encode the same input 4 times 
model.add(RNN(units=HIDDEN_SIZE, input_shape=(MAXLEN, len(chars))))
model.add(layers.RepeatVector(DIGITS + 1))

# decoder: have 4 temporal outputs one for each of the digits of the results
# return_sequences=True tells it to keep all 4 temporal outputs, not only the final one (we need all four digits for the results)
model.add(RNN(units=HIDDEN_SIZE, return_sequences=True))

model.add(layers.Dense(name='classifier', units=len(chars), activation='softmax'))

model.compile(loss='categorical_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])
model.summary()


Using TensorFlow backend.
Build model...
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
simple_rnn_1 (SimpleRNN)     (None, 128)               18048     
_________________________________________________________________
repeat_vector_1 (RepeatVecto (None, 4, 128)            0         
_________________________________________________________________
simple_rnn_2 (SimpleRNN)     (None, 4, 128)            32896     
_________________________________________________________________
classifier (Dense)           (None, 4, 12)             1548      
=================================================================
Total params: 52,492
Trainable params: 52,492
Non-trainable params: 0
_________________________________________________________________

In [42]:
model.predict(np.array([x_val[0]]))


Out[42]:
array([[[0.03990376, 0.08339092, 0.10174882, 0.0690695 , 0.11191454,
         0.06986454, 0.0719152 , 0.12844141, 0.03216231, 0.08586498,
         0.08835057, 0.11737339],
        [0.06463561, 0.08647179, 0.10332599, 0.05021508, 0.07692068,
         0.08913328, 0.05807737, 0.15744036, 0.04143989, 0.1009753 ,
         0.06894982, 0.10241491],
        [0.06044635, 0.05534412, 0.08035141, 0.04345266, 0.09017674,
         0.11381816, 0.07136583, 0.21808359, 0.04540214, 0.08416803,
         0.0657936 , 0.07159729],
        [0.05927669, 0.05155819, 0.07752077, 0.0526554 , 0.11733301,
         0.10098131, 0.08565692, 0.20166811, 0.05360102, 0.08825838,
         0.04696324, 0.06452689]]], dtype=float32)

In [43]:
model.predict_classes(np.array([x_val[0]]))


Out[43]:
array([[7, 7, 7, 7]])

Step 4: Train


In [44]:
# Train the model each generation and show predictions against the validation
# dataset.
for iteration in range(1, 20):
    print()
    print('-' * 50)
    print('Iteration', iteration)
    model.fit(x_train, y_train,
              batch_size=BATCH_SIZE,
              epochs=1,
              validation_data=(x_val, y_val))
    # Select 10 samples from the validation set at random so we can visualize
    # errors.
    for i in range(10):
        ind = np.random.randint(0, len(x_val))
        rowx, rowy = x_val[np.array([ind])], y_val[np.array([ind])]
        preds = model.predict_classes(rowx, verbose=0)
        q = ctable.decode(rowx[0])
        correct = ctable.decode(rowy[0])
        guess = ctable.decode(preds[0], calc_argmax=False)
        print('Q', q[::-1] if REVERSE else q, end=' ')
        print('T', correct, end=' ')
        if correct == guess:
            print(colors.ok + '☑' + colors.close, end=' ')
        else:
            print(colors.fail + '☒' + colors.close, end=' ')
        print(guess)


--------------------------------------------------
Iteration 1
Train on 45000 samples, validate on 5000 samples
Epoch 1/1
45000/45000 [==============================] - 5s 121us/step - loss: 1.6843 - acc: 0.3875 - val_loss: 1.5620 - val_acc: 0.4284
Q 343+18  T 361   326 
Q 53+532  T 585   666 
Q 957+28  T 985   900 
Q 72+549  T 621   762 
Q 19+77   T 96    118 
Q 94+519  T 613   604 
Q 788+200 T 988   1777
Q 54+795  T 849   771 
Q 826+212 T 1038  120 
Q 607+433 T 1040  717 

--------------------------------------------------
Iteration 2
Train on 45000 samples, validate on 5000 samples
Epoch 1/1
45000/45000 [==============================] - 5s 111us/step - loss: 1.4533 - acc: 0.4645 - val_loss: 1.3659 - val_acc: 0.4964
Q 951+9   T 960   989 
Q 45+225  T 270   389 
Q 134+576 T 710   177 
Q 10+141  T 151   211 
Q 116+46  T 162   253 
Q 677+80  T 757   779 
Q 56+64   T 120   111 
Q 981+956 T 1937  1895
Q 82+51   T 133   111 
Q 28+714  T 742   773 

--------------------------------------------------
Iteration 3
Train on 45000 samples, validate on 5000 samples
Epoch 1/1
45000/45000 [==============================] - 5s 112us/step - loss: 1.2711 - acc: 0.5260 - val_loss: 1.2043 - val_acc: 0.5445
Q 91+972  T 1063  1053
Q 41+544  T 585   595 
Q 709+105 T 814   801 
Q 6+379   T 385   376 
Q 21+78   T 99    10  
Q 62+539  T 601   601 
Q 831+2   T 833   837 
Q 591+344 T 935   100 
Q 623+545 T 1168  1177
Q 998+15  T 1013  101 

--------------------------------------------------
Iteration 4
Train on 45000 samples, validate on 5000 samples
Epoch 1/1
45000/45000 [==============================] - 5s 112us/step - loss: 1.0974 - acc: 0.5880 - val_loss: 1.0264 - val_acc: 0.6131
Q 698+30  T 728   739 
Q 95+166  T 261   251 
Q 631+8   T 639   631 
Q 55+894  T 949   939 
Q 4+908   T 912   113 
Q 30+173  T 203   204 
Q 85+25   T 110   121 
Q 33+176  T 209   209 
Q 70+535  T 605   514 
Q 6+795   T 801   701 

--------------------------------------------------
Iteration 5
Train on 45000 samples, validate on 5000 samples
Epoch 1/1
45000/45000 [==============================] - 5s 113us/step - loss: 0.9328 - acc: 0.6469 - val_loss: 0.8759 - val_acc: 0.6652
Q 287+72  T 359   359 
Q 879+898 T 1777  1774
Q 391+93  T 484   4734
Q 911+610 T 1521  1434
Q 89+740  T 829   839 
Q 96+58   T 154   164 
Q 964+1   T 965   967 
Q 1+329   T 330   329 
Q 263+4   T 267   276 
Q 971+399 T 1370  1479

--------------------------------------------------
Iteration 6
Train on 45000 samples, validate on 5000 samples
Epoch 1/1
45000/45000 [==============================] - 5s 112us/step - loss: 0.7843 - acc: 0.7029 - val_loss: 0.7267 - val_acc: 0.7232
Q 491+93  T 584   574 
Q 112+5   T 117   126 
Q 758+291 T 1049  1060
Q 913+9   T 922   922 
Q 302+3   T 305   315 
Q 982+810 T 1792  1802
Q 5+374   T 379   388 
Q 369+86  T 455   445 
Q 197+827 T 1024  1123
Q 743+6   T 749   749 

--------------------------------------------------
Iteration 7
Train on 45000 samples, validate on 5000 samples
Epoch 1/1
45000/45000 [==============================] - 5s 112us/step - loss: 0.6581 - acc: 0.7529 - val_loss: 0.6295 - val_acc: 0.7596
Q 663+792 T 1455  1435
Q 62+34   T 96    90  
Q 30+999  T 1029  1030
Q 87+523  T 610   600 
Q 950+50  T 1000  9000
Q 72+119  T 191   182 
Q 298+443 T 741   630 
Q 47+14   T 61    61  
Q 25+936  T 961   961 
Q 162+934 T 1096  1187

--------------------------------------------------
Iteration 8
Train on 45000 samples, validate on 5000 samples
Epoch 1/1
45000/45000 [==============================] - 5s 111us/step - loss: 0.5621 - acc: 0.7937 - val_loss: 0.5369 - val_acc: 0.8014
Q 883+86  T 969   979 
Q 77+81   T 158   158 
Q 52+509  T 561   551 
Q 843+514 T 1357  1367
Q 462+374 T 836   847 
Q 881+733 T 1614  1614
Q 285+161 T 446   445 
Q 630+761 T 1391  1392
Q 40+56   T 96    97  
Q 437+732 T 1169  1189

--------------------------------------------------
Iteration 9
Train on 45000 samples, validate on 5000 samples
Epoch 1/1
45000/45000 [==============================] - 5s 111us/step - loss: 0.4905 - acc: 0.8253 - val_loss: 0.4922 - val_acc: 0.8167
Q 118+37  T 155   154 
Q 438+85  T 523   513 
Q 5+746   T 751   761 
Q 67+874  T 941   941 
Q 62+539  T 601   501 
Q 360+958 T 1318  1228
Q 80+959  T 1039  1049
Q 216+14  T 230   239 
Q 50+555  T 605   505 
Q 448+582 T 1030  1020

--------------------------------------------------
Iteration 10
Train on 45000 samples, validate on 5000 samples
Epoch 1/1
45000/45000 [==============================] - 5s 112us/step - loss: 0.4426 - acc: 0.8421 - val_loss: 0.4383 - val_acc: 0.8384
Q 90+478  T 568   567 
Q 302+781 T 1083  1082
Q 93+601  T 694   695 
Q 6+314   T 320   320 
Q 326+753 T 1079  1080
Q 8+218   T 226   226 
Q 200+586 T 786   887 
Q 281+414 T 695   704 
Q 3+389   T 392   392 
Q 245+870 T 1115  1126

--------------------------------------------------
Iteration 11
Train on 45000 samples, validate on 5000 samples
Epoch 1/1
45000/45000 [==============================] - 5s 112us/step - loss: 0.3912 - acc: 0.8644 - val_loss: 0.3975 - val_acc: 0.8566
Q 73+997  T 1070  1061
Q 621+7   T 628   629 
Q 675+28  T 703   603 
Q 141+988 T 1129  1120
Q 367+40  T 407   317 
Q 390+21  T 411   411 
Q 118+90  T 208   218 
Q 31+759  T 790   780 
Q 3+738   T 741   731 
Q 51+917  T 968   968 

--------------------------------------------------
Iteration 12
Train on 45000 samples, validate on 5000 samples
Epoch 1/1
45000/45000 [==============================] - 5s 111us/step - loss: 0.3589 - acc: 0.8762 - val_loss: 0.3482 - val_acc: 0.8789
Q 920+46  T 966   976 
Q 844+6   T 850   840 
Q 180+1   T 181   180 
Q 92+649  T 741   741 
Q 897+558 T 1455  1455
Q 61+692  T 753   753 
Q 90+833  T 923   923 
Q 3+12    T 15    15  
Q 999+6   T 1005  1004
Q 207+454 T 661   651 

--------------------------------------------------
Iteration 13
Train on 45000 samples, validate on 5000 samples
Epoch 1/1
45000/45000 [==============================] - 5s 112us/step - loss: 0.3268 - acc: 0.8885 - val_loss: 0.3710 - val_acc: 0.8625
Q 98+519  T 617   627 
Q 289+43  T 332   332 
Q 181+414 T 595   605 
Q 98+651  T 749   759 
Q 30+75   T 105   106 
Q 10+565  T 575   575 
Q 269+855 T 1124  1124
Q 493+377 T 870   860 
Q 639+60  T 699   699 
Q 950+739 T 1689  1679

--------------------------------------------------
Iteration 14
Train on 45000 samples, validate on 5000 samples
Epoch 1/1
45000/45000 [==============================] - 5s 112us/step - loss: 0.3035 - acc: 0.8967 - val_loss: 0.3201 - val_acc: 0.8857
Q 69+297  T 366   366 
Q 39+726  T 765   765 
Q 341+572 T 913   803 
Q 95+513  T 608   608 
Q 0+8     T 8     8   
Q 568+63  T 631   631 
Q 51+503  T 554   554 
Q 67+840  T 907   807 
Q 524+82  T 606   606 
Q 42+857  T 899   809 

--------------------------------------------------
Iteration 15
Train on 45000 samples, validate on 5000 samples
Epoch 1/1
45000/45000 [==============================] - 5s 111us/step - loss: 0.2766 - acc: 0.9075 - val_loss: 0.2947 - val_acc: 0.8954
Q 79+792  T 871   871 
Q 410+8   T 418   418 
Q 62+438  T 500   500 
Q 492+43  T 535   535 
Q 822+95  T 917   918 
Q 97+194  T 291   291 
Q 46+79   T 125   125 
Q 423+8   T 431   431 
Q 16+99   T 115   115 
Q 591+228 T 819   819 

--------------------------------------------------
Iteration 16
Train on 45000 samples, validate on 5000 samples
Epoch 1/1
45000/45000 [==============================] - 5s 111us/step - loss: 0.2658 - acc: 0.9088 - val_loss: 0.2895 - val_acc: 0.8968
Q 353+485 T 838   838 
Q 666+254 T 920   9100
Q 12+791  T 803   803 
Q 670+4   T 674   674 
Q 57+179  T 236   236 
Q 958+1   T 959   959 
Q 0+901   T 901   901 
Q 65+260  T 325   325 
Q 758+291 T 1049  1049
Q 863+443 T 1306  1306

--------------------------------------------------
Iteration 17
Train on 45000 samples, validate on 5000 samples
Epoch 1/1
45000/45000 [==============================] - 5s 112us/step - loss: 0.2462 - acc: 0.9169 - val_loss: 0.2699 - val_acc: 0.9026
Q 740+962 T 1702  1702
Q 85+201  T 286   286 
Q 2+343   T 345   345 
Q 94+218  T 312   312 
Q 998+6   T 1004  1005
Q 35+210  T 245   245 
Q 0+902   T 902   901 
Q 77+83   T 160   160 
Q 97+837  T 934   925 
Q 18+924  T 942   932 

--------------------------------------------------
Iteration 18
Train on 45000 samples, validate on 5000 samples
Epoch 1/1
45000/45000 [==============================] - 5s 111us/step - loss: 0.2323 - acc: 0.9210 - val_loss: 0.2530 - val_acc: 0.9089
Q 551+29  T 580   570 
Q 5+16    T 21    28  
Q 823+90  T 913   913 
Q 11+301  T 312   312 
Q 71+45   T 116   116 
Q 85+75   T 160   150 
Q 23+43   T 66    66  
Q 380+737 T 1117  1117
Q 588+7   T 595   595 
Q 8+488   T 496   486 

--------------------------------------------------
Iteration 19
Train on 45000 samples, validate on 5000 samples
Epoch 1/1
45000/45000 [==============================] - 5s 111us/step - loss: 0.2212 - acc: 0.9253 - val_loss: 0.2318 - val_acc: 0.9176
Q 26+89   T 115   115 
Q 65+414  T 479   479 
Q 329+852 T 1181  1181
Q 815+158 T 973   973 
Q 245+870 T 1115  1125
Q 60+501  T 561   561 
Q 1+240   T 241   241 
Q 58+559  T 617   617 
Q 94+811  T 905   905 
Q 28+310  T 338   338 

In [45]:
x_val[0]


Out[45]:
array([[False, False, False,  True, False, False, False, False, False,
        False, False, False],
       [False, False, False, False,  True, False, False, False, False,
        False, False, False],
       [False, False, False, False, False, False, False, False, False,
        False, False,  True],
       [False,  True, False, False, False, False, False, False, False,
        False, False, False],
       [False, False, False, False, False, False, False, False, False,
        False, False,  True],
       [False, False, False, False, False, False,  True, False, False,
        False, False, False],
       [ True, False, False, False, False, False, False, False, False,
        False, False, False]])

In [46]:
model.predict(np.array([x_val[0]]))


Out[46]:
array([[[2.79526109e-08, 8.51895265e-09, 6.70795816e-06, 1.77009236e-02,
         9.68226492e-01, 1.40420068e-02, 2.21580922e-05, 1.37661075e-07,
         5.23054577e-09, 2.04132999e-09, 1.12506813e-08, 1.60851346e-06],
        [4.10865342e-10, 4.51209692e-09, 1.01068029e-02, 3.87964636e-01,
         5.59736788e-01, 4.18978445e-02, 2.39486151e-04, 1.12083796e-06,
         6.99106195e-08, 7.76496094e-08, 2.84406156e-07, 5.29415593e-05],
        [6.73183315e-11, 7.59089413e-15, 2.87622599e-11, 5.65917162e-06,
         1.37217447e-01, 8.62411082e-01, 3.65810702e-04, 8.57151239e-10,
         3.83236245e-15, 8.39013679e-18, 1.33594549e-18, 2.33289206e-16],
        [9.99999404e-01, 5.21824446e-13, 3.79854134e-11, 5.24088728e-09,
         5.95357790e-07, 2.15290505e-08, 1.03019586e-10, 7.87527595e-14,
         2.56646875e-15, 2.87176763e-14, 1.61396422e-13, 5.31796883e-13]]],
      dtype=float32)

In [47]:
model.predict_classes(np.array([x_val[0]]))


Out[47]:
array([[4, 4, 5, 0]])

In [0]: