In [0]:

    

! pip install nltk


    

In [0]:

    

import tensorflow as tf
import tensorflow_hub as hub
import tensorflow.keras.backend as K
import numpy as np


    

    




WARNING: Logging before flag parsing goes to stderr.
W0325 21:47:56.237649 139708550211456 __init__.py:56] Some hub symbols are not available because TensorFlow version is less than 1.14

In [0]:

    

tf.__version__, np.__version__


    

    
Out[0]:





('1.13.1', '1.14.6')

In [0]:

    

class ElmoEmbeddingLayer(tf.keras.layers.Layer):
    """Taken from: 
    https://github.com/strongio/keras-elmo/blob/master/Elmo%20Keras.ipynb"""
    def __init__(self, **kwargs):
        self.dimensions = 1024
        super(ElmoEmbeddingLayer, self).__init__(**kwargs)

    def build(self, input_shape):
        self.elmo = hub.Module(
            'https://tfhub.dev/google/elmo/2', 
            trainable=self.trainable,
            name="{}_module".format(self.name)
        )
        if self.trainable:
          self._trainable_weights.extend(
              tf.trainable_variables(scope="^{}_module/.*".format(self.name))
          )
        # Changed assuming trainable weights might be set using 
        super(ElmoEmbeddingLayer, self).build(input_shape)

    def call(self, x, mask=None):
        result = self.elmo(
            K.squeeze(K.cast(x, tf.string), axis=1),
            as_dict=True,
            signature='default',
        )['default']
        return result

    def compute_mask(self, inputs, mask=None):
        return K.not_equal(inputs, '--PAD--')

    def compute_output_shape(self, input_shape):
        return (input_shape[0], self.dimensions)

def create_model(train_elmo=False):
  # Create Sequential model
  model = tf.keras.Sequential([
      # Need to explicitly include input layer 
      # to allow keras to accept string input
      # Taken from:
      # https://gist.github.com/colinmorris/9183206284b4fe3179809098e809d009
      tf.keras.layers.InputLayer(dtype='string', input_shape=(1,)),
      ElmoEmbeddingLayer(trainable=train_elmo),
      tf.keras.layers.Dense(1)
  ])
  
  # Needed to initialize elmo variables
  sess = K.get_session()
  init = tf.global_variables_initializer()
  sess.run(init)
  
  # Compile model
  model.compile(
      optimizer="adam", 
      loss="binary_crossentropy", 
      metrics=["accuracy"]
  )
  return model


    

In [0]:

    

model = create_model(train_elmo=True)


    

    




WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/control_flow_ops.py:3632: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.






    




W0325 21:47:56.784782 139708550211456 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/control_flow_ops.py:3632: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.






    




INFO:tensorflow:Saver not created because there are no variables in the graph to restore






    




I0325 21:47:57.561886 139708550211456 saver.py:1483] Saver not created because there are no variables in the graph to restore

In [0]:

    

X = np.array([
    "This is good",
    "This is bad"
]).reshape(2, 1)
y = np.array([0, 1]).reshape(2, 1)
X.shape, y.shape


    

    
Out[0]:





((2, 1), (2, 1))

In [0]:

    

model.fit(X, y)


    

    




WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.






    




W0325 21:47:59.085000 139708550211456 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.






    




2/2 [==============================] - 2s 790ms/sample - loss: 8.0590 - acc: 0.5000






    
Out[0]:





<tensorflow.python.keras.callbacks.History at 0x7f0ff58115c0>

In [0]:

    

model.summary()


    

    




_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
elmo_embedding_layer (ElmoEm (None, 1024)              4         
_________________________________________________________________
dense (Dense)                (None, 1)                 1025      
=================================================================
Total params: 1,029
Trainable params: 1,029
Non-trainable params: 0
_________________________________________________________________

In [0]:

    

model.trainable_weights


    

    
Out[0]:





[<tf.Variable 'elmo_embedding_layer_module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'dense/kernel:0' shape=(1024, 1) dtype=float32>,
 <tf.Variable 'dense/bias:0' shape=(1,) dtype=float32>]

In [0]:

    

elmo = model.layers[0].elmo


    

In [0]:

    

elmo.variables


    

    
Out[0]:





[<tf.Variable 'elmo_embedding_layer_module/aggregation/scaling:0' shape=() dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/CNN/W_cnn_0:0' shape=(1, 1, 16, 32) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/CNN/W_cnn_1:0' shape=(1, 2, 16, 32) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/CNN/W_cnn_2:0' shape=(1, 3, 16, 64) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/CNN/W_cnn_3:0' shape=(1, 4, 16, 128) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/CNN/W_cnn_4:0' shape=(1, 5, 16, 256) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/CNN/W_cnn_5:0' shape=(1, 6, 16, 512) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/CNN/W_cnn_6:0' shape=(1, 7, 16, 1024) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/CNN/b_cnn_0:0' shape=(32,) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/CNN/b_cnn_1:0' shape=(32,) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/CNN/b_cnn_2:0' shape=(64,) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/CNN/b_cnn_3:0' shape=(128,) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/CNN/b_cnn_4:0' shape=(256,) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/CNN/b_cnn_5:0' shape=(512,) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/CNN/b_cnn_6:0' shape=(1024,) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/CNN_high_0/W_carry:0' shape=(2048, 2048) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/CNN_high_0/W_transform:0' shape=(2048, 2048) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/CNN_high_0/b_carry:0' shape=(2048,) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/CNN_high_0/b_transform:0' shape=(2048,) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/CNN_high_1/W_carry:0' shape=(2048, 2048) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/CNN_high_1/W_transform:0' shape=(2048, 2048) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/CNN_high_1/b_carry:0' shape=(2048,) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/CNN_high_1/b_transform:0' shape=(2048,) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/CNN_proj/W_proj:0' shape=(2048, 512) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/CNN_proj/b_proj:0' shape=(512,) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/RNN_0/RNN/MultiRNNCell/Cell0/rnn/lstm_cell/bias:0' shape=(16384,) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/RNN_0/RNN/MultiRNNCell/Cell0/rnn/lstm_cell/kernel:0' shape=(1024, 16384) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/RNN_0/RNN/MultiRNNCell/Cell0/rnn/lstm_cell/projection/kernel:0' shape=(4096, 512) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/RNN_0/RNN/MultiRNNCell/Cell1/rnn/lstm_cell/bias:0' shape=(16384,) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/RNN_0/RNN/MultiRNNCell/Cell1/rnn/lstm_cell/kernel:0' shape=(1024, 16384) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/RNN_0/RNN/MultiRNNCell/Cell1/rnn/lstm_cell/projection/kernel:0' shape=(4096, 512) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/RNN_1/RNN/MultiRNNCell/Cell0/rnn/lstm_cell/bias:0' shape=(16384,) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/RNN_1/RNN/MultiRNNCell/Cell0/rnn/lstm_cell/kernel:0' shape=(1024, 16384) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/RNN_1/RNN/MultiRNNCell/Cell0/rnn/lstm_cell/projection/kernel:0' shape=(4096, 512) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/RNN_1/RNN/MultiRNNCell/Cell1/rnn/lstm_cell/bias:0' shape=(16384,) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/RNN_1/RNN/MultiRNNCell/Cell1/rnn/lstm_cell/kernel:0' shape=(1024, 16384) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/RNN_1/RNN/MultiRNNCell/Cell1/rnn/lstm_cell/projection/kernel:0' shape=(4096, 512) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/bilm/char_embed:0' shape=(261, 16) dtype=float32>]

In [0]:

    

model.layers[0].trainable_weights


    

    
Out[0]:





[<tf.Variable 'elmo_embedding_layer_module/aggregation/weights:0' shape=(3,) dtype=float32>,
 <tf.Variable 'elmo_embedding_layer_module/aggregation/scaling:0' shape=() dtype=float32>]

In [0]:

    

model.predict([["This is so cool"]])


    

    
Out[0]:





array([[-0.363752]], dtype=float32)

In [0]: