In [1]:

    

import h5py
import numpy as np
from ccfd_dnn.model import *
from ccfd_dnn.model_eval import *


    

    




Using Theano backend.
Using gpu device 0: GeForce GTX 980 (CNMeM is disabled, cuDNN 5103)
/usr/local/lib/python2.7/dist-packages/theano/sandbox/cuda/__init__.py:600: UserWarning:

Your cuDNN version is more recent than the one Theano officially supports. If you see any problems, try updating Theano or downgrading cuDNN to version 5.

In [2]:

    

from keras.engine.training import *


    

In [3]:

    

h5py.enable_ipython_completer()


    

In [4]:

    

table = 'data_little_enc'
w_id = 1


    

In [5]:

    

disk_engine = get_engine()
    ml = ModelLoader(table,w_id)
    model = ml.model
    title = 'BiRNN-DO3-DLE'


    

    




Model Architecture LOADED!

In [6]:

    

model.__dict__


    

    
Out[6]:





{'_function_kwargs': {},
 '_output_mask_cache': {'140706648166992_140706630112336': All{2}.0,
  '140706648166992_9545840': All{2}.0},
 '_output_shape_cache': {'(None, 60, 44)': (None, 60, 3)},
 '_output_tensor_cache': {'140706648166992_9545840': Reshape{3}.0},
 'built': True,
 'container_nodes': {'gru_1_ib-0',
  'gru_2_ib-0',
  'gru_3_ib-0',
  'gru_4_ib-0',
  'gru_5_ib-0',
  'gru_6_ib-0',
  'gru_7_ib-0',
  'gru_8_ib-0',
  'main_input_ib-0',
  'masking_1_ib-0',
  'merge_1_ib-0',
  'merge_2_ib-0',
  'merge_3_ib-0',
  'merge_4_ib-0',
  'timedistributed_1_ib-0'},
 'inbound_nodes': [<keras.engine.topology.Node at 0x7ff8d0b49310>],
 'input_layers': [<keras.engine.topology.InputLayer at 0x7ff8d1c81410>],
 'input_layers_node_indices': [0],
 'input_layers_tensor_indices': [0],
 'input_names': ['main_input'],
 'inputs': [main_input],
 'internal_input_shapes': [(None, 60, 44)],
 'internal_output_shapes': [(None, 60, 3)],
 'layers': [<keras.engine.topology.InputLayer at 0x7ff8d1c81410>,
  <keras.layers.core.Masking at 0x7ff8d1bf2710>,
  <keras.layers.recurrent.GRU at 0x7ff8d1bf26d0>,
  <keras.layers.recurrent.GRU at 0x7ff8d1c46210>,
  <keras.engine.topology.Merge at 0x7ff8d1231250>,
  <keras.layers.recurrent.GRU at 0x7ff8d11922d0>,
  <keras.layers.recurrent.GRU at 0x7ff8d1192b10>,
  <keras.engine.topology.Merge at 0x7ff8d1113050>,
  <keras.layers.recurrent.GRU at 0x7ff8d0e7e090>,
  <keras.layers.recurrent.GRU at 0x7ff8d0e74650>,
  <keras.engine.topology.Merge at 0x7ff8d0d17e50>,
  <keras.layers.recurrent.GRU at 0x7ff8d0b9d390>,
  <keras.layers.recurrent.GRU at 0x7ff8d0b9db50>,
  <keras.engine.topology.Merge at 0x7ff8d0af81d0>,
  <keras.layers.wrappers.TimeDistributed at 0x7ff8d1738890>],
 'layers_by_depth': {0: [<keras.layers.wrappers.TimeDistributed at 0x7ff8d1738890>],
  1: [<keras.engine.topology.Merge at 0x7ff8d0af81d0>],
  2: [<keras.layers.recurrent.GRU at 0x7ff8d0b9d390>,
   <keras.layers.recurrent.GRU at 0x7ff8d0b9db50>],
  3: [<keras.engine.topology.Merge at 0x7ff8d0d17e50>],
  4: [<keras.layers.recurrent.GRU at 0x7ff8d0e7e090>,
   <keras.layers.recurrent.GRU at 0x7ff8d0e74650>],
  5: [<keras.engine.topology.Merge at 0x7ff8d1113050>],
  6: [<keras.layers.recurrent.GRU at 0x7ff8d11922d0>,
   <keras.layers.recurrent.GRU at 0x7ff8d1192b10>],
  7: [<keras.engine.topology.Merge at 0x7ff8d1231250>],
  8: [<keras.layers.recurrent.GRU at 0x7ff8d1bf26d0>,
   <keras.layers.recurrent.GRU at 0x7ff8d1c46210>],
  9: [<keras.layers.core.Masking at 0x7ff8d1bf2710>],
  10: [<keras.engine.topology.InputLayer at 0x7ff8d1c81410>]},
 'loss': 'sparse_categorical_crossentropy',
 'loss_functions': [<function keras.objectives.sparse_categorical_crossentropy>],
 'loss_weights': None,
 'metrics': [mean],
 'metrics_names': ['loss', 'acc'],
 'name': 'model_1',
 'nodes_by_depth': {0: [<keras.engine.topology.Node at 0x7ff8ce193a10>],
  1: [<keras.engine.topology.Node at 0x7ff8d0b49390>],
  2: [<keras.engine.topology.Node at 0x7ff8d0b49bd0>,
   <keras.engine.topology.Node at 0x7ff8d0aeed90>],
  3: [<keras.engine.topology.Node at 0x7ff8d0b9d950>],
  4: [<keras.engine.topology.Node at 0x7ff8d0d17a50>,
   <keras.engine.topology.Node at 0x7ff8d0c13f90>],
  5: [<keras.engine.topology.Node at 0x7ff8d0e749d0>],
  6: [<keras.engine.topology.Node at 0x7ff8d0e7e5d0>,
   <keras.engine.topology.Node at 0x7ff8d1176750>],
  7: [<keras.engine.topology.Node at 0x7ff8d1192910>],
  8: [<keras.engine.topology.Node at 0x7ff8d1188ed0>,
   <keras.engine.topology.Node at 0x7ff8d122bdd0>],
  9: [<keras.engine.topology.Node at 0x7ff8d1c4ce90>],
  10: [<keras.engine.topology.Node at 0x7ff8d1bf2110>]},
 'optimizer': <keras.optimizers.RMSprop at 0x7ff8ce193a90>,
 'outbound_nodes': [],
 'output_layers': [<keras.layers.wrappers.TimeDistributed at 0x7ff8d1738890>],
 'output_layers_node_indices': [0],
 'output_layers_tensor_indices': [0],
 'output_names': ['timedistributed_1'],
 'outputs': [Reshape{3}.0],
 'predict_function': None,
 'sample_weight_mode': 'temporal',
 'sample_weight_modes': ['temporal'],
 'sample_weights': [timedistributed_1_sample_weights],
 'supports_masking': False,
 'targets': [timedistributed_1_target],
 'test_function': None,
 'total_loss': Elemwise{mul,no_inplace}.0,
 'train_function': None}

In [7]:

    

model.layers[0].inbound_nodes[0].__dict__


    

    
Out[7]:





{'inbound_layers': [],
 'input_masks': [None],
 'input_shapes': [(None, 60, 44)],
 'input_tensors': [main_input],
 'node_indices': [],
 'outbound_layer': <keras.engine.topology.InputLayer at 0x7ff8d1c81410>,
 'output_masks': [None],
 'output_shapes': [(None, 60, 44)],
 'output_tensors': [main_input],
 'tensor_indices': []}

In [8]:

    

model.layers[2].__dict__


    

    
Out[8]:





{'U': gru_1_U,
 'U_regularizer': None,
 'W': gru_1_W,
 'W_regularizer': None,
 'activation': <function keras.activations.tanh>,
 'b': gru_1_b,
 'b_regularizer': None,
 'batch_input_shape': (None, None, 44),
 'built': True,
 'constraints': {},
 'consume_less': 'gpu',
 'dropout_U': 0.0,
 'dropout_W': 0.3,
 'go_backwards': False,
 'inbound_nodes': [<keras.engine.topology.Node at 0x7ff8d122bdd0>],
 'init': <function keras.initializations.glorot_uniform>,
 'initial_weights': None,
 'inner_activation': <function keras.activations.hard_sigmoid>,
 'inner_init': <function keras.initializations.orthogonal>,
 'input_dim': 44,
 'input_dtype': 'float32',
 'input_length': None,
 'input_spec': [<keras.engine.topology.InputSpec at 0x7ff8d1c46250>],
 'name': 'gru_1',
 'non_trainable_weights': [],
 'outbound_nodes': [<keras.engine.topology.Node at 0x7ff8d1192910>],
 'output_dim': 320,
 'regularizers': [],
 'return_sequences': True,
 'stateful': False,
 'states': [None],
 'supports_masking': True,
 'trainable': True,
 'trainable_weights': [gru_1_W, gru_1_U, gru_1_b],
 'unroll': False,
 'uses_learning_phase': True}

In [9]:

    

inter_models = []
for layer in model.layers:
    if type(layer) == keras.engine.topology.Merge:
        print layer.__dict__
        inter_models.append(Model(input=model.inputs, output=layer.output))


    

    




{'layers': None, 'regularizers': [], 'outbound_nodes': [<keras.engine.topology.Node object at 0x7ff8d1176750>, <keras.engine.topology.Node object at 0x7ff8d0e7e5d0>], 'trainable_weights': [], 'name': 'merge_1', 'concat_axis': -1, 'supports_masking': True, 'uses_learning_phase': False, 'non_trainable_weights': [], 'dot_axes': [-1, -1], '_output_shape': None, 'mode': 'concat', 'built': True, '_output_mask': None, 'inbound_nodes': [<keras.engine.topology.Node object at 0x7ff8d1192910>], 'input_spec': None, 'node_indices': None, 'constraints': {}}
{'layers': None, 'regularizers': [], 'outbound_nodes': [<keras.engine.topology.Node object at 0x7ff8d0d17a50>, <keras.engine.topology.Node object at 0x7ff8d0c13f90>], 'trainable_weights': [], 'name': 'merge_2', 'concat_axis': -1, 'supports_masking': True, 'uses_learning_phase': False, 'non_trainable_weights': [], 'dot_axes': [-1, -1], '_output_shape': None, 'mode': 'concat', 'built': True, '_output_mask': None, 'inbound_nodes': [<keras.engine.topology.Node object at 0x7ff8d0e749d0>], 'input_spec': None, 'node_indices': None, 'constraints': {}}
{'layers': None, 'regularizers': [], 'outbound_nodes': [<keras.engine.topology.Node object at 0x7ff8d0aeed90>, <keras.engine.topology.Node object at 0x7ff8d0b49bd0>], 'trainable_weights': [], 'name': 'merge_3', 'concat_axis': -1, 'supports_masking': True, 'uses_learning_phase': False, 'non_trainable_weights': [], 'dot_axes': [-1, -1], '_output_shape': None, 'mode': 'concat', 'built': True, '_output_mask': None, 'inbound_nodes': [<keras.engine.topology.Node object at 0x7ff8d0b9d950>], 'input_spec': None, 'node_indices': None, 'constraints': {}}
{'layers': None, 'regularizers': [], 'outbound_nodes': [<keras.engine.topology.Node object at 0x7ff8ce193a10>], 'trainable_weights': [], 'name': 'merge_4', 'concat_axis': -1, 'supports_masking': True, 'uses_learning_phase': False, 'non_trainable_weights': [], 'dot_axes': [-1, -1], '_output_shape': None, 'mode': 'concat', 'built': True, '_output_mask': None, 'inbound_nodes': [<keras.engine.topology.Node object at 0x7ff8d0b49390>], 'input_spec': None, 'node_indices': None, 'constraints': {}}

In [28]:

    

inter_models = []
for layer in model.merge_1_ib-0:
    if type(layer) == keras.engine.topology.Merge:
        print layer.__dict__
        print type(layer())
        inter_models.append(Model(input=model.inputs, output=layer.output))


    

    




---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-28-1d0d327b58fe> in <module>()
      1 inter_models = []
----> 2 for layer in model.merge_1_ib-0:
      3     if type(layer) == keras.engine.topology.Merge:
      4         print layer.__dict__
      5         print type(layer())

AttributeError: 'Model' object has no attribute 'merge_1_ib'

In [18]:

    

test_layer = Input(shape=(None,60,44),name='test')
type(test_layer)


    

    
Out[18]:





theano.tensor.var.TensorVariable

In [20]:

    

mask = Masking(mask_value=0)(test_layer)
type(mask)


    

    




---------------------------------------------------------------------------
Exception                                 Traceback (most recent call last)
<ipython-input-20-09dd718ac33a> in <module>()
----> 1 mask =Dense(64, activation='relu')(test_layer)
      2 type(mask)

/usr/local/lib/python2.7/dist-packages/keras/engine/topology.pyc in __call__(self, x, mask)
    439             # raise exceptions in case the input is not compatible
    440             # with the input_spec specified in the layer constructor
--> 441             self.assert_input_compatibility(x)
    442 
    443             # collect input shapes to build layer

/usr/local/lib/python2.7/dist-packages/keras/engine/topology.pyc in assert_input_compatibility(self, input)
    380                                         self.name + ': expected ndim=' +
    381                                         str(spec.ndim) + ', found ndim=' +
--> 382                                         str(K.ndim(x)))
    383             if spec.dtype is not None:
    384                 if K.dtype(x) != spec.dtype:

Exception: Input 0 is incompatible with layer dense_1: expected ndim=2, found ndim=4

In [ ]:

    

input_layer = Input(shape=(int(seq_len_param), 44),name='main_input')
mask = Masking(mask_value=0)(input_layer)


    

In [12]:

    

t_mid = dt.datetime.now()


    

In [41]:

    

(dt.datetime.now() - t_mid)


    

    
Out[41]:





datetime.timedelta(0, 118, 773899)

In [42]:

    

obj = (dt.datetime.now() - t_mid)


    

    




---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-42-90dcc186405e> in <module>()
      1 obj = (dt.datetime.now() - t_mid)
----> 2 obj.hours

AttributeError: 'datetime.timedelta' object has no attribute 'hours'

In [10]:

    

def days_hours_minutes_seconds(td):
    return td.days, td.seconds//3600, (td.seconds//60)%60, td.seconds%60


    

In [55]:

    

':'.join(days_hours_minutes(obj))


    

    




---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-55-2a3fc9cf33dc> in <module>()
----> 1 ':'.join(days_hours_minutes(obj))

TypeError: sequence item 0: expected string, int found

In [13]:

    

print 'time taken: {time}'.format(time=days_hours_minutes_seconds(dt.datetime.now()-t_mid))


    

    




time taken: (0, 0, 0, 4)

In [58]:

    

dir_child = '/home/botty/Documents/Add_Data/05childbook/'
f_child = 'states.h5'
f = h5py.File(dir_child+f_child, "r")


    

In [62]:

    

f.keys()


    

    
Out[62]:





[u'output1', u'states1']

In [63]:

    

np.array(f.get('states1')).shape


    

    
Out[63]:





(1271900, 200)

In [48]:

    

def export_generator(model, generator, val_samples, max_q_size=10000,remove_pad=False):
    processed_samples = 0
    wait_time = 0.01
    all_outs = []
    all_y_r = []
    all_y_hat = []
    x_exp = []
    y_exp = []
    y_hat_exp = []
    data_gen_queue, _stop = generator_queue(generator, max_q_size=max_q_size)

    while processed_samples < val_samples:
        print 'insdie'
        generator_output = None
        while not _stop.is_set():
            if not data_gen_queue.empty():
                generator_output = data_gen_queue.get()
                break
            else:
                time.sleep(wait_time)

        if isinstance(generator_output, tuple):
            if len(generator_output) == 2:
                x, y = generator_output
                sample_weight = None
            elif len(generator_output) == 3:
                x, y, sample_weight = generator_output
            else:
                _stop.set()
                raise Exception('output of generator should be a tuple '
                                '(x, y, sample_weight) '
                                'or (x, y). Found: ' + str(generator_output))
        else:
            _stop.set()
            raise Exception('output of generator should be a tuple '
                                '(x, y, sample_weight) '
                                'or (x, y). Found: ' + str(generator_output))

        try:
            if x.size != 0:
                print 'x',x.shape
                print 'y',y.shape
                y_hat = model.predict_on_batch(x)
                
                
                x_re = x.reshape(-1, x.shape[-1])
                y_re = y.reshape(-1, y.shape[-1])
                y_hat_re = y_hat.reshape(-1, y_hat.shape[-1])

                #####padding is not excluded in this version
                if remove_pad:
                    pad_ids = np.where(y_re[:,0]!=2)[0]
                    x_re = x_re[pad_ids,:]
                    y_re = y_re[pad_ids,:]
                    y_hat_re = y_hat_re[pad_ids,:]
                    
                print 'x',x_re.shape
                print 'y',y_re.shape
                print 'y_hat',y_hat_re.shape
                x_exp.extend(x_re)
                y_exp.extend(y_re)
                y_hat_exp.extend(y_hat_re)
        except:
            _stop.set()
            raise
        nb_samples = x.shape[0]   

        processed_samples += nb_samples

    _stop.set()
    x_exp = np.array(x_exp,dtype=np.dtype(float))
    y_exp = np.array(y_exp,dtype=np.dtype(float))
    y_hat_exp = np.array(y_hat_exp,dtype=np.dtype(float))
    print x_exp.shape
    print y_exp.shape
    print y_hat_exp.shape


    

In [50]:

    

export_generator(inter_models[0], dummy_gen(), val_samples,max_q_size=1,remove_pad=True)


    

    




insdie
x (101, 60, 44)
y (101, 60, 1)
x (3057, 44)
y (3057, 1)
y_hat (3057, 640)
insdie
x (103, 60, 44)
y (103, 60, 1)
x (3269, 44)
y (3269, 1)
y_hat (3269, 640)
(6326, 44)
(6326, 1)
(6326, 640)






    




Exception in thread Thread-20:
Traceback (most recent call last):
  File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
    self.run()
  File "/usr/lib/python2.7/threading.py", line 763, in run
    self.__target(*self.__args, **self.__kwargs)
  File "/usr/local/lib/python2.7/dist-packages/keras/engine/training.py", line 416, in data_generator_task
    generator_output = next(generator)
  File "<ipython-input-24-e7724c813ca0>", line 4, in dummy_gen
    yield dummy_list[i]
IndexError: list index out of range

In [22]:

    

user_mode = 'train'
trans_mode = 'train'
disk_engine = get_engine()
table = 'data_little_enc'
batch_size = 100
val_samples = 200
encoders = None
events_tbl = None
generator = data_generator(user_mode,trans_mode,disk_engine,encoders,table=table,
                         batch_size=batch_size,usr_ratio=80, class_weight=None,lbl_pad_val = 2, pad_val = -1,events_tbl=events_tbl)


    

    




insdie
last date
0    1396683949000000000
Name: authzn_rqst_proc_tm, dtype: int64
last_date calculated!
Users generator
new users
User List acquired
total # users: 8043
e.g. users [939010, 244963, 553865, 538125, 491888, 846899, 998746, 198975, 863803, 1192947]
total # unique users: 8043
total # sequences: 12263.0
used # sequences: 8084.0
return set cardinality: 5388
USERGEN: user set sequence length:  100.0
USERGEN: user set cardinality:  57
Number of users: 57






    




[==========================================================] 100% Time: 0:00:01






    




sequence acquired in: 1.95530319214
x (102, 60, 44)
y (102, 60, 1)
new users
USERGEN: user set sequence length:  100.0
USERGEN: user set cardinality:  74
Number of users: 74






    




[                                                         ]   0% ETA:  --:--:--






    




---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-22-2940095325c3> in <module>()
      9 generator = data_generator(user_mode,trans_mode,disk_engine,encoders,table=table,
     10                          batch_size=batch_size,usr_ratio=80, class_weight=None,lbl_pad_val = 2, pad_val = -1,events_tbl=events_tbl)
---> 11 export_generator(inter_models[0], generator, val_samples,max_q_size=1)

<ipython-input-21-0a592263f00b> in export_generator(model, generator, val_samples, max_q_size)
     44 
     45 
---> 46                 x = x.reshape(-1, arr.shape[-1])
     47                 y = y.reshape(-1, arr.shape[-1])
     48                 y_hat = y_hat.reshape(-1, arr.shape[-1])

ValueError: total size of new array must be unchanged





    




[==========================================================] 100% Time: 0:00:01






    




sequence acquired in: 1.19768595695

In [17]:

    

arr = np.zeros((102, 60, 640))
new_arr = arr.reshape(-1, arr.shape[-1])


    

In [23]:

    

dummy_list = []
dummy_list.append(next(generator))
dummy_list.append(next(generator))


    

    




new users
USERGEN: user set sequence length:  100.0
USERGEN: user set cardinality:  76
Number of users: 76






    




[==========================================================] 100% Time: 0:00:01






    




sequence acquired in: 1.57875800133
new users
USERGEN: user set sequence length:  101.0
USERGEN: user set cardinality:  69
Number of users: 69






    










    




sequence acquired in: 0.998779058456






    




[==========================================================] 100% Time: 0:00:00

In [24]:

    

def dummy_gen():
    i=0
    while True:
        yield dummy_list[i]
        i+=1


    

In [18]:

    

new_arr.shape


    

    
Out[18]:





(6120, 640)

In [ ]: