import fgm tables


In [1]:
!pip install gdown
!mkdir ./data
import gdown

def data_import():
  ids = {
      "tables_of_fgm.h5":"1omAitbGljJeoZPMJ9odkrF27zaoQm61X"
  }

  url = 'https://drive.google.com/uc?id='

  for title, g_id in ids.items(): 
    try:
      output_file = open("/content/data/" + title, 'wb')
      gdown.download(url + g_id, output_file, quiet=False)
    except IOError as e:
      print(e)
    finally:
      output_file.close()
      
data_import()


Collecting gdown
  Downloading https://files.pythonhosted.org/packages/87/fa/f6b4c5728f9828aa4b41b164125b505281bc428e6b13a05d2b210f7162e7/gdown-3.7.4.tar.gz
Requirement already satisfied: filelock in /home/eg/anaconda3/lib/python3.6/site-packages (from gdown) (3.0.8)
Requirement already satisfied: requests in /home/eg/anaconda3/lib/python3.6/site-packages (from gdown) (2.19.1)
Requirement already satisfied: six in /home/eg/anaconda3/lib/python3.6/site-packages (from gdown) (1.11.0)
Requirement already satisfied: tqdm in /home/eg/anaconda3/lib/python3.6/site-packages (from gdown) (4.26.0)
Requirement already satisfied: urllib3<1.24,>=1.21.1 in /home/eg/anaconda3/lib/python3.6/site-packages (from requests->gdown) (1.23)
Requirement already satisfied: idna<2.8,>=2.5 in /home/eg/anaconda3/lib/python3.6/site-packages (from requests->gdown) (2.7)
Requirement already satisfied: certifi>=2017.4.17 in /home/eg/anaconda3/lib/python3.6/site-packages (from requests->gdown) (2018.8.24)
Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /home/eg/anaconda3/lib/python3.6/site-packages (from requests->gdown) (3.0.4)
Building wheels for collected packages: gdown
  Running setup.py bdist_wheel for gdown ... done
  Stored in directory: /home/eg/.cache/pip/wheels/9f/84/05/42b4392bb9ec698e04903201b32305ecbe99acb6bc78a04908
Successfully built gdown
twisted 18.7.0 requires PyHamcrest>=1.9.0, which is not installed.
Installing collected packages: gdown
Successfully installed gdown-3.7.4
You are using pip version 18.1, however version 19.0.3 is available.
You should consider upgrading via the 'pip install --upgrade pip' command.
---------------------------------------------------------------------------
ModuleNotFoundError                       Traceback (most recent call last)
<ipython-input-1-9fc2d2fab0e9> in <module>
      1 get_ipython().system('pip install gdown')
      2 get_ipython().system('mkdir ./data')
----> 3 import gdown
      4 
      5 def data_import():

ModuleNotFoundError: No module named 'gdown'

Function libaries

ResBlock

res_block is the backbone of the resnet structure. The resblock has multi branch, bottle neck layer and skip connection build in. This modularized design has made create deep neural network easy.


In [1]:
import tensorflow as tf
import keras
from keras.layers import Dense, Activation, Input, BatchNormalization, Dropout, concatenate
from keras import layers

def res_branch(bi, conv_name_base, bn_name_base, scale, input_tensor, n_neuron, stage, block,dp1, bn=False):
    x_1 = Dense(scale * n_neuron, name=conv_name_base + '2a_'+str(bi))(input_tensor)
    if bn:
        x_1 = BatchNormalization(axis=-1, name=bn_name_base + '2a_'+str(bi))(x_1)
    x_1 = Activation('relu')(x_1)
    if dp1>0:
      x_1 = Dropout(dp1)(x_1)
    return x_1

def res_block(input_tensor,scale, n_neuron, stage, block, bn=False,branches=0):
    conv_name_base = 'res' + str(stage) + block + '_branch'
    bn_name_base = 'bn' + str(stage) + block + '_branch'

#     scale = 2
    x = Dense(scale * n_neuron, name=conv_name_base + '2a')(input_tensor)
    if bn:
        x = BatchNormalization(axis=-1, name=bn_name_base + '2a')(x)
    x = Activation('relu')(x)
    dp1=0.
    if dp1 >0:
        x = Dropout(dp1)(x)
    
    branch_list=[x]
    for i in range(branches-1):
        branch_list.append(res_branch(i,conv_name_base, bn_name_base, scale,input_tensor,n_neuron,stage,block,dp1,bn))
    if branches-1 > 0:
        x = Dense(n_neuron, name=conv_name_base + '2b')(concatenate(branch_list,axis=-1))
#         x = Dense(n_neuron, name=conv_name_base + '2b')(layers.add(branch_list))
    else:
        x = Dense(n_neuron, name=conv_name_base + '2b')(x)
    
    if bn:
        x = BatchNormalization(axis=-1, name=bn_name_base + '2b')(x)
    x = layers.add([x, input_tensor])
    x = Activation('relu')(x)
    if dp1 >0:
        x = Dropout(dp1)(x)

    return x


Using TensorFlow backend.

data_reader

The read_h5_data function read the table from the hdf5 file.

In the FGM case we chose not to scale the input features, since they all falls between 0 and 1. There are a great variety in the output features. In the reaction region close to stoichiometry the gradient in the output properties are great. A good example is the source term for progress variable, which rises from 0 to 1e5. So the output features are first transformed to logrithmic scale and then rearranged between 0 and 1. The outputs are normalised by its variance. This way the output value will be large where the gradient is great. So during training more focus would be put. The same 'focus design' has been put on the loss function selection as well. mse is selected over mae for that the squared error put more weights on the data samples that shows great changes.


In [10]:
import numpy as np
import pandas as pd
from sklearn.preprocessing import MinMaxScaler, StandardScaler


class data_scaler(object):
    def __init__(self):
        self.norm = None
        self.norm_1 = None
        self.std = None
        self.bc = None
        self.case = None
        self.scale = 1
        self.bias = 1e-20
#         self.bias = 1


        self.switcher = {
            'min_std': 'min_std',
            'std2': 'std2',
            'std_min':'std_min',
            'min': 'min',
            'no':'no',
            'log': 'log',
            'log_min':'log_min',
            'log_std':'log_std',
            'log2': 'log2',
            'sqrt_std': 'sqrt_std',
            'cbrt_std': 'cbrt_std',
            'nrt_std':'nrt_std',
            'bc':'bc',
            'tan': 'tan'
        }

    def fit_transform(self, input_data, case):
        self.case = case
        if self.switcher.get(self.case) == 'min_std':
            self.norm = MinMaxScaler()
            self.std = StandardScaler()
            out = self.norm.fit_transform(input_data)
            out = self.std.fit_transform(out)

        if self.switcher.get(self.case) == 'std2':
            self.std = StandardScaler()
            out = self.std.fit_transform(input_data)

        if self.switcher.get(self.case) == 'std_min':
            self.norm = MinMaxScaler()
            self.std = StandardScaler()
            out = self.std.fit_transform(input_data)
            out = self.norm.fit_transform(out)

        if self.switcher.get(self.case) == 'min':
            self.norm = MinMaxScaler()
            out = self.norm.fit_transform(input_data)

        if self.switcher.get(self.case) == 'no':
            self.norm = MinMaxScaler()
            self.std = StandardScaler()
            out = input_data

        if self.switcher.get(self.case) == 'log_min':
            out = - np.log(np.asarray(input_data / self.scale) + self.bias)
            self.norm = MinMaxScaler()
            out = self.norm.fit_transform(out)

        if self.switcher.get(self.case) == 'log_std':
            out = - np.log(np.asarray(input_data / self.scale) + self.bias)
            self.std = StandardScaler()
            out = self.std.fit_transform(out)

        if self.switcher.get(self.case) == 'log2':
            self.norm = MinMaxScaler()
            self.std = StandardScaler()
            out = self.norm.fit_transform(input_data)
            out = np.log(np.asarray(out) + self.bias)
            out = self.std.fit_transform(out)

        if self.switcher.get(self.case) == 'sqrt_std':
            self.norm = MinMaxScaler()
            self.std = StandardScaler()
            out = self.norm.fit_transform(input_data)
            out = np.sqrt(np.asarray(out / self.scale))
            out = self.std.fit_transform(out)


        if self.switcher.get(self.case) == 'cbrt_std':
            out = np.cbrt(np.asarray(input_data / self.scale))
            self.std = StandardScaler()
            out = self.std.fit_transform(out)

        if self.switcher.get(self.case) == 'nrt_std':
            out = np.power(np.asarray(input_data / self.scale),1/4)
            self.std = StandardScaler()
            out = self.std.fit_transform(out)

        if self.switcher.get(self.case) == 'bc':
            self.bc = PowerTransformer(method='box-cox')
            out = self.bc.fit_transform(input_data + self.bias)
            print('lambda:', self.bc.lambdas_)

        if self.switcher.get(self.case) == 'tan':
            self.norm = MaxAbsScaler()
            self.std = StandardScaler()
            out = self.std.fit_transform(input_data)
            out = self.norm.fit_transform(out)
            out = np.tan(out / (2 * np.pi + self.bias))

        return out

    def transform(self, input_data):
        if self.switcher.get(self.case) == 'min_std':
            out = self.norm.transform(input_data)
            out = self.std.transform(out)

        if self.switcher.get(self.case) == 'std2':
            out = self.std.transform(input_data)

        if self.switcher.get(self.case) == 'std_min':
            out = self.std.transform(input_data)
            out = self.norm.transform(out)

        if self.switcher.get(self.case) == 'min':
            out = self.norm.transform(input_data)

        if self.switcher.get(self.case) == 'no':
            out = input_data

        if self.switcher.get(self.case) == 'log_min':
            out = - np.log(np.asarray(input_data / self.scale) + self.bias)
            out = self.norm.transform(out)

        if self.switcher.get(self.case) == 'log_std':
            out = - np.log(np.asarray(input_data / self.scale) + self.bias)
            out = self.std.transform(out)

        if self.switcher.get(self.case) == 'log2':
            out = self.norm.transform(input_data)
            out = np.log(np.asarray(out) + self.bias)
            out = self.std.transform(out)

        if self.switcher.get(self.case) == 'sqrt_std':
            out = self.norm.transform(input_data)
            out = np.sqrt(np.asarray(out / self.scale))
            out = self.std.transform(out)

        if self.switcher.get(self.case) == 'cbrt_std':
            out = np.cbrt(np.asarray(input_data / self.scale))
            out = self.std.transform(out)

        if self.switcher.get(self.case) == 'nrt_std':
            out = np.power(np.asarray(input_data / self.scale),1/4)
            out = self.std.transform(out)

        if self.switcher.get(self.case) == 'bc':
            out = self.bc.transform(input_data + self.bias)

        if self.switcher.get(self.case) == 'tan':
            out = self.std.transform(input_data)
            out = self.norm.transform(out)
            out = np.tan(out / (2 * np.pi + self.bias))

        return out

    def inverse_transform(self, input_data):

        if self.switcher.get(self.case) == 'min_std':
            out = self.std.inverse_transform(input_data)
            out = self.norm.inverse_transform(out)

        if self.switcher.get(self.case) == 'std2':
            out = self.std.inverse_transform(input_data)

        if self.switcher.get(self.case) == 'std_min':
            out = self.norm.inverse_transform(input_data)
            out = self.std.inverse_transform(out)

        if self.switcher.get(self.case) == 'min':
            out = self.norm.inverse_transform(input_data)

        if self.switcher.get(self.case) == 'no':
            out = input_data

        if self.switcher.get(self.case) == 'log_min':
            out = self.norm.inverse_transform(input_data)
            out = (np.exp(-out) - self.bias) * self.scale

        if self.switcher.get(self.case) == 'log_std':
            out = self.std.inverse_transform(input_data)
            out = (np.exp(-out) - self.bias) * self.scale

        if self.switcher.get(self.case) == 'log2':
            out = self.std.inverse_transform(input_data)
            out = np.exp(out) - self.bias
            out = self.norm.inverse_transform(out)

        if self.switcher.get(self.case) == 'sqrt_std':
            out = self.std.inverse_transform(input_data)
            out = np.power(out,2) * self.scale
            out = self.norm.inverse_transform(out)

        if self.switcher.get(self.case) == 'cbrt_std':
            out = self.std.inverse_transform(input_data)
            out = np.power(out,3) * self.scale

        if self.switcher.get(self.case) == 'nrt_std':
            out = self.std.inverse_transform(input_data)
            out = np.power(out,4) * self.scale

        if self.switcher.get(self.case) == 'bc':
            out = self.bc.inverse_transform(input_data)
            out = out - self.bias

        if self.switcher.get(self.case) == 'tan':
            out = (2 * np.pi + self.bias) * np.arctan(input_data)
            out = self.norm.inverse_transform(out)
            out = self.std.inverse_transform(out)

        return out      

def read_h5_data(fileName, input_features, labels):
    df = pd.read_hdf(fileName)
#     df = df[df['f']<0.45]
    
    input_df=df[input_features]
    in_scaler = data_scaler()
    input_np = in_scaler.fit_transform(input_df.values,'no')

    label_df=df[labels].clip(0)
#     if 'PVs' in labels:
#       label_df['PVs']=np.log(label_df['PVs']+1)
    out_scaler = data_scaler()
    label_np = out_scaler.fit_transform(label_df.values,'cbrt_std')

    return input_np, label_np, df, in_scaler, out_scaler

model

load data


In [11]:
%matplotlib inline
import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt


# define the labels
col_labels=['C2H3', 'C2H6', 'CH2', 'H2CN', 'C2H4', 'H2O2', 'C2H', 'CN',
       'heatRelease', 'NCO', 'NNH', 'N2', 'AR', 'psi', 'CO', 'CH4', 'HNCO',
       'CH2OH', 'HCCO', 'CH2CO', 'CH', 'mu', 'C2H2', 'C2H5', 'H2', 'T', 'PVs',
       'O', 'O2', 'N2O', 'C', 'C3H7', 'CH2(S)', 'NH3', 'HO2', 'NO', 'HCO',
       'NO2', 'OH', 'HCNO', 'CH3CHO', 'CH3', 'NH', 'alpha', 'CH3O', 'CO2',
       'CH3OH', 'CH2CHO', 'CH2O', 'C3H8', 'HNO', 'NH2', 'HCN', 'H', 'N', 'H2O',
       'HCCOH', 'HCNN']

# Taking 0 out
col_labels.remove('AR')
col_labels.remove('heatRelease')

# labels = ['CH4','O2','H2O','CO','CO2','T','PVs','psi','mu','alpha']
# labels = ['T','PVs']
labels = ['CO']
# labels = ['T','CH4','O2','CO2','CO','H2O','H2','OH','psi']
# labels = ['CH2OH','HNCO','CH3OH', 'CH2CHO', 'CH2O', 'C3H8', 'HNO', 'NH2', 'HCN']

# labels = np.random.choice(col_labels,20,replace=False).tolist()
# labels.append('PVs')

# labels = col_labels

print(labels)

input_features=['f','pv','zeta']

# read in the data
x_input, y_label, df, in_scaler, out_scaler = read_h5_data('../data/tables_of_fgm.h5',input_features=input_features, labels = labels)


['CO']

build neural network model


In [12]:
from sklearn.model_selection import train_test_split

import tensorflow as tf
from keras.models import Model
from keras.layers import Dense, Input
from keras.callbacks import ModelCheckpoint

# split into train and test data
x_train, x_test, y_train, y_test = train_test_split(x_input,y_label, test_size=0.01)

n_neuron = 10
scale=3
branches=3
# %%
print('set up ANN')
# ANN parameters
dim_input = x_train.shape[1]
dim_label = y_train.shape[1]

batch_norm = False

# This returns a tensor
inputs = Input(shape=(dim_input,),name='input_1')

# a layer instance is callable on a tensor, and returns a tensor
x = Dense(n_neuron, activation='relu')(inputs)

# less then 2 res_block, there will be variance
x = res_block(x, scale, n_neuron, stage=1, block='a', bn=batch_norm,branches=branches)
x = res_block(x, scale, n_neuron, stage=1, block='b', bn=batch_norm,branches=branches)
# x = res_block(x, scale, n_neuron, stage=1, block='c', bn=batch_norm,branches=branches)


x = Dense(100, activation='relu')(x)
x = Dropout(0.1)(x)
predictions = Dense(dim_label, activation='linear', name='output_1')(x)

model = Model(inputs=inputs, outputs=predictions)
model.summary()


set up ANN
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_1 (InputLayer)            (None, 3)            0                                            
__________________________________________________________________________________________________
dense_3 (Dense)                 (None, 10)           40          input_1[0][0]                    
__________________________________________________________________________________________________
res1a_branch2a (Dense)          (None, 30)           330         dense_3[0][0]                    
__________________________________________________________________________________________________
res1a_branch2a_0 (Dense)        (None, 30)           330         dense_3[0][0]                    
__________________________________________________________________________________________________
res1a_branch2a_1 (Dense)        (None, 30)           330         dense_3[0][0]                    
__________________________________________________________________________________________________
activation_9 (Activation)       (None, 30)           0           res1a_branch2a[0][0]             
__________________________________________________________________________________________________
activation_10 (Activation)      (None, 30)           0           res1a_branch2a_0[0][0]           
__________________________________________________________________________________________________
activation_11 (Activation)      (None, 30)           0           res1a_branch2a_1[0][0]           
__________________________________________________________________________________________________
concatenate_3 (Concatenate)     (None, 90)           0           activation_9[0][0]               
                                                                 activation_10[0][0]              
                                                                 activation_11[0][0]              
__________________________________________________________________________________________________
res1a_branch2b (Dense)          (None, 10)           910         concatenate_3[0][0]              
__________________________________________________________________________________________________
add_3 (Add)                     (None, 10)           0           res1a_branch2b[0][0]             
                                                                 dense_3[0][0]                    
__________________________________________________________________________________________________
activation_12 (Activation)      (None, 10)           0           add_3[0][0]                      
__________________________________________________________________________________________________
res1b_branch2a (Dense)          (None, 30)           330         activation_12[0][0]              
__________________________________________________________________________________________________
res1b_branch2a_0 (Dense)        (None, 30)           330         activation_12[0][0]              
__________________________________________________________________________________________________
res1b_branch2a_1 (Dense)        (None, 30)           330         activation_12[0][0]              
__________________________________________________________________________________________________
activation_13 (Activation)      (None, 30)           0           res1b_branch2a[0][0]             
__________________________________________________________________________________________________
activation_14 (Activation)      (None, 30)           0           res1b_branch2a_0[0][0]           
__________________________________________________________________________________________________
activation_15 (Activation)      (None, 30)           0           res1b_branch2a_1[0][0]           
__________________________________________________________________________________________________
concatenate_4 (Concatenate)     (None, 90)           0           activation_13[0][0]              
                                                                 activation_14[0][0]              
                                                                 activation_15[0][0]              
__________________________________________________________________________________________________
res1b_branch2b (Dense)          (None, 10)           910         concatenate_4[0][0]              
__________________________________________________________________________________________________
add_4 (Add)                     (None, 10)           0           res1b_branch2b[0][0]             
                                                                 activation_12[0][0]              
__________________________________________________________________________________________________
activation_16 (Activation)      (None, 10)           0           add_4[0][0]                      
__________________________________________________________________________________________________
dense_4 (Dense)                 (None, 100)          1100        activation_16[0][0]              
__________________________________________________________________________________________________
dropout_2 (Dropout)             (None, 100)          0           dense_4[0][0]                    
__________________________________________________________________________________________________
output_1 (Dense)                (None, 1)            101         dropout_2[0][0]                  
==================================================================================================
Total params: 5,041
Trainable params: 5,041
Non-trainable params: 0
__________________________________________________________________________________________________

model training

gpu training


In [13]:
import keras.backend as K
from keras.callbacks import LearningRateScheduler
import math

def cubic_loss(y_true, y_pred):
    return K.mean(K.square(y_true - y_pred)*K.abs(y_true - y_pred), axis=-1)

def coeff_r2(y_true, y_pred):
    from keras import backend as K
    SS_res =  K.sum(K.square( y_true-y_pred ))
    SS_tot = K.sum(K.square( y_true - K.mean(y_true) ) )
    return ( 1 - SS_res/(SS_tot + K.epsilon()) )

  
def step_decay(epoch):
   initial_lrate = 0.001
   drop = 0.5
   epochs_drop = 200.0
   lrate = initial_lrate * math.pow(drop,math.floor((1+epoch)/epochs_drop))
   return lrate
  
lrate = LearningRateScheduler(step_decay)

In [14]:
from keras import optimizers

batch_size = 1024*32
epochs = 100
vsplit = 0.1

loss_type='mse'

adam_op = optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999,epsilon=1e-8, decay=0.0, amsgrad=True)

model.compile(loss=loss_type, optimizer=adam_op, metrics=[coeff_r2])
# model.compile(loss=cubic_loss, optimizer=adam_op, metrics=['accuracy'])

# checkpoint (save the best model based validate loss)
!mkdir ./tmp
filepath = "./tmp/weights.best.cntk.hdf5"

checkpoint = ModelCheckpoint(filepath,
                             monitor='val_loss',
                             verbose=1,
                             save_best_only=True,
                             mode='min',
                             period=20)

# callbacks_list = [checkpoint]
callbacks_list = [lrate]


# fit the model
history = model.fit(
    x_train, y_train,
    epochs=epochs,
    batch_size=batch_size,
    validation_split=vsplit,
    verbose=2,
#     callbacks=callbacks_list,
    shuffle=True)

model.save('trained_fgm_nn.h5')


mkdir: cannot create directory ‘./tmp’: File exists
Train on 2236418 samples, validate on 248491 samples
Epoch 1/100
 - 2s - loss: 0.4455 - coeff_r2: 0.5545 - val_loss: 0.1983 - val_coeff_r2: 0.8015
Epoch 2/100
 - 2s - loss: 0.1340 - coeff_r2: 0.8660 - val_loss: 0.0533 - val_coeff_r2: 0.9467
Epoch 3/100
 - 2s - loss: 0.0412 - coeff_r2: 0.9588 - val_loss: 0.0204 - val_coeff_r2: 0.9795
Epoch 4/100
 - 2s - loss: 0.0241 - coeff_r2: 0.9759 - val_loss: 0.0134 - val_coeff_r2: 0.9866
Epoch 5/100
 - 2s - loss: 0.0188 - coeff_r2: 0.9812 - val_loss: 0.0104 - val_coeff_r2: 0.9896
Epoch 6/100
 - 2s - loss: 0.0157 - coeff_r2: 0.9843 - val_loss: 0.0092 - val_coeff_r2: 0.9908
Epoch 7/100
 - 1s - loss: 0.0139 - coeff_r2: 0.9861 - val_loss: 0.0074 - val_coeff_r2: 0.9926
Epoch 8/100
 - 2s - loss: 0.0126 - coeff_r2: 0.9874 - val_loss: 0.0071 - val_coeff_r2: 0.9929
Epoch 9/100
 - 2s - loss: 0.0117 - coeff_r2: 0.9883 - val_loss: 0.0060 - val_coeff_r2: 0.9940
Epoch 10/100
 - 2s - loss: 0.0112 - coeff_r2: 0.9888 - val_loss: 0.0056 - val_coeff_r2: 0.9944
Epoch 11/100
 - 2s - loss: 0.0104 - coeff_r2: 0.9896 - val_loss: 0.0054 - val_coeff_r2: 0.9946
Epoch 12/100
 - 2s - loss: 0.0099 - coeff_r2: 0.9901 - val_loss: 0.0053 - val_coeff_r2: 0.9947
Epoch 13/100
 - 2s - loss: 0.0096 - coeff_r2: 0.9904 - val_loss: 0.0053 - val_coeff_r2: 0.9947
Epoch 14/100
 - 2s - loss: 0.0092 - coeff_r2: 0.9908 - val_loss: 0.0045 - val_coeff_r2: 0.9955
Epoch 15/100
 - 2s - loss: 0.0090 - coeff_r2: 0.9910 - val_loss: 0.0049 - val_coeff_r2: 0.9951
Epoch 16/100
 - 2s - loss: 0.0088 - coeff_r2: 0.9912 - val_loss: 0.0043 - val_coeff_r2: 0.9957
Epoch 17/100
 - 2s - loss: 0.0084 - coeff_r2: 0.9916 - val_loss: 0.0039 - val_coeff_r2: 0.9961
Epoch 18/100
 - 2s - loss: 0.0082 - coeff_r2: 0.9918 - val_loss: 0.0041 - val_coeff_r2: 0.9959
Epoch 19/100
 - 2s - loss: 0.0086 - coeff_r2: 0.9914 - val_loss: 0.0041 - val_coeff_r2: 0.9959
Epoch 20/100
 - 2s - loss: 0.0078 - coeff_r2: 0.9922 - val_loss: 0.0036 - val_coeff_r2: 0.9964
Epoch 21/100
 - 2s - loss: 0.0081 - coeff_r2: 0.9919 - val_loss: 0.0039 - val_coeff_r2: 0.9961
Epoch 22/100
 - 2s - loss: 0.0077 - coeff_r2: 0.9923 - val_loss: 0.0034 - val_coeff_r2: 0.9966
Epoch 23/100
 - 2s - loss: 0.0076 - coeff_r2: 0.9924 - val_loss: 0.0036 - val_coeff_r2: 0.9964
Epoch 24/100
 - 2s - loss: 0.0078 - coeff_r2: 0.9922 - val_loss: 0.0035 - val_coeff_r2: 0.9965
Epoch 25/100
 - 2s - loss: 0.0078 - coeff_r2: 0.9922 - val_loss: 0.0035 - val_coeff_r2: 0.9965
Epoch 26/100
 - 2s - loss: 0.0072 - coeff_r2: 0.9928 - val_loss: 0.0031 - val_coeff_r2: 0.9969
Epoch 27/100
 - 2s - loss: 0.0073 - coeff_r2: 0.9927 - val_loss: 0.0031 - val_coeff_r2: 0.9969
Epoch 28/100
 - 2s - loss: 0.0074 - coeff_r2: 0.9926 - val_loss: 0.0032 - val_coeff_r2: 0.9968
Epoch 29/100
 - 2s - loss: 0.0072 - coeff_r2: 0.9928 - val_loss: 0.0034 - val_coeff_r2: 0.9966
Epoch 30/100
 - 2s - loss: 0.0071 - coeff_r2: 0.9929 - val_loss: 0.0031 - val_coeff_r2: 0.9969
Epoch 31/100
 - 2s - loss: 0.0071 - coeff_r2: 0.9929 - val_loss: 0.0049 - val_coeff_r2: 0.9951
Epoch 32/100
 - 2s - loss: 0.0069 - coeff_r2: 0.9931 - val_loss: 0.0028 - val_coeff_r2: 0.9972
Epoch 33/100
 - 2s - loss: 0.0068 - coeff_r2: 0.9932 - val_loss: 0.0028 - val_coeff_r2: 0.9972
Epoch 34/100
 - 2s - loss: 0.0068 - coeff_r2: 0.9932 - val_loss: 0.0033 - val_coeff_r2: 0.9967
Epoch 35/100
 - 2s - loss: 0.0067 - coeff_r2: 0.9933 - val_loss: 0.0028 - val_coeff_r2: 0.9972
Epoch 36/100
 - 2s - loss: 0.0066 - coeff_r2: 0.9934 - val_loss: 0.0028 - val_coeff_r2: 0.9972
Epoch 37/100
 - 2s - loss: 0.0067 - coeff_r2: 0.9933 - val_loss: 0.0026 - val_coeff_r2: 0.9974
Epoch 38/100
 - 2s - loss: 0.0065 - coeff_r2: 0.9935 - val_loss: 0.0030 - val_coeff_r2: 0.9970
Epoch 39/100
 - 1s - loss: 0.0064 - coeff_r2: 0.9936 - val_loss: 0.0024 - val_coeff_r2: 0.9976
Epoch 40/100
 - 2s - loss: 0.0065 - coeff_r2: 0.9935 - val_loss: 0.0027 - val_coeff_r2: 0.9973
Epoch 41/100
 - 1s - loss: 0.0063 - coeff_r2: 0.9937 - val_loss: 0.0030 - val_coeff_r2: 0.9970
Epoch 42/100
 - 2s - loss: 0.0063 - coeff_r2: 0.9937 - val_loss: 0.0023 - val_coeff_r2: 0.9977
Epoch 43/100
 - 2s - loss: 0.0065 - coeff_r2: 0.9935 - val_loss: 0.0023 - val_coeff_r2: 0.9977
Epoch 44/100
 - 2s - loss: 0.0060 - coeff_r2: 0.9940 - val_loss: 0.0023 - val_coeff_r2: 0.9977
Epoch 45/100
 - 2s - loss: 0.0060 - coeff_r2: 0.9940 - val_loss: 0.0028 - val_coeff_r2: 0.9972
Epoch 46/100
 - 2s - loss: 0.0061 - coeff_r2: 0.9939 - val_loss: 0.0025 - val_coeff_r2: 0.9975
Epoch 47/100
 - 2s - loss: 0.0061 - coeff_r2: 0.9939 - val_loss: 0.0022 - val_coeff_r2: 0.9978
Epoch 48/100
 - 2s - loss: 0.0062 - coeff_r2: 0.9938 - val_loss: 0.0031 - val_coeff_r2: 0.9969
Epoch 49/100
 - 2s - loss: 0.0063 - coeff_r2: 0.9937 - val_loss: 0.0025 - val_coeff_r2: 0.9975
Epoch 50/100
 - 2s - loss: 0.0059 - coeff_r2: 0.9941 - val_loss: 0.0022 - val_coeff_r2: 0.9978
Epoch 51/100
 - 2s - loss: 0.0059 - coeff_r2: 0.9941 - val_loss: 0.0023 - val_coeff_r2: 0.9977
Epoch 52/100
 - 1s - loss: 0.0059 - coeff_r2: 0.9941 - val_loss: 0.0022 - val_coeff_r2: 0.9978
Epoch 53/100
 - 2s - loss: 0.0058 - coeff_r2: 0.9942 - val_loss: 0.0023 - val_coeff_r2: 0.9977
Epoch 54/100
 - 1s - loss: 0.0058 - coeff_r2: 0.9942 - val_loss: 0.0022 - val_coeff_r2: 0.9978
Epoch 55/100
 - 2s - loss: 0.0059 - coeff_r2: 0.9941 - val_loss: 0.0023 - val_coeff_r2: 0.9977
Epoch 56/100
 - 2s - loss: 0.0059 - coeff_r2: 0.9941 - val_loss: 0.0024 - val_coeff_r2: 0.9976
Epoch 57/100
 - 2s - loss: 0.0061 - coeff_r2: 0.9939 - val_loss: 0.0032 - val_coeff_r2: 0.9968
Epoch 58/100
 - 2s - loss: 0.0058 - coeff_r2: 0.9942 - val_loss: 0.0020 - val_coeff_r2: 0.9980
Epoch 59/100
 - 2s - loss: 0.0057 - coeff_r2: 0.9943 - val_loss: 0.0020 - val_coeff_r2: 0.9980
Epoch 60/100
 - 2s - loss: 0.0056 - coeff_r2: 0.9944 - val_loss: 0.0027 - val_coeff_r2: 0.9973
Epoch 61/100
 - 2s - loss: 0.0058 - coeff_r2: 0.9942 - val_loss: 0.0019 - val_coeff_r2: 0.9981
Epoch 62/100
 - 2s - loss: 0.0057 - coeff_r2: 0.9943 - val_loss: 0.0022 - val_coeff_r2: 0.9978
Epoch 63/100
 - 2s - loss: 0.0056 - coeff_r2: 0.9944 - val_loss: 0.0025 - val_coeff_r2: 0.9975
Epoch 64/100
 - 2s - loss: 0.0058 - coeff_r2: 0.9942 - val_loss: 0.0020 - val_coeff_r2: 0.9980
Epoch 65/100
 - 2s - loss: 0.0056 - coeff_r2: 0.9944 - val_loss: 0.0027 - val_coeff_r2: 0.9973
Epoch 66/100
 - 2s - loss: 0.0058 - coeff_r2: 0.9942 - val_loss: 0.0019 - val_coeff_r2: 0.9981
Epoch 67/100
 - 2s - loss: 0.0054 - coeff_r2: 0.9946 - val_loss: 0.0021 - val_coeff_r2: 0.9979
Epoch 68/100
 - 2s - loss: 0.0055 - coeff_r2: 0.9945 - val_loss: 0.0020 - val_coeff_r2: 0.9980
Epoch 69/100
 - 2s - loss: 0.0054 - coeff_r2: 0.9946 - val_loss: 0.0018 - val_coeff_r2: 0.9982
Epoch 70/100
 - 2s - loss: 0.0058 - coeff_r2: 0.9942 - val_loss: 0.0022 - val_coeff_r2: 0.9978
Epoch 71/100
 - 2s - loss: 0.0056 - coeff_r2: 0.9944 - val_loss: 0.0020 - val_coeff_r2: 0.9980
Epoch 72/100
 - 2s - loss: 0.0054 - coeff_r2: 0.9946 - val_loss: 0.0019 - val_coeff_r2: 0.9981
Epoch 73/100
 - 2s - loss: 0.0052 - coeff_r2: 0.9948 - val_loss: 0.0021 - val_coeff_r2: 0.9979
Epoch 74/100
 - 2s - loss: 0.0054 - coeff_r2: 0.9946 - val_loss: 0.0020 - val_coeff_r2: 0.9980
Epoch 75/100
 - 2s - loss: 0.0053 - coeff_r2: 0.9947 - val_loss: 0.0020 - val_coeff_r2: 0.9980
Epoch 76/100
 - 2s - loss: 0.0054 - coeff_r2: 0.9946 - val_loss: 0.0029 - val_coeff_r2: 0.9971
Epoch 77/100
 - 2s - loss: 0.0054 - coeff_r2: 0.9946 - val_loss: 0.0022 - val_coeff_r2: 0.9978
Epoch 78/100
 - 2s - loss: 0.0053 - coeff_r2: 0.9947 - val_loss: 0.0028 - val_coeff_r2: 0.9972
Epoch 79/100
 - 2s - loss: 0.0054 - coeff_r2: 0.9946 - val_loss: 0.0017 - val_coeff_r2: 0.9983
Epoch 80/100
 - 2s - loss: 0.0053 - coeff_r2: 0.9947 - val_loss: 0.0018 - val_coeff_r2: 0.9982
Epoch 81/100
 - 2s - loss: 0.0053 - coeff_r2: 0.9947 - val_loss: 0.0028 - val_coeff_r2: 0.9972
Epoch 82/100
 - 2s - loss: 0.0053 - coeff_r2: 0.9947 - val_loss: 0.0021 - val_coeff_r2: 0.9979
Epoch 83/100
 - 2s - loss: 0.0052 - coeff_r2: 0.9948 - val_loss: 0.0017 - val_coeff_r2: 0.9983
Epoch 84/100
 - 2s - loss: 0.0057 - coeff_r2: 0.9943 - val_loss: 0.0018 - val_coeff_r2: 0.9982
Epoch 85/100
 - 2s - loss: 0.0052 - coeff_r2: 0.9948 - val_loss: 0.0019 - val_coeff_r2: 0.9981
Epoch 86/100
 - 2s - loss: 0.0052 - coeff_r2: 0.9948 - val_loss: 0.0016 - val_coeff_r2: 0.9984
Epoch 87/100
 - 2s - loss: 0.0051 - coeff_r2: 0.9949 - val_loss: 0.0016 - val_coeff_r2: 0.9984
Epoch 88/100
 - 2s - loss: 0.0051 - coeff_r2: 0.9949 - val_loss: 0.0016 - val_coeff_r2: 0.9984
Epoch 89/100
 - 2s - loss: 0.0052 - coeff_r2: 0.9948 - val_loss: 0.0017 - val_coeff_r2: 0.9983
Epoch 90/100
 - 2s - loss: 0.0053 - coeff_r2: 0.9947 - val_loss: 0.0018 - val_coeff_r2: 0.9982
Epoch 91/100
 - 2s - loss: 0.0052 - coeff_r2: 0.9948 - val_loss: 0.0016 - val_coeff_r2: 0.9984
Epoch 92/100
 - 2s - loss: 0.0053 - coeff_r2: 0.9947 - val_loss: 0.0017 - val_coeff_r2: 0.9983
Epoch 93/100
 - 2s - loss: 0.0052 - coeff_r2: 0.9948 - val_loss: 0.0019 - val_coeff_r2: 0.9981
Epoch 94/100
 - 2s - loss: 0.0050 - coeff_r2: 0.9950 - val_loss: 0.0024 - val_coeff_r2: 0.9976
Epoch 95/100
 - 2s - loss: 0.0056 - coeff_r2: 0.9944 - val_loss: 0.0016 - val_coeff_r2: 0.9984
Epoch 96/100
 - 2s - loss: 0.0049 - coeff_r2: 0.9951 - val_loss: 0.0022 - val_coeff_r2: 0.9978
Epoch 97/100
 - 2s - loss: 0.0050 - coeff_r2: 0.9950 - val_loss: 0.0015 - val_coeff_r2: 0.9985
Epoch 98/100
 - 2s - loss: 0.0050 - coeff_r2: 0.9950 - val_loss: 0.0016 - val_coeff_r2: 0.9984
Epoch 99/100
 - 2s - loss: 0.0050 - coeff_r2: 0.9950 - val_loss: 0.0017 - val_coeff_r2: 0.9983
Epoch 100/100
 - 2s - loss: 0.0050 - coeff_r2: 0.9950 - val_loss: 0.0014 - val_coeff_r2: 0.9986

Training loss plot


In [15]:
fig = plt.figure()
plt.semilogy(history.history['loss'])
if vsplit:
    plt.semilogy(history.history['val_loss'])
plt.title(loss_type)
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper right')
plt.show()


Inference test

prepare frontend for plotting


In [8]:
#@title import plotly
import plotly.plotly as py
import numpy as np
from plotly.offline import init_notebook_mode, iplot
# from plotly.graph_objs import Contours, Histogram2dContour, Marker, Scatter
import plotly.graph_objs as go

def configure_plotly_browser_state():
  import IPython
  display(IPython.core.display.HTML('''
        <script src="/static/components/requirejs/require.js"></script>
        <script>
          requirejs.config({
            paths: {
              base: '/static/base',
              plotly: 'https://cdn.plot.ly/plotly-1.5.1.min.js?noext',
            },
          });
        </script>
        '''))

prepare data for plotting

GPU data prepare


In [17]:
from sklearn.metrics import r2_score
# model.load_weights("./tmp/weights.best.cntk.hdf5")

x_test_df = pd.DataFrame(in_scaler.inverse_transform(x_test),columns=input_features)
y_test_df = pd.DataFrame(out_scaler.inverse_transform(y_test),columns=labels)

predict_val = model.predict(x_test,batch_size=1024*8)
predict_df = pd.DataFrame(out_scaler.inverse_transform(predict_val), columns=labels)

test_data=pd.concat([x_test_df,y_test_df],axis=1)
pred_data=pd.concat([x_test_df,predict_df],axis=1)

!rm sim_check.h5
test_data.to_hdf('sim_check.h5',key='test')
pred_data.to_hdf('sim_check.h5',key='pred')

df_test=pd.read_hdf('sim_check.h5',key='test')
df_pred=pd.read_hdf('sim_check.h5',key='pred')

zeta_level=list(set(df_test['zeta']))
zeta_level.sort()


res_sum=pd.DataFrame()
r2s=[]
r2s_i=[]

names=[]
maxs_0=[]
maxs_9=[]

for r2,name in zip(r2_score(df_test,df_pred,multioutput='raw_values'),df_test.columns):
  names.append(name)
  r2s.append(r2)
  maxs_0.append(df_test[df_test['zeta']==zeta_level[0]][name].max())
  maxs_9.append(df_test[df_test['zeta']==zeta_level[8]][name].max())
  for i in zeta_level:
    r2s_i.append(r2_score(df_pred[df_pred['zeta']==i][name],
                          df_test[df_test['zeta']==i][name]))

res_sum['name']=names
# res_sum['max_0']=maxs_0
# res_sum['max_9']=maxs_9
res_sum['z_scale']=[m_9/(m_0+1e-20) for m_9,m_0 in zip(maxs_9,maxs_0)]
# res_sum['r2']=r2s


tmp=np.asarray(r2s_i).reshape(-1,10)
for idx,z in enumerate(zeta_level):
  res_sum['r2s_'+str(z)]=tmp[:,idx]

res_sum[3:]


Out[17]:
name z_scale r2s_0.0 r2s_0.11 r2s_0.22 r2s_0.33 r2s_0.44 r2s_0.55 r2s_0.66 r2s_0.77 r2s_0.88 r2s_0.99
3 CO 0.18086 0.998438 0.999448 0.999486 0.999608 0.999614 0.999224 0.999255 0.999058 0.998379 0.990823

In [16]:
scaler_std2=res_sum[3:]
scaler_std2


Out[16]:
name z_scale r2s_0.0 r2s_0.11 r2s_0.22 r2s_0.33 r2s_0.44 r2s_0.55 r2s_0.66 r2s_0.77 r2s_0.88 r2s_0.99
3 CO 0.183845 0.999775 0.999817 0.999823 0.999764 0.99972 0.999659 0.999489 0.999323 0.997707 0.889935

In [18]:
scaler_cbrt_std=res_sum[3:]
scaler_cbrt_std


Out[18]:
name z_scale r2s_0.0 r2s_0.11 r2s_0.22 r2s_0.33 r2s_0.44 r2s_0.55 r2s_0.66 r2s_0.77 r2s_0.88 r2s_0.99
3 CO 0.18086 0.998438 0.999448 0.999486 0.999608 0.999614 0.999224 0.999255 0.999058 0.998379 0.990823

In [40]:
scaler_tb=pd.concat([scaler_std2,scaler_cbrt_std],axis=0)
scaler_tb=scaler_tb.drop(['z_scale','name'],axis=1)
scaler_tb


Out[40]:
r2s_0.0 r2s_0.11 r2s_0.22 r2s_0.33 r2s_0.44 r2s_0.55 r2s_0.66 r2s_0.77 r2s_0.88 r2s_0.99
3 0.999775 0.999817 0.999823 0.999764 0.999720 0.999659 0.999489 0.999323 0.997707 0.889935
3 0.998438 0.999448 0.999486 0.999608 0.999614 0.999224 0.999255 0.999058 0.998379 0.990823

In [41]:
with open('tb.txt','wb') as f:
    scaler_tb.to_latex('tb.txt')

interactive plot


In [14]:
#@title Default title text
# species = np.random.choice(labels)
species = 'T' #@param {type:"string"}
z_level =  0 #@param {type:"integer"}

# configure_plotly_browser_state()
# init_notebook_mode(connected=False)

from sklearn.metrics import r2_score


df_t=df_test[df_test['zeta']==zeta_level[z_level]].sample(frac=1)
# df_p=df_pred.loc[df_pred['zeta']==zeta_level[1]].sample(frac=0.1)
df_p=df_pred.loc[df_t.index]
error=df_p[species]-df_t[species]
r2=round(r2_score(df_p[species],df_t[species]),4)

print(species,'r2:',r2,'max:',df_t[species].max())

fig_db = {
    'data': [       
        {'name':'test data from table',
         'x': df_t['f'],
         'y': df_t['pv'],
         'z': df_t[species],
         'type':'scatter3d', 
        'mode': 'markers',
          'marker':{
              'size':1
          }
        },
        {'name':'prediction from neural networks',
         'x': df_p['f'],
         'y': df_p['pv'],
         'z': df_p[species],
         'type':'scatter3d', 
        'mode': 'markers',
          'marker':{
              'size':1
          },
        },
        {'name':'error in difference',
         'x': df_p['f'],
         'y': df_p['pv'],
         'z': error,
         'type':'scatter3d', 
         'mode': 'markers',
          'marker':{
              'size':1
          },
         }       
    ],
    'layout': {
        'scene':{
            'xaxis': {'title':'mixture fraction'},
            'yaxis': {'title':'progress variable'},
            'zaxis': {'title': species+'_r2:'+str(r2)}
                 }
    }
}
# iplot(fig_db, filename='multiple-scatter')
iplot(fig_db)


T r2: 0.9717 max: 2151.56

In [56]:
model.save('trained_fgm_nn.h5')

In [99]:
model.save('trained_fgm_nn.h5')
%run -i k2tf.py --input_model='trained_fgm_nn.h5' --output_model='exported/fgm.pb'


---------------------------------------------------------------------------
DuplicateFlagError                        Traceback (most recent call last)
~/repos/combustionML/FPV_ANN/k2tf.py in <module>
     25 FLAGS = flags.FLAGS
     26 
---> 27 flags.DEFINE_string('input_model', None, 'Path to the input model.')
     28 flags.DEFINE_string('input_model_json', None, 'Path to the input model '
     29                                               'architecture in json format.')

~/anaconda3/envs/my_dev/lib/python3.6/site-packages/absl/flags/_defines.py in DEFINE_string(name, default, help, flag_values, **args)
    239   parser = _argument_parser.ArgumentParser()
    240   serializer = _argument_parser.ArgumentSerializer()
--> 241   DEFINE(parser, name, default, help, flag_values, serializer, **args)
    242 
    243 

~/anaconda3/envs/my_dev/lib/python3.6/site-packages/absl/flags/_defines.py in DEFINE(parser, name, default, help, flag_values, serializer, module_name, **args)
     80   """
     81   DEFINE_flag(_flag.Flag(parser, serializer, name, default, help, **args),
---> 82               flag_values, module_name)
     83 
     84 

~/anaconda3/envs/my_dev/lib/python3.6/site-packages/absl/flags/_defines.py in DEFINE_flag(flag, flag_values, module_name)
    102   # Copying the reference to flag_values prevents pychecker warnings.
    103   fv = flag_values
--> 104   fv[flag.name] = flag
    105   # Tell flag_values who's defining the flag.
    106   if module_name:

~/anaconda3/envs/my_dev/lib/python3.6/site-packages/absl/flags/_flagvalues.py in __setitem__(self, name, flag)
    425         # module is simply being imported a subsequent time.
    426         return
--> 427       raise _exceptions.DuplicateFlagError.from_flag(name, self)
    428     short_name = flag.short_name
    429     # If a new flag overrides an old one, we need to cleanup the old flag's

DuplicateFlagError: The flag 'input_model' is defined twice. First from k2tf.py, Second from k2tf.py.  Description from first occurrence: Path to the input model.
<Figure size 432x288 with 0 Axes>

Stutdent networ

The student network is trained on the synsetic data generated from the full teacher network. It is mean to simplify the final model used in production.


In [114]:
from keras.models import Model
from keras.layers import Dense, Input
from keras.callbacks import ModelCheckpoint


n_neuron = 50
# %%
print('set up student network')
# ANN parameters
dim_input = x_train.shape[1]
dim_label = y_train.shape[1]

batch_norm = False

# This returns a tensor
inputs = Input(shape=(dim_input,),name='input_1')

# a layer instance is callable on a tensor, and returns a tensor
x = Dense(n_neuron, activation='relu',name='l1')(inputs)
x = Dense(n_neuron, activation='relu',name='l2')(x)
x = Dropout(0.1)(x)
predictions = Dense(dim_label, activation='linear', name='output_1')(x)

student_model = Model(inputs=inputs, outputs=predictions)
student_model.summary()


set up student network
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_1 (InputLayer)         (None, 3)                 0         
_________________________________________________________________
l1 (Dense)                   (None, 50)                200       
_________________________________________________________________
l2 (Dense)                   (None, 50)                2550      
_________________________________________________________________
dropout_4 (Dropout)          (None, 50)                0         
_________________________________________________________________
output_1 (Dense)             (None, 2)                 102       
=================================================================
Total params: 2,852
Trainable params: 2,852
Non-trainable params: 0
_________________________________________________________________

In [115]:
batch_size = 1024*32
epochs = 60
vsplit = 0.1

loss_type='mse'

adam_op = optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999,epsilon=1e-8, decay=0.0, amsgrad=True)

student_model.compile(loss=loss_type, optimizer=adam_op, metrics=[coeff_r2])
# model.compile(loss=cubic_loss, optimizer=adam_op, metrics=['accuracy'])

# checkpoint (save the best model based validate loss)
!mkdir ./tmp
filepath = "./tmp/student_weights.best.cntk.hdf5"

checkpoint = ModelCheckpoint(filepath,
                             monitor='val_loss',
                             verbose=1,
                             save_best_only=True,
                             mode='min',
                             period=20)

# callbacks_list = [checkpoint]
callbacks_list = [lrate]

x_train_teacher = x_train
y_train_teacher = model.predict(x_train, batch_size=1024*8)
# fit the model
history = student_model.fit(
    x_train_teacher, y_train_teacher,
    epochs=epochs,
    batch_size=batch_size,
    validation_split=vsplit,
    verbose=2,
#     callbacks=callbacks_list,
    shuffle=True)


mkdir: cannot create directory ‘./tmp’: File exists
Train on 1004379 samples, validate on 111598 samples
Epoch 1/60
 - 1s - loss: 0.9016 - coeff_r2: 0.0952 - val_loss: 0.7900 - val_coeff_r2: 0.2183
Epoch 2/60
 - 1s - loss: 0.6610 - coeff_r2: 0.3373 - val_loss: 0.5672 - val_coeff_r2: 0.4392
Epoch 3/60
 - 1s - loss: 0.5310 - coeff_r2: 0.4677 - val_loss: 0.5122 - val_coeff_r2: 0.4936
Epoch 4/60
 - 1s - loss: 0.4893 - coeff_r2: 0.5094 - val_loss: 0.4779 - val_coeff_r2: 0.5277
Epoch 5/60
 - 1s - loss: 0.4603 - coeff_r2: 0.5386 - val_loss: 0.4548 - val_coeff_r2: 0.5506
Epoch 6/60
 - 1s - loss: 0.4413 - coeff_r2: 0.5575 - val_loss: 0.4401 - val_coeff_r2: 0.5651
Epoch 7/60
 - 1s - loss: 0.4297 - coeff_r2: 0.5693 - val_loss: 0.4295 - val_coeff_r2: 0.5756
Epoch 8/60
 - 1s - loss: 0.4196 - coeff_r2: 0.5793 - val_loss: 0.4199 - val_coeff_r2: 0.5850
Epoch 9/60
 - 1s - loss: 0.4101 - coeff_r2: 0.5892 - val_loss: 0.4103 - val_coeff_r2: 0.5946
Epoch 10/60
 - 1s - loss: 0.4007 - coeff_r2: 0.5984 - val_loss: 0.4007 - val_coeff_r2: 0.6041
Epoch 11/60
 - 1s - loss: 0.3908 - coeff_r2: 0.6086 - val_loss: 0.3907 - val_coeff_r2: 0.6140
Epoch 12/60
 - 1s - loss: 0.3809 - coeff_r2: 0.6182 - val_loss: 0.3797 - val_coeff_r2: 0.6248
Epoch 13/60
 - 1s - loss: 0.3707 - coeff_r2: 0.6284 - val_loss: 0.3691 - val_coeff_r2: 0.6353
Epoch 14/60
 - 1s - loss: 0.3602 - coeff_r2: 0.6390 - val_loss: 0.3593 - val_coeff_r2: 0.6450
Epoch 15/60
 - 1s - loss: 0.3509 - coeff_r2: 0.6483 - val_loss: 0.3499 - val_coeff_r2: 0.6543
Epoch 16/60
 - 1s - loss: 0.3420 - coeff_r2: 0.6574 - val_loss: 0.3409 - val_coeff_r2: 0.6632
Epoch 17/60
 - 1s - loss: 0.3330 - coeff_r2: 0.6661 - val_loss: 0.3320 - val_coeff_r2: 0.6720
Epoch 18/60
 - 1s - loss: 0.3240 - coeff_r2: 0.6755 - val_loss: 0.3234 - val_coeff_r2: 0.6806
Epoch 19/60
 - 1s - loss: 0.3154 - coeff_r2: 0.6841 - val_loss: 0.3150 - val_coeff_r2: 0.6889
Epoch 20/60
 - 1s - loss: 0.3075 - coeff_r2: 0.6919 - val_loss: 0.3066 - val_coeff_r2: 0.6972
Epoch 21/60
 - 1s - loss: 0.2994 - coeff_r2: 0.6999 - val_loss: 0.2984 - val_coeff_r2: 0.7053
Epoch 22/60
 - 1s - loss: 0.2911 - coeff_r2: 0.7083 - val_loss: 0.2901 - val_coeff_r2: 0.7135
Epoch 23/60
 - 1s - loss: 0.2840 - coeff_r2: 0.7154 - val_loss: 0.2823 - val_coeff_r2: 0.7213
Epoch 24/60
 - 1s - loss: 0.2762 - coeff_r2: 0.7232 - val_loss: 0.2747 - val_coeff_r2: 0.7287
Epoch 25/60
 - 1s - loss: 0.2696 - coeff_r2: 0.7298 - val_loss: 0.2672 - val_coeff_r2: 0.7362
Epoch 26/60
 - 1s - loss: 0.2622 - coeff_r2: 0.7373 - val_loss: 0.2595 - val_coeff_r2: 0.7437
Epoch 27/60
 - 1s - loss: 0.2550 - coeff_r2: 0.7445 - val_loss: 0.2515 - val_coeff_r2: 0.7517
Epoch 28/60
 - 1s - loss: 0.2474 - coeff_r2: 0.7522 - val_loss: 0.2438 - val_coeff_r2: 0.7593
Epoch 29/60
 - 1s - loss: 0.2406 - coeff_r2: 0.7590 - val_loss: 0.2362 - val_coeff_r2: 0.7668
Epoch 30/60
 - 1s - loss: 0.2329 - coeff_r2: 0.7669 - val_loss: 0.2288 - val_coeff_r2: 0.7741
Epoch 31/60
 - 1s - loss: 0.2268 - coeff_r2: 0.7728 - val_loss: 0.2217 - val_coeff_r2: 0.7811
Epoch 32/60
 - 1s - loss: 0.2200 - coeff_r2: 0.7798 - val_loss: 0.2149 - val_coeff_r2: 0.7879
Epoch 33/60
 - 1s - loss: 0.2136 - coeff_r2: 0.7859 - val_loss: 0.2080 - val_coeff_r2: 0.7947
Epoch 34/60
 - 1s - loss: 0.2076 - coeff_r2: 0.7919 - val_loss: 0.2010 - val_coeff_r2: 0.8016
Epoch 35/60
 - 1s - loss: 0.2013 - coeff_r2: 0.7984 - val_loss: 0.1940 - val_coeff_r2: 0.8085
Epoch 36/60
 - 1s - loss: 0.1946 - coeff_r2: 0.8051 - val_loss: 0.1876 - val_coeff_r2: 0.8149
Epoch 37/60
 - 1s - loss: 0.1888 - coeff_r2: 0.8110 - val_loss: 0.1821 - val_coeff_r2: 0.8203
Epoch 38/60
 - 1s - loss: 0.1833 - coeff_r2: 0.8164 - val_loss: 0.1761 - val_coeff_r2: 0.8263
Epoch 39/60
 - 1s - loss: 0.1776 - coeff_r2: 0.8221 - val_loss: 0.1706 - val_coeff_r2: 0.8317
Epoch 40/60
 - 1s - loss: 0.1723 - coeff_r2: 0.8274 - val_loss: 0.1654 - val_coeff_r2: 0.8368
Epoch 41/60
 - 1s - loss: 0.1681 - coeff_r2: 0.8318 - val_loss: 0.1601 - val_coeff_r2: 0.8421
Epoch 42/60
 - 1s - loss: 0.1633 - coeff_r2: 0.8365 - val_loss: 0.1553 - val_coeff_r2: 0.8468
Epoch 43/60
 - 0s - loss: 0.1590 - coeff_r2: 0.8407 - val_loss: 0.1508 - val_coeff_r2: 0.8512
Epoch 44/60
 - 1s - loss: 0.1543 - coeff_r2: 0.8455 - val_loss: 0.1462 - val_coeff_r2: 0.8557
Epoch 45/60
 - 1s - loss: 0.1505 - coeff_r2: 0.8493 - val_loss: 0.1420 - val_coeff_r2: 0.8599
Epoch 46/60
 - 1s - loss: 0.1461 - coeff_r2: 0.8537 - val_loss: 0.1380 - val_coeff_r2: 0.8638
Epoch 47/60
 - 1s - loss: 0.1428 - coeff_r2: 0.8570 - val_loss: 0.1340 - val_coeff_r2: 0.8678
Epoch 48/60
 - 1s - loss: 0.1391 - coeff_r2: 0.8607 - val_loss: 0.1302 - val_coeff_r2: 0.8715
Epoch 49/60
 - 1s - loss: 0.1352 - coeff_r2: 0.8645 - val_loss: 0.1263 - val_coeff_r2: 0.8754
Epoch 50/60
 - 1s - loss: 0.1329 - coeff_r2: 0.8669 - val_loss: 0.1230 - val_coeff_r2: 0.8787
Epoch 51/60
 - 1s - loss: 0.1295 - coeff_r2: 0.8702 - val_loss: 0.1200 - val_coeff_r2: 0.8817
Epoch 52/60
 - 1s - loss: 0.1265 - coeff_r2: 0.8734 - val_loss: 0.1166 - val_coeff_r2: 0.8850
Epoch 53/60
 - 1s - loss: 0.1244 - coeff_r2: 0.8755 - val_loss: 0.1136 - val_coeff_r2: 0.8880
Epoch 54/60
 - 1s - loss: 0.1207 - coeff_r2: 0.8791 - val_loss: 0.1106 - val_coeff_r2: 0.8909
Epoch 55/60
 - 1s - loss: 0.1180 - coeff_r2: 0.8818 - val_loss: 0.1078 - val_coeff_r2: 0.8937
Epoch 56/60
 - 1s - loss: 0.1152 - coeff_r2: 0.8845 - val_loss: 0.1046 - val_coeff_r2: 0.8968
Epoch 57/60
 - 1s - loss: 0.1126 - coeff_r2: 0.8872 - val_loss: 0.1029 - val_coeff_r2: 0.8985
Epoch 58/60
 - 1s - loss: 0.1101 - coeff_r2: 0.8897 - val_loss: 0.1000 - val_coeff_r2: 0.9014
Epoch 59/60
 - 1s - loss: 0.1083 - coeff_r2: 0.8916 - val_loss: 0.0966 - val_coeff_r2: 0.9047
Epoch 60/60
 - 1s - loss: 0.1056 - coeff_r2: 0.8942 - val_loss: 0.0943 - val_coeff_r2: 0.9070

In [13]:
from sklearn.metrics import r2_score

x_test_df = pd.DataFrame(in_scaler.inverse_transform(x_test),columns=input_features)

predict_val = student_model.predict(x_test,batch_size=1024*8)
predict_df = pd.DataFrame(out_scaler.inverse_transform(predict_val), columns=labels)

pred_data=pd.concat([x_test_df,predict_df],axis=1)

!rm sim_check.h5
pred_data.to_hdf('sim_check.h5',key='pred')
df_pred=pd.read_hdf('sim_check.h5',key='pred')

zeta_level=list(set(df_test['zeta']))
zeta_level.sort()


res_sum=pd.DataFrame()
r2s=[]
r2s_i=[]

names=[]
maxs_0=[]
maxs_9=[]

for r2,name in zip(r2_score(df_test,df_pred,multioutput='raw_values'),df_test.columns):
  names.append(name)
  r2s.append(r2)
  maxs_0.append(df_test[df_test['zeta']==zeta_level[0]][name].max())
  maxs_9.append(df_test[df_test['zeta']==zeta_level[8]][name].max())
  for i in zeta_level:
    r2s_i.append(r2_score(df_pred[df_pred['zeta']==i][name],
                          df_test[df_test['zeta']==i][name]))

res_sum['name']=names
# res_sum['max_0']=maxs_0
# res_sum['max_9']=maxs_9
res_sum['z_scale']=[m_9/(m_0+1e-20) for m_9,m_0 in zip(maxs_9,maxs_0)]
# res_sum['r2']=r2s


tmp=np.asarray(r2s_i).reshape(-1,10)
for idx,z in enumerate(zeta_level):
  res_sum['r2s_'+str(z)]=tmp[:,idx]

res_sum[3:]


Out[13]:
name z_scale r2s_0.0 r2s_0.11 r2s_0.22 r2s_0.33 r2s_0.44 r2s_0.55 r2s_0.66 r2s_0.77 r2s_0.88 r2s_0.99
3 CH4 1.008505 0.971500 0.987782 0.993755 0.992194 0.988337 0.990516 0.992561 0.995902 0.991570 0.995409
4 O2 1.000000 0.987625 0.991136 0.993322 0.995098 0.996573 0.998220 0.998468 0.997902 0.994167 0.988721
5 H2O 0.222239 0.983944 0.993169 0.993735 0.995091 0.996454 0.996701 0.996895 0.988193 0.943449 0.446626
6 CO 0.181322 0.952372 0.969151 0.984470 0.990651 0.994517 0.992602 0.991570 0.984378 0.939984 0.384443
7 CO2 0.143767 0.857708 0.911901 0.963990 0.963058 0.967662 0.966969 0.952636 0.938890 0.944349 0.577656
8 T 0.295670 0.971685 0.988545 0.993414 0.993663 0.994816 0.995572 0.995843 0.992465 0.969658 0.493271
9 PVs 0.030422 -1.259126 0.552150 0.756739 0.702127 0.595810 0.500029 0.427537 0.359685 -0.169924 -0.275544
10 psi 0.461212 0.964584 0.983095 0.991711 0.991252 0.992692 0.992830 0.994645 0.991764 0.973992 0.560999
11 mu 0.462740 0.972904 0.986585 0.993010 0.994129 0.995148 0.995341 0.995629 0.992655 0.974620 0.550289
12 alpha 1.000000 0.948433 0.965166 0.976732 0.973191 0.980955 0.986462 0.988344 0.984737 0.974464 0.941372

save student network weights


In [71]:
import h5py
!rm student_model_weights.h5
student_model.save('student_model_weights.h5')
f = h5py.File('student_model_weights.h5','r')
dset=f['model_weights']
list(dset)


Out[71]:
['dense_5', 'dense_6', 'dropout_3', 'input_1', 'output_1']

In [92]:
l1_w=dset['dense_5']['dense_5']['kernel:0'][:]
l1_b=dset['dense_5']['dense_5']['bias:0'][:]
l1_c=np.vstack([l1_w,l1_b])
l1_c=pd.Series(list(l1_c)).to_json()

l2_w=dset['dense_6']['dense_6']['kernel:0'][:]
l2_b=dset['dense_6']['dense_6']['bias:0'][:]
l2_c=np.vstack([l2_w,l2_b])
l2_c=pd.Series(list(l2_c)).to_json()

l3_w=dset['output_1']['output_1_2']['kernel:0'][:]
l3_b=dset['output_1']['output_1_2']['bias:0'][:]
l3_c=np.vstack([l3_w,l3_b])
l3_c=pd.Series(list(l3_c)).to_json()

In [94]:
!rm data.json
print("{",file=open('data.json','w'))
print('"l1":',l1_c,file=open('data.json','a'))
print(',"l2":',l2_c,file=open('data.json','a'))
print(',"output":',l3_c,file=open('data.json','a'))
print("}",file=open('data.json','a'))

In [113]:
test_id=888
print(x_test[test_id])
print(student_model.predict(x_test[test_id].reshape(-1,3)))
print(y_test[test_id])


[0.37  0.124 0.55 ]
[[-0.5438185  -0.38415462]]
[-0.78356046 -0.33965215]

In [90]:
l1_b


Out[90]:
array([ 0.2093982 , -0.05028194,  0.44980595,  0.1120008 , -0.17164879],
      dtype=float32)

In [89]:
np.vstack([l1,l1_b])


Out[89]:
array([[-1.505183  , -0.48669735, -1.0311114 , -0.83720607, -2.1831415 ],
       [-0.49262458,  0.5467909 ,  0.41370386,  0.7635428 ,  0.3367838 ],
       [ 1.2676779 ,  0.44149137, -0.19589986,  0.00307888,  0.67396855],
       [ 0.2093982 , -0.05028194,  0.44980595,  0.1120008 , -0.17164879]],
      dtype=float32)

In [110]:
student_model.predict(np.asarray([0.5,0.1,0.1]).reshape(-1,3))


Out[110]:
array([[-0.47643796, -0.39353642]], dtype=float32)

In [106]:
student_model.save_weights('student_weights.h5')

In [ ]: