In [1]:
from IPython.core.debugger import Tracer
#Tracer()()
import os, sys, time
### prevent the dying jupyter notebook
stdout = sys.stdout
#sys.stdout = sys.__stdout__ # did not work to restoure print -> console
#sys.stdout = open('keras_output.txt', 'a+')
#sys.stdout = stdout
import utils
import importlib
importlib.reload(utils)
#Allow relative imports to directories above cwd/
sys.path.insert(1, os.path.join(sys.path[0], '..'))
#import modules
from utils import *
%matplotlib inline
np.random.seed(42)
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import mean_squared_error
%pwd #Verify we are in the right directory
!uname -a
Out[1]:
Out[1]:
In [15]:
#Create references to important directories we will use over and over
current_dir = os.getcwd()
dataname = 'rain-s'
sample = False
if sample:
dpath = "./data/rain-s/"
else:
dpath = "./data/rain/"
seq_len = 19
In [25]:
import twModel
importlib.reload(twModel)
Out[25]:
In [ ]:
from keras.models import load_model
model = load_model('xxx.h5')
In [9]:
#model = twModel.RNNModel()
model = twModel.BidiRNNModel()
In [ ]:
In [11]:
model.init_model(rnn_size=256,
dense_size=512,
#optimizer='Adam',
init='uniform',
activation='relu',
layers=[False, False],
dropout_rate=0)
In [13]:
from IPython.display import SVG
from keras.utils.visualize_util import model_to_dot
SVG(model_to_dot(model.model).create(prog='dot', format='svg'))
Out[13]:
In [52]:
X_train, y_train, X_valid, y_valid = twModel.data(sample)
X_train.shape, y_train.shape, X_valid.shape, y_valid.shape
Out[52]:
In [53]:
%%time
#%%capture output
sys.stdout = open('keras_output.txt', 'a+')
m_, hist = model.train(X_train, y_train, epochs=35, batch_size=128, verbose=1, validation_data=(X_valid, y_valid))
sys.stdout = stdout
hist
model.save()
In [ ]:
In [26]:
#model = twModel.RNNModel()
model = twModel.BidiRNNModel()
model.load()
In [27]:
model.model.summary()
In [28]:
X_test = twModel.data_test()
X_test.shape
Out[28]:
In [39]:
X_test[:3,0,0]
X_test[:,0,0].max()
Out[39]:
Out[39]:
In [29]:
%%time
preds = model.model.predict(X_test, batch_size=128, verbose=2)
In [48]:
ids = np.load('test/obs_ids_test.npy')
ids.shape
Out[48]:
In [49]:
df = pd.DataFrame(data=preds, index=ids[:], columns=['Expected'])
In [50]:
df.info()
df.to_csv('df.csv', index_label='Id')
In [ ]:
%mkdir -p $dpath
In [ ]:
# run everything on a 10% data subset
# 13765202 line in train
!head -n 1000000 ./data/rain/train.csv > ./data/rain-s/train.csv
!head -n 800000 ./data/rain/test.csv > ./data/rain-s/test.csv
In [ ]:
# manual downlaod, kg did not work
!python data_preprocessing.py
In [ ]:
obs_ids_train = np.load('train/obs_ids_train_cv0.npy')
train = np.load(dpath+'processed_train.npy')
In [ ]:
train.shape, obs_ids_train.shape
train[0]
obs_ids_train[0]
In [ ]:
COLUMNS = ['Id','minutes_past', 'radardist_km', 'Ref', 'Ref_5x5_10th',
'Ref_5x5_50th', 'Ref_5x5_90th', 'RefComposite',
'RefComposite_5x5_10th', 'RefComposite_5x5_50th',
'RefComposite_5x5_90th', 'RhoHV', 'RhoHV_5x5_10th',
'RhoHV_5x5_50th', 'RhoHV_5x5_90th', 'Zdr', 'Zdr_5x5_10th',
'Zdr_5x5_50th', 'Zdr_5x5_90th', 'Kdp', 'Kdp_5x5_10th',
'Kdp_5x5_50th', 'Kdp_5x5_90th', 'Expected']
data_pd = pd.DataFrame(data=train[:,0:], columns=COLUMNS)
data_pd.info()
data_pd.head(2)
In [ ]:
data_pd.groupby('Id').count().head(3)
data_pd.groupby('Id').mean().head(3)
In [ ]:
# generate sequence data
!python data_augmentation_train.py
In [ ]:
# generate sequence data
!python data_augmentation_valid.py
In [ ]:
X_train = np.load('train/data_train_augmented_cv0_t19_rand0.npy')
y_train = np.load('train/data_train_expected_cv0.npy')
X_train.shape, y_train.shape
X_train[0, 0], y_train[0]
In [45]:
ids = np.load('valid/obs_ids_valid_cv0-s.npy')
X_valid = np.load('valid/data_valid_augmented_cv0_t19_rand0-s.npy')
y_valid = np.load('valid/data_valid_expected_cv0-s.npy')
X_valid.shape, y_valid.shape
X_valid[0, 0], y_valid[0]
Out[45]:
Out[45]:
In [46]:
ids.shape
Out[46]:
In [ ]:
plot2(X_train[0], y_train)
In [ ]:
class AbstractModel():
def __init__(self):
self.modelPath = dpath+self.get_classname()+'.h5'
self.checkpoint = ModelCheckpoint(filepath= dpath + 'checkpoint-{epoch:02d}-{loss:.3f}-{val_loss:.3f}.hdf5')
@classmethod #classmethod always gets class as parameter
def get_classname(cls):
return cls.__name__
def save(self):
self.model.save(self.modelPath)
print("model saved to: ", self.modelPath)
def load(self):
self.model = load_model(modelPath)
print("model loaded from: ", self.modelPath)
def train(self, X_train, y_train, epochs=1, batch_size=64, val_split=0.05, verbose=1):
global_start_time = time.time()
try:
history = self.model.fit(
X_train, y_train,
batch_size=batch_size,
nb_epoch=epochs,
validation_split=val_split,
verbose=verbose,
#callbacks=[self.checkpoint, self.csvLogger]) #BUG: csv logger not rerun capable
callbacks=[self.checkpoint])
except KeyboardInterrupt:
print('Training duration (s) : ', time.time() - global_start_time)
return self.model, history
print('Training duration (s) : ', time.time() - global_start_time)
return self.model, history
In [ ]:
class BaselineModel(AbstractModel):
def __init__(self):
super(BaselineModel, self).__init__()
self.model = None
def __call__(self, neurons=12, optimizer='RMSprop', tw_init='uniform', activation='relu', dropout_rate=0, weight_constraint=2, layers=[False]):
""" returns the model, e.g. for GridSearch"""
self.model = None # make sure, that instance is pickable
return self.create_model(neurons, optimizer, tw_init, activation, dropout_rate, weight_constraint, layers)
def init_model(self, neurons=12, optimizer='RMSprop', tw_init='uniform', activation='relu', dropout_rate=0, weight_constraint=2, layers=[False]):
self.model = self.create_model(neurons, optimizer, tw_init, activation, dropout_rate, weight_constraint, layers)
self.model.summary()
def create_model(self, neurons=12, optimizer='RMSprop', tw_init='uniform', activation='relu', dropout_rate=0, weight_constraint=2, layers=[False]):
''' creates functional KERAS model
parameters are passed from GridSearch, always provide default to call the model without GridSearch
'''
inputs = Input(shape=(40,))
x = Dense(neurons, init=tw_init, activation=activation, W_constraint=maxnorm(weight_constraint))(inputs)
x = Dropout(dropout_rate)(x)
if layers[0]:
x = Dense(neurons, init=tw_init, activation=activation, W_constraint=maxnorm(weight_constraint))(x)
x = Dropout(dropout_rate)(x)
if layers[1]:
x = Dense(neurons, init=tw_init, activation=activation, W_constraint=maxnorm(weight_constraint))(x)
x = Dropout(dropout_rate)(x)
preds = Dense(1, activation='linear')(x)
model = Model(input=inputs, output=preds)
model.compile(optimizer=optimizer, loss='mse', metrics=['mse'])
return model
m = BaselineModel([seq_len, seq_len+1, (seq_len+1)*2, 1]) #bias added
m.classtype
m.modelPath
m.model.summary()
In [ ]:
class DenseModel(BaseModel):
"""take the seq_len as feature input and predict one output"""
classtype = 'dense' # class variable
def __init__(self, layers):
super(DenseModel, self).__init__(layers)
self.create(layers)
def create(self, layers):
if verbose: print('> Create Model', self.get_classname(), ' type: ', self.classtype)
start = time.time()
inputs = Input(shape=(layers[0],))
x = Dense(layers[1], activation='relu')(inputs)
x = Dropout(0.25)(x)
x = Dense(layers[2], activation='relu')(x)
x = Dropout(0.25)(x)
preds = Dense(layers[3], activation='linear')(x)
self.model = Model(input=inputs, output=preds)
self.model.compile(optimizer='adam', loss='mse', metrics=['mse', 'mape'])
if verbose: print("Compilation Time : ", time.time() - start)
m = DenseModel([seq_len, 500, 250, 1]) #bias added
m.model.summary()
In [ ]:
%%time
#%%capture output
#sys.stdout = open('keras_output.txt', 'a+')
#m.compile()
model, hist = m.train(X_train, y_train, epochs=1, verbose=1)
#sys.stdout = stdout
hist
#m.save()
In [ ]:
class BaseLTSM(BaseModel):
classtype = 'ltsm' # class variable
classname = 'BaseLTSM'
def __init__(self, layers):
super(BaseLTSM, self).__init__(layers)
self.create(layers)
def create(self, layers):
print('> Create Model', self.get_classname(), ' type: ', self.classtype)
start = time.time()
inputs = Input(shape=(seq_len, layers[0]))
x = LSTM(layers[1], activation='relu', return_sequences=True)(inputs)
#x = Dropout(0.1)(x)
x = LSTM(layers[2], activation='relu', return_sequences=False)(x)
#x = Dropout(0.1)(x)
preds = Dense(layers[3], activation='linear')(x)
self.model = Model(input=inputs, output=preds)
self.model.compile(optimizer='rmsprop', loss='mse', metrics=['mse'])
print("Compilation Time : ", time.time() - start)
#m = BaseLTSM([1, seq_len+1, (seq_len+1)*2, 1]) #bias added
m = BaseLTSM([1, 50, 50, 1]) #bias added
m.modelPath
m.model.summary()
In [ ]:
%%time
%%capture output
#sys.stdout = open('keras_output.txt', 'a+')
#m.compile()
model, hist = m.train(X_train, y_train, epochs=1, verbose=1)
#sys.stdout = stdout
hist
#m.save()
In [ ]:
output.show()
In [ ]:
from keras.models import load_model
# returns a compiled model
# identical to the previous one
model = load_model(h5PathModel)
In [ ]:
from sklearn.model_selection import GridSearchCV
from keras.wrappers.scikit_learn import KerasClassifier, KerasRegressor
def _model():
return m.model
model = KerasRegressor(build_fn=_model, verbose=1)
# __call__
#model = KerasRegressor(build_fn=m, verbose=1)
In [ ]:
%%capture output
grid = GridSearchCV(model,
param_grid={'nb_epoch': [1],
'batch_size': [10]},
#scoring='mse',
scoring='neg_mean_squared_error',
n_jobs=1)
grid_result = grid.fit(X_train, y_train)
In [ ]:
output.show()
In [ ]:
# summarize results
# The unified scoring API always maximizes the score, so scores which need to be minimized are negated in order for the unified scoring API to work correctly. The score that is returned is therefore negated when it is a score that should be minimized and left positive if it is a score that should be maximized.
print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']
for mean, stdev, param in zip(means, stds, params):
print("%f (%f) with: %r" % (mean, stdev, param))
In [ ]:
# validator.best_estimator_ returns sklearn-wrapped version of best model.
# validator.best_estimator_.model returns the (unwrapped) keras model
best_model = grid.best_estimator_.model
metric_names = best_model.metrics_names
metric_values = best_model.evaluate(X_test, y_test)
for metric, value in zip(metric_names, metric_values):
print(metric, ': ', value)
In [ ]:
grid_result.best_score_
grid_result.best_params_
grid_result.cv_results_
In [ ]:
seq_len, X_test.shape, y_test.shape
X_test[0, :10], y_test[0]
X_test[1, :10], y_test[1]
In [ ]:
def predict_sequence_full(model, data, window_size):
#Shift the window by 1 new prediction each time, re-run predictions on new window
curr_frame = data[0]
predicted = []
# loop over entire testdata
for i in range(len(data)):
if m.classtype == 'ltsm':
predicted.append(model.predict(curr_frame[np.newaxis,:,:])[0,0]) #get element from shape(1,1,:)
elif m.classtype == 'dense':
predicted.append(model.predict(curr_frame[np.newaxis,:])[0][0]) #get element from shape(1,1)
else:
raise ValueError('Wrong model type: ', modelType)
curr_frame = curr_frame[1:] #move window
#Tracer()()
curr_frame = np.insert(curr_frame, [window_size-2], predicted[-1], axis=0) #fill frame with prediction
return predicted
In [ ]:
%%time
length = len(X_test)
if m.classtype == 'ltsm':
predicted = predict_sequence_full(model, X_test[:length], seq_len)
elif m.classtype == 'dense':
predicted = predict_sequence_full(model, np.squeeze(X_test[:length]), seq_len)
else:
raise ValueError('Wrong model type: ', modelType)
#predicted = predict_sequence_full(model, np.squeeze(X_test[:length]), seq_len)
print(len(predicted), predicted)
plot_result(y_test, predicted, length)
In [ ]:
y_test.shape
y_test
np.array(predicted)
In [ ]:
def evaluate(X, y, verbose=0):
# gives the metrics, defined during compile
for n in zip(model.metrics_names, model.evaluate(X, y, verbose=verbose)):
print("{}:\t{}".format(n[0], n[1]))
evaluate(X_test, y_test, 1)
In [ ]: