In [ ]:
from IPython.core.debugger import Tracer
#Tracer()()
import os, sys, time
### prevent the dying jupyter notebook
stdout = sys.stdout
#sys.stdout = sys.__stdout__ # did not work to restoure print -> console
#sys.stdout = open('keras_output.txt', 'a+')
#sys.stdout = stdout
import utils
import importlib
importlib.reload(utils)
#Allow relative imports to directories above cwd/
sys.path.insert(1, os.path.join(sys.path[0], '..'))
#import modules
from utils import *
%matplotlib inline
np.random.seed(42)
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import mean_squared_error
%pwd #Verify we are in the right directory
!uname -a
In [ ]:
# argparse
import argparse
parser = argparse.ArgumentParser(description="programpurpose")
parser.add_argument("-s", "--sample", help="run on sample", action="store_true")
args = parser.parse_args(["-s"])
In [2]:
#Create references to important directories we will use over and over
current_dir = os.getcwd()
dataname = 'airline'
dpath = "{}/data/{}/".format(current_dir, dataname)
#h5Path = '{}weights_TF.h5'.format(dpath)
#h5PathModel = '{}model_TF.h5'.format(dpath)
#h5Path, h5PathModel
#
#modelType = 'dense'
seq_len = 40
In [ ]:
%mkdir -p $dpath
In [ ]:
zipname = 'xxx.zip'
!wget -P $dpath https://datamarket.com/data/set/22u3/international-airline-passengers-monthly-totals-in-thousands-jan-49-dec-60#
!unzip -d $dpath data/power/household_power_consumption.zip
In [3]:
# read data
df = pd.read_csv(dpath+'international-airline-passengers.csv', sep=';', usecols=[1], engine='python', skipfooter=3)
df.info()
ts = df.values
ts = ts.astype('float32')
ts.shape
# normalize the dataset
scaler = MinMaxScaler(feature_range=(0, 1))
ts = scaler.fit_transform(ts)
plot1(ts)
Out[3]:
In [ ]:
autocorrelation_plot2d(df)
In [ ]:
autocorrelation_plot_all(df.values[:,0])
In [ ]:
def autocorrelation_plotPD_all(series):
# solid: 95% dashed: 99% confidence interval for correlation values
from pandas.tools.plotting import autocorrelation_plot
autocorrelation_plot(series)
In [ ]:
print(test_corr(df))
autocorrelation_plotPD_all(df)
In [ ]:
df.values[:,0]
In [ ]:
test_stationary(df.values[:,0])
In [4]:
def load_data(ts, seq_len, normalise_window, split=0.9, shuffle=False):
print('> Loading data...')
sequence_length = seq_len + 1
result = []
# create gliding window
for index in range(len(ts) - sequence_length):
result.append(ts[index: index + sequence_length])
if normalise_window:
result = normalise_windows(result)
result = np.array(result)
#print(result[:4, :])
row = round(split * result.shape[0])
train = result[:row, :]
test = result[row:, :]
print("Data shape: ", result.shape, "Train: ", train.shape, "Test: ", test.shape)
if shuffle:
np.random.shuffle(train)
x_train = train[:, :-1]
y_train = train[:, -1]
x_test = test[:, :-1]
y_test = test[:, -1]
x_train = np.reshape(x_train, (x_train.shape[0], x_train.shape[1], 1))
x_test = np.reshape(x_test, (x_test.shape[0], x_test.shape[1], 1))
return [x_train, y_train, x_test, y_test]
def normalise_windows(window_data):
normalised_data = []
for window in window_data:
normalised_window = [((float(p) / float(window[0])) - 1) for p in window]
normalised_data.append(normalised_window)
return normalised_data
#X_train, y_train, X_test, y_test = load_data(ts, seq_len, False)
X_train, y_train, X_test, y_test = load_data(ts, seq_len, False)
X_train.shape, y_train.shape, X_test.shape, y_test.shape
X_train[0,:seq_len, 0]
X_train[1,:seq_len, 0]
Out[4]:
Out[4]:
Out[4]:
In [ ]:
plot2(X_train[0], y_train)
In [ ]:
if m.classtype == 'dense':
print("Squeezing X_train")
X_train = np.squeeze(X_train)
X_test = np.squeeze(X_test)
elif modelType == 'ltsm':
pass
else:
raise ValueError('Wrong model type: ', modelType)
In [ ]:
class BaseModel():
def __init__(self, layers, verbose=1):
#self.FILE_PATH = 'http://www.platform.ai/models/'
self.modelPath = dpath+self.get_classname()+'.h5'
self.checkpoint = ModelCheckpoint(filepath= dpath + 'checkpoint-{epoch:02d}-{loss:.3f}-{val_loss:.3f}.hdf5')
#self.csvLogger = CSVLogger(dpath+'trainingLog.csv', separator=';', append=True)
self.verbose = verbose
@classmethod #classmethod always gets class as parameter
def get_classname(cls):
return cls.__name__
def save(self):
self.model.save(self.modelPath)
print("model saved to: ", self.modelPath)
def load(self):
self.model = load_model(modelPath)
print("model loaded from: ", self.modelPath)
def train(self, X_train, y_train, epochs=1, batch_size=64, val_split=0.05, verbose=1):
global_start_time = time.time()
try:
history = self.model.fit(
X_train, y_train,
batch_size=batch_size,
nb_epoch=epochs,
validation_split=val_split,
verbose=verbose,
#callbacks=[self.checkpoint, self.csvLogger]) #BUG: csv logger not rerun capable
callbacks=[self.checkpoint])
except KeyboardInterrupt:
print('Training duration (s) : ', time.time() - global_start_time)
return self.model, history
print('Training duration (s) : ', time.time() - global_start_time)
return self.model, history
In [ ]:
def preprocess(x):
return x
class BaseModel():
classtype = 'dense' # class variable
classname = 'BaseModel'
def __init__(self, layers):
#self.FILE_PATH = 'http://www.platform.ai/models/'
self.modelPath = dpath+self.classname+'.h5'
self.checkpoint = ModelCheckpoint(filepath= dpath + 'checkpoint-{epoch:02d}-{loss:.3f}-{val_loss:.3f}.hdf5')
self.csvLogger = CSVLogger(dpath+'trainingLog.csv', separator=';', append=True)
self.create(layers)
def create(self, layers):
print('> Create Model')
start = time.time()
model = self.model = Sequential()
model.add(Dense(
input_dim=layers[0],
output_dim=layers[1],
activation='relu'))
model.add(Dense(
output_dim=layers[2],
activation='relu'))
model.add(Dense(
output_dim=layers[3],
activation='linear'))
model.compile(optimizer='adam', loss='mse', metrics=['mse', 'mape'])
print("Compilation Time : ", time.time() - start)
def save(self):
self.model.save(modelPath)
print("model saved to: ", self.modelPath)
def load(self):
self.model = load_model(modelPath)
print("model loaded from: ", self.modelPath)
def train(self, X_train, y_train, epochs=1, batch_size=64, val_split=0.05, verbose=1):
global_start_time = time.time()
try:
history = self.model.fit(
X_train, y_train,
batch_size=batch_size,
nb_epoch=epochs,
validation_split=0.05,
verbose=verbose,
callbacks=[self.checkpoint, self.csvLogger])
except KeyboardInterrupt:
print('Training duration (s) : ', time.time() - global_start_time)
return self.model, history
print('Training duration (s) : ', time.time() - global_start_time)
return self.model, history
m = BaseModel([seq_len, seq_len+1, (seq_len+1)*2, 1]) #bias added
m.classtype
m.modelPath
m.model.summary()
In [ ]:
class BaselineModel(BaseModel):
classtype = 'dense' # class variable
#classname = 'BaselineModel'
def __init__(self, layers):
super(BaselineModel, self).__init__(layers)
#self.FILE_PATH = 'http://www.platform.ai/models/'
#self.modelPath = dpath+self.classname+'.h5'
self.create(layers)
def __call__(self, layers):
self.__init(layers, verbose=0)
def create(self, layers):
print('> Create Model', self.get_classname(), ' type: ', self.classtype)
start = time.time()
inputs = Input(shape=(layers[0],))
x = Dense(layers[1], activation='relu')(inputs)
x = Dense(layers[2], activation='relu')(x)
preds = Dense(layers[3], activation='linear')(x)
self.model = Model(input=inputs, output=preds)
self.model.compile(optimizer='adam', loss='mse', metrics=['mse', 'mape'])
print("Compilation Time : ", time.time() - start)
m = BaselineModel([seq_len, seq_len+1, (seq_len+1)*2, 1]) #bias added
m.classtype
m.modelPath
m.model.summary()
In [ ]:
class DenseModel(BaseModel):
"""take the seq_len as feature input and predict one output"""
classtype = 'dense' # class variable
def __init__(self, layers):
super(DenseModel, self).__init__(layers)
self.create(layers)
def create(self, layers):
if verbose: print('> Create Model', self.get_classname(), ' type: ', self.classtype)
start = time.time()
inputs = Input(shape=(layers[0],))
x = Dense(layers[1], activation='relu')(inputs)
x = Dropout(0.25)(x)
x = Dense(layers[2], activation='relu')(x)
x = Dropout(0.25)(x)
preds = Dense(layers[3], activation='linear')(x)
self.model = Model(input=inputs, output=preds)
self.model.compile(optimizer='adam', loss='mse', metrics=['mse', 'mape'])
if verbose: print("Compilation Time : ", time.time() - start)
m = DenseModel([seq_len, 500, 250, 1]) #bias added
m.model.summary()
In [ ]:
%%time
#%%capture output
#sys.stdout = open('keras_output.txt', 'a+')
#m.compile()
model, hist = m.train(X_train, y_train, epochs=1, verbose=1)
#sys.stdout = stdout
hist
#m.save()
In [ ]:
class BaseLTSM(BaseModel):
classtype = 'ltsm' # class variable
classname = 'BaseLTSM'
def __init__(self, layers):
super(BaseLTSM, self).__init__(layers)
self.create(layers)
def create(self, layers):
print('> Create Model', self.get_classname(), ' type: ', self.classtype)
start = time.time()
inputs = Input(shape=(seq_len, layers[0]))
x = LSTM(layers[1], activation='relu', return_sequences=True)(inputs)
#x = Dropout(0.1)(x)
x = LSTM(layers[2], activation='relu', return_sequences=False)(x)
#x = Dropout(0.1)(x)
preds = Dense(layers[3], activation='linear')(x)
self.model = Model(input=inputs, output=preds)
self.model.compile(optimizer='rmsprop', loss='mse', metrics=['mse'])
print("Compilation Time : ", time.time() - start)
#m = BaseLTSM([1, seq_len+1, (seq_len+1)*2, 1]) #bias added
m = BaseLTSM([1, 50, 50, 1]) #bias added
m.modelPath
m.model.summary()
In [ ]:
%%time
%%capture output
#sys.stdout = open('keras_output.txt', 'a+')
#m.compile()
model, hist = m.train(X_train, y_train, epochs=1, verbose=1)
#sys.stdout = stdout
hist
#m.save()
In [ ]:
output.show()
In [ ]:
from keras.models import load_model
# returns a compiled model
# identical to the previous one
model = load_model(h5PathModel)
In [ ]:
from sklearn.model_selection import GridSearchCV
from keras.wrappers.scikit_learn import KerasClassifier, KerasRegressor
def _model():
return m.model
model = KerasRegressor(build_fn=_model, verbose=1)
# __call__
#model = KerasRegressor(build_fn=m, verbose=1)
In [ ]:
%%capture output
grid = GridSearchCV(model,
param_grid={'nb_epoch': [1],
'batch_size': [10]},
#scoring='mse',
scoring='neg_mean_squared_error',
n_jobs=1)
grid_result = grid.fit(X_train, y_train)
In [ ]:
output.show()
In [ ]:
# summarize results
# The unified scoring API always maximizes the score, so scores which need to be minimized are negated in order for the unified scoring API to work correctly. The score that is returned is therefore negated when it is a score that should be minimized and left positive if it is a score that should be maximized.
print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']
for mean, stdev, param in zip(means, stds, params):
print("%f (%f) with: %r" % (mean, stdev, param))
In [ ]:
# validator.best_estimator_ returns sklearn-wrapped version of best model.
# validator.best_estimator_.model returns the (unwrapped) keras model
best_model = grid.best_estimator_.model
metric_names = best_model.metrics_names
metric_values = best_model.evaluate(X_test, y_test)
for metric, value in zip(metric_names, metric_values):
print(metric, ': ', value)
In [ ]:
grid_result.best_score_
grid_result.best_params_
grid_result.cv_results_
In [ ]:
seq_len, X_test.shape, y_test.shape
X_test[0, :10], y_test[0]
X_test[1, :10], y_test[1]
In [ ]:
def predict_sequence_full(model, data, window_size):
#Shift the window by 1 new prediction each time, re-run predictions on new window
curr_frame = data[0]
predicted = []
# loop over entire testdata
for i in range(len(data)):
if m.classtype == 'ltsm':
predicted.append(model.predict(curr_frame[np.newaxis,:,:])[0,0]) #get element from shape(1,1,:)
elif m.classtype == 'dense':
predicted.append(model.predict(curr_frame[np.newaxis,:])[0][0]) #get element from shape(1,1)
else:
raise ValueError('Wrong model type: ', modelType)
curr_frame = curr_frame[1:] #move window
#Tracer()()
curr_frame = np.insert(curr_frame, [window_size-2], predicted[-1], axis=0) #fill frame with prediction
return predicted
In [ ]:
%%time
length = len(X_test)
if m.classtype == 'ltsm':
predicted = predict_sequence_full(model, X_test[:length], seq_len)
elif m.classtype == 'dense':
predicted = predict_sequence_full(model, np.squeeze(X_test[:length]), seq_len)
else:
raise ValueError('Wrong model type: ', modelType)
#predicted = predict_sequence_full(model, np.squeeze(X_test[:length]), seq_len)
print(len(predicted), predicted)
plot_result(y_test, predicted, length)
In [ ]:
y_test.shape
y_test
np.array(predicted)
In [ ]:
def evaluate(X, y, verbose=0):
# gives the metrics, defined during compile
for n in zip(model.metrics_names, model.evaluate(X, y, verbose=verbose)):
print("{}:\t{}".format(n[0], n[1]))
evaluate(X_test, y_test, 1)
In [ ]: