training module: shl_tm
prediction module: shl_pm
simulation module: shl_sm
misc module: shl_mm
historical bidding price, per second, time series
live bidding price, per second, time series
parm_si (seasonality index per second)
parm_month (parameter like alpha, beta, gamma, etc. per month)
In [1]:
import pandas as pd
In [2]:
# function to fetch Seasonality-Index
def shl_intra_fetch_si(ccyy_mm, time, shl_data_parm_si):
# return shl_data_parm_si[(shl_data_parm_si['ccyy-mm'] == '2017-09') & (shl_data_parm_si['time'] == '11:29:00')]
return shl_data_parm_si[(shl_data_parm_si['ccyy-mm'] == ccyy_mm) & (shl_data_parm_si['time'] == time)].iloc[0]['si']
In [3]:
# function to fetch Dynamic-Increment
def shl_intra_fetch_di(ccyy_mm, shl_data_parm_month):
return shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == ccyy_mm].iloc[0]['di']
In [4]:
def shl_intra_fetch_previous_n_sec_time_as_str(shl_data_time_field, n):
return str((pd.to_datetime(shl_data_time_field, format='%H:%M:%S') - pd.Timedelta(seconds=n)).time())
def shl_intra_fetch_future_n_sec_time_as_str(shl_data_time_field, n):
return str((pd.to_datetime(shl_data_time_field, format='%H:%M:%S') - pd.Timedelta(seconds=-n)).time())
In [5]:
def shl_initialize(in_ccyy_mm='2017-07'):
print()
print('+-----------------------------------------------+')
print('| shl_initialize() |')
print('+-----------------------------------------------+')
print()
global shl_data_parm_si
global shl_data_parm_month
shl_data_parm_si = pd.read_csv('data/parm_si.csv')
shl_data_parm_month = pd.read_csv('data/parm_month.csv')
global shl_global_parm_ccyy_mm
shl_global_parm_ccyy_mm = in_ccyy_mm
# create default global base price
global shl_global_parm_base_price
shl_global_parm_base_price = 10000000
global shl_global_parm_dynamic_increment
shl_global_parm_dynamic_increment = shl_intra_fetch_di(shl_global_parm_ccyy_mm, shl_data_parm_month)
global shl_global_parm_alpha
shl_global_parm_alpha = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == shl_global_parm_ccyy_mm].iloc[0]['alpha']
global shl_global_parm_beta
shl_global_parm_beta = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == shl_global_parm_ccyy_mm].iloc[0]['beta']
global shl_global_parm_gamma
shl_global_parm_gamma = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == shl_global_parm_ccyy_mm].iloc[0]['gamma']
global shl_global_parm_sec57_weight
shl_global_parm_sec57_weight = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == shl_global_parm_ccyy_mm].iloc[0]['sec57-weight']
global shl_global_parm_month_weight
shl_global_parm_month_weight = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == shl_global_parm_ccyy_mm].iloc[0]['month-weight']
global shl_global_parm_short_weight
shl_global_parm_short_weight = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == shl_global_parm_ccyy_mm].iloc[0]['short-weight']
# default = 0
global shl_global_parm_short_weight_ratio
shl_global_parm_short_weight_ratio = 0
# create default average error between 46~50 seconds:
global shl_global_parm_short_weight_misc
shl_global_parm_short_weight_misc = 0
print('shl_global_parm_ccyy_mm : %s' % shl_global_parm_ccyy_mm)
print('-------------------------------------------------')
print('shl_global_parm_alpha : %0.15f' % shl_global_parm_alpha) # used in forecasting
print('shl_global_parm_beta : %0.15f' % shl_global_parm_beta) # used in forecasting
print('shl_global_parm_gamma : %0.15f' % shl_global_parm_gamma) # used in forecasting
print('shl_global_parm_short_weight : %f' % shl_global_parm_short_weight) # used in forecasting
print('shl_global_parm_short_weight_ratio: %f' % shl_global_parm_short_weight) # used in forecasting
print('shl_global_parm_sec57_weight : %f' % shl_global_parm_sec57_weight) # used in training a model
print('shl_global_parm_month_weight : %f' % shl_global_parm_month_weight) # used in training a model
print('shl_global_parm_dynamic_increment : %d' % shl_global_parm_dynamic_increment)
print('-------------------------------------------------')
# plt.figure(figsize=(6,3)) # plot seasonality index
# plt.plot(shl_data_parm_si[(shl_data_parm_si['ccyy-mm'] == shl_global_parm_ccyy_mm)]['si'])
global shl_data_pm_1_step
shl_data_pm_1_step = pd.DataFrame() # initialize dataframe of prediction results
print()
print('prediction results dataframe: shl_data_pm_1_step')
print(shl_data_pm_1_step)
global shl_data_pm_k_step
shl_data_pm_k_step = pd.DataFrame() # initialize dataframe of prediction results
print()
print('prediction results dataframe: shl_data_pm_k_step')
print(shl_data_pm_k_step)
In [6]:
shl_initialize()
shl_predict_price(in_current_time, in_current_price, in_k_sec) # return k-seconrds Predicted Prices, in a list format
shl_predict_set_price(in_current_time, in_current_price, in_k_sec) # return k-second Predicted Price + Dynamic Increment, in a list format
call k times of shl_predict_price()
shl_data_pm_1_step_itr = pd.DataFrame() # initialize prediction dataframe at 11:29:00
In [28]:
(shl_data_pm_1_step['f_1_step_pred_price_inc'].shift(1)[46:50] - shl_data_pm_1_step['f_actual_price4pm'][46:50]).sum()
Out[28]:
In [29]:
(shl_data_pm_k_step['f_1_step_pred_price_inc'].shift(1)[46:50] - shl_data_pm_k_step['f_actual_price4pm'][46:50]).sum()
Out[29]:
In [ ]:
In [48]:
def shl_predict_price_1_step(in_current_time, in_current_price):
# 11:29:00~11:29:50
global shl_data_pm_k_step
global shl_global_parm_short_weight_misc
if in_current_time < '11:29:50': shl_global_parm_short_weight_misc = 0
global shl_global_parm_short_weight_ratio
global shl_global_parm_base_price
print()
print('+-----------------------------------------------+')
print('| shl_predict_price() |')
print('+-----------------------------------------------+')
print()
print('current_ccyy_mm : %s' % shl_global_parm_ccyy_mm) # str, format: ccyy-mm
print('in_current_time : %s' % in_current_time) # str, format: hh:mm:ss
print('in_current_price : %d' % in_current_price) # number, format: integer
print('-------------------------------------------------')
# capture & calculate 11:29:00 bid price - 1 as base price
if in_current_time == '11:29:00':
shl_global_parm_base_price = in_current_price -1
print('*INFO* At time [ %s ] Set shl_global_parm_base_price : %d ' % (in_current_time, shl_global_parm_base_price)) # Debug
# f_actual_datetime = shl_global_parm_ccyy_mm + ' ' + in_current_time
# print('*INFO* f_actual_datetime : %s ' % f_actual_datetime)
# get Seasonality-Index, for current second
f_actual_si = shl_intra_fetch_si(shl_global_parm_ccyy_mm, in_current_time, shl_data_parm_si)
print('*INFO* f_actual_si : %0.10f ' % f_actual_si) # Debug
# get Seasonality-Index, for current second + 1
f_1_step_time = shl_intra_fetch_future_n_sec_time_as_str(in_current_time, 1)
f_1_step_si = shl_intra_fetch_si(shl_global_parm_ccyy_mm, f_1_step_time, shl_data_parm_si)
print('*INFO* f_1_step_si : %0.10f ' % f_1_step_si) # Debug
# calculate price increment: f_actual_price4pm
f_actual_price4pm = in_current_price - shl_global_parm_base_price
print('*INFO* f_actual_price4pm : %d ' % f_actual_price4pm) # Debug
# calculate seasonality adjusted price increment: f_actual_price4pmsi
f_actual_price4pmsi = f_actual_price4pm / f_actual_si
print('*INFO* f_actual_price4pmsi : %0.10f ' % f_actual_price4pmsi) # Debug
if in_current_time == '11:29:00':
# shl_data_pm_k_step_itr = pd.DataFrame() # initialize prediction dataframe at 11:29:00
print('---- call prediction function shl_pm ---- %s' % in_current_time)
f_1_step_pred_les_level = f_actual_price4pmsi # special handling for 11:29:00
f_1_step_pred_les_trend = 0 # special handling for 11:29:00
f_1_step_pred_les = f_1_step_pred_les_level + f_1_step_pred_les_trend
f_1_step_pred_adj_misc = 0
f_1_step_pred_price_inc = (f_1_step_pred_les + f_1_step_pred_adj_misc) * f_1_step_si
f_1_step_pred_price = f_1_step_pred_price_inc + shl_global_parm_base_price
f_1_step_pred_price_rounded = round(f_1_step_pred_price/100, 0) * 100
f_1_step_pred_set_price_rounded = f_1_step_pred_price_rounded + shl_global_parm_dynamic_increment
else:
print('---- call prediction function shl_pm ---- %s' % in_current_time)
# function to get average forecast error between 46~50 seconds: mean(f_current_step_error)
if in_current_time == '11:29:50':
sec50_pred_price_inc = shl_data_pm_k_step[(shl_data_pm_k_step['ccyy-mm'] == shl_global_parm_ccyy_mm) \
& (shl_data_pm_k_step['time'] ==in_current_time)].iloc[0]['f_1_step_pred_price_inc']
sec50_error = sec50_pred_price_inc - f_actual_price4pm
sec46_49_error = (shl_data_pm_k_step['f_1_step_pred_price_inc'].shift(1)[46:50] - shl_data_pm_k_step['f_actual_price4pm'][46:50]).sum()
print('*INFO* sec50_error : %f' % sec50_error)
print('*INFO* sec46_49_error : %f' % sec46_49_error)
shl_global_parm_short_weight_misc = (sec50_error + sec46_49_error) / 5
print('*INFO* shl_global_parm_short_weight_misc : %f' % shl_global_parm_short_weight_misc)
# ----------------------------------------------------------------------------------------------------
# if in_current_time == '11:29:50':
shl_global_parm_short_weight_ratio = 1
print('*INFO* shl_global_parm_short_weight_ratio : %d' % shl_global_parm_short_weight_ratio)
if in_current_time == '11:29:51':
shl_global_parm_short_weight_ratio = 2
print('*INFO* shl_global_parm_short_weight_ratio : %d' % shl_global_parm_short_weight_ratio)
if in_current_time == '11:29:52':
shl_global_parm_short_weight_ratio = 3
print('*INFO* shl_global_parm_short_weight_ratio : %d' % shl_global_parm_short_weight_ratio)
if in_current_time == '11:29:53':
shl_global_parm_short_weight_ratio = 4
print('*INFO* shl_global_parm_short_weight_ratio : %d' % shl_global_parm_short_weight_ratio)
if in_current_time == '11:29:54':
shl_global_parm_short_weight_ratio = 5
print('*INFO* shl_global_parm_short_weight_ratio : %d' % shl_global_parm_short_weight_ratio)
if in_current_time == '11:29:55':
shl_global_parm_short_weight_ratio = 6
print('*INFO* shl_global_parm_short_weight_ratio : %d' % shl_global_parm_short_weight_ratio)
if in_current_time == '11:29:56':
shl_global_parm_short_weight_ratio = 7
print('*INFO* shl_global_parm_short_weight_ratio : %d' % shl_global_parm_short_weight_ratio)
if in_current_time == '11:29:57':
shl_global_parm_short_weight_ratio = 8
print('*INFO* shl_global_parm_short_weight_ratio : %d' % shl_global_parm_short_weight_ratio)
if in_current_time == '11:29:58':
shl_global_parm_short_weight_ratio = 9
print('*INFO* shl_global_parm_short_weight_ratio : %d' % shl_global_parm_short_weight_ratio)
if in_current_time == '11:29:59':
shl_global_parm_short_weight_ratio = 10
print('*INFO* shl_global_parm_short_weight_ratio : %d' % shl_global_parm_short_weight_ratio)
if in_current_time == '11:29:60':
shl_global_parm_short_weight_ratio = 11
print('*INFO* shl_global_parm_short_weight_ratio : %d' % shl_global_parm_short_weight_ratio)
# ----------------------------------------------------------------------------------------------------
previous_pred_les_level = shl_data_pm_k_step[(shl_data_pm_k_step['ccyy-mm'] == shl_global_parm_ccyy_mm) \
& (shl_data_pm_k_step['time'] ==in_current_time)].iloc[0]['f_1_step_pred_les_level']
print(' previous_pred_les_level : %f' % previous_pred_les_level)
previous_pred_les_trend = shl_data_pm_k_step[(shl_data_pm_k_step['ccyy-mm'] == shl_global_parm_ccyy_mm) \
& (shl_data_pm_k_step['time'] ==in_current_time)].iloc[0]['f_1_step_pred_les_trend']
print(' previous_pred_les_trend : %f' % previous_pred_les_trend)
f_1_step_pred_les_level = shl_global_parm_alpha * f_actual_price4pmsi \
+ (1 - shl_global_parm_alpha) * (previous_pred_les_level + previous_pred_les_trend)
print(' f_1_step_pred_les_level : %f' % f_1_step_pred_les_level)
f_1_step_pred_les_trend = shl_global_parm_beta * (f_1_step_pred_les_level - previous_pred_les_level) \
+ (1 - shl_global_parm_beta) * previous_pred_les_trend
print(' f_1_step_pred_les_trend : %f' % f_1_step_pred_les_trend)
f_1_step_pred_les = f_1_step_pred_les_level + f_1_step_pred_les_trend
f_1_step_pred_adj_misc = shl_global_parm_short_weight_misc * shl_global_parm_short_weight * shl_global_parm_short_weight_ratio * shl_global_parm_gamma
print(' les + misc : %f' % (f_1_step_pred_adj_misc+f_1_step_pred_les))
f_1_step_pred_price_inc = (f_1_step_pred_les + f_1_step_pred_adj_misc) * f_1_step_si
print(' f_1_step_pred_price_inc : %f' % f_1_step_pred_price_inc)
print(' f_1_step_si : %f' % f_1_step_si)
f_1_step_pred_price = f_1_step_pred_price_inc + shl_global_parm_base_price
f_1_step_pred_price_rounded = round(f_1_step_pred_price/100, 0) * 100
f_1_step_pred_set_price_rounded = f_1_step_pred_price_rounded + shl_global_parm_dynamic_increment
# write results to shl_pm dataframe
shl_data_pm_k_step_itr_dict = {
'ccyy-mm' : shl_global_parm_ccyy_mm
,'time' : f_1_step_time # in_current_time + 1 second
,'bid' : in_current_price
,'f_actual_si' : f_actual_si
,'f_1_step_si' : f_1_step_si
,'f_actual_price4pm' : f_actual_price4pm
,'f_actual_price4pmsi' : f_actual_price4pmsi
,'f_1_step_pred_les_level' : f_1_step_pred_les_level
,'f_1_step_pred_les_trend' : f_1_step_pred_les_trend
,'f_1_step_pred_les' : f_1_step_pred_les
,'f_1_step_pred_adj_misc' : f_1_step_pred_adj_misc
,'f_1_step_pred_price_inc' : f_1_step_pred_price_inc
,'f_1_step_pred_price' : f_1_step_pred_price
,'f_1_step_pred_price_rounded' : f_1_step_pred_price_rounded
,'f_1_step_pred_set_price_rounded' : f_1_step_pred_set_price_rounded
}
# shl_data_pm_k_step_itr = shl_data_pm_k_step_itr.append(shl_data_pm_k_step_itr_dict, ignore_index=True)
# shl_data_pm_k_step = shl_data_pm_k_step.append(shl_data_pm_k_step_itr_dict, ignore_index=True)
return shl_data_pm_k_step_itr_dict
In [15]:
def shl_predict_price_k_step(in_current_time, in_current_price, in_k_seconds=1):
global shl_data_pm_1_step
global shl_data_pm_k_step
shl_data_pm_k_step = shl_data_pm_1_step.copy()
shl_data_pm_itr_dict = {}
for k in range(1,in_k_seconds+1):
print()
print('==>> Forecasting next %3d second/step... ' % k)
if k == 1:
print('k = ', k)
input_price = in_current_price
input_time = in_current_time
shl_data_pm_itr_dict = shl_predict_price_1_step(input_time, input_price)
shl_data_pm_1_step = shl_data_pm_1_step.append(shl_data_pm_itr_dict, ignore_index=True)
else:
print('k = ', k)
input_price = shl_data_pm_itr_dict['f_1_step_pred_price']
input_time = shl_data_pm_itr_dict['time']
shl_data_pm_itr_dict = shl_predict_price_1_step(input_time, input_price)
shl_data_pm_k_step = shl_data_pm_k_step.append(shl_data_pm_itr_dict, ignore_index=True)
In [16]:
shl_data_pm_1_step
Out[16]:
In [17]:
shl_data_pm_k_step
Out[17]:
In [ ]:
In [ ]:
# shl_data_pm_1_step['f_actual_price4pm'][46:50]
In [ ]:
# shl_data_pm_1_step['f_1_step_pred_price_inc'].shift(1)[46:50]
In [ ]:
# shl_data_pm_1_step['f_1_step_pred_price_inc'].shift(1)[46:50] - shl_data_pm_1_step['f_actual_price4pm'][46:50]
In [ ]:
In [ ]:
In [19]:
shl_data_history_ts_process = pd.read_csv('data/history_ts.csv')
shl_data_history_ts_process.tail()
Out[19]:
In [104]:
# which month to predict?
# shl_global_parm_ccyy_mm = '2017-04'
# shl_global_parm_ccyy_mm_offset = 1647
# shl_global_parm_ccyy_mm = '2017-05'
# shl_global_parm_ccyy_mm_offset = 1708
shl_global_parm_ccyy_mm = '2017-06'
shl_global_parm_ccyy_mm_offset = 1769
# shl_global_parm_ccyy_mm = '2017-07'
# shl_global_parm_ccyy_mm_offset = 1830
In [105]:
shl_data_pm_1_step = pd.DataFrame() # initialize dataframe of prediction results
shl_data_pm_k_step = pd.DataFrame()
In [106]:
# Upon receiving 11:29:00 second price, to predict till 11:29:49 <- one-step forward price forecasting
shl_data_pm_1_step = pd.DataFrame() # initialize dataframe of prediction results
for i in range(shl_global_parm_ccyy_mm_offset, shl_global_parm_ccyy_mm_offset+50): # use July 2015 data as simulatino
print('\n<<<< Record No.: %5d >>>>' % i)
print(shl_data_history_ts_process['ccyy-mm'][i]) # format: ccyy-mm
print(shl_data_history_ts_process['time'][i]) # format: hh:mm:ss
print(shl_data_history_ts_process['bid-price'][i]) # format: integer
shl_predict_price_k_step(shl_data_history_ts_process['time'][i], shl_data_history_ts_process['bid-price'][i],1) # <- one-step forward price forecasting
In [107]:
# Upon receiving 11:29:50 second price, to predict till 11:30:00 <- ten-step forward price forecasting
for i in range(shl_global_parm_ccyy_mm_offset+50, shl_global_parm_ccyy_mm_offset+51): # use July 2015 data as simulatino
print('\n<<<< Record No.: %5d >>>>' % i)
print(shl_data_history_ts_process['ccyy-mm'][i]) # format: ccyy-mm
print(shl_data_history_ts_process['time'][i]) # format: hh:mm:ss
print(shl_data_history_ts_process['bid-price'][i]) # format: integer
shl_predict_price_k_step(shl_data_history_ts_process['time'][i], shl_data_history_ts_process['bid-price'][i],10) # <- ten-step forward price forecasting
In [108]:
# shl_data_pm_1_step.tail(11)
In [109]:
# shl_data_pm_k_step.tail(11)
In [ ]:
In [44]:
%matplotlib inline
import matplotlib.pyplot as plt
In [110]:
shl_data_pm_k_step_test = shl_data_pm_k_step.copy()
shl_data_pm_k_step_test.index = shl_data_pm_k_step_test.index + 1
shl_data_pm_k_step_test
Out[110]:
In [111]:
# bid is predicted bid-price from shl_pm
plt.figure(figsize=(12,6))
plt.plot(shl_data_pm_k_step['bid'])
# plt.plot(shl_data_pm_1_step_k_step['f_1_step_pred_price'].shift(1))
plt.plot(shl_data_pm_k_step_test['f_1_step_pred_price'])
# bid is actual bid-price from raw dataset
shl_data_actual_bid = shl_data_history_ts_process[shl_global_parm_ccyy_mm_offset:shl_global_parm_ccyy_mm_offset+61].copy()
shl_data_actual_bid.reset_index(inplace=True)
plt.figure(figsize=(12,6))
plt.plot(shl_data_actual_bid['bid-price'])
plt.plot(shl_data_pm_k_step_test['f_1_step_pred_price'])
Out[111]:
In [ ]:
In [112]:
# pd.concat([shl_data_actual_bid['bid-price'], shl_data_pm_k_step_test['f_1_step_pred_price'], shl_data_pm_k_step_test['f_1_step_pred_price'] - shl_data_actual_bid['bid-price']], axis=1, join='inner')
pd.concat([shl_data_actual_bid['bid-price'].tail(11), shl_data_pm_k_step_test['f_1_step_pred_price'].tail(11), shl_data_pm_k_step_test['f_1_step_pred_price'].tail(11) - shl_data_actual_bid['bid-price'].tail(11)], axis=1, join='inner')
Out[112]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
# create default global base price
shl_global_parm_base_price = 10000000
shl_global_parm_dynamic_increment = shl_intra_fetch_di(shl_global_parm_ccyy_mm, shl_data_parm_month)
shl_global_parm_alpha = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == shl_global_parm_ccyy_mm].iloc[0]['alpha']
shl_global_parm_beta = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == shl_global_parm_ccyy_mm].iloc[0]['beta']
shl_global_parm_gamma = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == shl_global_parm_ccyy_mm].iloc[0]['gamma']
shl_global_parm_sec57_weight = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == shl_global_parm_ccyy_mm].iloc[0]['sec57-weight']
shl_global_parm_month_weight = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == shl_global_parm_ccyy_mm].iloc[0]['month-weight']
shl_global_parm_short_weight = shl_data_parm_month[shl_data_parm_month['ccyy-mm'] == shl_global_parm_ccyy_mm].iloc[0]['short-weight']
# create default average error between 46~50 seconds:
shl_global_parm_short_weight_misc = 0
print('=================================================')
print(' Global Parameters for Month : %s' % shl_global_parm_ccyy_mm)
print('-------------------------------------------------')
print('shl_global_parm_dynamic_increment : %d' % shl_global_parm_dynamic_increment)
print('shl_global_parm_alpha : %0.15f' % shl_global_parm_alpha) # used in forecasting
print('shl_global_parm_beta : %0.15f' % shl_global_parm_beta) # used in forecasting
print('shl_global_parm_gamma : %0.15f' % shl_global_parm_gamma) # used in forecasting
print('shl_global_parm_sec57_weight : %f' % shl_global_parm_sec57_weight) # used in training a model
print('shl_global_parm_month_weight : %f' % shl_global_parm_month_weight) # used in training a model
print('shl_global_parm_short_weight : %f' % shl_global_parm_short_weight) # used in training a model
print('=================================================')
# plot seasonality index
plt.figure(figsize=(6,3))
plt.plot(shl_data_parm_si[(shl_data_parm_si['ccyy-mm'] == shl_global_parm_ccyy_mm)]['si'])
In [ ]:
In [ ]:
# 11:29:00~11:29:50
shl_global_parm_short_weight_misc = 0
# for i in range(1830, 1830+51): # use July 2015 data as simulatino
for i in range(shl_global_parm_ccyy_mm_offset, shl_global_parm_ccyy_mm_offset+51): # use July 2015 data as simulatino
print('\n<<<< Record No.: %5d >>>>' % i)
print(shl_data_history_ts_process['ccyy-mm'][i]) # format: ccyy-mm
print(shl_data_history_ts_process['time'][i]) # format: hh:mm:ss
print(shl_data_history_ts_process['bid-price'][i]) # format: integer
# print(shl_data_history_ts_process['ref-price'][i])
# capture & calculate 11:29:00 bid price - 1 = base price
if shl_data_history_ts_process['time'][i] == '11:29:00':
shl_global_parm_base_price = shl_data_history_ts_process['bid-price'][i] -1
print('*INFO* shl_global_parm_base_price : %d ' % shl_global_parm_base_price)
print('---- Pre-Process ---')
# pre-process: ccyy-mm-hh:mm:ss
f_actual_datetime = shl_data_history_ts_process['ccyy-mm'][i] + ' ' + shl_data_history_ts_process['time'][i]
f_actual_price4pm = shl_data_history_ts_process['bid-price'][i] - shl_global_parm_base_price
print('*INFO* f_actual_datetime : %s ' % f_actual_datetime)
print('*INFO* f_actual_price4pm : %d ' % f_actual_price4pm)
# get Seasonality-Index
f_actual_si = shl_intra_fetch_si(shl_data_history_ts_process['ccyy-mm'][i]
,shl_data_history_ts_process['time'][i]
,shl_data_parm_si)
print('*INFO* f_actual_si : %0.10f ' % f_actual_si)
f_1_step_si = shl_intra_fetch_si(shl_data_history_ts_process['ccyy-mm'][i]
,shl_data_history_ts_process['time'][i+1]
,shl_data_parm_si)
print('*INFO* f_1_step_si : %0.10f ' % f_1_step_si)
# get de-seasoned price: price4pmsi
f_actual_price4pmsi = f_actual_price4pm / f_actual_si
print('*INFO* f_actual_price4pmsi : %0.10f ' % f_actual_price4pmsi)
if shl_data_history_ts_process['time'][i] == '11:29:00':
shl_data_pm_1_step = pd.DataFrame() # initialize prediction dataframe at 11:29:00
print('---- call prediction function shl_pm ---- %s' % shl_data_history_ts_process['time'][i])
f_1_step_pred_les_level = f_actual_price4pmsi
f_1_step_pred_les_trend = 0
f_1_step_pred_les = f_1_step_pred_les_level + f_1_step_pred_les_trend
f_1_step_pred_adj_misc = 0
# f_1_step_pred_price_inc = (f_1_step_pred_les + f_1_step_pred_adj_misc) * f_actual_si
f_1_step_pred_price_inc = (f_1_step_pred_les + f_1_step_pred_adj_misc) * f_1_step_si
f_1_step_pred_price = f_1_step_pred_price_inc + shl_global_parm_base_price
f_1_step_pred_price_rounded = round(f_1_step_pred_price/100, 0) * 100
f_1_step_pred_dynamic_increment = shl_global_parm_dynamic_increment
f_1_step_pred_set_price_rounded = f_1_step_pred_price_rounded + f_1_step_pred_dynamic_increment
f_current_step_pred_les = f_1_step_pred_les
f_current_step_pred_les_misc = f_1_step_pred_adj_misc
f_current_step_pred_price_inc = f_1_step_pred_price_inc
f_current_step_pred_price = f_1_step_pred_price
f_current_step_pred_price_rounded = f_1_step_pred_price_rounded
f_current_step_pred_dynamic_increment = f_1_step_pred_dynamic_increment # +200 or + 300
f_current_step_pred_set_price_rounded = f_1_step_pred_set_price_rounded
f_current_step_error = f_current_step_pred_price_inc - f_actual_price4pm # current second forecast error
else:
previous_time = shl_intra_fetch_previous_n_sec_time_as_str(shl_data_history_ts_process['time'][i], 1)
previous_pred_les_level = shl_data_pm_1_step[(shl_data_pm_1_step['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_pm_1_step['time'] ==previous_time)].iloc[0]['f_1_step_pred_les_level']
print(' previous_pred_les_level : %f' % previous_pred_les_level)
previous_pred_les_trend = shl_data_pm_1_step[(shl_data_pm_1_step['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_pm_1_step['time'] ==previous_time)].iloc[0]['f_1_step_pred_les_trend']
print(' previous_pred_les_trend : %f' % previous_pred_les_trend)
f_current_step_pred_les = shl_data_pm_1_step[(shl_data_pm_1_step['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_pm_1_step['time'] ==previous_time)].iloc[0]['f_1_step_pred_les']
f_current_step_pred_les_misc = shl_data_pm_1_step[(shl_data_pm_1_step['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_pm_1_step['time'] ==previous_time)].iloc[0]['f_1_step_pred_adj_misc']
f_current_step_pred_price_inc = shl_data_pm_1_step[(shl_data_pm_1_step['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_pm_1_step['time'] ==previous_time)].iloc[0]['f_1_step_pred_price_inc']
f_current_step_pred_price = shl_data_pm_1_step[(shl_data_pm_1_step['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_pm_1_step['time'] ==previous_time)].iloc[0]['f_1_step_pred_price']
f_current_step_pred_price_rounded = shl_data_pm_1_step[(shl_data_pm_1_step['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_pm_1_step['time'] ==previous_time)].iloc[0]['f_1_step_pred_price_rounded']
f_current_step_pred_dynamic_increment = shl_data_pm_1_step[(shl_data_pm_1_step['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_pm_1_step['time'] ==previous_time)].iloc[0]['f_1_step_pred_dynamic_increment']
f_current_step_pred_set_price_rounded = shl_data_pm_1_step[(shl_data_pm_1_step['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_pm_1_step['time'] ==previous_time)].iloc[0]['f_1_step_pred_set_price_rounded']
f_current_step_error = f_current_step_pred_price_inc - f_actual_price4pm # current second forecast error
if shl_data_history_ts_process['time'][i] == '11:29:50':
# function to get average forecast error between 46~50 seconds: mean(f_current_step_error)
shl_global_parm_short_weight_misc = (shl_data_pm_1_step.iloc[46:50]['f_current_step_error'].sum() \
+ f_current_step_error) / 5
print('*INFO* shl_global_parm_short_weight_misc : %f' % shl_global_parm_short_weight_misc)
# call prediction functino shl_pm, forcaste next k=1 step
print('---- call prediction function shl_pm ---- %s' % shl_data_history_ts_process['time'][i])
f_1_step_pred_les_level = shl_global_parm_alpha * f_actual_price4pmsi \
+ (1 - shl_global_parm_alpha) * (previous_pred_les_level + previous_pred_les_trend)
print(' f_1_step_pred_les_level : %f' % f_1_step_pred_les_level)
f_1_step_pred_les_trend = shl_global_parm_beta * (f_1_step_pred_les_level - previous_pred_les_level) \
+ (1 - shl_global_parm_beta) * previous_pred_les_trend
print(' f_1_step_pred_les_trend : %f' % f_1_step_pred_les_trend)
print('global shl_global_parm_alpha : ', shl_global_parm_alpha)
print('global shl_global_parm_beta : ', shl_global_parm_beta)
f_1_step_pred_les = f_1_step_pred_les_level + f_1_step_pred_les_trend
f_1_step_pred_adj_misc = shl_global_parm_short_weight_misc * shl_global_parm_short_weight * shl_global_parm_gamma
# f_1_step_pred_price_inc = (f_1_step_pred_les + f_1_step_pred_adj_misc) * f_actual_si
f_1_step_pred_price_inc = (f_1_step_pred_les + f_1_step_pred_adj_misc) * f_1_step_si
f_1_step_pred_price = f_1_step_pred_price_inc + shl_global_parm_base_price
f_1_step_pred_price_rounded = round(f_1_step_pred_price/100, 0) * 100
f_1_step_pred_dynamic_increment = shl_global_parm_dynamic_increment
f_1_step_pred_set_price_rounded = f_1_step_pred_price_rounded + f_1_step_pred_dynamic_increment
# write results to shl_pm dataframe
shl_data_pm_1_step_current = {
'ccyy-mm' : shl_data_history_ts_process['ccyy-mm'][i]
,'time' : shl_data_history_ts_process['time'][i]
,'bid' : shl_data_history_ts_process['bid-price'][i]
,'datetime' : f_actual_datetime
,'f_actual_price4pm' : f_actual_price4pm
,'f_actual_si' : f_actual_si
,'f_1_step_si' : f_1_step_si
,'f_actual_price4pmsi' : f_actual_price4pmsi
,'f_1_step_pred_les_level' : f_1_step_pred_les_level
,'f_1_step_pred_les_trend' : f_1_step_pred_les_trend
,'f_1_step_pred_les' : f_1_step_pred_les
,'f_1_step_pred_adj_misc' : f_1_step_pred_adj_misc
,'f_1_step_pred_price_inc' : f_1_step_pred_price_inc
,'f_1_step_pred_price' : f_1_step_pred_price
,'f_1_step_pred_price_rounded' : f_1_step_pred_price_rounded
,'f_1_step_pred_dynamic_increment' : f_1_step_pred_dynamic_increment # +200 or + 300
,'f_1_step_pred_set_price_rounded' : f_1_step_pred_set_price_rounded
,'f_current_step_pred_les' : f_current_step_pred_les
,'f_current_step_pred_les_misc' : f_current_step_pred_les_misc
,'f_current_step_pred_price_inc' : f_current_step_pred_price_inc
,'f_current_step_pred_price' : f_current_step_pred_price
,'f_current_step_pred_price_rounded' : f_current_step_pred_price_rounded
,'f_current_step_pred_dynamic_increment' : f_current_step_pred_dynamic_increment # +200 or + 300
,'f_current_step_pred_set_price_rounded' : f_current_step_pred_set_price_rounded
,'f_current_step_error' : f_current_step_error
}
shl_data_pm_1_step = shl_data_pm_1_step.append(shl_data_pm_1_step_current, ignore_index=True)
In [ ]:
# shl_data_pm_1_step.iloc[2]
shl_data_pm_1_step.head()
In [ ]:
shl_data_pm_1_step.tail()
In [ ]:
plt.figure(figsize=(12,6))
plt.plot(shl_data_pm_1_step['bid'])
plt.plot(shl_data_pm_1_step['f_current_step_pred_price'])
# plt.plot(shl_data_pm_1_step['f_1_step_pred_price'].shift(1))
plt.figure(figsize=(12,6))
plt.plot(shl_data_pm_1_step['bid'])
plt.plot(shl_data_pm_1_step['f_current_step_pred_price'])
plt.plot(shl_data_pm_1_step['f_1_step_pred_price'])
In [ ]:
In [ ]:
# 11:29:51~
def predict_k_step_price(shl_data_pm_1_step, ccyy_mm, time, k):
print('month & time : ', ccyy_mm, time)
print()
# shl_data_pm_1_step_k = pd.DataFrame() # initialize prediction dataframe
for sec in range(0, k):
print('delta second(s) : ', sec)
current_time = shl_intra_fetch_future_n_sec_time_as_str(time, sec)
print('current_time : %s' % current_time)
f_1_step_time = shl_intra_fetch_future_n_sec_time_as_str(current_time, 1)
print('f_1_step_time : %s' % f_1_step_time)
previous_time = shl_intra_fetch_previous_n_sec_time_as_str(current_time, 1)
print('previous_time : %s' % previous_time)
previous_pred_les_level = shl_data_pm_1_step[(shl_data_pm_1_step['ccyy-mm'] == shl_global_parm_ccyy_mm) \
& (shl_data_pm_1_step['time'] ==previous_time)].iloc[0]['f_1_step_pred_les_level']
print(' previous_pred_les_level : %f' % previous_pred_les_level)
previous_pred_les_trend = shl_data_pm_1_step[(shl_data_pm_1_step['ccyy-mm'] == shl_global_parm_ccyy_mm) \
& (shl_data_pm_1_step['time'] ==previous_time)].iloc[0]['f_1_step_pred_les_trend']
print(' previous_pred_les_trend : %f' % previous_pred_les_trend)
print('---- Pre-Process ---')
############ use predicted value for boost-trap
previous_pred_price = shl_data_pm_1_step[(shl_data_pm_1_step['ccyy-mm'] == shl_global_parm_ccyy_mm) \
& (shl_data_pm_1_step['time'] == previous_time)].iloc[0]['f_1_step_pred_price']
# pre-process: ccyy-mm-hh:mm:ss
f_actual_datetime = shl_global_parm_ccyy_mm + ' ' + current_time
# f_actual_price4pm = shl_data_history_ts_process['bid-price'][i] - shl_global_parm_base_price
f_actual_price4pm = previous_pred_price - shl_global_parm_base_price
print('*INFO* f_actual_datetime : %s ' % f_actual_datetime)
print('*INFO* previous_pred_price: %s ' % previous_pred_price)
print('*INFO* f_actual_price4pm : %d ' % f_actual_price4pm)
# get Seasonality-Index
f_actual_si = shl_intra_fetch_si(shl_global_parm_ccyy_mm
,current_time
,shl_data_parm_si)
try:
f_1_step_si = shl_intra_fetch_si(shl_global_parm_ccyy_mm
,f_1_step_time
,shl_data_parm_si)
except:
f_1_step_si = shl_intra_fetch_si(shl_global_parm_ccyy_mm
,current_time
,shl_data_parm_si)
print('*INFO* f_actual_si : %0.10f ' % f_actual_si)
# get de-seasoned price: price4pmsi
f_actual_price4pmsi = f_actual_price4pm / f_actual_si
print('*INFO* f_actual_price4pmsi : %0.10f ' % f_actual_price4pmsi)
f_current_step_pred_les = shl_data_pm_1_step[(shl_data_pm_1_step['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_pm_1_step['time'] ==previous_time)].iloc[0]['f_1_step_pred_les']
f_current_step_pred_les_misc = shl_data_pm_1_step[(shl_data_pm_1_step['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_pm_1_step['time'] ==previous_time)].iloc[0]['f_1_step_pred_adj_misc']
f_current_step_pred_price_inc = shl_data_pm_1_step[(shl_data_pm_1_step['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_pm_1_step['time'] ==previous_time)].iloc[0]['f_1_step_pred_price_inc']
f_current_step_pred_price = shl_data_pm_1_step[(shl_data_pm_1_step['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_pm_1_step['time'] ==previous_time)].iloc[0]['f_1_step_pred_price']
f_current_step_pred_price_rounded = shl_data_pm_1_step[(shl_data_pm_1_step['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_pm_1_step['time'] ==previous_time)].iloc[0]['f_1_step_pred_price_rounded']
f_current_step_pred_dynamic_increment = shl_data_pm_1_step[(shl_data_pm_1_step['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_pm_1_step['time'] ==previous_time)].iloc[0]['f_1_step_pred_dynamic_increment']
f_current_step_pred_set_price_rounded = shl_data_pm_1_step[(shl_data_pm_1_step['ccyy-mm'] == shl_data_history_ts_process['ccyy-mm'][i]) \
& (shl_data_pm_1_step['time'] ==previous_time)].iloc[0]['f_1_step_pred_set_price_rounded']
f_current_step_error = f_current_step_pred_price_inc - f_actual_price4pm # current second forecast error
f_1_step_pred_les_level = shl_global_parm_alpha * f_actual_price4pmsi \
+ (1 - shl_global_parm_alpha) * (previous_pred_les_level + previous_pred_les_trend)
print(' f_1_step_pred_les_level : %f' % f_1_step_pred_les_level)
f_1_step_pred_les_trend = shl_global_parm_beta * (f_1_step_pred_les_level - previous_pred_les_level) \
+ (1 - shl_global_parm_beta) * previous_pred_les_trend
print(' f_1_step_pred_les_trend : %f' % f_1_step_pred_les_trend)
f_1_step_pred_les = f_1_step_pred_les_level + f_1_step_pred_les_trend
# f_1_step_pred_adj_misc = 0
f_1_step_pred_adj_misc = shl_global_parm_short_weight_misc * shl_global_parm_short_weight * (sec+2) * shl_global_parm_gamma
# f_1_step_pred_price_inc = (f_1_step_pred_les + f_1_step_pred_adj_misc) * f_actual_si
f_1_step_pred_price_inc = (f_1_step_pred_les + f_1_step_pred_adj_misc) * f_1_step_si
f_1_step_pred_price = f_1_step_pred_price_inc + shl_global_parm_base_price
f_1_step_pred_price_rounded = round(f_1_step_pred_price/100, 0) * 100
f_1_step_pred_dynamic_increment = shl_global_parm_dynamic_increment
f_1_step_pred_set_price_rounded = f_1_step_pred_price_rounded + f_1_step_pred_dynamic_increment
# write results to shl_pm dataframe
shl_data_pm_1_step_current = {
'ccyy-mm' : shl_global_parm_ccyy_mm
,'time' : current_time
,'bid' : previous_pred_price
,'datetime' : f_actual_datetime
,'f_actual_price4pm' : f_actual_price4pm
,'f_actual_si' : f_actual_si
,'f_1_step_si' : f_1_step_si
,'f_actual_price4pmsi' : f_actual_price4pmsi
,'f_1_step_pred_les_level' : f_1_step_pred_les_level
,'f_1_step_pred_les_trend' : f_1_step_pred_les_trend
,'f_1_step_pred_les' : f_1_step_pred_les
,'f_1_step_pred_adj_misc' : f_1_step_pred_adj_misc
,'f_1_step_pred_price_inc' : f_1_step_pred_price_inc
,'f_1_step_pred_price' : f_1_step_pred_price
,'f_1_step_pred_price_rounded' : f_1_step_pred_price_rounded
,'f_1_step_pred_dynamic_increment' : f_1_step_pred_dynamic_increment # +200 or + 300
,'f_1_step_pred_set_price_rounded' : f_1_step_pred_set_price_rounded
,'f_current_step_pred_les' : f_current_step_pred_les
,'f_current_step_pred_les_misc' : f_current_step_pred_les_misc
,'f_current_step_pred_price_inc' : f_current_step_pred_price_inc
,'f_current_step_pred_price' : f_current_step_pred_price
,'f_current_step_pred_price_rounded' : f_current_step_pred_price_rounded
,'f_current_step_pred_dynamic_increment' : f_current_step_pred_dynamic_increment # +200 or + 300
,'f_current_step_pred_set_price_rounded' : f_current_step_pred_set_price_rounded
,'f_current_step_error' : f_current_step_error
}
print('---------------------------')
shl_data_pm_1_step = shl_data_pm_1_step.append(shl_data_pm_1_step_current, ignore_index=True)
return shl_data_pm_1_step
In [ ]:
shl_data_pm_1_step_k_step = predict_k_step_price(shl_data_pm_1_step, shl_global_parm_ccyy_mm, '11:29:51', 10)
In [ ]:
shl_data_pm_1_step_k_step['f_current_step_pred_les_misc'].tail(11)
In [ ]:
# bid is predicted bid-price from shl_pm
plt.figure(figsize=(12,6))
plt.plot(shl_data_pm_1_step_k_step['bid'])
# plt.plot(shl_data_pm_1_step_k_step['f_1_step_pred_price'].shift(1))
plt.plot(shl_data_pm_1_step_k_step['f_current_step_pred_price'])
# bid is actual bid-price from raw dataset
shl_data_actual_bid = shl_data_history_ts_process[shl_global_parm_ccyy_mm_offset:shl_global_parm_ccyy_mm_offset+61].copy()
shl_data_actual_bid.reset_index(inplace=True)
plt.figure(figsize=(12,6))
plt.plot(shl_data_actual_bid['bid-price'])
# plt.plot(shl_data_pm_1_step_k_step['f_1_step_pred_price'].shift(1))
plt.plot(shl_data_pm_1_step_k_step['f_current_step_pred_price'])
# plt.plot(shl_data_pm_1_step_k_step['bid'])
In [ ]:
In [ ]:
# actual price & oredicted price & error
pd.concat([shl_data_actual_bid['bid-price'].tail(11), shl_data_pm_1_step_k_step['f_current_step_pred_price'].tail(11), shl_data_pm_1_step_k_step['f_current_step_pred_price'].tail(11) - shl_data_actual_bid['bid-price'].tail(11)], axis=1, join='inner')
In [ ]:
shl_data_pm_1_step_k_step.iloc[57]
# shl_data_pm_1_step_k_step.iloc[50:61]
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]: