notebook.community

Edit and run

By: 顾 瞻 GU Zhan (Sam)

July 2017

[1] Import useful reference packages





In [1]:



    


# from __future__ import print_function, division
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns; sns.set()
import pandas as pd
import operator
from scipy import interp
from itertools import cycle
from sklearn import svm
from sklearn.utils.validation import check_random_state
from sklearn.model_selection import StratifiedKFold, cross_val_score
from sklearn.preprocessing import StandardScaler

from sklearn.ensemble import GradientBoostingRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import AdaBoostRegressor
from sklearn.ensemble import ExtraTreesRegressor
from sklearn.ensemble import BaggingRegressor
from sklearn.linear_model import LinearRegression
from sklearn.neighbors import KNeighborsRegressor
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.ensemble import BaggingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier

from sklearn.metrics import roc_curve, auc
from statsmodels.graphics.mosaicplot import mosaic
print(__doc__)



    


















    






Automatically created module for IPython interactive environment

Read raw data





In [2]:



    


df_history_ts_process = pd.read_csv('data/history_ts.csv') 
df_history_ts_process.tail()



    


















    
Out[2]:











  
    

      
      ccyy-mm
      time
      bid-price
    

  
  
    

      1886
      2017-07
      11:29:56
      92100
    

    

      1887
      2017-07
      11:29:57
      92100
    

    

      1888
      2017-07
      11:29:58
      92100
    

    

      1889
      2017-07
      11:29:59
      92200
    

    

      1890
      2017-07
      11:30:00
      92200
    

  




















In [3]:



    


df_history_table_process = pd.read_csv('data/history_table.csv') 
df_history_table_process.tail()



    


















    
Out[3]:











  
    

      
      ccyy-mm
      volume-plate
      deal-price-low
      deal-price-avg
      deal-early-second
      volume-bidder
    

  
  
    

      26
      2017-03
      10356
      87800
      87916
      55
      262010
    

    

      27
      2017-04
      12196
      89800
      89850
      59
      252273
    

    

      28
      2017-05
      10316
      90100
      90209
      55
      270197
    

    

      29
      2017-06
      10312
      89400
      89532
      45
      244349
    

    

      30
      2017-07
      10325
      92200
      92250
      57
      269189

[2] Data pre-porcessing

Explore and visualize data

Parameters





In [4]:



    


parm_calculate_base_price_second = 15 # Use the current month's bid-price as base-price at this seconds. Later to derive increment-price
parm_calculate_target_second = 7 # How many seconds in future to predict: target variable
parm_calculate_prev_bp = 15 # Number of previous price/increment to include, i.e. previous 2sec, 3sec, 4sec, 5sec ... 15sec
parm_calculate_mv = 15 # Number of  previous price/increment Moving Average to calculate, i.e. previous 2sec, 3sec, 4sec, 5sec ... 15sec
parm_calculate_prev_month = 3 # Number of previous month to include (need to remove earliest x month from training data)
print('parm_calculate_base_price_second : %3d seconds' % parm_calculate_base_price_second)
print('parm_calculate_target_second     : %3d seconds' % parm_calculate_target_second)
print('parm_calculate_prev_bp           : %3d seconds' % parm_calculate_prev_bp)
print('parm_calculate_mv                : %3d seconds' % parm_calculate_mv)
print('parm_calculate_prev_month        : %3d months' % parm_calculate_prev_month)

print('' )
parm_ts_cycle = 61 # seconds/records per month
print('parm_ts_cycle                    : %3d seconds' % parm_ts_cycle)
parm_ts_month = int(len(df_history_ts_process) / parm_ts_cycle)
print('parm_ts_month                    : %3d months' %  parm_ts_month)

parm_record_cut_row_head = max(parm_calculate_base_price_second, parm_calculate_prev_bp, parm_calculate_mv)
parm_record_cut_row_tail = parm_calculate_target_second
parm_record_cut_month_head = parm_calculate_prev_month + 1

parm_ts_valid_cycle = parm_ts_cycle - parm_record_cut_row_head - parm_record_cut_row_tail
print('parm_ts_valid_cycle              : %3d seconds' % parm_ts_valid_cycle)
parm_ts_valid_month = parm_ts_month - parm_record_cut_month_head
print('parm_ts_valid_month              : %3d months' % parm_ts_valid_month)

if parm_record_cut_month_head < 10:
    parm_record_cut_ccyy = pd.to_datetime('2015-0'+str(parm_record_cut_month_head))
else:
    parm_record_cut_ccyy = pd.to_datetime('2015-'+str(parm_record_cut_month_head))

print('' )
print('parm_record_cut_ccyy             : %s' % parm_record_cut_ccyy)

print('parm_record_cut_month_head       : %3d months' % parm_record_cut_month_head)
print('parm_record_cut_row_head         : %3d seconds' % parm_record_cut_row_head)
print('parm_record_cut_row_tail         : %3d seconds' % parm_record_cut_row_tail)
print('' )



    


















    






parm_calculate_base_price_second :  15 seconds
parm_calculate_target_second     :   7 seconds
parm_calculate_prev_bp           :  15 seconds
parm_calculate_mv                :  15 seconds
parm_calculate_prev_month        :   3 months

parm_ts_cycle                    :  61 seconds
parm_ts_month                    :  31 months
parm_ts_valid_cycle              :  39 seconds
parm_ts_valid_month              :  27 months

parm_record_cut_ccyy             : 2015-04-01 00:00:00
parm_record_cut_month_head       :   4 months
parm_record_cut_row_head         :  15 seconds
parm_record_cut_row_tail         :   7 seconds


















In [5]:



    


df_history_ts_process.head()



    


















    
Out[5]:











  
    

      
      ccyy-mm
      time
      bid-price
    

  
  
    

      0
      2015-01
      11:29:00
      74000
    

    

      1
      2015-01
      11:29:01
      74000
    

    

      2
      2015-01
      11:29:02
      74000
    

    

      3
      2015-01
      11:29:03
      74000
    

    

      4
      2015-01
      11:29:04
      74000
    

  




















In [ ]:

Prepare derived features

Process: df_history_ts_process





In [6]:



    


# date of current month
df_history_ts_process['date-curr'] = df_history_ts_process.apply(lambda row: pd.to_datetime(row['ccyy-mm']), axis=1)

# date of previous month
df_history_ts_process['date-prev'] = df_history_ts_process.apply(lambda row: row['date-curr'] - pd.offsets.MonthBegin(1), axis=1)


# Year
df_history_ts_process['year'] = df_history_ts_process.apply(lambda row: row['ccyy-mm'][0:4], axis=1)

# Month
df_history_ts_process['month'] = df_history_ts_process.apply(lambda row: row['ccyy-mm'][5:7], axis=1)

# Hour
df_history_ts_process['hour'] = df_history_ts_process.apply(lambda row: row['time'][0:2], axis=1)

# Minute
df_history_ts_process['minute'] = df_history_ts_process.apply(lambda row: row['time'][3:5], axis=1)

# Second
df_history_ts_process['second'] = df_history_ts_process.apply(lambda row: row['time'][6:8], axis=1)


# datetime of current month
df_history_ts_process['datetime-curr'] = df_history_ts_process.apply(lambda row: str(row['date-curr']) + ' ' + row['time'], axis=1)

# datetime of previous month
df_history_ts_process['datetime-prev'] = df_history_ts_process.apply(lambda row: str(row['date-prev']) + ' ' + row['time'], axis=1)



    











In [7]:



    


df_history_ts_process.tail()



    


















    
Out[7]:











  
    

      
      ccyy-mm
      time
      bid-price
      date-curr
      date-prev
      year
      month
      hour
      minute
      second
      datetime-curr
      datetime-prev
    

  
  
    

      1886
      2017-07
      11:29:56
      92100
      2017-07-01
      2017-06-01
      2017
      07
      11
      29
      56
      2017-07-01 00:00:00 11:29:56
      2017-06-01 00:00:00 11:29:56
    

    

      1887
      2017-07
      11:29:57
      92100
      2017-07-01
      2017-06-01
      2017
      07
      11
      29
      57
      2017-07-01 00:00:00 11:29:57
      2017-06-01 00:00:00 11:29:57
    

    

      1888
      2017-07
      11:29:58
      92100
      2017-07-01
      2017-06-01
      2017
      07
      11
      29
      58
      2017-07-01 00:00:00 11:29:58
      2017-06-01 00:00:00 11:29:58
    

    

      1889
      2017-07
      11:29:59
      92200
      2017-07-01
      2017-06-01
      2017
      07
      11
      29
      59
      2017-07-01 00:00:00 11:29:59
      2017-06-01 00:00:00 11:29:59
    

    

      1890
      2017-07
      11:30:00
      92200
      2017-07-01
      2017-06-01
      2017
      07
      11
      30
      00
      2017-07-01 00:00:00 11:30:00
      2017-06-01 00:00:00 11:30:00
    

  




















In [8]:



    


# df_history_ts_process
# df_history_ts_process[1768:]



    











In [9]:



    


# new ['base-price']
gap = 1 # only one new feature/column

for gap in range(1, gap+1):
    col_name = 'base-price'+str(parm_calculate_base_price_second)+'sec'
    col_name_base_price = col_name
    col_data = pd.DataFrame(columns=[col_name])
    print('Creating : ', col_name)  

    for month in range(0, parm_ts_month):
        for i in range(0, parm_ts_cycle):
            col_data.loc[month*parm_ts_cycle+i] = df_history_ts_process['bid-price'][month*parm_ts_cycle+parm_calculate_base_price_second]
  
    df_history_ts_process[col_name] = col_data

print('Total records processed : ', len(col_data))



    


















    






Creating :  base-price15sec
Total records processed :  1891

















In [10]:



    


# df_history_ts_process
# df_history_ts_process[1768:]



    











In [11]:



    


# new ['increment-price'] = ['bid-price'] - ['base-price']

df_history_ts_process['increment-price'] = df_history_ts_process.apply(lambda row: row['bid-price'] - row[col_name_base_price], axis=1)



    











In [12]:



    


# df_history_ts_process
# df_history_ts_process[1768:]



    











In [13]:



    


plt.figure()
plt.plot(df_history_ts_process['bid-price'])
plt.plot(df_history_ts_process[col_name_base_price])
plt.plot()
plt.figure()
plt.plot(df_history_ts_process['increment-price'])
plt.plot()



    


















    
Out[13]:








[]










    






/home/user/env_py3/lib/python3.5/site-packages/matplotlib/font_manager.py:1297: UserWarning: findfont: Font family ['sans-serif'] not found. Falling back to DejaVu Sans
  (prop.get_family(), self.defaultFamily[fontext]))

['increment-price-target']





In [14]:



    


# previous N sec ['increment-price-target']

for gap in range(1, 2):
    col_name = 'increment-price-target'
    col_data = pd.DataFrame(columns=[col_name])
    print('Creating : ', col_name)  

    for month in range(0, parm_ts_month):
    #     print('month : ', month)
        for i in range(0, (parm_ts_cycle - parm_calculate_target_second)):
            col_data.loc[month*parm_ts_cycle+i] = df_history_ts_process['increment-price'][month*parm_ts_cycle+i+parm_calculate_target_second]
        for i in range((parm_ts_cycle - parm_calculate_target_second), parm_ts_cycle):
            col_data.loc[month*parm_ts_cycle+i] = 0
  
    df_history_ts_process[col_name] = col_data

print('Total records processed : ', len(col_data))



    


















    






Creating :  increment-price-target
Total records processed :  1891

















In [15]:



    


plt.figure()
plt.plot(df_history_ts_process['increment-price'])
plt.plot(df_history_ts_process['increment-price-target'])
plt.plot()

plt.figure()
plt.plot(df_history_ts_process['increment-price'][1768:])
plt.plot(df_history_ts_process['increment-price-target'][1768:])
plt.plot()



    


















    
Out[15]:








[]










    






/home/user/env_py3/lib/python3.5/site-packages/matplotlib/font_manager.py:1297: UserWarning: findfont: Font family ['sans-serif'] not found. Falling back to DejaVu Sans
  (prop.get_family(), self.defaultFamily[fontext]))










    

















    
























In [ ]:



    






    











In [16]:



    


# previous 'parm_calculate_prev_bp' sec ['increment-price']
gap = parm_calculate_prev_bp

for gap in range(1, gap+1):
    col_name = 'increment-price-prev'+str(gap)+'sec'
    col_data = pd.DataFrame(columns=[col_name])
#     col_data_zeros = pd.DataFrame({col_name: np.zeros(gap)})
    print('Creating : ', col_name)  

    for month in range(0, parm_ts_month):
    #     print('month : ', month)
#         col_data.append(col_data_zeros)
        for i in range(0, gap):
            col_data.loc[month*parm_ts_cycle+i] = 0
        for i in range(gap, parm_ts_cycle):
            col_data.loc[month*parm_ts_cycle+i] = df_history_ts_process['increment-price'][month*parm_ts_cycle+i-gap]
  
    df_history_ts_process[col_name] = col_data

print('Total records processed : ', len(col_data))



    


















    






Creating :  increment-price-prev1sec
Creating :  increment-price-prev2sec
Creating :  increment-price-prev3sec
Creating :  increment-price-prev4sec
Creating :  increment-price-prev5sec
Creating :  increment-price-prev6sec
Creating :  increment-price-prev7sec
Creating :  increment-price-prev8sec
Creating :  increment-price-prev9sec
Creating :  increment-price-prev10sec
Creating :  increment-price-prev11sec
Creating :  increment-price-prev12sec
Creating :  increment-price-prev13sec
Creating :  increment-price-prev14sec
Creating :  increment-price-prev15sec
Total records processed :  1891

















In [17]:



    


# previous 'parm_calculate_mv' sec Moving Average ['increment-price']

gap = parm_calculate_mv

for gap in range(2, gap+1): # MV starts from 2 seconds, till parm_calculate_mv
    col_name = 'increment-price-mv'+str(gap)+'sec'
    col_data = pd.DataFrame(columns=[col_name])
    print('Creating : ', col_name)  

    for month in range(0, parm_ts_month):
    #     print('month : ', month)
        for i in range(0, gap):
            col_data.loc[month*parm_ts_cycle+i] = 0
        for i in range(gap, parm_ts_cycle):
            col_data.loc[month*parm_ts_cycle+i] = \
            np.mean(df_history_ts_process['increment-price'][month*parm_ts_cycle+i-gap:month*parm_ts_cycle+i])
  
    df_history_ts_process[col_name] = col_data

print('Total records processed : ', len(col_data))



    


















    






Creating :  increment-price-mv2sec
Creating :  increment-price-mv3sec
Creating :  increment-price-mv4sec
Creating :  increment-price-mv5sec
Creating :  increment-price-mv6sec
Creating :  increment-price-mv7sec
Creating :  increment-price-mv8sec
Creating :  increment-price-mv9sec
Creating :  increment-price-mv10sec
Creating :  increment-price-mv11sec
Creating :  increment-price-mv12sec
Creating :  increment-price-mv13sec
Creating :  increment-price-mv14sec
Creating :  increment-price-mv15sec
Total records processed :  1891

















In [18]:



    


# df_history_ts_process[1768:]



    











In [19]:



    


plt.figure()
plt.plot(df_history_ts_process['increment-price'][1768:])
plt.plot(df_history_ts_process['increment-price-prev3sec'][1768:])
plt.plot(df_history_ts_process['increment-price-prev7sec'][1768:])
plt.plot(df_history_ts_process['increment-price-prev11sec'][1768:])
plt.plot(df_history_ts_process['increment-price-prev15sec'][1768:])
plt.plot()



    


















    
Out[19]:








[]










    






/home/user/env_py3/lib/python3.5/site-packages/matplotlib/font_manager.py:1297: UserWarning: findfont: Font family ['sans-serif'] not found. Falling back to DejaVu Sans
  (prop.get_family(), self.defaultFamily[fontext]))










    
























In [20]:



    


plt.figure()
plt.plot(df_history_ts_process['increment-price'][1768:])
plt.plot(df_history_ts_process['increment-price-mv3sec'][1768:])
plt.plot(df_history_ts_process['increment-price-mv7sec'][1768:])
plt.plot(df_history_ts_process['increment-price-mv11sec'][1768:])
plt.plot(df_history_ts_process['increment-price-mv15sec'][1768:])
plt.plot()



    


















    
Out[20]:








[]










    






/home/user/env_py3/lib/python3.5/site-packages/matplotlib/font_manager.py:1297: UserWarning: findfont: Font family ['sans-serif'] not found. Falling back to DejaVu Sans
  (prop.get_family(), self.defaultFamily[fontext]))










    
























In [ ]:

Process: df_history_table_process





In [21]:



    


df_history_table_process.tail()



    


















    
Out[21]:











  
    

      
      ccyy-mm
      volume-plate
      deal-price-low
      deal-price-avg
      deal-early-second
      volume-bidder
    

  
  
    

      26
      2017-03
      10356
      87800
      87916
      55
      262010
    

    

      27
      2017-04
      12196
      89800
      89850
      59
      252273
    

    

      28
      2017-05
      10316
      90100
      90209
      55
      270197
    

    

      29
      2017-06
      10312
      89400
      89532
      45
      244349
    

    

      30
      2017-07
      10325
      92200
      92250
      57
      269189
    

  




















In [22]:



    


# date of current month
df_history_table_process['date-curr'] = df_history_table_process.apply(lambda row: pd.to_datetime(row['ccyy-mm']), axis=1)
df_history_table_process['d-avg-low-price'] = df_history_table_process.apply(lambda row: row['deal-price-avg'] - row['deal-price-low'], axis=1)
df_history_table_process['ratio-bid'] = df_history_table_process.apply(lambda row: row['volume-plate'] / row['volume-bidder'], axis=1)



    











In [23]:



    


df_history_table_process.tail()



    


















    
Out[23]:











  
    

      
      ccyy-mm
      volume-plate
      deal-price-low
      deal-price-avg
      deal-early-second
      volume-bidder
      date-curr
      d-avg-low-price
      ratio-bid
    

  
  
    

      26
      2017-03
      10356
      87800
      87916
      55
      262010
      2017-03-01
      116
      0.039525
    

    

      27
      2017-04
      12196
      89800
      89850
      59
      252273
      2017-04-01
      50
      0.048344
    

    

      28
      2017-05
      10316
      90100
      90209
      55
      270197
      2017-05-01
      109
      0.038180
    

    

      29
      2017-06
      10312
      89400
      89532
      45
      244349
      2017-06-01
      132
      0.042202
    

    

      30
      2017-07
      10325
      92200
      92250
      57
      269189
      2017-07-01
      50
      0.038356

Merge dataframe





In [24]:



    


df_history_ts_process_tmp2 = df_history_ts_process.copy()



    











In [25]:



    


df_history_ts_process = df_history_ts_process_tmp2.copy()



    











In [26]:



    


# look up current month table data: 'volume-plate', 'ratio-bid'
df_history_ts_process = pd.merge(df_history_ts_process, df_history_table_process[['date-curr', 'volume-plate', 'ratio-bid']], how = 'left', left_on = 'date-curr', right_on = 'date-curr', suffixes=['', '_table'])



    











In [27]:



    


for i in range(0, len(df_history_ts_process.columns)): print(df_history_ts_process.columns[i])



    


















    
Out[27]:








Index(['ccyy-mm', 'time', 'bid-price', 'date-curr', 'date-prev', 'year',
       'month', 'hour', 'minute', 'second', 'datetime-curr', 'datetime-prev',
       'base-price15sec', 'increment-price', 'increment-price-target',
       'increment-price-prev1sec', 'increment-price-prev2sec',
       'increment-price-prev3sec', 'increment-price-prev4sec',
       'increment-price-prev5sec', 'increment-price-prev6sec',
       'increment-price-prev7sec', 'increment-price-prev8sec',
       'increment-price-prev9sec', 'increment-price-prev10sec',
       'increment-price-prev11sec', 'increment-price-prev12sec',
       'increment-price-prev13sec', 'increment-price-prev14sec',
       'increment-price-prev15sec', 'increment-price-mv2sec',
       'increment-price-mv3sec', 'increment-price-mv4sec',
       'increment-price-mv5sec', 'increment-price-mv6sec',
       'increment-price-mv7sec', 'increment-price-mv8sec',
       'increment-price-mv9sec', 'increment-price-mv10sec',
       'increment-price-mv11sec', 'increment-price-mv12sec',
       'increment-price-mv13sec', 'increment-price-mv14sec',
       'increment-price-mv15sec', 'volume-plate', 'ratio-bid'],
      dtype='object')

















In [28]:



    


# look up pevious month table data: 'volume-plate', 'ratio-bid', 'deal-early-second', 'deal-price-avg', 'd-avg-low-price'
df_history_ts_process = pd.merge(df_history_ts_process, df_history_table_process[['date-curr', 'volume-plate', 'ratio-bid', 'deal-early-second', 'deal-price-avg', 'd-avg-low-price']], how = 'left', left_on = 'date-prev', right_on = 'date-curr', suffixes=['', '_m0'])



    











In [29]:



    


for i in range(0, len(df_history_ts_process.columns)): print(df_history_ts_process.columns[i])



    


















    






ccyy-mm
time
bid-price
date-curr
date-prev
year
month
hour
minute
second
datetime-curr
datetime-prev
base-price15sec
increment-price
increment-price-target
increment-price-prev1sec
increment-price-prev2sec
increment-price-prev3sec
increment-price-prev4sec
increment-price-prev5sec
increment-price-prev6sec
increment-price-prev7sec
increment-price-prev8sec
increment-price-prev9sec
increment-price-prev10sec
increment-price-prev11sec
increment-price-prev12sec
increment-price-prev13sec
increment-price-prev14sec
increment-price-prev15sec
increment-price-mv2sec
increment-price-mv3sec
increment-price-mv4sec
increment-price-mv5sec
increment-price-mv6sec
increment-price-mv7sec
increment-price-mv8sec
increment-price-mv9sec
increment-price-mv10sec
increment-price-mv11sec
increment-price-mv12sec
increment-price-mv13sec
increment-price-mv14sec
increment-price-mv15sec
volume-plate
ratio-bid
date-curr_m0
volume-plate_m0
ratio-bid_m0
deal-early-second
deal-price-avg
d-avg-low-price

Shift to copy previous 'parm_calculate_prev_month' month's data into current row





In [30]:



    


# df_history_ts_process = df_history_ts_process_lookup.copy()



    











In [31]:



    


df_history_ts_process_lookup = df_history_ts_process.copy()
df_history_ts_process_lookup.tail()



    


















    
Out[31]:











  
    

      
      ccyy-mm
      time
      bid-price
      date-curr
      date-prev
      year
      month
      hour
      minute
      second
      ...
      increment-price-mv14sec
      increment-price-mv15sec
      volume-plate
      ratio-bid
      date-curr_m0
      volume-plate_m0
      ratio-bid_m0
      deal-early-second
      deal-price-avg
      d-avg-low-price
    

  
  
    

      1886
      2017-07
      11:29:56
      92100
      2017-07-01
      2017-06-01
      2017
      07
      11
      29
      56
      ...
      828.571
      800
      10325
      0.038356
      2017-06-01
      10312.0
      0.042202
      45.0
      89532.0
      132.0
    

    

      1887
      2017-07
      11:29:57
      92100
      2017-07-01
      2017-06-01
      2017
      07
      11
      29
      57
      ...
      907.143
      873.333
      10325
      0.038356
      2017-06-01
      10312.0
      0.042202
      45.0
      89532.0
      132.0
    

    

      1888
      2017-07
      11:29:58
      92100
      2017-07-01
      2017-06-01
      2017
      07
      11
      29
      58
      ...
      985.714
      946.667
      10325
      0.038356
      2017-06-01
      10312.0
      0.042202
      45.0
      89532.0
      132.0
    

    

      1889
      2017-07
      11:29:59
      92200
      2017-07-01
      2017-06-01
      2017
      07
      11
      29
      59
      ...
      1057.14
      1020
      10325
      0.038356
      2017-06-01
      10312.0
      0.042202
      45.0
      89532.0
      132.0
    

    

      1890
      2017-07
      11:30:00
      92200
      2017-07-01
      2017-06-01
      2017
      07
      11
      30
      00
      ...
      1135.71
      1093.33
      10325
      0.038356
      2017-06-01
      10312.0
      0.042202
      45.0
      89532.0
      132.0
    

  



5 rows × 52 columns



















In [32]:



    


# _m1
df_history_ts_process = pd.merge(df_history_ts_process, df_history_ts_process_lookup[[ \
        'datetime-curr', 'datetime-prev', 
        'base-price15sec', 'increment-price', 'increment-price-target',
        'increment-price-prev1sec', 'increment-price-prev2sec',
        'increment-price-prev3sec', 'increment-price-prev4sec',
        'increment-price-prev5sec', 'increment-price-prev6sec',
        'increment-price-prev7sec', 'increment-price-prev8sec',
        'increment-price-prev9sec', 'increment-price-prev10sec',
        'increment-price-prev11sec', 'increment-price-prev12sec',
        'increment-price-prev13sec', 'increment-price-prev14sec',
        'increment-price-prev15sec', 
        'increment-price-mv2sec',
        'increment-price-mv3sec', 'increment-price-mv4sec',
        'increment-price-mv5sec', 'increment-price-mv6sec',
        'increment-price-mv7sec', 'increment-price-mv8sec',
        'increment-price-mv9sec', 'increment-price-mv10sec',
        'increment-price-mv11sec', 'increment-price-mv12sec',
        'increment-price-mv13sec', 'increment-price-mv14sec',
        'increment-price-mv15sec', 
        'volume-plate_m0', 
        'ratio-bid_m0', 
        'deal-early-second',
        'deal-price-avg',
        'd-avg-low-price'
        ]], how = 'left', left_on = 'datetime-prev', right_on = 'datetime-curr', suffixes=['', '_m1'])
df_history_ts_process.tail()



    


















    
Out[32]:











  
    

      
      ccyy-mm
      time
      bid-price
      date-curr
      date-prev
      year
      month
      hour
      minute
      second
      ...
      increment-price-mv11sec_m1
      increment-price-mv12sec_m1
      increment-price-mv13sec_m1
      increment-price-mv14sec_m1
      increment-price-mv15sec_m1
      volume-plate_m0_m1
      ratio-bid_m0_m1
      deal-early-second_m1
      deal-price-avg_m1
      d-avg-low-price_m1
    

  
  
    

      1886
      2017-07
      11:29:56
      92100
      2017-07-01
      2017-06-01
      2017
      07
      11
      29
      56
      ...
      627.273
      616.667
      607.692
      592.857
      580
      10316.0
      0.03818
      55.0
      90209.0
      109.0
    

    

      1887
      2017-07
      11:29:57
      92100
      2017-07-01
      2017-06-01
      2017
      07
      11
      29
      57
      ...
      636.364
      633.333
      623.077
      614.286
      600
      10316.0
      0.03818
      55.0
      90209.0
      109.0
    

    

      1888
      2017-07
      11:29:58
      92100
      2017-07-01
      2017-06-01
      2017
      07
      11
      29
      58
      ...
      654.545
      650
      646.154
      635.714
      626.667
      10316.0
      0.03818
      55.0
      90209.0
      109.0
    

    

      1889
      2017-07
      11:29:59
      92200
      2017-07-01
      2017-06-01
      2017
      07
      11
      29
      59
      ...
      672.727
      666.667
      661.538
      657.143
      646.667
      10316.0
      0.03818
      55.0
      90209.0
      109.0
    

    

      1890
      2017-07
      11:30:00
      92200
      2017-07-01
      2017-06-01
      2017
      07
      11
      30
      00
      ...
      690.909
      683.333
      676.923
      671.429
      666.667
      10316.0
      0.03818
      55.0
      90209.0
      109.0
    

  



5 rows × 91 columns



















In [33]:



    


# _m2
df_history_ts_process = pd.merge(df_history_ts_process, df_history_ts_process_lookup[[ \
        'datetime-curr', 'datetime-prev', 
        'base-price15sec', 'increment-price', 'increment-price-target',
        'increment-price-prev1sec', 'increment-price-prev2sec',
        'increment-price-prev3sec', 'increment-price-prev4sec',
        'increment-price-prev5sec', 'increment-price-prev6sec',
        'increment-price-prev7sec', 'increment-price-prev8sec',
        'increment-price-prev9sec', 'increment-price-prev10sec',
        'increment-price-prev11sec', 'increment-price-prev12sec',
        'increment-price-prev13sec', 'increment-price-prev14sec',
        'increment-price-prev15sec', 
        'increment-price-mv2sec',
        'increment-price-mv3sec', 'increment-price-mv4sec',
        'increment-price-mv5sec', 'increment-price-mv6sec',
        'increment-price-mv7sec', 'increment-price-mv8sec',
        'increment-price-mv9sec', 'increment-price-mv10sec',
        'increment-price-mv11sec', 'increment-price-mv12sec',
        'increment-price-mv13sec', 'increment-price-mv14sec',
        'increment-price-mv15sec', 
        'volume-plate_m0', 
        'ratio-bid_m0', 
        'deal-early-second',
        'deal-price-avg',
        'd-avg-low-price'                                                                                   
        ]], how = 'left', left_on = 'datetime-prev_m1', right_on = 'datetime-curr', suffixes=['', '_m2'])
df_history_ts_process.tail()



    


















    
Out[33]:











  
    

      
      ccyy-mm
      time
      bid-price
      date-curr
      date-prev
      year
      month
      hour
      minute
      second
      ...
      increment-price-mv11sec_m2
      increment-price-mv12sec_m2
      increment-price-mv13sec_m2
      increment-price-mv14sec_m2
      increment-price-mv15sec_m2
      volume-plate_m0_m2
      ratio-bid_m0_m2
      deal-early-second_m2
      deal-price-avg_m2
      d-avg-low-price_m2
    

  
  
    

      1886
      2017-07
      11:29:56
      92100
      2017-07-01
      2017-06-01
      2017
      07
      11
      29
      56
      ...
      681.818
      658.333
      630.769
      607.143
      586.667
      12196.0
      0.048344
      59.0
      89850.0
      50.0
    

    

      1887
      2017-07
      11:29:57
      92100
      2017-07-01
      2017-06-01
      2017
      07
      11
      29
      57
      ...
      745.455
      716.667
      692.308
      664.286
      640
      12196.0
      0.048344
      59.0
      89850.0
      50.0
    

    

      1888
      2017-07
      11:29:58
      92100
      2017-07-01
      2017-06-01
      2017
      07
      11
      29
      58
      ...
      809.091
      783.333
      753.846
      728.571
      700
      12196.0
      0.048344
      59.0
      89850.0
      50.0
    

    

      1889
      2017-07
      11:29:59
      92200
      2017-07-01
      2017-06-01
      2017
      07
      11
      29
      59
      ...
      881.818
      850
      823.077
      792.857
      766.667
      12196.0
      0.048344
      59.0
      89850.0
      50.0
    

    

      1890
      2017-07
      11:30:00
      92200
      2017-07-01
      2017-06-01
      2017
      07
      11
      30
      00
      ...
      945.455
      916.667
      884.615
      857.143
      826.667
      12196.0
      0.048344
      59.0
      89850.0
      50.0
    

  



5 rows × 130 columns



















In [34]:



    


# _m3
df_history_ts_process = pd.merge(df_history_ts_process, df_history_ts_process_lookup[[ \
        'datetime-curr', 'datetime-prev', 
        'base-price15sec', 'increment-price', 'increment-price-target',
        'increment-price-prev1sec', 'increment-price-prev2sec',
        'increment-price-prev3sec', 'increment-price-prev4sec',
        'increment-price-prev5sec', 'increment-price-prev6sec',
        'increment-price-prev7sec', 'increment-price-prev8sec',
        'increment-price-prev9sec', 'increment-price-prev10sec',
        'increment-price-prev11sec', 'increment-price-prev12sec',
        'increment-price-prev13sec', 'increment-price-prev14sec',
        'increment-price-prev15sec', 
        'increment-price-mv2sec',
        'increment-price-mv3sec', 'increment-price-mv4sec',
        'increment-price-mv5sec', 'increment-price-mv6sec',
        'increment-price-mv7sec', 'increment-price-mv8sec',
        'increment-price-mv9sec', 'increment-price-mv10sec',
        'increment-price-mv11sec', 'increment-price-mv12sec',
        'increment-price-mv13sec', 'increment-price-mv14sec',
        'increment-price-mv15sec', 
        'volume-plate_m0', 
        'ratio-bid_m0', 
        'deal-early-second',
        'deal-price-avg',
        'd-avg-low-price'                                                                                  
        ]], how = 'left', left_on = 'datetime-prev_m2', right_on = 'datetime-curr', suffixes=['', '_m3'])
df_history_ts_process.tail()



    


















    
Out[34]:











  
    

      
      ccyy-mm
      time
      bid-price
      date-curr
      date-prev
      year
      month
      hour
      minute
      second
      ...
      increment-price-mv11sec_m3
      increment-price-mv12sec_m3
      increment-price-mv13sec_m3
      increment-price-mv14sec_m3
      increment-price-mv15sec_m3
      volume-plate_m0_m3
      ratio-bid_m0_m3
      deal-early-second_m3
      deal-price-avg_m3
      d-avg-low-price_m3
    

  
  
    

      1886
      2017-07
      11:29:56
      92100
      2017-07-01
      2017-06-01
      2017
      07
      11
      29
      56
      ...
      709.091
      691.667
      669.231
      650
      626.667
      10356.0
      0.039525
      55.0
      87916.0
      116.0
    

    

      1887
      2017-07
      11:29:57
      92100
      2017-07-01
      2017-06-01
      2017
      07
      11
      29
      57
      ...
      736.364
      725
      707.692
      685.714
      666.667
      10356.0
      0.039525
      55.0
      87916.0
      116.0
    

    

      1888
      2017-07
      11:29:58
      92100
      2017-07-01
      2017-06-01
      2017
      07
      11
      29
      58
      ...
      772.727
      758.333
      746.154
      728.571
      706.667
      10356.0
      0.039525
      55.0
      87916.0
      116.0
    

    

      1889
      2017-07
      11:29:59
      92200
      2017-07-01
      2017-06-01
      2017
      07
      11
      29
      59
      ...
      809.091
      791.667
      776.923
      764.286
      746.667
      10356.0
      0.039525
      55.0
      87916.0
      116.0
    

    

      1890
      2017-07
      11:30:00
      92200
      2017-07-01
      2017-06-01
      2017
      07
      11
      30
      00
      ...
      845.455
      833.333
      815.385
      800
      786.667
      10356.0
      0.039525
      55.0
      87916.0
      116.0
    

  



5 rows × 169 columns



















In [35]:



    


plt.figure()
plt.plot(df_history_ts_process['increment-price-mv10sec'][1768:])
plt.plot(df_history_ts_process['increment-price-mv10sec_m1'][1768:])
plt.plot(df_history_ts_process['increment-price-mv10sec_m2'][1768:])
plt.plot(df_history_ts_process['increment-price-mv10sec_m3'][1768:])
plt.figure()
plt.plot(df_history_ts_process['increment-price-prev10sec'][1768:])
plt.plot(df_history_ts_process['increment-price-prev10sec_m1'][1768:])
plt.plot(df_history_ts_process['increment-price-prev10sec_m2'][1768:])
plt.plot(df_history_ts_process['increment-price-prev10sec_m3'][1768:])
plt.figure()
plt.plot(df_history_ts_process['increment-price'][1768:])
plt.plot(df_history_ts_process['increment-price_m1'][1768:])
plt.plot(df_history_ts_process['increment-price_m2'][1768:])
plt.plot(df_history_ts_process['increment-price_m3'][1768:])
plt.figure()
plt.plot(df_history_ts_process['increment-price-target'][1768:])
plt.plot(df_history_ts_process['increment-price-target_m1'][1768:])
plt.plot(df_history_ts_process['increment-price-target_m2'][1768:])
plt.plot(df_history_ts_process['increment-price-target_m3'][1768:])

plt.plot()



    


















    
Out[35]:








[]










    






/home/user/env_py3/lib/python3.5/site-packages/matplotlib/font_manager.py:1297: UserWarning: findfont: Font family ['sans-serif'] not found. Falling back to DejaVu Sans
  (prop.get_family(), self.defaultFamily[fontext]))










    

















    

















    

















    
























In [ ]:

Housekeeping to remove some invald data during pre-processing





In [36]:



    


for i in range(0, len(df_history_ts_process.columns)): print(df_history_ts_process.columns[i])



    


















    






ccyy-mm
time
bid-price
date-curr
date-prev
year
month
hour
minute
second
datetime-curr
datetime-prev
base-price15sec
increment-price
increment-price-target
increment-price-prev1sec
increment-price-prev2sec
increment-price-prev3sec
increment-price-prev4sec
increment-price-prev5sec
increment-price-prev6sec
increment-price-prev7sec
increment-price-prev8sec
increment-price-prev9sec
increment-price-prev10sec
increment-price-prev11sec
increment-price-prev12sec
increment-price-prev13sec
increment-price-prev14sec
increment-price-prev15sec
increment-price-mv2sec
increment-price-mv3sec
increment-price-mv4sec
increment-price-mv5sec
increment-price-mv6sec
increment-price-mv7sec
increment-price-mv8sec
increment-price-mv9sec
increment-price-mv10sec
increment-price-mv11sec
increment-price-mv12sec
increment-price-mv13sec
increment-price-mv14sec
increment-price-mv15sec
volume-plate
ratio-bid
date-curr_m0
volume-plate_m0
ratio-bid_m0
deal-early-second
deal-price-avg
d-avg-low-price
datetime-curr_m1
datetime-prev_m1
base-price15sec_m1
increment-price_m1
increment-price-target_m1
increment-price-prev1sec_m1
increment-price-prev2sec_m1
increment-price-prev3sec_m1
increment-price-prev4sec_m1
increment-price-prev5sec_m1
increment-price-prev6sec_m1
increment-price-prev7sec_m1
increment-price-prev8sec_m1
increment-price-prev9sec_m1
increment-price-prev10sec_m1
increment-price-prev11sec_m1
increment-price-prev12sec_m1
increment-price-prev13sec_m1
increment-price-prev14sec_m1
increment-price-prev15sec_m1
increment-price-mv2sec_m1
increment-price-mv3sec_m1
increment-price-mv4sec_m1
increment-price-mv5sec_m1
increment-price-mv6sec_m1
increment-price-mv7sec_m1
increment-price-mv8sec_m1
increment-price-mv9sec_m1
increment-price-mv10sec_m1
increment-price-mv11sec_m1
increment-price-mv12sec_m1
increment-price-mv13sec_m1
increment-price-mv14sec_m1
increment-price-mv15sec_m1
volume-plate_m0_m1
ratio-bid_m0_m1
deal-early-second_m1
deal-price-avg_m1
d-avg-low-price_m1
datetime-curr_m2
datetime-prev_m2
base-price15sec_m2
increment-price_m2
increment-price-target_m2
increment-price-prev1sec_m2
increment-price-prev2sec_m2
increment-price-prev3sec_m2
increment-price-prev4sec_m2
increment-price-prev5sec_m2
increment-price-prev6sec_m2
increment-price-prev7sec_m2
increment-price-prev8sec_m2
increment-price-prev9sec_m2
increment-price-prev10sec_m2
increment-price-prev11sec_m2
increment-price-prev12sec_m2
increment-price-prev13sec_m2
increment-price-prev14sec_m2
increment-price-prev15sec_m2
increment-price-mv2sec_m2
increment-price-mv3sec_m2
increment-price-mv4sec_m2
increment-price-mv5sec_m2
increment-price-mv6sec_m2
increment-price-mv7sec_m2
increment-price-mv8sec_m2
increment-price-mv9sec_m2
increment-price-mv10sec_m2
increment-price-mv11sec_m2
increment-price-mv12sec_m2
increment-price-mv13sec_m2
increment-price-mv14sec_m2
increment-price-mv15sec_m2
volume-plate_m0_m2
ratio-bid_m0_m2
deal-early-second_m2
deal-price-avg_m2
d-avg-low-price_m2
datetime-curr_m3
datetime-prev_m3
base-price15sec_m3
increment-price_m3
increment-price-target_m3
increment-price-prev1sec_m3
increment-price-prev2sec_m3
increment-price-prev3sec_m3
increment-price-prev4sec_m3
increment-price-prev5sec_m3
increment-price-prev6sec_m3
increment-price-prev7sec_m3
increment-price-prev8sec_m3
increment-price-prev9sec_m3
increment-price-prev10sec_m3
increment-price-prev11sec_m3
increment-price-prev12sec_m3
increment-price-prev13sec_m3
increment-price-prev14sec_m3
increment-price-prev15sec_m3
increment-price-mv2sec_m3
increment-price-mv3sec_m3
increment-price-mv4sec_m3
increment-price-mv5sec_m3
increment-price-mv6sec_m3
increment-price-mv7sec_m3
increment-price-mv8sec_m3
increment-price-mv9sec_m3
increment-price-mv10sec_m3
increment-price-mv11sec_m3
increment-price-mv12sec_m3
increment-price-mv13sec_m3
increment-price-mv14sec_m3
increment-price-mv15sec_m3
volume-plate_m0_m3
ratio-bid_m0_m3
deal-early-second_m3
deal-price-avg_m3
d-avg-low-price_m3

















In [ ]:



    


# housekeeping: delete some columns
# df_history_ts_process.drop('date-curr_y', axis=1, inplace=True)



    











In [37]:



    


parm_record_cut_ccyy



    


















    
Out[37]:








Timestamp('2015-04-01 00:00:00')

















In [38]:



    


# remove first 'parm_record_cut_ccyy' months from dataset
df_history_ts_process = df_history_ts_process[df_history_ts_process['date-curr'] > parm_record_cut_ccyy]



    











In [39]:



    


# total 61 seconds/rows per month:
# remove first 'parm_record_cut_row_head' reconds
# remove last 'parm_record_cut_row_tail' reconds
df_history_ts_process = df_history_ts_process[df_history_ts_process['second'] >= str(parm_record_cut_row_head) ]
df_history_ts_process = df_history_ts_process[df_history_ts_process['second'] <= str(60 - parm_record_cut_row_tail) ]
# df_history_ts_process = df_history_ts_process[df_history_ts_process['second'] > parm_record_cut_row_head ]



    











In [40]:



    


# Reset index after housekeeping
df_history_ts_process = df_history_ts_process.reset_index(drop=True)



    











In [41]:



    


df_history_ts_process.head()



    


















    
Out[41]:











  
    

      
      ccyy-mm
      time
      bid-price
      date-curr
      date-prev
      year
      month
      hour
      minute
      second
      ...
      increment-price-mv11sec_m3
      increment-price-mv12sec_m3
      increment-price-mv13sec_m3
      increment-price-mv14sec_m3
      increment-price-mv15sec_m3
      volume-plate_m0_m3
      ratio-bid_m0_m3
      deal-early-second_m3
      deal-price-avg_m3
      d-avg-low-price_m3
    

  
  
    

      0
      2015-05
      11:29:15
      78400
      2015-05-01
      2015-04-01
      2015
      05
      11
      29
      15
      ...
      0
      0
      0
      0
      -6.66667
      7990.0
      0.081362
      48.0
      74216.0
      216.0
    

    

      1
      2015-05
      11:29:16
      78400
      2015-05-01
      2015-04-01
      2015
      05
      11
      29
      16
      ...
      0
      0
      0
      0
      0
      7990.0
      0.081362
      48.0
      74216.0
      216.0
    

    

      2
      2015-05
      11:29:17
      78400
      2015-05-01
      2015-04-01
      2015
      05
      11
      29
      17
      ...
      0
      0
      0
      0
      0
      7990.0
      0.081362
      48.0
      74216.0
      216.0
    

    

      3
      2015-05
      11:29:18
      78400
      2015-05-01
      2015-04-01
      2015
      05
      11
      29
      18
      ...
      0
      0
      0
      0
      0
      7990.0
      0.081362
      48.0
      74216.0
      216.0
    

    

      4
      2015-05
      11:29:19
      78500
      2015-05-01
      2015-04-01
      2015
      05
      11
      29
      19
      ...
      0
      0
      0
      0
      0
      7990.0
      0.081362
      48.0
      74216.0
      216.0
    

  



5 rows × 169 columns



















In [42]:



    


df_history_ts_process.tail()



    


















    
Out[42]:











  
    

      
      ccyy-mm
      time
      bid-price
      date-curr
      date-prev
      year
      month
      hour
      minute
      second
      ...
      increment-price-mv11sec_m3
      increment-price-mv12sec_m3
      increment-price-mv13sec_m3
      increment-price-mv14sec_m3
      increment-price-mv15sec_m3
      volume-plate_m0_m3
      ratio-bid_m0_m3
      deal-early-second_m3
      deal-price-avg_m3
      d-avg-low-price_m3
    

  
  
    

      1048
      2017-07
      11:29:49
      91400
      2017-07-01
      2017-06-01
      2017
      07
      11
      29
      49
      ...
      454.545
      441.667
      430.769
      421.429
      413.333
      10356.0
      0.039525
      55.0
      87916.0
      116.0
    

    

      1049
      2017-07
      11:29:50
      91500
      2017-07-01
      2017-06-01
      2017
      07
      11
      29
      50
      ...
      490.909
      475
      461.538
      450
      440
      10356.0
      0.039525
      55.0
      87916.0
      116.0
    

    

      1050
      2017-07
      11:29:51
      91600
      2017-07-01
      2017-06-01
      2017
      07
      11
      29
      51
      ...
      527.273
      508.333
      492.308
      478.571
      466.667
      10356.0
      0.039525
      55.0
      87916.0
      116.0
    

    

      1051
      2017-07
      11:29:52
      91700
      2017-07-01
      2017-06-01
      2017
      07
      11
      29
      52
      ...
      563.636
      541.667
      523.077
      507.143
      493.333
      10356.0
      0.039525
      55.0
      87916.0
      116.0
    

    

      1052
      2017-07
      11:29:53
      91800
      2017-07-01
      2017-06-01
      2017
      07
      11
      29
      53
      ...
      600
      575
      553.846
      535.714
      520
      10356.0
      0.039525
      55.0
      87916.0
      116.0
    

  



5 rows × 169 columns



















In [43]:



    


plt.figure()
plt.plot(df_history_ts_process['increment-price'][974:])
plt.plot(df_history_ts_process['increment-price-mv3sec'][974:])
plt.plot(df_history_ts_process['increment-price-mv7sec'][974:])
plt.plot(df_history_ts_process['increment-price-mv11sec'][974:])
plt.plot(df_history_ts_process['increment-price-mv15sec'][974:])
plt.figure()
plt.plot(df_history_ts_process['increment-price-mv15sec'][974:])
plt.plot(df_history_ts_process['increment-price-mv15sec_m1'][974:])
plt.plot(df_history_ts_process['increment-price-mv15sec_m2'][974:])
plt.plot(df_history_ts_process['increment-price-mv15sec_m3'][974:])
plt.plot()



    


















    
Out[43]:








[]










    






/home/user/env_py3/lib/python3.5/site-packages/matplotlib/font_manager.py:1297: UserWarning: findfont: Font family ['sans-serif'] not found. Falling back to DejaVu Sans
  (prop.get_family(), self.defaultFamily[fontext]))










    

















    
























In [ ]:

[3] Modeling Part 2: Python scikit-learn

Models to use:

  • GradientBoostingClassifier
  • RandomForestClassifier
  • AdaBoostClassifier
  • ExtraTreesClassifier
  • BaggingClassifier
  • LogisticRegression
  • SVM kernal RBF
  • SVM kernal Linear
  • KNeighborsClassifier

Import pre-processed data





In [68]:



    


plt.plot(df_history_ts_process['d-avg-low-price'])
plt.figure()
plt.figure()
plt.plot(df_history_ts_process['d-avg-low-price_m1'])
plt.figure()
plt.plot(df_history_ts_process['d-avg-low-price_m2'])
plt.figure()
plt.plot(df_history_ts_process['d-avg-low-price_m3'])



    


















    
Out[68]:








[<matplotlib.lines.Line2D at 0x7f9fd14b8588>]










    






/home/user/env_py3/lib/python3.5/site-packages/matplotlib/font_manager.py:1297: UserWarning: findfont: Font family ['sans-serif'] not found. Falling back to DejaVu Sans
  (prop.get_family(), self.defaultFamily[fontext]))










    

















    








<matplotlib.figure.Figure at 0x7f9fd1249b70>










    

















    

















    
























In [45]:



    


for i in range(0, len(df_history_ts_process.columns)): print(df_history_ts_process.columns[i])



    


















    






ccyy-mm
time
bid-price
date-curr
date-prev
year
month
hour
minute
second
datetime-curr
datetime-prev
base-price15sec
increment-price
increment-price-target
increment-price-prev1sec
increment-price-prev2sec
increment-price-prev3sec
increment-price-prev4sec
increment-price-prev5sec
increment-price-prev6sec
increment-price-prev7sec
increment-price-prev8sec
increment-price-prev9sec
increment-price-prev10sec
increment-price-prev11sec
increment-price-prev12sec
increment-price-prev13sec
increment-price-prev14sec
increment-price-prev15sec
increment-price-mv2sec
increment-price-mv3sec
increment-price-mv4sec
increment-price-mv5sec
increment-price-mv6sec
increment-price-mv7sec
increment-price-mv8sec
increment-price-mv9sec
increment-price-mv10sec
increment-price-mv11sec
increment-price-mv12sec
increment-price-mv13sec
increment-price-mv14sec
increment-price-mv15sec
volume-plate
ratio-bid
date-curr_m0
volume-plate_m0
ratio-bid_m0
deal-early-second
deal-price-avg
d-avg-low-price
datetime-curr_m1
datetime-prev_m1
base-price15sec_m1
increment-price_m1
increment-price-target_m1
increment-price-prev1sec_m1
increment-price-prev2sec_m1
increment-price-prev3sec_m1
increment-price-prev4sec_m1
increment-price-prev5sec_m1
increment-price-prev6sec_m1
increment-price-prev7sec_m1
increment-price-prev8sec_m1
increment-price-prev9sec_m1
increment-price-prev10sec_m1
increment-price-prev11sec_m1
increment-price-prev12sec_m1
increment-price-prev13sec_m1
increment-price-prev14sec_m1
increment-price-prev15sec_m1
increment-price-mv2sec_m1
increment-price-mv3sec_m1
increment-price-mv4sec_m1
increment-price-mv5sec_m1
increment-price-mv6sec_m1
increment-price-mv7sec_m1
increment-price-mv8sec_m1
increment-price-mv9sec_m1
increment-price-mv10sec_m1
increment-price-mv11sec_m1
increment-price-mv12sec_m1
increment-price-mv13sec_m1
increment-price-mv14sec_m1
increment-price-mv15sec_m1
volume-plate_m0_m1
ratio-bid_m0_m1
deal-early-second_m1
deal-price-avg_m1
d-avg-low-price_m1
datetime-curr_m2
datetime-prev_m2
base-price15sec_m2
increment-price_m2
increment-price-target_m2
increment-price-prev1sec_m2
increment-price-prev2sec_m2
increment-price-prev3sec_m2
increment-price-prev4sec_m2
increment-price-prev5sec_m2
increment-price-prev6sec_m2
increment-price-prev7sec_m2
increment-price-prev8sec_m2
increment-price-prev9sec_m2
increment-price-prev10sec_m2
increment-price-prev11sec_m2
increment-price-prev12sec_m2
increment-price-prev13sec_m2
increment-price-prev14sec_m2
increment-price-prev15sec_m2
increment-price-mv2sec_m2
increment-price-mv3sec_m2
increment-price-mv4sec_m2
increment-price-mv5sec_m2
increment-price-mv6sec_m2
increment-price-mv7sec_m2
increment-price-mv8sec_m2
increment-price-mv9sec_m2
increment-price-mv10sec_m2
increment-price-mv11sec_m2
increment-price-mv12sec_m2
increment-price-mv13sec_m2
increment-price-mv14sec_m2
increment-price-mv15sec_m2
volume-plate_m0_m2
ratio-bid_m0_m2
deal-early-second_m2
deal-price-avg_m2
d-avg-low-price_m2
datetime-curr_m3
datetime-prev_m3
base-price15sec_m3
increment-price_m3
increment-price-target_m3
increment-price-prev1sec_m3
increment-price-prev2sec_m3
increment-price-prev3sec_m3
increment-price-prev4sec_m3
increment-price-prev5sec_m3
increment-price-prev6sec_m3
increment-price-prev7sec_m3
increment-price-prev8sec_m3
increment-price-prev9sec_m3
increment-price-prev10sec_m3
increment-price-prev11sec_m3
increment-price-prev12sec_m3
increment-price-prev13sec_m3
increment-price-prev14sec_m3
increment-price-prev15sec_m3
increment-price-mv2sec_m3
increment-price-mv3sec_m3
increment-price-mv4sec_m3
increment-price-mv5sec_m3
increment-price-mv6sec_m3
increment-price-mv7sec_m3
increment-price-mv8sec_m3
increment-price-mv9sec_m3
increment-price-mv10sec_m3
increment-price-mv11sec_m3
increment-price-mv12sec_m3
increment-price-mv13sec_m3
increment-price-mv14sec_m3
increment-price-mv15sec_m3
volume-plate_m0_m3
ratio-bid_m0_m3
deal-early-second_m3
deal-price-avg_m3
d-avg-low-price_m3

















In [69]:



    


X = df_history_ts_process[[
#          ,'ccyy-mm'
#         ,'time'
#         ,'bid-price'
#         ,'date-curr'
#         ,'date-prev'
#         ,'year'
         'month'
#         ,'hour'
#         ,'minute'
        ,'second'
#         ,'datetime-curr'
#         ,'datetime-prev'
        ,'base-price15sec'
        ,'increment-price'
#         ,'increment-price-target'   # <<<<<<< This is target 
        ,'increment-price-prev1sec'
        ,'increment-price-prev2sec'
        ,'increment-price-prev3sec'
        ,'increment-price-prev4sec'
        ,'increment-price-prev5sec'
        ,'increment-price-prev6sec'
        ,'increment-price-prev7sec'
        ,'increment-price-prev8sec'
        ,'increment-price-prev9sec'
        ,'increment-price-prev10sec'
        ,'increment-price-prev11sec'
        ,'increment-price-prev12sec'
        ,'increment-price-prev13sec'
        ,'increment-price-prev14sec'
        ,'increment-price-prev15sec'
        ,'increment-price-mv2sec'
        ,'increment-price-mv3sec'
        ,'increment-price-mv4sec'
        ,'increment-price-mv5sec'
        ,'increment-price-mv6sec'
        ,'increment-price-mv7sec'
        ,'increment-price-mv8sec'
        ,'increment-price-mv9sec'
        ,'increment-price-mv10sec'
        ,'increment-price-mv11sec'
        ,'increment-price-mv12sec'
        ,'increment-price-mv13sec'
        ,'increment-price-mv14sec'
        ,'increment-price-mv15sec'
        ,'volume-plate'
        ,'ratio-bid'
#         ,'date-curr_m0'
        ,'volume-plate_m0'
        ,'ratio-bid_m0'
        ,'deal-early-second'
        ,'deal-price-avg'
        ,'d-avg-low-price'
    
#         ,'datetime-curr_m1'
#         ,'datetime-prev_m1'
        ,'base-price15sec_m1'
        ,'increment-price_m1'
        ,'increment-price-target_m1'
        ,'increment-price-prev1sec_m1'
        ,'increment-price-prev2sec_m1'
        ,'increment-price-prev3sec_m1'
        ,'increment-price-prev4sec_m1'
        ,'increment-price-prev5sec_m1'
        ,'increment-price-prev6sec_m1'
        ,'increment-price-prev7sec_m1'
        ,'increment-price-prev8sec_m1'
        ,'increment-price-prev9sec_m1'
        ,'increment-price-prev10sec_m1'
        ,'increment-price-prev11sec_m1'
        ,'increment-price-prev12sec_m1'
        ,'increment-price-prev13sec_m1'
        ,'increment-price-prev14sec_m1'
        ,'increment-price-prev15sec_m1'
        ,'increment-price-mv2sec_m1'
        ,'increment-price-mv3sec_m1'
        ,'increment-price-mv4sec_m1'
        ,'increment-price-mv5sec_m1'
        ,'increment-price-mv6sec_m1'
        ,'increment-price-mv7sec_m1'
        ,'increment-price-mv8sec_m1'
        ,'increment-price-mv9sec_m1'
        ,'increment-price-mv10sec_m1'
        ,'increment-price-mv11sec_m1'
        ,'increment-price-mv12sec_m1'
        ,'increment-price-mv13sec_m1'
        ,'increment-price-mv14sec_m1'
        ,'increment-price-mv15sec_m1'
        ,'volume-plate_m0_m1'
        ,'ratio-bid_m0_m1'
        ,'deal-early-second_m1'
        ,'deal-price-avg_m1'
        ,'d-avg-low-price_m1'
    
#         ,'datetime-curr_m2'
#         ,'datetime-prev_m2'
        ,'base-price15sec_m2'
        ,'increment-price_m2'
        ,'increment-price-target_m2'
        ,'increment-price-prev1sec_m2'
        ,'increment-price-prev2sec_m2'
        ,'increment-price-prev3sec_m2'
        ,'increment-price-prev4sec_m2'
        ,'increment-price-prev5sec_m2'
        ,'increment-price-prev6sec_m2'
        ,'increment-price-prev7sec_m2'
        ,'increment-price-prev8sec_m2'
        ,'increment-price-prev9sec_m2'
        ,'increment-price-prev10sec_m2'
        ,'increment-price-prev11sec_m2'
        ,'increment-price-prev12sec_m2'
        ,'increment-price-prev13sec_m2'
        ,'increment-price-prev14sec_m2'
        ,'increment-price-prev15sec_m2'
        ,'increment-price-mv2sec_m2'
        ,'increment-price-mv3sec_m2'
        ,'increment-price-mv4sec_m2'
        ,'increment-price-mv5sec_m2'
        ,'increment-price-mv6sec_m2'
        ,'increment-price-mv7sec_m2'
        ,'increment-price-mv8sec_m2'
        ,'increment-price-mv9sec_m2'
        ,'increment-price-mv10sec_m2'
        ,'increment-price-mv11sec_m2'
        ,'increment-price-mv12sec_m2'
        ,'increment-price-mv13sec_m2'
        ,'increment-price-mv14sec_m2'
        ,'increment-price-mv15sec_m2'
        ,'volume-plate_m0_m2'
        ,'ratio-bid_m0_m2'
        ,'deal-early-second_m2'
        ,'deal-price-avg_m2'
        ,'d-avg-low-price_m2'
    
#         ,'datetime-curr_m3'
#         ,'datetime-prev_m3'
        ,'base-price15sec_m3'
        ,'increment-price_m3'
        ,'increment-price-target_m3'
        ,'increment-price-prev1sec_m3'
        ,'increment-price-prev2sec_m3'
        ,'increment-price-prev3sec_m3'
        ,'increment-price-prev4sec_m3'
        ,'increment-price-prev5sec_m3'
        ,'increment-price-prev6sec_m3'
        ,'increment-price-prev7sec_m3'
        ,'increment-price-prev8sec_m3'
        ,'increment-price-prev9sec_m3'
        ,'increment-price-prev10sec_m3'
        ,'increment-price-prev11sec_m3'
        ,'increment-price-prev12sec_m3'
        ,'increment-price-prev13sec_m3'
        ,'increment-price-prev14sec_m3'
        ,'increment-price-prev15sec_m3'
        ,'increment-price-mv2sec_m3'
        ,'increment-price-mv3sec_m3'
        ,'increment-price-mv4sec_m3'
        ,'increment-price-mv5sec_m3'
        ,'increment-price-mv6sec_m3'
        ,'increment-price-mv7sec_m3'
        ,'increment-price-mv8sec_m3'
        ,'increment-price-mv9sec_m3'
        ,'increment-price-mv10sec_m3'
        ,'increment-price-mv11sec_m3'
        ,'increment-price-mv12sec_m3'
        ,'increment-price-mv13sec_m3'
        ,'increment-price-mv14sec_m3'
        ,'increment-price-mv15sec_m3'
        ,'volume-plate_m0_m3'
        ,'ratio-bid_m0_m3'
        ,'deal-early-second_m3'
        ,'deal-price-avg_m3'
        ,'d-avg-low-price_m3'
        ]]

X_col = X.columns # get the column list

# X = StandardScaler().fit_transform(X.as_matrix())
X = X.as_matrix()

# y = StandardScaler().fit_transform(df_wnv_raw[['increment-price-target']].as_matrix()).reshape(len(df_wnv_raw),)
y = df_history_ts_process[['increment-price-target']].as_matrix().reshape(len(df_history_ts_process),)



    











In [70]:



    


X_col



    


















    
Out[70]:








Index(['month', 'second', 'base-price15sec', 'increment-price',
       'increment-price-prev1sec', 'increment-price-prev2sec',
       'increment-price-prev3sec', 'increment-price-prev4sec',
       'increment-price-prev5sec', 'increment-price-prev6sec',
       ...
       'increment-price-mv11sec_m3', 'increment-price-mv12sec_m3',
       'increment-price-mv13sec_m3', 'increment-price-mv14sec_m3',
       'increment-price-mv15sec_m3', 'volume-plate_m0_m3', 'ratio-bid_m0_m3',
       'deal-early-second_m3', 'deal-price-avg_m3', 'd-avg-low-price_m3'],
      dtype='object', length=151)

















In [71]:



    


plt.figure()
plt.plot(X)
plt.figure()
plt.plot(y)



    


















    
Out[71]:








[<matplotlib.lines.Line2D at 0x7f9fd111bc88>]










    






/home/user/env_py3/lib/python3.5/site-packages/matplotlib/font_manager.py:1297: UserWarning: findfont: Font family ['sans-serif'] not found. Falling back to DejaVu Sans
  (prop.get_family(), self.defaultFamily[fontext]))

[4] Evaluation

K-fold Cross-Validation





In [72]:



    


rng = check_random_state(0)



    











In [73]:



    


# GB
classifier_GB = GradientBoostingRegressor(n_estimators=1500, # score: 0.94608 (AUC 0.81419), learning_rate=0.001, max_features=8 <<< Best
#                                    loss='deviance',
#                                    subsample=1,
#                                    max_depth=5,
#                                    min_samples_split=20,
                                   learning_rate=0.002,
#                                    max_features=10,
                                   random_state=rng)



    











In [74]:



    


# AB
classifier_AB = AdaBoostRegressor(n_estimators=1500, # score: 0.93948 (AUC 0.88339), learning_rate=0.004 <<< Best
                                   learning_rate=0.002,
                                   random_state=rng)



    











In [75]:



    


# RF
classifier_RF = RandomForestRegressor(n_estimators=1500, # score: 0.94207 (AUC 0.81870), max_depth=3, min_samples_split=20, <<< Best
#                                     max_features=10,
#                                     max_depth=3,
#                                     min_samples_split=20,
                                    random_state=rng)



    











In [76]:



    


# ET
classifier_ET = ExtraTreesRegressor(n_estimators=1000, # score: 0.94655 (AUC 0.84364), max_depth=3, min_samples_split=20, max_features=10 <<< Best
#                                     max_depth=3,
#                                     min_samples_split=20,
#                                     max_features=10,
                                    random_state=rng)



    











In [77]:



    


# BG
classifier_BG = BaggingRegressor(n_estimators=500, # score: 0.70725 (AUC 0.63729) <<< Best
#                                     max_features=10,
                                    random_state=rng)

LR





In [78]:



    


classifier_LR = LinearRegression() # score: 0.90199 (AUC 0.80569)

SVM Linear





In [79]:



    


# classifier_SVCL = svm.SVC(kernel='linear', probability=True, random_state=rng) # score: 0.89976 (AUC 0.70524)
classifier_SVRL = svm.SVR() # score: 0.89976 (AUC 0.70524)

SVM





In [80]:



    


classifier_SVRR = svm.SVR(kernel='rbf') # score: 0.80188 (AUC 0.50050)
# classifier_SVRR = svm.SVR(kernel='poly') # score: 0.80188 (AUC 0.50050)

KNN





In [81]:



    


classifier_KNN = KNeighborsRegressor(n_neighbors=2) # score: 0.94018 (AUC 0.72792)
cv = cross_val_score(classifier_KNN,
                            X,
                            y,
                            cv=StratifiedKFold(parm_ts_valid_month))
print('KNN CV score: {0:.5f}'.format(cv.mean()))



    


















    






/home/user/env_py3/lib/python3.5/site-packages/sklearn/model_selection/_split.py:581: Warning: The least populated class in y has only 1 members, which is too few. The minimum number of groups for any class cannot be less than n_splits=27.
  % (min_groups, self.n_splits)), Warning)










    






KNN CV score: 0.97322

















In [ ]:

Select Model





In [82]:



    


# classifier = classifier_GB   # 324.632308296
# classifier = classifier_AB   # 429.646733221
# classifier = classifier_RF   # 175.504322802
classifier = classifier_ET   # 172.097916817, 0.0724812030075
# classifier = classifier_BG   # 175.451381872
# classifier = classifier_LR     # 128.465059749, 0.11
# classifier = classifier_SVRL # 3789.82169312
# classifier = classifier_SVRR # 3789.82169312, 0.10754224349

Split Data





In [83]:



    


n_splits = parm_ts_valid_cycle
print('cycle seconds : %d' % n_splits)
# n_splits=54 # 19 seconds/records for each bidding month
# n_splits=19 # 19 seconds/records for each bidding month
n_fold = parm_ts_valid_month
print('cycle month   : %d' % n_fold)


# X_train_1 = X[0:(len(X)-batch*n_splits)]
# y_train_1 = y[0:(len(X)-batch*n_splits)]

# X_test_1 = X[(len(X)-batch*n_splits):((len(X)-batch*n_splits)+n_splits)]
# y_test_1 = y[(len(X)-batch*n_splits):((len(X)-batch*n_splits)+n_splits)]



    


















    






cycle seconds : 39
cycle month   : 27

CV





In [84]:



    


n_fold=5



    











In [85]:



    


y_pred = {}
y_test = {}

y_pred_org = {}
y_test_org = {}

i = 0
for batch in range(1, n_fold):
    X_train_1 = X[0:(len(X)-batch*n_splits)]
    y_train_1 = y[0:(len(X)-batch*n_splits)]
    X_test_1  = X[(len(X)-batch*n_splits):((len(X)-batch*n_splits)+n_splits)]
    y_test_1  = y[(len(X)-batch*n_splits):((len(X)-batch*n_splits)+n_splits)]
    print(len(X_train_1))
    
    # ReScale
    ScalerX = StandardScaler()
    ScalerX.fit(X_train_1)
    X_train_1 = ScalerX.transform(X_train_1)
    X_test_1  = ScalerX.transform(X_test_1)
    
    ScalerY = StandardScaler()
    ScalerY.fit(y_train_1.reshape(-1, 1))
    y_train_1 = ScalerY.transform(y_train_1.reshape(-1, 1))
    y_test_1  = ScalerY.transform(y_test_1.reshape(-1, 1))
    
    y_pred[i] = classifier.fit(X_train_1, y_train_1).predict(X_test_1)
    y_test[i] = y_test_1  

    y_pred_org[i] = ScalerY.inverse_transform(y_pred[i])
    y_test_org[i] = ScalerY.inverse_transform(y_test[i])
    
    plt.figure()
    plt.plot(y_train_1)
    plt.plot()
    plt.figure()
    plt.plot(y_test[i])
    plt.plot(y_pred[i])
    plt.plot()
    i += 1



    


















    






1014










    






/home/user/env_py3/lib/python3.5/site-packages/sklearn/utils/validation.py:429: DataConversionWarning: Data with input dtype object was converted to float64 by StandardScaler.
  warnings.warn(msg, _DataConversionWarning)
/home/user/env_py3/lib/python3.5/site-packages/ipykernel_launcher.py:26: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().










    






975










    






/home/user/env_py3/lib/python3.5/site-packages/sklearn/utils/validation.py:429: DataConversionWarning: Data with input dtype object was converted to float64 by StandardScaler.
  warnings.warn(msg, _DataConversionWarning)
/home/user/env_py3/lib/python3.5/site-packages/ipykernel_launcher.py:26: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().










    






936










    






/home/user/env_py3/lib/python3.5/site-packages/sklearn/utils/validation.py:429: DataConversionWarning: Data with input dtype object was converted to float64 by StandardScaler.
  warnings.warn(msg, _DataConversionWarning)
/home/user/env_py3/lib/python3.5/site-packages/ipykernel_launcher.py:26: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().










    






897










    






/home/user/env_py3/lib/python3.5/site-packages/sklearn/utils/validation.py:429: DataConversionWarning: Data with input dtype object was converted to float64 by StandardScaler.
  warnings.warn(msg, _DataConversionWarning)
/home/user/env_py3/lib/python3.5/site-packages/ipykernel_launcher.py:26: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().
/home/user/env_py3/lib/python3.5/site-packages/matplotlib/font_manager.py:1297: UserWarning: findfont: Font family ['sans-serif'] not found. Falling back to DejaVu Sans
  (prop.get_family(), self.defaultFamily[fontext]))

no inverse-scale





In [86]:



    


k = []
for i in range(0, len(y_test)):
    k.append(np.mean(np.sqrt(np.square(y_test[i] - y_pred[i]))))

k_mean = np.mean(k)

print(k_mean)
print()
print(k)



    


















    






1.05844200322

[1.3494853891905882, 0.79940316962453462, 1.1700861231095021, 0.91479333095321491]

















In [87]:



    


k = []
for i in range(0, len(y_test)):
    k.append(np.mean(np.sqrt(np.square(y_test[i][35:37] - y_pred[i][35:37]))))

k_mean = np.mean(k)

print(k_mean)
print()
print(k)



    


















    






0.394769379894

[0.49166020170674574, 0.46279808460546001, 0.28888358420588789, 0.33573564905761766]

inverse-scale





In [88]:



    


k = []
for i in range(0, len(y_test)):
    k.append(np.mean(np.sqrt(np.square(y_test_org[i] - y_pred_org[i]))))

k_mean = np.mean(k)

print(k_mean)
print()
print(k)



    


















    






407.095266272

[514.64102564102609, 308.84589086127522, 449.99638395792397, 354.89776462853507]

















In [89]:



    


k = []
for i in range(0, len(y_test)):
    k.append(np.mean(np.sqrt(np.square(y_test_org[i][35:37] - y_pred_org[i][35:37]))))

k_mean = np.mean(k)

print(k_mean)
print()
print(k)



    


















    






151.9125

[187.49999999999864, 178.79999999999797, 111.0999999999899, 130.25000000000284]

















In [96]:



    


# 49 second predicts 56 second
k = []
for i in range(0, len(y_test)):
    k.append(np.mean(np.sqrt(np.square(y_test_org[i][34:35] - y_pred_org[i][34:35]))))

k_mean = np.mean(k)

print(k_mean)
print()
print(k)



    


















    






218.3

[266.59999999999673, 253.39999999999861, 132.29999999999518, 220.90000000000782]

















In [97]:



    


# 50 second predicts 57 second
k = []
for i in range(0, len(y_test)):
    k.append(np.mean(np.sqrt(np.square(y_test_org[i][35:36] - y_pred_org[i][35:36]))))

k_mean = np.mean(k)

print(k_mean)
print()
print(k)



    


















    






149.725

[220.699999999998, 169.39999999999827, 84.299999999991087, 124.50000000000296]

















In [98]:



    


# 51 second predicts 58 second
k = []
for i in range(0, len(y_test)):
    k.append(np.mean(np.sqrt(np.square(y_test_org[i][36:37] - y_pred_org[i][36:37]))))

k_mean = np.mean(k)

print(k_mean)
print()
print(k)



    


















    






154.1

[154.29999999999927, 188.19999999999766, 137.89999999998872, 136.00000000000273]

















In [91]:



    


plt.plot(y_test_org[0])
plt.plot(y_pred_org[0])



    


















    
Out[91]:








[<matplotlib.lines.Line2D at 0x7f9fd10470f0>]










    






/home/user/env_py3/lib/python3.5/site-packages/matplotlib/font_manager.py:1297: UserWarning: findfont: Font family ['sans-serif'] not found. Falling back to DejaVu Sans
  (prop.get_family(), self.defaultFamily[fontext]))










    
























In [92]:



    


plt.plot(k)



    


















    
Out[92]:








[<matplotlib.lines.Line2D at 0x7f9fd14f6438>]










    






/home/user/env_py3/lib/python3.5/site-packages/matplotlib/font_manager.py:1297: UserWarning: findfont: Font family ['sans-serif'] not found. Falling back to DejaVu Sans
  (prop.get_family(), self.defaultFamily[fontext]))










    
























In [ ]:



    






    











In [ ]:

Model Feature Importances:





In [93]:



    


def util_feature_importances(classifier):
    print(classifier)
    dict_importance ={}
    for i in range(len(X_col)):
        dict_importance[X_col[i]] = classifier.feature_importances_[i]
        dict_importance_sort = sorted(dict_importance.items(), key=operator.itemgetter(1), reverse=True)
    return dict_importance_sort



    











In [94]:



    


util_feature_importances(classifier_ET)



    


















    






ExtraTreesRegressor(bootstrap=False, criterion='mse', max_depth=None,
          max_features='auto', max_leaf_nodes=None,
          min_impurity_split=1e-07, min_samples_leaf=1,
          min_samples_split=2, min_weight_fraction_leaf=0.0,
          n_estimators=1000, n_jobs=1, oob_score=False,
          random_state=<mtrand.RandomState object at 0x7f9fd103d8b8>,
          verbose=0, warm_start=False)










    
Out[94]:








[('increment-price', 0.18294549102800434),
 ('increment-price-prev1sec', 0.105402142267844),
 ('increment-price-mv2sec', 0.083837879547327035),
 ('increment-price-mv3sec', 0.062382040047606266),
 ('increment-price-prev2sec', 0.057690328276618276),
 ('increment-price-mv4sec', 0.054858414785140856),
 ('increment-price-mv5sec', 0.047279873103841213),
 ('second', 0.03790516685112006),
 ('increment-price-prev3sec', 0.036785994784607742),
 ('increment-price-mv6sec', 0.031549961484273498),
 ('increment-price-mv7sec', 0.027336924508331611),
 ('increment-price-mv8sec', 0.023419253850687174),
 ('increment-price-prev4sec', 0.020294545682587373),
 ('increment-price-mv9sec', 0.019889910112709356),
 ('increment-price-mv10sec', 0.019534450973090974),
 ('increment-price_m1', 0.017256270873197553),
 ('increment-price-target_m1', 0.015959577554173859),
 ('increment-price-mv11sec', 0.011559900504960764),
 ('increment-price-mv13sec', 0.0080422962563815124),
 ('increment-price-mv14sec', 0.0075189035453956344),
 ('increment-price-mv12sec', 0.0073148918895939276),
 ('increment-price-mv15sec', 0.0065347322250807627),
 ('increment-price-prev5sec', 0.0064083060475180953),
 ('increment-price-prev1sec_m1', 0.005613545514259387),
 ('increment-price-mv2sec_m1', 0.0050077437509522486),
 ('increment-price_m2', 0.0040614865551303345),
 ('increment-price-prev2sec_m1', 0.0035505731441588242),
 ('increment-price-prev15sec', 0.0033445163778018584),
 ('increment-price-mv3sec_m1', 0.0032123793148919771),
 ('increment-price-target_m2', 0.0025099914149186342),
 ('increment-price-prev1sec_m2', 0.0024794799457975784),
 ('deal-early-second_m2', 0.0023490723974972077),
 ('increment-price-target_m3', 0.0021354929884488294),
 ('ratio-bid', 0.0021139364959787891),
 ('increment-price-prev6sec', 0.0021100988802971623),
 ('increment-price-mv4sec_m1', 0.0020093366231844073),
 ('increment-price-mv2sec_m2', 0.0018780907900070604),
 ('base-price15sec_m3', 0.0018476591772042456),
 ('increment-price-prev7sec', 0.0017650527071293046),
 ('volume-plate_m0_m2', 0.0017193757373332931),
 ('deal-price-avg_m2', 0.0017120185297707167),
 ('deal-price-avg_m3', 0.0015070987606420124),
 ('d-avg-low-price_m2', 0.0014708279236036799),
 ('increment-price-prev14sec', 0.0013905993133820737),
 ('base-price15sec_m1', 0.0012399457335588631),
 ('increment-price-prev2sec_m2', 0.0011908244539672777),
 ('volume-plate_m0_m1', 0.0011521346185752925),
 ('ratio-bid_m0_m1', 0.0011201291799760346),
 ('volume-plate_m0', 0.0011025634458052863),
 ('increment-price-mv3sec_m2', 0.001095056836847204),
 ('deal-price-avg', 0.0010207886754407915),
 ('increment-price-prev13sec_m2', 0.00096923020822686963),
 ('month', 0.00095103659464793999),
 ('base-price15sec', 0.00089678409598121969),
 ('increment-price-prev8sec', 0.00084213231649510887),
 ('volume-plate', 0.00081812036210153146),
 ('increment-price-prev9sec', 0.00080076784611740582),
 ('increment-price-prev15sec_m3', 0.00078024073627103261),
 ('increment-price-prev14sec_m2', 0.00077851299639896798),
 ('increment-price-prev15sec_m2', 0.00077784764436333788),
 ('increment-price-mv9sec_m1', 0.0007631306023950608),
 ('deal-price-avg_m1', 0.00075554475620501433),
 ('increment-price-prev11sec_m2', 0.00074692569575010559),
 ('base-price15sec_m2', 0.00074514418245183456),
 ('increment-price-mv8sec_m1', 0.00073384774622020654),
 ('increment-price-prev12sec_m2', 0.00069314373181263861),
 ('ratio-bid_m0_m2', 0.00067540202806073665),
 ('d-avg-low-price', 0.00067069038395540571),
 ('increment-price-mv4sec_m2', 0.00066253955077126808),
 ('increment-price-prev13sec', 0.00065160523630056612),
 ('increment-price-prev4sec_m3', 0.00062366588403944909),
 ('volume-plate_m0_m3', 0.00061713787510175855),
 ('d-avg-low-price_m1', 0.00058409548264678004),
 ('ratio-bid_m0', 0.0005706990265636233),
 ('increment-price-mv10sec_m3', 0.00056716947294113533),
 ('deal-early-second', 0.00056314260899548293),
 ('increment-price-prev10sec_m2', 0.00055872035720311026),
 ('increment-price-mv12sec_m3', 0.00055420527292492517),
 ('increment-price-prev3sec_m2', 0.00055277043068829585),
 ('increment-price-mv9sec_m3', 0.00054854696757201536),
 ('increment-price-prev6sec_m2', 0.00053519631488971504),
 ('increment-price-mv8sec_m3', 0.00052908068312536327),
 ('increment-price-mv7sec_m3', 0.00052862687426850268),
 ('increment-price-mv5sec_m2', 0.00052244830325003197),
 ('increment-price-mv6sec_m3', 0.00052055254065726275),
 ('increment-price-prev10sec', 0.0005099812161959309),
 ('increment-price-mv11sec_m3', 0.00049704458132380494),
 ('ratio-bid_m0_m3', 0.00049525135103882588),
 ('increment-price-mv15sec_m2', 0.00049422027040089739),
 ('increment-price_m3', 0.00048940232753904341),
 ('increment-price-mv13sec_m2', 0.00047866375730288744),
 ('increment-price-mv14sec_m2', 0.00047792131151159914),
 ('increment-price-prev15sec_m1', 0.0004777668292510698),
 ('increment-price-prev7sec_m2', 0.00046918916754038788),
 ('increment-price-prev3sec_m1', 0.00045831626065427931),
 ('deal-early-second_m1', 0.00045204816452371343),
 ('increment-price-prev14sec_m3', 0.00045142132979124008),
 ('increment-price-mv13sec_m3', 0.00044959770332597579),
 ('increment-price-mv14sec_m3', 0.00044662869447710074),
 ('increment-price-mv12sec_m2', 0.00044381298010512583),
 ('increment-price-prev7sec_m3', 0.00044271182613983231),
 ('increment-price-mv5sec_m3', 0.00042342071991812396),
 ('increment-price-mv15sec_m3', 0.00042303455307412938),
 ('increment-price-prev2sec_m3', 0.00041867121257052224),
 ('increment-price-prev12sec', 0.0004162491904867499),
 ('increment-price-prev8sec_m2', 0.0004130535584700903),
 ('increment-price-mv11sec_m2', 0.00040652584704866876),
 ('increment-price-prev9sec_m2', 0.00040202076524700509),
 ('increment-price-mv4sec_m3', 0.00040157047163028243),
 ('increment-price-mv9sec_m2', 0.00040005878955338888),
 ('increment-price-mv10sec_m2', 0.00039652923293149133),
 ('increment-price-prev5sec_m3', 0.00039641094782907508),
 ('increment-price-prev5sec_m2', 0.0003953595973168542),
 ('increment-price-prev4sec_m2', 0.00038980689631666323),
 ('increment-price-mv6sec_m2', 0.00037817427242082722),
 ('increment-price-mv2sec_m3', 0.00037753111040926714),
 ('increment-price-mv8sec_m2', 0.00036984133647415587),
 ('increment-price-prev1sec_m3', 0.00036546565583034157),
 ('deal-early-second_m3', 0.00036372647275916128),
 ('increment-price-mv7sec_m2', 0.00035143659848571352),
 ('increment-price-prev8sec_m1', 0.00034331894352192125),
 ('increment-price-prev9sec_m1', 0.00033906037185127522),
 ('increment-price-prev8sec_m3', 0.00033858584233680033),
 ('increment-price-prev6sec_m3', 0.00033285569323726735),
 ('d-avg-low-price_m3', 0.00032164623438691616),
 ('increment-price-mv3sec_m3', 0.00032011862124602894),
 ('increment-price-prev11sec_m1', 0.00031910274055455592),
 ('increment-price-prev13sec_m1', 0.0003188858598114887),
 ('increment-price-prev14sec_m1', 0.00031853609917488349),
 ('increment-price-prev12sec_m1', 0.00030522081100442348),
 ('increment-price-prev6sec_m1', 0.00029283243678640748),
 ('increment-price-prev11sec', 0.00029233094508057661),
 ('increment-price-mv5sec_m1', 0.00028553544832215415),
 ('increment-price-mv11sec_m1', 0.00028334219009392721),
 ('increment-price-mv12sec_m1', 0.00028142587178943548),
 ('increment-price-mv14sec_m1', 0.00027403850656429929),
 ('increment-price-mv15sec_m1', 0.00027397020979941824),
 ('increment-price-prev7sec_m1', 0.0002725550374487783),
 ('increment-price-mv6sec_m1', 0.0002701888017743156),
 ('increment-price-mv13sec_m1', 0.0002698705815302529),
 ('increment-price-mv10sec_m1', 0.00025642810749696945),
 ('increment-price-mv7sec_m1', 0.00025495755642367937),
 ('increment-price-prev13sec_m3', 0.00025047918653189324),
 ('increment-price-prev9sec_m3', 0.00024641086738164819),
 ('increment-price-prev10sec_m1', 0.00023456332907039485),
 ('increment-price-prev11sec_m3', 0.0002294459773553411),
 ('increment-price-prev10sec_m3', 0.0002145402562077868),
 ('increment-price-prev4sec_m1', 0.00019790464886721822),
 ('increment-price-prev12sec_m3', 0.00019706927899359393),
 ('increment-price-prev3sec_m3', 0.00018911599495918954),
 ('increment-price-prev5sec_m1', 0.00014701825445721567)]

















In [ ]:



    






    











In [ ]:



    






    











In [ ]:



    






    











In [ ]:



    






    











In [ ]:



    






    











In [ ]:



    






    











In [ ]:



    






    











In [ ]:

The End

Content source: telescopeuser/uat_shl

Similar notebooks:

notebook.community | gallery | about