In [1]:
%matplotlib inline

import math
import pytz 
import traceback
import time
import pandas as pd
import numpy as np
import seaborn as sns
from datetime import datetime
import matplotlib.pyplot as plt
import cPickle as pickle

In [2]:
%run src/data/helper.py

In [3]:
start_time = time.time()

with open('data/parsed/stations_dataset_final.p', 'rb') as f:
    stations = pickle.load(f)

with open('data/parsed/readings_clean.p', "rb") as f:
    readings = pickle.load(f)

end_time = time.time()
print 'Opening redistribution data took %s' % (end_time - start_time)


Opening redistribution data took 279.28757906

Split Dataset


In [4]:
split_training = lambda df: df[datetime(2016,5,15,0,0,0,0):datetime(2016,6,12,23,59,59,999999)]
split_validation = lambda df: df[datetime(2016,6,13,0,0,0,0):datetime(2016,6,19,23,59,59,999999)]
split_test = lambda df: df[datetime(2016,5,20,0,0,0,0):datetime(2016,6,26,23,59,59,999999)]

In [5]:
def split_datasets(df, station_id):
    station_df = df.loc[station_id]
    training = split_training(station_df)
    validation = split_validation(station_df)
    test = split_test(station_df)
    
    return training, validation, test

Model Definitions


In [6]:
import sys

def clip_and_round(arr):
    arr = np.clip(arr, 0, sys.maxint)
    return np.round(arr)

In [7]:
import inspect

from sklearn.base import BaseEstimator, RegressorMixin
from sklearn.metrics import mean_squared_error

from rpy2 import robjects
from rpy2.robjects import pandas2ri
from rpy2.robjects.packages import importr
from rpy2.robjects import IntVector, Formula

pandas2ri.activate()

r = robjects.r
base = importr('base')
stats = importr('stats')
mgcv = importr('mgcv')

class GAMRegressor(BaseEstimator, RegressorMixin):      
    last_data = None

    def __init__(self, features=None, formula_str=None,
                 FogTMinus2_parametric=None, RainTMinus2_parametric=None,
                 DistNbBikes_parametric=None, CollNbBikes_parametric=None, 
                 TempTMinus2AndHumidityTMinus2_sp=None,
                 NbBikesTMinus2_sp=None, NbBikesTMinus3_sp=None, 
                 NbBikesTMinus12_sp=None, NbBikesTMinus18_sp=None,             
                 TimeOfDay_1_sp=None, TimeOfDay_2_sp=None, TimeOfDay_3_sp=None,
                 TimeOfDay_1_by=None, TimeOfDay_2_by=None, TimeOfDay_3_by=None):
        
        args, _, _, values = inspect.getargvalues(inspect.currentframe())
        values.pop("self")

        self.args_to_set = []
        for arg, val in values.items():
            # save a list of the arguments
            if arg != 'features' and arg != 'formula_str':
                self.args_to_set.append(arg)
            setattr(self, arg, val)

    def fit(self, X, y=None): 
        if self.formula_str is None:
            features_dicts = self.build_features_dicts()
            self.formula_str = self.build_formula_str(features_dicts)       
            
        GAMRegressor.last_data=X
            
        frm = Formula(self.formula_str)
        self.gam = mgcv.gam(frm, data=X)
        
        return self

    def predict(self, X):
        assert (self.gam is not None), "GAM must be set"
        p_val = clip_and_round(stats.predict(self.gam, newdata=X))
        return p_val
    
    def score(self, X):
        p_val = self.predict(X)
        y_val = X.NbBikes
        rmse = mean_squared_error(y_val, p_val)**0.5
        return rmse * (-1)
    
    def build_features_dicts(self):
        assert (self.features is not None), "features must be set"
        
        # initialize the dictionaries
        features_dicts = {}
        for feature in self.features:
            features_dicts[feature] = {
                'name': feature,
                'bs': 'tp',
                'sp': None,
                'by': None,
                'k': None,
                'parametric': False
            }
            
        # set parameter values
        for arg in self.args_to_set:
            val = getattr(self, arg)
            if val is None:
                continue
            feature, parameter = arg.rsplit('_',1)
            features_dicts[feature][parameter] = val
            
        return features_dicts
    
    def build_formula_str(self, features_dicts):
        formula = 'NbBikes ~ '
        for feature, feature_dict in features_dicts.iteritems():
            if feature_dict['parametric']:
                formula += '%(name)s+' % feature_dict
                continue
                                
            tokens = feature_dict['name'].split('_')
            name, index = (tokens[0],None) if len(tokens) == 1 else (tokens[0], tokens[1])
            formula += "s(%s" % name.replace('And', ',')
            
            if feature_dict['bs'] is not None:
                formula += ", bs='%s'" % feature_dict['bs']
            if feature_dict['sp'] is not None:
                formula += ", sp=%f" % feature_dict['sp']
            if feature_dict['by'] is not None:
                formula += ", by=%s" % feature_dict['by']
            if feature_dict['k'] is not None:
                formula += ", k=%s" % feature_dict['k']
                
            formula += ")+" % feature_dict
        return formula[:-1]
    
class LRegressor(BaseEstimator, RegressorMixin):  
    def __init__(self, formula_str):
    	self.formula_str = formula_str

    def fit(self, X, y=None):            
        self.lr = stats.lm(Formula(self.formula_str), data=X)        
        return self

    def predict(self, X):
        assert (self.lr is not None), "LR must be set"
        p_val = clip_and_round(stats.predict(self.lr, newdata=X))
        return p_val
    
    def score(self, X):
        p_val = self.predict(X)
        y_val = X.NbBikes
        rmse = mean_squared_error(y_val, p_val)**0.5
        return rmse * (-1)

In [8]:
from sklearn.linear_model import LinearRegression

def fit_and_predict_lm(training, validation, test, formula):
    lm = LRegressor(formula_str=formula)
    lm.fit(training)
    return lm, clip_and_round(lm.predict(training)), clip_and_round(lm.predict(validation)), clip_and_round(lm.predict(test))

In [9]:
def fit_and_predict_gam(training, validation, test, formula):
    gam = GAMRegressor(formula_str=formula)
    gam.fit(training)
    return gam, clip_and_round(gam.predict(training)), clip_and_round(gam.predict(validation)), clip_and_round(gam.predict(test))

In [10]:
def fit_and_predict_hist_avg(training, validation, test):
    return None, training.HistAvg, validation.HistAvg, test.HistAvg

In [11]:
def model(df, station_ids, gam_formula, lr_formula, pred_col):
    results = []

    for station_id in station_ids:
        print 'Fitting %s' % station_id
            
        training, validation, test = split_datasets(df, station_id)      
        t_train = training[pred_col]
        t_validation = validation[pred_col]
        t_test = test[pred_col]
        
        try:            
            # Linear Model
            lm_fit = fit_and_predict_lm(training, validation, test, lr_formula)
            lm_rmse_train = mean_squared_error(t_train, lm_fit[1])**0.5
            lm_rmse_val = mean_squared_error(t_validation, lm_fit[2])**0.5
            lm_rmse_test = mean_squared_error(t_test, lm_fit[3])**0.5
        
            # GAM Model
            gam_fit = fit_and_predict_gam(training, validation, test, gam_formula)
            gam_rmse_train = mean_squared_error(t_train, gam_fit[1])**0.5
            gam_rmse_val = mean_squared_error(t_validation, gam_fit[2])**0.5
            gam_rmse_test = mean_squared_error(t_test, gam_fit[3])**0.5
            
            # Hist Avg
            havg_fit = fit_and_predict_hist_avg(training, validation, test)
            havg_rmse_train = mean_squared_error(t_train, havg_fit[1])**0.5
            havg_rmse_val = mean_squared_error(t_validation, havg_fit[2])**0.5
            havh_rmse_test = mean_squared_error(t_test, havg_fit[3])**0.5
        except Exception as e:
            logging.error(traceback.format_exc())
        
        results.append({'Id': station_id, 
                        'LR-TRAIN-ERR': lm_rmse_train, 'LR-VAL-ERR': lm_rmse_val, 'LR-TEST-ERR': lm_rmse_test,
                        'GAM-TRAIN-ERR': gam_rmse_train, 'GAM-VAL-ERR': gam_rmse_val, 'GAM-TEST-ERR': gam_rmse_test,
                        'HAVG-TRAIN-ERR': havg_rmse_train, 'HAVG-VAL-ERR': havg_rmse_val, 'HAVG-TEST-ERR': havh_rmse_test,#})
                        'LR-MOD': lm_fit[0], 'GAM-MOD': gam_fit[0]})
        
    return results

In [12]:
def convert_results_to_df(results, name):
    dfs = [pd.DataFrame(result).set_index('Id') for result in results]
    for i,df in enumerate(dfs):
        df.columns = ['%s-EXP%d-%s' % (name, i,col) for col in df.columns]
    return pd.concat(dfs, axis=1)

Use Samples?


In [13]:
stations_to_use = readings.index.get_level_values(0).unique().tolist()

Short Term Predictions


In [14]:
gam_formula_short = "NbBikes ~ s(TempTMinus2, HumidityTMinus2, bs='tp', sp=30.0) + s(TimeOfDay, by=Weekday, bs='tp', sp=1.1) "  
gam_formula_short += "+ s(TimeOfDay, by=Weekend, bs='tp', sp=50.0) + s(TimeOfDay, by=Holiday, bs='tp', sp=0.2) + s(NbBikesTMinus2, bs='tp', sp=8.0) "
gam_formula_short += "+ s(NbBikesTMinus3, bs='tp', sp=11.0) + RainTMinus2 + FogTMinus2 "

In [15]:
lr_formula_short = "NbBikes ~ TempTMinus2 + HumidityTMinus2 + TimeOfDay + Weekday "
lr_formula_short += "+ NbBikesTMinus2 + NbBikesTMinus3 + RainTMinus2 + FogTMinus2 "

Baseline


In [16]:
# choose the columns to use in the model
boolean_cols_short = ['Weekday', 'Weekend', 'Holiday', 'RainTMinus2', 'FogTMinus2']
numeric_cols_short = ['HumidityTMinus2', 'TempTMinus2', 'TimeOfDay',
                      'NbBikesTMinus2', 'NbBikesTMinus3', 'HistAvg']                       
pred_col_short = 'NbBikes'

feature_cols_short = numeric_cols_short + boolean_cols_short
cols_short = [pred_col_short] + feature_cols_short

# select the columns chosen columns
readings_short = readings.loc[stations_to_use][cols_short]

# remove na
readings_short.dropna(inplace=True)

In [17]:
baseline_short = [model(readings_short, stations_to_use, gam_formula_short, lr_formula_short, 'NbBikes')]


Fitting BikePoints_1
Fitting BikePoints_10
Fitting BikePoints_100
Fitting BikePoints_101
Fitting BikePoints_102
Fitting BikePoints_103
Fitting BikePoints_104
Fitting BikePoints_105
Fitting BikePoints_106
Fitting BikePoints_107
Fitting BikePoints_108
Fitting BikePoints_109
Fitting BikePoints_11
Fitting BikePoints_110
Fitting BikePoints_111
Fitting BikePoints_112
Fitting BikePoints_113
Fitting BikePoints_114
Fitting BikePoints_115
Fitting BikePoints_116
Fitting BikePoints_117
Fitting BikePoints_118
Fitting BikePoints_119
Fitting BikePoints_12
Fitting BikePoints_120
Fitting BikePoints_121
Fitting BikePoints_122
Fitting BikePoints_123
Fitting BikePoints_124
Fitting BikePoints_125
Fitting BikePoints_126
Fitting BikePoints_127
Fitting BikePoints_128
Fitting BikePoints_129
Fitting BikePoints_13
Fitting BikePoints_130
Fitting BikePoints_131
Fitting BikePoints_132
Fitting BikePoints_133
Fitting BikePoints_134
Fitting BikePoints_135
Fitting BikePoints_136
Fitting BikePoints_137
Fitting BikePoints_138
Fitting BikePoints_139
Fitting BikePoints_14
Fitting BikePoints_140
Fitting BikePoints_141
Fitting BikePoints_142
Fitting BikePoints_143
Fitting BikePoints_144
Fitting BikePoints_145
Fitting BikePoints_146
Fitting BikePoints_147
Fitting BikePoints_148
Fitting BikePoints_149
Fitting BikePoints_15
Fitting BikePoints_150
Fitting BikePoints_151
Fitting BikePoints_152
Fitting BikePoints_153
Fitting BikePoints_154
Fitting BikePoints_155
Fitting BikePoints_156
Fitting BikePoints_157
Fitting BikePoints_158
Fitting BikePoints_159
Fitting BikePoints_16
Fitting BikePoints_160
Fitting BikePoints_161
Fitting BikePoints_162
Fitting BikePoints_163
Fitting BikePoints_164
Fitting BikePoints_165
Fitting BikePoints_166
Fitting BikePoints_167
Fitting BikePoints_168
Fitting BikePoints_169
Fitting BikePoints_17
Fitting BikePoints_170
Fitting BikePoints_171
Fitting BikePoints_172
Fitting BikePoints_173
Fitting BikePoints_174
Fitting BikePoints_175
Fitting BikePoints_176
Fitting BikePoints_177
Fitting BikePoints_178
Fitting BikePoints_18
Fitting BikePoints_180
Fitting BikePoints_181
Fitting BikePoints_182
Fitting BikePoints_183
Fitting BikePoints_184
Fitting BikePoints_185
Fitting BikePoints_186
Fitting BikePoints_187
Fitting BikePoints_188
Fitting BikePoints_189
Fitting BikePoints_19
Fitting BikePoints_190
Fitting BikePoints_191
Fitting BikePoints_192
Fitting BikePoints_193
Fitting BikePoints_194
Fitting BikePoints_195
Fitting BikePoints_196
Fitting BikePoints_197
Fitting BikePoints_199
Fitting BikePoints_2
Fitting BikePoints_20
Fitting BikePoints_200
Fitting BikePoints_201
Fitting BikePoints_202
Fitting BikePoints_203
Fitting BikePoints_204
Fitting BikePoints_206
Fitting BikePoints_207
Fitting BikePoints_208
Fitting BikePoints_209
Fitting BikePoints_21
Fitting BikePoints_210
Fitting BikePoints_211
Fitting BikePoints_212
Fitting BikePoints_213
Fitting BikePoints_214
Fitting BikePoints_215
Fitting BikePoints_216
Fitting BikePoints_217
Fitting BikePoints_218
Fitting BikePoints_219
Fitting BikePoints_22
Fitting BikePoints_220
Fitting BikePoints_221
Fitting BikePoints_222
Fitting BikePoints_223
Fitting BikePoints_224
Fitting BikePoints_225
Fitting BikePoints_226
Fitting BikePoints_227
Fitting BikePoints_228
Fitting BikePoints_229
Fitting BikePoints_23
Fitting BikePoints_230
Fitting BikePoints_231
Fitting BikePoints_232
Fitting BikePoints_233
Fitting BikePoints_234
Fitting BikePoints_235
Fitting BikePoints_236
Fitting BikePoints_238
Fitting BikePoints_239
Fitting BikePoints_24
Fitting BikePoints_240
Fitting BikePoints_242
Fitting BikePoints_243
Fitting BikePoints_244
Fitting BikePoints_245
Fitting BikePoints_246
Fitting BikePoints_247
Fitting BikePoints_248
Fitting BikePoints_249
Fitting BikePoints_25
Fitting BikePoints_250
Fitting BikePoints_251
Fitting BikePoints_252
Fitting BikePoints_253
Fitting BikePoints_254
Fitting BikePoints_255
Fitting BikePoints_256
Fitting BikePoints_257
Fitting BikePoints_258
Fitting BikePoints_26
Fitting BikePoints_260
Fitting BikePoints_261
Fitting BikePoints_262
Fitting BikePoints_263
Fitting BikePoints_264
Fitting BikePoints_265
Fitting BikePoints_266
Fitting BikePoints_267
Fitting BikePoints_268
Fitting BikePoints_269
Fitting BikePoints_27
Fitting BikePoints_270
Fitting BikePoints_271
Fitting BikePoints_272
Fitting BikePoints_273
Fitting BikePoints_274
Fitting BikePoints_275
Fitting BikePoints_276
Fitting BikePoints_277
Fitting BikePoints_278
Fitting BikePoints_279
Fitting BikePoints_28
Fitting BikePoints_280
Fitting BikePoints_281
Fitting BikePoints_282
Fitting BikePoints_284
Fitting BikePoints_286
Fitting BikePoints_287
Fitting BikePoints_288
Fitting BikePoints_289
Fitting BikePoints_29
Fitting BikePoints_290
Fitting BikePoints_291
Fitting BikePoints_292
Fitting BikePoints_293
Fitting BikePoints_294
Fitting BikePoints_295
Fitting BikePoints_296
Fitting BikePoints_297
Fitting BikePoints_298
Fitting BikePoints_299
Fitting BikePoints_3
Fitting BikePoints_30
Fitting BikePoints_300
Fitting BikePoints_301
Fitting BikePoints_302
Fitting BikePoints_303
Fitting BikePoints_304
Fitting BikePoints_305
Fitting BikePoints_306
Fitting BikePoints_307
Fitting BikePoints_308
Fitting BikePoints_309
Fitting BikePoints_31
Fitting BikePoints_310
Fitting BikePoints_311
Fitting BikePoints_312
Fitting BikePoints_313
Fitting BikePoints_314
Fitting BikePoints_315
Fitting BikePoints_316
Fitting BikePoints_317
Fitting BikePoints_318
Fitting BikePoints_319
Fitting BikePoints_32
Fitting BikePoints_320
Fitting BikePoints_321
Fitting BikePoints_322
Fitting BikePoints_323
Fitting BikePoints_324
Fitting BikePoints_325
Fitting BikePoints_326
Fitting BikePoints_327
Fitting BikePoints_328
Fitting BikePoints_329
Fitting BikePoints_33
Fitting BikePoints_330
Fitting BikePoints_331
Fitting BikePoints_332
Fitting BikePoints_333
Fitting BikePoints_334
Fitting BikePoints_335
Fitting BikePoints_336
Fitting BikePoints_337
Fitting BikePoints_338
Fitting BikePoints_339
Fitting BikePoints_34
Fitting BikePoints_340
Fitting BikePoints_341
Fitting BikePoints_343
Fitting BikePoints_344
Fitting BikePoints_345
Fitting BikePoints_347
Fitting BikePoints_348
Fitting BikePoints_349
Fitting BikePoints_350
Fitting BikePoints_351
Fitting BikePoints_352
Fitting BikePoints_353
Fitting BikePoints_354
Fitting BikePoints_355
Fitting BikePoints_356
Fitting BikePoints_357
Fitting BikePoints_359
Fitting BikePoints_36
Fitting BikePoints_360
Fitting BikePoints_361
Fitting BikePoints_362
Fitting BikePoints_363
Fitting BikePoints_364
Fitting BikePoints_365
Fitting BikePoints_366
Fitting BikePoints_367
Fitting BikePoints_368
Fitting BikePoints_37
Fitting BikePoints_370
Fitting BikePoints_371
Fitting BikePoints_372
Fitting BikePoints_373
Fitting BikePoints_374
Fitting BikePoints_375
Fitting BikePoints_376
Fitting BikePoints_377
Fitting BikePoints_378
Fitting BikePoints_379
Fitting BikePoints_38
Fitting BikePoints_380
Fitting BikePoints_381
Fitting BikePoints_382
Fitting BikePoints_383
Fitting BikePoints_384
Fitting BikePoints_386
Fitting BikePoints_387
Fitting BikePoints_388
Fitting BikePoints_389
Fitting BikePoints_39
Fitting BikePoints_390
Fitting BikePoints_391
Fitting BikePoints_392
Fitting BikePoints_393
Fitting BikePoints_394
Fitting BikePoints_395
Fitting BikePoints_396
Fitting BikePoints_397
Fitting BikePoints_398
Fitting BikePoints_399
Fitting BikePoints_4
Fitting BikePoints_40
Fitting BikePoints_400
Fitting BikePoints_401
Fitting BikePoints_402
Fitting BikePoints_403
Fitting BikePoints_404
Fitting BikePoints_405
Fitting BikePoints_406
Fitting BikePoints_407
Fitting BikePoints_408
Fitting BikePoints_409
Fitting BikePoints_41
Fitting BikePoints_410
Fitting BikePoints_411
Fitting BikePoints_412
Fitting BikePoints_419
Fitting BikePoints_42
Fitting BikePoints_420
Fitting BikePoints_421
Fitting BikePoints_423
Fitting BikePoints_424
Fitting BikePoints_425
Fitting BikePoints_426
Fitting BikePoints_427
Fitting BikePoints_428
Fitting BikePoints_43
Fitting BikePoints_430
Fitting BikePoints_431
Fitting BikePoints_432
Fitting BikePoints_433
Fitting BikePoints_435
Fitting BikePoints_436
Fitting BikePoints_437
Fitting BikePoints_44
Fitting BikePoints_440
Fitting BikePoints_441
Fitting BikePoints_442
Fitting BikePoints_443
Fitting BikePoints_444
Fitting BikePoints_445
Fitting BikePoints_446
Fitting BikePoints_447
Fitting BikePoints_448
Fitting BikePoints_449
Fitting BikePoints_45
Fitting BikePoints_450
Fitting BikePoints_451
Fitting BikePoints_452
Fitting BikePoints_453
Fitting BikePoints_454
Fitting BikePoints_455
Fitting BikePoints_456
Fitting BikePoints_457
Fitting BikePoints_458
Fitting BikePoints_459
Fitting BikePoints_46
Fitting BikePoints_460
Fitting BikePoints_461
Fitting BikePoints_462
Fitting BikePoints_463
Fitting BikePoints_464
Fitting BikePoints_465
Fitting BikePoints_466
Fitting BikePoints_467
Fitting BikePoints_468
Fitting BikePoints_469
Fitting BikePoints_47
Fitting BikePoints_470
Fitting BikePoints_471
Fitting BikePoints_472
Fitting BikePoints_473
Fitting BikePoints_474
Fitting BikePoints_475
Fitting BikePoints_476
Fitting BikePoints_477
Fitting BikePoints_478
Fitting BikePoints_479
Fitting BikePoints_48
Fitting BikePoints_480
Fitting BikePoints_481
Fitting BikePoints_482
Fitting BikePoints_483
Fitting BikePoints_484
Fitting BikePoints_485
Fitting BikePoints_486
Fitting BikePoints_487
Fitting BikePoints_488
Fitting BikePoints_489
Fitting BikePoints_49
Fitting BikePoints_490
Fitting BikePoints_491
Fitting BikePoints_492
Fitting BikePoints_494
Fitting BikePoints_495
Fitting BikePoints_496
Fitting BikePoints_498
Fitting BikePoints_499
Fitting BikePoints_5
Fitting BikePoints_50
Fitting BikePoints_500
Fitting BikePoints_501
Fitting BikePoints_503
Fitting BikePoints_504
Fitting BikePoints_505
Fitting BikePoints_506
Fitting BikePoints_507
Fitting BikePoints_508
Fitting BikePoints_509
Fitting BikePoints_51
Fitting BikePoints_510
Fitting BikePoints_511
Fitting BikePoints_512
Fitting BikePoints_513
Fitting BikePoints_514
Fitting BikePoints_515
Fitting BikePoints_516
Fitting BikePoints_517
Fitting BikePoints_518
Fitting BikePoints_519
Fitting BikePoints_52
Fitting BikePoints_520
Fitting BikePoints_521
Fitting BikePoints_522
Fitting BikePoints_523
Fitting BikePoints_524
Fitting BikePoints_526
Fitting BikePoints_527
Fitting BikePoints_528
Fitting BikePoints_529
Fitting BikePoints_53
Fitting BikePoints_530
Fitting BikePoints_531
Fitting BikePoints_532
Fitting BikePoints_533
Fitting BikePoints_534
Fitting BikePoints_535
Fitting BikePoints_536
Fitting BikePoints_537
Fitting BikePoints_538
Fitting BikePoints_539
Fitting BikePoints_54
Fitting BikePoints_540
Fitting BikePoints_541
Fitting BikePoints_542
Fitting BikePoints_543
Fitting BikePoints_544
Fitting BikePoints_545
Fitting BikePoints_546
Fitting BikePoints_547
Fitting BikePoints_548
Fitting BikePoints_549
Fitting BikePoints_55
Fitting BikePoints_550
Fitting BikePoints_551
Fitting BikePoints_552
Fitting BikePoints_553
Fitting BikePoints_554
Fitting BikePoints_555
Fitting BikePoints_556
Fitting BikePoints_557
Fitting BikePoints_558
Fitting BikePoints_559
Fitting BikePoints_56
Fitting BikePoints_560
Fitting BikePoints_561
Fitting BikePoints_562
Fitting BikePoints_563
Fitting BikePoints_564
Fitting BikePoints_565
Fitting BikePoints_566
Fitting BikePoints_568
Fitting BikePoints_569
Fitting BikePoints_57
Fitting BikePoints_570
Fitting BikePoints_571
Fitting BikePoints_572
Fitting BikePoints_573
Fitting BikePoints_574
Fitting BikePoints_576
Fitting BikePoints_577
Fitting BikePoints_578
Fitting BikePoints_579
Fitting BikePoints_58
Fitting BikePoints_580
Fitting BikePoints_581
Fitting BikePoints_583
Fitting BikePoints_584
Fitting BikePoints_586
Fitting BikePoints_587
Fitting BikePoints_588
Fitting BikePoints_589
Fitting BikePoints_59
Fitting BikePoints_590
Fitting BikePoints_591
Fitting BikePoints_592
Fitting BikePoints_593
Fitting BikePoints_594
Fitting BikePoints_595
Fitting BikePoints_596
Fitting BikePoints_597
Fitting BikePoints_598
Fitting BikePoints_599
Fitting BikePoints_6
Fitting BikePoints_60
Fitting BikePoints_600
Fitting BikePoints_601
Fitting BikePoints_602
Fitting BikePoints_603
Fitting BikePoints_604
Fitting BikePoints_605
Fitting BikePoints_606
Fitting BikePoints_607
Fitting BikePoints_608
Fitting BikePoints_609
Fitting BikePoints_61
Fitting BikePoints_610
Fitting BikePoints_611
Fitting BikePoints_612
Fitting BikePoints_613
Fitting BikePoints_614
Fitting BikePoints_615
Fitting BikePoints_616
Fitting BikePoints_617
Fitting BikePoints_618
Fitting BikePoints_619
Fitting BikePoints_62
Fitting BikePoints_620
Fitting BikePoints_621
Fitting BikePoints_622
Fitting BikePoints_623
Fitting BikePoints_624
Fitting BikePoints_625
Fitting BikePoints_626
Fitting BikePoints_627
Fitting BikePoints_628
Fitting BikePoints_629
Fitting BikePoints_63
Fitting BikePoints_630
Fitting BikePoints_631
Fitting BikePoints_632
Fitting BikePoints_633
Fitting BikePoints_634
Fitting BikePoints_635
Fitting BikePoints_636
Fitting BikePoints_637
Fitting BikePoints_638
Fitting BikePoints_639
Fitting BikePoints_64
Fitting BikePoints_640
Fitting BikePoints_641
Fitting BikePoints_642
Fitting BikePoints_643
Fitting BikePoints_644
Fitting BikePoints_645
Fitting BikePoints_646
Fitting BikePoints_647
Fitting BikePoints_648
Fitting BikePoints_649
Fitting BikePoints_65
Fitting BikePoints_650
Fitting BikePoints_651
Fitting BikePoints_652
Fitting BikePoints_653
Fitting BikePoints_654
Fitting BikePoints_655
Fitting BikePoints_656
Fitting BikePoints_657
Fitting BikePoints_658
Fitting BikePoints_659
Fitting BikePoints_66
Fitting BikePoints_660
Fitting BikePoints_661
Fitting BikePoints_662
Fitting BikePoints_663
Fitting BikePoints_664
Fitting BikePoints_665
Fitting BikePoints_666
Fitting BikePoints_667
Fitting BikePoints_668
Fitting BikePoints_669
Fitting BikePoints_67
Fitting BikePoints_670
Fitting BikePoints_671
Fitting BikePoints_672
Fitting BikePoints_673
Fitting BikePoints_674
Fitting BikePoints_675
Fitting BikePoints_676
Fitting BikePoints_677
Fitting BikePoints_678
Fitting BikePoints_679
Fitting BikePoints_68
Fitting BikePoints_680
Fitting BikePoints_681
Fitting BikePoints_682
Fitting BikePoints_683
Fitting BikePoints_684
Fitting BikePoints_685
Fitting BikePoints_686
Fitting BikePoints_687
Fitting BikePoints_688
Fitting BikePoints_689
Fitting BikePoints_69
Fitting BikePoints_690
Fitting BikePoints_691
Fitting BikePoints_692
Fitting BikePoints_693
Fitting BikePoints_694
Fitting BikePoints_695
Fitting BikePoints_696
Fitting BikePoints_697
Fitting BikePoints_698
Fitting BikePoints_699
Fitting BikePoints_7
Fitting BikePoints_70
Fitting BikePoints_700
Fitting BikePoints_701
Fitting BikePoints_702
Fitting BikePoints_703
Fitting BikePoints_704
Fitting BikePoints_705
Fitting BikePoints_706
Fitting BikePoints_707
Fitting BikePoints_708
Fitting BikePoints_709
Fitting BikePoints_71
Fitting BikePoints_710
Fitting BikePoints_711
Fitting BikePoints_712
Fitting BikePoints_713
Fitting BikePoints_714
Fitting BikePoints_715
Fitting BikePoints_716
Fitting BikePoints_717
Fitting BikePoints_718
Fitting BikePoints_719
Fitting BikePoints_72
Fitting BikePoints_720
Fitting BikePoints_721
Fitting BikePoints_722
Fitting BikePoints_723
Fitting BikePoints_724
Fitting BikePoints_725
Fitting BikePoints_726
Fitting BikePoints_727
Fitting BikePoints_728
Fitting BikePoints_729
Fitting BikePoints_73
Fitting BikePoints_730
Fitting BikePoints_731
Fitting BikePoints_732
Fitting BikePoints_733
Fitting BikePoints_734
Fitting BikePoints_735
Fitting BikePoints_736
Fitting BikePoints_737
Fitting BikePoints_738
Fitting BikePoints_739
Fitting BikePoints_74
Fitting BikePoints_740
Fitting BikePoints_741
Fitting BikePoints_742
Fitting BikePoints_743
Fitting BikePoints_744
Fitting BikePoints_745
Fitting BikePoints_746
Fitting BikePoints_747
Fitting BikePoints_748
Fitting BikePoints_749
Fitting BikePoints_75
Fitting BikePoints_750
Fitting BikePoints_751
Fitting BikePoints_752
Fitting BikePoints_753
Fitting BikePoints_754
Fitting BikePoints_755
Fitting BikePoints_756
Fitting BikePoints_757
Fitting BikePoints_758
Fitting BikePoints_759
Fitting BikePoints_76
Fitting BikePoints_760
Fitting BikePoints_761
Fitting BikePoints_762
Fitting BikePoints_763
Fitting BikePoints_764
Fitting BikePoints_765
Fitting BikePoints_766
Fitting BikePoints_767
Fitting BikePoints_768
Fitting BikePoints_769
Fitting BikePoints_77
Fitting BikePoints_770
Fitting BikePoints_771
Fitting BikePoints_772
Fitting BikePoints_773
Fitting BikePoints_774
Fitting BikePoints_775
Fitting BikePoints_776
Fitting BikePoints_777
Fitting BikePoints_779
Fitting BikePoints_78
Fitting BikePoints_780
Fitting BikePoints_781
Fitting BikePoints_782
Fitting BikePoints_783
Fitting BikePoints_784
Fitting BikePoints_785
Fitting BikePoints_786
Fitting BikePoints_787
Fitting BikePoints_788
Fitting BikePoints_789
Fitting BikePoints_790
Fitting BikePoints_792
Fitting BikePoints_793
Fitting BikePoints_794
Fitting BikePoints_795
Fitting BikePoints_8
Fitting BikePoints_80
Fitting BikePoints_800
Fitting BikePoints_801
Fitting BikePoints_804
Fitting BikePoints_808
Fitting BikePoints_81
Fitting BikePoints_815
Fitting BikePoints_818
Fitting BikePoints_82
Fitting BikePoints_83
Fitting BikePoints_84
Fitting BikePoints_85
Fitting BikePoints_86
Fitting BikePoints_87
Fitting BikePoints_88
Fitting BikePoints_89
Fitting BikePoints_9
Fitting BikePoints_90
Fitting BikePoints_91
Fitting BikePoints_92
Fitting BikePoints_93
Fitting BikePoints_94
Fitting BikePoints_95
Fitting BikePoints_96
Fitting BikePoints_97
Fitting BikePoints_98
Fitting BikePoints_99

In [18]:
baseline_short_df = convert_results_to_df(baseline_short, 'SHORT')

In [19]:
baseline_short_df


Out[19]:
SHORT-EXP0-GAM-TEST-ERR SHORT-EXP0-GAM-TRAIN-ERR SHORT-EXP0-GAM-VAL-ERR SHORT-EXP0-HAVG-TEST-ERR SHORT-EXP0-HAVG-TRAIN-ERR SHORT-EXP0-HAVG-VAL-ERR SHORT-EXP0-LR-TEST-ERR SHORT-EXP0-LR-TRAIN-ERR SHORT-EXP0-LR-VAL-ERR
Id
BikePoints_1 0.868446 0.895192 0.829156 4.807993 4.846394 4.315554 0.866595 0.893963 0.828258
BikePoints_10 0.935212 0.962700 0.920177 4.576382 4.456970 4.907174 0.936583 0.964665 0.920177
BikePoints_100 1.019852 1.035073 0.879100 5.029238 5.191576 4.234214 1.019897 1.035073 0.879100
BikePoints_101 1.644451 1.609409 1.798092 4.110624 3.820384 3.998698 1.667633 1.651647 1.801399
BikePoints_102 0.906125 0.895260 0.888641 2.846146 2.880637 2.107715 0.941805 0.938234 0.935679
BikePoints_103 0.769301 0.658364 0.886125 4.201037 4.104669 3.874328 0.770014 0.659477 0.886405
BikePoints_104 1.600881 1.649362 1.747021 6.800566 6.724007 6.153340 1.639654 1.695109 1.792566
BikePoints_105 1.054203 1.028562 1.271638 6.546863 6.423480 6.651657 1.052945 1.033600 1.269296
BikePoints_106 0.815013 0.836485 0.836008 3.415377 3.739926 2.545226 0.837922 0.862313 0.857680
BikePoints_107 1.118044 1.109129 1.100370 5.212214 5.248558 4.757461 1.119107 1.113364 1.107783
BikePoints_108 1.927086 1.943979 1.840355 6.222126 6.034920 5.824440 1.975393 1.992072 1.905376
BikePoints_109 1.886537 1.978030 1.965223 11.214401 11.007143 9.760602 1.998355 2.101507 2.076417
BikePoints_11 0.937378 0.898463 1.112697 5.063484 5.951409 4.869528 0.937378 0.898463 1.112697
BikePoints_110 0.806280 0.843034 0.818317 4.138832 4.025174 5.231449 0.808544 0.847444 0.814368
BikePoints_111 1.114860 1.260007 0.865739 6.521253 6.800561 6.103969 1.115354 1.260636 0.867742
BikePoints_112 1.207004 1.305492 1.146726 6.745412 7.495175 4.052789 1.302222 1.398704 1.290802
BikePoints_113 0.869936 0.880252 0.743023 4.889022 5.026698 3.390360 0.869936 0.880252 0.743023
BikePoints_114 1.204345 1.234066 1.090635 6.371169 6.496631 6.729798 1.204991 1.234611 1.091998
BikePoints_115 1.199277 1.145157 1.456887 5.616933 4.502219 8.205840 1.223101 1.196134 1.439418
BikePoints_116 1.632189 1.704882 1.658611 4.031637 3.955849 3.911851 1.669765 1.757444 1.706954
BikePoints_117 0.995599 1.019250 1.023775 4.684648 4.327730 5.808749 1.020826 1.056872 1.042975
BikePoints_118 0.890423 0.901134 0.832440 2.829188 2.857660 2.648094 0.902026 0.917985 0.839560
BikePoints_119 1.086625 1.112789 1.011344 4.109330 4.338624 3.686096 1.090194 1.117392 1.009872
BikePoints_12 1.576216 1.786762 1.366405 5.947783 6.513871 5.867375 1.666712 1.892316 1.444235
BikePoints_120 1.030827 0.989130 1.156846 2.765397 2.771823 2.990892 1.071199 1.039326 1.195436
BikePoints_121 0.841126 0.882431 0.855653 3.324019 3.260829 3.397887 0.843573 0.885806 0.856812
BikePoints_122 1.111184 1.051480 1.167729 3.381227 3.059848 3.071244 1.166349 1.116543 1.210896
BikePoints_123 0.909103 0.902831 0.899735 4.063213 4.135461 3.626744 0.912570 0.907489 0.899184
BikePoints_124 0.831881 0.793569 0.752311 4.880346 4.439051 4.932028 0.831991 0.793569 0.752641
BikePoints_125 0.861954 0.869036 0.912871 4.716623 4.739041 3.855397 0.861954 0.869036 0.912871
... ... ... ... ... ... ... ... ... ...
BikePoints_794 0.460265 0.482426 0.466114 3.462328 3.919415 2.906445 0.460265 0.482426 0.466114
BikePoints_795 1.628979 1.578799 1.786190 6.477659 5.912535 6.353039 1.657097 1.631046 1.795884
BikePoints_8 0.447254 0.455375 0.487441 3.409182 3.555598 3.613179 0.447561 0.455775 0.487441
BikePoints_80 0.969854 1.004079 0.875142 8.609593 8.794784 8.372231 0.974990 1.011645 0.874291
BikePoints_800 0.964257 0.938177 0.883322 6.622969 6.078414 6.756133 0.964542 0.938177 0.883322
BikePoints_801 1.216494 1.212334 1.406123 6.502172 6.346082 6.562467 1.256297 1.269213 1.436659
BikePoints_804 1.139798 1.147676 1.153626 6.591449 6.553598 6.959812 1.141964 1.150547 1.153626
BikePoints_808 0.578776 0.671922 0.442082 3.923583 5.025528 2.644063 0.579883 0.673375 0.441521
BikePoints_81 0.921543 0.845690 1.102621 3.313329 3.238346 3.369447 0.952713 0.881094 1.134112
BikePoints_815 1.266699 1.334503 1.052444 7.554426 8.337984 6.681123 1.270490 1.338341 1.047721
BikePoints_818 0.811762 0.811416 0.790569 2.772201 3.291906 2.517104 0.811762 0.811416 0.790569
BikePoints_82 0.941025 0.951269 0.888082 2.452172 2.325373 2.216573 0.963951 0.976815 0.907421
BikePoints_83 1.295898 1.317730 1.248610 4.653495 4.706469 4.738447 1.313916 1.346884 1.246622
BikePoints_84 0.944362 0.945766 0.937004 3.071797 3.048468 3.389409 1.011197 1.022081 0.992032
BikePoints_85 1.150511 1.155371 1.164539 8.214923 9.414541 6.497748 1.150670 1.156058 1.164539
BikePoints_86 0.896029 0.894604 0.919908 6.815257 6.866708 6.656279 0.903225 0.898876 0.925820
BikePoints_87 0.930884 0.928193 0.942020 2.667203 3.075449 1.933544 0.936936 0.935667 0.944911
BikePoints_88 1.097720 1.105818 1.126212 3.676000 3.343530 2.970340 1.155953 1.177655 1.196266
BikePoints_89 0.993901 1.058143 0.927159 4.406807 4.286067 3.669439 1.003286 1.073967 0.939119
BikePoints_9 1.217018 1.277192 1.175772 4.165178 3.783334 4.746082 1.243615 1.322634 1.162834
BikePoints_90 0.725253 0.786072 0.639320 5.564285 5.340441 6.150397 0.726900 0.790637 0.629941
BikePoints_91 0.872954 0.895820 0.856233 3.941342 4.085989 3.171441 0.902220 0.931470 0.890871
BikePoints_92 0.886742 0.833698 0.943598 8.815352 7.986511 10.174568 0.887672 0.836624 0.939119
BikePoints_93 1.357611 1.399836 1.243834 7.627754 7.303545 8.307818 1.406458 1.452882 1.285797
BikePoints_94 1.202396 1.199764 1.150181 4.884100 4.573404 5.126911 1.244635 1.250889 1.212738
BikePoints_95 1.462609 1.477387 1.413863 2.995698 2.968699 3.149153 1.479213 1.504433 1.415441
BikePoints_96 1.016671 1.024740 0.974883 6.193414 5.451637 6.509418 1.012608 1.026408 0.974629
BikePoints_97 0.999268 1.015885 0.967477 4.424608 4.131101 4.434488 1.000595 1.017568 0.965681
BikePoints_98 1.620800 1.610329 1.668600 9.952717 10.309279 9.249115 1.626326 1.632893 1.670383
BikePoints_99 0.979027 0.997308 0.949885 3.064492 2.923763 3.771368 0.977952 0.999878 0.935944

765 rows × 9 columns


In [20]:
with open("data/parsed/models_short.p", "wb") as f:
    pickle.dump(baseline_short, f)

In [103]:
baseline_short_df.mean()


Out[103]:
SHORT-EXP0-GAM-TEST-ERR      1.024929
SHORT-EXP0-GAM-TRAIN-ERR     1.031017
SHORT-EXP0-GAM-VAL-ERR       1.015606
SHORT-EXP0-HAVG-TEST-ERR     5.399333
SHORT-EXP0-HAVG-TRAIN-ERR    5.342800
SHORT-EXP0-HAVG-VAL-ERR      5.228157
SHORT-EXP0-LR-TEST-ERR       1.040086
SHORT-EXP0-LR-TRAIN-ERR      1.050169
SHORT-EXP0-LR-VAL-ERR        1.028567
dtype: float64

Mid Term Predictions


In [14]:
lr_formula_mid = "NbBikes ~ TempTMinus12 + HumidityTMinus12 + TimeOfDay + Weekday + DistNbBikes + CollNbBikes"
lr_formula_mid += "+ NbBikesTMinus12 + NbBikesTMinus18 + Near1TMinus12 + Near2TMinus12 + Near1TMinus18 + Near2TMinus18 "

In [15]:
gam_formula_mid = "NbBikes ~ s(TempTMinus12, HumidityTMinus12, bs='tp') + s(TimeOfDay, by=Weekday, bs='tp') "
gam_formula_mid += "+ s(TimeOfDay, by=Weekend, bs='tp') + s(TimeOfDay, by=Holiday, bs='tp') + s(NbBikesTMinus12, bs='tp') "
gam_formula_mid += "+ s(NbBikesTMinus18, bs='tp') "

Baseline


In [16]:
# choose the columns to use in the model
boolean_cols_mid = ['Weekday', 'Weekend', 'Holiday']
numeric_cols_mid = ['HumidityTMinus12', 'TempTMinus12', 'TimeOfDay', 'NbBikesTMinus12', 'NbBikesTMinus18', 'HistAvg', 
                    'Near1TMinus12', 'Near2TMinus12', 'Near1TMinus18', 'Near2TMinus18', 'DistNbBikes', 'CollNbBikes']
pred_col_mid = 'NbBikes'

feature_cols_mid = numeric_cols_mid + boolean_cols_mid
cols_mid = [pred_col_mid] + feature_cols_mid

# select the columns chosen columns
readings_mid = readings.loc[stations_to_use][cols_mid]

# remove na
readings_mid.dropna(inplace=True)

In [17]:
baseline_mid = [model(readings_mid, stations_to_use, gam_formula_mid, lr_formula_mid, 'NbBikes')]


Fitting BikePoints_1
Fitting BikePoints_10
Fitting BikePoints_100
Fitting BikePoints_101
Fitting BikePoints_102
Fitting BikePoints_103
Fitting BikePoints_104
Fitting BikePoints_105
Fitting BikePoints_106
Fitting BikePoints_107
Fitting BikePoints_108
Fitting BikePoints_109
Fitting BikePoints_11
Fitting BikePoints_110
Fitting BikePoints_111
Fitting BikePoints_112
Fitting BikePoints_113
Fitting BikePoints_114
Fitting BikePoints_115
Fitting BikePoints_116
Fitting BikePoints_117
Fitting BikePoints_118
Fitting BikePoints_119
Fitting BikePoints_12
Fitting BikePoints_120
Fitting BikePoints_121
Fitting BikePoints_122
Fitting BikePoints_123
Fitting BikePoints_124
Fitting BikePoints_125
Fitting BikePoints_126
Fitting BikePoints_127
Fitting BikePoints_128
Fitting BikePoints_129
Fitting BikePoints_13
Fitting BikePoints_130
Fitting BikePoints_131
Fitting BikePoints_132
Fitting BikePoints_133
Fitting BikePoints_134
Fitting BikePoints_135
Fitting BikePoints_136
Fitting BikePoints_137
Fitting BikePoints_138
Fitting BikePoints_139
Fitting BikePoints_14
Fitting BikePoints_140
Fitting BikePoints_141
Fitting BikePoints_142
Fitting BikePoints_143
Fitting BikePoints_144
Fitting BikePoints_145
Fitting BikePoints_146
Fitting BikePoints_147
Fitting BikePoints_148
Fitting BikePoints_149
Fitting BikePoints_15
Fitting BikePoints_150
Fitting BikePoints_151
Fitting BikePoints_152
Fitting BikePoints_153
Fitting BikePoints_154
Fitting BikePoints_155
Fitting BikePoints_156
Fitting BikePoints_157
Fitting BikePoints_158
Fitting BikePoints_159
Fitting BikePoints_16
Fitting BikePoints_160
Fitting BikePoints_161
Fitting BikePoints_162
Fitting BikePoints_163
Fitting BikePoints_164
Fitting BikePoints_165
Fitting BikePoints_166
Fitting BikePoints_167
Fitting BikePoints_168
Fitting BikePoints_169
Fitting BikePoints_17
Fitting BikePoints_170
Fitting BikePoints_171
Fitting BikePoints_172
Fitting BikePoints_173
Fitting BikePoints_174
Fitting BikePoints_175
Fitting BikePoints_176
Fitting BikePoints_177
Fitting BikePoints_178
Fitting BikePoints_18
Fitting BikePoints_180
Fitting BikePoints_181
Fitting BikePoints_182
Fitting BikePoints_183
Fitting BikePoints_184
Fitting BikePoints_185
Fitting BikePoints_186
Fitting BikePoints_187
Fitting BikePoints_188
Fitting BikePoints_189
Fitting BikePoints_19
Fitting BikePoints_190
Fitting BikePoints_191
Fitting BikePoints_192
Fitting BikePoints_193
Fitting BikePoints_194
Fitting BikePoints_195
Fitting BikePoints_196
Fitting BikePoints_197
Fitting BikePoints_199
Fitting BikePoints_2
Fitting BikePoints_20
Fitting BikePoints_200
Fitting BikePoints_201
Fitting BikePoints_202
Fitting BikePoints_203
Fitting BikePoints_204
Fitting BikePoints_206
Fitting BikePoints_207
Fitting BikePoints_208
Fitting BikePoints_209
Fitting BikePoints_21
Fitting BikePoints_210
Fitting BikePoints_211
Fitting BikePoints_212
Fitting BikePoints_213
Fitting BikePoints_214
Fitting BikePoints_215
Fitting BikePoints_216
Fitting BikePoints_217
Fitting BikePoints_218
Fitting BikePoints_219
Fitting BikePoints_22
Fitting BikePoints_220
Fitting BikePoints_221
Fitting BikePoints_222
Fitting BikePoints_223
Fitting BikePoints_224
Fitting BikePoints_225
Fitting BikePoints_226
Fitting BikePoints_227
Fitting BikePoints_228
Fitting BikePoints_229
Fitting BikePoints_23
Fitting BikePoints_230
Fitting BikePoints_231
Fitting BikePoints_232
Fitting BikePoints_233
Fitting BikePoints_234
Fitting BikePoints_235
Fitting BikePoints_236
Fitting BikePoints_238
Fitting BikePoints_239
Fitting BikePoints_24
Fitting BikePoints_240
Fitting BikePoints_242
Fitting BikePoints_243
Fitting BikePoints_244
Fitting BikePoints_245
Fitting BikePoints_246
Fitting BikePoints_247
Fitting BikePoints_248
Fitting BikePoints_249
Fitting BikePoints_25
Fitting BikePoints_250
Fitting BikePoints_251
Fitting BikePoints_252
Fitting BikePoints_253
Fitting BikePoints_254
Fitting BikePoints_255
Fitting BikePoints_256
Fitting BikePoints_257
Fitting BikePoints_258
Fitting BikePoints_26
Fitting BikePoints_260
Fitting BikePoints_261
Fitting BikePoints_262
Fitting BikePoints_263
Fitting BikePoints_264
Fitting BikePoints_265
Fitting BikePoints_266
Fitting BikePoints_267
Fitting BikePoints_268
Fitting BikePoints_269
Fitting BikePoints_27
Fitting BikePoints_270
Fitting BikePoints_271
Fitting BikePoints_272
Fitting BikePoints_273
Fitting BikePoints_274
Fitting BikePoints_275
Fitting BikePoints_276
Fitting BikePoints_277
Fitting BikePoints_278
Fitting BikePoints_279
Fitting BikePoints_28
Fitting BikePoints_280
Fitting BikePoints_281
Fitting BikePoints_282
Fitting BikePoints_284
Fitting BikePoints_286
Fitting BikePoints_287
Fitting BikePoints_288
Fitting BikePoints_289
Fitting BikePoints_29
Fitting BikePoints_290
Fitting BikePoints_291
Fitting BikePoints_292
Fitting BikePoints_293
Fitting BikePoints_294
Fitting BikePoints_295
Fitting BikePoints_296
Fitting BikePoints_297
Fitting BikePoints_298
Fitting BikePoints_299
Fitting BikePoints_3
Fitting BikePoints_30
Fitting BikePoints_300
Fitting BikePoints_301
Fitting BikePoints_302
Fitting BikePoints_303
Fitting BikePoints_304
Fitting BikePoints_305
Fitting BikePoints_306
Fitting BikePoints_307
Fitting BikePoints_308
Fitting BikePoints_309
Fitting BikePoints_31
Fitting BikePoints_310
Fitting BikePoints_311
Fitting BikePoints_312
Fitting BikePoints_313
Fitting BikePoints_314
Fitting BikePoints_315
Fitting BikePoints_316
Fitting BikePoints_317
Fitting BikePoints_318
Fitting BikePoints_319
Fitting BikePoints_32
Fitting BikePoints_320
Fitting BikePoints_321
Fitting BikePoints_322
Fitting BikePoints_323
Fitting BikePoints_324
Fitting BikePoints_325
Fitting BikePoints_326
Fitting BikePoints_327
Fitting BikePoints_328
Fitting BikePoints_329
Fitting BikePoints_33
Fitting BikePoints_330
Fitting BikePoints_331
Fitting BikePoints_332
Fitting BikePoints_333
Fitting BikePoints_334
Fitting BikePoints_335
Fitting BikePoints_336
Fitting BikePoints_337
Fitting BikePoints_338
Fitting BikePoints_339
Fitting BikePoints_34
Fitting BikePoints_340
Fitting BikePoints_341
Fitting BikePoints_343
Fitting BikePoints_344
Fitting BikePoints_345
Fitting BikePoints_347
Fitting BikePoints_348
Fitting BikePoints_349
Fitting BikePoints_350
Fitting BikePoints_351
Fitting BikePoints_352
Fitting BikePoints_353
Fitting BikePoints_354
Fitting BikePoints_355
Fitting BikePoints_356
Fitting BikePoints_357
Fitting BikePoints_359
Fitting BikePoints_36
Fitting BikePoints_360
Fitting BikePoints_361
Fitting BikePoints_362
Fitting BikePoints_363
Fitting BikePoints_364
Fitting BikePoints_365
Fitting BikePoints_366
Fitting BikePoints_367
Fitting BikePoints_368
Fitting BikePoints_37
Fitting BikePoints_370
Fitting BikePoints_371
Fitting BikePoints_372
Fitting BikePoints_373
Fitting BikePoints_374
Fitting BikePoints_375
Fitting BikePoints_376
Fitting BikePoints_377
Fitting BikePoints_378
Fitting BikePoints_379
Fitting BikePoints_38
Fitting BikePoints_380
Fitting BikePoints_381
Fitting BikePoints_382
Fitting BikePoints_383
Fitting BikePoints_384
Fitting BikePoints_386
Fitting BikePoints_387
Fitting BikePoints_388
Fitting BikePoints_389
Fitting BikePoints_39
Fitting BikePoints_390
Fitting BikePoints_391
Fitting BikePoints_392
Fitting BikePoints_393
Fitting BikePoints_394
Fitting BikePoints_395
Fitting BikePoints_396
Fitting BikePoints_397
Fitting BikePoints_398
Fitting BikePoints_399
Fitting BikePoints_4
Fitting BikePoints_40
Fitting BikePoints_400
Fitting BikePoints_401
Fitting BikePoints_402
Fitting BikePoints_403
Fitting BikePoints_404
Fitting BikePoints_405
Fitting BikePoints_406
Fitting BikePoints_407
Fitting BikePoints_408
Fitting BikePoints_409
Fitting BikePoints_41
Fitting BikePoints_410
Fitting BikePoints_411
Fitting BikePoints_412
Fitting BikePoints_419
Fitting BikePoints_42
Fitting BikePoints_420
Fitting BikePoints_421
Fitting BikePoints_423
Fitting BikePoints_424
Fitting BikePoints_425
Fitting BikePoints_426
Fitting BikePoints_427
Fitting BikePoints_428
Fitting BikePoints_43
Fitting BikePoints_430
Fitting BikePoints_431
Fitting BikePoints_432
Fitting BikePoints_433
Fitting BikePoints_435
Fitting BikePoints_436
Fitting BikePoints_437
Fitting BikePoints_44
Fitting BikePoints_440
Fitting BikePoints_441
Fitting BikePoints_442
Fitting BikePoints_443
Fitting BikePoints_444
Fitting BikePoints_445
Fitting BikePoints_446
Fitting BikePoints_447
Fitting BikePoints_448
Fitting BikePoints_449
Fitting BikePoints_45
Fitting BikePoints_450
Fitting BikePoints_451
Fitting BikePoints_452
Fitting BikePoints_453
Fitting BikePoints_454
Fitting BikePoints_455
Fitting BikePoints_456
Fitting BikePoints_457
Fitting BikePoints_458
Fitting BikePoints_459
Fitting BikePoints_46
Fitting BikePoints_460
Fitting BikePoints_461
Fitting BikePoints_462
Fitting BikePoints_463
Fitting BikePoints_464
Fitting BikePoints_465
Fitting BikePoints_466
Fitting BikePoints_467
Fitting BikePoints_468
Fitting BikePoints_469
Fitting BikePoints_47
Fitting BikePoints_470
Fitting BikePoints_471
Fitting BikePoints_472
Fitting BikePoints_473
Fitting BikePoints_474
Fitting BikePoints_475
Fitting BikePoints_476
Fitting BikePoints_477
Fitting BikePoints_478
Fitting BikePoints_479
Fitting BikePoints_48
Fitting BikePoints_480
Fitting BikePoints_481
Fitting BikePoints_482
Fitting BikePoints_483
Fitting BikePoints_484
Fitting BikePoints_485
Fitting BikePoints_486
Fitting BikePoints_487
Fitting BikePoints_488
Fitting BikePoints_489
Fitting BikePoints_49
Fitting BikePoints_490
Fitting BikePoints_491
Fitting BikePoints_492
Fitting BikePoints_494
Fitting BikePoints_495
Fitting BikePoints_496
Fitting BikePoints_498
Fitting BikePoints_499
Fitting BikePoints_5
Fitting BikePoints_50
Fitting BikePoints_500
Fitting BikePoints_501
Fitting BikePoints_503
Fitting BikePoints_504
Fitting BikePoints_505
Fitting BikePoints_506
Fitting BikePoints_507
Fitting BikePoints_508
Fitting BikePoints_509
Fitting BikePoints_51
Fitting BikePoints_510
Fitting BikePoints_511
Fitting BikePoints_512
Fitting BikePoints_513
Fitting BikePoints_514
Fitting BikePoints_515
Fitting BikePoints_516
Fitting BikePoints_517
Fitting BikePoints_518
Fitting BikePoints_519
Fitting BikePoints_52
Fitting BikePoints_520
Fitting BikePoints_521
Fitting BikePoints_522
Fitting BikePoints_523
Fitting BikePoints_524
Fitting BikePoints_526
Fitting BikePoints_527
Fitting BikePoints_528
Fitting BikePoints_529
Fitting BikePoints_53
Fitting BikePoints_530
Fitting BikePoints_531
Fitting BikePoints_532
Fitting BikePoints_533
Fitting BikePoints_534
Fitting BikePoints_535
Fitting BikePoints_536
Fitting BikePoints_537
Fitting BikePoints_538
Fitting BikePoints_539
Fitting BikePoints_54
Fitting BikePoints_540
Fitting BikePoints_541
Fitting BikePoints_542
Fitting BikePoints_543
Fitting BikePoints_544
Fitting BikePoints_545
Fitting BikePoints_546
Fitting BikePoints_547
Fitting BikePoints_548
Fitting BikePoints_549
Fitting BikePoints_55
Fitting BikePoints_550
Fitting BikePoints_551
Fitting BikePoints_552
Fitting BikePoints_553
Fitting BikePoints_554
Fitting BikePoints_555
Fitting BikePoints_556
Fitting BikePoints_557
Fitting BikePoints_558
Fitting BikePoints_559
Fitting BikePoints_56
Fitting BikePoints_560
Fitting BikePoints_561
Fitting BikePoints_562
Fitting BikePoints_563
Fitting BikePoints_564
Fitting BikePoints_565
Fitting BikePoints_566
Fitting BikePoints_568
Fitting BikePoints_569
Fitting BikePoints_57
Fitting BikePoints_570
Fitting BikePoints_571
Fitting BikePoints_572
Fitting BikePoints_573
Fitting BikePoints_574
Fitting BikePoints_576
Fitting BikePoints_577
Fitting BikePoints_578
Fitting BikePoints_579
Fitting BikePoints_58
Fitting BikePoints_580
Fitting BikePoints_581
Fitting BikePoints_583
Fitting BikePoints_584
Fitting BikePoints_586
Fitting BikePoints_587
Fitting BikePoints_588
Fitting BikePoints_589
Fitting BikePoints_59
Fitting BikePoints_590
Fitting BikePoints_591
Fitting BikePoints_592
Fitting BikePoints_593
Fitting BikePoints_594
Fitting BikePoints_595
Fitting BikePoints_596
Fitting BikePoints_597
Fitting BikePoints_598
Fitting BikePoints_599
Fitting BikePoints_6
Fitting BikePoints_60
Fitting BikePoints_600
Fitting BikePoints_601
Fitting BikePoints_602
Fitting BikePoints_603
Fitting BikePoints_604
Fitting BikePoints_605
Fitting BikePoints_606
Fitting BikePoints_607
Fitting BikePoints_608
Fitting BikePoints_609
Fitting BikePoints_61
Fitting BikePoints_610
Fitting BikePoints_611
Fitting BikePoints_612
Fitting BikePoints_613
Fitting BikePoints_614
Fitting BikePoints_615
Fitting BikePoints_616
Fitting BikePoints_617
Fitting BikePoints_618
Fitting BikePoints_619
Fitting BikePoints_62
Fitting BikePoints_620
Fitting BikePoints_621
Fitting BikePoints_622
Fitting BikePoints_623
Fitting BikePoints_624
Fitting BikePoints_625
Fitting BikePoints_626
Fitting BikePoints_627
Fitting BikePoints_628
Fitting BikePoints_629
Fitting BikePoints_63
Fitting BikePoints_630
Fitting BikePoints_631
Fitting BikePoints_632
Fitting BikePoints_633
Fitting BikePoints_634
Fitting BikePoints_635
Fitting BikePoints_636
Fitting BikePoints_637
Fitting BikePoints_638
Fitting BikePoints_639
Fitting BikePoints_64
Fitting BikePoints_640
Fitting BikePoints_641
Fitting BikePoints_642
Fitting BikePoints_643
Fitting BikePoints_644
Fitting BikePoints_645
Fitting BikePoints_646
Fitting BikePoints_647
Fitting BikePoints_648
Fitting BikePoints_649
Fitting BikePoints_65
Fitting BikePoints_650
Fitting BikePoints_651
Fitting BikePoints_652
Fitting BikePoints_653
Fitting BikePoints_654
Fitting BikePoints_655
Fitting BikePoints_656
Fitting BikePoints_657
Fitting BikePoints_658
Fitting BikePoints_659
Fitting BikePoints_66
Fitting BikePoints_660
Fitting BikePoints_661
Fitting BikePoints_662
Fitting BikePoints_663
Fitting BikePoints_664
Fitting BikePoints_665
Fitting BikePoints_666
Fitting BikePoints_667
Fitting BikePoints_668
Fitting BikePoints_669
Fitting BikePoints_67
Fitting BikePoints_670
Fitting BikePoints_671
Fitting BikePoints_672
Fitting BikePoints_673
Fitting BikePoints_674
Fitting BikePoints_675
Fitting BikePoints_676
Fitting BikePoints_677
Fitting BikePoints_678
Fitting BikePoints_679
Fitting BikePoints_68
Fitting BikePoints_680
Fitting BikePoints_681
Fitting BikePoints_682
Fitting BikePoints_683
Fitting BikePoints_684
Fitting BikePoints_685
Fitting BikePoints_686
Fitting BikePoints_687
Fitting BikePoints_688
Fitting BikePoints_689
Fitting BikePoints_69
Fitting BikePoints_690
Fitting BikePoints_691
Fitting BikePoints_692
Fitting BikePoints_693
Fitting BikePoints_694
Fitting BikePoints_695
Fitting BikePoints_696
Fitting BikePoints_697
Fitting BikePoints_698
Fitting BikePoints_699
Fitting BikePoints_7
Fitting BikePoints_70
Fitting BikePoints_700
Fitting BikePoints_701
Fitting BikePoints_702
Fitting BikePoints_703
Fitting BikePoints_704
Fitting BikePoints_705
Fitting BikePoints_706
Fitting BikePoints_707
Fitting BikePoints_708
Fitting BikePoints_709
Fitting BikePoints_71
Fitting BikePoints_710
Fitting BikePoints_711
Fitting BikePoints_712
Fitting BikePoints_713
Fitting BikePoints_714
Fitting BikePoints_715
Fitting BikePoints_716
Fitting BikePoints_717
Fitting BikePoints_718
Fitting BikePoints_719
Fitting BikePoints_72
Fitting BikePoints_720
Fitting BikePoints_721
Fitting BikePoints_722
Fitting BikePoints_723
Fitting BikePoints_724
Fitting BikePoints_725
Fitting BikePoints_726
Fitting BikePoints_727
Fitting BikePoints_728
Fitting BikePoints_729
Fitting BikePoints_73
Fitting BikePoints_730
Fitting BikePoints_731
Fitting BikePoints_732
Fitting BikePoints_733
Fitting BikePoints_734
Fitting BikePoints_735
Fitting BikePoints_736
Fitting BikePoints_737
Fitting BikePoints_738
Fitting BikePoints_739
Fitting BikePoints_74
Fitting BikePoints_740
Fitting BikePoints_741
Fitting BikePoints_742
Fitting BikePoints_743
Fitting BikePoints_744
Fitting BikePoints_745
Fitting BikePoints_746
Fitting BikePoints_747
Fitting BikePoints_748
Fitting BikePoints_749
Fitting BikePoints_75
Fitting BikePoints_750
Fitting BikePoints_751
Fitting BikePoints_752
Fitting BikePoints_753
Fitting BikePoints_754
Fitting BikePoints_755
Fitting BikePoints_756
Fitting BikePoints_757
Fitting BikePoints_758
Fitting BikePoints_759
Fitting BikePoints_76
Fitting BikePoints_760
Fitting BikePoints_761
Fitting BikePoints_762
Fitting BikePoints_763
Fitting BikePoints_764
Fitting BikePoints_765
Fitting BikePoints_766
Fitting BikePoints_767
Fitting BikePoints_768
Fitting BikePoints_769
Fitting BikePoints_77
Fitting BikePoints_770
Fitting BikePoints_771
Fitting BikePoints_772
Fitting BikePoints_773
Fitting BikePoints_774
Fitting BikePoints_775
Fitting BikePoints_776
Fitting BikePoints_777
Fitting BikePoints_779
Fitting BikePoints_78
Fitting BikePoints_780
Fitting BikePoints_781
Fitting BikePoints_782
Fitting BikePoints_783
Fitting BikePoints_784
Fitting BikePoints_785
Fitting BikePoints_786
Fitting BikePoints_787
Fitting BikePoints_788
Fitting BikePoints_789
Fitting BikePoints_790
Fitting BikePoints_792
Fitting BikePoints_793
Fitting BikePoints_794
Fitting BikePoints_795
Fitting BikePoints_8
Fitting BikePoints_80
Fitting BikePoints_800
Fitting BikePoints_801
Fitting BikePoints_804
Fitting BikePoints_808
Fitting BikePoints_81
Fitting BikePoints_815
Fitting BikePoints_818
Fitting BikePoints_82
Fitting BikePoints_83
Fitting BikePoints_84
Fitting BikePoints_85
Fitting BikePoints_86
Fitting BikePoints_87
Fitting BikePoints_88
Fitting BikePoints_89
Fitting BikePoints_9
Fitting BikePoints_90
Fitting BikePoints_91
Fitting BikePoints_92
Fitting BikePoints_93
Fitting BikePoints_94
Fitting BikePoints_95
Fitting BikePoints_96
Fitting BikePoints_97
Fitting BikePoints_98
Fitting BikePoints_99

In [21]:
baseline_mid_df = convert_results_to_df(baseline_mid, 'MID')

In [ ]:
with open("data/parsed/models_mid.p", "wb") as f:
    pickle.dump(baseline_mid, f)

Long Term Predictions

All


In [157]:
# choose the columns to use in the model
redistribution_cols = ['CollNbBikesCum6', 'DistNbBikesCum6']
boolean_cols_long = ['Weekday', 'Weekend', 'Holiday']
numeric_cols_long = ['TimeOfDay', 'HistAvg']
pred_col_long = 'NbBikes'

feature_cols_long = numeric_cols_long + boolean_cols_long + redistribution_cols
cols_long = [pred_col_long] + feature_cols_long

# select the columns chosen columns
readings_long = readings.loc[stations_to_use][cols_long]

# remove na
readings_long.dropna(inplace=True)

In [158]:
gam_formula_long = "NbBikes ~ s(TimeOfDay, by=Weekday, bs='tp', sp=35.0) + s(TimeOfDay, by=Weekend, bs='tp', sp=0.7) "
gam_formula_long += "+ s(TimeOfDay, by=Holiday, bs='tp', sp=3.0) + s(HistAvg, bs='tp', k=5) + CollNbBikesCum6 + DistNbBikesCum6 "

In [159]:
lr_formula_long = "NbBikes ~ TimeOfDay + Weekday + Holiday + Weekend + HistAvg + CollNbBikesCum6 + DistNbBikesCum6"

In [160]:
all_long = [model(readings_long, stations_to_use, gam_formula_long, lr_formula_long, 'NbBikes')]


Fitting BikePoints_1
Fitting BikePoints_10
Fitting BikePoints_100
Fitting BikePoints_101
Fitting BikePoints_102
Fitting BikePoints_103
Fitting BikePoints_104
Fitting BikePoints_105
Fitting BikePoints_106
Fitting BikePoints_107
Fitting BikePoints_108
Fitting BikePoints_109
Fitting BikePoints_11
Fitting BikePoints_110
Fitting BikePoints_111
Fitting BikePoints_112
Fitting BikePoints_113
Fitting BikePoints_114
Fitting BikePoints_115
Fitting BikePoints_116
Fitting BikePoints_117
Fitting BikePoints_118
Fitting BikePoints_119
Fitting BikePoints_12
Fitting BikePoints_120
Fitting BikePoints_121
Fitting BikePoints_122
Fitting BikePoints_123
Fitting BikePoints_124
Fitting BikePoints_125
Fitting BikePoints_126
Fitting BikePoints_127
Fitting BikePoints_128
Fitting BikePoints_129
Fitting BikePoints_13
Fitting BikePoints_130
Fitting BikePoints_131
Fitting BikePoints_132
Fitting BikePoints_133
Fitting BikePoints_134
Fitting BikePoints_135
Fitting BikePoints_136
Fitting BikePoints_137
Fitting BikePoints_138
Fitting BikePoints_139
Fitting BikePoints_14
Fitting BikePoints_140
Fitting BikePoints_141
Fitting BikePoints_142
Fitting BikePoints_143
Fitting BikePoints_144
Fitting BikePoints_145
Fitting BikePoints_146
Fitting BikePoints_147
Fitting BikePoints_148
Fitting BikePoints_149
Fitting BikePoints_15
Fitting BikePoints_150
Fitting BikePoints_151
Fitting BikePoints_152
Fitting BikePoints_153
Fitting BikePoints_154
Fitting BikePoints_155
Fitting BikePoints_156
Fitting BikePoints_157
Fitting BikePoints_158
Fitting BikePoints_159
Fitting BikePoints_16
Fitting BikePoints_160
Fitting BikePoints_161
Fitting BikePoints_162
Fitting BikePoints_163
Fitting BikePoints_164
Fitting BikePoints_165
Fitting BikePoints_166
Fitting BikePoints_167
Fitting BikePoints_168
Fitting BikePoints_169
Fitting BikePoints_17
Fitting BikePoints_170
Fitting BikePoints_171
Fitting BikePoints_172
Fitting BikePoints_173
Fitting BikePoints_174
Fitting BikePoints_175
Fitting BikePoints_176
Fitting BikePoints_177
Fitting BikePoints_178
Fitting BikePoints_18
Fitting BikePoints_180
Fitting BikePoints_181
Fitting BikePoints_182
Fitting BikePoints_183
Fitting BikePoints_184
Fitting BikePoints_185
Fitting BikePoints_186
Fitting BikePoints_187
Fitting BikePoints_188
Fitting BikePoints_189
Fitting BikePoints_19
Fitting BikePoints_190
Fitting BikePoints_191
Fitting BikePoints_192
Fitting BikePoints_193
Fitting BikePoints_194
Fitting BikePoints_195
Fitting BikePoints_196
Fitting BikePoints_197
Fitting BikePoints_199
Fitting BikePoints_2
Fitting BikePoints_20
Fitting BikePoints_200
Fitting BikePoints_201
Fitting BikePoints_202
Fitting BikePoints_203
Fitting BikePoints_204
Fitting BikePoints_206
Fitting BikePoints_207
Fitting BikePoints_208
Fitting BikePoints_209
Fitting BikePoints_21
Fitting BikePoints_210
Fitting BikePoints_211
Fitting BikePoints_212
Fitting BikePoints_213
Fitting BikePoints_214
Fitting BikePoints_215
Fitting BikePoints_216
Fitting BikePoints_217
Fitting BikePoints_218
Fitting BikePoints_219
Fitting BikePoints_22
Fitting BikePoints_220
Fitting BikePoints_221
Fitting BikePoints_222
Fitting BikePoints_223
Fitting BikePoints_224
Fitting BikePoints_225
Fitting BikePoints_226
Fitting BikePoints_227
Fitting BikePoints_228
Fitting BikePoints_229
Fitting BikePoints_23
Fitting BikePoints_230
Fitting BikePoints_231
Fitting BikePoints_232
Fitting BikePoints_233
Fitting BikePoints_234
Fitting BikePoints_235
Fitting BikePoints_236
Fitting BikePoints_238
Fitting BikePoints_239
Fitting BikePoints_24
Fitting BikePoints_240
Fitting BikePoints_242
Fitting BikePoints_243
Fitting BikePoints_244
Fitting BikePoints_245
Fitting BikePoints_246
Fitting BikePoints_247
Fitting BikePoints_248
Fitting BikePoints_249
Fitting BikePoints_25
Fitting BikePoints_250
Fitting BikePoints_251
Fitting BikePoints_252
Fitting BikePoints_253
Fitting BikePoints_254
Fitting BikePoints_255
Fitting BikePoints_256
Fitting BikePoints_257
Fitting BikePoints_258
Fitting BikePoints_26
Fitting BikePoints_260
Fitting BikePoints_261
Fitting BikePoints_262
Fitting BikePoints_263
Fitting BikePoints_264
Fitting BikePoints_265
Fitting BikePoints_266
Fitting BikePoints_267
Fitting BikePoints_268
Fitting BikePoints_269
Fitting BikePoints_27
Fitting BikePoints_270
Fitting BikePoints_271
Fitting BikePoints_272
Fitting BikePoints_273
Fitting BikePoints_274
Fitting BikePoints_275
Fitting BikePoints_276
Fitting BikePoints_277
Fitting BikePoints_278
Fitting BikePoints_279
Fitting BikePoints_28
Fitting BikePoints_280
Fitting BikePoints_281
Fitting BikePoints_282
Fitting BikePoints_284
Fitting BikePoints_286
Fitting BikePoints_287
Fitting BikePoints_288
Fitting BikePoints_289
Fitting BikePoints_29
Fitting BikePoints_290
Fitting BikePoints_291
Fitting BikePoints_292
Fitting BikePoints_293
Fitting BikePoints_294
Fitting BikePoints_295
Fitting BikePoints_296
Fitting BikePoints_297
Fitting BikePoints_298
Fitting BikePoints_299
Fitting BikePoints_3
Fitting BikePoints_30
Fitting BikePoints_300
Fitting BikePoints_301
Fitting BikePoints_302
Fitting BikePoints_303
Fitting BikePoints_304
Fitting BikePoints_305
Fitting BikePoints_306
Fitting BikePoints_307
Fitting BikePoints_308
Fitting BikePoints_309
Fitting BikePoints_31
Fitting BikePoints_310
Fitting BikePoints_311
Fitting BikePoints_312
Fitting BikePoints_313
Fitting BikePoints_314
Fitting BikePoints_315
Fitting BikePoints_316
Fitting BikePoints_317
Fitting BikePoints_318
Fitting BikePoints_319
Fitting BikePoints_32
Fitting BikePoints_320
Fitting BikePoints_321
Fitting BikePoints_322
Fitting BikePoints_323
Fitting BikePoints_324
Fitting BikePoints_325
Fitting BikePoints_326
Fitting BikePoints_327
Fitting BikePoints_328
Fitting BikePoints_329
Fitting BikePoints_33
Fitting BikePoints_330
Fitting BikePoints_331
Fitting BikePoints_332
Fitting BikePoints_333
Fitting BikePoints_334
Fitting BikePoints_335
Fitting BikePoints_336
Fitting BikePoints_337
Fitting BikePoints_338
Fitting BikePoints_339
Fitting BikePoints_34
Fitting BikePoints_340
Fitting BikePoints_341
Fitting BikePoints_343
Fitting BikePoints_344
Fitting BikePoints_345
Fitting BikePoints_347
Fitting BikePoints_348
Fitting BikePoints_349
Fitting BikePoints_350
Fitting BikePoints_351
Fitting BikePoints_352
Fitting BikePoints_353
Fitting BikePoints_354
Fitting BikePoints_355
Fitting BikePoints_356
Fitting BikePoints_357
Fitting BikePoints_359
Fitting BikePoints_36
Fitting BikePoints_360
Fitting BikePoints_361
Fitting BikePoints_362
Fitting BikePoints_363
Fitting BikePoints_364
Fitting BikePoints_365
Fitting BikePoints_366
Fitting BikePoints_367
Fitting BikePoints_368
Fitting BikePoints_37
Fitting BikePoints_370
Fitting BikePoints_371
Fitting BikePoints_372
Fitting BikePoints_373
Fitting BikePoints_374
Fitting BikePoints_375
Fitting BikePoints_376
Fitting BikePoints_377
Fitting BikePoints_378
Fitting BikePoints_379
Fitting BikePoints_38
Fitting BikePoints_380
Fitting BikePoints_381
Fitting BikePoints_382
Fitting BikePoints_383
Fitting BikePoints_384
Fitting BikePoints_386
Fitting BikePoints_387
Fitting BikePoints_388
Fitting BikePoints_389
Fitting BikePoints_39
Fitting BikePoints_390
Fitting BikePoints_391
Fitting BikePoints_392
Fitting BikePoints_393
Fitting BikePoints_394
Fitting BikePoints_395
Fitting BikePoints_396
Fitting BikePoints_397
Fitting BikePoints_398
Fitting BikePoints_399
Fitting BikePoints_4
Fitting BikePoints_40
Fitting BikePoints_400
Fitting BikePoints_401
Fitting BikePoints_402
Fitting BikePoints_403
Fitting BikePoints_404
Fitting BikePoints_405
Fitting BikePoints_406
Fitting BikePoints_407
Fitting BikePoints_408
Fitting BikePoints_409
Fitting BikePoints_41
Fitting BikePoints_410
Fitting BikePoints_411
Fitting BikePoints_412
Fitting BikePoints_419
Fitting BikePoints_42
Fitting BikePoints_420
Fitting BikePoints_421
Fitting BikePoints_423
Fitting BikePoints_424
Fitting BikePoints_425
Fitting BikePoints_426
Fitting BikePoints_427
Fitting BikePoints_428
Fitting BikePoints_43
Fitting BikePoints_430
Fitting BikePoints_431
Fitting BikePoints_432
Fitting BikePoints_433
Fitting BikePoints_435
Fitting BikePoints_436
Fitting BikePoints_437
Fitting BikePoints_44
Fitting BikePoints_440
Fitting BikePoints_441
Fitting BikePoints_442
Fitting BikePoints_443
Fitting BikePoints_444
Fitting BikePoints_445
Fitting BikePoints_446
Fitting BikePoints_447
Fitting BikePoints_448
Fitting BikePoints_449
Fitting BikePoints_45
Fitting BikePoints_450
Fitting BikePoints_451
Fitting BikePoints_452
Fitting BikePoints_453
Fitting BikePoints_454
Fitting BikePoints_455
Fitting BikePoints_456
Fitting BikePoints_457
Fitting BikePoints_458
Fitting BikePoints_459
Fitting BikePoints_46
Fitting BikePoints_460
Fitting BikePoints_461
Fitting BikePoints_462
Fitting BikePoints_463
Fitting BikePoints_464
Fitting BikePoints_465
Fitting BikePoints_466
Fitting BikePoints_467
Fitting BikePoints_468
Fitting BikePoints_469
Fitting BikePoints_47
Fitting BikePoints_470
Fitting BikePoints_471
Fitting BikePoints_472
Fitting BikePoints_473
Fitting BikePoints_474
Fitting BikePoints_475
Fitting BikePoints_476
Fitting BikePoints_477
Fitting BikePoints_478
Fitting BikePoints_479
Fitting BikePoints_48
Fitting BikePoints_480
Fitting BikePoints_481
Fitting BikePoints_482
Fitting BikePoints_483
Fitting BikePoints_484
Fitting BikePoints_485
Fitting BikePoints_486
Fitting BikePoints_487
Fitting BikePoints_488
Fitting BikePoints_489
Fitting BikePoints_49
Fitting BikePoints_490
Fitting BikePoints_491
Fitting BikePoints_492
Fitting BikePoints_494
Fitting BikePoints_495
Fitting BikePoints_496
Fitting BikePoints_498
Fitting BikePoints_499
Fitting BikePoints_5
Fitting BikePoints_50
Fitting BikePoints_500
Fitting BikePoints_501
Fitting BikePoints_503
Fitting BikePoints_504
Fitting BikePoints_505
Fitting BikePoints_506
Fitting BikePoints_507
Fitting BikePoints_508
Fitting BikePoints_509
Fitting BikePoints_51
Fitting BikePoints_510
Fitting BikePoints_511
Fitting BikePoints_512
Fitting BikePoints_513
Fitting BikePoints_514
Fitting BikePoints_515
Fitting BikePoints_516
Fitting BikePoints_517
Fitting BikePoints_518
Fitting BikePoints_519
Fitting BikePoints_52
Fitting BikePoints_520
Fitting BikePoints_521
Fitting BikePoints_522
Fitting BikePoints_523
Fitting BikePoints_524
Fitting BikePoints_526
Fitting BikePoints_527
Fitting BikePoints_528
Fitting BikePoints_529
Fitting BikePoints_53
Fitting BikePoints_530
Fitting BikePoints_531
Fitting BikePoints_532
Fitting BikePoints_533
Fitting BikePoints_534
Fitting BikePoints_535
Fitting BikePoints_536
Fitting BikePoints_537
Fitting BikePoints_538
Fitting BikePoints_539
Fitting BikePoints_54
Fitting BikePoints_540
Fitting BikePoints_541
Fitting BikePoints_542
Fitting BikePoints_543
Fitting BikePoints_544
Fitting BikePoints_545
Fitting BikePoints_546
Fitting BikePoints_547
Fitting BikePoints_548
Fitting BikePoints_549
Fitting BikePoints_55
Fitting BikePoints_550
Fitting BikePoints_551
Fitting BikePoints_552
Fitting BikePoints_553
Fitting BikePoints_554
Fitting BikePoints_555
Fitting BikePoints_556
Fitting BikePoints_557
Fitting BikePoints_558
Fitting BikePoints_559
Fitting BikePoints_56
Fitting BikePoints_560
Fitting BikePoints_561
Fitting BikePoints_562
Fitting BikePoints_563
Fitting BikePoints_564
Fitting BikePoints_565
Fitting BikePoints_566
Fitting BikePoints_568
Fitting BikePoints_569
Fitting BikePoints_57
Fitting BikePoints_570
Fitting BikePoints_571
Fitting BikePoints_572
Fitting BikePoints_573
Fitting BikePoints_574
Fitting BikePoints_576
Fitting BikePoints_577
Fitting BikePoints_578
Fitting BikePoints_579
Fitting BikePoints_58
Fitting BikePoints_580
Fitting BikePoints_581
Fitting BikePoints_583
Fitting BikePoints_584
Fitting BikePoints_586
Fitting BikePoints_587
Fitting BikePoints_588
Fitting BikePoints_589
Fitting BikePoints_59
Fitting BikePoints_590
Fitting BikePoints_591
Fitting BikePoints_592
Fitting BikePoints_593
Fitting BikePoints_594
Fitting BikePoints_595
Fitting BikePoints_596
Fitting BikePoints_597
Fitting BikePoints_598
Fitting BikePoints_599
Fitting BikePoints_6
Fitting BikePoints_60
Fitting BikePoints_600
Fitting BikePoints_601
Fitting BikePoints_602
Fitting BikePoints_603
Fitting BikePoints_604
Fitting BikePoints_605
Fitting BikePoints_606
Fitting BikePoints_607
Fitting BikePoints_608
Fitting BikePoints_609
Fitting BikePoints_61
Fitting BikePoints_610
Fitting BikePoints_611
Fitting BikePoints_612
Fitting BikePoints_613
Fitting BikePoints_614
Fitting BikePoints_615
Fitting BikePoints_616
Fitting BikePoints_617
Fitting BikePoints_618
Fitting BikePoints_619
Fitting BikePoints_62
Fitting BikePoints_620
Fitting BikePoints_621
Fitting BikePoints_622
Fitting BikePoints_623
Fitting BikePoints_624
Fitting BikePoints_625
Fitting BikePoints_626
Fitting BikePoints_627
Fitting BikePoints_628
Fitting BikePoints_629
Fitting BikePoints_63
Fitting BikePoints_630
Fitting BikePoints_631
Fitting BikePoints_632
Fitting BikePoints_633
Fitting BikePoints_634
Fitting BikePoints_635
Fitting BikePoints_636
Fitting BikePoints_637
Fitting BikePoints_638
Fitting BikePoints_639
Fitting BikePoints_64
Fitting BikePoints_640
Fitting BikePoints_641
Fitting BikePoints_642
Fitting BikePoints_643
Fitting BikePoints_644
Fitting BikePoints_645
Fitting BikePoints_646
Fitting BikePoints_647
Fitting BikePoints_648
Fitting BikePoints_649
Fitting BikePoints_65
Fitting BikePoints_650
Fitting BikePoints_651
Fitting BikePoints_652
Fitting BikePoints_653
Fitting BikePoints_654
Fitting BikePoints_655
Fitting BikePoints_656
Fitting BikePoints_657
Fitting BikePoints_658
Fitting BikePoints_659
Fitting BikePoints_66
Fitting BikePoints_660
Fitting BikePoints_661
Fitting BikePoints_662
Fitting BikePoints_663
Fitting BikePoints_664
Fitting BikePoints_665
Fitting BikePoints_666
Fitting BikePoints_667
Fitting BikePoints_668
Fitting BikePoints_669
Fitting BikePoints_67
Fitting BikePoints_670
Fitting BikePoints_671
Fitting BikePoints_672
Fitting BikePoints_673
Fitting BikePoints_674
Fitting BikePoints_675
Fitting BikePoints_676
Fitting BikePoints_677
Fitting BikePoints_678
Fitting BikePoints_679
Fitting BikePoints_68
Fitting BikePoints_680
Fitting BikePoints_681
Fitting BikePoints_682
Fitting BikePoints_683
Fitting BikePoints_684
Fitting BikePoints_685
Fitting BikePoints_686
Fitting BikePoints_687
Fitting BikePoints_688
Fitting BikePoints_689
Fitting BikePoints_69
Fitting BikePoints_690
Fitting BikePoints_691
Fitting BikePoints_692
Fitting BikePoints_693
Fitting BikePoints_694
Fitting BikePoints_695
Fitting BikePoints_696
Fitting BikePoints_697
Fitting BikePoints_698
Fitting BikePoints_699
Fitting BikePoints_7
Fitting BikePoints_70
Fitting BikePoints_700
Fitting BikePoints_701
Fitting BikePoints_702
Fitting BikePoints_703
Fitting BikePoints_704
Fitting BikePoints_705
Fitting BikePoints_706
Fitting BikePoints_707
Fitting BikePoints_708
Fitting BikePoints_709
Fitting BikePoints_71
Fitting BikePoints_710
Fitting BikePoints_711
Fitting BikePoints_712
Fitting BikePoints_713
Fitting BikePoints_714
Fitting BikePoints_715
Fitting BikePoints_716
Fitting BikePoints_717
Fitting BikePoints_718
Fitting BikePoints_719
Fitting BikePoints_72
Fitting BikePoints_720
Fitting BikePoints_721
Fitting BikePoints_722
Fitting BikePoints_723
Fitting BikePoints_724
Fitting BikePoints_725
Fitting BikePoints_726
Fitting BikePoints_727
Fitting BikePoints_728
Fitting BikePoints_729
Fitting BikePoints_73
Fitting BikePoints_730
Fitting BikePoints_731
Fitting BikePoints_732
Fitting BikePoints_733
Fitting BikePoints_734
Fitting BikePoints_735
Fitting BikePoints_736
Fitting BikePoints_737
Fitting BikePoints_738
Fitting BikePoints_739
Fitting BikePoints_74
Fitting BikePoints_740
Fitting BikePoints_741
Fitting BikePoints_742
Fitting BikePoints_743
Fitting BikePoints_744
Fitting BikePoints_745
Fitting BikePoints_746
Fitting BikePoints_747
Fitting BikePoints_748
Fitting BikePoints_749
Fitting BikePoints_75
Fitting BikePoints_750
Fitting BikePoints_751
Fitting BikePoints_752
Fitting BikePoints_753
Fitting BikePoints_754
Fitting BikePoints_755
Fitting BikePoints_756
Fitting BikePoints_757
Fitting BikePoints_758
Fitting BikePoints_759
Fitting BikePoints_76
Fitting BikePoints_760
Fitting BikePoints_761
Fitting BikePoints_762
Fitting BikePoints_763
Fitting BikePoints_764
Fitting BikePoints_765
Fitting BikePoints_766
Fitting BikePoints_767
Fitting BikePoints_768
Fitting BikePoints_769
Fitting BikePoints_77
Fitting BikePoints_770
Fitting BikePoints_771
Fitting BikePoints_772
Fitting BikePoints_773
Fitting BikePoints_774
Fitting BikePoints_775
Fitting BikePoints_776
Fitting BikePoints_777
Fitting BikePoints_779
Fitting BikePoints_78
Fitting BikePoints_780
Fitting BikePoints_781
Fitting BikePoints_782
Fitting BikePoints_783
Fitting BikePoints_784
Fitting BikePoints_785
Fitting BikePoints_786
Fitting BikePoints_787
Fitting BikePoints_788
Fitting BikePoints_789
Fitting BikePoints_790
Fitting BikePoints_792
Fitting BikePoints_793
Fitting BikePoints_794
Fitting BikePoints_795
Fitting BikePoints_8
Fitting BikePoints_80
Fitting BikePoints_800
Fitting BikePoints_801
Fitting BikePoints_804
Fitting BikePoints_808
Fitting BikePoints_81
Fitting BikePoints_815
Fitting BikePoints_818
Fitting BikePoints_82
Fitting BikePoints_83
Fitting BikePoints_84
Fitting BikePoints_85
Fitting BikePoints_86
Fitting BikePoints_87
Fitting BikePoints_88
Fitting BikePoints_89
Fitting BikePoints_9
Fitting BikePoints_90
Fitting BikePoints_91
Fitting BikePoints_92
Fitting BikePoints_93
Fitting BikePoints_94
Fitting BikePoints_95
Fitting BikePoints_96
Fitting BikePoints_97
Fitting BikePoints_98
Fitting BikePoints_99

In [162]:
all_long_df = convert_results_to_df(all_long, 'LONG')

In [86]:
with open("data/parsed/models_long.p", "wb") as f:
    pickle.dump(all_long, f)


---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-86-fa10f8b1544e> in <module>()
      1 with open("data/parsed/models_long.p", "wb") as f:
----> 2     pickle.dump(all_long, f)

NameError: name 'all_long' is not defined

Present


In [21]:
with open("data/parsed/models_short.p", "rb") as f:
    models_short = pickle.load(f)
with open("data/parsed/models_mid.p", "rb") as f:
    models_mid = pickle.load(f)

In [163]:
short_df = convert_results_to_df(models_short, 'SHORT')
mid_df = convert_results_to_df(models_mid, 'MID')
long_df = convert_results_to_df(all_long, 'LONG')

In [164]:
long_df.describe()


Out[164]:
LONG-EXP0-GAM-TEST-ERR LONG-EXP0-GAM-TRAIN-ERR LONG-EXP0-GAM-VAL-ERR LONG-EXP0-HAVG-TEST-ERR LONG-EXP0-HAVG-TRAIN-ERR LONG-EXP0-HAVG-VAL-ERR LONG-EXP0-LR-TEST-ERR LONG-EXP0-LR-TRAIN-ERR LONG-EXP0-LR-VAL-ERR
count 765.000000 765.000000 765.000000 765.000000 765.000000 765.000000 765.000000 765.000000 765.000000
mean 5.545743 5.216309 5.736651 5.407190 5.348678 5.239801 5.519201 5.229735 5.651938
std 1.926344 1.777650 2.467123 1.821642 1.843964 2.042587 1.918340 1.779272 2.449580
min 2.294893 2.198929 1.766785 2.229457 2.237461 1.758201 2.294632 2.219586 1.800849
25% 4.182835 3.954905 4.044029 4.110624 4.046713 3.850569 4.167532 3.958752 3.972437
50% 5.229283 4.901323 5.190230 5.123685 5.042512 4.784700 5.224243 4.919091 5.156047
75% 6.541918 6.089167 6.839392 6.384363 6.273090 6.339870 6.515230 6.107812 6.755583
max 16.236486 14.388968 22.562941 14.011610 14.828793 16.309278 16.173550 14.426999 22.077257

In [165]:
short_df['Scenario'] = 'Short-Term'
short_df.set_index('Scenario', append=True, inplace=True)
short_df = short_df.reorder_levels(['Scenario', 'Id'])[['SHORT-EXP0-HAVG-TEST-ERR', 'SHORT-EXP0-LR-TEST-ERR', 'SHORT-EXP0-GAM-TEST-ERR']]
short_df.columns=['HAVG-TEST-ERR', 'LR-TEST-ERR', 'GAM-TEST-ERR']

mid_df['Scenario'] = 'Mid-Term'
mid_df.set_index('Scenario', append=True, inplace=True)
mid_df = mid_df.reorder_levels(['Scenario', 'Id'])[['MID-EXP0-HAVG-TEST-ERR', 'MID-EXP0-LR-TEST-ERR', 'MID-EXP0-GAM-TEST-ERR']]
mid_df.columns=['HAVG-TEST-ERR', 'LR-TEST-ERR', 'GAM-TEST-ERR']

long_df['Scenario'] = 'Long-Term'
long_df.set_index('Scenario', append=True, inplace=True)
long_df = long_df.reorder_levels(['Scenario', 'Id'])[['LONG-EXP0-HAVG-TEST-ERR', 'LONG-EXP0-LR-TEST-ERR', 'LONG-EXP0-GAM-TEST-ERR']]
long_df.columns=['HAVG-TEST-ERR', 'LR-TEST-ERR', 'GAM-TEST-ERR']

In [166]:
aggregations = {
    'HAVG-TEST-ERR': {
        'mean': 'mean',
        'std': 'std'
    },
    'LR-TEST-ERR': {
        'mean': 'mean',
        'std': 'std'
    },
    'GAM-TEST-ERR': {
        'mean': 'mean',
        'std': 'std'
    }
}

In [167]:
rmse_results = pd.concat([short_df, mid_df, long_df])
rmse_results['Metric'] = 'RMSE'

In [168]:
wrmse_results = pd.concat([short_df, mid_df, long_df]).merge(readings.groupby(level='Id')['NbDocks'].mean().to_frame(), how='inner', right_index=True, left_index=True)
wrmse_results['HAVG-TEST-ERR'] = wrmse_results['HAVG-TEST-ERR'] / wrmse_results.NbDocks
wrmse_results['LR-TEST-ERR'] = wrmse_results['LR-TEST-ERR'] / wrmse_results.NbDocks
wrmse_results['GAM-TEST-ERR'] = wrmse_results['GAM-TEST-ERR'] / wrmse_results.NbDocks
wrmse_results['Metric'] = 'WRMSE'
wrmse_results.drop('NbDocks', axis=1, inplace=True)

In [169]:
results = pd.concat([rmse_results, wrmse_results]).reset_index().groupby(['Metric', 'Scenario']).agg(aggregations)
results['HAVG-TEST-ERR', 'HAVG-TEST-ERR'] = results['HAVG-TEST-ERR', 'mean'].map(str) + " (" + results['HAVG-TEST-ERR', 'std'].map(str) + ")"
results['LR-TEST-ERR', 'LR-TEST-ERR'] = results['LR-TEST-ERR', 'mean'].map(str) + " (" + results['LR-TEST-ERR', 'std'].map(str) + ")"
results['GAM-TEST-ERR', 'GAM-TEST-ERR'] = results['GAM-TEST-ERR', 'mean'].map(str) + " (" + results['GAM-TEST-ERR', 'std'].map(str) + ")"
results.columns = results.columns.droplevel()
results = results[['HAVG-TEST-ERR', 'LR-TEST-ERR', 'GAM-TEST-ERR']]
results.index.name = None

In [170]:
results


Out[170]:
HAVG-TEST-ERR LR-TEST-ERR GAM-TEST-ERR
Metric Scenario
RMSE Long-Term 5.40719022993 (1.82164249869) 5.51920066142 (1.91833971277) 5.54574287899 (1.92634384331)
Mid-Term 5.4027172564 (1.82225202537) 2.60386000789 (0.993319219906) 2.37085914916 (0.85810201434)
Short-Term 5.39933259074 (1.81863390546) 1.04008594605 (0.409696989696) 1.02492873261 (0.396915776099)
WRMSE Long-Term 0.207424850855 (0.0379533695221) 0.211385316337 (0.0397903305265) 0.212344534885 (0.0396780703194)
Mid-Term 0.207443103395 (0.039066845878) 0.104578996929 (0.0394646190669) 0.0953142976369 (0.0348835951717)
Short-Term 0.2072065831 (0.0381709451086) 0.04231023103 (0.0183284081812) 0.0417400851698 (0.0180038973045)

In [121]:
print results.to_latex()


\begin{tabular}{lllll}
\toprule
     &           &                     HAVG-TEST-ERR &                       LR-TEST-ERR &                       GAM-TEST-ERR \\
Metric & Scenario &                                   &                                   &                                    \\
\midrule
RMSE & Long-Term &     5.40096005358 (1.82362774841) &     5.51920066142 (1.91833971277) &      5.53988620589 (1.93049824071) \\
     & Mid-Term &      5.4027172564 (1.82225202537) &    2.60386000789 (0.993319219906) &      2.37085914916 (0.85810201434) \\
     & Short-Term &     5.39933259074 (1.81863390546) &    1.04008594605 (0.409696989696) &     1.02492873261 (0.396915776099) \\
WRMSE & Long-Term &  0.207155675284 (0.0379307023412) &  0.211385316337 (0.0397903305265) &   0.212087492103 (0.0398891487769) \\
     & Mid-Term &   0.207443103395 (0.039066845878) &  0.104578996929 (0.0394646190669) &  0.0953142976369 (0.0348835951717) \\
     & Short-Term &    0.2072065831 (0.0381709451086) &   0.04231023103 (0.0183284081812) &  0.0417400851698 (0.0180038973045) \\
\bottomrule
\end{tabular}

Analysis


In [ ]:
with open("data/parsed/r_eval_short.p", "wb") as f:
    pickle.dump(training, f)

Short


In [81]:
baseline_short_df['SHORT-EXP0-GAM-TEST-ERR'].sort_values()[765/2:765/2 + 1]


Out[81]:
Id
BikePoints_148    0.956842
Name: SHORT-EXP0-GAM-TEST-ERR, dtype: float64

In [82]:
training, validation, test = split_datasets(readings_short, 'BikePoints_148')
with open("data/parsed/r_eval_short.p", "wb") as f:
    pickle.dump(training, f)

In [99]:
with open("data/parsed/r_eval_short.p", "rb") as f:
    r_eval_short = pickle.load(f)

In [100]:
%load_ext rpy2.ipython

robjects.globalenv['data'] = r_eval_short


The rpy2.ipython extension is already loaded. To reload it, use:
  %reload_ext rpy2.ipython

In [101]:
lr_formula_short


Out[101]:
'NbBikes ~ TempTMinus2 + HumidityTMinus2 + TimeOfDay + Weekday + NbBikesTMinus2 + NbBikesTMinus3 + RainTMinus2 + FogTMinus2 '

In [102]:
%%R

lrModel <- lm(NbBikes ~ TempTMinus2 + HumidityTMinus2 + TimeOfDay + Weekday + NbBikesTMinus2 + NbBikesTMinus3 + RainTMinus2 + FogTMinus2 , data=data)

summary(lrModel)


Call:
lm(formula = NbBikes ~ TempTMinus2 + HumidityTMinus2 + TimeOfDay + 
    Weekday + NbBikesTMinus2 + NbBikesTMinus3 + RainTMinus2 + 
    FogTMinus2, data = data)

Residuals:
    Min      1Q  Median      3Q     Max 
-8.9630 -0.1247  0.0250  0.1430  8.8390 

Coefficients: (1 not defined because of singularities)
                  Estimate Std. Error t value Pr(>|t|)    
(Intercept)      7.250e-02  1.170e-01   0.619   0.5356    
TempTMinus2     -6.350e-05  3.925e-03  -0.016   0.9871    
HumidityTMinus2  1.041e-03  8.439e-04   1.234   0.2172    
TimeOfDay        2.390e-06  4.427e-07   5.400 6.86e-08 ***
Weekday         -4.372e-02  2.412e-02  -1.812   0.0700 .  
NbBikesTMinus2   1.019e+00  1.581e-02  64.475  < 2e-16 ***
NbBikesTMinus3  -4.041e-02  1.584e-02  -2.552   0.0107 *  
RainTMinus2      7.854e-05  3.484e-02   0.002   0.9982    
FogTMinus2              NA         NA      NA       NA    
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.9008 on 8173 degrees of freedom
Multiple R-squared:  0.9624,	Adjusted R-squared:  0.9624 
F-statistic: 2.989e+04 on 7 and 8173 DF,  p-value: < 2.2e-16


In [85]:
%%R

library(mgcv)

gamModel <- gam(NbBikes ~ s(TempTMinus2, HumidityTMinus2, bs='tp', sp=30.0) + s(TimeOfDay, by=Weekday, bs='tp', sp=1.1) 
                + s(TimeOfDay, by=Weekend, bs='tp', sp=50.0) + s(TimeOfDay, by=Holiday, bs='tp', sp=0.2) 
                + s(NbBikesTMinus2, bs='tp', sp=8.0) + s(NbBikesTMinus3, bs='tp', sp=11.0) 
                + RainTMinus2 + FogTMinus2, data=data)

summary(gamModel)


Family: gaussian 
Link function: identity 

Formula:
NbBikes ~ s(TempTMinus2, HumidityTMinus2, bs = "tp", sp = 30) + 
    s(TimeOfDay, by = Weekday, bs = "tp", sp = 1.1) + s(TimeOfDay, 
    by = Weekend, bs = "tp", sp = 50) + s(TimeOfDay, by = Holiday, 
    bs = "tp", sp = 0.2) + s(NbBikesTMinus2, bs = "tp", sp = 8) + 
    s(NbBikesTMinus3, bs = "tp", sp = 11) + RainTMinus2 + FogTMinus2

Parametric coefficients:
            Estimate Std. Error t value Pr(>|t|)    
(Intercept)  6.84670    0.07532   90.90   <2e-16 ***
RainTMinus2 -0.00211    0.03535   -0.06    0.952    
FogTMinus2   0.00000    0.00000      NA       NA    
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Approximate significance of smooth terms:
                                 edf Ref.df       F p-value    
s(TempTMinus2,HumidityTMinus2) 6.597  8.952   1.003 0.41786    
s(TimeOfDay):Weekday           6.456  7.619 384.035 < 2e-16 ***
s(TimeOfDay):Weekend           2.562  3.033 723.942 < 2e-16 ***
s(TimeOfDay):Holiday           5.384  6.388   0.554 0.77827    
s(NbBikesTMinus2)              3.726  4.624 871.456 < 2e-16 ***
s(NbBikesTMinus3)              2.966  3.731   4.623 0.00398 ** 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Rank: 78/80
R-sq.(adj) =  0.964   Deviance explained = 96.4%
GCV = 0.78999  Scale est. = 0.78715   n = 8181

In [86]:
%%R

postscript("reports/images/short-check.eps",horizontal=FALSE, paper="special",height=6,width=10, onefile=TRUE)

gam.check(gamModel)

dev.off()


Method: GCV   Optimizer: magic
Model required no smoothing parameter selectionModel rank =  78 / 80 

Basis dimension (k) checking results. Low p-value (k-index<1) may
indicate that k is too low, especially if edf is close to k'.

                                   k'    edf k-index p-value
s(TempTMinus2,HumidityTMinus2) 29.000  6.597   1.002    0.56
s(TimeOfDay):Weekday           10.000  6.456   0.993    0.24
s(TimeOfDay):Weekend           10.000  2.562   0.993    0.33
s(TimeOfDay):Holiday           10.000  5.384   0.993    0.28
s(NbBikesTMinus2)               9.000  3.726   0.994    0.32
s(NbBikesTMinus3)               9.000  2.966   0.993    0.28
png 
  2 

In [87]:
%%R

postscript("reports/images/short-splines.eps",horizontal=FALSE, paper="special",height=6,width=10, onefile=TRUE)

layout(matrix(1:2, ncol = 2))
plot(gamModel, scale = 0)
layout(1)

dev.off()


png 
  2 

In [88]:
%%R

postscript("reports/images/short-correlation.eps",horizontal=FALSE, paper="special",height=6,width=10, onefile=TRUE)

layout(matrix(1:2, ncol = 2))
acf(resid(gamModel), lag.max = 36, main = "ACF")
pacf(resid(gamModel), lag.max = 36, main = "pACF")
layout(1)

dev.off()


png 
  2 

Mid


In [89]:
baseline_mid_df['MID-EXP0-GAM-TEST-ERR'].sort_values()[765/2:765/2 + 1]


Out[89]:
Id
BikePoints_231    2.224621
Name: MID-EXP0-GAM-TEST-ERR, dtype: float64

In [90]:
training, validation, test = split_datasets(readings_mid, 'BikePoints_231')
with open("data/parsed/r_eval_mid.p", "wb") as f:
    pickle.dump(training, f)

In [91]:
with open("data/parsed/r_eval_mid.p", "rb") as f:
    r_eval_mid = pickle.load(f)

In [92]:
%load_ext rpy2.ipython

robjects.globalenv['data'] = r_eval_mid


The rpy2.ipython extension is already loaded. To reload it, use:
  %reload_ext rpy2.ipython

In [93]:
gam_formula_mid


Out[93]:
"NbBikes ~ s(TempTMinus12, HumidityTMinus12, bs='tp') + s(TimeOfDay, by=Weekday, bs='tp') + s(TimeOfDay, by=Weekend, bs='tp') + s(TimeOfDay, by=Holiday, bs='tp') + s(NbBikesTMinus12, bs='tp') + s(NbBikesTMinus18, bs='tp') "

In [94]:
%%R

library(mgcv)

gamModel <- gam(NbBikes ~ s(TempTMinus12, HumidityTMinus12, bs='tp') + s(TimeOfDay, by=Weekday, bs='tp') 
                + s(TimeOfDay, by=Weekend, bs='tp') + s(TimeOfDay, by=Holiday, bs='tp') 
                + s(NbBikesTMinus12, bs='tp') + s(NbBikesTMinus18, bs='tp'), data=data)

summary(gamModel)


Family: gaussian 
Link function: identity 

Formula:
NbBikes ~ s(TempTMinus12, HumidityTMinus12, bs = "tp") + s(TimeOfDay, 
    by = Weekday, bs = "tp") + s(TimeOfDay, by = Weekend, bs = "tp") + 
    s(TimeOfDay, by = Holiday, bs = "tp") + s(NbBikesTMinus12, 
    bs = "tp") + s(NbBikesTMinus18, bs = "tp")

Parametric coefficients:
            Estimate Std. Error t value Pr(>|t|)    
(Intercept)  11.6694     0.5091   22.92   <2e-16 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Approximate significance of smooth terms:
                                    edf Ref.df       F  p-value    
s(TempTMinus12,HumidityTMinus12) 27.064 28.738   5.207  < 2e-16 ***
s(TimeOfDay):Weekday              9.101  9.576  71.227  < 2e-16 ***
s(TimeOfDay):Weekend              9.322  9.632  19.966  < 2e-16 ***
s(TimeOfDay):Holiday              3.566  4.203  13.835 1.58e-11 ***
s(NbBikesTMinus12)                8.135  8.781 369.863  < 2e-16 ***
s(NbBikesTMinus18)                7.925  8.691   4.188 1.73e-05 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Rank: 77/78
R-sq.(adj) =  0.923   Deviance explained = 92.4%
GCV = 4.9535  Scale est. = 4.9136    n = 8166

In [95]:
%%R

postscript("reports/images/mid-check.eps",horizontal=FALSE, paper="special",height=6,width=10, onefile=TRUE)

gam.check(gamModel)

dev.off()


Method: GCV   Optimizer: magic
Smoothing parameter selection converged after 21 iterations.
The RMS GCV score gradiant at convergence was 1.054994e-06 .
The Hessian was positive definite.
The estimated model rank was 77 (maximum possible: 78)
Model rank =  77 / 78 

Basis dimension (k) checking results. Low p-value (k-index<1) may
indicate that k is too low, especially if edf is close to k'.

                                     k'    edf k-index p-value
s(TempTMinus12,HumidityTMinus12) 29.000 27.064   0.819    0.00
s(TimeOfDay):Weekday             10.000  9.101   1.010    0.74
s(TimeOfDay):Weekend             10.000  9.322   1.010    0.72
s(TimeOfDay):Holiday             10.000  3.566   1.010    0.72
s(NbBikesTMinus12)                9.000  8.135   0.986    0.12
s(NbBikesTMinus18)                9.000  7.925   0.993    0.30
png 
  2 

In [96]:
%%R

postscript("reports/images/mid-splines.eps",horizontal=FALSE, paper="special",height=6,width=10, onefile=TRUE)

layout(matrix(1:2, ncol = 2))
plot(gamModel, scale = 0)
layout(1)

dev.off()


png 
  2 

In [97]:
%%R

postscript("reports/images/mid-correlation.eps",horizontal=FALSE, paper="special",height=6,width=10, onefile=TRUE)

layout(matrix(1:2, ncol = 2))
acf(resid(gamModel), lag.max = 36, main = "ACF")
pacf(resid(gamModel), lag.max = 36, main = "pACF")
layout(1)

dev.off()


png 
  2 

Long


In [171]:
all_long_df['LONG-EXP0-GAM-TEST-ERR'].sort_values()[765/2:765/2 + 1]


Out[171]:
Id
BikePoints_610    5.229283
Name: LONG-EXP0-GAM-TEST-ERR, dtype: float64

In [40]:
gam_formula_long


Out[40]:
"NbBikes ~ s(TimeOfDay, by=Weekday, bs='tp', sp=35.0) + s(TimeOfDay, by=Weekend, bs='tp', sp=0.7) + s(TimeOfDay, by=Holiday, bs='tp', sp=3.0) + s(HistAvg, bs='tp', k=5) + CollNbBikesCum6 + DistNbBikesCum6 "

In [172]:
training, validation, test = split_datasets(readings_long, 'BikePoints_610')
with open("data/parsed/r_eval_long.p", "wb") as f:
    pickle.dump(training, f)

In [173]:
with open("data/parsed/r_eval_long.p", "rb") as f:
    r_eval_long = pickle.load(f)

In [174]:
%load_ext rpy2.ipython

robjects.globalenv['data'] = r_eval_long

In [175]:
%%R

library(mgcv)

gamModel <- gam(NbBikes ~ s(TimeOfDay, by=Weekday, bs='tp', sp=35.0) + s(TimeOfDay, by=Weekend, bs='tp', sp=0.7) 
                + s(TimeOfDay, by=Holiday, bs='tp', sp=3.0) + s(HistAvg, bs='tp', k=5) 
                + CollNbBikesCum6 + DistNbBikesCum6 , data=data)

summary(gamModel)


Family: gaussian 
Link function: identity 

Formula:
NbBikes ~ s(TimeOfDay, by = Weekday, bs = "tp", sp = 35) + s(TimeOfDay, 
    by = Weekend, bs = "tp", sp = 0.7) + s(TimeOfDay, by = Holiday, 
    bs = "tp", sp = 3) + s(HistAvg, bs = "tp", k = 5) + CollNbBikesCum6 + 
    DistNbBikesCum6

Parametric coefficients:
                Estimate Std. Error t value Pr(>|t|)    
(Intercept)       7.8811     0.5587  14.105  < 2e-16 ***
CollNbBikesCum6   0.3442     0.2183   1.577  0.11490    
DistNbBikesCum6  -0.3834     0.1373  -2.792  0.00526 ** 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Approximate significance of smooth terms:
                       edf Ref.df       F  p-value    
s(TimeOfDay):Weekday 3.250  3.864  11.927 3.41e-09 ***
s(TimeOfDay):Weekend 6.037  7.170  10.201 6.76e-13 ***
s(TimeOfDay):Holiday 3.281  3.852   0.274    0.889    
s(HistAvg)           1.000  1.000 146.366  < 2e-16 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Rank: 36/37
R-sq.(adj) =  0.0784   Deviance explained = 8.01%
GCV = 23.913  Scale est. = 23.866    n = 8184

In [177]:
%%R

postscript("reports/images/long-check.eps",horizontal=FALSE, paper="special",height=6,width=10, onefile=TRUE)

gam.check(gamModel)

dev.off()


Method: GCV   Optimizer: magic
Smoothing parameter selection converged after 8 iterations.
The RMS GCV score gradiant at convergence was 1.684521e-06 .
The Hessian was positive definite.
The estimated model rank was 36 (maximum possible: 37)
Model rank =  36 / 37 

Basis dimension (k) checking results. Low p-value (k-index<1) may
indicate that k is too low, especially if edf is close to k'.

                        k'   edf k-index p-value
s(TimeOfDay):Weekday 10.00  3.25    1.05    1.00
s(TimeOfDay):Weekend 10.00  6.04    1.05    1.00
s(TimeOfDay):Holiday 10.00  3.28    1.05    1.00
s(HistAvg)            4.00  1.00    0.97    0.02
png 
  2 

In [178]:
%%R

postscript("reports/images/long-splines.eps",horizontal=FALSE, paper="special",height=6,width=10, onefile=TRUE)

layout(matrix(1:2, ncol = 2))
plot(gamModel, scale = 0)
layout(1)

dev.off()


png 
  2 

In [179]:
%%R

postscript("reports/images/long-correlation.eps",horizontal=FALSE, paper="special",height=6,width=10, onefile=TRUE)

layout(matrix(1:2, ncol = 2))
acf(resid(gamModel), lag.max = 36, main = "ACF")
pacf(resid(gamModel), lag.max = 36, main = "pACF")
layout(1)

dev.off()


png 
  2 

Visualization Demo


In [48]:
%run src/data/visualization.py

In [93]:
now = datetime(year=2016, month=6, day=22, hour=9, minute=0, second=0)
single_reading = readings.xs(now, level='Timestamp')

In [102]:
predictions = []
for idx, row in single_reading.iterrows():
    gam = baseline_mid_df.loc[idx]['MID-EXP0-GAM-MOD']
    prediction = gam.predict(row.to_frame().transpose())[0]
    predictions.append({'Prediction': prediction, 'Id': idx, 
                        'NbDocks': row.NbDocks, 
                        'HistAvg': row.HistAvg,
                        'Current': row.NbBikesTMinus12})
predictions = pd.DataFrame(predictions).set_index('Id')

In [103]:
predictions_df = add_station_info(predictions, stations.set_index('Id'), use_indexes=True)
predictions_df['NormalizedPrediction'] = predictions_df.Prediction.astype(float) / predictions_df.NbDocks.astype(float)

In [168]:
from palettable.colorbrewer.diverging import RdYlBu_10

def create_prediction_marker(station):
    
    font_color_now = 'red' if station.Current < 5 else 'black'
    font_color_mid = 'red' if station.Prediction < 5 else 'black'
    font_color_long = 'red' if station.HistAvg < 5 else 'black'
    
    html="""
    <h1>%s</h1>
    <p><strong>Id</strong>: %s</p>
    <p>
    <strong>Bicycle Availability:</strong>
    <ul>
        <font color="%s"><li><strong>%s:</strong> %d bicycles</li></font>
        <font color="%s"><li><strong>%s: </strong> %d bicycles</li></font>
        <font color="%s"><li><strong>%s: </strong> %d bicycles</li></font>
    </ul>
    </p>
    <p>
        <strong>Availability Trend:</strong><br/><br/>
        <img src="http://localhost:8888/files/reports/images/trend.png" width=210 height=210 hspace=16/>
    </p>
    """ % (station.Name, station.name, 
           font_color_now, now.strftime('%d/%m - %H:%M'), station.Current, 
           font_color_mid, (now + timedelta(hours=1)).strftime('%d/%m - %H:%M'), station.Prediction,
           font_color_long, (now + timedelta(hours=3)).strftime('%d/%m - %H:%M'), station.HistAvg)
    iframe = folium.element.IFrame(html=html, width=350, height=530)
    popup = folium.Popup(iframe, max_width=2650)
        
    fill_color = cmap_to_hex(RdYlBu_10.mpl_colormap, station.NormalizedPrediction)
    label = "%s - %s: %d bicycle(s)" 
    
    return folium.CircleMarker(location=[station['Latitude'], station['Longitude']], radius=100,
                        popup=popup, fill_color=fill_color, color=fill_color, fill_opacity=0.99)

predictions_map = draw_stations_map(predictions_df, create_prediction_marker)

In [169]:
folium.TileLayer('stamentoner').add_to(predictions_map)
folium.Map.save(predictions_map, 'reports/maps/predicted_availability_map.html')
predictions_map


Out[169]:

In [ ]: