In [ ]:
%load_ext autoreload
%autoreload 2
In [ ]:
import numpy as np
import pandas as pd
import sys
import datetime
import os
import matplotlib.pyplot as plt
import matplotlib
import networkx as nx
import pickle
from collections import OrderedDict
import copy
from scipy.sparse import csr_matrix
from scipy import io
import seaborn as sns
import joblib
# from base import *
from joblib import Parallel, delayed
import random
import scipy
In [ ]:
import matplotlib.pyplot as plt
from matplotlib import colors
import matplotlib
import six
import matplotlib.dates as mdates
import datetime
import pandas as pd
import seaborn as sns
sns.set()
plt.style.use('seaborn-poster')
from sklearn.metrics import r2_score
In [ ]:
MNM_nb_folder = os.path.join('..', '..', '..', 'side_project', 'network_builder')
sys.path.append(MNM_nb_folder)
python_lib_folder = os.path.join('..', '..', 'pylib')
sys.path.append(python_lib_folder)
In [ ]:
from MNMAPI import *
from MNM_mcnb import *
from mcDODE import *
In [ ]:
data_folder = os.path.join('/home/lemma/Documents/MAC-POSTS/data/input_files_7link_multiclass_new')
In [ ]:
nb = MNM_network_builder()
nb.load_from_folder(data_folder)
In [ ]:
# nb.dump_to_folder('test')
In [ ]:
from sklearn.metrics import r2_score
from scipy import stats
def r2(predictions, targets):
y_bar = np.mean(targets)
# diff = np.minimum(np.abs(predictions - targets), targets)
diff = predictions - targets
ss_e = np.sum(diff ** 2)
ss_t = np.sum((targets) ** 2)
return 1 - ss_e / ss_t
def rsquared(x, y):
slope, intercept, r_value, p_value, std_err = stats.linregress(x, y)
return r_value**2
def rmse(predictions, targets):
return np.sqrt(((predictions - targets) ** 2).mean())
def rmsn(predictions, targets):
return np.sqrt(np.sum((predictions - targets) ** 2) * len(predictions)) / np.sum(targets)
In [ ]:
observed_link_list = [3, 4, 5, 6]
ml_car = 6
ml_truck = 5
data_dict = dict()
num_interval = nb.config.config_dict['DTA']['max_interval']
# true_car_f = np.random.rand(num_interval * nb.config.config_dict['FIXED']['num_path']) * 300
# true_truck_f = np.random.rand(num_interval * nb.config.config_dict['FIXED']['num_path']) * 30
true_car_f, true_truck_f, _, _, _, _ = pickle.load(open('final_use.pickle', 'r'))
# true_car_x = np.random.rand(num_interval * len(observed_link_list)) * 100
# true_truck_x = np.random.rand(num_interval * len(observed_link_list)) * 10
# L_car_one = np.random.randint(2, size = (ml_car, len(observed_link_list)))
L_car_one = np.array([[1, 0, 0, 1],
[0, 0, 1, 1],
[1, 1, 0, 1],
[1, 0, 1, 1],
[1, 0, 0, 0],
[0, 1, 0, 1]])
L_truck_one = np.array([[1, 0, 0, 1],
[0, 0, 0, 1],
[1, 1, 0, 1],
[1, 0, 1, 0],
[0, 1, 0, 1]])
# L_truck_one = np.random.randint(2, size = (ml_truck, len(observed_link_list)))
L_car = csr_matrix(scipy.linalg.block_diag(*[L_car_one for i in range(num_interval)]))
L_truck = csr_matrix(scipy.linalg.block_diag(*[L_truck_one for i in range(num_interval)]))
config = dict()
config['use_car_link_flow'] = True
config['use_truck_link_flow'] = True
config['use_car_link_tt'] = True
config['use_truck_link_tt'] = True
config['car_count_agg'] = True
config['truck_count_agg'] = True
config['link_car_flow_weight'] = 1
config['link_truck_flow_weight'] = 1
config['link_car_tt_weight'] = 0.1
config['link_truck_tt_weight'] = 0.1
config['num_data'] = 8
config['observed_links'] = observed_link_list
config['paths_list'] = range(nb.config.config_dict['FIXED']['num_path'])
config['compute_car_link_flow_loss'] = True
config['compute_truck_link_flow_loss'] = True
config['compute_car_link_tt_loss'] = True
config['compute_truck_link_tt_loss'] = True
dode = MCDODE(nb, config)
dta = dode._run_simulation(true_car_f, true_truck_f)
true_car_dar, true_truck_dar = dode.get_dar(dta, true_car_f, true_truck_f)
noise_level = 0.1
(true_dar_car, true_dar_truck) = dode.get_dar(dta, true_car_f, true_truck_f)
true_car_x = true_dar_car.dot(true_car_f)
true_truck_x = true_dar_truck.dot(true_truck_f)
data_dict['car_count_agg_L_list'] = list()
data_dict['truck_count_agg_L_list'] = list()
data_dict['car_link_flow'] = []
data_dict['truck_link_flow'] = []
data_dict['car_link_tt'] = []
data_dict['truck_link_tt'] = []
for i in range(config['num_data']):
true_car_x = dta.get_link_car_inflow(np.arange(0, dode.num_loading_interval, dode.ass_freq),
np.arange(0, dode.num_loading_interval, dode.ass_freq) + dode.ass_freq).flatten(order = 'F')
true_truck_x = dta.get_link_truck_inflow(np.arange(0, dode.num_loading_interval, dode.ass_freq),
np.arange(0, dode.num_loading_interval, dode.ass_freq) + dode.ass_freq).flatten(order = 'F')
# true_car_tt = dta.get_car_link_tt_robust(np.arange(0, dode.num_loading_interval, dode.ass_freq),
# np.arange(0, dode.num_loading_interval, dode.ass_freq) + dode.ass_freq).flatten(order = 'F')
true_car_tt = dta.get_car_link_tt(np.arange(0, dode.num_loading_interval, dode.ass_freq)).flatten(order = 'F')
true_truck_tt = dta.get_truck_link_tt(np.arange(0, dode.num_loading_interval, dode.ass_freq)).flatten(order = 'F')
m_car = L_car.dot(true_car_x)
m_truck = L_truck.dot(true_truck_x)
data_dict['car_count_agg_L_list'].append(L_car)
data_dict['truck_count_agg_L_list'].append(L_truck)
data_dict['car_link_flow'].append(m_car + np.random.uniform(-1, 1, m_car.shape) * noise_level * m_car)
data_dict['truck_link_flow'].append(m_truck + np.random.uniform(-1, 1, m_truck.shape) * noise_level * m_truck)
data_dict['car_link_tt'].append(true_car_tt + np.random.uniform(-1, 1, true_car_tt.shape) * noise_level * true_car_tt)
data_dict['truck_link_tt'].append(true_truck_tt + np.random.uniform(-1, 1, true_truck_tt.shape) * noise_level * true_truck_tt)
In [ ]:
np.max(np.abs(true_car_dar.dot(true_car_f) - true_car_x))
In [ ]:
np.max(np.abs(true_truck_dar.dot(true_truck_f) - true_truck_x))
In [ ]:
true_truck_x
In [ ]:
np.linalg.matrix_rank(np.matrix(L_car_one))
In [ ]:
np.linalg.matrix_rank(np.matrix(L_truck_one))
In [ ]:
dode.paths_list
In [ ]:
true_car_tt
In [ ]:
true_truck_x
In [ ]:
true_car_x
In [ ]:
m_truck
In [ ]:
In [ ]:
dode = MCDODE(nb, config)
In [ ]:
dode.add_data(data_dict)
In [ ]:
# nb.update_demand_path2(true_car_f, true_truck_f)
# nb.dump_to_folder("one")
# nb.update_demand_path2(car_flow, truck_flow)
# nb.dump_to_folder("two")
In [ ]:
# (car_flow, truck_flow) = dode.init_path_flow(car_scale = 10, truck_scale = 1)
In [ ]:
# pickle.dump((car_flow, truck_flow, None), open('test.pickle', 'w'))
In [ ]:
(car_flow, truck_flow, l_list) = dode.estimate_path_flow(max_epoch = 100, car_step_size = 1,
truck_step_size = 0.1, car_init_scale = 5,
truck_init_scale = 1, adagrad = True)
print r2_score(car_flow, true_car_f), r2_score(truck_flow, true_truck_f)
In [ ]:
pickle.dump([true_car_f, true_truck_f, car_flow, truck_flow, l_list, data_dict], open('final_use2.pickle', 'w'))
In [ ]:
color_list = ['teal', 'tomato', 'blue', 'sienna', 'plum', 'red', 'yellowgreen', 'khaki']
marker_list = ["o", "v", "^", "<", ">", "p", "D","*","s", "D", "p"]
In [ ]:
_, _, car_flow, truck_flow, l_list, _ = pickle.load(open('final_use2.pickle', 'r'))
In [ ]:
plt.figure(figsize = (16,9), dpi=300)
plt.plot(np.arange(len(l_list))+1, list(map(lambda x: x[1]['truck_tt_loss']/l_list[0][1]['truck_tt_loss'], l_list)),
color = color_list[0], linewidth = 3, label = "Truck observed travel cost")
plt.plot(np.arange(len(l_list))+1, list(map(lambda x: x[1]['car_tt_loss']/l_list[0][1]['car_tt_loss'], l_list)),
color = color_list[1], linewidth = 3, label = "Car observed travel cost")
plt.plot(np.arange(len(l_list))+1, list(map(lambda x: x[1]['car_count_loss']/l_list[0][1]['car_count_loss'], l_list)),
color = color_list[2], linewidth = 3, label = "Car observed flow")
plt.plot(np.arange(len(l_list))+1, list(map(lambda x: x[1]['truck_count_loss']/l_list[0][1]['truck_count_loss'], l_list)),
color = color_list[3], linewidth = 3, label = "Truck observed flow")
# plt.plot(range(len(l_list)), list(map(lambda x: x[0]/8, l_list)),
# color = color_list[4], linewidth = 3, label = "Total cost")
plt.ylabel('Loss function')
plt.xlabel('Iteration')
plt.legend()
plt.ylim([0, 1])
plt.xlim([1, 100])
plt.show()
In [ ]:
# (car_flow, truck_flow, l_list) = dode.estimate_path_flow_mp(max_epoch = 200, step_size = 0.1, car_init_scale = 5,
# truck_init_scale = 0.1,
# adagrad = True, n_process = 8)
In [ ]:
# (car_flow, truck_flow, l_list) = dode.estimate_path_flow(max_epoch = 200, step_size = 0.1, car_init_scale = 5,
# truck_init_scale = 0.1, adagrad = True)
In [ ]:
# plt.figure(figsize = (16,9), dpi=300)
# plt.plot(range(1,151), l_list, color = 'tomato', linewidth = 3)
# plt.ylabel('Loss')
# plt.xlabel('Iteration')
# plt.xlim([1, 150])
# plt.show()
In [ ]:
dta = dode._run_simulation(true_car_f, truck_flow)
estimated_car_x = dta.get_link_car_inflow(np.arange(0, dode.num_loading_interval, dode.ass_freq),
np.arange(0, dode.num_loading_interval, dode.ass_freq) + dode.ass_freq).flatten(order = 'F')
estimated_truck_x = dta.get_link_truck_inflow(np.arange(0, dode.num_loading_interval, dode.ass_freq),
np.arange(0, dode.num_loading_interval, dode.ass_freq) + dode.ass_freq).flatten(order = 'F')
In [ ]:
fig, axes = plt.subplots(1,2, figsize=(16, 9), dpi=300)
# plt.figure(num=None, figsize=(16, 9), dpi=300, facecolor='w', edgecolor='k')
axes[0].scatter(m_car, L_car.dot(estimated_car_x), label = "Proposed direct solution", color = 'teal', marker = '*', s = 100)
axes[0].plot(range(350), range(350), label = "Convex optimization solution", color = 'gray')
axes[1].scatter(m_truck, L_truck.dot(estimated_truck_x), label = "Proposed closed-form method", color = 'tomato',
marker = "o", s = 100)
axes[1].plot(range(350), range(350), label = "Convex optimization solution", color = 'gray')
# axes[1].plot(time_list2, gap_rec2, label = "Quadratic solver method", color = 'tomato')
# plt.ylabel("Equilibrium Gap")
# plt.xlabel("Iterations")
# plt.legend()
# plt.xlim([1, 100])
# axes[1].legend(loc='center left', bbox_to_anchor=(1, 0.5))
# axes[0].set_title('Path flow rate (vehs/15-min)')
# axes[1].set_title('Path choice probablity')
axes[0].set_ylabel('Estimated otrue_car_ttbserved flow for car')
axes[0].set_xlabel('True observed flow for car')
axes[1].set_ylabel('Estimated observed flow for truck')
axes[1].set_xlabel('True observed flow for truck')
axes[0].set_xlim([0, 350])
axes[0].set_ylim([0, 350])
axes[1].set_xlim([0, 50])
axes[1].set_ylim([0, 50])
plt.show()
In [ ]:
In [ ]:
e_m_truck =L_truck.dot(estimated_truck_x)
print rmsn(m_car, L_car.dot(estimated_car_x))
print rmsn(m_truck[e_m_truck < 50], e_m_truck[e_m_truck < 50])
In [ ]:
fig, axes = plt.subplots(1,2, figsize=(16, 9), dpi=300)
# plt.figure(num=None, figsize=(16, 9), dpi=300, facecolor='w', edgecolor='k')
axes[0].scatter(true_car_tt, dta.get_car_link_tt(np.arange(0, dode.num_loading_interval, dode.ass_freq)).flatten(order = 'F'), label = "Proposed direct solution", color = 'teal', marker = '*', s = 100)
axes[0].plot(range(350), range(350), label = "Convex optimization solution", color = 'gray')
axes[1].scatter(true_truck_tt, dta.get_truck_link_tt(np.arange(0, dode.num_loading_interval, dode.ass_freq)).flatten(order = 'F'), label = "Proposed closed-form method", color = 'tomato',
marker = "o", s = 100)
axes[1].plot(range(350), range(350), label = "Convex optimization solution", color = 'gray')
# axes[1].plot(time_list2, gap_rec2, label = "Quadratic solver method", color = 'tomato')
# plt.ylabel("Equilibrium Gap")
# plt.xlabel("Iterations")
# plt.legend()
# plt.xlim([1, 100])
# axes[1].legend(loc='center left', bbox_to_anchor=(1, 0.5))
# axes[0].set_title('Path flow rate (vehs/15-min)')
# axes[1].set_title('Path choice probablity')
axes[0].set_ylabel('Estimated observed flow for car')
axes[0].set_xlabel('True observed flow for car')
axes[1].set_ylabel('Estimated observed flow for truck')
axes[1].set_xlabel('True observed flow for truck')
axes[0].set_xlim([0, 120])
axes[0].set_ylim([0, 120])
axes[1].set_xlim([0, 200])
axes[1].set_ylim([0, 200])
plt.show()
In [ ]:
est_car_tt = dta.get_car_link_tt(np.arange(0, dode.num_loading_interval, dode.ass_freq)).flatten(order = 'F')
print r2(true_car_tt, est_car_tt)
est_truck_tt = dta.get_truck_link_tt(np.arange(0, dode.num_loading_interval, dode.ass_freq)).flatten(order = 'F')
print r2(true_truck_tt, est_truck_tt)
In [ ]:
dta.get_car_link_tt(np.arange(0, dode.num_loading_interval, dode.ass_freq)).flatten(order = 'F')
In [ ]:
fig, axes = plt.subplots(1,2, figsize=(16, 9), dpi=300)
# plt.figure(num=None, figsize=(16, 9), dpi=300, facecolor='w', edgecolor='k')
axes[0].scatter(true_car_x, estimated_car_x, label = "Proposed direct solution", color = 'teal', marker = '*', s = 100)
axes[0].plot(range(350), range(350), label = "Convex optimization solution", color = 'gray')
axes[1].scatter(true_truck_x, estimated_truck_x, label = "Proposed closed-form method", color = 'tomato',
marker = "o", s = 100)
axes[1].plot(range(350), range(350), label = "Convex optimization solution", color = 'gray')
# axes[1].plot(time_list2, gap_rec2, label = "Quadratic solver method", color = 'tomato')
# plt.ylabel("Equilibrium Gap")
# plt.xlabel("Iterations")
# plt.legend()
# plt.xlim([1, 100])
# axes[1].legend(loc='center left', bbox_to_anchor=(1, 0.5))
# axes[0].set_title('Path flow rate (vehs/15-min)')
# axes[1].set_title('Path choice probablity')
axes[0].set_ylabel('Estimated link flow for car')
axes[0].set_xlabel('True link flow for car')
axes[1].set_ylabel('Estimated link flow for truck')
axes[1].set_xlabel('True link flow for truck')
axes[0].set_xlim([0, 200])
axes[0].set_ylim([0, 200])
axes[1].set_xlim([0, 50])
axes[1].set_ylim([0, 50])
plt.show()
In [ ]:
print r2(true_car_x, estimated_car_x)
print r2(true_truck_x[estimated_truck_x<30], estimated_truck_x[estimated_truck_x<30])
In [ ]:
true_car_q = true_car_f.reshape(3, -1, order = 'F').sum(axis = 0)
est_car_q = car_flow.reshape(3, -1, order = 'F').sum(axis = 0)
true_truck_q = true_truck_f.reshape(3, -1, order = 'F').sum(axis = 0)
est_truck_q = truck_flow.reshape(3, -1, order = 'F').sum(axis = 0)
fig, axes = plt.subplots(1,2, figsize=(16, 9), dpi=300)
# truck_flow = true_truck_f + np.random.rand(len(true_car_f)) * 2- 1
# plt.figure(num=None, figsize=(16, 9), dpi=300, facecolor='w', edgecolor='k')
axes[0].scatter(true_car_q, est_car_q, label = "Proposed direct solution", color = 'teal', marker = '*', s = 100)
axes[0].plot(range(350), range(350), label = "Convex optimization solution", color = 'gray')
axes[1].scatter(true_truck_q, est_truck_q, label = "Proposed closed-form method", color = 'tomato',
marker = "o", s = 100)
axes[1].plot(range(350), range(350), label = "Convex optimization solution", color = 'gray')
# axes[1].plot(time_list2, gap_rec2, label = "Quadratic solver method", color = 'tomato')
# plt.ylabel("Equilibrium Gap")
# plt.xlabel("Iterations")
# plt.legend()
# plt.xlim([1, 100])
# axes[1].legend(loc='center left', bbox_to_anchor=(1, 0.5))
# axes[0].set_title('Path flow rate (vehs/15-min)')
# axes[1].set_title('Path choice probablity')
axes[0].set_ylabel('Estimated OD flow for car')
axes[0].set_xlabel('True OD flow for car')
axes[1].set_ylabel('Estimated OD flow for truck')
axes[1].set_xlabel('True OD flow for truck')
axes[0].set_xlim([0, 350])
axes[0].set_ylim([0, 350])
axes[1].set_xlim([0, 80])
axes[1].set_ylim([0, 80])
plt.show()
In [ ]:
print r2(true_car_q, est_car_q)
print r2(true_truck_q, est_truck_q)
In [ ]:
config = dict()
config['use_car_link_flow'] = True
config['use_truck_link_flow'] = True
config['use_car_link_tt'] = False
config['use_truck_link_tt'] = False
config['car_count_agg'] = True
config['truck_count_agg'] = True
config['link_car_flow_weight'] = 1
config['link_truck_flow_weight'] = 1
config['link_car_tt_weight'] = 0.1
config['link_truck_tt_weight'] = 0.1
config['num_data'] = 8
config['observed_links'] = observed_link_list
config['paths_list'] = range(nb.config.config_dict['FIXED']['num_path'])
config['compute_car_link_flow_loss'] = True
config['compute_truck_link_flow_loss'] = True
config['compute_car_link_tt_loss'] = True
config['compute_truck_link_tt_loss'] = True
In [ ]:
dode = MCDODE(nb, config)
dode.add_data(data_dict)
In [ ]:
(car_flow, truck_flow, l_list) = dode.estimate_path_flow(max_epoch = 100, car_step_size = 1,
truck_step_size = 0.1, car_init_scale = 5,
truck_init_scale = 1, adagrad = True)
print r2_score(car_flow, true_car_f), r2_score(truck_flow, true_truck_f)
In [ ]:
# pickle.dump([true_car_f, true_truck_f, car_flow, truck_flow, l_list, data_dict], open('no_spd.pickle', 'w'))
In [ ]:
true_car_q = true_car_f.reshape(3, -1, order = 'F').sum(axis = 0)
est_car_q = car_flow.reshape(3, -1, order = 'F').sum(axis = 0)
true_truck_q = true_truck_f.reshape(3, -1, order = 'F').sum(axis = 0)
est_truck_q = truck_flow.reshape(3, -1, order = 'F').sum(axis = 0)
In [ ]:
print r2_score(true_car_q, est_car_q)
print r2_score(true_truck_q, est_truck_q)
In [ ]:
_, _, _, _, l_list, _ = pickle.load(open('final_use.pickle', 'r'))
_, _, _, _, l_list_nospd, _ = pickle.load(open('no_spd.pickle', 'r'))
In [ ]:
plt.figure(figsize = (16,9), dpi=300)
plt.plot(np.arange(len(l_list))+1, list(map(lambda x: x[1]['car_count_loss'], l_list)),
color = color_list[0], linewidth = 3, label = "Truck observed travel cost")
plt.plot(np.arange(len(l_list))+1, list(map(lambda x: x[1]['car_count_loss'], l_list_nospd)),
color = color_list[1], linewidth = 3, label = "ooo")
# plt.plot(np.arange(len(l_list))+1, list(map(lambda x: x[1]['car_count_loss']/10, l_list)),
# color = color_list[2], linewidth = 3, label = "Car observed flow")
# plt.plot(np.arange(len(l_list))+1, list(map(lambda x: x[1]['truck_count_loss'], l_list)),
# color = color_list[3], linewidth = 3, label = "Truck observed flow")
# plt.plot(range(len(l_list)), list(map(lambda x: x[0]/8, l_list)),
# color = color_list[4], linewidth = 3, label = "Total cost")
plt.ylabel('Loss function')
plt.xlabel('Iteration')
plt.legend()
plt.xlim([1, 100])
plt.show()
In [ ]:
observed_link_list = [3, 4, 5, 6]
ml_car = 6
ml_truck = 5
data_dict = dict()
num_interval = nb.config.config_dict['DTA']['max_interval']
# true_car_f = np.random.rand(num_interval * nb.config.config_dict['FIXED']['num_path']) * 300
# true_truck_f = np.random.rand(num_interval * nb.config.config_dict['FIXED']['num_path']) * 30
true_car_f, true_truck_f, _, _, _, _ = pickle.load(open('final_use.pickle', 'r'))
# true_car_x = np.random.rand(num_interval * len(observed_link_list)) * 100
# true_truck_x = np.random.rand(num_interval * len(observed_link_list)) * 10
# L_car_one = np.random.randint(2, size = (ml_car, len(observed_link_list)))
L_car_one = np.array([[1, 0, 0, 1],
[0, 0, 1, 1],
[1, 1, 0, 1],
[1, 0, 1, 1],
[1, 0, 0, 0],
[0, 1, 0, 1]])
L_truck_one = np.array([[1, 0, 0, 1],
[0, 0, 0, 1],
[1, 1, 0, 1],
[1, 0, 1, 0],
[0, 1, 0, 1]])
# L_truck_one = np.random.randint(2, size = (ml_truck, len(observed_link_list)))
L_car = csr_matrix(scipy.linalg.block_diag(*[L_car_one for i in range(num_interval)]))
L_truck = csr_matrix(scipy.linalg.block_diag(*[L_truck_one for i in range(num_interval)]))
config = dict()
config['use_car_link_flow'] = True
config['use_truck_link_flow'] = True
config['use_car_link_tt'] = True
config['use_truck_link_tt'] = True
config['car_count_agg'] = True
config['truck_count_agg'] = True
config['link_car_flow_weight'] = 1
config['link_truck_flow_weight'] = 1
config['link_car_tt_weight'] = 0.1
config['link_truck_tt_weight'] = 0.1
config['num_data'] = 8
config['observed_links'] = observed_link_list
config['paths_list'] = range(nb.config.config_dict['FIXED']['num_path'])
config['compute_car_link_flow_loss'] = True
config['compute_truck_link_flow_loss'] = True
config['compute_car_link_tt_loss'] = True
config['compute_truck_link_tt_loss'] = True
dode = MCDODE(nb, config)
dta = dode._run_simulation(true_car_f, true_truck_f)
true_car_dar, true_truck_dar = dode.get_dar(dta, true_car_f, true_truck_f)
noise_level = 0.1
# (true_dar_car, true_dar_truck) = dode.get_dar(dta, true_car_f, true_truck_f)
# true_car_x = true_dar_car.dot(true_car_f)
# true_truck_x = true_dar_truck.dot(true_truck_f)
data_dict['car_count_agg_L_list'] = list()
data_dict['truck_count_agg_L_list'] = list()
data_dict['car_link_flow'] = []
data_dict['truck_link_flow'] = []
data_dict['car_link_tt'] = []
data_dict['truck_link_tt'] = []
for i in range(config['num_data']):
true_car_x = dta.get_link_car_inflow(np.arange(0, dode.num_loading_interval, dode.ass_freq),
np.arange(0, dode.num_loading_interval, dode.ass_freq) + dode.ass_freq).flatten(order = 'F')
true_truck_x = dta.get_link_truck_inflow(np.arange(0, dode.num_loading_interval, dode.ass_freq),
np.arange(0, dode.num_loading_interval, dode.ass_freq) + dode.ass_freq).flatten(order = 'F')
# true_car_tt = dta.get_car_link_tt_robust(np.arange(0, dode.num_loading_interval, dode.ass_freq),
# np.arange(0, dode.num_loading_interval, dode.ass_freq) + dode.ass_freq).flatten(order = 'F')
true_car_tt = dta.get_car_link_tt(np.arange(0, dode.num_loading_interval, dode.ass_freq)).flatten(order = 'F')
true_truck_tt = dta.get_truck_link_tt(np.arange(0, dode.num_loading_interval, dode.ass_freq)).flatten(order = 'F')
m_car = L_car.dot(true_car_x)
m_truck = L_truck.dot(true_truck_x)
data_dict['car_count_agg_L_list'].append(L_car)
data_dict['truck_count_agg_L_list'].append(L_truck)
data_dict['car_link_flow'].append(m_car + np.random.uniform(-1, 1, m_car.shape) * noise_level * m_car)
data_dict['truck_link_flow'].append(m_truck + np.random.uniform(-1, 1, m_truck.shape) * noise_level * m_truck)
data_dict['car_link_tt'].append(true_car_tt + np.random.uniform(-1, 1, true_car_tt.shape) * noise_level * true_car_tt)
data_dict['truck_link_tt'].append(true_truck_tt + np.random.uniform(-1, 1, true_truck_tt.shape) * noise_level * true_truck_tt)
In [ ]:
dode = MCDODE(nb, config)
dode.add_data(data_dict)
In [ ]:
for n in [1,2,4,8]:
(car_flow, truck_flow, l_list) = dode.estimate_path_flow_mp(max_epoch = 100, car_step_size = 1,
truck_step_size = 0.1, car_init_scale = 5,
truck_init_scale = 1, adagrad = True,
record_time = True, n_process = n)
# print r2_score(car_flow, true_car_f), r2_score(truck_flow, true_truck_f)
print l_list[99][2]
pickle.dump([true_car_f, true_truck_f, car_flow, truck_flow, l_list, data_dict], open("cpu" + str(n)+ '.pickle', 'w'))
In [ ]:
plt.figure(figsize=(16, 9), dpi=300)
for i, n in enumerate([1,2,4,8]):
[_, _, _, _, l_list, _] = pickle.load(open("cpu" + str(n)+ '.pickle', 'r'))
# axes[0].plot(np.arange(100)+1, list(map(lambda x: x[0]/8, l_list)) , label = "cpu" + str(n), color = color_list[i])
plt.plot(list(map(lambda x: x[2], l_list)), list(map(lambda x: x[0]/8, l_list)), label = "cpu" + str(n), color = color_list[i])
plt.legend()
plt.show()
In [ ]:
dode = MCDODE(nb, config)
dode.add_data(data_dict)
In [ ]:
(car_flow, truck_flow, l_list_gd) = dode.estimate_path_flow(max_epoch = 100, car_step_size = 0.01,
truck_step_size = 0.001, car_init_scale = 5,
truck_init_scale = 0.1, adagrad = False)
In [ ]:
pickle.dump([true_car_f, true_truck_f, car_flow, truck_flow, l_list_gd, data_dict], open("sgd.pickle", 'w'))
In [ ]:
(car_flow, truck_flow, l_list_gd) = dode.estimate_path_flow_gd(max_epoch = 100, car_step_size = 0.01,
truck_step_size = 0.001, car_init_scale = 5,
truck_init_scale = 0.1, adagrad = False)
In [ ]:
pickle.dump([true_car_f, true_truck_f, car_flow, truck_flow, l_list_gd, data_dict], open("gd.pickle", 'w'))
In [ ]:
[_, _, _, _, l_list1, _] = pickle.load(open("final_use.pickle", 'r'))
[_, _, _, _, l_list2, _] = pickle.load(open("sgd.pickle", 'r'))
[_, _, _, _, l_list3, _] = pickle.load(open("gd.pickle", 'r'))
In [ ]:
plt.figure(figsize = (16,9), dpi = 300)
plt.plot(np.arange(100)+1, list(map(lambda x: x[0]/8, l_list3)), color = 'teal', label ="GD", linewidth = 3)
plt.plot(np.arange(101)+1, [l_list3[0][0]/8]+list(map(lambda x: x[0]/8, l_list2)), color = 'tomato', label = "SGD", linewidth = 3)
plt.plot(np.arange(101)+1, [l_list3[0][0]/8]+list(map(lambda x: x[0]/8, l_list1)), color = 'blue', label = "Adagrad", linewidth = 3)
plt.legend()
plt.ylabel('Loss')
plt.xlabel('Iteration')
plt.xlim([1, 100])
plt.show()
In [ ]:
In [ ]:
dode = MCDODE(nb, config)
dode.add_data(data_dict)
res_df = pd.DataFrame(index = range(100), columns = ['y_car', 'y_truck', 'x_car', 'x_truck', 'q_car', 'q_truck',
'c_car', 'c_truck'])
for i in range(100):
dode = MCDODE(nb, config)
dode.add_data(data_dict)
(car_flow, truck_flow, l_list) = dode.estimate_path_flow(max_epoch = 100, car_step_size = 1,
truck_step_size = 0.1, car_init_scale = 5,
truck_init_scale = 0.1, adagrad = True)
dta = dode._run_simulation(car_flow, truck_flow)
estimated_car_x = dta.get_link_car_inflow(np.arange(0, dode.num_loading_interval, dode.ass_freq),
np.arange(0, dode.num_loading_interval, dode.ass_freq) + dode.ass_freq).flatten(order = 'F')
estimated_truck_x = dta.get_link_truck_inflow(np.arange(0, dode.num_loading_interval, dode.ass_freq),
np.arange(0, dode.num_loading_interval, dode.ass_freq) + dode.ass_freq).flatten(order = 'F')
true_car_q = true_car_f.reshape(3, -1, order = 'F').sum(axis = 0)
est_car_q = car_flow.reshape(3, -1, order = 'F').sum(axis = 0)
true_truck_q = true_truck_f.reshape(3, -1, order = 'F').sum(axis = 0)
est_truck_q = truck_flow.reshape(3, -1, order = 'F').sum(axis = 0)
est_car_tt = dta.get_car_link_tt(np.arange(0, dode.num_loading_interval, dode.ass_freq)).flatten(order = 'F')
est_truck_tt = dta.get_truck_link_tt(np.arange(0, dode.num_loading_interval, dode.ass_freq)).flatten(order = 'F')
y_car = r2_score(m_car, L_car.dot(estimated_car_x))
e_m_truck =L_truck.dot(estimated_truck_x)
y_truck = r2_score(m_truck, e_m_truck)
# truck_flow = true_truck_f + (np.random.rand(len(true_car_f)) * 2- 1) * np.random.rand() * 2
x_car = r2_score(true_car_x, estimated_car_x)
x_truck = r2_score(true_truck_x, estimated_truck_x)
q_car = r2_score(true_car_q, est_car_q)
q_truck = r2_score(true_truck_q, est_truck_q)
c_car = r2(true_car_tt, est_car_tt)
c_truck = r2(true_truck_tt, est_truck_tt)
print [y_car, y_truck, x_car, x_truck, q_car, q_truck, c_car, c_truck]
res_df.iloc[i] = [y_car, y_truck, x_car, x_truck, q_car, q_truck, c_car, c_truck]
In [ ]:
[y_car, y_truck, x_car, x_truck, q_car, q_truck, c_car, c_truck]
In [ ]:
pickle.dump(res_df, open('initf_res_df.pickle', 'wb'))
In [ ]:
res_df
In [ ]:
res_rdf = pd.DataFrame(index = range(800), columns = ['R-square', 'Vehicle Class', 'Flow'])
for i in range(100):
# print i
res_rdf.iloc[i * 8 + 0] = [res_df.iloc[i, 0], 'car', 'Observed Flow']
res_rdf.iloc[i * 8 + 1] = [res_df.iloc[i, 1], 'truck', 'Observed Flow']
res_rdf.iloc[i * 8 + 2] = [res_df.iloc[i, 2], 'car', 'Link Flow']
res_rdf.iloc[i * 8 + 3] = [res_df.iloc[i, 3], 'truck', 'Link Flow']
res_rdf.iloc[i * 8 + 4] = [res_df.iloc[i, 4], 'car', 'OD Demand']
res_rdf.iloc[i * 8 + 5] = [res_df.iloc[i, 5], 'truck', 'OD Demand']
res_rdf.iloc[i * 8 + 6] = [res_df.iloc[i, 6], 'car', 'Link Cost']
res_rdf.iloc[i * 8 + 7] = [res_df.iloc[i, 7], 'truck', 'Link Cost']
In [ ]:
res_rdf['R-square'] = res_rdf['R-square'].astype(np.float32)
In [ ]:
plt.figure(figsize = (16,9), dpi=300)
sns.boxplot(x="Flow", y="R-square", hue="Vehicle Class",
# split=True, inner="quart",
# palette={"Yes": "y", "No": "b"},
palette=["teal", "tomato"], linewidth=1.5, fliersize = 5,
data=res_rdf)
plt.show()
In [ ]:
dode = MCDODE(nb, config)
(f_car, f_truck) = dode.init_path_flow(car_scale = 5, truck_scale = 0.1)
pickle.dump((f_car, f_truck, None), open('init4obs.pickle', 'w'))
In [ ]:
# res2_df = pd.DataFrame(index = range(100), columns = ['y_car', 'y_truck', 'x_car', 'x_truck', 'q_car', 'q_truck',
# 'c_car', 'c_truck'])
for jjj in range(97, 100):
observed_link_list = [3, 4, 5, 6]
ml_car = 6
ml_truck = 5
data_dict = dict()
num_interval = nb.config.config_dict['DTA']['max_interval']
# true_car_f = np.random.rand(num_interval * nb.config.config_dict['FIXED']['num_path']) * 300
# true_truck_f = np.random.rand(num_interval * nb.config.config_dict['FIXED']['num_path']) * 30
true_car_f, true_truck_f, _, _, _, _ = pickle.load(open('final_use.pickle', 'r'))
# true_car_x = np.random.rand(num_interval * len(observed_link_list)) * 100
# true_truck_x = np.random.rand(num_interval * len(observed_link_list)) * 10
# L_car_one = np.random.randint(2, size = (ml_car, len(observed_link_list)))
L_car_one = np.array([[1, 0, 0, 1],
[0, 0, 1, 1],
[1, 1, 0, 1],
[1, 0, 1, 1],
[1, 0, 0, 0],
[0, 1, 0, 1]])
L_truck_one = np.array([[1, 0, 0, 1],
[0, 0, 0, 1],
[1, 1, 0, 1],
[1, 0, 1, 0],
[0, 1, 0, 1]])
# L_truck_one = np.random.randint(2, size = (ml_truck, len(observed_link_list)))
L_car = csr_matrix(scipy.linalg.block_diag(*[L_car_one for i in range(num_interval)]))
L_truck = csr_matrix(scipy.linalg.block_diag(*[L_truck_one for i in range(num_interval)]))
config = dict()
config['use_car_link_flow'] = True
config['use_truck_link_flow'] = True
config['use_car_link_tt'] = True
config['use_truck_link_tt'] = True
config['car_count_agg'] = True
config['truck_count_agg'] = True
config['link_car_flow_weight'] = 1
config['link_truck_flow_weight'] = 1
config['link_car_tt_weight'] = 0.1
config['link_truck_tt_weight'] = 0.1
config['num_data'] = 8
config['observed_links'] = observed_link_list
config['paths_list'] = range(nb.config.config_dict['FIXED']['num_path'])
config['compute_car_link_flow_loss'] = True
config['compute_truck_link_flow_loss'] = True
config['compute_car_link_tt_loss'] = True
config['compute_truck_link_tt_loss'] = True
dode = MCDODE(nb, config)
dta = dode._run_simulation(true_car_f, true_truck_f)
true_car_dar, true_truck_dar = dode.get_dar(dta, true_car_f, true_truck_f)
noise_level = 0.1
(true_dar_car, true_dar_truck) = dode.get_dar(dta, true_car_f, true_truck_f)
true_car_x = true_dar_car.dot(true_car_f)
true_truck_x = true_dar_truck.dot(true_truck_f)
true_car_tt = dta.get_car_link_tt(np.arange(0, dode.num_loading_interval, dode.ass_freq)).flatten(order = 'F')
true_truck_tt = dta.get_truck_link_tt(np.arange(0, dode.num_loading_interval, dode.ass_freq)).flatten(order = 'F')
data_dict['car_count_agg_L_list'] = list()
data_dict['truck_count_agg_L_list'] = list()
data_dict['car_link_flow'] = []
data_dict['truck_link_flow'] = []
data_dict['car_link_tt'] = []
data_dict['truck_link_tt'] = []
for i in range(config['num_data']):
true_car_x = dta.get_link_car_inflow(np.arange(0, dode.num_loading_interval, dode.ass_freq),
np.arange(0, dode.num_loading_interval, dode.ass_freq) + dode.ass_freq).flatten(order = 'F')
true_truck_x = dta.get_link_truck_inflow(np.arange(0, dode.num_loading_interval, dode.ass_freq),
np.arange(0, dode.num_loading_interval, dode.ass_freq) + dode.ass_freq).flatten(order = 'F')
# true_car_tt = dta.get_car_link_tt_robust(np.arange(0, dode.num_loading_interval, dode.ass_freq),
# np.arange(0, dode.num_loading_interval, dode.ass_freq) + dode.ass_freq).flatten(order = 'F')
m_car = L_car.dot(true_car_x)
m_truck = L_truck.dot(true_truck_x)
data_dict['car_count_agg_L_list'].append(L_car)
data_dict['truck_count_agg_L_list'].append(L_truck)
data_dict['car_link_flow'].append(m_car + np.random.uniform(-1, 1, m_car.shape) * noise_level * m_car)
data_dict['truck_link_flow'].append(m_truck + np.random.uniform(-1, 1, m_truck.shape) * noise_level * m_truck)
data_dict['car_link_tt'].append(true_car_tt + np.random.uniform(-1, 1, true_car_tt.shape) * noise_level * true_car_tt)
data_dict['truck_link_tt'].append(true_truck_tt + np.random.uniform(-1, 1, true_truck_tt.shape) * noise_level * true_truck_tt)
# data_dict['car_link_tt'] = [m_spd_car]
# data_dict['truck_link_tt'] = [m_spd_truck]
dode = MCDODE(nb, config)
dode.add_data(data_dict)
(car_flow, truck_flow, l_list) = dode.estimate_path_flow(max_epoch = 100, car_step_size = 1,
truck_step_size = 0.1, car_init_scale = 5,
truck_init_scale = 0.1, adagrad = True,
use_file_as_init = 'init4obs.pickle')
dta = dode._run_simulation(car_flow, truck_flow)
estimated_car_x = dta.get_link_car_inflow(np.arange(0, dode.num_loading_interval, dode.ass_freq),
np.arange(0, dode.num_loading_interval, dode.ass_freq) + dode.ass_freq).flatten(order = 'F')
estimated_truck_x = dta.get_link_truck_inflow(np.arange(0, dode.num_loading_interval, dode.ass_freq),
np.arange(0, dode.num_loading_interval, dode.ass_freq) + dode.ass_freq).flatten(order = 'F')
true_car_q = true_car_f.reshape(3, -1, order = 'F').sum(axis = 0)
est_car_q = car_flow.reshape(3, -1, order = 'F').sum(axis = 0)
true_truck_q = true_truck_f.reshape(3, -1, order = 'F').sum(axis = 0)
est_truck_q = truck_flow.reshape(3, -1, order = 'F').sum(axis = 0)
est_car_tt = dta.get_car_link_tt(np.arange(0, dode.num_loading_interval, dode.ass_freq)).flatten(order = 'F')
est_truck_tt = dta.get_truck_link_tt(np.arange(0, dode.num_loading_interval, dode.ass_freq)).flatten(order = 'F')
y_car = r2_score(m_car, L_car.dot(estimated_car_x))
e_m_truck =L_truck.dot(estimated_truck_x)
y_truck = r2_score(m_truck, e_m_truck)
# truck_flow = true_truck_f + (np.random.rand(len(true_car_f)) * 2- 1) * np.random.rand() * 2
x_car = r2_score(true_car_x, estimated_car_x)
x_truck = r2_score(true_truck_x, estimated_truck_x)
q_car = r2_score(true_car_q, est_car_q)
q_truck = r2_score(true_truck_q, est_truck_q)
c_car = r2(true_car_tt, est_car_tt)
c_truck = r2(true_truck_tt, est_truck_tt)
print [y_car, y_truck, x_car, x_truck, q_car, q_truck, c_car, c_truck]
res2_df.iloc[jjj] = [y_car, y_truck, x_car, x_truck, q_car, q_truck, c_car, c_truck]
In [ ]:
res2_df
In [ ]:
pickle.dump(res2_df, open('obs_res2_df.pickle', 'wb'))
In [ ]:
res2_rdf = pd.DataFrame(index = range(800), columns = ['R-square', 'Vehicle Class', 'Flow'])
for i in range(100):
# print i
res2_rdf.iloc[i * 8 + 0] = [res2_df.iloc[i, 0], 'car', 'Observed Flow']
res2_rdf.iloc[i * 8 + 1] = [res2_df.iloc[i, 1], 'truck', 'Observed Flow']
res2_rdf.iloc[i * 8 + 2] = [res2_df.iloc[i, 2], 'car', 'Link Flow']
res2_rdf.iloc[i * 8 + 3] = [res2_df.iloc[i, 3], 'truck', 'Link Flow']
res2_rdf.iloc[i * 8 + 4] = [res2_df.iloc[i, 4], 'car', 'OD Demand']
res2_rdf.iloc[i * 8 + 5] = [res2_df.iloc[i, 5], 'truck', 'OD Demand']
res2_rdf.iloc[i * 8 + 6] = [res2_df.iloc[i, 6], 'car', 'Link Cost']
res2_rdf.iloc[i * 8 + 7] = [res2_df.iloc[i, 7], 'truck', 'Link Cost']
In [ ]:
res2_rdf['R-square'] = res2_rdf['R-square'].astype(np.float32)
In [ ]:
plt.figure(figsize = (16,9), dpi=300)
sns.boxplot(x="Flow", y="R-square", hue="Vehicle Class",
# split=True, inner="quart",
# palette={"Yes": "y", "No": "b"},
palette=["teal", "tomato"], linewidth=1.5, fliersize = 5,
data=res2_rdf)
plt.show()