In [ ]:
import tqdm
import tarfile
import pandas
from urllib.request import urlopen
# 檔案名稱格式
filename_format="M06A_{year:04d}{month:02d}{day:02d}.tar.gz".format
xz_filename_format="xz/M06A_{year:04d}{month:02d}{day:02d}.tar.xz".format
csv_format = "M06A/{year:04d}{month:02d}{day:02d}/{hour:02d}/TDCS_M06A_{year:04d}{month:02d}{day:02d}_{hour:02d}0000.csv".format
In [ ]:
# 打開剛才下載的檔案試試
data_config ={"year":2016, "month":12, "day":18}
tar = tarfile.open(filename_format(**data_config), 'r')
In [ ]:
# 如果沒有下載，可以試試看 xz 檔案
#data_dconfig ={"year":2016, "month":11, "day":18}
#tar = tarfile.open(xz_filename_format(**data_config), 'r')
In [ ]:
# 設定欄位名稱
M06A_fields = ['VehicleType',
'DetectionTime_O','GantryID_O',
'DetectionTime_D','GantryID_D ',
'TripLength', 'TripEnd', 'TripInformation']
# 打開裡面 10 點鐘的資料
csv = tar.extractfile(csv_format(hour=10, **data_config))
# 讀進資料
data = pandas.read_csv(csv, names=M06A_fields)
# 檢查異常的資料
print("異常資料數:", data[data.TripEnd == 'N'].shape[0])
# 去除異常資料
data = data[data.TripEnd == 'Y']
# 只保留 TripInformation 和 VehicleType
data = data[['VehicleType', "TripInformation"]]
# 看前五筆
data.head(5)
In [ ]:
import datetime
# 用來解析時間格式
def strptime(x):
return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S")
In [ ]:
data.iloc[0].TripInformation
In [ ]:
In [ ]:
# 合在一起看看
for idx, row in data.head(10).iterrows():
# 處理過程
In [ ]:
# 節省記憶體
del data
In [ ]:
node_data_url = "http://www.freeway.gov.tw/Upload/DownloadFiles/%e5%9c%8b%e9%81%93%e8%a8%88%e8%b2%bb%e9%96%80%e6%9e%b6%e5%ba%a7%e6%a8%99%e5%8f%8a%e9%87%8c%e7%a8%8b%e7%89%8c%e5%83%b9%e8%a1%a8104.09.04%e7%89%88.csv"
node_data = pandas.read_csv(urlopen(node_data_url), encoding='big5', header=1)
# 簡單清理資料
node_data = node_data[node_data["方向"].apply(lambda x:x in 'NS')]
node_data.head(10)
In [ ]:
In [ ]:
%matplotlib inline
In [ ]:
node_data['經度(東經)'] = node_data['經度(東經)'].astype(float)
node_data['緯度(北緯)'] = node_data['緯度(北緯)'].astype(float)
In [ ]:
node_data.plot.scatter(x='經度(東經)', y='緯度(北緯)')
In [ ]:
from PIL import Image
import numpy as np
import matplotlib.pyplot as plt
# 網路上的台灣地圖，有經緯度
taiwan_img_url="http://gallery.mjes.ntpc.edu.tw/gallery2/main.php?g2_view=core.DownloadItem&g2_itemId=408&g2_serialNumber=1"
taiwan_img = Image.open(urlopen(taiwan_img_url))
taiwan_img
In [ ]:
# 查看編號的前置碼
set(node_data['編號'].str[:3].tolist())
In [ ]:
# 依照路線編號
cfunc = {'01F':"green", '01H':"blue", '03A':"yellow", '03F':"red", '05F':"purple"}.get
colors = node_data['編號'].str[:3].apply(cfunc)
In [ ]:
fig = plt.gcf()
fig.set_size_inches(8,8)
extent=[118.75,123.05,21.45,25.75]
plt.xlim(*extent[:2])
plt.ylim(*extent[2:])
plt.scatter(node_data['經度(東經)'], node_data['緯度(北緯)'], c=colors, alpha=1)
plt.imshow(np.array(taiwan_img), extent=extent);
In [ ]:
node_data[node_data.編號=="03F-318.7S"]
In [ ]:
node_data[node_data.編號=="03F-321.1S"]