In [1]:
import tqdm
import tarfile
import pandas
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
from urllib.request import urlopen
from IPython.display import display
%matplotlib inline
matplotlib.style.use('ggplot')
# 檔案名稱格式
filename_format="../Week03/M06A_{year:04d}{month:02d}{day:02d}.tar.gz".format
xz_filename_format="xz/M06A_{year:04d}{month:02d}{day:02d}.tar.xz".format
csv_format = "M06A/{year:04d}{month:02d}{day:02d}/{hour:02d}/TDCS_M06A_{year:04d}{month:02d}{day:02d}_{hour:02d}0000.csv".format
In [2]:
# 打開剛才下載的檔案試試
data_config ={"year":2016, "month":12, "day":18}
tar = tarfile.open(filename_format(**data_config), 'r')
In [3]:
# 如果沒有下載，可以試試看 xz 檔案
#data_dconfig ={"year":2016, "month":11, "day":18}
#tar = tarfile.open(xz_filename_format(**data_config), 'r')
In [4]:
# 設定欄位名稱
M06A_fields = ['VehicleType',
'DetectionTime_O','GantryID_O',
'DetectionTime_D','GantryID_D ',
'TripLength', 'TripEnd', 'TripInformation']
# 打開裡面 10 點鐘的資料
csv = tar.extractfile(csv_format(hour=10, **data_config))
# 讀進資料
data = pandas.read_csv(csv, names=M06A_fields)
# 檢查異常的資料
print("異常資料數:", data[data.TripEnd == 'N'].shape[0])
# 去除異常資料
data = data[data.TripEnd == 'Y']
# 只保留 TripInformation 和 VehicleType
data = data[['VehicleType', "TripInformation"]]
# 看前五筆
data.head(5)
Out[4]:
In [5]:
import datetime
# 用來解析時間格式
def strptime(x):
return datetime.datetime(int(x[:4]), int(x[5:7]), int(x[8:10]),
int(x[11:13]), int(x[14:16]), int(x[17:19]) )
def parse_tripinfo(tripinfo):
split1 = tripinfo.split("; ")
return [(strptime(t), t[20:]) for t in split1]
data.head(10).TripInformation.apply(parse_tripinfo)
# progress bar
tqdm.tqdm.pandas()
# 新增一欄
data['Trip'] = data.TripInformation.progress_apply(parse_tripinfo)
In [6]:
node_data = pandas.read_json("../Week03/node_data.json")
node_data.head(10)
Out[6]:
In [7]:
position_dict=node_data.apply(lambda t: [t["經度(東經)"], t["緯度(北緯)"]], axis=1).to_dict()
In [8]:
start_time = datetime.datetime(data_config['year'],data_config['month'],data_config['day'])
In [9]:
t = [(a[0]-start_time).total_seconds()/60/60 for a in data.Trip[0]]
position = np.array([position_dict[a[1]] for a in data.Trip[0]])
In [10]:
# 時間 - X 座標
plt.plot(t,position[:,0])
Out[10]:
In [11]:
# 時間 - Y 座標
plt.plot(t,position[:,1])
Out[11]:
In [12]:
from PIL import Image
import numpy as np
import matplotlib.pyplot as plt
# 網路上的台灣地圖，有經緯度
taiwan_img_url="http://gallery.mjes.ntpc.edu.tw/gallery2/main.php?g2_view=core.DownloadItem&g2_itemId=408&g2_serialNumber=1"
taiwan_img = Image.open(urlopen(taiwan_img_url))
#taiwan_img = Image.open("taiwan.png")
In [13]:
fig = plt.gcf()
fig.set_size_inches(8,8)
extent=[118.75,123.05,21.45,25.75]
plt.xlim(*extent[:2])
plt.ylim(*extent[2:])
plt.plot(position[:,0], position[:,1])
plt.imshow(np.array(taiwan_img), extent=extent);
In [14]:
from scipy.interpolate import interp1d
In [15]:
t_space = np.arange(int(min(t)*60+1)/60,max(t),1/60)
In [16]:
f1 = interp1d(t, position, 'linear', axis=0)
f3 = interp1d(t, position, 'cubic', axis=0)
plt.plot(t,position[:,0])
plt.plot(t_space, f1(t_space)[:,0])
plt.plot(t_space, f3(t_space)[:,0]);
In [17]:
plt.plot(t,position[:,1])
plt.plot(t_space, f1(t_space)[:,1])
plt.plot(t_space, f3(t_space)[:,1]);
In [18]:
# 打底
scatters = plt.scatter(position[:,0], position[:,1])
plt.margins(0,0)
fig = plt.gcf()
In [19]:
# 開始動畫
from matplotlib import animation, rc
rc('animation', html='html5')
curv = f1(t_space)
def animate(tick):
scatters.set_offsets([curv[tick]])
return [scatters]
anim = animation.FuncAnimation(fig, animate,
frames=len(t_space), interval=100, blit=True)
anim
Out[19]:
In [20]:
# 打底
fig = plt.gcf()
fig.set_size_inches(5,5)
extent=[118.75,123.05,21.45,25.75]
plt.xlim(*extent[:2])
plt.ylim(*extent[2:])
fig = plt.gcf()
scatters = plt.scatter([],[], alpha=0.5, color='red')
desc_text = plt.text(119,25.3,"",size=25, color="#ff00ff")
plt.axis('off')
plt.subplots_adjust(top = 1, bottom = 0, right = 1, left = 0,
hspace = 0, wspace = 0)
plt.margins(0,0)
bg = plt.imshow(np.array(taiwan_img), extent=extent)
In [21]:
# 動畫
curv = f1(t_space)
def animate(tick):
scatters.set_offsets([curv[tick]])
return [scatters]
anim = animation.FuncAnimation(fig, animate,
frames=29, interval=100, blit=True)
anim
Out[21]:
In [22]:
from random import randint
scatter_data = [[] for i in range(24*60)]
scatter_color = [[] for i in range(24*60)]
for trip in tqdm.tqdm(data.Trip.sample(3000)):
t = [(a[0]-start_time).total_seconds()/60 for a in trip]
position = np.array([position_dict[a[1]] for a in trip])
color="#{:02x}{:02x}{:02x}".format(randint(0,255),randint(0,255),randint(0,255))
if len(t)<2:
continue
kind = "slinear"
inter_f = interp1d(t, position, kind, axis=0)
t_space = np.arange(int(min(t)+1),int(max(t))+1)
curve = inter_f(t_space)
for j,p in enumerate(t_space):
scatter_data[p%(24*60)].append(curve[j])
scatter_color[p%(24*60)].append(color)
for i in range(24*60):
scatter_data[i]=np.array(scatter_data[i])
In [23]:
# 先畫出 10:00-12:00
with tqdm.tqdm(total=2*60) as pbar:
def animate(tick):
pbar.update(1)
scatters.set_color(scatter_color[tick+600])
scatters.set_offsets(scatter_data[tick+600])
scatters.set_alpha(0.3)
scatters.set
hh=int((tick+600)/60)
mm = tick%60
desc_text.set_text("{:02d}:{:02d}".format(hh,mm))
return [scatters, desc_text]
anim = animation.FuncAnimation(fig, animate,
frames=2*60, interval=100, blit=True)
display(anim)
In [24]:
csvs = (tar.extractfile(csv_format(hour=hr, **data_config)) for hr in tqdm.trange(24))
data = pandas.concat([pandas.read_csv(csv, names=M06A_fields) for csv in csvs], ignore_index=True)
print("資料大小", data.shape)
# 檢查異常的資料
print("異常資料數:", data[data.TripEnd == 'N'].shape[0])
# 去除異常資料
data = data[data.TripEnd == 'Y']
# 把焦點放在 TripInformation 和 VehicleType
data = data[['VehicleType', "TripInformation"]]
data['Trip'] = data.TripInformation.progress_apply(parse_tripinfo)
In [25]:
from random import randint
scatter_data = [[] for i in range(24*60)]
scatter_color = [[] for i in range(24*60)]
for trip in tqdm.tqdm(data.Trip.sample(3000*24)):
t = [(a[0]-start_time).total_seconds()/60 for a in trip]
position = np.array([position_dict[a[1]] for a in trip])
color="#{:02x}{:02x}{:02x}".format(randint(0,255),randint(0,255),randint(0,255))
if len(t)<2:
continue
kind = "linear"
inter_f = interp1d(t, position, kind, axis=0)
t_space = np.arange(int(min(t)+1),int(max(t))+1)
curve = inter_f(t_space)
for j,p in enumerate(t_space):
scatter_data[p%(24*60)].append(curve[j])
scatter_color[p%(24*60)].append(color)
for i in range(24*60):
scatter_data[i]=np.array(scatter_data[i])
In [26]:
# 先畫出 0:00-23:59
def animate(tick):
pbar.update(1)
scatters.set_color(scatter_color[tick])
scatters.set_offsets(scatter_data[tick])
scatters.set_alpha(0.3)
scatters.set
hh=int(tick/60)
mm = tick%60
desc_text.set_text("{:02d}:{:02d}".format(hh,mm))
return [scatters, desc_text]
with tqdm.tqdm(total=24*60) as pbar:
display(animation.FuncAnimation(fig, animate, frames=24*60, interval=100, blit=True))
In [ ]:
from ipywidgets import Image
import time
from io import BytesIO
anim = animation.FuncAnimation(fig, animate, frames=24*60, interval=100, blit=True)
img=Image(width=400)
display(img)
target_time = time.time() + anim._interval/1000
for d in anim.new_saved_frame_seq():
anim._draw_next_frame(d, blit=False)
with BytesIO() as bio:
fig.savefig(bio, format="png")
img.value = bio.getvalue()
now = time.time()
if target_time > now:
time.sleep(target_time-now)
target_time += anim._interval/1000
In [ ]: