In [1]:
%pylab inline
import pandas as pd
from pykalman import KalmanFilter
In [2]:
df = pd.read_csv("../data/ChungCheonDC/CompositeETCdata.csv")
df_DC = pd.read_csv("../data/ChungCheonDC/CompositeDCdata.csv")
df_DCprc = pd.read_csv("../data/ChungCheonDC/CompositeDCdata_processed1.csv")
df_DCstd = pd.read_csv("../data/ChungCheonDC/CompositeDCstddata.csv")
In [3]:
# missininds = np.arange(df_DC[electrodeID[elecind]].values.size)[np.isnan(df_DC[electrodeID[elecind]].values)]
electrodeID = df_DC.keys()[1:-1]
In [4]:
from scipy import interpolate
sys.path.append("../codes/")
from DCdata import readReservoirDC_all
directory = "../data/ChungCheonDC/"
dat_temp,height_temp, ID = readReservoirDC_all(directory+"20151231180000.apr")
locs = dat_temp[:,:4]
mida = locs[:,:2].sum(axis=1)
midb = locs[:,2:].sum(axis=1)
mid = (mida + midb)*0.5
dz = mida-midb
x = np.linspace(mid.min(), mid.max(), 100)
z = np.linspace(dz.min(), dz.max(), 100)
grid_x, grid_z = np.meshgrid(x,z)
def vizDCtimeSeries(idatum, itime, itime_ref, colors, flag, df_DC):
fig = plt.figure(figsize = (12, 12))
ax1 = plt.subplot(411)
ax2 = plt.subplot(412)
valsratio = df_DC[electrodeID].values[itime,:].flatten() / df_DC[electrodeID].values[itime_ref,:].flatten()
valsDC = np.log10(df_DC[electrodeID].values[itime,:].flatten())
valsDCstd = df_DCstd[electrodeID].values[itime,:].flatten()
grid_rho_ratio = griddata(mid, dz, valsratio, grid_x, grid_z, interp='linear')
grid_rho_ratio = grid_rho_ratio.reshape(grid_x.shape)
if flag =="std":
vmin, vmax = 0, 10
grid_rho = griddata(mid, dz, valsDCstd, grid_x, grid_z, interp='linear')
elif flag =="rho":
vmin, vmax = np.log10(20), np.log10(200)
grid_rho = griddata(mid, dz, valsDC, grid_x, grid_z, interp='linear')
grid_rho = grid_rho.reshape(grid_x.shape)
ax1.contourf(grid_x, grid_z, grid_rho, 200, vmin =vmin, vmax = vmax, clim=(vmin, vmax), cmap="jet")
vmin, vmax = 0.9, 1.1
ax2.contourf(grid_x, grid_z, grid_rho_ratio, 200, vmin =vmin, vmax = vmax, clim=(vmin, vmax), cmap="jet")
ax1.scatter(mid, dz, s=20, c = valsDC, edgecolor="None", vmin =vmin, vmax = vmax, clim=(vmin, vmax))
ax1.plot(mid, dz, 'k.')
ax2.scatter(mid, dz, s=20, c = valsratio, edgecolor="None", vmin =vmin, vmax = vmax, clim=(vmin, vmax))
ax2.plot(mid, dz, 'k.')
for i in range(len(colors)):
ax1.plot(mid[idatum[i]], dz[idatum[i]], 'o', color=colors[i])
ax2.plot(mid[idatum[i]], dz[idatum[i]], 'o', color=colors[i])
ax3 = plt.subplot(413)
ax3_1 = ax3.twinx()
df.plot(x='date', y='reservoirH', ax=ax3_1, color='k', linestyle='-', lw=2)
df.plot(x='date', y='upperH_med', ax=ax3_1, color='b', linestyle='-', lw=2)
df.plot(x='date', y='Temp (degree)', ax=ax3, color='r', linestyle='-', lw=2)
df.plot(x='date', y='Rainfall (mm)', ax=ax3, color='b', linestyle='-', marker="o", ms=4)
ax3.legend(loc=3, bbox_to_anchor=(1.05, 0.7))
ax3_1.legend(loc=3, bbox_to_anchor=(1.05, 0.4))
itime_ref0 = itime_ref
itime_ref1 = itime
ax3.plot(np.r_[itime_ref0, itime_ref0], np.r_[-5, 40], 'k--', lw=2)
ax3.plot(np.r_[itime_ref1, itime_ref1], np.r_[-5, 40], 'k--', lw=2)
ax4 = plt.subplot(414)
df_DC.plot(x='date', y=electrodeID[idatum], ax=ax4)
ax4.legend(loc=3, bbox_to_anchor=(1.05, 0.7))
ax4.set_yscale('log')
temp = df_DC[electrodeID[elecind]].values
vmax = np.median(temp[~np.isnan(temp)]) + np.std(temp[~np.isnan(temp)])*3
vmin = np.median(temp[~np.isnan(temp)]) - np.std(temp[~np.isnan(temp)])*3
ax4.plot(np.r_[itime_ref1, itime_ref1], np.r_[vmin, vmax], 'k--', lw=2)
ax4.plot(np.r_[itime_ref0, itime_ref0], np.r_[vmin, vmax], 'k--', lw=2)
ax4.set_ylim(vmin, vmax)
In [5]:
ax1 = plt.subplot(111)
ax1_1 = ax1.twinx()
df.plot(figsize=(12,3), x='date', y='reservoirH', ax=ax1_1, color='k', linestyle='-', lw=2)
df.plot(figsize=(12,3), x='date', y='upperH_med', ax=ax1_1, color='b', linestyle='-', lw=2)
df.plot(figsize=(12,3), x='date', y='Temp (degree)', ax=ax1, color='r', linestyle='-', lw=2)
ax1.legend(loc=3, bbox_to_anchor=(1.05, 0.7))
ax1_1.legend(loc=3, bbox_to_anchor=(1.05, 0.4))
itime_ref0 = 255
itime_ref1 = 115
ax1.plot(np.r_[itime_ref0, itime_ref0], np.r_[-5, 35], 'k-')
ax1.plot(np.r_[itime_ref1, itime_ref1], np.r_[-5, 35], 'k-')
# print df['date'].values[itime_ref]
Out[5]:
In [6]:
# ax1 = plt.subplot(111)
# ax1_1 = ax1.twinx()
# df_DC.plot(figsize=(12,3), x='date', y=electrodeID[elecind], ax=ax1, colors=['k', 'b', 'r'])
# df.plot(figsize=(12,3), x='date', y='reservoirH', ax=ax1_1, color='k', linestyle='-', lw=2)
# ax1.legend(loc=3, bbox_to_anchor=(1.05, 0.7))
# ax1_1.legend(loc=3, bbox_to_anchor=(1.05, 0.4))
# ax1.set_yscale('linear')
In [7]:
# ax1 = plt.subplot(111)
# df_DCstd.plot(figsize=(12,3), x='date', y=electrodeID[elecind], ax=ax1, colors=['k', 'b', 'r'], linestyle="-", marker='.', lw=1)
# ax1.set_yscale('log')
# ax1.legend(loc=3, bbox_to_anchor=(1.05, 0.7))
In [8]:
txrxID = df_DC.keys()[1:-1]
xmasking = lambda x: np.ma.masked_where(np.isnan(x.values), x.values)
In [9]:
#x= electrodeID[elecind]
x= df_DC[txrxID]
max3 = pd.rolling_max(x, 3)
In [10]:
#pd.rolling_max??
In [11]:
# plt.plot(x)
# plt.plot(max3)
In [12]:
from ipywidgets import interact
In [13]:
# making matrix like max3 (but with zeros)
newdata = np.zeros_like(max3)
In [14]:
newdata.shape
Out[14]:
In [15]:
ndata = newdata.shape[1]
for i in range(ndata):
x= df_DC[txrxID[i]]
#median10 = pd.rolling_median(x, 6)
mean10 = pd.rolling_max(x, 3)
# Masking array having NaN
xm = xmasking(mean10)
kf = KalmanFilter(transition_matrices = [1],
observation_matrices = [1],
initial_state_mean = x[0],
initial_state_covariance = 1,
observation_covariance=1,
transition_covariance=1)
# Use the observed values of the price to get a rolling mean
state_means, _ = kf.filter(xm)
newdata[:,i] = state_means.flatten()
In [16]:
df_DC_new = df_DC.copy()
for i,index in enumerate(txrxID):
df_DC_new.loc[:,index] = newdata[:,i].flatten()
# df_DC_new.to_csv("../data/ChungCheonDC/CompositeDCdata_processed.csv")
In [33]:
from ipywidgets import interact, IntSlider, ToggleButtons
itime = 201
itime_ref = 201 # 86 = Date 2015. 3.28, 201 = Date 2015. 7. 21
print df['date'].values[itime]
elecind = [5, 150,200]
# vizDCtimeSeries(elecind, itime, itime_ref, ['k','b','r'])
viz = lambda idatum, itime, flag: vizDCtimeSeries([idatum], itime, itime_ref, ['r'], flag, df_DC_new)
interact(viz, idatum=IntSlider(min=0, max=379, step=1, value=201)\
,itime=IntSlider(min=0, max=360, step=1, value=200)\
,flag=ToggleButtons(options=["std", "rho"]))
print df['date'].values[itime]
In [27]:
print df_DC[electrodeID].values[228,:].flatten() / df_DC[electrodeID].values[201,:].flatten()
In [28]:
for i in range(0,379,100):
x= df_DC[txrxID[i]]
x1 = df_DC_new[txrxID[i]]
plt.plot(newdata[:,i], 'k')
plt.plot(x1, 'ro')
plt.plot(x, 'k.', ms=2)
In [29]:
plt.plot(newdata[:,i], 'k')
x1 = df_DC_new[txrxID[i]]
In [21]:
# for index in txrxID:
# df_DC_new.loc[:,index] = newdata[:,i].flatten()
In [22]:
i = 112
def viz(i):
x= df_DC[txrxID[i]]
#median10 = pd.rolling_median(x, 6)
mean10 = pd.rolling_max(x, 3)
#x1 = median10
#x2 = mean10
# Masking array having NaN
xm = xmasking(mean10)
# Construct a Kalman filter
# kf = KalmanFilter(transition_matrices = [1],
# observation_matrices = [1],
# initial_state_mean = x[0],
# initial_state_covariance = 1,
# observation_covariance=1,
# transition_covariance=1)
# # Use the observed values of the price to get a rolling mean
# state_means, _ = kf.filter(xm)
state_means= df_DC_new[txrxID[i]]
plt.plot(x)
plt.plot(mean10, 'k.')
#plt.plot(x1)
#plt.plot(x2)
plt.plot(state_means)
# plt.legend([ i, 'Kalman Estimate'])
# print df_DC[txrxID[i]]
interact(viz, i=(0,389,10))
Out[22]:
In [23]:
i = 105
x= df_DC[txrxID[i]]
#median10 = pd.rolling_median(x, 6)
mean10 = pd.rolling_max(x, 3)
#x1 = median10
#x2 = mean10
# Masking array having NaN
xm = xmasking(mean10)
# Construct a Kalman filter
kf = KalmanFilter(transition_matrices = [1],
observation_matrices = [1],
initial_state_mean = 67.6,
initial_state_covariance = 1,
observation_covariance=1,
transition_covariance=1)
# Use the observed values of the price to get a rolling mean
state_means, _ = kf.filter(xm)
#plt.plot(x1)
plt.plot(x)
#plt.plot(x1)
#plt.plot(x2)
plt.plot(state_means)
plt.legend([ 'origin x','Kalman Estimate'])
Out[23]:
In [23]:
i = 300
x= df_DC[txrxID[i]]
#median10 = pd.rolling_median(x, 6)
mean10 = pd.rolling_max(x, 3)
#x1 = median10
#x2 = mean10
# Masking array having NaN
xm = xmasking(mean10)
# Construct a Kalman filter
kf = KalmanFilter(transition_matrices = [1],
observation_matrices = [1],
initial_state_mean = 67.6,
initial_state_covariance = 1,
observation_covariance=1,
transition_covariance=1)
# Use the observed values of the price to get a rolling mean
state_means, _ = kf.filter(xm)
#plt.plot(x1)
plt.plot(x)
#plt.plot(x1)
#plt.plot(x2)
plt.plot(state_means)
plt.legend([ 'origin x','Kalman Estimate'])
Out[23]:
In [ ]:
In [ ]: