In [1]:
%run basics
%matplotlib
import xlwt
In [2]:
def xl_write_data(xlSheet,data):
xlCol = 0
# write the data to the xl file
series_list = data.keys()
xlSheet.write(1,xlCol,data["DateTime"]["units"])
nrows = len(data["DateTime"]["data"])
ncols = len(series_list)
d_xf = xlwt.easyxf(num_format_str=data["DateTime"]["format"])
for j in range(nrows):
xlSheet.write(j+2,xlCol,data["DateTime"]["data"][j],d_xf)
series_list.remove("DateTime")
series_list.sort()
for item in series_list:
xlCol = xlCol + 1
xlSheet.write(0,xlCol,data[item]["units"])
xlSheet.write(1,xlCol,item)
d_xf = xlwt.easyxf(num_format_str=data[item]["format"])
for j in range(nrows):
xlSheet.write(j+2,xlCol,float(data[item]["data"][j]),d_xf)
In [3]:
def smooth(x,window_len=11,window='hanning'):
"""
Purpose:
Smooth the data using a window with requested size.
This method is based on the convolution of a scaled window with the signal.
The signal is prepared by introducing reflected copies of the signal
(with the window size) in both ends so that transient parts are minimized
in the begining and end part of the output signal.
Input:
x: the input signal
window_len: the dimension of the smoothing window; should be an odd integer
window: the type of window from 'flat', 'hanning', 'hamming', 'bartlett', 'blackman'
flat window will produce a moving average smoothing.
Output:
the smoothed signal
Example:
t=linspace(-2,2,0.1)
x=sin(t)+randn(len(t))*0.1
y=smooth(x)
See also:
numpy.hanning, numpy.hamming, numpy.bartlett, numpy.blackman, numpy.convolve
scipy.signal.lfilter
TODO: the window parameter could be the window itself if an array instead of a string
Note:
1) length(output) != length(input), to correct this: return y[(window_len/2-1):-(window_len/2)] instead of just y.
2) odd values for window_len return output with different length from input
Source:
Lifted from scipy Cookbook (http://wiki.scipy.org/Cookbook/SignalSmooth)
"""
if x.ndim != 1:
raise ValueError, "smooth only accepts 1 dimension arrays."
if x.size < window_len:
raise ValueError, "Input vector needs to be bigger than window size."
if window_len<3:
return x
if not window in ['flat', 'hanning', 'hamming', 'bartlett', 'blackman']:
raise ValueError, "Window is on of 'flat', 'hanning', 'hamming', 'bartlett', 'blackman'"
s=numpy.r_[x[window_len-1:0:-1],x,x[-1:-window_len:-1]]
if window == 'flat': #moving average
w=numpy.ones(window_len,'d')
else:
w=eval('numpy.'+window+'(window_len)')
y=numpy.convolve(w/w.sum(),s,mode='valid')
# return y
return y[(window_len/2-1):-(window_len/2)]
In [4]:
#nc_name = "../../Sites/HowardSprings/Data/Processed/all/HowardSprings_2002_to_2014_L6.nc"
nc_name = "../../Sites/Tumbarumba/Data/Processed/all/no_storage/yearly/Tumbarumba_2001_to_2014_L6.nc"
xl_name = nc_name.replace(".nc","_Summary.xls")
cf = qcio.get_controlfilecontents("../controlfiles/standard/L6_summary.txt")
In [5]:
ds = qcio.nc_read_series(nc_name)
dt = ds.series["DateTime"]["Data"]
ts = int(ds.globalattributes["time_step"])
In [6]:
# calculate ET if it is not present
if "ET" not in ds.series.keys():
Fe,flag,attr = qcutils.GetSeriesasMA(ds,"Fe")
ET = Fe*ts*60/c.Lv
attr["long_name"] = "Evapo-transpiration calculated from latent heat flux"
attr["units"] = "mm"
qcutils.CreateSeries(ds,"ET",ET,Flag=flag,Attr=attr)
In [7]:
# adjust units of NEE, NEP, GPP and Fre
nee_list = [item for item in cf["Variables"].keys() if "NEE" in item]
nep_list = [item for item in cf["Variables"].keys() if "NEP" in item]
gpp_list = [item for item in cf["Variables"].keys() if "GPP" in item]
fre_list = [item for item in cf["Variables"].keys() if "Fre" in item]
co2_list = nee_list+nep_list+gpp_list+fre_list
for item in co2_list:
data,flag,attr = qcutils.GetSeriesasMA(ds,item)
data = data*12.01*ts*60/1E6
attr["units"] = "gC/m2"
qcutils.CreateSeries(ds,item,data,Flag=flag,Attr=attr)
In [12]:
# open the Excel workbook
xl_file = xlwt.Workbook()
In [13]:
# daily averages and totals
first_date = dt[0]
si = qcutils.GetDateIndex(dt,str(first_date),ts=ts,default=0,match="startnextday")
start_date = dt[si]
last_date = dt[-1]
ei = qcutils.GetDateIndex(dt,str(last_date),ts=ts,default=len(dt)-1,match="endpreviousday")
end_date = dt[ei]
ldt = dt[si:ei+1]
ntsInDay = int(24.0*60.0/float(ts))
nDays = int(len(ldt))/ntsInDay
ldt_daily = [ldt[0]+datetime.timedelta(days=i) for i in range(0,nDays)]
daily_dict = {}
daily_dict["DateTime"] = {"data":ldt_daily,"units":"Days","format":"dd/mm/yyyy"}
for item in cf["Variables"].keys():
if item not in ds.series.keys(): continue
daily_dict[item] = {}
data_1d,flag,attr = qcutils.GetSeriesasMA(ds,item,si=si,ei=ei)
daily_dict[item]["units"] = attr["units"]
data_2d = data_1d.reshape(nDays,ntsInDay)
if cf["Variables"][item]["operator"].lower()=="average":
daily_dict[item]["data"] = numpy.ma.average(data_2d,axis=1)
elif cf["Variables"][item]["operator"].lower()=="sum":
daily_dict[item]["data"] = numpy.ma.sum(data_2d,axis=1)
daily_dict[item]["units"] = daily_dict[item]["units"]+"/day"
else:
print "unrecognised operator"
daily_dict[item]["format"] = cf["Variables"][item]["format"]
# add the daily worksheet to the summary Excel file
xl_sheet = xl_file.add_sheet("Daily")
xl_write_data(xl_sheet,daily_dict)
In [ ]:
# monthly averages and totals
In [14]:
# annual averages and totals
start_year = ldt[0].year
end_year = ldt[-1].year
year_list = range(start_year,end_year+1,1)
annual_dict = {}
annual_dict["DateTime"] = {"data":[datetime.datetime(yr,1,1) for yr in year_list],
"units":"Days","format":"dd/mm/yyyy"}
# create arrays in annual_dict
for item in cf["Variables"].keys():
annual_dict[item] = {"data":numpy.array([float(-9999)]*len(year_list))}
annual_dict["ET"] = {"data":numpy.array([float(-9999)]*len(year_list))}
for i,year in enumerate(year_list):
if ts==30:
start_date = str(year)+"-01-01 00:30"
elif ts==60:
start_date = str(year)+"-01-01 01:00"
end_date = str(year+1)+"-01-01 00:00"
si = qcutils.GetDateIndex(dt,start_date,ts=ts,default=0)
ei = qcutils.GetDateIndex(dt,end_date,ts=ts,default=len(dt)-1)
for item in cf["Variables"].keys():
if item not in ds.series.keys(): continue
data_1d,flag,attr = qcutils.GetSeriesasMA(ds,item,si=si,ei=ei)
annual_dict[item]["units"] = attr["units"]
if cf["Variables"][item]["operator"].lower()=="average":
annual_dict[item]["data"][i] = numpy.ma.average(data_1d)
elif cf["Variables"][item]["operator"].lower()=="sum":
annual_dict[item]["data"][i] = numpy.ma.sum(data_1d)
annual_dict[item]["units"] = annual_dict[item]["units"]+"/year"
else:
print "unrecognised operator"
annual_dict[item]["format"] = cf["Variables"][item]["format"]
# add the annual worksheet to the summary Excel file
xl_sheet = xl_file.add_sheet("Annual")
xl_write_data(xl_sheet,annual_dict)
In [15]:
# cumulative totals
h2o_list = ["Precip","ET"]
nee_list = [item for item in cf["Variables"].keys() if "NEE" in item]
nep_list = [item for item in cf["Variables"].keys() if "NEP" in item]
gpp_list = [item for item in cf["Variables"].keys() if "GPP" in item]
fre_list = [item for item in cf["Variables"].keys() if "Fre" in item]
co2_list = nee_list+nep_list+gpp_list+fre_list
series_list = h2o_list+co2_list
cumulative_dict = {}
for i,year in enumerate(year_list):
cumulative_dict[str(year)] = {}
if ts==30:
start_date = str(year)+"-01-01 00:30"
elif ts==60:
start_date = str(year)+"-01-01 01:00"
end_date = str(year+1)+"-01-01 00:00"
si = qcutils.GetDateIndex(dt,start_date,ts=ts,default=0)
ei = qcutils.GetDateIndex(dt,end_date,ts=ts,default=len(dt)-1)
ldt = dt[si:ei+1]
cumulative_dict[str(year)] = {}
cumulative_dict[str(year)]["DateTime"] = {"data":ldt,"units":"Year",
"format":"dd/mm/yyyy HH:MM"}
for item in series_list:
cumulative_dict[str(year)][item] = {}
data,flag,attr = qcutils.GetSeriesasMA(ds,item,si=si,ei=ei)
cumulative_dict[str(year)][item]["data"] = numpy.ma.cumsum(data)
cumulative_dict[str(year)][item]["units"] = attr["units"]+"/year"
cumulative_dict[str(year)][item]["format"] = cf["Variables"][item]["format"]
xl_sheet = xl_file.add_sheet("Cumulative("+str(year)+")")
xl_write_data(xl_sheet,cumulative_dict[str(year)])
In [16]:
# close the Excel workbook
xl_file.save(xl_name)
In [20]:
# plot time series of NEE, GPP and Reco
fig = plt.figure(1,figsize=(16,4))
plt.plot(daily_dict["DateTime"]["data"],daily_dict["NEE_SOLO"]["data"],'b-',alpha=0.3)
plt.plot(daily_dict["DateTime"]["data"],smooth(daily_dict["NEE_SOLO"]["data"],window_len=30),
'b-',linewidth=2,label="NEE_SOLO (30 day filter)")
plt.plot(daily_dict["DateTime"]["data"],daily_dict["GPP_SOLO"]["data"],'g-',alpha=0.3)
plt.plot(daily_dict["DateTime"]["data"],smooth(daily_dict["GPP_SOLO"]["data"],window_len=30),
'g-',linewidth=2,label="GPP_SOLO (30 day filter)")
plt.plot(daily_dict["DateTime"]["data"],daily_dict["Fre_SOLO"]["data"],'r-',alpha=0.3)
plt.plot(daily_dict["DateTime"]["data"],smooth(daily_dict["Fre_SOLO"]["data"],window_len=30),
'r-',linewidth=2,label="Fre_SOLO (30 day filter)")
plt.axhline(0)
plt.xlabel("Date")
plt.ylabel(daily_dict["NEE_SOLO"]["units"])
plt.legend(loc='upper left',prop={'size':8})
plt.tight_layout()
plt.show()
fig = plt.figure(2,figsize=(16,4))
plt.plot(daily_dict["DateTime"]["data"],daily_dict["NEE_FFNET"]["data"],'b-',alpha=0.3)
plt.plot(daily_dict["DateTime"]["data"],smooth(daily_dict["NEE_FFNET"]["data"],window_len=30),
'b-',linewidth=2,label="NEE_FFNET (30 day filter)")
plt.plot(daily_dict["DateTime"]["data"],daily_dict["GPP_FFNET"]["data"],'g-',alpha=0.3)
plt.plot(daily_dict["DateTime"]["data"],smooth(daily_dict["GPP_FFNET"]["data"],window_len=30),
'g-',linewidth=2,label="GPP_FFNET (30 day filter)")
plt.plot(daily_dict["DateTime"]["data"],daily_dict["Fre_FFNET"]["data"],'r-',alpha=0.3)
plt.plot(daily_dict["DateTime"]["data"],smooth(daily_dict["Fre_FFNET"]["data"],window_len=30),
'r-',linewidth=2,label="Fre_FFNET (30 day filter)")
plt.axhline(0)
plt.xlabel("Date")
plt.ylabel(daily_dict["NEE_FFNET"]["units"])
plt.legend(loc='upper left',prop={'size':8})
plt.tight_layout()
plt.show()
# plot time series of Fn,Fg,Fh,Fe
fig = plt.figure(3,figsize=(16,4))
plt.plot(daily_dict["DateTime"]["data"],daily_dict["Fn"]["data"],'k-',alpha=0.3)
plt.plot(daily_dict["DateTime"]["data"],smooth(daily_dict["Fn"]["data"],window_len=30),
'k-',linewidth=2,label="Fn (30 day filter)")
plt.plot(daily_dict["DateTime"]["data"],daily_dict["Fg"]["data"],'g-',alpha=0.3)
plt.plot(daily_dict["DateTime"]["data"],smooth(daily_dict["Fg"]["data"],window_len=30),
'g-',linewidth=2,label="Fg (30 day filter)")
plt.plot(daily_dict["DateTime"]["data"],daily_dict["Fh"]["data"],'r-',alpha=0.3)
plt.plot(daily_dict["DateTime"]["data"],smooth(daily_dict["Fh"]["data"],window_len=30),
'r-',linewidth=2,label="Fh (30 day filter)")
plt.plot(daily_dict["DateTime"]["data"],daily_dict["Fe"]["data"],'b-',alpha=0.3)
plt.plot(daily_dict["DateTime"]["data"],smooth(daily_dict["Fe"]["data"],window_len=30),
'b-',linewidth=2,label="Fe (30 day filter)")
plt.xlabel("Date")
plt.ylabel(daily_dict["Fn"]["units"])
plt.legend(loc='upper left',prop={'size':8})
plt.tight_layout()
plt.show()
In [60]:
# cumulative plots
color_list = ["blue","red","green","yellow","magenta","black","cyan","brown"]
year_list = cumulative_dict.keys()
year_list.sort()
# NEE_SOLO
fig = plt.figure()
plt.title("Net Ecosystem Exchange: SOLO")
for n,year in enumerate(year_list):
x = numpy.arange(0,len(cumulative_dict[year]["NEE_SOLO"]["data"]))*ts/float(60)
plt.plot(x,cumulative_dict[year]["NEE_SOLO"]["data"],color=color_list[numpy.mod(n,8)],
label=str(year))
plt.xlabel("Hour of Year")
plt.ylabel(cumulative_dict[year]["NEE_SOLO"]["units"])
plt.legend(loc='lower left',prop={'size':8})
plt.tight_layout()
plt.show()
# GPP_SOLO
fig = plt.figure()
plt.title("Gross Primary Productivity: SOLO")
for n,year in enumerate(year_list):
x = numpy.arange(0,len(cumulative_dict[year]["GPP_SOLO"]["data"]))*ts/float(60)
plt.plot(x,cumulative_dict[year]["GPP_SOLO"]["data"],color=color_list[numpy.mod(n,8)],
label=str(year))
plt.xlabel("Hour of Year")
plt.ylabel(cumulative_dict[year]["GPP_SOLO"]["units"])
plt.legend(loc='lower right',prop={'size':8})
plt.tight_layout()
plt.show()
# Fre_SOLO
fig = plt.figure()
plt.title("Ecosystem Respiration: SOLO")
for n,year in enumerate(year_list):
x = numpy.arange(0,len(cumulative_dict[year]["Fre_SOLO"]["data"]))*ts/float(60)
plt.plot(x,cumulative_dict[year]["Fre_SOLO"]["data"],color=color_list[numpy.mod(n,8)],
label=str(year))
plt.xlabel("Hour of Year")
plt.ylabel(cumulative_dict[year]["Fre_SOLO"]["units"])
plt.legend(loc='lower right',prop={'size':8})
plt.tight_layout()
plt.show()
# ET and precipitation
fig = plt.figure()
plt.title("ET and precipitation: SOLO")
for n,year in enumerate(year_list):
x = numpy.arange(0,len(cumulative_dict[year]["ET"]["data"]))*ts/float(60)
plt.plot(x,cumulative_dict[year]["ET"]["data"],color=color_list[numpy.mod(n,8)],
label=str(year))
plt.plot(x,cumulative_dict[year]["Precip"]["data"],color=color_list[numpy.mod(n,8)],
linestyle='--',label=str(year))
plt.xlabel("Hour of Year")
plt.ylabel(cumulative_dict[year]["ET"]["units"])
plt.legend(loc='upper left',prop={'size':8})
plt.tight_layout()
plt.show()
In [24]:
fre_list = ["Fre_SOLO","Fre_FFNET"]
for item in fre_list:
us_index = item.index("Fre")
print item[0:3],item[us_index+3:]
In [ ]: