In [1]:
import csv
import datetime
import numpy
import matplotlib.pyplot as plt
import sys
sys.path.append('../scripts')
import qcio
import qcutils
In [3]:
def get_datetimefromymdhms(ds):
''' Creates a series of Python datetime objects from the year, month,
day, hour, minute and second series stored in the netCDF file.'''
SeriesList = ds.series.keys()
if 'Year' not in SeriesList or 'Month' not in SeriesList or 'Day' not in SeriesList or 'Hour' not in SeriesList or 'Minute' not in SeriesList or 'Second' not in SeriesList:
#log.info(' get_datetimefromymdhms: unable to find all datetime fields required')
print 'get_datetimefromymdhms: unable to find all datetime fields required'
return
#log.info(' Getting the date and time series')
print 'Getting the date and time series'
nRecs = ds.globalattributes["nc_nrecs"]
ds.series[unicode('DateTime')] = {}
ds.series['DateTime']['Data'] = [None]*nRecs
ds.series[unicode('Date')] = {}
ds.series['Date']['Data'] = [None]*nRecs
ds.series[unicode('Time')] = {}
ds.series['Time']['Data'] = [None]*nRecs
if "Microsecond" in ds.series.keys():
microseconds = ds.series["Microsecond"]["Data"]
else:
microseconds = numpy.zeros(nRecs,dtype=numpy.float64)
for i in range(nRecs):
ds.series['DateTime']['Data'][i] = datetime.datetime(int(ds.series['Year']['Data'][i]),
int(ds.series['Month']['Data'][i]),
int(ds.series['Day']['Data'][i]),
int(ds.series['Hour']['Data'][i]),
int(ds.series['Minute']['Data'][i]),
int(ds.series['Second']['Data'][i]),
int(microseconds[i]))
ds.series['Date']['Data'][i] = datetime.date(int(ds.series['Year']['Data'][i]),
int(ds.series['Month']['Data'][i]),
int(ds.series['Day']['Data'][i]))
ds.series['Time']['Data'][i] = datetime.time(int(ds.series['Hour']['Data'][i]),
int(ds.series['Minute']['Data'][i]),
int(ds.series['Second']['Data'][i]),
int(microseconds[i]))
ds.series['DateTime']['Flag'] = numpy.zeros(nRecs)
ds.series['DateTime']['Attr'] = {}
ds.series['DateTime']['Attr']['long_name'] = 'Date-time object'
ds.series['DateTime']['Attr']['units'] = 'None'
ds.series['Date']['Flag'] = numpy.zeros(nRecs)
ds.series['Date']['Attr'] = {}
ds.series['Date']['Attr']['long_name'] = 'Date object'
ds.series['Date']['Attr']['units'] = 'None'
ds.series['Time']['Flag'] = numpy.zeros(nRecs)
ds.series['Time']['Attr'] = {}
ds.series['Time']['Attr']['long_name'] = 'Time object'
ds.series['Time']['Attr']['units'] = 'None'
In [4]:
infilename = "../../Sites/WombatStateForest/Data/Fast/3952/test_3952.dat"
# columns are:
# data file description
# 0 0 Year
# 1 1 Month
# 2 2 Day
# 3 3 Hour
# 4 4 Minute
# 5 5 Second
# 6 7 Ux
# 7 8 Uy
# 8 9 Uz
# 9 12 Tv
#10 13 Diag_CSAT
data_3952 = numpy.genfromtxt(infilename,skip_header=4,delimiter=",",usecols=(0,1,2,3,4,5,7,8,9,12,13),
missing_values=-9999,filling_values=-9999)
data_3952 = numpy.ma.masked_equal(data,float(-9999),copy=True)
nRecs = data_3952.shape[0]
flag = numpy.zeros(nRecs,dtype=numpy.int32)
ds_3952=qcio.DataStructure()
ds_3952.globalattributes["nc_nrecs"] = nRecs
seconds = numpy.array(data[:,5],dtype=numpy.int32)
microseconds = numpy.rint((data[:,5]%1)*float(1000000))
qcutils.CreateSeries(ds_3952,'Year',data[:,0],Flag=flag,Attr=qcutils.MakeAttributeDictionary(long_name='Year',units='years'))
qcutils.CreateSeries(ds_3952,'Month',data[:,1],Flag=flag,Attr=qcutils.MakeAttributeDictionary(long_name='Month',units='months'))
qcutils.CreateSeries(ds_3952,'Day',data[:,2],Flag=flag,Attr=qcutils.MakeAttributeDictionary(long_name='Day',units='days'))
qcutils.CreateSeries(ds_3952,'Hour',data[:,3],Flag=flag,Attr=qcutils.MakeAttributeDictionary(long_name='Hour',units='hours'))
qcutils.CreateSeries(ds_3952,'Minute',data[:,4],Flag=flag,Attr=qcutils.MakeAttributeDictionary(long_name='Minute',units='minutes'))
qcutils.CreateSeries(ds_3952,'Second',seconds,Flag=flag,Attr=qcutils.MakeAttributeDictionary(long_name='Second',units='seconds'))
qcutils.CreateSeries(ds_3952,'Microsecond',microseconds,Flag=flag,Attr=qcutils.MakeAttributeDictionary(long_name='Microseconds',units='microseconds'))
get_datetimefromymdhms(ds_3952)
In [21]:
infilename = "../../Sites/WombatStateForest/Data/Fast/6281/test_6281.dat"
# columns are:
# data file description
# 0 0 Year
# 1 1 Month
# 2 2 Day
# 3 3 Hour
# 4 4 Minute
# 5 5 Second
# 6 7 Ux
# 7 8 Uy
# 8 9 Uz
# 9 10 Cc_7500
#10 11 Ah_7500
#11 12 Tv
#12 13 Diag_CSAT
#13 14 Diag_7500
data_6281 = numpy.genfromtxt(infilename,skip_header=4,delimiter=",",usecols=(0,1,2,3,4,5,7,8,9,10,11,12,13,14),
missing_values=-9999,filling_values=-9999)
data_6281 = numpy.ma.masked_equal(data,float(-9999),copy=True)
nRecs = data_6281.shape[0]
seconds = numpy.array(data[:,5],dtype=numpy.int32)
microseconds = numpy.rint((data[:,5]%1)*float(1000000))
ldt_6281 = [None]*nRecs
for i in range(0,nRecs):
ldt_6281[i] = datetime.datetime(int(data_6281[i,0]),int(data_6281[i,1]),int(data_6281[i,2]),
int(data_6281[i,3]),int(data_6281[i,4]),int(seconds[i]),int(microseconds[i]))
In [22]:
ldt_3952=ds_3952.series["DateTime"]["Data"]
ldt_6281=ds_6281.series["DateTime"]["Data"]
ld_3952=ds_3952.series["Date"]["Data"]
ld_6281=ds_6281.series["Date"]["Data"]
lt_3952=ds_3952.series["Time"]["Data"]
lt_6281=ds_6281.series["Time"]["Data"]
In [23]:
ldt_10hz = [x for x in qcutils.perdelta(ldt_3952[0],ldt_3952[-1],datetime.timedelta(seconds=0.1))]
In [24]:
for i in range(10,20):
print ldt_10hz[i],ldt_3952[i],ldt_6281[i],microseconds[i]
In [10]:
data_10hz = numpy.ma.array((len(ldt_10hz),13))
In [12]:
idx=qcutils.find_indices(ldt_10hz,ldt_3952)
In [15]:
data_10hz[idx,0]=data_3952[:,0]
In [ ]: