In [1]:
import urllib2
import xmltodict
import pandas as pd
from datetime import datetime
from httplib import BadStatusLine
import matplotlib.pyplot as plt
import numpy as np
import requests
import wellapplication as wa
%matplotlib inline
In [2]:
drive = 'E:'
raw_archive_folder = drive + '/PROJECTS/Snake Valley Water/Transducer Data/Raw_data_archive'
folder = raw_archive_folder + '/2016/2016 q2/'
enteredFolder = folder + '/entered/'
checkFolder = folder + '/toCheck/'
wellinfofile = drive + raw_archive_folder + '/well table 2015-03-23.csv'
In [3]:
inputfile = "E:/PROJECTS/Snake Valley Water/Transducer Data/Raw_data_archive/2016/2016 q2/ag13c 2016-08-02.xle"
manualwls = raw_archive_folder + '/All tape measurements.csv'
manual = pd.read_csv(manualwls, index_col="DateTime", engine="python")
barofile = raw_archive_folder + '/baro.csv'
baro = pd.read_csv(barofile,index_col=0, parse_dates=True)
wellinfo = pd.read_csv(folder + '/wellinfo4.csv')
In [14]:
g, drift, wellname = wa.imp_new_well(inputfile, wellinfo, manual, baro)
In [19]:
df = wa.new_xle_imp(inputfile)
jf = wa.jumpfix(df, 'Level', threashold=0.005)
jf['newVal'][-1] > 10
Out[19]:
In [12]:
xle = inputfile
xle_df = wa.new_xle_imp(xle)
manual35 = manual[manual['WellID']==35]
manual35['dt'] = pd.to_datetime(manual35.index)
manual_35 = manual35.reset_index()
manual_35.set_index('dt',inplace=True)
fd = wa.fix_drift(xle_df, manual_35, meas='Level', manmeas='MeasuredDTW', outcolname='DriftCorrection')
'DriftCorrection' in list(fd[0].columns)
Out[12]:
In [9]:
manual
In [10]:
test_fix_drift(xle_df, manual)
In [21]:
well = wa.new_xle_imp(inputfile)
In [9]:
baro['Level'] = baro['pw03']
In [10]:
wellbaro = wa.well_baro_merge(well, baro)
In [22]:
type(well)
Out[22]:
In [31]:
wellbaro[['corrwl','Level']].plot()
Out[31]:
In [20]:
def fix_drift(well, manualfile, meas= 'Level', manmeas = 'MeasuredDTW', outcolname = 'DriftCorrection'):
"""Remove transducer drift from nonvented transducer data.
Args:
well (pandas.core.frame.DataFrame):
Pandas DataFrame of merged water level and barometric data; index must be datetime
manualfile (pandas.core.frame.DataFrame):
Pandas DataFrame of manual measurements
meas (str):
name of column in well DataFrame containing transducer data to be corrected
manmeas (str):
name of column in manualfile Dataframe containing manual measurement data
outcolname (str):
name of column resulting from correction
Returns:
wellbarofixed (pandas.core.frame.DataFrame):
corrected water levels with bp removed
driftinfo (pandas.core.frame.DataFrame):
dataframe of correction parameters
"""
#breakpoints = self.get_breakpoints(wellbaro, manualfile)
breakpoints = []
for i in range(len(manualfile)):
breakpoints.append(wa.fcl(wellbaro, pd.to_datetime(manualfile.index)[i]).name)
breakpoints = sorted(list(set(breakpoints)))
bracketedwls, drift_features = {},{}
dtnm = wellbaro.index.name
manualfile['julian'] = manualfile.index.to_julian_date()
for i in range(len(breakpoints) - 1):
# Break up pandas dataframe time series into pieces based on timing of manual measurements
bracketedwls[i] = wellbaro.loc[(wellbaro.index.to_datetime() > breakpoints[i]) & (wellbaro.index.to_datetime() < breakpoints[i + 1])]
if len(bracketedwls[i]) > 0:
bracketedwls[i].loc[:, 'julian'] = bracketedwls[i].index.to_julian_date()
last_trans = bracketedwls[i].ix[-1, meas] # last transducer measurment
first_trans = bracketedwls[i].ix[0, meas] # first transducer measurement
last_man = wa.fcl(manualfile, breakpoints[i + 1]) # first manual measurment
first_man = wa.fcl(manualfile, breakpoints[i]) # last manual mesurement
# intercept of line = value of first manual measurement
b = first_man[manmeas] - first_trans
# slope of line = change in difference between manual and transducer over time
m = ((last_man[manmeas] - last_trans) - (first_man[manmeas]-first_trans)) / (last_man['julian'] - first_man['julian'])
# datechange = amount of time between manual measurements
bracketedwls[i].loc[:, 'datechange'] = bracketedwls[i].ix[:, 'julian'] - bracketedwls[i].ix[0, 'julian']
#bracketedwls[i].loc[:, 'wldiff'] = bracketedwls[i].loc[:, meas] - first_trans
bracketedwls[i].loc[:, outcolname] = bracketedwls[i][['datechange', meas]].apply(lambda x: x[1] + (m * x[0] + b), 1)
drift_features[i] = {'begining':first_man, 'end':last_man, 'intercept':b, 'slope':m,
'first_meas':first_man[manmeas],'last_meas':last_man[manmeas]}
else:
pass
wellbarofixed = pd.concat(bracketedwls)
wellbarofixed.reset_index(inplace=True)
wellbarofixed.set_index(dtnm, inplace=True)
drift_info = pd.DataFrame(drift_features)
return wellbarofixed, drift_info
In [5]:
manual35 = manual[manual['WellID']==35]
manual35['dt'] = pd.to_datetime(manual35.index)
manual_35 = manual35.reset_index()
manual_35.set_index('dt',inplace=True)
In [21]:
fixed= fix_drift_slope(wellbaro, manual_35, 'Level')[0]
In [165]:
fixed[2]
Out[165]:
In [145]:
fixed
Out[145]:
In [22]:
x = fixed.index
y = fixed['DriftCorrection']
x1 = manual_35.index
y1 = manual_35['MeasuredDTW']
plt.figure()
plt.plot(x,y)
plt.scatter(x1,y1)
plt.xlim('5/1/2016','8/15/2016')
Out[22]:
In [43]:
wellbaro['dbp'] = wellbaro.barometer.diff()
wellbaro['dwl'] = wellbaro.Level.diff()
In [49]:
wellbaro['dbp'] = wellbaro.barometer.diff()
wellbaro['dwl'] = wellbaro.Level.diff()
wellbaro['corrwl'] = wellbaro[['dbp','dwl']].apply(lambda x: x[1]-x[0],1).cumsum()
In [107]:
wellbaro['corrwl'].plot()
Out[107]:
In [28]:
def well_baro_merge(wellfile, barofile, sampint=60):
"""Remove barometric pressure from nonvented transducers.
Args:
wellfile (pandas.core.frame.DataFrame):
Pandas DataFrame of water level data labeled 'Level'; index must be datetime
barofile (pandas.core.frame.DataFrame):
Pandas DataFrame barometric data labeled 'Level'; index must be datetime
sampint (int):
sampling interval in minutes; default 60
Returns:
wellbaro (Pandas DataFrame):
corrected water levels with bp removed
"""
# resample data to make sample interval consistent
baro = wa.hourly_resample(barofile, 0, sampint)
well = wa.hourly_resample(wellfile, 0, sampint)
# reassign `Level` to reduce ambiguity
if 'Level' in list(baro.columns):
baro= baro.rename(columns = {'Level':'barometer'})
# combine baro and well data for easy calculations, graphing, and manipulation
wellbaro = pd.merge(well, baro, left_index=True, right_index=True, how='inner')
wellbaro['dbp'] = wellbaro.barometer.diff()
wellbaro['dwl'] = wellbaro.Level.diff()
first_well = wellbaro.Level[0]
wellbaro['corrwl'] = wellbaro[['dbp','dwl']].apply(lambda x: x[1]-x[0],1).cumsum()+first_well
return wellbaro
In [12]:
def baro_drift_correct(wellfile, barofile, manualfile, sampint=60, wellelev=4800, stickup=0):
"""Remove barometric pressure and corrects drift.
Args:
wellfile (pandas.core.frame.DataFrame):
Pandas DataFrame of water level data labeled 'Level'; index must be datetime
barofile (pandas.core.frame.DataFrame):
Pandas DataFrame barometric data labeled 'Level'; index must be datetime
manualfile (pandas.core.frame.DataFrame):
Pandas DataFrame manual level data in the first column after the index; index must be datetime
sampint (int):
sampling interval in minutes; default 60
wellelev (int):
site ground surface elevation in feet
stickup (float):
offset of measure point from ground in feet
Returns:
wellbarofinal (Pandas DataFrame):
corrected water levels
"""
# Remove dangling ends
baroclean = dataendclean(barofile, 'Level')
wellclean = dataendclean(wellfile, 'Level')
# resample data to make sample interval consistent
baro = hourly_resample(baroclean, 0, sampint)
well = hourly_resample(wellclean, 0, sampint)
# reassign `Level` to reduce ambiguity
well['abs_feet_above_levelogger'] = well['Level']
baro['abs_feet_above_barologger'] = baro['Level']
# combine baro and well data for easy calculations, graphing, and manipulation
wellbaro = pd.merge(well, baro, left_index=True, right_index=True, how='inner')
# subtract barometric pressure from total pressure measured by transducer
wellbaro['adjusted_levelogger'] = wellbaro['abs_feet_above_levelogger'] - wellbaro['abs_feet_above_barologger']
breakpoints = []
for i in range(len(manualfile) + 1):
breakpoints.append(fcl(wellbaro, manualfile.index.to_datetime()[i - 1]).name)
last_man_wl, first_man_wl, last_tran_wl, driftlen = [], [], [], []
bracketedwls = {}
for i in range(len(manualfile) - 1):
# Break up time series into pieces based on timing of manual measurements
bracketedwls[i + 1] = wellbaro.loc[(wellbaro.index.to_datetime() > breakpoints[i + 1]) & (wellbaro.index.to_datetime() < breakpoints[i + 2])]
# get difference in transducer water measurements
bracketedwls[i + 1]['diff_wls'] = bracketedwls[i + 1]['abs_feet_above_levelogger'].diff()
# get difference of each depth to water from initial measurement
bracketedwls[i + 1].loc[:, 'DeltaLevel'] = bracketedwls[i + 1].loc[:, 'adjusted_levelogger'] - \
bracketedwls[i + 1].ix[0, 'adjusted_levelogger']
bracketedwls[i + 1].loc[:, 'MeasuredDTW'] = fcl(manualfile, breakpoints[i + 1])[0] - \
bracketedwls[i + 1].loc[:,'DeltaLevel']
last_man_wl.append(fcl(manualfile, breakpoints[i + 2])[0])
first_man_wl.append(fcl(manualfile, breakpoints[i + 1])[0])
last_tran_wl.append(
float(bracketedwls[i + 1].loc[max(bracketedwls[i + 1].index.to_datetime()), 'MeasuredDTW']))
driftlen.append(len(bracketedwls[i + 1].index))
bracketedwls[i + 1].loc[:, 'last_diff_int'] = np.round((last_tran_wl[i] - last_man_wl[i]), 4) / np.round(
driftlen[i] - 1.0, 4)
bracketedwls[i + 1].loc[:, 'DriftCorrection'] = np.round(
bracketedwls[i + 1].loc[:, 'last_diff_int'].cumsum() - bracketedwls[i + 1].loc[:, 'last_diff_int'], 4)
wellbarofixed = pd.concat(bracketedwls)
wellbarofixed.reset_index(inplace=True)
wellbarofixed.set_index('DateTime', inplace=True)
# Get Depth to water below casing
wellbarofixed.loc[:, 'DTWBelowCasing'] = wellbarofixed['MeasuredDTW'] - wellbarofixed['DriftCorrection']
# subtract casing height from depth to water below casing
wellbarofixed.loc[:, 'DTWBelowGroundSurface'] = wellbarofixed.loc[:,
'DTWBelowCasing'] - stickup # well riser height
# subtract depth to water below ground surface from well surface elevation
wellbarofixed.loc[:, 'WaterElevation'] = wellelev - wellbarofixed.loc[:, 'DTWBelowGroundSurface']
wellbarofinal = smoother(wellbarofixed, 'WaterElevation')
return wellbarofinal