In [41]:

    

import urllib2
import xmltodict
import pandas as pd
from datetime import datetime 
from httplib import BadStatusLine
import matplotlib.pyplot as plt
import numpy as np
import requests
import wellapplication as wa
%matplotlib inline


    

In [4]:

    

drive = 'E:'
raw_archive_folder = drive + '/PROJECTS/Snake Valley Water/Transducer Data/Raw_data_archive'
folder = raw_archive_folder + '/2016/2016 q2/'
enteredFolder = folder + '/entered/'
checkFolder = folder + '/toCheck/'
wellinfofile = drive + raw_archive_folder + '/well table 2015-03-23.csv'


    

In [17]:

    

inputfile = "E:/PROJECTS/Snake Valley Water/Transducer Data/Raw_data_archive/2016/2016 q2/ag13c 2016-08-02.xle"
manualwls = raw_archive_folder + '/All tape measurements.csv'
manual = pd.read_csv(manualwls, index_col="DateTime", engine="python")
barofile = raw_archive_folder + '/baro.csv'
baro = pd.read_csv(barofile,index_col=0, parse_dates=True)
wellinfo = pd.read_csv(folder + '/wellinfo4.csv')


    

In [19]:

    

g, drift, wellname = wa.imp_new_well(inputfile, wellinfo, manual, baro)


    

    




Well = ag13c
Efficiency = 0.35
Max Drift = 1.0855

In [23]:

    

well = wa.new_xle_imp(inputfile)


    

In [26]:

    

baro['Level'] = baro['pw03']


    

In [37]:

    

wellbaro = well_baro_merge(well, baro)


    

In [38]:

    

wellbaro.columns


    

    
Out[38]:





Index([u'Temperature', u'Level', u'name', u'pw03', u'pw10', u'pw19', u'integr',
       u'barometer'],
      dtype='object')

In [42]:

    

wellbaro[['barometer','Level']].plot()


    

    
Out[42]:





<matplotlib.axes._subplots.AxesSubplot at 0x23345668>






    

In [100]:

    

def fix_drift(wellbaro, manualfile, manmeas, meas, outcolname): 
    
    #breakpoints = self.get_breakpoints(wellbaro, manualfile)
    breakpoints = []
    manualfile['julian'] = manualfile.index.to_julian_date()
    for i in range(len(manualfile)):
        breakpoints.append(wa.fcl(wellbaro, pd.to_datetime(manualfile.index)[i]).name)
    breakpoints = sorted(list(set(breakpoints)))

    bracketedwls = {}
    dtnm = wellbaro.index.name
    manualfile['julian'] = manualfile.index.to_julian_date()


    drift_features = {}

    for i in range(len(breakpoints) - 1):
        # Break up pandas dataframe time series into pieces based on timing of manual measurements
        bracketedwls[i] = wellbaro.loc[(wellbaro.index.to_datetime() > breakpoints[i]) & (wellbaro.index.to_datetime() < breakpoints[i + 1])]

        if len(bracketedwls[i]) > 0:
            bracketedwls[i].loc[:, 'julian'] = bracketedwls[i].index.to_julian_date()

            last_man = wa.fcl(manualfile, breakpoints[i + 1])
            first_man = wa.fcl(manualfile, breakpoints[i])
            
            b = first_man[manmeas] - bracketedwls[i].ix[0, meas]
            
            m = (last_man[manmeas] - first_man[manmeas]) / (last_man['julian'] - first_man['julian'])
            bracketedwls[i].loc[:, 'datechange'] = bracketedwls[i].ix[:, 'julian'] - bracketedwls[i].ix[0, 'julian']
            bracketedwls[i].loc[:, 'wldiff'] = bracketedwls[i].loc[:, meas] - bracketedwls[i].ix[0, meas]
            bracketedwls[i].loc[:, outcolname] = bracketedwls[i][['datechange', meas]].apply(lambda x: x[1] + (m * x[0] + b), 1)
            drift_features[i] = {'begining':first_man, 'end':last_man, 'intercept':b, 'slope':m,
                                'first_meas':first_man[manmeas],'last_meas':last_man[manmeas]}
        else:
            pass
    wellbarofixed = pd.concat(bracketedwls)
    wellbarofixed.reset_index(inplace=True)
    wellbarofixed.set_index(dtnm, inplace=True)
    drift_info = pd.DataFrame(drift_features)
    return wellbarofixed, drift_info


    

In [180]:

    

def fix_drift(wellbaro, manualfile, meas= 'Level', manmeas =  'MeasuredDTW'):
    
    breakpoints = []
    manualfile['julian'] = manualfile.index.to_julian_date()
    for i in range(len(manualfile)):
        breakpoints.append(wa.fcl(wellbaro, pd.to_datetime(manualfile.index)[i]).name)
    breakpoints = sorted(list(set(breakpoints)))

    dtnm = wellbaro.index.name

    last_man_wl, first_man_wl, last_tran_wl, driftlen = [], [], [], []
    bracketedwls = {}
    for i in range(len(breakpoints) - 1):
        # Break up time series into pieces based on timing of manual measurements
        bracketedwls[i] = wellbaro[(pd.to_datetime(wellbaro.index) >= breakpoints[i]) & (pd.to_datetime(wellbaro.index) <= breakpoints[i+1])]
        print(len(bracketedwls[i]))
        if len(bracketedwls[i]) > 0:
            # get difference in transducer water measurements
            bracketedwls[i].loc[:,'diff_wls'] = bracketedwls[i].loc[:,meas].diff()
            # get difference of each depth to water from initial measurement
            first_man = wa.fcl(manualfile, breakpoints[i])
            bracketedwls[i].loc[:, 'DeltaLevel'] = bracketedwls[i].loc[:, meas] - bracketedwls[i].ix[0, meas]
            bracketedwls[i].loc[:, 'MeasuredDTW'] = first_man - bracketedwls[i].loc[:,'DeltaLevel']
            last_man = wa.fcl(manualfile, breakpoints[i + 1])
            first_man = wa.fcl(manualfile, breakpoints[i])
            last_tran = float(bracketedwls[i].loc[breakpoints[i+1], meas])
            driftlen = len(bracketedwls[i].index)
            print(last_man)
            bracketedwls[i].loc[:, 'last_diff_int'] = np.round((last_tran - last_man[manmeas]), 4) / np.round(driftlen - 1.0, 4)
            bracketedwls[i].loc[:, 'DriftCorrection'] = np.round(
                bracketedwls[i].loc[:, 'last_diff_int'].cumsum() - bracketedwls[i].loc[:, 'last_diff_int'], 4)
        else:
            pass
    wellbarofixed = pd.concat(bracketedwls)
    wellbarofixed.reset_index(inplace=True)
    wellbarofixed.set_index(dtnm, inplace=True)
    return wellbarofixed#,bracketedwls,breakpoints


    

In [71]:

    

manual35 = manual[manual['WellID']==35]
manual35['dt'] = pd.to_datetime(manual35.index)
manual_35 = manual35.reset_index()
manual_35.set_index('dt',inplace=True)


    

    




c:\winpython-64bit-2.7.10.3\python-2.7.10.amd64\lib\site-packages\ipykernel\__main__.py:1: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  if __name__ == '__main__':

In [181]:

    

fixed= fix_drift(wellbaro, manual_35, 'Level')


    

    




2184
DateTime       8/2/2016 9:39
WellID                    35
MeasuredDTW            86.63
WellName               AG13C
julian            2.4576e+06
Name: 2016-08-02 09:39:00, dtype: object

In [165]:

    

fixed[2]


    

    
Out[165]:





[Timestamp('2016-05-03 10:00:00', freq='60T'),
 Timestamp('2016-08-02 09:00:00', freq='60T')]

In [145]:

    

fixed


    

    
Out[145]:






  

    

      

      
level_0
      
Temperature
      
Level
      
name
      
pw03
      
pw10
      
pw19
      
integr
      
barometer
      
dbp
      
dwl
      
corrwl
    
    

      
DateTime
      

      

      

      

      

      

      

      

      

      

      

      

    
  
  

  

In [182]:

    

x = fixed.index
y = fixed['DriftCorrection']

x1 = manual_35.index
y1 = manual_35['MeasuredDTW']

plt.figure()

plt.plot(x,y)
plt.scatter(x1,y1)
plt.xlim('5/1/2016','8/15/2016')


    

    
Out[182]:





(736085.0, 736191.0)






    

In [43]:

    

wellbaro['dbp'] = wellbaro.barometer.diff()
wellbaro['dwl'] = wellbaro.Level.diff()


    

In [49]:

    

wellbaro['dbp'] = wellbaro.barometer.diff()
wellbaro['dwl'] = wellbaro.Level.diff()
wellbaro['corrwl'] = wellbaro[['dbp','dwl']].apply(lambda x: x[1]-x[0],1).cumsum()


    

In [107]:

    

wellbaro['corrwl'].plot()


    

    
Out[107]:





<matplotlib.axes._subplots.AxesSubplot at 0x25066908>






    

In [36]:

    

def well_baro_merge(wellfile, barofile, sampint=60):
    """Remove barometric pressure from nonvented transducers.
    Args:
        wellfile (pandas.core.frame.DataFrame):
            Pandas DataFrame of water level data labeled 'Level'; index must be datetime
        barofile (pandas.core.frame.DataFrame):
            Pandas DataFrame barometric data labeled 'Level'; index must be datetime
        sampint (int):
            sampling interval in minutes; default 60

    Returns:
        wellbaro (Pandas DataFrame):
           corrected water levels with bp removed
    """

    # resample data to make sample interval consistent
    baro = wa.hourly_resample(barofile, 0, sampint)
    well = wa.hourly_resample(wellfile, 0, sampint)

    # reassign `Level` to reduce ambiguity
    if 'Level' in list(baro.columns):
        baro= baro.rename(columns = {'Level':'barometer'})
    # combine baro and well data for easy calculations, graphing, and manipulation
    wellbaro = pd.merge(well, baro, left_index=True, right_index=True, how='inner')
    
    wellbaro['dbp'] = wellbaro.barometer.diff()
    wellbaro['dwl'] = wellbaro.Level.diff()
    wellbaro['corrwl'] = wellbaro[['dbp','dwl']].apply(lambda x: x[1]-x[0],1).cumsum()
    
    return wellbaro


    

In [ ]:

    

def baro_drift_correct(wellfile, barofile, manualfile, sampint=60, wellelev=4800, stickup=0):
    """Remove barometric pressure and corrects drift.
    Args:
        wellfile (pandas.core.frame.DataFrame):
            Pandas DataFrame of water level data labeled 'Level'; index must be datetime
        barofile (pandas.core.frame.DataFrame):
            Pandas DataFrame barometric data labeled 'Level'; index must be datetime
        manualfile (pandas.core.frame.DataFrame):
            Pandas DataFrame manual level data in the first column after the index; index must be datetime
        sampint (int):
            sampling interval in minutes; default 60
        wellelev (int):
            site ground surface elevation in feet
        stickup (float):
            offset of measure point from ground in feet

    Returns:
        wellbarofinal (Pandas DataFrame):
           corrected water levels
    """
    # Remove dangling ends
    baroclean = dataendclean(barofile, 'Level')
    wellclean = dataendclean(wellfile, 'Level')

    # resample data to make sample interval consistent
    baro = hourly_resample(baroclean, 0, sampint)
    well = hourly_resample(wellclean, 0, sampint)

    # reassign `Level` to reduce ambiguity
    well['abs_feet_above_levelogger'] = well['Level']
    baro['abs_feet_above_barologger'] = baro['Level']

    # combine baro and well data for easy calculations, graphing, and manipulation
    wellbaro = pd.merge(well, baro, left_index=True, right_index=True, how='inner')

    # subtract barometric pressure from total pressure measured by transducer
    wellbaro['adjusted_levelogger'] = wellbaro['abs_feet_above_levelogger'] - wellbaro['abs_feet_above_barologger']

    breakpoints = []
    for i in range(len(manualfile) + 1):
        breakpoints.append(fcl(wellbaro, manualfile.index.to_datetime()[i - 1]).name)

    last_man_wl, first_man_wl, last_tran_wl, driftlen = [], [], [], []
    bracketedwls = {}
    for i in range(len(manualfile) - 1):
        # Break up time series into pieces based on timing of manual measurements
        bracketedwls[i + 1] = wellbaro.loc[(wellbaro.index.to_datetime() > breakpoints[i + 1]) & (wellbaro.index.to_datetime() < breakpoints[i + 2])]
        # get difference in transducer water measurements
        bracketedwls[i + 1]['diff_wls'] = bracketedwls[i + 1]['abs_feet_above_levelogger'].diff()
        # get difference of each depth to water from initial measurement
        bracketedwls[i + 1].loc[:, 'DeltaLevel'] = bracketedwls[i + 1].loc[:, 'adjusted_levelogger'] - \
                                                   bracketedwls[i + 1].ix[0, 'adjusted_levelogger']
        bracketedwls[i + 1].loc[:, 'MeasuredDTW'] = fcl(manualfile, breakpoints[i + 1])[0] - \
                                                    bracketedwls[i + 1].loc[:,'DeltaLevel']
        last_man_wl.append(fcl(manualfile, breakpoints[i + 2])[0])
        first_man_wl.append(fcl(manualfile, breakpoints[i + 1])[0])
        last_tran_wl.append(
            float(bracketedwls[i + 1].loc[max(bracketedwls[i + 1].index.to_datetime()), 'MeasuredDTW']))
        driftlen.append(len(bracketedwls[i + 1].index))
        bracketedwls[i + 1].loc[:, 'last_diff_int'] = np.round((last_tran_wl[i] - last_man_wl[i]), 4) / np.round(
            driftlen[i] - 1.0, 4)
        bracketedwls[i + 1].loc[:, 'DriftCorrection'] = np.round(
            bracketedwls[i + 1].loc[:, 'last_diff_int'].cumsum() - bracketedwls[i + 1].loc[:, 'last_diff_int'], 4)

    wellbarofixed = pd.concat(bracketedwls)
    wellbarofixed.reset_index(inplace=True)
    wellbarofixed.set_index('DateTime', inplace=True)
    # Get Depth to water below casing
    wellbarofixed.loc[:, 'DTWBelowCasing'] = wellbarofixed['MeasuredDTW'] - wellbarofixed['DriftCorrection']

    # subtract casing height from depth to water below casing
    wellbarofixed.loc[:, 'DTWBelowGroundSurface'] = wellbarofixed.loc[:,
                                                    'DTWBelowCasing'] - stickup  # well riser height

    # subtract depth to water below ground surface from well surface elevation
    wellbarofixed.loc[:, 'WaterElevation'] = wellelev - wellbarofixed.loc[:, 'DTWBelowGroundSurface']

    wellbarofinal = smoother(wellbarofixed, 'WaterElevation')

    return wellbarofinal