Accessing ncSOS with OWSLib


In [1]:
from owslib.sos import SensorObservationService
import pdb
from owslib.etree import etree
import pandas as pd
import datetime as dt

In [12]:
# usgs woods hole
# buoy data (ADCP)
url='http://geoport.whoi.edu/thredds/sos/usgs/data2/emontgomery/stellwagen/CF-1.6/HURRIRENE_BB/9141wh-a.nc'
usgs = SensorObservationService(url)
contents = usgs.contents

In [13]:
usgs.contents


Out[13]:
{'network-all': <owslib.swe.observation.sos100.SosObservationOffering at 0x7f1a94461210>,
 'urn_ioos_station_gov.usgs_9141wh-a': <owslib.swe.observation.sos100.SosObservationOffering at 0x7f1a94461a50>}

In [14]:
off = usgs.offerings[1]
off.name


Out[14]:
'urn:ioos:station:gov.usgs:9141wh-a'

In [15]:
off.response_formats


Out[15]:
['text/xml;schema="om/1.0.0"']

In [16]:
off.observed_properties


Out[16]:
['urn:ioos:sensor:gov.usgs:9141wh-a:Rec',
 'urn:ioos:sensor:gov.usgs:9141wh-a:brange',
 'urn:ioos:sensor:gov.usgs:9141wh-a:Tx_1211',
 'urn:ioos:sensor:gov.usgs:9141wh-a:SV_80',
 'urn:ioos:sensor:gov.usgs:9141wh-a:Hdg_1215',
 'urn:ioos:sensor:gov.usgs:9141wh-a:Ptch_1216',
 'urn:ioos:sensor:gov.usgs:9141wh-a:Roll_1217',
 'urn:ioos:sensor:gov.usgs:9141wh-a:P_1294',
 'urn:ioos:sensor:gov.usgs:9141wh-a:SDP_850',
 'urn:ioos:sensor:gov.usgs:9141wh-a:u_1205',
 'urn:ioos:sensor:gov.usgs:9141wh-a:v_1206',
 'urn:ioos:sensor:gov.usgs:9141wh-a:w_1204',
 'urn:ioos:sensor:gov.usgs:9141wh-a:Werr_1201',
 'urn:ioos:sensor:gov.usgs:9141wh-a:AGC_1202',
 'urn:ioos:sensor:gov.usgs:9141wh-a:PGd_1203']

In [17]:
# the get observation request below works.  How can we recreate this using OWSLib?
# http://geoport-dev.whoi.edu/thredds/sos/usgs/data2/notebook/1211-A1H.cdf?service=SOS&version=1.0.0&request=GetObservation&responseFormat=text%2Fxml%3Bsubtype%3D%22om%2F1.0.0%22&offering=1211-A1H&observedProperty=u_1205&procedure=urn:ioos:station:gov.usgs:1211-A1H

In [25]:
start = '2011-08-26T16:00:00Z'
stop = '2011-08-26T17:00:00Z'
response = usgs.get_observation(offerings=['9141wh-a'],
                                 responseFormat='text/xml;schema="om/1.0.0"',
                                 observedProperties=['Tx_1211'],
                                 procedure='urn:ioos:station:gov.usgs:9141wh-a',
                                 eventTime='%s/%s' % (start,stop))


---------------------------------------------------------------------------
timeout                                   Traceback (most recent call last)
<ipython-input-25-3c95b8d4fb95> in <module>()
      5                                  observedProperties=['Tx_1211'],
      6                                  procedure='urn:ioos:station:gov.usgs:9141wh-a',
----> 7                                  eventTime='%s/%s' % (start,stop))

/home/usgs/anaconda/lib/python2.7/site-packages/OWSLib-0.8.7-py2.7.egg/owslib/swe/observation/sos100.py in get_observation(self, responseFormat, offerings, observedProperties, eventTime, method, **kwargs)
    163         data = urlencode(request)
    164 
--> 165         response = openURL(base_url, data, method, username=self.username, password=self.password).read()
    166         try:
    167             tr = etree.fromstring(response)

/home/usgs/anaconda/lib/python2.7/site-packages/OWSLib-0.8.7-py2.7.egg/owslib/util.py in openURL(url_base, data, method, cookies, username, password, timeout)
    155         if cookies is not None:
    156             req.add_header('Cookie', cookies)
--> 157         u = openit(req, timeout=timeout)
    158     except HTTPError, e: #Some servers may set the http header to 400 if returning an OGC service exception or 401 if unauthorised.
    159         if e.code in [400, 401]:

/home/usgs/anaconda/lib/python2.7/urllib2.pyc in urlopen(url, data, timeout)
    125     if _opener is None:
    126         _opener = build_opener()
--> 127     return _opener.open(url, data, timeout)
    128 
    129 def install_opener(opener):

/home/usgs/anaconda/lib/python2.7/urllib2.pyc in open(self, fullurl, data, timeout)
    402             req = meth(req)
    403 
--> 404         response = self._open(req, data)
    405 
    406         # post-process response

/home/usgs/anaconda/lib/python2.7/urllib2.pyc in _open(self, req, data)
    420         protocol = req.get_type()
    421         result = self._call_chain(self.handle_open, protocol, protocol +
--> 422                                   '_open', req)
    423         if result:
    424             return result

/home/usgs/anaconda/lib/python2.7/urllib2.pyc in _call_chain(self, chain, kind, meth_name, *args)
    380             func = getattr(handler, meth_name)
    381 
--> 382             result = func(*args)
    383             if result is not None:
    384                 return result

/home/usgs/anaconda/lib/python2.7/urllib2.pyc in http_open(self, req)
   1212 
   1213     def http_open(self, req):
-> 1214         return self.do_open(httplib.HTTPConnection, req)
   1215 
   1216     http_request = AbstractHTTPHandler.do_request_

/home/usgs/anaconda/lib/python2.7/urllib2.pyc in do_open(self, http_class, req)
   1185         else:
   1186             try:
-> 1187                 r = h.getresponse(buffering=True)
   1188             except TypeError: # buffering kw not supported
   1189                 r = h.getresponse()

/home/usgs/anaconda/lib/python2.7/httplib.pyc in getresponse(self, buffering)
   1065         response = self.response_class(*args, **kwds)
   1066 
-> 1067         response.begin()
   1068         assert response.will_close != _UNKNOWN
   1069         self.__state = _CS_IDLE

/home/usgs/anaconda/lib/python2.7/httplib.pyc in begin(self)
    407         # read until we get a non-100 response
    408         while True:
--> 409             version, status, reason = self._read_status()
    410             if status != CONTINUE:
    411                 break

/home/usgs/anaconda/lib/python2.7/httplib.pyc in _read_status(self)
    363     def _read_status(self):
    364         # Initialize with Simple-Response defaults
--> 365         line = self.fp.readline(_MAXLINE + 1)
    366         if len(line) > _MAXLINE:
    367             raise LineTooLong("header line")

/home/usgs/anaconda/lib/python2.7/socket.pyc in readline(self, size)
    474             while True:
    475                 try:
--> 476                     data = self._sock.recv(self._rbufsize)
    477                 except error, e:
    478                     if e.args[0] == EINTR:

timeout: timed out

In [19]:
#pdb.set_trace()
start = '2011-08-26T16:00:00Z'
stop = '2011-08-26T17:00:00Z'
response = usgs.get_observation(offerings=['9141wh-a'],
                                 responseFormat='text/xml;schema="om/1.0.0"',
                                 observedProperties=['u_1205'],
                                  eventTime='%s/%s' % (start,stop))


---------------------------------------------------------------------------
timeout                                   Traceback (most recent call last)
<ipython-input-19-c3c5f3c96733> in <module>()
      5                                  responseFormat='text/xml;schema="om/1.0.0"',
      6                                  observedProperties=['u_1205'],
----> 7                                   eventTime='%s/%s' % (start,stop))

/home/usgs/anaconda/lib/python2.7/site-packages/OWSLib-0.8.7-py2.7.egg/owslib/swe/observation/sos100.py in get_observation(self, responseFormat, offerings, observedProperties, eventTime, method, **kwargs)
    163         data = urlencode(request)
    164 
--> 165         response = openURL(base_url, data, method, username=self.username, password=self.password).read()
    166         try:
    167             tr = etree.fromstring(response)

/home/usgs/anaconda/lib/python2.7/site-packages/OWSLib-0.8.7-py2.7.egg/owslib/util.py in openURL(url_base, data, method, cookies, username, password, timeout)
    155         if cookies is not None:
    156             req.add_header('Cookie', cookies)
--> 157         u = openit(req, timeout=timeout)
    158     except HTTPError, e: #Some servers may set the http header to 400 if returning an OGC service exception or 401 if unauthorised.
    159         if e.code in [400, 401]:

/home/usgs/anaconda/lib/python2.7/urllib2.pyc in urlopen(url, data, timeout)
    125     if _opener is None:
    126         _opener = build_opener()
--> 127     return _opener.open(url, data, timeout)
    128 
    129 def install_opener(opener):

/home/usgs/anaconda/lib/python2.7/urllib2.pyc in open(self, fullurl, data, timeout)
    402             req = meth(req)
    403 
--> 404         response = self._open(req, data)
    405 
    406         # post-process response

/home/usgs/anaconda/lib/python2.7/urllib2.pyc in _open(self, req, data)
    420         protocol = req.get_type()
    421         result = self._call_chain(self.handle_open, protocol, protocol +
--> 422                                   '_open', req)
    423         if result:
    424             return result

/home/usgs/anaconda/lib/python2.7/urllib2.pyc in _call_chain(self, chain, kind, meth_name, *args)
    380             func = getattr(handler, meth_name)
    381 
--> 382             result = func(*args)
    383             if result is not None:
    384                 return result

/home/usgs/anaconda/lib/python2.7/urllib2.pyc in http_open(self, req)
   1212 
   1213     def http_open(self, req):
-> 1214         return self.do_open(httplib.HTTPConnection, req)
   1215 
   1216     http_request = AbstractHTTPHandler.do_request_

/home/usgs/anaconda/lib/python2.7/urllib2.pyc in do_open(self, http_class, req)
   1185         else:
   1186             try:
-> 1187                 r = h.getresponse(buffering=True)
   1188             except TypeError: # buffering kw not supported
   1189                 r = h.getresponse()

/home/usgs/anaconda/lib/python2.7/httplib.pyc in getresponse(self, buffering)
   1065         response = self.response_class(*args, **kwds)
   1066 
-> 1067         response.begin()
   1068         assert response.will_close != _UNKNOWN
   1069         self.__state = _CS_IDLE

/home/usgs/anaconda/lib/python2.7/httplib.pyc in begin(self)
    407         # read until we get a non-100 response
    408         while True:
--> 409             version, status, reason = self._read_status()
    410             if status != CONTINUE:
    411                 break

/home/usgs/anaconda/lib/python2.7/httplib.pyc in _read_status(self)
    363     def _read_status(self):
    364         # Initialize with Simple-Response defaults
--> 365         line = self.fp.readline(_MAXLINE + 1)
    366         if len(line) > _MAXLINE:
    367             raise LineTooLong("header line")

/home/usgs/anaconda/lib/python2.7/socket.pyc in readline(self, size)
    474             while True:
    475                 try:
--> 476                     data = self._sock.recv(self._rbufsize)
    477                 except error, e:
    478                     if e.args[0] == EINTR:

timeout: timed out

In [ ]:
print response[0:4000]

In [10]:
# usgs woods hole ADCP data
# url='http://geoport-dev.whoi.edu/thredds/sos/usgs/data2/notebook/9111aqd-a.nc'
# adcp = SensorObservationService(url)

In [11]:
def parse_om_xml(response):
    # extract data and time from OM-XML response
    root = etree.fromstring(response)
    # root.findall(".//{%(om)s}Observation" % root.nsmap )
    values = root.find(".//{%(swe)s}values" % root.nsmap )
    date_value = array( [ (dt.datetime.strptime(d,"%Y-%m-%dT%H:%M:%SZ"),float(v))
            for d,v in [l.split(',') for l in values.text.split()]] )
    time = date_value[:,0]
    data = date_value[:,1]
    return data,time

In [12]:
data, time = parse_om_xml(response)

In [13]:
ts = pd.Series(data,index=time)

In [14]:
ts.plot(figsize(12,4));



In [15]:
''' this doesn't work
# Try adding a time range to the getobs request:

start = '1976-12-302T00:00:00Z'
stop = '1977-01-07T00:00:00Z'

response = usgs.get_observation(offerings=['1211-A1H'],
                                 responseFormat='text/xml;subtype="om/1.0.0"',
                                 observedProperties=['u_1205'],
                                 procedure='urn:ioos:station:gov.usgs:1211-A1H',
                                 eventTime='%s/%s' % (start,stop)
''';

Try adding a subset on time:


In [16]:
start = '1977-01-01T00:00:00Z'
stop = '1977-01-04T00:00:00Z'
response = usgs.get_observation(offerings=['1211-A1H'],
                                 responseFormat='text/xml;schema="om/1.0.0"',
                                 observedProperties=['u_1205'],
                                 procedure='urn:ioos:station:gov.usgs:1211-A1H',
                                 eventTime='%s/%s' % (start,stop))

In [17]:
data, time = parse_om_xml(response)
ts = pd.Series(data,index=time)
ts.plot(figsize(12,4));