Use CSW to find model data at NODC, NGDC, and CATALOG.DATA.GOV



In [16]:

    
from owslib.csw import CatalogueServiceWeb
from owslib import fes
import netCDF4



In [17]:

    
def service_urls(records,service_string='urn:x-esri:specification:ServiceType:odp:url'):
    """
    Get all URLs matching a specific ServiceType 
 
    Unfortunately these seem to differ between different CSW-ISO services.
    For example, OpenDAP is specified:
    NODC geoportal: 'urn:x-esri:specification:ServiceType:OPeNDAP'
    NGDC geoportal: 'urn:x-esri:specification:ServiceType:odp:url'
    """

    urls=[]
    for key,rec in records.iteritems():
        #create a generator object, and iterate through it until the match is found
        #if not found, gets the default value (here "none")
        url = next((d['url'] for d in rec.references if d['scheme'] == service_string), None)
        if url is not None:
            urls.append(url)
    return urls



In [23]:

    
filt=[]
search_text = ['roms','selfe','adcirc','ncom','hycom','fvcom']
for val in search_text:
    filt.append(fes.PropertyIsLike(propertyname='apiso:AnyText', 
                                   literal=('*%s*' % val),escapeChar='\\',wildCard='*',singleChar='?'))

val='sea_water_temperature'
filt.append(fes.PropertyIsLike(propertyname='apiso:AnyText', 
                                   literal=('*%s*' % val),escapeChar='\\',wildCard='*',singleChar='?'))

Find model results at NODC



In [24]:

    
endpoint = 'http://www.nodc.noaa.gov/geoportal/csw'   # NODC/UAF Geoportal: granule level
csw = CatalogueServiceWeb(endpoint,timeout=60)
print csw.version



In [25]:

    
for oper in csw.operations:
    if oper.name == 'GetRecords':
        print '\nISO Queryables:\n',oper.constraints['SupportedISOQueryables']['values']









    



ISO Queryables:
['apiso:Subject', 'apiso:Title', 'apiso:Abstract', 'apiso:AnyText', 'apiso:Format', 'apiso:Identifier', 'apiso:Modified', 'apiso:Type', 'apiso:BoundingBox', 'apiso:CRS.Authority', 'apiso:CRS.ID', 'apiso:CRS.Version', 'apiso:RevisionDate', 'apiso:AlternateTitle', 'apiso:CreationDate', 'apiso:PublicationDate', 'apiso:OrganizationName', 'apiso:HasSecurityConstraints', 'apiso:Language', 'apiso:ResourceIdentifier', 'apiso:ParentIdentifier', 'apiso:KeywordType', 'apiso:TopicCategory', 'apiso:ResourceLanguage', 'apiso:GeographicDescriptionCode', 'apiso:Denominator', 'apiso:DistanceValue', 'apiso:DistanceUOM', 'apiso:TempExtent_begin', 'apiso:TempExtent_end', 'apiso:ServiceType', 'apiso:ServiceTypeVersion', 'apiso:Operation', 'apiso:OperatesOn', 'apiso:OperatesOnIdentifier', 'apiso:OperatesOnName', 'apiso:CouplingType']



In [26]:

    
csw.getrecords2(constraints=filt,maxrecords=1000,esn='full')
len(csw.records.keys())









    Out[26]:





331



In [27]:

    
dap_urls = service_urls(csw.records,service_string='urn:x-esri:specification:ServiceType:OPeNDAP')
len(dap_urls)









    Out[27]:





331



In [ ]:

    
bad_urls=[]
for url in dap_urls:
    try:
        nc = netCDF4.Dataset(url)
        #print url
        nc.close()
    except:
        bad_urls.append(url)
            

print 'NODC: bad urls:',len(bad_urls),'/',len(dap_urls)
print 'First five bad URLS:',"\n".join(bad_urls[0:5])

Find model results at NGDC



In [ ]:

    
endpoint = 'http://www.ngdc.noaa.gov/geoportal/csw' #  NGDC/IOOS Geoportal
csw = CatalogueServiceWeb(endpoint,timeout=60)
csw.version



In [ ]:

    
for oper in csw.operations:
    if oper.name == 'GetRecords':
        print '\nISO Queryables:\n',oper.constraints['SupportedISOQueryables']['values']



In [ ]:

    
csw.getrecords2(constraints=filt,maxrecords=1000,esn='full')
len(csw.records.keys())



In [ ]:

    
choice=random.choice(list(csw.records.keys()))
print choice
csw.records[choice].references



In [ ]:

    
#dap_urls = service_urls(csw.records,service_string='urn:x-esri:specification:ServiceType:OPeNDAP')
dap_urls = service_urls(csw.records,service_string='urn:x-esri:specification:ServiceType:odp:url')
len(dap_urls)



In [ ]:

    
bad_urls=[]
for url in dap_urls:
    try:
        nc = netCDF4.Dataset(url)
        #print url
        nc.close()
    except:
        bad_urls.append(url)
print 'NGDC: bad urls:',len(bad_urls),'/',len(dap_urls)
print 'First five bad URLS:',"\n".join(bad_urls[0:5])

Find model data at CATALOG.DATA.GOV



In [ ]:

    
endpoint = 'http://uat-catalog-fe-data.reisys.com/csw-all' #  catalog.data.gov CSW
csw = CatalogueServiceWeb(endpoint,timeout=60)
csw.version



In [ ]:

    
for oper in csw.operations:
    if oper.name == 'GetRecords':
        print '\nISO Queryables:\n',oper.constraints['SupportedISOQueryables']['values']



In [ ]:

    
csw.getrecords2(constraints=filt,maxrecords=1000,esn='full')
len(csw.records.keys())



In [ ]:

    
choice=random.choice(list(csw.records.keys()))
print choice
csw.records[choice].references

From the above, we can see that because the 'scheme' is 'None' on all the references, we can't extract the different service types, like OPeNDAP, WCS, etc.



In [ ]:

    
#dap_urls = service_urls(csw.records,service_string='urn:x-esri:specification:ServiceType:odp:url')  #NGDC
#dap_urls = service_urls(csw.records,service_string='urn:x-esri:specification:ServiceType:OPeNDAP')  #NODC
dap_urls = service_urls(csw.records,service_string='?????????')    #CATALOG.DATA.GOV
len(dap_urls)



In [ ]: