Use CSW to find model data at NODC, NGDC, DATA.GOV, and PACIOOS



In [1]:

    
from owslib.csw import CatalogueServiceWeb
from owslib import fes
import netCDF4

Find model results at NODC (geoportal)



In [2]:

    
endpoint = 'http://www.nodc.noaa.gov/geoportal/csw'   # NODC/UAF Geoportal: granule level
csw = CatalogueServiceWeb(endpoint,timeout=60)
print csw.version



In [3]:

    
csw.get_operation_by_name('GetRecords').constraints









    Out[3]:





[Constraint: SupportedCommonQueryables - ['Subject', 'Title', 'Abstract', 'AnyText', 'Format', 'Identifier', 'Modified', 'Type', 'BoundingBox'],
 Constraint: SupportedISOQueryables - ['apiso:Subject', 'apiso:Title', 'apiso:Abstract', 'apiso:AnyText', 'apiso:Format', 'apiso:Identifier', 'apiso:Modified', 'apiso:Type', 'apiso:BoundingBox', 'apiso:CRS.Authority', 'apiso:CRS.ID', 'apiso:CRS.Version', 'apiso:RevisionDate', 'apiso:AlternateTitle', 'apiso:CreationDate', 'apiso:PublicationDate', 'apiso:OrganizationName', 'apiso:HasSecurityConstraints', 'apiso:Language', 'apiso:ResourceIdentifier', 'apiso:ParentIdentifier', 'apiso:KeywordType', 'apiso:TopicCategory', 'apiso:ResourceLanguage', 'apiso:GeographicDescriptionCode', 'apiso:Denominator', 'apiso:DistanceValue', 'apiso:DistanceUOM', 'apiso:TempExtent_begin', 'apiso:TempExtent_end', 'apiso:ServiceType', 'apiso:ServiceTypeVersion', 'apiso:Operation', 'apiso:OperatesOn', 'apiso:OperatesOnIdentifier', 'apiso:OperatesOnName', 'apiso:CouplingType'],
 Constraint: AdditionalQueryables - ['apiso:Degree', 'apiso:AccessConstraints', 'apiso:OtherConstraints', 'apiso:Classification', 'apiso:ConditionApplyingToAccessAndUse', 'apiso:Lineage', 'apiso:ResponsiblePartyRole', 'apiso:SpecificationTitle', 'apiso:SpecificationDate', 'apiso:SpecificationDateType']]



In [4]:

    
val = 'ghrsst'
filter1 = fes.PropertyIsLike(propertyname='apiso:AnyText',literal=('*%s*' % val),
                        escapeChar='\\',wildCard='*',singleChar='?')
filter_list = [ filter1 ]



In [5]:

    
csw.getrecords2(constraints=filter_list,maxrecords=1000,esn='full')
len(csw.records.keys())









    Out[5]:





1000



In [6]:

    
choice=random.choice(list(csw.records.keys()))
print choice
csw.records[choice].references









    



JPL-L2P-MODIS_A.20130208-MODIS_A-JPL-L2P-A2013039122000.L2_LAC_GHRSST_D-v01.nc.bz2






    Out[6]:





[{'scheme': 'urn:x-esri:specification:ServiceType:ArcIMS:Metadata:Onlink',
  'url': 'http://www.nodc.noaa.gov/SatelliteData/ghrsst/'},
 {'scheme': 'urn:x-esri:specification:ServiceType:ArcIMS:Metadata:Document',
  'url': 'http://www.nodc.noaa.gov/geoportal/csw?getxml=%7BD1340852-9D8E-45ED-A55F-31352F035372%7D'},
 {'scheme': 'urn:x-esri:specification:ServiceType:TDS',
  'url': 'http://data.nodc.noaa.gov/thredds/catalog/ghrsst/L2P/MODIS_A/JPL/2013/039/catalog.html?dataset=ghrsst/L2P/MODIS_A/JPL/2013/039/20130208-MODIS_A-JPL-L2P-A2013039122000.L2_LAC_GHRSST_D-v01.nc.bz2'},
 {'scheme': 'urn:x-esri:specification:ServiceType:FTP',
  'url': 'ftp://ftp.nodc.noaa.gov/pub/data.nodc/ghrsst/L2P/MODIS_A/JPL/2013/039/20130208-MODIS_A-JPL-L2P-A2013039122000.L2_LAC_GHRSST_D-v01.nc.bz2'},
 {'scheme': 'urn:x-esri:specification:ServiceType:OPeNDAP',
  'url': 'http://data.nodc.noaa.gov/opendap/ghrsst/L2P/MODIS_A/JPL/2013/039/20130208-MODIS_A-JPL-L2P-A2013039122000.L2_LAC_GHRSST_D-v01.nc.bz2.html'},
 {'scheme': 'urn:x-esri:specification:ServiceType:Download',
  'url': 'http://data.nodc.noaa.gov/ghrsst/L2P/MODIS_A/JPL/2013/039/20130208-MODIS_A-JPL-L2P-A2013039122000.L2_LAC_GHRSST_D-v01.nc.bz2'},
 {'scheme': 'urn:x-esri:specification:ServiceType:WMS',
  'url': 'http://data.nodc.noaa.gov/thredds/wms/ghrsst/L2P/MODIS_A/JPL/2013/039/20130208-MODIS_A-JPL-L2P-A2013039122000.L2_LAC_GHRSST_D-v01.nc.bz2?service=WMS&version=1.3.0&request=GetCapabilities'},
 {'scheme': 'urn:x-esri:specification:ServiceType:WCS',
  'url': 'http://data.nodc.noaa.gov/thredds/wcs/ghrsst/L2P/MODIS_A/JPL/2013/039/20130208-MODIS_A-JPL-L2P-A2013039122000.L2_LAC_GHRSST_D-v01.nc.bz2?service=WCS&version=1.0.0&request=GetCapabilities'}]

Find model results at NGDC (geoportal)



In [8]:

    
endpoint = 'http://www.ngdc.noaa.gov/geoportal/csw' #  NGDC/IOOS Geoportal
csw = CatalogueServiceWeb(endpoint,timeout=60)
csw.version









    Out[8]:





'2.0.2'



In [9]:

    
csw.get_operation_by_name('GetRecords').constraints









    Out[9]:





[Constraint: SupportedCommonQueryables - ['Subject', 'Title', 'Abstract', 'AnyText', 'Format', 'Identifier', 'Modified', 'Type', 'BoundingBox'],
 Constraint: SupportedISOQueryables - ['apiso:Subject', 'apiso:Title', 'apiso:Abstract', 'apiso:AnyText', 'apiso:Format', 'apiso:Identifier', 'apiso:Modified', 'apiso:Type', 'apiso:BoundingBox', 'apiso:CRS.Authority', 'apiso:CRS.ID', 'apiso:CRS.Version', 'apiso:RevisionDate', 'apiso:AlternateTitle', 'apiso:CreationDate', 'apiso:PublicationDate', 'apiso:OrganizationName', 'apiso:HasSecurityConstraints', 'apiso:Language', 'apiso:ResourceIdentifier', 'apiso:ParentIdentifier', 'apiso:KeywordType', 'apiso:TopicCategory', 'apiso:ResourceLanguage', 'apiso:GeographicDescriptionCode', 'apiso:Denominator', 'apiso:DistanceValue', 'apiso:DistanceUOM', 'apiso:TempExtent_begin', 'apiso:TempExtent_end', 'apiso:ServiceType', 'apiso:ServiceTypeVersion', 'apiso:Operation', 'apiso:OperatesOn', 'apiso:OperatesOnIdentifier', 'apiso:OperatesOnName', 'apiso:CouplingType'],
 Constraint: AdditionalQueryables - ['apiso:Degree', 'apiso:AccessConstraints', 'apiso:OtherConstraints', 'apiso:Classification', 'apiso:ConditionApplyingToAccessAndUse', 'apiso:Lineage', 'apiso:ResponsiblePartyRole', 'apiso:ResponsiblePartyName', 'apiso:SpecificationTitle', 'apiso:SpecificationDate', 'apiso:SpecificationDateType']]



In [10]:

    
try:
    csw.get_operation_by_name('GetDomain')
    csw.getdomain('apiso:ServiceType', 'property')
    print(csw.results['values'])
except:
    print('GetDomain not supported')









    



GetDomain not supported



In [11]:

    
csw.getrecords2(constraints=filter_list,maxrecords=1000,esn='full')
len(csw.records.keys())









    Out[11]:





1



In [12]:

    
choice=random.choice(list(csw.records.keys()))
print choice
csw.records[choice].references









    



G1_SST_US_WEST_COAST






    Out[12]:





[{'scheme': 'urn:x-esri:specification:ServiceType:distribution:url',
  'url': 'http://thredds.axiomalaska.com/thredds/dodsC/G1_SST_US_WEST_COAST.nc.html'},
 {'scheme': 'urn:x-esri:specification:ServiceType:distribution:url',
  'url': 'http://www.ncdc.noaa.gov/oa/wct/wct-jnlp-beta.php?singlefile=http://thredds.axiomalaska.com/thredds/dodsC/G1_SST_US_WEST_COAST.nc'},
 {'scheme': 'urn:x-esri:specification:ServiceType:odp:url',
  'url': 'http://thredds.axiomalaska.com/thredds/dodsC/G1_SST_US_WEST_COAST.nc'},
 {'scheme': 'urn:x-esri:specification:ServiceType:download:url',
  'url': 'http://thredds.axiomalaska.com/thredds/dodsC/G1_SST_US_WEST_COAST.nc.html'}]

Find model data at CATALOG.DATA.GOV (pycsw)



In [ ]:

    
endpoint = 'http://catalog.data.gov/csw-all' #  catalog.data.gov CSW
#endpoint = 'http://csw.data.gov.uk/geonetwork/srv/en/csw' # data.gov.uk
#endpoint = 'http://www.nationaalgeoregister.nl/geonetwork/srv/eng/csw'
#endpoint = 'http://www.rndt.gov.it/RNDT/CSW'
csw = CatalogueServiceWeb(endpoint,timeout=60)
csw.version



In [ ]:

    
for oper in csw.operations:
    print(oper.name)



In [ ]:

    
csw.get_operation_by_name('GetRecords').constraints



In [ ]:

    
try:
    csw.get_operation_by_name('GetDomain')
    csw.getdomain('apiso:ServiceType', 'property')
    print(csw.results['values'])
except:
    print('GetDomain not supported')



In [ ]:

    
val = 'sea surface'
filter1 = fes.PropertyIsLike(propertyname='apiso:AnyText',literal=('*%s*' % val),
                        escapeChar='\\',wildCard='*',singleChar='?')
filter_list = [ filter1 ]



In [ ]:

    
csw.getrecords2(constraints=filter_list,maxrecords=1000,esn='full')
len(csw.records.keys())



In [ ]:

    
choice=random.choice(list(csw.records.keys()))
print csw.records[choice].title
print csw.records[choice].references

Search at geoport.whoi.edu (geoportal)



In [ ]:

    
endpoint = 'http://geoport.whoi.edu/geoportal/csw' #  catalog.data.gov CSW
csw = CatalogueServiceWeb(endpoint,timeout=60)
print csw.version



In [ ]:

    
for oper in csw.operations:
    if oper.name == 'GetRecords':
        print '\nISO Queryables:\n',oper.constraints['SupportedISOQueryables']['values']



In [ ]:

    
csw.getrecords2(constraints=filter_list,maxrecords=1000,esn='full')
len(csw.records.keys())



In [ ]:

    
choice=random.choice(list(csw.records.keys()))
print choice
csw.records[choice].references

Search at PACIOOS (pycsw)



In [ ]:

    
endpoint='http://oos.soest.hawaii.edu/pacioos/ogc/csw.py'



In [ ]:

    
csw = CatalogueServiceWeb(endpoint,timeout=60)
csw.version



In [ ]:

    
csw.get_operation_by_name('GetRecords').constraints



In [ ]:

    
try:
    csw.get_operation_by_name('GetDomain')
    csw.getdomain('apiso:ServiceType', 'property')
    print(csw.results['values'])
except:
    print('GetDomain not supported')



In [ ]:

    
val = 'ROMS'
filter1 = fes.PropertyIsLike(propertyname='apiso:AnyText',literal=('*%s*' % val),
                        escapeChar='\\',wildCard='*',singleChar='?')
filter_list = [ filter1 ]



In [ ]:

    
csw.getrecords2(constraints=filter_list,maxrecords=1000,esn='full')
len(csw.records.keys())



In [ ]:

    
choice=random.choice(list(csw.records.keys()))
print choice
csw.records[choice].references

Working! Woo hoo!!!

EPA



In [ ]:

    
endpoint = 'https://edg.epa.gov/metadata/csw'
csw = CatalogueServiceWeb(endpoint,timeout=60)
csw.version



In [ ]:

    
# trying to do this search:
# ('roms' OR 'selfe' OR 'adcirc' OR 'ncom' OR 'hycom' OR 'fvcom') AND 'ocean' NOT 'regridded' NOT 'espresso'
# should return 11 records from NODC geoportal

search_text = ['waves','selfe','adcirc','ncom','hycom','fvcom']
filt=[]
for val in search_text:
    filt.append(fes.PropertyIsLike(propertyname='apiso:AnyText',literal=('*%s*' % val),
                                   escapeChar='\\',wildCard='*',singleChar='?'))
filter1=fes.Or(filt)

val = 'ocean'
filter2=fes.PropertyIsLike(propertyname='apiso:AnyText',literal=('*%s*' % val),
                        escapeChar='\\',wildCard='*',singleChar='?')

val = 'regridded'
filt=fes.PropertyIsLike(propertyname='apiso:AnyText',literal=('*%s*' % val),
                        escapeChar='\\',wildCard='*',singleChar='?')
filter3 = fes.Not([filt])

val = 'espresso'
filt=fes.PropertyIsLike(propertyname='apiso:AnyText',literal=('*%s*' % val),
                        escapeChar='\\',wildCard='*',singleChar='?')
filter4 = fes.Not([filt])


filter_list = [fes.And([filter1, filter2, filter3, filter4])]



In [ ]:

    
csw.getrecords2(constraints=filter_list,maxrecords=1000,esn='full')
len(csw.records.keys())



In [ ]:

    
choice=random.choice(list(csw.records.keys()))
print choice
csw.records[choice].references



In [ ]:

    
try:
    csw.get_operation_by_name('GetDomain')
    csw.getdomain('apiso:SupportedISOQueryables', 'apiso:ServiceType')
    print(csw.results['values'])
except:
    print('GetDomain not supported')

USGS CGDMS server (geonetwork)



In [ ]:

    
endpoint = 'http://cmgds.marine.usgs.gov/geonetwork/srv/en/csw'
csw = CatalogueServiceWeb(endpoint,timeout=60)
csw.version

USGS CIDA Server (geonetwork)



In [ ]:

    
endpoint = 'http://cida.usgs.gov/gdp/geonetwork/srv/en/csw'
csw = CatalogueServiceWeb(endpoint,timeout=60)
csw.version



In [ ]:

    
foo=csw.get_operation_by_name('GetRecords')
foo.constraints



In [ ]:

    
try:
    csw.get_operation_by_name('GetDomain')
    csw.getdomain('SupportedISOQueryables', 'ServiceType')
    print(csw.results['values'])
except:
    print('GetDomain not supported')



In [ ]:

    
val = 'dap'
service_type = fes.PropertyIsLike(propertyname='ServiceType',literal=('*%s*' % val),
                        escapeChar='\\',wildCard='*',singleChar='?')
filter_list = [ service_type]



In [ ]:

    
csw.getrecords2(constraints=filter_list,maxrecords=1000,esn='full')
len(csw.records.keys())



In [ ]:

    
val = 'climate'
filter1 = fes.PropertyIsLike(propertyname='AnyText',literal=('*%s*' % val),
                        escapeChar='\\',wildCard='*',singleChar='?')
filter_list = [ filter1 ]



In [ ]:

    
csw.getrecords2(constraints=filter_list,maxrecords=1000,esn='full')
len(csw.records.keys())



In [ ]:

    
choice=random.choice(list(csw.records.keys()))
print choice
csw.records[choice].references

INSPIRE



In [ ]:

    
endpoint ='http://inspire-geoportal.ec.europa.eu/GeoportalProxyWebServices/resources/OGCCSW202/AT'
csw = CatalogueServiceWeb(endpoint,timeout=60)
csw.version



In [ ]:

    
csw.get_operation_by_name('GetRecords').constraints



In [14]:

    
try:
    csw.get_operation_by_name('GetDomain')
    csw.getdomain('apiso:ServiceType', 'property')
    print(csw.results['values'])
except:
    print('GetDomain not supported')









    



GetDomain not supported



In [15]:

    
val = 'data'
filter1 = fes.PropertyIsLike(propertyname='AnyText',literal=('*%s*' % val),
                        escapeChar='\\',wildCard='*',singleChar='?')
filter_list = [ filter1 ]



In [16]:

    
csw.getrecords2(constraints=filter_list,maxrecords=1000,esn='full')
len(csw.records.keys())









    Out[16]:





1000



In [17]:

    
choice=random.choice(list(csw.records.keys()))
print(csw.records[choice].title)
print(csw.records[choice].references)









    



gov.noaa.ngdc.mgg.multibeam:EW0114_Multibeam






    Out[17]:





[{'scheme': 'urn:x-esri:specification:ServiceType:ArcIMS:Metadata:Onlink',
  'url': 'http://www.ngdc.noaa.gov/mgg/bathymetry/multibeam.html'},
 {'scheme': 'urn:x-esri:specification:ServiceType:download:url',
  'url': 'http://www.ngdc.noaa.gov/nndc/struts/results?op_0=l&v_0=NEW557&t=101378&s=8&d=70&d=75&d=76&d=91&d=74&d=73&d=72&d=81&d=82&d=85&d=86&d=79&no_data=suppress'},
 {'scheme': 'urn:x-esri:specification:ServiceType:download:url',
  'url': 'http://www.ngdc.noaa.gov/nndc/struts/results?op_0=l&v_0=NEW557&op_1=l&v_1=&op_2=l&v_2=&t=101378&s=2&d=6'}]

Italy



In [18]:

    
endpoint = 'http://www.rndt.gov.it/RNDT/CSW'
csw = CatalogueServiceWeb(endpoint,timeout=60)
csw.version









    Out[18]:





'2.0.2'



In [19]:

    
csw.get_operation_by_name('GetRecords').constraints









    Out[19]:





[Constraint: SupportedISOQueryables - ['OrganisationName', 'Type', 'ServiceType', 'RevisionDate', 'AnyText', 'PublicationDate', 'ResourceIdentifier', 'ParentIdentifier', 'Identifier', 'TopicCategory', 'DistanceValue', 'DistanceUOM', 'TempExtent_end', 'Subject', 'CreationDate', 'Title', 'Denominator', 'Language', 'TempExtent_begin', 'Abstract'],
 Constraint: AdditionalQueryables - ['SpecificationDate', 'SpecificationDateType', 'ConditionApplyingToAccessAndUse', 'AccessConstraints', 'Classification', 'OtherConstraints', 'Degree', 'Lineage', 'SpecificationTitle', 'ResponsiblePartyRole']]



In [27]:

    
val = 'water'
filter1 = fes.PropertyIsLike(propertyname='AnyText',literal=('*%s*' % val),
                        escapeChar='\\',wildCard='*',singleChar='?')
filter_list = [ filter1 ]



In [28]:

    
csw.getrecords2(constraints=filter_list,maxrecords=100,esn='full')
len(csw.records.keys())









    Out[28]:





28



In [53]:

    
choice=random.choice(list(csw.records.keys()))
print(csw.records[choice].title)
print(csw.records[choice].references)









    



MOLISEDB.GIS.MO_PTA24G_staz_monit
[]



In [ ]: