CMRE search, access and visualize demo

Search for data using OGC Catalog Service for the Web (CSW)


In [1]:
from owslib.csw import CatalogueServiceWeb
from owslib import fes
import netCDF4
import numpy as np

In [2]:
endpoint='http://scsrv26v:8000/pycsw'
#endpoint='http://www.ngdc.noaa.gov/geoportal/csw'
csw = CatalogueServiceWeb(endpoint,timeout=60)
csw.version


Out[2]:
'2.0.2'

In [3]:
csw.get_operation_by_name('GetRecords').constraints


Out[3]:
[Constraint: SupportedISOQueryables - ['apiso:DistanceValue', 'apiso:Abstract', 'apiso:RevisionDate', 'apiso:Subject', 'apiso:KeywordType', 'apiso:Title', 'apiso:CRS', 'apiso:PublicationDate', 'apiso:Type', 'apiso:AlternateTitle', 'apiso:BoundingBox', 'apiso:AnyText', 'apiso:ParentIdentifier', 'apiso:Modified', 'apiso:Operation', 'apiso:Format', 'apiso:TempExtent_end', 'apiso:DistanceUOM', 'apiso:OrganisationName', 'apiso:ServiceType', 'apiso:TempExtent_begin', 'apiso:ResourceLanguage', 'apiso:ServiceTypeVersion', 'apiso:OperatesOn', 'apiso:Denominator', 'apiso:HasSecurityConstraints', 'apiso:OperatesOnIdentifier', 'apiso:GeographicDescriptionCode', 'apiso:Language', 'apiso:Identifier', 'apiso:OperatesOnName', 'apiso:TopicCategory', 'apiso:CreationDate', 'apiso:CouplingType'],
 Constraint: AdditionalQueryables - ['apiso:Lineage', 'apiso:Classification', 'apiso:Creator', 'apiso:Relation', 'apiso:OtherConstraints', 'apiso:SpecificationTitle', 'apiso:ResponsiblePartyRole', 'apiso:SpecificationDateType', 'apiso:Degree', 'apiso:Contributor', 'apiso:ConditionApplyingToAccessAndUse', 'apiso:SpecificationDate', 'apiso:AccessConstraints', 'apiso:Publisher'],
 Constraint: SupportedDublinCoreQueryables - ['dc:contributor', 'dc:source', 'dc:language', 'dc:title', 'dc:subject', 'dc:creator', 'dc:type', 'ows:BoundingBox', 'dct:modified', 'dct:abstract', 'dc:relation', 'dc:date', 'dc:identifier', 'dc:publisher', 'dc:format', 'csw:AnyText', 'dc:rights']]

In [4]:
def dateRange(start_date='1900-01-01',stop_date='2100-01-01',constraint='overlaps'):
    if constraint == 'overlaps':
        start = fes.PropertyIsLessThanOrEqualTo(propertyname='apiso:TempExtent_begin', literal=stop_date)
        stop = fes.PropertyIsGreaterThanOrEqualTo(propertyname='apiso:TempExtent_end', literal=start_date)
    elif constraint == 'within':
        start = fes.PropertyIsGreaterThanOrEqualTo(propertyname='apiso:TempExtent_begin', literal=start_date)
        stop = fes.PropertyIsLessThanOrEqualTo(propertyname='apiso:TempExtent_end', literal=stop_date)
    return start,stop

In [5]:
box=[38., 6., 41., 9.]     #  lon_min lat_min lon_max lat_max
start_date='2014-03-12 18:00'
stop_date='2014-09-18 18:00'
val = 'sea_water_potential_temperature'

In [6]:
# convert User Input into FES filters
start,stop = dateRange(start_date,stop_date)
bbox = fes.BBox(box)
any_text = fes.PropertyIsLike(propertyname='apiso:AnyText',literal=('*%s*' % val),
                        escapeChar='\\',wildCard='*',singleChar='?')

In [7]:
# combine filters into a list
filter_list = [fes.And([ start, stop, bbox,any_text]) ]

In [8]:
csw.getrecords2(constraints=filter_list,maxrecords=100,esn='full')
len(csw.records.keys())


Out[8]:
9

In [9]:
choice=np.random.choice(list(csw.records.keys()))
print(csw.records[choice].title)
csw.records[choice].references


REP14:Model:Mercator
Out[9]:
[{'scheme': 'WWW:LINK',
  'url': 'http://scsrv26v:8080/thredds/dodsC/mercator/fmrc/mercator_best.ncd.html'},
 {'scheme': 'WWW:LINK',
  'url': 'http://www.ncdc.noaa.gov/oa/wct/wct-jnlp-beta.php?singlefile=http://scsrv26v:8080/thredds/dodsC/mercator/fmrc/mercator_best.ncd'},
 {'scheme': 'OPeNDAP:OPeNDAP',
  'url': 'http://scsrv26v:8080/thredds/dodsC/mercator/fmrc/mercator_best.ncd'},
 {'scheme': 'OGC:WMS',
  'url': 'http://scsrv26v:8080/thredds/wms/mercator/fmrc/mercator_best.ncd?service=WMS&version=1.3.0&request=GetCapabilities'},
 {'scheme': 'UNIDATA:NCSS',
  'url': 'http://scsrv26v:8080/thredds/ncss/grid/mercator/fmrc/mercator_best.ncd/dataset.html'}]

In [10]:
# get specific ServiceType URL from records
def service_urls(records,service_string='OPeNDAP:OPeNDAP'):
    urls=[]
    for key,rec in records.iteritems():
        #create a generator object, and iterate through it until the match is found
        #if not found, gets the default value (here "none")
        url = next((d['url'] for d in rec.references if d['scheme'] == service_string), None)
        if url is not None:
            urls.append(url)
    return urls

In [11]:
#scheme='urn:x-esri:specification:ServiceType:odp:url'
scheme='OPeNDAP:OPeNDAP'
urls = service_urls(csw.records,service_string=scheme)
print "\n".join(urls)


http://scsrv26v:8080/thredds/dodsC/cmre_roms/fmrc/cmre_roms_best.ncd
http://scsrv26v:8080/thredds/dodsC/gliders/GL-20140608-zoe-MEDREP14depl001-grid-D.nc.ncml
http://scsrv26v:8080/thredds/dodsC/gliders/GL-20140609-noa-MEDREP14depl001-grid-D.nc.ncml
http://scsrv26v:8080/thredds/dodsC/socib_roms/fmrc/socib_roms_best.ncd
http://scsrv26v:8080/thredds/dodsC/cmre_roms_regular/fmrc/cmre_roms_regular_best.ncd
http://scsrv26v:8080/thredds/dodsC/mfs/fmrc/mfs_best.ncd
http://scsrv26v:8080/thredds/dodsC/mercator/fmrc/mercator_best.ncd
http://scsrv26v:8080/thredds/dodsC/nrl/fmrc/nrl_best.ncd
http://thredds.socib.es/thredds/dodsC/operational_models/oceanographical/hydrodynamics/model_run_aggregation/wmopv2/wmopv2_best.ncd

Use Iris to access CF data


In [12]:
import iris
import iris.plot as iplt
import iris.quickplot as qplt
import cartopy.crs as ccrs
import matplotlib.pyplot as plt
%matplotlib inline

In [13]:
cube = iris.load_cube(urls[7],'sea_water_potential_temperature')


/home/signell/anaconda/envs/ioos/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1291: UserWarning: Gracefully filling 'time' dimension coordinate masked points
  warnings.warn(msg.format(str(cf_coord_var.cf_name)))
/home/signell/anaconda/envs/ioos/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1196: UserWarning: Ignoring netCDF variable 'surf_salt_flux' invalid units 'psu-m/s'
  warnings.warn(msg.format(msg_name, msg_units))
/home/signell/anaconda/envs/ioos/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1196: UserWarning: Ignoring netCDF variable 'salinity' invalid units 'psu'
  warnings.warn(msg.format(msg_name, msg_units))

In [14]:
print cube


sea_water_potential_temperature / (degC) (time: 131; depth: 31; latitude: 108; longitude: 83)
     Dimension coordinates:
          time                                x           -             -               -
          depth                               -           x             -               -
          latitude                            -           -             x               -
          longitude                           -           -             -               x
     Auxiliary coordinates:
          forecast_reference_time             x           -             -               -
     Attributes:
          Conventions: CF-1.4, _Coordinates
          NAVO_code: 15
          NCO: 4.3.7
          _CoordSysBuilder: ucar.nc2.dataset.conv.CF1Convention
          cdm_data_type: GRID
          classification_authority: not applicable
          classification_level: not applicable
          contact: NRL Code 7323
          detide_tide_file: /u/realt3/relo/med_ens/run/med_3km/input2d/otdet_1.D
          distribution_statement: Distribution limited to Department of Defense and U.S. DOD contractors...
          downgrade_date: not applicable
          featureType: GRID
          generating_model: NCOM NOKI
          history: Fri Nov  7 08:02:17 2014: ncrcat med_3km_2014062000_t0000.nc med_3km_2014062000_t0024.nc...
          initial_time: 2000010100
          input_data_source: not_applicable
          institution: Naval Research Laboratory
          location: Proto fmrc:nrl
          model_type: NCOM_2.3
          nco_openmp_thread_number: 1
          operational_status: development
          summary: insert blurb from data report here. 
          time_origin: 2014-06-20 00:00:00
          title: REP14:Model:NRL long-range forecast

Make a nice plot using Cartopy (better than Basemap)


In [15]:
from matplotlib import rcParams
from matplotlib.ticker import MultipleLocator
rcParams['xtick.direction'] = 'out'
rcParams['ytick.direction'] = 'out'

fig=plt.figure(figsize=(12,8))

# set the projection
ax1 = plt.axes(projection=ccrs.Mercator())

# color filled contour plot
h = iplt.contourf(cube[1,0,:,:],64)

from cartopy.mpl.gridliner import LONGITUDE_FORMATTER, LATITUDE_FORMATTER
gl=ax1.gridlines(crs=ccrs.PlateCarree(),draw_labels=True)
gl.xlabels_top = False
gl.ylabels_right = False 
gl.xformatter = LONGITUDE_FORMATTER
gl.yformatter = LATITUDE_FORMATTER
# add coastlines, colorbar and title
plt.gca().coastlines(resolution='10m')
plt.colorbar(h,orientation='vertical');
plt.title(cube.attributes['title']);



In [15]: