COMT Standards Compliance Test

Crawl a THREDDS catalog, and try loading all OPeNDAP urls with pyugrid and Iris


In [1]:
import matplotlib.tri as tri
import datetime as dt

In [2]:
import cartopy.crs as ccrs
import iris
iris.FUTURE.netcdf_promote = True
import pyugrid

In [3]:
from thredds_crawler.crawl import Crawl

In [4]:
catalog='http://comt.sura.org/thredds/comt_2_current.xml'
catalog='http://comt.sura.org/thredds/comt_1_archive_summary.xml'
#c = Crawl(catalog, select=[".*-Agg"])
c = Crawl(catalog)

In [5]:
name_list=['water_surface_height_above_reference_datum',
    'sea_surface_height_above_geoid','sea_surface_elevation',
    'sea_surface_height_above_reference_ellipsoid','sea_surface_height_above_sea_level',
    'sea_surface_height','water level']

In [6]:
def name_in_list(cube):
    return cube.standard_name in name_list

In [7]:
name_constraint = iris.Constraint(cube_func=name_in_list)

In [8]:
def my_test(cube):
    return cube.standard_name in name_list & cube.cell_methods[0].method=='point'

In [9]:
def my_func(cube):
    b = False
    if cube.standard_name in name_list:
        if hasattr(cube,'cell_methods'):
            b = not any(m.method == 'maximum' for m in cube.cell_methods)
        else:
            b = True
    return b

In [10]:
my_constraint = iris.Constraint(cube_func=my_func)

In [11]:
dap_urls = [s.get("url") for d in c.datasets for s in d.services if s.get("service").lower() == "opendap"]

In [12]:
ugrid=[]
not_ugrid=[]
cf=[]
not_cf=[]
for url in dap_urls:
    try:
        ug = pyugrid.UGrid.from_ncfile(url)
        ugrid.append(url)
    except:
        not_ugrid.append(url)

    try:
        cube = iris.load_cube(url,my_constraint)
        cf.append(url)
    except:
        not_cf.append(url)


/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'swan_TMM10' invalid units 'metric'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'swan_TMM10_max' invalid units 'metric'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'radstress_y' invalid units 'metric'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'radstress_x' invalid units 'metric'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'windy' invalid units 'metric'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'windx' invalid units 'metric'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'zeta' invalid units 'metric'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'swan_TPS_max' invalid units 'metric'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'wind_max' invalid units 'metric'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'zeta_max' invalid units 'metric'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'nbdv' invalid units 'nondimensional'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'vel_max' invalid units 'metric'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'swan_HS' invalid units 'metric'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'nvell' invalid units 'nondimensional'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'swan_TPS' invalid units 'metric'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'nvdll' invalid units 'nondimensional'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'radstress_max' invalid units 'metric'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'swan_DIR_max' invalid units 'metric'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'swan_DIR' invalid units 'metric'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'pressure' invalid units 'meters of water'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'swan_HS_max' invalid units 'metric'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'ibtype' invalid units 'nondimensional'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'nbvv' invalid units 'nondimensional'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'u-vel' invalid units 'metric'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'ibtypee' invalid units 'nondimensional'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'pressure_min' invalid units 'meters of water'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'element' invalid units 'nondimensional'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'v-vel' invalid units 'metric'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'zeta2' invalid units 'metric'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'zeta1' invalid units 'metric'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'zetad' invalid units 'metric'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/cf.py:1004: UserWarning: Ignoring variable u'siglay' referenced by variable u'v': Dimensions (u'siglay', u'node') do not span (u'time', u'siglay', u'nele')
  warnings.warn(msg)
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/cf.py:1004: UserWarning: Ignoring variable u'siglay' referenced by variable u'u': Dimensions (u'siglay', u'node') do not span (u'time', u'siglay', u'nele')
  warnings.warn(msg)
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'Itime2' invalid units 'msec since 00:00:00'
  warnings.warn(msg.format(msg_name, msg_units))
Exception AttributeError: "'CFReader' object has no attribute '_dataset'" in <bound method CFReader.__del__ of CFReader('http://comt.sura.org/thredds/dodsC/data/comt_1_archive/inundation_tropical/USF_FVCOM/Tides_only_3D_final_run')> ignored
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'ele' invalid units 'non-dimensional'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'Cs' invalid units 'non-dimensional'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'sigma' invalid units 'non-dimensional'
  warnings.warn(msg.format(msg_name, msg_units))
Exception AttributeError: "'CFReader' object has no attribute '_dataset'" in <bound method CFReader.__del__ of CFReader('http://comt.sura.org/thredds/dodsC/data/comt_1_archive/inundation_tropical/USF_FVCOM/Tides_only_3D_preliminary_run_1_fixed_roughness')> ignored
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1357: UserWarning: Failed to create 'time' dimension coordinate: The points array must be strictly monotonic.
Gracefully creating 'time' auxiliary coordinate instead.
  error=e_msg))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'xc' invalid units 'metersp'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'yc' invalid units 'metersp'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/cf.py:1038: UserWarning: Ignoring formula terms variable u'h' referenced by data variable u'Cs_w' via variable u's_w': Dimensions (u'eta_rho', u'xi_rho') do not span (u's_w',)
  warnings.warn(msg)
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/cf.py:1038: UserWarning: Ignoring formula terms variable u'zeta' referenced by data variable u'Cs_w' via variable u's_w': Dimensions (u'ocean_time', u'eta_rho', u'xi_rho') do not span (u's_w',)
  warnings.warn(msg)
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/cf.py:1038: UserWarning: Ignoring formula terms variable u'h' referenced by data variable u'Cs_r' via variable u's_rho': Dimensions (u'eta_rho', u'xi_rho') do not span (u's_rho',)
  warnings.warn(msg)
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/cf.py:1038: UserWarning: Ignoring formula terms variable u'zeta' referenced by data variable u'Cs_r' via variable u's_rho': Dimensions (u'ocean_time', u'eta_rho', u'xi_rho') do not span (u's_rho',)
  warnings.warn(msg)
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1357: UserWarning: Failed to create 'ocean_time' dimension coordinate: The points array must be strictly monotonic.
Gracefully creating 'ocean_time' auxiliary coordinate instead.
  error=e_msg))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/cf.py:1038: UserWarning: Ignoring formula terms variable u'h' referenced by data variable u'v' via variable u's_rho': Dimensions (u'eta_rho', u'xi_rho') do not span (u'ocean_time', u's_rho', u'eta_v', u'xi_v')
  warnings.warn(msg)
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/cf.py:1038: UserWarning: Ignoring formula terms variable u'zeta' referenced by data variable u'v' via variable u's_rho': Dimensions (u'ocean_time', u'eta_rho', u'xi_rho') do not span (u'ocean_time', u's_rho', u'eta_v', u'xi_v')
  warnings.warn(msg)
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/cf.py:1038: UserWarning: Ignoring formula terms variable u'h' referenced by data variable u'u' via variable u's_rho': Dimensions (u'eta_rho', u'xi_rho') do not span (u'ocean_time', u's_rho', u'eta_u', u'xi_u')
  warnings.warn(msg)
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/cf.py:1038: UserWarning: Ignoring formula terms variable u'zeta' referenced by data variable u'u' via variable u's_rho': Dimensions (u'ocean_time', u'eta_rho', u'xi_rho') do not span (u'ocean_time', u's_rho', u'eta_u', u'xi_u')
  warnings.warn(msg)
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'chlorophyll' invalid units 'milligrams_chlorophyll meter-3'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'phytoplankton' invalid units 'millimole_nitrogen meter-3'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'LdetritusN' invalid units 'millimole_nitrogen meter-3'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'NO3' invalid units 'millimole_N03 meter-3'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'ssflux' invalid units 'PSU meter second-1'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'zooplankton' invalid units 'millimole_nitrogen meter-3'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'NH4' invalid units 'millimole_NH4 meter-3'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'oxygen' invalid units 'millimole_oxygen meter-3'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'SdetritusN' invalid units 'millimole_nitrogen meter-3'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'sigma' invalid units 'sigma_level'
  warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'mask' invalid units 'nondimensional'
  warnings.warn(msg.format(msg_name, msg_units))

In [13]:
len(ugrid)


Out[13]:
81

In [14]:
len(not_ugrid)


Out[14]:
32

In [15]:
len(cf)


Out[15]:
54

In [16]:
len(not_cf)


Out[16]:
59