Crawl a THREDDS catalog, and try loading all OPeNDAP urls with pyugrid and Iris
In [1]:
import matplotlib.tri as tri
import datetime as dt
In [2]:
import cartopy.crs as ccrs
import iris
iris.FUTURE.netcdf_promote = True
import pyugrid
In [3]:
from thredds_crawler.crawl import Crawl
In [4]:
catalog='http://comt.sura.org/thredds/comt_2_current.xml'
catalog='http://comt.sura.org/thredds/comt_1_archive_summary.xml'
#c = Crawl(catalog, select=[".*-Agg"])
c = Crawl(catalog)
In [5]:
name_list=['water_surface_height_above_reference_datum',
'sea_surface_height_above_geoid','sea_surface_elevation',
'sea_surface_height_above_reference_ellipsoid','sea_surface_height_above_sea_level',
'sea_surface_height','water level']
In [6]:
def name_in_list(cube):
return cube.standard_name in name_list
In [7]:
name_constraint = iris.Constraint(cube_func=name_in_list)
In [8]:
def my_test(cube):
return cube.standard_name in name_list & cube.cell_methods[0].method=='point'
In [9]:
def my_func(cube):
b = False
if cube.standard_name in name_list:
if hasattr(cube,'cell_methods'):
b = not any(m.method == 'maximum' for m in cube.cell_methods)
else:
b = True
return b
In [10]:
my_constraint = iris.Constraint(cube_func=my_func)
In [11]:
dap_urls = [s.get("url") for d in c.datasets for s in d.services if s.get("service").lower() == "opendap"]
In [12]:
ugrid=[]
not_ugrid=[]
cf=[]
not_cf=[]
for url in dap_urls:
try:
ug = pyugrid.UGrid.from_ncfile(url)
ugrid.append(url)
except:
not_ugrid.append(url)
try:
cube = iris.load_cube(url,my_constraint)
cf.append(url)
except:
not_cf.append(url)
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'swan_TMM10' invalid units 'metric'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'swan_TMM10_max' invalid units 'metric'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'radstress_y' invalid units 'metric'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'radstress_x' invalid units 'metric'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'windy' invalid units 'metric'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'windx' invalid units 'metric'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'zeta' invalid units 'metric'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'swan_TPS_max' invalid units 'metric'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'wind_max' invalid units 'metric'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'zeta_max' invalid units 'metric'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'nbdv' invalid units 'nondimensional'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'vel_max' invalid units 'metric'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'swan_HS' invalid units 'metric'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'nvell' invalid units 'nondimensional'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'swan_TPS' invalid units 'metric'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'nvdll' invalid units 'nondimensional'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'radstress_max' invalid units 'metric'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'swan_DIR_max' invalid units 'metric'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'swan_DIR' invalid units 'metric'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'pressure' invalid units 'meters of water'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'swan_HS_max' invalid units 'metric'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'ibtype' invalid units 'nondimensional'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'nbvv' invalid units 'nondimensional'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'u-vel' invalid units 'metric'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'ibtypee' invalid units 'nondimensional'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'pressure_min' invalid units 'meters of water'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'element' invalid units 'nondimensional'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'v-vel' invalid units 'metric'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'zeta2' invalid units 'metric'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'zeta1' invalid units 'metric'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'zetad' invalid units 'metric'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/cf.py:1004: UserWarning: Ignoring variable u'siglay' referenced by variable u'v': Dimensions (u'siglay', u'node') do not span (u'time', u'siglay', u'nele')
warnings.warn(msg)
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/cf.py:1004: UserWarning: Ignoring variable u'siglay' referenced by variable u'u': Dimensions (u'siglay', u'node') do not span (u'time', u'siglay', u'nele')
warnings.warn(msg)
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'Itime2' invalid units 'msec since 00:00:00'
warnings.warn(msg.format(msg_name, msg_units))
Exception AttributeError: "'CFReader' object has no attribute '_dataset'" in <bound method CFReader.__del__ of CFReader('http://comt.sura.org/thredds/dodsC/data/comt_1_archive/inundation_tropical/USF_FVCOM/Tides_only_3D_final_run')> ignored
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'ele' invalid units 'non-dimensional'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'Cs' invalid units 'non-dimensional'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'sigma' invalid units 'non-dimensional'
warnings.warn(msg.format(msg_name, msg_units))
Exception AttributeError: "'CFReader' object has no attribute '_dataset'" in <bound method CFReader.__del__ of CFReader('http://comt.sura.org/thredds/dodsC/data/comt_1_archive/inundation_tropical/USF_FVCOM/Tides_only_3D_preliminary_run_1_fixed_roughness')> ignored
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1357: UserWarning: Failed to create 'time' dimension coordinate: The points array must be strictly monotonic.
Gracefully creating 'time' auxiliary coordinate instead.
error=e_msg))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'xc' invalid units 'metersp'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'yc' invalid units 'metersp'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/cf.py:1038: UserWarning: Ignoring formula terms variable u'h' referenced by data variable u'Cs_w' via variable u's_w': Dimensions (u'eta_rho', u'xi_rho') do not span (u's_w',)
warnings.warn(msg)
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/cf.py:1038: UserWarning: Ignoring formula terms variable u'zeta' referenced by data variable u'Cs_w' via variable u's_w': Dimensions (u'ocean_time', u'eta_rho', u'xi_rho') do not span (u's_w',)
warnings.warn(msg)
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/cf.py:1038: UserWarning: Ignoring formula terms variable u'h' referenced by data variable u'Cs_r' via variable u's_rho': Dimensions (u'eta_rho', u'xi_rho') do not span (u's_rho',)
warnings.warn(msg)
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/cf.py:1038: UserWarning: Ignoring formula terms variable u'zeta' referenced by data variable u'Cs_r' via variable u's_rho': Dimensions (u'ocean_time', u'eta_rho', u'xi_rho') do not span (u's_rho',)
warnings.warn(msg)
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1357: UserWarning: Failed to create 'ocean_time' dimension coordinate: The points array must be strictly monotonic.
Gracefully creating 'ocean_time' auxiliary coordinate instead.
error=e_msg))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/cf.py:1038: UserWarning: Ignoring formula terms variable u'h' referenced by data variable u'v' via variable u's_rho': Dimensions (u'eta_rho', u'xi_rho') do not span (u'ocean_time', u's_rho', u'eta_v', u'xi_v')
warnings.warn(msg)
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/cf.py:1038: UserWarning: Ignoring formula terms variable u'zeta' referenced by data variable u'v' via variable u's_rho': Dimensions (u'ocean_time', u'eta_rho', u'xi_rho') do not span (u'ocean_time', u's_rho', u'eta_v', u'xi_v')
warnings.warn(msg)
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/cf.py:1038: UserWarning: Ignoring formula terms variable u'h' referenced by data variable u'u' via variable u's_rho': Dimensions (u'eta_rho', u'xi_rho') do not span (u'ocean_time', u's_rho', u'eta_u', u'xi_u')
warnings.warn(msg)
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/cf.py:1038: UserWarning: Ignoring formula terms variable u'zeta' referenced by data variable u'u' via variable u's_rho': Dimensions (u'ocean_time', u'eta_rho', u'xi_rho') do not span (u'ocean_time', u's_rho', u'eta_u', u'xi_u')
warnings.warn(msg)
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'chlorophyll' invalid units 'milligrams_chlorophyll meter-3'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'phytoplankton' invalid units 'millimole_nitrogen meter-3'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'LdetritusN' invalid units 'millimole_nitrogen meter-3'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'NO3' invalid units 'millimole_N03 meter-3'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'ssflux' invalid units 'PSU meter second-1'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'zooplankton' invalid units 'millimole_nitrogen meter-3'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'NH4' invalid units 'millimole_NH4 meter-3'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'oxygen' invalid units 'millimole_oxygen meter-3'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'SdetritusN' invalid units 'millimole_nitrogen meter-3'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'sigma' invalid units 'sigma_level'
warnings.warn(msg.format(msg_name, msg_units))
/home/testbed/anaconda/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1195: UserWarning: Ignoring netCDF variable 'mask' invalid units 'nondimensional'
warnings.warn(msg.format(msg_name, msg_units))
In [13]:
len(ugrid)
Out[13]:
81
In [14]:
len(not_ugrid)
Out[14]:
32
In [15]:
len(cf)
Out[15]:
54
In [16]:
len(not_cf)
Out[16]:
59
Content source: ioos/comt_notebooks
Similar notebooks: