Using Iris to access data from US-IOOS models


In [1]:
from IPython.core.display import HTML
HTML('<iframe src=http://scitools.org.uk/iris/ width=800 height=350></iframe>')


Out[1]:

In [2]:
import datetime as dt
import time

import iris

In [3]:
def time_near(cube,start):
    #    coord_names = [coord.name() for coord in cube.coords()]
    #    timevar = cube.coord(coord_names[0]))
    timevar=cube.coord('time')
    try:
        itime = timevar.nearest_neighbour_index(timevar.units.date2num(start))
    except:
        itime = -1
    return timevar.points[itime]

In [4]:
def var_lev_date(url=None,var=None,mytime=None,lev=0,subsample=1):
    time0=time.time()
    cubes = iris.load(url)
    cube = cubes.extract(iris.Constraint(name=var.strip()))[0]
    try:
        cube.coord(axis='T').rename('time')
    except:
        pass
    slice = cube.extract(iris.Constraint(time=time_near(cube,mytime)))
    slice = slice[lev,::subsample,::subsample]  
    print 'slice retrieved in %f seconds' % (time.time()-time0)
    return slice

In [5]:
def myplot(slice,model=None):
    # make the plot
    figure(figsize=(12,8))
    lat=slice.coord(axis='Y').points
    lon=slice.coord(axis='X').points
    time=slice.coord('time')[0]
    subplot(111,aspect=(1.0/cos(mean(lat)*pi/180.0)))
    lon = mod(lon,360.)-360.
    pcolormesh(lon,lat,ma.masked_invalid(slice.data),vmin=5.,vmax=30.);
    axis([-78., -68., 34., 42. ]);
    colorbar()
    grid()
    date=time.units.num2date(time.points)
    date_str=date[0].strftime('%Y-%m-%d %H:%M:%S %Z')
    plt.title('%s: %s: %s' % (model,slice.long_name,date_str));

In [6]:
# use contraints to select nearest time
#mytime=dt.datetime(2008,7,28,12)  #specified time...
mytime=dt.datetime.utcnow()      # .... or now

In [7]:
model='MARACOOS/ESPRESSO'
url='http://tds.marine.rutgers.edu/thredds/dodsC/roms/espresso/2009_da/his'
var='potential temperature'
lev=0
slice=var_lev_date(url=url,var=var, mytime=mytime, lev=lev)
myplot(slice,model=model)


slice retrieved in 36.458963 seconds

In [8]:
model='USGS/COAWST'
url='http://geoport.whoi.edu/thredds/dodsC/coawst_4/use/fmrc/coawst_4_use_best.ncd'
var='potential temperature'
lev=0
slice=var_lev_date(url=url,var=var, mytime=mytime, lev=lev, subsample=1)
myplot(slice,model=model)


slice retrieved in 382.020732 seconds

In [10]:
model='MARACOOS/NYHOPS'
url='http://colossus.dl.stevens-tech.edu/thredds/dodsC/fmrc/NYBight/NYHOPS_Forecast_Collection_for_the_New_York_Bight_best.ncd'
var='Water Temperature'
lev=-1
slice=var_lev_date(url=url,var=var, mytime=mytime, lev=lev)
myplot(slice,model=model)


---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-10-1e3dff0de36e> in <module>()
      3 var='Water Temperature'
      4 lev=-1
----> 5 slice=var_lev_date(url=url,var=var, mytime=mytime, lev=lev)
      6 myplot(slice,model=model)

<ipython-input-4-d9023d1bf69c> in var_lev_date(url, var, mytime, lev, subsample)
      1 def var_lev_date(url=None,var=None,mytime=None,lev=0,subsample=1):
      2     time0=time.time()
----> 3     cubes = iris.load(url)
      4     cube = cubes.extract(iris.Constraint(name=var.strip()))[0]
      5     try:

/home/local/python27_epd/lib/python2.7/site-packages/Iris-1.5.0_dev-py2.7.egg/iris/__init__.pyc in load(uris, constraints, callback)
    196 
    197     """
--> 198     return _load_collection(uris, constraints, callback).merged().cubes()
    199 
    200 

/home/local/python27_epd/lib/python2.7/site-packages/Iris-1.5.0_dev-py2.7.egg/iris/__init__.pyc in _load_collection(uris, constraints, callback)
    166     try:
    167         cubes = _generate_cubes(uris, callback)
--> 168         result = iris.cube._CubeFilterCollection.from_cubes(cubes, constraints)
    169     except EOFError as e:
    170         raise iris.exceptions.TranslationError(

/home/local/python27_epd/lib/python2.7/site-packages/Iris-1.5.0_dev-py2.7.egg/iris/cube.pyc in from_cubes(cubes, constraints)
    133         pairs = [_CubeFilter(constraint) for constraint in constraints]
    134         collection = _CubeFilterCollection(pairs)
--> 135         for cube in cubes:
    136             collection.add_cube(cube)
    137         return collection

/home/local/python27_epd/lib/python2.7/site-packages/Iris-1.5.0_dev-py2.7.egg/iris/__init__.pyc in _generate_cubes(uris, callback)
    157         elif scheme in ['http', 'https']:
    158             urls = [':'.join(x) for x in groups]
--> 159             for cube in iris.io.load_http(urls, callback):
    160                 yield cube
    161         else:

/home/local/python27_epd/lib/python2.7/site-packages/Iris-1.5.0_dev-py2.7.egg/iris/io/__init__.pyc in load_http(urls, callback)
    198     # Call each iris format handler with the appropriate filenames
    199     for handling_format_spec, fnames in handler_map.iteritems():
--> 200         for cube in handling_format_spec.handler(fnames, callback):
    201             yield cube
    202 

/home/local/python27_epd/lib/python2.7/site-packages/Iris-1.5.0_dev-py2.7.egg/iris/fileformats/netcdf.pyc in load_cubes(filenames, callback)
    461             # formula term.
    462             if not cf_var.has_formula_terms():
--> 463                 cube = _load_cube(engine, cf, cf_var, filename)
    464 
    465                 # Process any associated formula terms and attach

/home/local/python27_epd/lib/python2.7/site-packages/Iris-1.5.0_dev-py2.7.egg/iris/fileformats/netcdf.pyc in _load_cube(engine, cf, cf_var, filename)
    366 
    367     # Run pyke inference engine with forward chaining rules.
--> 368     engine.activate(_PYKE_RULE_BASE)
    369 
    370     # Populate coordinate attributes with the untouched attributes from the

/home/local/python27_epd/lib/python2.7/site-packages/pyke/knowledge_engine.pyc in activate(self, *rb_names)
    294         add your facts before doing this!
    295         '''
--> 296         for rb_name in rb_names: self.get_rb(rb_name).activate()
    297 
    298     def lookup(self, kb_name, entity_name, pat_context, patterns):

/home/local/python27_epd/lib/python2.7/site-packages/pyke/rule_base.pyc in activate(self)
    157         self.engine.knowledge_bases[self.root_name] = self
    158         self.register_fc_rules(current_rb)
--> 159         self.run_fc_rules(current_rb)
    160 
    161     def reset(self):

/home/local/python27_epd/lib/python2.7/site-packages/pyke/rule_base.pyc in run_fc_rules(self, stop_at_rb)
    145         rb = self
    146         while rb is not stop_at_rb:
--> 147             for fc_rule in rb.fc_rules: fc_rule.run()
    148             if not rb.parent: break
    149             rb = rb.parent

/home/local/python27_epd/lib/python2.7/site-packages/pyke/fc_rule.pyc in run(self)
     88     def run(self):
     89         self.ran = True
---> 90         self.rule_fn(self)
     91 
     92     def new_fact(self, fact_args, n):

/home/local/python27_epd/lib/python2.7/site-packages/Iris-1.5.0_dev-py2.7.egg/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.pyc in fc_build_auxiliary_coordinate_latitude(rule, context, index)
    257             cf_coord_var = engine.cf_var.cf_group.auxiliary_coordinates[context.lookup_data('coordinate')]
    258             build_auxiliary_coordinate(engine, cf_coord_var,
--> 259             coord_name=CF_VALUE_STD_NAME_LAT)
    260             engine.rule_triggered.add(rule.name)
    261             rule.rule_base.num_fc_rules_triggered += 1

/home/local/python27_epd/lib/python2.7/site-packages/Iris-1.5.0_dev-py2.7.egg/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.pyc in build_auxiliary_coordinate(engine, cf_coord_var, coord_name, coord_system)
   1337                                  bounds=bounds_data,
   1338                                  attributes=attributes,
-> 1339                                  coord_system=coord_system)
   1340     cube.add_aux_coord(coord, data_dims)
   1341     engine.provides['coordinates'].append((coord, cf_coord_var.cf_name))

/home/local/python27_epd/lib/python2.7/site-packages/Iris-1.5.0_dev-py2.7.egg/iris/coords.pyc in __init__(self, points, standard_name, long_name, var_name, units, bounds, attributes, coord_system)
    344 
    345         self.points = points
--> 346         self.bounds = bounds
    347 
    348     def __getitem__(self, key):

/home/local/python27_epd/lib/python2.7/site-packages/Iris-1.5.0_dev-py2.7.egg/iris/coords.pyc in bounds(self, bounds)
   1384             # NB. Use _points to avoid triggering any lazy array.
   1385             if self._points.shape != bounds.shape[:-1]:
-> 1386                 raise ValueError("Bounds shape must be compatible with points "
   1387                                  "shape.")
   1388         self._bounds = bounds

ValueError: Bounds shape must be compatible with points shape.

In [ ]:
model='Global RTOFS/NCEP'
url='http://ecowatch.ncddc.noaa.gov/thredds/dodsC/hycom/hycom_reg1_agg/HYCOM_Region_1_Aggregation_best.ncd'
var='sea_water_temperature'  
lev=1
subsample=1
slice=var_lev_date(url=url,var=var, mytime=mytime, lev=lev, subsample=subsample)
myplot(slice,model=model)

In [7]:
model='RUTGERS/NWA'
url='http://oceanus.esm.rutgers.edu:8090/thredds/dodsC/ROMS/NWA/Run03/Output'
var='time-averaged potential temperature'
lev=0
slice=var_lev_date(url=url,var=var, mytime=mytime, lev=lev, subsample=1)


/home/local/python27_epd/lib/python2.7/site-packages/Iris-1.5.0_dev-py2.7.egg/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1163: UserWarning: Ignoring netCDF variable 'TIC' invalid units 'millimole_carbon meter-3'
  warnings.warn(msg.format(msg_name, msg_units))
/home/local/python27_epd/lib/python2.7/site-packages/Iris-1.5.0_dev-py2.7.egg/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1163: UserWarning: Ignoring netCDF variable 'SiOH' invalid units 'millimole_silica meter-3'
  warnings.warn(msg.format(msg_name, msg_units))
/home/local/python27_epd/lib/python2.7/site-packages/Iris-1.5.0_dev-py2.7.egg/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1163: UserWarning: Ignoring netCDF variable 'NO3' invalid units 'millimole_N03 meter-3'
  warnings.warn(msg.format(msg_name, msg_units))
/home/local/python27_epd/lib/python2.7/site-packages/Iris-1.5.0_dev-py2.7.egg/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1163: UserWarning: Ignoring netCDF variable 'NH4' invalid units 'millimole_NH4 meter-3'
  warnings.warn(msg.format(msg_name, msg_units))
/home/local/python27_epd/lib/python2.7/site-packages/Iris-1.5.0_dev-py2.7.egg/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1163: UserWarning: Ignoring netCDF variable 'PO4' invalid units 'millimole_PO4 meter-3'
  warnings.warn(msg.format(msg_name, msg_units))
slice retrieved in 338.939230 seconds
/home/local/python27_epd/lib/python2.7/site-packages/Iris-1.5.0_dev-py2.7.egg/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1163: UserWarning: Ignoring netCDF variable 'oxyg' invalid units 'millimole_O2 meter-3'
  warnings.warn(msg.format(msg_name, msg_units))

In [13]:
myplot(slice,model=model)



In [ ]:
print cubes

In [ ]:
nc = netCDF4.Dataset(url)

In [1]:
import netCDF4

In [2]:
url='[prefetch]http://colossus.dl.stevens-tech.edu/thredds/dodsC/fmrc/NYBight/NYHOPS_Forecast_Collection_for_the_New_York_Bight_best.ncd'

In [3]:
nc = netCDF4.Dataset(url)

In [4]:
print nc.variables['temp']


<type 'netCDF4.Variable'>
float32 temp(u'time', u'sigma', u'ypos', u'xpos')
    long_name: Water Temperature
    units: Celsius
    standard_name: sea_water_temperature
    coordinates: lon lat sigma time 
unlimited dimensions = ()
current size = (165954, 11, 22, 124)


In [5]:
t = nc.variables['temp'][-1,0,:,:]


---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-5-7c4314f018a8> in <module>()
----> 1 t = nc.variables['temp'][-1,0,:,:]

/home/local/python27_epd/lib/python2.7/site-packages/netCDF4.so in netCDF4.Variable.__getitem__ (netCDF4.c:29289)()

/home/local/python27_epd/lib/python2.7/site-packages/netCDF4.so in netCDF4.Variable._get (netCDF4.c:34647)()

RuntimeError: NetCDF: Malformed or inaccessible DAP DATADDS

In [4]:
print nc.variables['xpos']


<type 'netCDF4.Variable'>
float32 xpos(u'xpos',)
    long_name: X-coordinate in Cartesian system
    units: m
    axis: X
    _CoordinateAxisType: GeoX
unlimited dimensions = ()
current size = (124,)


In [5]:
print nc.variables['xpos'][:]


---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-5-0ecdc13e1e89> in <module>()
----> 1 print nc.variables['xpos'][:]

/home/local/python27_epd/lib/python2.7/site-packages/netCDF4.so in netCDF4.Variable.__getitem__ (netCDF4.c:29289)()

/home/local/python27_epd/lib/python2.7/site-packages/netCDF4.so in netCDF4.Variable._get (netCDF4.c:34647)()

RuntimeError: NetCDF: Malformed or inaccessible DAP DATADDS

In [7]:
url='http://oceanus.esm.rutgers.edu:8090/thredds/dodsC/ROMS/NWA/Run03/Output'
var='time-averaged potential temperature'

In [8]:
temp = iris.load_cube(url,var)

In [11]:
url='http://colossus.dl.stevens-tech.edu/thredds/dodsC/fmrc/NYBight/NYHOPS_Forecast_Collection_for_the_New_York_Bight_best.ncd'

In [12]:
cubes=iris.load(url)


---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-12-1913e1e486e3> in <module>()
----> 1 cubes=iris.load(url)

/home/local/python27_epd/lib/python2.7/site-packages/Iris-1.5.0_dev-py2.7.egg/iris/__init__.pyc in load(uris, constraints, callback)
    196 
    197     """
--> 198     return _load_collection(uris, constraints, callback).merged().cubes()
    199 
    200 

/home/local/python27_epd/lib/python2.7/site-packages/Iris-1.5.0_dev-py2.7.egg/iris/__init__.pyc in _load_collection(uris, constraints, callback)
    166     try:
    167         cubes = _generate_cubes(uris, callback)
--> 168         result = iris.cube._CubeFilterCollection.from_cubes(cubes, constraints)
    169     except EOFError as e:
    170         raise iris.exceptions.TranslationError(

/home/local/python27_epd/lib/python2.7/site-packages/Iris-1.5.0_dev-py2.7.egg/iris/cube.pyc in from_cubes(cubes, constraints)
    133         pairs = [_CubeFilter(constraint) for constraint in constraints]
    134         collection = _CubeFilterCollection(pairs)
--> 135         for cube in cubes:
    136             collection.add_cube(cube)
    137         return collection

/home/local/python27_epd/lib/python2.7/site-packages/Iris-1.5.0_dev-py2.7.egg/iris/__init__.pyc in _generate_cubes(uris, callback)
    157         elif scheme in ['http', 'https']:
    158             urls = [':'.join(x) for x in groups]
--> 159             for cube in iris.io.load_http(urls, callback):
    160                 yield cube
    161         else:

/home/local/python27_epd/lib/python2.7/site-packages/Iris-1.5.0_dev-py2.7.egg/iris/io/__init__.pyc in load_http(urls, callback)
    198     # Call each iris format handler with the appropriate filenames
    199     for handling_format_spec, fnames in handler_map.iteritems():
--> 200         for cube in handling_format_spec.handler(fnames, callback):
    201             yield cube
    202 

/home/local/python27_epd/lib/python2.7/site-packages/Iris-1.5.0_dev-py2.7.egg/iris/fileformats/netcdf.pyc in load_cubes(filenames, callback)
    461             # formula term.
    462             if not cf_var.has_formula_terms():
--> 463                 cube = _load_cube(engine, cf, cf_var, filename)
    464 
    465                 # Process any associated formula terms and attach

/home/local/python27_epd/lib/python2.7/site-packages/Iris-1.5.0_dev-py2.7.egg/iris/fileformats/netcdf.pyc in _load_cube(engine, cf, cf_var, filename)
    366 
    367     # Run pyke inference engine with forward chaining rules.
--> 368     engine.activate(_PYKE_RULE_BASE)
    369 
    370     # Populate coordinate attributes with the untouched attributes from the

/home/local/python27_epd/lib/python2.7/site-packages/pyke/knowledge_engine.pyc in activate(self, *rb_names)
    294         add your facts before doing this!
    295         '''
--> 296         for rb_name in rb_names: self.get_rb(rb_name).activate()
    297 
    298     def lookup(self, kb_name, entity_name, pat_context, patterns):

/home/local/python27_epd/lib/python2.7/site-packages/pyke/rule_base.pyc in activate(self)
    157         self.engine.knowledge_bases[self.root_name] = self
    158         self.register_fc_rules(current_rb)
--> 159         self.run_fc_rules(current_rb)
    160 
    161     def reset(self):

/home/local/python27_epd/lib/python2.7/site-packages/pyke/rule_base.pyc in run_fc_rules(self, stop_at_rb)
    145         rb = self
    146         while rb is not stop_at_rb:
--> 147             for fc_rule in rb.fc_rules: fc_rule.run()
    148             if not rb.parent: break
    149             rb = rb.parent

/home/local/python27_epd/lib/python2.7/site-packages/pyke/fc_rule.pyc in run(self)
     88     def run(self):
     89         self.ran = True
---> 90         self.rule_fn(self)
     91 
     92     def new_fact(self, fact_args, n):

/home/local/python27_epd/lib/python2.7/site-packages/Iris-1.5.0_dev-py2.7.egg/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.pyc in fc_build_auxiliary_coordinate_latitude(rule, context, index)
    257             cf_coord_var = engine.cf_var.cf_group.auxiliary_coordinates[context.lookup_data('coordinate')]
    258             build_auxiliary_coordinate(engine, cf_coord_var,
--> 259             coord_name=CF_VALUE_STD_NAME_LAT)
    260             engine.rule_triggered.add(rule.name)
    261             rule.rule_base.num_fc_rules_triggered += 1

/home/local/python27_epd/lib/python2.7/site-packages/Iris-1.5.0_dev-py2.7.egg/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.pyc in build_auxiliary_coordinate(engine, cf_coord_var, coord_name, coord_system)
   1337                                  bounds=bounds_data,
   1338                                  attributes=attributes,
-> 1339                                  coord_system=coord_system)
   1340     cube.add_aux_coord(coord, data_dims)
   1341     engine.provides['coordinates'].append((coord, cf_coord_var.cf_name))

/home/local/python27_epd/lib/python2.7/site-packages/Iris-1.5.0_dev-py2.7.egg/iris/coords.pyc in __init__(self, points, standard_name, long_name, var_name, units, bounds, attributes, coord_system)
    344 
    345         self.points = points
--> 346         self.bounds = bounds
    347 
    348     def __getitem__(self, key):

/home/local/python27_epd/lib/python2.7/site-packages/Iris-1.5.0_dev-py2.7.egg/iris/coords.pyc in bounds(self, bounds)
   1384             # NB. Use _points to avoid triggering any lazy array.
   1385             if self._points.shape != bounds.shape[:-1]:
-> 1386                 raise ValueError("Bounds shape must be compatible with points "
   1387                                  "shape.")
   1388         self._bounds = bounds

ValueError: Bounds shape must be compatible with points shape.

In [ ]: