This notebook generates the map and table HTMLs for the SECOORA Model vs Observed Sea Surface Height

Based on IOOS system-test notebook.


In [1]:
import os
import time

import iris
import pyoos
import owslib

from utilities import timeit

start_time = time.time()

In [2]:
import pytz
from datetime import datetime, timedelta

from utilities import CF_names

# Choose the date range.
stop = datetime(2014, 7, 7, 12)

stop = stop.replace(tzinfo=pytz.utc)
start = stop - timedelta(days=7)

# SECOORA region (NC, SC GA, FL).
bbox = [-87.40, 24.25, -74.70, 36.70]

# CF-names to look for (Sea Surface Height).
name_list = CF_names['water level']

In [3]:
import logging as log
reload(log)

fmt = '{:*^64}'.format
log.captureWarnings(True)
LOG_FILENAME = 'log'
LOG_FILENAME = os.path.join(LOG_FILENAME)
log.basicConfig(filename=LOG_FILENAME,
                filemode='w',
                format='%(asctime)s %(levelname)s: %(message)s',
                datefmt='%I:%M:%S',
                level=log.INFO,
                stream=None)

log.info(fmt(' Run information '))
log.info('Run date: {:%Y-%m-%d %H:%M:%S}'.format(datetime.utcnow()))
log.info('Download start: {:%Y-%m-%d %H:%M:%S}'.format(start))
log.info('Download stop: {:%Y-%m-%d %H:%M:%S}'.format(stop))
log.info('Bounding box: {0:3.2f}, {1:3.2f},'
         '{2:3.2f}, {3:3.2f}'.format(*bbox))
log.info(fmt(' Software version '))
log.info('Iris version: {}'.format(iris.__version__))
log.info('owslib version: {}'.format(owslib.__version__))
log.info('pyoos version: {}'.format(pyoos.__version__))

In [4]:
from owslib import fes
from utilities import fes_date_filter

kw = dict(wildCard='*',
          escapeChar='\\',
          singleChar='?',
          propertyname='apiso:AnyText')

or_filt = fes.Or([fes.PropertyIsLike(literal=('*%s*' % val), **kw)
                  for val in name_list])

not_filt = fes.Not([fes.PropertyIsLike(literal='*Averages*', **kw)])

begin, end = fes_date_filter(start, stop)
filter_list = [fes.And([fes.BBox(bbox), begin, end, or_filt, not_filt])]

In [5]:
from owslib.csw import CatalogueServiceWeb

endpoint = 'http://www.ngdc.noaa.gov/geoportal/csw'
csw = CatalogueServiceWeb(endpoint, timeout=60)
csw.getrecords2(constraints=filter_list, maxrecords=1000, esn='full')

log.info(fmt(' Catalog information '))
log.info("URL: {}".format(endpoint))
log.info("CSW version: {}".format(csw.version))
log.info("Number of datasets available: {}".format(len(csw.records.keys())))

In [6]:
from utilities import service_urls

dap_urls = service_urls(csw.records, service='odp:url')
sos_urls = service_urls(csw.records, service='sos:url')

log.info(fmt(' CSW URLs '))
for rec, item in csw.records.items():
    log.info('{}'.format(item.title))

log.info(fmt(' DAP URLs '))
for url in dap_urls:
    log.info('{}.html'.format(url))

log.info(fmt(' SOS URLs '))
for url in sos_urls:
    log.info('{}'.format(url))

In [7]:
from pyoos.collectors.coops.coops_sos import CoopsSos

collector = CoopsSos()
sos_name = 'water_surface_height_above_reference_datum'

datum = 'NAVD'
collector.set_datum(datum)
collector.end_time = stop
collector.start_time = start
collector.variables = [sos_name]

ofrs = collector.server.offerings
title = collector.server.identification.title
log.info(fmt(' Collector offerings '))
log.info('{}: {} offerings'.format(title, len(ofrs)))

In [8]:
from pandas import read_csv
from utilities import sos_request

params = dict(observedProperty=sos_name,
              eventTime=start.strftime('%Y-%m-%dT%H:%M:%SZ'),
              featureOfInterest='BBOX:{0},{1},{2},{3}'.format(*bbox),
              offering='urn:ioos:network:NOAA.NOS.CO-OPS:WaterLevelActive')

uri = 'http://opendap.co-ops.nos.noaa.gov/ioos-dif-sos/SOS'
url = sos_request(uri, **params)
observations = read_csv(url)

log.info('SOS URL request: {}'.format(url))

Clean the dataframe (visualization purpose only)


In [9]:
from utilities import get_coops_longname, to_html

columns = {'datum_id': 'datum',
           'sensor_id': 'sensor',
           'station_id': 'station',
           'latitude (degree)': 'lat',
           'longitude (degree)': 'lon',
           'vertical_position (m)': 'height',
           'water_surface_height_above_reference_datum (m)': 'ssh above datum'}

observations.rename(columns=columns, inplace=True)

observations['datum'] = [s.split(':')[-1] for s in observations['datum']]
observations['sensor'] = [s.split(':')[-1] for s in observations['sensor']]
observations['station'] = [s.split(':')[-1] for s in observations['station']]
observations['name'] = [get_coops_longname(s) for s in observations['station']]

observations.set_index('name', inplace=True)
to_html(observations.head(), 'style.css')


Out[9]:
station sensor lat lon date_time ssh above datum datum height
name
Duck, NC 8651370 A1 36.1833 -75.7467 2014-06-30T12:00:00Z 0.761 MLLW 5.663
Oregon Inlet Marina, NC 8652587 A1 35.7950 -75.5483 2014-06-30T12:00:00Z 0.242 MLLW 0.788
USCG Station Hatteras, NC 8654467 A1 35.2086 -75.7042 2014-06-30T12:00:00Z 0.216 MLLW 8.345
Beaufort, NC 8656483 A1 34.7200 -76.6700 2014-06-30T12:00:00Z 0.703 MLLW 0.562
Wilmington, NC 8658120 A1 34.2267 -77.9533 2014-06-30T12:00:00Z 0.391 MLLW 0.747

Generate a uniform 6-min time base for model/data comparison


In [10]:
import iris
from pandas import DataFrame
from owslib.ows import ExceptionReport
from utilities import coops2df, save_timeseries

iris.FUTURE.netcdf_promote = True

log.info(fmt(' Observations (station data) '))
fname = '{:%Y-%m-%d}-OBS_DATA.nc'.format(stop)

log.info(fmt(' Downloading to file {} '.format(fname)))
data = dict()
bad_datum = []
for station in observations.station:
    try:
        df = coops2df(collector, station)
        col = 'water_surface_height_above_reference_datum (m)'
        data.update({station: df[col]})
    except ExceptionReport as e:
        bad_datum.append(station)
        name = get_coops_longname(station)
        log.warning("[{}] {}:\n{}".format(station, name, e))
obs_data = DataFrame.from_dict(data)

# Split good and bad vertical datum stations.
pattern = '|'.join(bad_datum)
if pattern:
    non_navd = observations.station.str.contains(pattern)
    bad_datum = observations[non_navd]
    observations = observations[~non_navd]

comment = "Several stations from http://opendap.co-ops.nos.noaa.gov"
kw = dict(longitude=observations.lon,
          latitude=observations.lat,
          station_attr=dict(cf_role="timeseries_id"),
          cube_attr=dict(featureType='timeSeries',
                         Conventions='CF-1.6',
                         standard_name_vocabulary='CF-1.6',
                         cdm_data_type="Station",
                         comment=comment,
                         datum=datum,
                         url=url))

save_timeseries(obs_data, outfile=fname,
                standard_name=sos_name, **kw)

to_html(obs_data.head(), 'style.css')


Out[10]:
8651370 8652587 8658163 8661070 8662245 8665530 8720030 8720218 8720219 8720226 8720357 8720503 8720625 8721604 8722670 8723970 8724580 8725110 8725520 8726384 8727520 8728690 8729108 8729840
date_time
2014-06-30 12:00:00 0.094 0.027 0.186 0.220 0.104 0.299 -0.044 0.023 -0.145 -0.074 0.079 0.124 0.240 -0.056 -0.142 -0.161 -0.221 -0.216 -0.028 -0.102 -0.097 0.122 0.064 0.122
2014-06-30 12:06:00 0.133 0.036 0.219 0.249 0.125 0.325 -0.002 0.046 -0.124 -0.071 0.078 0.120 0.232 -0.060 -0.124 -0.166 -0.213 -0.239 -0.029 -0.098 -0.116 0.123 0.069 0.122
2014-06-30 12:12:00 0.150 0.042 0.224 0.263 0.145 0.352 0.028 0.071 -0.105 -0.061 0.077 0.116 0.225 -0.010 -0.107 -0.169 -0.204 -0.225 -0.030 -0.100 -0.137 0.130 0.073 0.125
2014-06-30 12:18:00 0.161 0.050 0.249 0.301 0.165 0.379 0.063 0.100 -0.083 -0.055 0.077 0.110 0.217 -0.026 -0.087 -0.173 -0.192 -0.249 -0.031 -0.102 -0.157 0.133 0.070 0.132
2014-06-30 12:24:00 0.206 0.059 0.273 0.323 0.182 0.402 0.096 0.126 -0.058 -0.046 0.077 0.110 0.209 -0.055 -0.077 -0.176 -0.183 -0.228 -0.033 -0.105 -0.170 0.131 0.077 0.140

Loop discovered models and save the nearest time-series


In [11]:
%matplotlib inline

import numpy as np
from iris.pandas import as_series
from iris.exceptions import (CoordinateNotFoundError, ConstraintMismatchError,
                             MergeError)

from utilities import (standardize_fill_value, plt_grid, get_cube,
                       get_model_name, make_tree, get_nearest_water,
                       add_station, ensure_timeseries)

# FIXME: Filtering out NECOFS  and estofs.
dap_urls = [url for url in dap_urls
            if 'NECOFS' not in url  # cartesian coords are not implemented.
            if 'estofs' not in url]  # download is taking forver today!


log.info(fmt(' Models (simulated data) '))
for k, url in enumerate(dap_urls):
    with timeit(log):
        log.info('\n[Reading url {}/{}]: {}'.format(k+1, len(dap_urls), url))
        try:
            cube = get_cube(url, name_list=name_list, bbox=bbox,
                            time=(start, stop), units=iris.unit.Unit('meters'))
            if cube.ndim == 1:  # We Need a better way to identify model data.
                log.warning('url {} is probably a timeSeries!'.format(url))
                continue
        except (RuntimeError, ValueError, MergeError,
                ConstraintMismatchError, CoordinateNotFoundError) as e:
            log.warning('Cannot get cube for: {}\n{}'.format(url, e))
            continue

        mod_name, model_full_name = get_model_name(cube, url)

        fname = '{:%Y-%m-%d}-{}.nc'.format(stop, mod_name)
        log.info(fmt(' Downloading to file {} '.format(fname)))
        try:  # Make tree.
            tree, lon, lat = make_tree(cube)
            fig, ax = plt_grid(lon, lat)
        except CoordinateNotFoundError as e:
            log.warning('Cannot make KDTree for: {}'.format(mod_name))
            continue
        # Get model series at observed locations.
        raw_series = dict()
        for station, obs in observations.iterrows():
            a = obs_data[obs['station']]
            try:
                kw = dict(k=10, max_dist=0.04, min_var=0.01)
                args = cube, tree, obs.lon, obs.lat
                series, dist, idx = get_nearest_water(*args, **kw)
            # RuntimeError may occurs, but you should run it again!
            except ValueError as e:
                log.warning(e)
                continue
            if not series:
                status = "Found Land"
            else:
                raw_series.update({obs['station']: series})
                series = as_series(series)
                status = "Found Water"
                ax.plot(lon[idx], lat[idx], 'g.')

            log.info('[{}] {}'.format(status, obs.name))

        if raw_series:  # Save cube.
            for station, cube in raw_series.items():
                cube = standardize_fill_value(cube)
                cube = add_station(cube, station)
            try:
                cube = iris.cube.CubeList(raw_series.values()).merge_cube()
            except MergeError as e:
                log.warning(e)

            ensure_timeseries(cube)
            iris.save(cube, fname)
            del cube

        size = len(raw_series)
        ax.set_title('{}: Points found {}'.format(mod_name, size))
        ax.plot(observations.lon, observations.lat, 'ro',
                zorder=1, label='Observation', alpha=0.25)
        ax.set_extent([bbox[0], bbox[2], bbox[1], bbox[3]])

    log.info('[{}]: {}'.format(mod_name, url))


Load saved files and interpolate to the observations time interval


In [12]:
from glob import glob
from operator import itemgetter

from pandas import Panel
from utilities import nc2df

fname = '{:%Y-%m-%d}-OBS_DATA.nc'.format(stop)
OBS_DATA = nc2df(fname)
index = OBS_DATA.index

dfs = dict(OBS_DATA=OBS_DATA)
for fname in glob("*.nc"):
    if 'OBS_DATA' in fname:
        continue
    else:
        model = fname.split('.')[0].split('-')[-1]
        df = nc2df(fname)
        kw = dict(method='time', limit=30)
        df = df.reindex(index).interpolate(**kw).ix[index]
        dfs.update({model: df})

dfs = Panel.fromDict(dfs).swapaxes(0, 2)

# Clusters.
big_list = []
for fname in glob("*.nc"):
    if 'OBS_DATA' in fname:
        continue
    nc = iris.load_cube(fname)
    model = fname.split('-')[-1].split('.')[0]
    lons = nc.coord(axis='X').points
    lats = nc.coord(axis='Y').points
    stations = nc.coord('station name').points
    models = [model]*lons.size
    lista = zip(models, lons.tolist(), lats.tolist(), stations.tolist())
    big_list.extend(lista)

big_list.sort(key=itemgetter(3))
df = DataFrame(big_list, columns=['name', 'lon', 'lat', 'station'])
df.set_index('station', drop=True, inplace=True)
groups = df.groupby(df.index)

In [13]:
import folium
import vincent
from utilities import get_coordinates, inline_map

lon_center, lat_center = np.array(bbox).reshape(2, 2).mean(axis=0)
ssh = folium.Map(width=750, height=500,
                 location=[lat_center, lon_center], zoom_start=5)

# Create the map and add the bounding box line.
kw = dict(line_color='#FF0000', line_weight=2)
ssh.line(get_coordinates(bbox), **kw)

# Clusters.
for station, info in groups:
    station = get_coops_longname(station)
    for lat, lon, name in zip(info.lat, info.lon, info.name):
        location = lat, lon
        popup = '<b>{}</b>\n{}'.format(station, name)
        ssh.simple_marker(location=location, popup=popup,
                          clustered_marker=True)

# Model and observations.
for station in dfs:
    sta_name = get_coops_longname(station)
    df = dfs[station].dropna(axis=1, how='all')
    # FIXME: This is bad!  But I cannot represent NaN with Vega!
    df.fillna(value='null', inplace=True)
    vis = vincent.Line(df, width=500, height=150)
    vis.axis_titles(x='Time', y='Sea surface height (m)')
    vis.legend(title=sta_name)
    vis.name = sta_name
    json = 'station_{}.json'.format(station)
    vis.to_json(json)
    obs = observations[observations['station'] == station].squeeze()
    popup = (vis, json)
    if (df.columns == 'OBS_DATA').all():
        kw = dict(popup=popup, marker_color="blue", marker_icon="ok")
    else:
        if 'SABGOM' in df.columns:
            kw = dict(popup=popup, marker_color="green", marker_icon="ok-sign")
        else:
            kw = dict(popup=popup, marker_color="green", marker_icon="ok")
    ssh.simple_marker(location=[obs['lat'], obs['lon']], **kw)

# Bad datum.
if isinstance(bad_datum, DataFrame):
    for station, obs in bad_datum.iterrows():
        popup = '<b>Station:</b> {}<br><b>Datum:</b> {}<br>'
        popup = popup.format(station, obs['datum'])
        kw = dict(popup=popup, marker_color="red", marker_icon="question-sign")
        ssh.simple_marker(location=[obs['lat'], obs['lon']], **kw)

ssh.create_map(path=os.path.join('ssh.html'))
inline_map('ssh.html')


Out[13]:

In [14]:
elapsed = time.time() - start_time
log.info(elapsed)
log.info('EOF')

Compute bias


In [15]:
import os
import sys
root = os.path.abspath(os.path.join(os.getcwd(), os.pardir))
sys.path.append(root)

from utilities import nc2df

In [16]:
fname = '{:%Y-%m-%d}-OBS_DATA.nc'.format(stop)

OBS_DATA = nc2df(fname)
index = OBS_DATA.index

In [17]:
from glob import glob
from pandas import Panel

dfs = dict(OBS_DATA=OBS_DATA)
for fname in glob("*.nc"):
    if 'OBS_DATA' in fname:
        pass
    else:
        model = fname.split('.')[0].split('-')[-1]
        df = nc2df(fname)
        if False:
            kw = dict(method='time')
            df = df.reindex(index).interpolate(**kw).ix[index]
        dfs.update({model: df})

dfs = Panel.fromDict(dfs).swapaxes(0, 2)

In [18]:
from pandas import DataFrame

means = dict()
for station, df in dfs.iteritems():
    df.dropna(axis=1, how='all', inplace=True)
    mean = df.mean()
    df = df - mean + mean['OBS_DATA']
    means.update({station: mean['OBS_DATA'] - mean.drop('OBS_DATA')})

bias = DataFrame.from_dict(means).dropna(axis=1, how='all')
bias = bias.applymap('{:.2f}'.format).replace('nan', '--')

columns = dict()
[columns.update({station: get_coops_longname(station)}) for
 station in bias.columns.values]

bias.rename(columns=columns, inplace=True)

to_html(bias.T, 'style.css')


Out[18]:
COAWST_4 ESPRESSO HYCOM SABGOM USF
Duck, NC -- -- -- 0.50 --
Oregon Inlet Marina, NC 0.08 0.44 0.61 -- --
Wrightsville Beach, NC -0.07 -- 0.39 0.49 --
Springmaid Pier, SC -0.10 -- -- 0.43 --
Lake Worth Pier, FL -0.32 -- 0.14 -- --
Key West, FL -0.34 -- -- 0.19 --
Naples, FL -- -- -- -- 0.14
Fort Myers, FL -- -- -- -- 0.18
Port Manatee, FL -- -- -- -- 0.20
Cedar Key, FL -0.11 -- -- 0.40 0.28
Apalachicola, FL -0.08 -- -- -- 0.24
Panama City, FL -0.05 -- -- -- 0.32
Pensacola, FL -0.00 -- -- -- --

In [19]:
with open(LOG_FILENAME) as f:
    print(''.join(f.readlines()))


05:33:13 INFO: *********************** Run information ************************
05:33:13 INFO: Run date: 2014-09-29 20:33:13
05:33:13 INFO: Download start: 2014-06-30 12:00:00
05:33:13 INFO: Download stop: 2014-07-07 12:00:00
05:33:13 INFO: Bounding box: -87.40, 24.25,-74.70, 36.70
05:33:13 INFO: *********************** Software version ***********************
05:33:13 INFO: Iris version: 1.7.2-DEV
05:33:13 INFO: owslib version: 0.8.7
05:33:13 INFO: pyoos version: 0.6.1
05:33:15 INFO: ********************* Catalog information **********************
05:33:15 INFO: URL: http://www.ngdc.noaa.gov/geoportal/csw
05:33:15 INFO: CSW version: 2.0.2
05:33:15 INFO: Number of datasets available: 9
05:33:15 INFO: *************************** CSW URLs ***************************
05:33:15 INFO: USF FVCOM - Nowcast Aggregation
05:33:15 INFO: ROMS/TOMS 3.0 - New Floria Shelf Application
05:33:15 INFO: ROMS/TOMS 3.0 - South-Atlantic Bight and Gulf of Mexico
05:33:15 INFO: COAWST Forecast System : USGS : US East Coast and Gulf of Mexico (Experimental)
05:33:15 INFO: ROMS ESPRESSO Real-Time Operational IS4DVAR Forecast System Version 2 (NEW) 2013-present FMRC History (Best)
05:33:15 INFO: Barotropic Tide Model for the Pacific Basin
05:33:15 INFO: HYbrid Coordinate Ocean Model (HYCOM): Global
05:33:15 INFO: ESTOFS Storm Surge Model - Atlantic - v1.0.0 - NOAA - NCEP - ADCIRC
05:33:15 INFO: NECOFS GOM3 Wave - Northeast US - Latest Forecast
05:33:15 INFO: *************************** DAP URLs ***************************
05:33:15 INFO: http://crow.marine.usf.edu:8080/thredds/dodsC/FVCOM-Nowcast-Agg.nc.html
05:33:15 INFO: http://crow.marine.usf.edu:8080/thredds/dodsC/WFS_ROMS_NF_model/USF_Ocean_Circulation_Group_West_Florida_Shelf_Daily_ROMS_Nowcast_Forecast_Model_Data_best.ncd.html
05:33:15 INFO: http://geoport-dev.whoi.edu/thredds/dodsC/estofs/atlantic.html
05:33:15 INFO: http://geoport.whoi.edu/thredds/dodsC/coawst_4/use/fmrc/coawst_4_use_best.ncd.html
05:33:15 INFO: http://omgsrv1.meas.ncsu.edu:8080/thredds/dodsC/fmrc/sabgom/SABGOM_Forecast_Model_Run_Collection_best.ncd.html
05:33:15 INFO: http://oos.soest.hawaii.edu/thredds/dodsC/hioos/tide_pac.html
05:33:15 INFO: http://oos.soest.hawaii.edu/thredds/dodsC/pacioos/hycom/global.html
05:33:15 INFO: http://tds.marine.rutgers.edu/thredds/dodsC/roms/espresso/2013_da/his_Best/ESPRESSO_Real-Time_v2_History_Best_Available_best.ncd.html
05:33:15 INFO: http://www.smast.umassd.edu:8080/thredds/dodsC/FVCOM/NECOFS/Forecasts/NECOFS_WAVE_FORECAST.nc.html
05:33:15 INFO: *************************** SOS URLs ***************************
05:33:23 INFO: ********************* Collector offerings **********************
05:33:23 INFO: NOAA.NOS.CO-OPS SOS: 1025 offerings
05:33:25 INFO: SOS URL request: http://opendap.co-ops.nos.noaa.gov/ioos-dif-sos/SOS?eventTime=2014-06-30T12%3A00%3A00Z&service=SOS&offering=urn%3Aioos%3Anetwork%3ANOAA.NOS.CO-OPS%3AWaterLevelActive&request=GetObservation&version=1.0.0&responseFormat=text%2Fcsv&featureOfInterest=BBOX%3A-87.4%2C24.25%2C-74.7%2C36.7&observedProperty=water_surface_height_above_reference_datum
05:33:38 INFO: ***************** Observations (station data) ******************
05:33:38 INFO: ********** Downloading to file 2014-07-07-OBS_DATA.nc **********
05:33:44 WARNING: [8654467] USCG Station Hatteras, NC:
'Wrong Datum for this station: The correct Datum values are: MHHW, MHW, MTL, MSL, MLW, MLLW, STND'
05:33:45 WARNING: [8656483] Beaufort, NC:
'Wrong Datum for this station: The correct Datum values are: MHHW, MHW, MTL, MSL, MLW, MLLW, STND'
05:33:46 WARNING: [8658120] Wilmington, NC:
'Wrong Datum for this station: The correct Datum values are: MHHW, MHW, MTL, MSL, MLW, MLLW, STND'
05:34:35 WARNING: [8726520] St Petersburg, FL:
'Wrong Datum for this station: The correct Datum values are: MHHW, MHW, MTL, MSL, MLW, MLLW, STND'
05:34:37 WARNING: [8726607] Old Port Tampa, FL:
'Wrong Datum for this station: The correct Datum values are: MHHW, MHW, MTL, MSL, MLW, MLLW, STND'
05:34:52 WARNING: [8729210] Panama City Beach, FL:
'Wrong Datum for this station: The correct Datum values are: MHHW, MHW, MTL, MSL, MLW, MLLW, STND'
05:34:54 INFO: ******************* Models (simulated data) ********************
05:34:54 INFO: 
[Reading url 1/7]: http://crow.marine.usf.edu:8080/thredds/dodsC/FVCOM-Nowcast-Agg.nc
05:34:55 WARNING: /home/filipe/miniconda/envs/ioos/lib/python2.7/site-packages/iris/fileformats/cf.py:1004: UserWarning: Ignoring variable u'lat' referenced by variable u'hc': Dimensions (u'node',) do not span (u'nele',)
  warnings.warn(msg)

05:34:55 WARNING: /home/filipe/miniconda/envs/ioos/lib/python2.7/site-packages/iris/fileformats/cf.py:1004: UserWarning: Ignoring variable u'lon' referenced by variable u'hc': Dimensions (u'node',) do not span (u'nele',)
  warnings.warn(msg)

05:34:55 WARNING: /home/filipe/miniconda/envs/ioos/lib/python2.7/site-packages/iris/fileformats/cf.py:1004: UserWarning: Ignoring variable u'siglay' referenced by variable u'u': Dimensions (u'siglay', u'node') do not span (u'time', u'siglay', u'nele')
  warnings.warn(msg)

05:34:55 WARNING: /home/filipe/miniconda/envs/ioos/lib/python2.7/site-packages/iris/fileformats/cf.py:1004: UserWarning: Ignoring variable u'siglay' referenced by variable u'v': Dimensions (u'siglay', u'node') do not span (u'time', u'siglay', u'nele')
  warnings.warn(msg)

05:34:57 WARNING: /home/filipe/miniconda/envs/ioos/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1359: UserWarning: Failed to create 'time' dimension coordinate: The points array must be strictly monotonic.
Gracefully creating 'time' auxiliary coordinate instead.
  error=e_msg))

05:34:57 WARNING: /home/filipe/miniconda/envs/ioos/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1196: UserWarning: Ignoring netCDF variable 'Itime2' invalid units 'msecs since 00:00:00'
  warnings.warn(msg.format(msg_name, msg_units))

05:34:57 WARNING: /home/filipe/miniconda/envs/ioos/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1196: UserWarning: Ignoring netCDF variable 'salinity' invalid units 'psu'
  warnings.warn(msg.format(msg_name, msg_units))

05:34:57 WARNING: Cannot get cube for: http://crow.marine.usf.edu:8080/thredds/dodsC/FVCOM-Nowcast-Agg.nc
Cube does not contain ['sea_surface_height', 'sea_surface_elevation', 'sea_surface_height_above_geoid', 'sea_surface_height_above_sea_level', 'water_surface_height_above_reference_datum', 'sea_surface_height_above_reference_ellipsoid']
05:34:57 INFO: 00:00:02
05:34:57 INFO: 
[Reading url 2/7]: http://crow.marine.usf.edu:8080/thredds/dodsC/WFS_ROMS_NF_model/USF_Ocean_Circulation_Group_West_Florida_Shelf_Daily_ROMS_Nowcast_Forecast_Model_Data_best.ncd
05:34:59 WARNING: /home/filipe/miniconda/envs/ioos/lib/python2.7/site-packages/iris/fileformats/cf.py:1038: UserWarning: Ignoring formula terms variable u'h' referenced by data variable u'v' via variable u's_rho': Dimensions (u'eta_rho', u'xi_rho') do not span (u'time', u's_rho', u'eta_v', u'xi_v')
  warnings.warn(msg)

05:34:59 WARNING: /home/filipe/miniconda/envs/ioos/lib/python2.7/site-packages/iris/fileformats/cf.py:1038: UserWarning: Ignoring formula terms variable u'zeta' referenced by data variable u'v' via variable u's_rho': Dimensions (u'time', u'eta_rho', u'xi_rho') do not span (u'time', u's_rho', u'eta_v', u'xi_v')
  warnings.warn(msg)

05:34:59 WARNING: /home/filipe/miniconda/envs/ioos/lib/python2.7/site-packages/iris/fileformats/cf.py:1038: UserWarning: Ignoring formula terms variable u'h' referenced by data variable u'u' via variable u's_rho': Dimensions (u'eta_rho', u'xi_rho') do not span (u'time', u's_rho', u'eta_u', u'xi_u')
  warnings.warn(msg)

05:34:59 WARNING: /home/filipe/miniconda/envs/ioos/lib/python2.7/site-packages/iris/fileformats/cf.py:1038: UserWarning: Ignoring formula terms variable u'zeta' referenced by data variable u'u' via variable u's_rho': Dimensions (u'time', u'eta_rho', u'xi_rho') do not span (u'time', u's_rho', u'eta_u', u'xi_u')
  warnings.warn(msg)

05:34:59 WARNING: /home/filipe/miniconda/envs/ioos/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1291: UserWarning: Gracefully filling 'time' dimension coordinate masked points
  warnings.warn(msg.format(str(cf_coord_var.cf_name)))

05:34:59 WARNING: /home/filipe/miniconda/envs/ioos/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1196: UserWarning: Ignoring netCDF variable 'Cs_r' invalid units 'nondimensional'
  warnings.warn(msg.format(msg_name, msg_units))

05:34:59 WARNING: /home/filipe/miniconda/envs/ioos/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1196: UserWarning: Ignoring netCDF variable 's_rho' invalid units 'nondimensional'
  warnings.warn(msg.format(msg_name, msg_units))

05:34:59 WARNING: /home/filipe/miniconda/envs/ioos/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1196: UserWarning: Ignoring netCDF variable 'Cs_w' invalid units 'nondimensional'
  warnings.warn(msg.format(msg_name, msg_units))

05:34:59 WARNING: /home/filipe/miniconda/envs/ioos/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1196: UserWarning: Ignoring netCDF variable 's_w' invalid units 'nondimensional'
  warnings.warn(msg.format(msg_name, msg_units))

05:34:59 WARNING: /home/filipe/miniconda/envs/ioos/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1196: UserWarning: Ignoring netCDF variable 'salt' invalid units 'PSU'
  warnings.warn(msg.format(msg_name, msg_units))

05:35:03 INFO: ************ Downloading to file 2014-07-07-USF.nc *************
05:35:03 WARNING: No data found for (-75.7467,36.1833) using max_dist=0.04.
05:35:03 WARNING: No data found for (-75.5483,35.795) using max_dist=0.04.
05:35:03 WARNING: No data found for (-77.7867,34.2133) using max_dist=0.04.
05:35:03 WARNING: No data found for (-78.9183,33.655) using max_dist=0.04.
05:35:03 WARNING: No data found for (-79.1867,33.3517) using max_dist=0.04.
05:35:03 WARNING: No data found for (-79.925,32.7817) using max_dist=0.04.
05:35:03 WARNING: No data found for (-81.465,30.6717) using max_dist=0.04.
05:35:03 WARNING: No data found for (-81.43,30.3967) using max_dist=0.04.
05:35:03 WARNING: No data found for (-81.5583,30.3867) using max_dist=0.04.
05:35:03 WARNING: No data found for (-81.6583,30.32) using max_dist=0.04.
05:35:03 WARNING: No data found for (-81.6917,30.1917) using max_dist=0.04.
05:35:03 WARNING: No data found for (-81.6283,29.9783) using max_dist=0.04.
05:35:03 WARNING: No data found for (-81.5483,29.8017) using max_dist=0.04.
05:35:03 WARNING: No data found for (-80.5931,28.4158) using max_dist=0.04.
05:35:03 WARNING: No data found for (-80.0333,26.6117) using max_dist=0.04.
05:35:03 WARNING: No data found for (-81.105,24.7117) using max_dist=0.04.
05:35:03 WARNING: No data found for (-81.8079,24.5557) using max_dist=0.04.
05:35:05 INFO: [Found Water] Naples, FL
05:35:07 INFO: [Found Water] Fort Myers, FL
05:35:10 INFO: [Found Water] Port Manatee, FL
05:35:14 INFO: [Found Water] Cedar Key, FL
05:35:17 INFO: [Found Water] Apalachicola, FL
05:35:20 INFO: [Found Water] Panama City, FL
05:35:20 WARNING: No data found for (-87.2112,30.4044) using max_dist=0.04.
05:35:29 INFO: 00:00:32
05:35:29 INFO: [USF]: http://crow.marine.usf.edu:8080/thredds/dodsC/WFS_ROMS_NF_model/USF_Ocean_Circulation_Group_West_Florida_Shelf_Daily_ROMS_Nowcast_Forecast_Model_Data_best.ncd
05:35:29 INFO: 
[Reading url 3/7]: http://geoport.whoi.edu/thredds/dodsC/coawst_4/use/fmrc/coawst_4_use_best.ncd
05:35:55 INFO: ********** Downloading to file 2014-07-07-COAWST_4.nc **********
05:36:01 INFO: [Found Land] Duck, NC
05:36:11 INFO: [Found Water] Oregon Inlet Marina, NC
05:36:17 INFO: [Found Water] Wrightsville Beach, NC
05:36:26 INFO: [Found Water] Springmaid Pier, SC
05:36:31 INFO: [Found Land] Oyster Landing (N Inlet Estuary), SC
05:36:36 INFO: [Found Land] Charleston, SC
05:36:41 INFO: [Found Land] Fernandina Beach, FL
05:36:46 INFO: [Found Land] Mayport (Bar Pilots Dock), FL
05:36:51 INFO: [Found Land] Dames Point, FL
05:36:57 INFO: [Found Land] Southbank Riverwalk, St Johns River, FL
05:37:02 INFO: [Found Land] I-295 Bridge, St Johns River, FL
05:37:07 INFO: [Found Land] Red Bay Point, St Johns River, FL
05:37:12 INFO: [Found Land] Racy Point, St Johns River, FL
05:37:17 INFO: [Found Land] Trident Pier, FL
05:37:28 INFO: [Found Water] Lake Worth Pier, FL
05:37:33 INFO: [Found Land] Vaca Key, FL
05:37:44 INFO: [Found Water] Key West, FL
05:37:50 INFO: [Found Land] Naples, FL
05:37:55 INFO: [Found Land] Fort Myers, FL
05:38:01 INFO: [Found Land] Port Manatee, FL
05:38:12 INFO: [Found Water] Cedar Key, FL
05:38:17 INFO: [Found Water] Apalachicola, FL
05:38:23 INFO: [Found Water] Panama City, FL
05:38:29 INFO: [Found Water] Pensacola, FL
05:39:12 INFO: 00:03:43
05:39:12 INFO: [COAWST_4]: http://geoport.whoi.edu/thredds/dodsC/coawst_4/use/fmrc/coawst_4_use_best.ncd
05:39:12 INFO: 
[Reading url 4/7]: http://omgsrv1.meas.ncsu.edu:8080/thredds/dodsC/fmrc/sabgom/SABGOM_Forecast_Model_Run_Collection_best.ncd
05:39:14 WARNING: /home/filipe/miniconda/envs/ioos/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1196: UserWarning: Ignoring netCDF variable 'chlorophyll' invalid units 'milligrams_chlorophyll meter-3'
  warnings.warn(msg.format(msg_name, msg_units))

05:39:15 WARNING: /home/filipe/miniconda/envs/ioos/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1196: UserWarning: Ignoring netCDF variable 'phytoplankton' invalid units 'millimole_nitrogen meter-3'
  warnings.warn(msg.format(msg_name, msg_units))

05:39:15 WARNING: /home/filipe/miniconda/envs/ioos/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1196: UserWarning: Ignoring netCDF variable 'NO3' invalid units 'millimole_N03 meter-3'
  warnings.warn(msg.format(msg_name, msg_units))

05:39:15 WARNING: /home/filipe/miniconda/envs/ioos/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1196: UserWarning: Ignoring netCDF variable 'zooplankton' invalid units 'millimole_nitrogen meter-3'
  warnings.warn(msg.format(msg_name, msg_units))

05:39:15 WARNING: /home/filipe/miniconda/envs/ioos/lib/python2.7/site-packages/iris/fileformats/_pyke_rules/compiled_krb/fc_rules_cf_fc.py:1196: UserWarning: Ignoring netCDF variable 'NH4' invalid units 'millimole_NH4 meter-3'
  warnings.warn(msg.format(msg_name, msg_units))

05:39:31 INFO: *********** Downloading to file 2014-07-07-SABGOM.nc ***********
05:39:37 INFO: [Found Water] Duck, NC
05:39:39 INFO: [Found Land] Oregon Inlet Marina, NC
05:39:41 INFO: [Found Water] Wrightsville Beach, NC
05:39:43 INFO: [Found Water] Springmaid Pier, SC
05:39:45 INFO: [Found Land] Oyster Landing (N Inlet Estuary), SC
05:39:47 INFO: [Found Land] Charleston, SC
05:39:50 INFO: [Found Land] Fernandina Beach, FL
05:39:52 INFO: [Found Land] Mayport (Bar Pilots Dock), FL
05:39:54 INFO: [Found Land] Dames Point, FL
05:39:56 INFO: [Found Land] Southbank Riverwalk, St Johns River, FL
05:39:58 INFO: [Found Land] I-295 Bridge, St Johns River, FL
05:40:00 INFO: [Found Land] Red Bay Point, St Johns River, FL
05:40:02 INFO: [Found Land] Racy Point, St Johns River, FL
05:40:06 INFO: [Found Land] Trident Pier, FL
05:40:11 INFO: [Found Land] Lake Worth Pier, FL
05:40:15 INFO: [Found Land] Vaca Key, FL
05:40:17 INFO: [Found Water] Key West, FL
05:40:20 INFO: [Found Land] Naples, FL
05:40:22 INFO: [Found Land] Fort Myers, FL
05:40:25 INFO: [Found Land] Port Manatee, FL
05:40:28 INFO: [Found Water] Cedar Key, FL
05:40:30 INFO: [Found Land] Apalachicola, FL
05:40:31 INFO: [Found Land] Panama City, FL
05:40:34 INFO: [Found Land] Pensacola, FL
05:40:44 INFO: 00:01:31
05:40:44 INFO: [SABGOM]: http://omgsrv1.meas.ncsu.edu:8080/thredds/dodsC/fmrc/sabgom/SABGOM_Forecast_Model_Run_Collection_best.ncd
05:40:44 INFO: 
[Reading url 5/7]: http://oos.soest.hawaii.edu/thredds/dodsC/hioos/tide_pac
05:40:48 INFO: *********** Downloading to file 2014-07-07-BTMPB.nc ************
05:40:48 WARNING: No data found for (-75.7467,36.1833) using max_dist=0.04.
05:40:48 WARNING: No data found for (-75.5483,35.795) using max_dist=0.04.
05:40:48 WARNING: No data found for (-77.7867,34.2133) using max_dist=0.04.
05:40:48 WARNING: No data found for (-78.9183,33.655) using max_dist=0.04.
05:40:48 WARNING: No data found for (-79.1867,33.3517) using max_dist=0.04.
05:40:48 WARNING: No data found for (-79.925,32.7817) using max_dist=0.04.
05:40:48 WARNING: No data found for (-81.465,30.6717) using max_dist=0.04.
05:40:48 WARNING: No data found for (-81.43,30.3967) using max_dist=0.04.
05:40:48 WARNING: No data found for (-81.5583,30.3867) using max_dist=0.04.
05:40:48 WARNING: No data found for (-81.6583,30.32) using max_dist=0.04.
05:40:48 WARNING: No data found for (-81.6917,30.1917) using max_dist=0.04.
05:40:48 WARNING: No data found for (-81.6283,29.9783) using max_dist=0.04.
05:40:48 WARNING: No data found for (-81.5483,29.8017) using max_dist=0.04.
05:40:48 WARNING: No data found for (-80.5931,28.4158) using max_dist=0.04.
05:40:48 WARNING: No data found for (-80.0333,26.6117) using max_dist=0.04.
05:40:48 WARNING: No data found for (-81.105,24.7117) using max_dist=0.04.
05:40:48 WARNING: No data found for (-81.8079,24.5557) using max_dist=0.04.
05:40:48 WARNING: No data found for (-81.8075,26.1317) using max_dist=0.04.
05:40:48 WARNING: No data found for (-81.8712,26.6477) using max_dist=0.04.
05:40:48 WARNING: No data found for (-82.5621,27.6387) using max_dist=0.04.
05:40:48 WARNING: No data found for (-83.0317,29.135) using max_dist=0.04.
05:40:48 WARNING: No data found for (-84.9817,29.7267) using max_dist=0.04.
05:40:48 WARNING: No data found for (-85.6669,30.1523) using max_dist=0.04.
05:40:48 WARNING: No data found for (-87.2112,30.4044) using max_dist=0.04.
05:40:48 INFO: 00:00:04
05:40:48 INFO: [BTMPB]: http://oos.soest.hawaii.edu/thredds/dodsC/hioos/tide_pac
05:40:48 INFO: 
[Reading url 6/7]: http://oos.soest.hawaii.edu/thredds/dodsC/pacioos/hycom/global
05:41:07 INFO: *********** Downloading to file 2014-07-07-HYCOM.nc ************
05:41:10 INFO: [Found Land] Duck, NC
05:41:15 INFO: [Found Water] Oregon Inlet Marina, NC
05:41:19 INFO: [Found Water] Wrightsville Beach, NC
05:41:19 WARNING: No data found for (-78.9183,33.655) using max_dist=0.04.
05:41:22 INFO: [Found Land] Oyster Landing (N Inlet Estuary), SC
05:41:25 INFO: [Found Land] Charleston, SC
05:41:29 INFO: [Found Land] Fernandina Beach, FL
05:41:32 INFO: [Found Land] Mayport (Bar Pilots Dock), FL
05:41:35 INFO: [Found Land] Dames Point, FL
05:41:38 INFO: [Found Land] Southbank Riverwalk, St Johns River, FL
05:41:40 INFO: [Found Land] I-295 Bridge, St Johns River, FL
05:41:43 INFO: [Found Land] Red Bay Point, St Johns River, FL
05:41:43 WARNING: No data found for (-81.5483,29.8017) using max_dist=0.04.
05:41:45 INFO: [Found Land] Trident Pier, FL
05:41:48 INFO: [Found Water] Lake Worth Pier, FL
05:41:51 INFO: [Found Land] Vaca Key, FL
05:41:54 INFO: [Found Land] Key West, FL
05:41:54 WARNING: No data found for (-81.8075,26.1317) using max_dist=0.04.
05:41:58 INFO: [Found Land] Fort Myers, FL
05:42:00 INFO: [Found Land] Port Manatee, FL
05:42:03 INFO: [Found Land] Cedar Key, FL
05:42:06 INFO: [Found Land] Apalachicola, FL
05:42:09 INFO: [Found Land] Panama City, FL
05:42:12 INFO: [Found Land] Pensacola, FL
05:42:12 INFO: 00:01:23
05:42:12 INFO: [HYCOM]: http://oos.soest.hawaii.edu/thredds/dodsC/pacioos/hycom/global
05:42:12 INFO: 
[Reading url 7/7]: http://tds.marine.rutgers.edu/thredds/dodsC/roms/espresso/2013_da/his_Best/ESPRESSO_Real-Time_v2_History_Best_Available_best.ncd
05:42:21 INFO: ********** Downloading to file 2014-07-07-ESPRESSO.nc **********
05:42:23 INFO: [Found Land] Duck, NC
05:42:26 INFO: [Found Water] Oregon Inlet Marina, NC
05:42:26 WARNING: No data found for (-77.7867,34.2133) using max_dist=0.04.
05:42:26 WARNING: No data found for (-78.9183,33.655) using max_dist=0.04.
05:42:26 WARNING: No data found for (-79.1867,33.3517) using max_dist=0.04.
05:42:26 WARNING: No data found for (-79.925,32.7817) using max_dist=0.04.
05:42:26 WARNING: No data found for (-81.465,30.6717) using max_dist=0.04.
05:42:26 WARNING: No data found for (-81.43,30.3967) using max_dist=0.04.
05:42:26 WARNING: No data found for (-81.5583,30.3867) using max_dist=0.04.
05:42:26 WARNING: No data found for (-81.6583,30.32) using max_dist=0.04.
05:42:26 WARNING: No data found for (-81.6917,30.1917) using max_dist=0.04.
05:42:26 WARNING: No data found for (-81.6283,29.9783) using max_dist=0.04.
05:42:26 WARNING: No data found for (-81.5483,29.8017) using max_dist=0.04.
05:42:26 WARNING: No data found for (-80.5931,28.4158) using max_dist=0.04.
05:42:26 WARNING: No data found for (-80.0333,26.6117) using max_dist=0.04.
05:42:26 WARNING: No data found for (-81.105,24.7117) using max_dist=0.04.
05:42:26 WARNING: No data found for (-81.8079,24.5557) using max_dist=0.04.
05:42:26 WARNING: No data found for (-81.8075,26.1317) using max_dist=0.04.
05:42:26 WARNING: No data found for (-81.8712,26.6477) using max_dist=0.04.
05:42:26 WARNING: No data found for (-82.5621,27.6387) using max_dist=0.04.
05:42:26 WARNING: No data found for (-83.0317,29.135) using max_dist=0.04.
05:42:26 WARNING: No data found for (-84.9817,29.7267) using max_dist=0.04.
05:42:26 WARNING: No data found for (-85.6669,30.1523) using max_dist=0.04.
05:42:26 WARNING: No data found for (-87.2112,30.4044) using max_dist=0.04.
05:42:28 INFO: 00:00:16
05:42:28 INFO: [ESPRESSO]: http://tds.marine.rutgers.edu/thredds/dodsC/roms/espresso/2013_da/his_Best/ESPRESSO_Real-Time_v2_History_Best_Available_best.ncd
05:42:42 WARNING: -c:27: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame

05:43:03 INFO: 590.127684116
05:43:03 INFO: EOF