Data and Visualization Integration Via Web-based Resources

Massimo Di Stefano

Software Engineer

Rensselaer Polytechnic Institute, Troy, NY

Woods Hole Oceanographic Institution, Woods Hole, MA

1991 The Perfect Storm

Follow up from the Rich Signell talk

Dataset provided by Signell Richard - USGS

Background


In [1]:
from IPython.core.display import HTML
HTML("<iframe src='http://www.ncdc.noaa.gov/oa/satellite/satelliteseye/cyclones/pfctstorm91/pfctstorm.html' width='700' height='600'></iframe>")


Out[1]:

In [2]:
from IPython import parallel

rc = parallel.Client()
view = rc.load_balanced_view()

In [3]:
%%px

import sys
import os

from pylab import *
import netCDF4
import datetime as dt

import numpy.ma as ma
from mpl_toolkits.basemap import Basemap

In [4]:
import sys
import os

from pylab import *
import netCDF4
import datetime as dt

import numpy.ma as ma
from mpl_toolkits.basemap import Basemap

In [5]:
def perdelta(start, end, delta):
    curr = start
    while curr < end:
        yield curr
        curr += delta
        
    
def plotWave(args):
    start = args["date"]
    url = args["url"]
    box = args["box"]
    vname = args["vname"]
    pvar = args["pvar"]
    uu = args["u"]
    vv = args["v"]
    directory = args["savepath"]
    nc = netCDF4.Dataset(url)
    lat = nc.variables['lat'][:]
    lon = nc.variables['lon'][:]
    u = nc.variables[str(uu)]
    v = nc.variables[str(vv)]
    time_var = nc.variables[str(pvar)]
    dtime = netCDF4.num2date(time_var[:],time_var.units)
    istart = netCDF4.date2index(start,time_var,select='nearest')
    if box is not None:
        bi=(lon>=box[0])&(lon<=box[2])
        bj=(lat>=box[1])&(lat<=box[3])
        var = nc.variables[vname][istart,bj,bi]
        uin = u[istart,bj,bi]
        vin = v[istart,bj,bi]
    else:
        var = nc.variables[vname][istart,:,:]
        uin = u[istart,:,:]
        vin = v[istart,:,:]
    hs = ma.masked_where(np.isnan(var),var)
    figure(figsize=(12,12)) 
    m = Basemap(resolution='i', llcrnrlon=-100.0,llcrnrlat=0.0,urcrnrlon=-50.0,urcrnrlat=60.0, \
                lat_0=20.,lon_0=-60.,lat_ts=20.)
    m.drawcoastlines(color='#6D5F47', linewidth=.4)
    m.drawrivers(color='#3366FF', linewidth=.4)
    m.fillcontinents(color='#99CC33',lake_color='#85A6D9')
    if box is not None:
        ff = plt.pcolormesh(lon[bi], lat[bj], hs, vmin=0, vmax=8)
        plt.contour(lon[bi], lat[bj], hs, 20, colors='w', linestyles='-', linewidth=0.1)
        #plt.barbs(lon[bi], lat[bj], uin, vin, length=5, barbcolor='k', flagcolor='r', linewidth=0.5)
        lons, lats = np.meshgrid(lon[bi], lat[bj])
        x, y = m(lons, lats)
        plt.quiver(x[::5,::5], y[::5,::5], uin[::5,::5], vin[::5,::5], scale=200);
    else :
        ff = plt.pcolormesh(lon[:], lat[:], hs, vmin=0, vmax=8)
        plt.contour(lon[:], lat[:], hs, 10, colors='w', linestyles='-', linewidth=0.1)
        lons, lats = np.meshgrid(lon[:], lat[:])
        x, y = m(lons, lats)
        plt.quiver(x[::5,::5], y[::5,::5], uin[::5,::5], vin[::5,::5], scale=400);
    #plt.colorbar()
    parallels = np.arange(0.,60,5.)
    meridians = np.arange(-100.,-50.,5.)
    m.drawparallels(parallels,labels=[1,0,0,0],fontsize=10);
    m.drawmeridians(meridians,labels=[0,0,0,1],fontsize=10);
    cb = m.colorbar(ff,"bottom", size="4%", pad="3%")
    cb.set_label('wave height (m)')
    title('The Perfect Storm - %s' % start.strftime('%Y-%m-%d-%H-%M'))
    name = str(pvar)+'_0000'+start.strftime('%Y%m%d%H%M')+'.png'
    filesave = os.path.join(directory,name)
    plt.savefig(filesave)
    print 'processed %s' % start

def gen_dates4Now(past,url, box, vname, pvar):
    for result in perdelta(dt.datetime.now() - dt.timedelta(days=past), dt.datetime.now(), dt.timedelta(hours=1)):
        yield {"date":result,"url":url, "box":box, "vname":vname, "pvar":pvar}
        
def gen_dates(tstart,tend,url, box, vname, pvar, u, v, tres, savepath):
    day = tend.toordinal() - tstart.toordinal()
    for result in perdelta(tend - dt.timedelta(days=day), tend, dt.timedelta(hours=tres)):
        yield {"date":result,"url":url, "box":box, "vname":vname, "pvar":pvar, "u":u, "v":v, 'savepath':savepath}

In [6]:
#url='http://geoport.whoi.edu/thredds/dodsC/fmrc/NCEP/ww3/cfsr/4m/best'
url='http://geoport.whoi.edu/thredds/dodsC/fmrc/NCEP/ww3/cfsr/10m/best'
box = None
#box = [-71.5, 39.5, -63.0, 46.0]
vname = 'Significant_height_of_combined_wind_waves_and_swell_surface'
u = 'u-component_of_wind_surface'
v = 'v-component_of_wind_surface'
pvar='time'
tres = 1
savepath = '/Users/epi/mp_video_unh'
tstart = dt.datetime(1991,10,24,12)
tend = dt.datetime(1991,11,4,12)

In [7]:
!mkdir '/Users/epi/mp_video_unh'


mkdir: /Users/epi/mp_video_unh: File exists

In [8]:
results = view.map(plotWave, list(gen_dates(tstart, tend, url, box, vname, pvar, u, v, tres, savepath)), ordered=False)
results


Out[8]:
<AsyncMapResult: plotWave>

In [9]:
for r in results:
    print r
    sys.stdout.flush()


None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None
None

In [10]:
results.display_outputs()


[stdout:1] processed 1991-10-24 12:00:00
[stdout:5] processed 1991-10-24 13:00:00
[stdout:0] processed 1991-10-24 14:00:00
[stdout:6] processed 1991-10-24 15:00:00
[stdout:4] processed 1991-10-24 16:00:00
[stdout:3] processed 1991-10-24 17:00:00
[stdout:7] processed 1991-10-24 18:00:00
[stdout:2] processed 1991-10-24 19:00:00
[stdout:4] processed 1991-10-24 20:00:00
[stdout:5] processed 1991-10-24 21:00:00
[stdout:7] processed 1991-10-24 22:00:00
[stdout:0] processed 1991-10-24 23:00:00
[stdout:3] processed 1991-10-25 00:00:00
[stdout:6] processed 1991-10-25 01:00:00
[stdout:2] processed 1991-10-25 02:00:00
[stdout:1] processed 1991-10-25 03:00:00
[stdout:4] processed 1991-10-25 04:00:00
[stdout:0] processed 1991-10-25 05:00:00
[stdout:5] processed 1991-10-25 06:00:00
[stdout:3] processed 1991-10-25 07:00:00
[stdout:6] processed 1991-10-25 08:00:00
[stdout:7] processed 1991-10-25 09:00:00
[stdout:2] processed 1991-10-25 10:00:00
[stdout:1] processed 1991-10-25 11:00:00
[stdout:4] processed 1991-10-25 12:00:00
[stdout:0] processed 1991-10-25 13:00:00
[stdout:6] processed 1991-10-25 14:00:00
[stdout:3] processed 1991-10-25 15:00:00
[stdout:1] processed 1991-10-25 16:00:00
[stdout:5] processed 1991-10-25 17:00:00
[stdout:2] processed 1991-10-25 18:00:00
[stdout:7] processed 1991-10-25 19:00:00
[stdout:4] processed 1991-10-25 20:00:00
[stdout:0] processed 1991-10-25 21:00:00
[stdout:3] processed 1991-10-25 22:00:00
[stdout:6] processed 1991-10-25 23:00:00
[stdout:5] processed 1991-10-26 00:00:00
[stdout:1] processed 1991-10-26 01:00:00
[stdout:2] processed 1991-10-26 02:00:00
[stdout:7] processed 1991-10-26 03:00:00
[stdout:3] processed 1991-10-26 04:00:00
[stdout:0] processed 1991-10-26 05:00:00
[stdout:4] processed 1991-10-26 06:00:00
[stdout:6] processed 1991-10-26 07:00:00
[stdout:2] processed 1991-10-26 08:00:00
[stdout:1] processed 1991-10-26 09:00:00
[stdout:5] processed 1991-10-26 10:00:00
[stdout:7] processed 1991-10-26 11:00:00
[stdout:3] processed 1991-10-26 12:00:00
[stdout:2] processed 1991-10-26 13:00:00
[stdout:4] processed 1991-10-26 14:00:00
[stdout:0] processed 1991-10-26 15:00:00
[stdout:6] processed 1991-10-26 16:00:00
[stdout:1] processed 1991-10-26 17:00:00
[stdout:5] processed 1991-10-26 18:00:00
[stdout:7] processed 1991-10-26 19:00:00
[stdout:3] processed 1991-10-26 20:00:00
[stdout:4] processed 1991-10-26 21:00:00
[stdout:2] processed 1991-10-26 22:00:00
[stdout:0] processed 1991-10-26 23:00:00
[stdout:6] processed 1991-10-27 00:00:00
[stdout:7] processed 1991-10-27 01:00:00
[stdout:1] processed 1991-10-27 02:00:00
[stdout:5] processed 1991-10-27 03:00:00
[stdout:3] processed 1991-10-27 04:00:00
[stdout:2] processed 1991-10-27 05:00:00
[stdout:0] processed 1991-10-27 06:00:00
[stdout:4] processed 1991-10-27 07:00:00
[stdout:6] processed 1991-10-27 08:00:00
[stdout:1] processed 1991-10-27 09:00:00
[stdout:5] processed 1991-10-27 10:00:00
[stdout:7] processed 1991-10-27 11:00:00
[stdout:3] processed 1991-10-27 12:00:00
[stdout:2] processed 1991-10-27 13:00:00
[stdout:0] processed 1991-10-27 14:00:00
[stdout:5] processed 1991-10-27 15:00:00
[stdout:6] processed 1991-10-27 16:00:00
[stdout:1] processed 1991-10-27 17:00:00
[stdout:7] processed 1991-10-27 18:00:00
[stdout:4] processed 1991-10-27 19:00:00
[stdout:3] processed 1991-10-27 20:00:00
[stdout:2] processed 1991-10-27 21:00:00
[stdout:0] processed 1991-10-27 22:00:00
[stdout:5] processed 1991-10-27 23:00:00
[stdout:6] processed 1991-10-28 00:00:00
[stdout:1] processed 1991-10-28 01:00:00
[stdout:7] processed 1991-10-28 02:00:00
[stdout:4] processed 1991-10-28 03:00:00
[stdout:3] processed 1991-10-28 04:00:00
[stdout:2] processed 1991-10-28 05:00:00
[stdout:0] processed 1991-10-28 06:00:00
[stdout:5] processed 1991-10-28 07:00:00
[stdout:6] processed 1991-10-28 08:00:00
[stdout:1] processed 1991-10-28 09:00:00
[stdout:7] processed 1991-10-28 10:00:00
[stdout:4] processed 1991-10-28 11:00:00
[stdout:3] processed 1991-10-28 12:00:00
[stdout:0] processed 1991-10-28 13:00:00
[stdout:2] processed 1991-10-28 14:00:00
[stdout:5] processed 1991-10-28 15:00:00
[stdout:6] processed 1991-10-28 16:00:00
[stdout:1] processed 1991-10-28 17:00:00
[stdout:7] processed 1991-10-28 18:00:00
[stdout:4] processed 1991-10-28 19:00:00
[stdout:3] processed 1991-10-28 20:00:00
[stdout:0] processed 1991-10-28 21:00:00
[stdout:5] processed 1991-10-28 22:00:00
[stdout:2] processed 1991-10-28 23:00:00
[stdout:6] processed 1991-10-29 00:00:00
[stdout:7] processed 1991-10-29 01:00:00
[stdout:1] processed 1991-10-29 02:00:00
[stdout:4] processed 1991-10-29 03:00:00
[stdout:3] processed 1991-10-29 04:00:00
[stdout:5] processed 1991-10-29 05:00:00
[stdout:0] processed 1991-10-29 06:00:00
[stdout:2] processed 1991-10-29 07:00:00
[stdout:6] processed 1991-10-29 08:00:00
[stdout:7] processed 1991-10-29 09:00:00
[stdout:1] processed 1991-10-29 10:00:00
[stdout:4] processed 1991-10-29 11:00:00
[stdout:3] processed 1991-10-29 12:00:00
[stdout:5] processed 1991-10-29 13:00:00
[stdout:2] processed 1991-10-29 14:00:00
[stdout:0] processed 1991-10-29 15:00:00
[stdout:6] processed 1991-10-29 16:00:00
[stdout:7] processed 1991-10-29 17:00:00
[stdout:1] processed 1991-10-29 18:00:00
[stdout:4] processed 1991-10-29 19:00:00
[stdout:3] processed 1991-10-29 20:00:00
[stdout:5] processed 1991-10-29 21:00:00
[stdout:2] processed 1991-10-29 22:00:00
[stdout:0] processed 1991-10-29 23:00:00
[stdout:6] processed 1991-10-30 00:00:00
[stdout:7] processed 1991-10-30 01:00:00
[stdout:1] processed 1991-10-30 02:00:00
[stdout:4] processed 1991-10-30 03:00:00
[stdout:3] processed 1991-10-30 04:00:00
[stdout:5] processed 1991-10-30 05:00:00
[stdout:2] processed 1991-10-30 06:00:00
[stdout:6] processed 1991-10-30 07:00:00
[stdout:7] processed 1991-10-30 08:00:00
[stdout:0] processed 1991-10-30 09:00:00
[stdout:1] processed 1991-10-30 10:00:00
[stdout:4] processed 1991-10-30 11:00:00
[stdout:3] processed 1991-10-30 12:00:00
[stdout:5] processed 1991-10-30 13:00:00
[stdout:2] processed 1991-10-30 14:00:00
[stdout:6] processed 1991-10-30 15:00:00
[stdout:7] processed 1991-10-30 16:00:00
[stdout:1] processed 1991-10-30 17:00:00
[stdout:0] processed 1991-10-30 18:00:00
[stdout:4] processed 1991-10-30 19:00:00
[stdout:3] processed 1991-10-30 20:00:00
[stdout:5] processed 1991-10-30 21:00:00
[stdout:2] processed 1991-10-30 22:00:00
[stdout:7] processed 1991-10-30 23:00:00
[stdout:6] processed 1991-10-31 00:00:00
[stdout:0] processed 1991-10-31 01:00:00
[stdout:4] processed 1991-10-31 02:00:00
[stdout:1] processed 1991-10-31 03:00:00
[stdout:3] processed 1991-10-31 04:00:00
[stdout:5] processed 1991-10-31 05:00:00
[stdout:2] processed 1991-10-31 06:00:00
[stdout:7] processed 1991-10-31 07:00:00
[stdout:6] processed 1991-10-31 08:00:00
[stdout:0] processed 1991-10-31 09:00:00
[stdout:1] processed 1991-10-31 10:00:00
[stdout:4] processed 1991-10-31 11:00:00
[stdout:3] processed 1991-10-31 12:00:00
[stdout:5] processed 1991-10-31 13:00:00
[stdout:2] processed 1991-10-31 14:00:00
[stdout:7] processed 1991-10-31 15:00:00
[stdout:6] processed 1991-10-31 16:00:00
[stdout:1] processed 1991-10-31 17:00:00
[stdout:0] processed 1991-10-31 18:00:00
[stdout:4] processed 1991-10-31 19:00:00
[stdout:3] processed 1991-10-31 20:00:00
[stdout:5] processed 1991-10-31 21:00:00
[stdout:2] processed 1991-10-31 22:00:00
[stdout:7] processed 1991-10-31 23:00:00
[stdout:6] processed 1991-11-01 00:00:00
[stdout:1] processed 1991-11-01 01:00:00
[stdout:4] processed 1991-11-01 02:00:00
[stdout:0] processed 1991-11-01 03:00:00
[stdout:3] processed 1991-11-01 04:00:00
[stdout:5] processed 1991-11-01 05:00:00
[stdout:7] processed 1991-11-01 06:00:00
[stdout:2] processed 1991-11-01 07:00:00
[stdout:6] processed 1991-11-01 08:00:00
[stdout:1] processed 1991-11-01 09:00:00
[stdout:4] processed 1991-11-01 10:00:00
[stdout:0] processed 1991-11-01 11:00:00
[stdout:3] processed 1991-11-01 12:00:00
[stdout:5] processed 1991-11-01 13:00:00
[stdout:7] processed 1991-11-01 14:00:00
[stdout:2] processed 1991-11-01 15:00:00
[stdout:6] processed 1991-11-01 16:00:00
[stdout:1] processed 1991-11-01 17:00:00
[stdout:0] processed 1991-11-01 18:00:00
[stdout:4] processed 1991-11-01 19:00:00
[stdout:3] processed 1991-11-01 20:00:00
[stdout:5] processed 1991-11-01 21:00:00
[stdout:7] processed 1991-11-01 22:00:00
[stdout:2] processed 1991-11-01 23:00:00
[stdout:6] processed 1991-11-02 00:00:00
[stdout:1] processed 1991-11-02 01:00:00
[stdout:0] processed 1991-11-02 02:00:00
[stdout:4] processed 1991-11-02 03:00:00
[stdout:3] processed 1991-11-02 04:00:00
[stdout:5] processed 1991-11-02 05:00:00
[stdout:7] processed 1991-11-02 06:00:00
[stdout:2] processed 1991-11-02 07:00:00
[stdout:6] processed 1991-11-02 08:00:00
[stdout:1] processed 1991-11-02 09:00:00
[stdout:4] processed 1991-11-02 10:00:00
[stdout:0] processed 1991-11-02 11:00:00
[stdout:3] processed 1991-11-02 12:00:00
[stdout:5] processed 1991-11-02 13:00:00
[stdout:7] processed 1991-11-02 14:00:00
[stdout:2] processed 1991-11-02 15:00:00
[stdout:6] processed 1991-11-02 16:00:00
[stdout:1] processed 1991-11-02 17:00:00
[stdout:4] processed 1991-11-02 18:00:00
[stdout:0] processed 1991-11-02 19:00:00
[stdout:3] processed 1991-11-02 20:00:00
[stdout:5] processed 1991-11-02 21:00:00
[stdout:7] processed 1991-11-02 22:00:00
[stdout:2] processed 1991-11-02 23:00:00
[stdout:6] processed 1991-11-03 00:00:00
[stdout:1] processed 1991-11-03 01:00:00
[stdout:4] processed 1991-11-03 02:00:00
[stdout:0] processed 1991-11-03 03:00:00
[stdout:3] processed 1991-11-03 04:00:00
[stdout:5] processed 1991-11-03 05:00:00
[stdout:7] processed 1991-11-03 06:00:00
[stdout:2] processed 1991-11-03 07:00:00
[stdout:6] processed 1991-11-03 08:00:00
[stdout:4] processed 1991-11-03 09:00:00
[stdout:1] processed 1991-11-03 10:00:00
[stdout:3] processed 1991-11-03 11:00:00
[stdout:0] processed 1991-11-03 12:00:00
[stdout:5] processed 1991-11-03 13:00:00
[stdout:7] processed 1991-11-03 14:00:00
[stdout:2] processed 1991-11-03 15:00:00
[stdout:6] processed 1991-11-03 16:00:00
[stdout:4] processed 1991-11-03 17:00:00
[stdout:1] processed 1991-11-03 18:00:00
[stdout:3] processed 1991-11-03 19:00:00
[stdout:0] processed 1991-11-03 20:00:00
[stdout:5] processed 1991-11-03 21:00:00
[stdout:7] processed 1991-11-03 22:00:00
[stdout:2] processed 1991-11-03 23:00:00
[stdout:4] processed 1991-11-04 00:00:00
[stdout:6] processed 1991-11-04 01:00:00
[stdout:1] processed 1991-11-04 02:00:00
[stdout:3] processed 1991-11-04 03:00:00
[stdout:0] processed 1991-11-04 04:00:00
[stdout:5] processed 1991-11-04 05:00:00
[stdout:7] processed 1991-11-04 06:00:00
[stdout:2] processed 1991-11-04 07:00:00
[stdout:4] processed 1991-11-04 08:00:00
[stdout:6] processed 1991-11-04 09:00:00
[stdout:1] processed 1991-11-04 10:00:00
[stdout:0] processed 1991-11-04 11:00:00

In [11]:
cd /Users/epi/mp_video_unh


/Users/epi/mp_video_unh

In [12]:
os.system('cat *.png | ffmpeg -r 24 -y -f image2pipe -c:v png -i - -c:v libx264 -preset ultrafast -qp 0 -movflags +faststart -pix_fmt yuv420p time.mkv')


Out[12]:
0

In [13]:
from IPython.core.display import HTML
video = open("time.mkv", "rb").read()
video_encoded = video.encode("base64")
video_tag = '<video autoplay loop controls alt="test" src="data:video/x-m4v;base64,{0}">'.format(video_encoded)
HTML(data=video_tag)


Out[13]:

In [ ]:
lines = !jist -p The_Perfect_Storm_1991.ipynb
print lines[0].replace("https://gist.github.com", "http://nbviewer.ipython.org")