Convert OOI Parsed JSON to NetCDF file

In [2]:

    

%matplotlib inline
import json
import pandas as pd
import numpy as np

from pyaxiom.netcdf.sensors import TimeSeries


    

In [6]:

    

infile = '/usgs/data2/notebook/data/20170130.superv.json'
infile = '/sand/usgs/users/rsignell/data/ooi/endurance/cg_proc/ce02shsm/D00004/buoy/pwrsys/20170208.pwrsys.json'

outfile = '/usgs/data2/notebook/data/20170130.superv.nc'
with open(infile) as jf:
    js = json.load(jf)
    df = pd.DataFrame({})
    for k, v in js.items():
        df[k] = v
    df['time'] = pd.to_datetime(df.time, unit='s')
    df['depth'] = 0.
df.head()


    

    
Out[6]:






  

    

      

      
battery_bank3_temperature
      
fuel_cell1_voltage
      
fuel_cell1_state
      
wind_turbine1_current
      
battery_bank2_current
      
cvt_voltage
      
external_voltage
      
solar_panel3_state
      
battery_bank1_voltage
      
wind_turbine1_state
      
...
      
solar_panel4_current
      
internal_current
      
battery_bank2_voltage
      
error_flag3
      
seawater_ground_state
      
fuel_cell2_current
      
battery_bank1_temperature
      
solar_panel4_voltage
      
dcl_date_time_string
      
depth
    
  
  

    

      
0
      
10.06
      
0.0
      
1
      
0.0
      
365.0
      
0.19
      
24.00
      
1
      
24.09
      
1
      
...
      
446.0
      
55.75
      
24.05
      
0f200000
      
0
      
0.0
      
10.06
      
24.02
      
2017/02/08 00:00:25.947
      
0.0
    
    

      
1
      
10.06
      
0.0
      
1
      
72.0
      
353.0
      
0.19
      
24.00
      
1
      
24.08
      
1
      
...
      
454.0
      
56.00
      
24.04
      
0f200000
      
0
      
0.0
      
10.06
      
24.02
      
2017/02/08 00:01:25.948
      
0.0
    
    

      
2
      
10.06
      
0.0
      
1
      
0.0
      
335.0
      
0.19
      
24.00
      
1
      
24.06
      
1
      
...
      
336.0
      
56.00
      
24.03
      
0f200000
      
0
      
0.0
      
10.06
      
24.00
      
2017/02/08 00:02:25.946
      
0.0
    
    

      
3
      
10.06
      
0.0
      
1
      
0.0
      
317.0
      
0.19
      
24.00
      
1
      
24.08
      
1
      
...
      
410.0
      
56.00
      
24.03
      
0f200000
      
0
      
0.0
      
10.06
      
24.00
      
2017/02/08 00:03:25.946
      
0.0
    
    

      
4
      
10.06
      
0.0
      
1
      
0.0
      
234.0
      
0.19
      
24.02
      
1
      
24.09
      
1
      
...
      
382.0
      
55.75
      
24.05
      
0f200000
      
0
      
0.0
      
10.06
      
24.02
      
2017/02/08 00:04:25.946
      
0.0
    
  

5 rows × 60 columns

In [7]:

    

df['solar_panel4_voltage'].plot();


    

In [8]:

    

df.index = df['time']
df['solar_panel4_voltage'].plot();


    

In [10]:

    

global_attributes = {
    'institution':'Oregon State University', 
    'title':'OOI CE02SHSM Pwrsys Data',
    'summary':'OOI Pwrsys data from Coastal Endurance Oregon Shelf Surface Mooring',
    'creator_name':'Chris Wingard',
    'creator_email':'cwingard@coas.oregonstate.edu',
    'creator_url':'http://ceoas.oregonstate.edu/ooi'
}


    

In [11]:

    

ts = TimeSeries(
    output_directory='.',
    latitude=44.64,
    longitude=-124.31,
    station_name='ce02shsm',
    global_attributes=global_attributes,
    times=df.time.values.astype(np.int64) // 10**9,
    verticals=df.depth.values,
    output_filename=outfile,
    vertical_positive='down'
)


    

In [12]:

    

for c in df.columns:
    if c in ts._nc.variables:
        print("Skipping '{}' (already in file)".format(c))
        continue
    if c in ['time', 'lat', 'lon', 'depth', 'cpm_date_time_string']:
        print("Skipping axis '{}' (already in file)".format(c))
        continue
    ts.add_variable(c, df[c].values)
    print("Added {}".format(c))


    

    




Added battery_bank3_temperature
Added fuel_cell1_voltage
Added fuel_cell1_state
Added wind_turbine1_current
Added battery_bank2_current
Added cvt_voltage
Added external_voltage
Added solar_panel3_state
Added battery_bank1_voltage
Added wind_turbine1_state
Added solar_panel1_current
Added fuel_cell2_state
Added external_current
Added wind_turbine2_current
Added solar_panel4_state
Added solar_panel1_state
Added main_current
Skipping 'time' (already in file)
Added cvt_temperature
Added battery_bank2_temperature
Added battery_bank3_voltage
Added wind_turbine2_voltage
Added battery_bank4_current






    




---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-12-efb911a9e934> in <module>()
      6         print("Skipping axis '{}' (already in file)".format(c))
      7         continue
----> 8     ts.add_variable(c, df[c].values)
      9     print("Added {}".format(c))
     10 

/home/usgs/miniconda/envs/IOOS3/lib/python3.5/site-packages/pyaxiom/netcdf/sensors/timeseries.py in add_variable(self, variable_name, values, times, verticals, sensor_vertical_datum, attributes, unlink_from_profile, fillvalue, raise_on_error, create_instrument_variable)
    352         logger.info("Setting values for {}...".format(variable_name))
    353         if len(used_values.shape) == 1:
--> 354             var = self._nc.createVariable(variable_name, get_type(used_values), ("time",), fill_value=fillvalue, chunksizes=(self.time_chunk,), zlib=True)
    355             self._nc.setncattr('ncei_template_version', 'NCEI_NetCDF_TimeSeries_Orthogonal_Template_v2.0')
    356             if vertical_axis.size == 1:

netCDF4/_netCDF4.pyx in netCDF4._netCDF4.Dataset.createVariable (netCDF4/_netCDF4.c:18749)()

netCDF4/_netCDF4.pyx in netCDF4._netCDF4.Variable.__init__ (netCDF4/_netCDF4.c:32122)()

AttributeError: cannot set _FillValue attribute for VLEN or compound variable

In [ ]:

    




    

In [9]:

    

import netCDF4
nc = netCDF4.Dataset(outfile)


    

In [10]:

    

nc


    

    
Out[10]:





<class 'netCDF4._netCDF4.Dataset'>
root group (NETCDF4 data model, file format HDF5):
    creator_email: cwingard@coas.oregonstate.edu
    creator_url: http://ceoas.oregonstate.edu/ooi
    institution: Oregon State University
    summary: OOI Supervisor data from Coastal Endurance Inshore Surface Mooring
    creator_name: Chris Wingard
    title: OOI CE01ISSM Supervisor Data
    Conventions: CF-1.6,ACDD-1.3
    date_created: 2017-02-01T16:08:00Z
    date_modified: 2017-02-01T16:08:00Z
    date_issued: 2017-02-01T16:08:00Z
    date_metadata_modified: 2017-02-01T16:08:00Z
    cdm_data_type: Station
    history: 2017-02-01T16:08:00Z - pyaxiom - File created using pyaxiom
    geospatial_lat_min: 44.65833
    geospatial_lat_max: 44.65833
    geospatial_lat_resolution: 0
    geospatial_lat_units: degrees_north
    geospatial_lon_min: -124.0953
    geospatial_lon_max: -124.0953
    geospatial_lon_resolution: 0
    geospatial_lon_units: degrees_east
    geospatial_bounds: POINT(-124.0953 44.65833)
    geospatial_bounds_crs: 4326
    time_coverage_start: 2017-01-30T00:26:49
    time_coverage_end: 2017-01-30T12:31:52
    time_coverage_duration: PT43503S
    time_coverage_resolution: PT60S
    geospatial_vertical_units: meters
    geospatial_vertical_positive: down
    featureType: timeSeries
    geospatial_vertical_resolution: 0
    geospatial_vertical_min: 0.0
    geospatial_vertical_max: 0.0
    ncei_template_version: NCEI_NetCDF_TimeSeries_Orthogonal_Template_v2.0
    dimensions(sizes): feature_type_instance(8), time(186)
    variables(dimensions): |S1 feature_type_instance(feature_type_instance), float64 latitude(), float64 longitude(), int32 crs(), int32 platform(), int32 time(time), float64 z(), int32 wake_power_count(time), int32 leak_detect_voltage2(time), int32 esw_power_state(time), int32 ground_fault_enable(time), int32 fwwf_power_flag(time), float64 sbd_power_state(time), float64 ground_fault_9522_fw(time), float64 iridium_current(time), int32 wake_code(time), int32 heartbeat_threshold(time), int32 error_flags(time), int32 heartbeat_delta(time), float64 sbd_message_pending(time), float64 main_voltage(time), float64 fwwf_current(time), float64 temperature1(time), int32 leak_detect_voltage1(time), float64 backup_battery_voltage(time), float64 gps_power_state(time), float64 wake_time_count(time), float64 backup_battery_current(time), float64 ground_fault_sbd(time), float64 fwwf_voltage(time), float64 pps_source(time), int32 dsl_power_state(time), float64 main_current(time), float64 temperature2(time), float64 humidity(time), float64 iridium_voltage(time), float64 ground_fault_main(time), float64 ground_fault_gps(time), int32 iridium_power_state(time), int32 dcl_power_state(time), int32 leak_detect_enable(time), int32 fwwf_power_state(time), float64 pressure(time), int32 heartbeat_enable(time), int32 iridium_error_flag(time)
    groups: 

In [11]:

    

nc.close()


    

In [12]:

    

import netCDF4
nc = netCDF4.Dataset(outfile)


    

In [15]:

    

nc['z']


    

    
Out[15]:





<class 'netCDF4._netCDF4.Variable'>
float64 z()
    _FillValue: -9999.9
    valid_min: 0.0
    valid_max: 0.0
    grid_mapping: crs
    long_name: z of the sensor relative to the water surface
    standard_name: depth
    positive: down
    units: m
    axis: Z
unlimited dimensions: 
current shape = ()
filling on

In [16]:

    

nc['crs']


    

    
Out[16]:





<class 'netCDF4._netCDF4.Variable'>
int32 crs()
    long_name: http://www.opengis.net/def/crs/EPSG/0/4326
    grid_mapping_name: latitude_longitude
    epsg_code: EPSG:4326
    semi_major_axis: 6378137.0
    inverse_flattening: 298.257223563
unlimited dimensions: 
current shape = ()
filling on, default _FillValue of -2147483647 used

In [19]:

    

nc['iridium_current']


    

    
Out[19]:





<class 'netCDF4._netCDF4.Variable'>
float64 iridium_current(time)
    _FillValue: -9999.9
    coordinates: time z latitude longitude
    missing_value: -9999.9
    grid_mapping: crs
    platform: platform
    ancillary_variables: platform
    coverage_content_type: physicalMeasurement
unlimited dimensions: 
current shape = (186,)
filling on

In [ ]: