In [2]:
%matplotlib inline

from dask.distributed import Client
import numpy as np
import pandas as pd
import xarray as xr

import importlib
import ELMlib
importlib.reload(ELMlib)


Out[2]:
<module 'ELMlib' from '/home/hmetzler/SOIL-R/Manuscripts/Berkeley/2019/Code/ELMlib.py'>

In [2]:
client = Client(n_workers=2, threads_per_worker=2, memory_limit='1GB')
client


Out[2]:

Client

Cluster

  • Workers: 2
  • Cores: 4
  • Memory: 2.00 GB

In [3]:
#ds = xr.open_dataset('../Data/14C_spinup_holger_fire.2x2_small.nc')
from netCDF4 import Dataset
ds = Dataset('../Data/14C_spinup_holger_fire.2x2_small.nc')

In [4]:
#lat, lon = ds.coords['lat'], ds.coords['lon']
lat, lon = ds['lat'][:], ds['lon'][:]
lat_indices, lon_indices = np.meshgrid(
    range(len(lat)),
    range(len(lon)),
    indexing='ij'
)

In [5]:
lats, lons = np.meshgrid(lat, lon, indexing='ij')
df_pd = pd.DataFrame({
    'cell_nr': range(len(lat)*len(lon)),
    'lat_index': lat_indices.flatten(),
    'lon_index': lon_indices.flatten(),
    'lat': lats.flatten(),
    'lon': lons.flatten()
})
df_pd


Out[5]:
cell_nr lat_index lon_index lat lon
0 0 0 0 36.947369 272.5
1 1 0 1 36.947369 275.0
2 2 1 0 38.842106 272.5
3 3 1 1 38.842106 275.0

In [6]:
import dask.dataframe as dask_df

In [7]:
df_dask = dask_df.from_pandas(df_pd, npartitions=4)
df_dask


Out[7]:
Dask DataFrame Structure:
cell_nr lat_index lon_index lat lon
npartitions=3
0 int64 int64 int64 float32 float32
1 ... ... ... ... ...
2 ... ... ... ... ...
3 ... ... ... ... ...
Dask Name: from_pandas, 3 tasks

In [8]:
parameter_set = ELMlib.load_parameter_set(
    ds_filename = '../Data/14C_spinup_holger_fire.2x2_small.nc',
    time_shift  = -198*365,
    nstep       = 10
)

In [14]:
def func(line):
    #pass  
    location_dict = {
        'cell_nr':   int(line.cell_nr),
        'lat_index': int(line.lat_index),
        'lon_index': int(line.lon_index),    
    }
    
    #print(line, flush=True)
    cell_nr, log, xs_12C_data, us_12C_data, rs_12C_data= ELMlib.load_model_12C_data(parameter_set, location_dict)
    #return (1, 'log', [1,2,3], [4,5,6], [7,8,9])
    return cell_nr, log, xs_12C_data, us_12C_data, rs_12C_data

df_dask_2 = df_dask.apply(func, axis=1, meta=('A', 'object'))

In [15]:
df_dask_2.compute()
type(df_dask_2)


CWD CWDC_vr
Litter 1 LITR1C_vr
Litter 2 LITR2C_vr
Litter 3 LITR3C_vr
Soil 1 SOIL1C_vr
Soil 2 SOIL2C_vr
Soil 3 SOIL3C_vr
CWD CWDC_vr
CWD CWDC_vr
Litter 1 LITR1C_vrLitter 1 LITR1C_vr

CWD fire_mortality_c_to_cwdc
CWD gap_mortality_c_to_cwdc
Litter 2 LITR2C_vr
Litter 2 LITR2C_vr
Litter 3 LITR3C_vr
CWD harvest_c_to_cwdc
Litter 3 LITR3C_vr
Litter 1 gap_mortality_c_to_litr_met_c
Soil 1Litter 1 harvest_c_to_litr_met_c
 SOIL1C_vr
Soil 1 SOIL1C_vr
Soil 2 SOIL2C_vr
Litter 1 m_c_to_litr_met_fire
Soil 2 SOIL2C_vr
Litter 1Soil 3  phenology_c_to_litr_met_cSOIL3C_vr

Soil 3 SOIL3C_vr
CWDLitter 2 fire_mortality_c_to_cwdc
CWD  fire_mortality_c_to_cwdc
gap_mortality_c_to_litr_cel_c
CWD gap_mortality_c_to_cwdc
Litter 2 harvest_c_to_litr_cel_c
CWD gap_mortality_c_to_cwdc
CWD harvest_c_to_cwdc
Litter 2 m_c_to_litr_cel_fire
CWD harvest_c_to_cwdc
Litter 1 gap_mortality_c_to_litr_met_c
Litter 2 phenology_c_to_litr_cel_c
Litter 3 gap_mortality_c_to_litr_lig_c
Litter 1 harvest_c_to_litr_met_c
Litter 1 gap_mortality_c_to_litr_met_c
Litter 1 m_c_to_litr_met_fire
Litter 1 harvest_c_to_litr_met_c
Litter 1 phenology_c_to_litr_met_c
Litter 3 harvest_c_to_litr_lig_c
Litter 3 m_c_to_litr_lig_fire
Litter 2 gap_mortality_c_to_litr_cel_c
Litter 2 harvest_c_to_litr_cel_c
Litter 3 phenology_c_to_litr_lig_c
Litter 1 m_c_to_litr_met_fire
Litter 2 m_c_to_litr_cel_fire
Litter 1 phenology_c_to_litr_met_c
CWD CWD_HR_L2_vr
Litter 2 gap_mortality_c_to_litr_cel_c
CWDLitter 2 phenology_c_to_litr_cel_c
 CWD_HR_L3_vr
Litter 2 harvest_c_to_litr_cel_c
CWD M_CWDC_TO_FIRE_vr
Litter 3 gap_mortality_c_to_litr_lig_c
Litter 2 m_c_to_litr_cel_fire
Litter 1 LITR1_HR_vr
Litter 3 harvest_c_to_litr_lig_c
Litter 2 phenology_c_to_litr_cel_c
Litter 1 M_LITR1C_TO_FIRE_vr
Litter 3 m_c_to_litr_lig_fire
Litter 2Litter 3Litter 3 LITR2_HR_vr
 phenology_c_to_litr_lig_c
 gap_mortality_c_to_litr_lig_c
Litter 2 Litter 3M_LITR2C_TO_FIRE_vr
 harvest_c_to_litr_lig_c
Litter 3Litter 3 LITR3_HR_vr
 m_c_to_litr_lig_fire
CWD CWD_HR_L2_vr
CWD CWD_HR_L3_vr
Litter 3 phenology_c_to_litr_lig_c
Litter 3 M_LITR3C_TO_FIRE_vr
CWDCWD M_CWDC_TO_FIRE_vr
 CWD_HR_L2_vr
Soil 1 SOIL1_HR_S2_vr
CWD CWD_HR_L3_vr
Litter 1 LITR1_HR_vr
Soil 1 SOIL1_HR_S3_vr
Litter 1 M_LITR1C_TO_FIRE_vr
CWD M_CWDC_TO_FIRE_vr
Litter 2 LITR2_HR_vr
Soil 2 SOIL2_HR_S1_vr
Litter 2 M_LITR2C_TO_FIRE_vr
Litter 1 LITR1_HR_vr
Soil 2 SOIL2_HR_S3_vr
Soil 3 SOIL3_HR_vr
Litter 3 LITR3_HR_vr
Litter 1 M_LITR1C_TO_FIRE_vr
Litter 3 M_LITR3C_TO_FIRE_vr
Litter 2 LITR2_HR_vr
Soil 1 SOIL1_HR_S2_vr
Litter 2 M_LITR2C_TO_FIRE_vr
Soil 1 SOIL1_HR_S3_vr
Litter 3 LITR3_HR_vr
Litter 3Soil 2 SOIL2_HR_S1_vr
 M_LITR3C_TO_FIRE_vr
Soil 2Soil 1 SOIL2_HR_S3_vr SOIL1_HR_S2_vr

Soil 1 SOIL1_HR_S3_vr
Soil 3 SOIL3_HR_vr
Soil 2 SOIL2_HR_S1_vr
Soil 2 SOIL2_HR_S3_vr
Soil 3 SOIL3_HR_vr
CWD CWDC_vr
Litter 1 LITR1C_vr
Litter 2 LITR2C_vr
Litter 3 LITR3C_vr
Soil 1 SOIL1C_vr
Soil 2 SOIL2C_vr
Soil 3 SOIL3C_vr
CWD fire_mortality_c_to_cwdc
CWD gap_mortality_c_to_cwdc
CWD harvest_c_to_cwdc
Litter 1 gap_mortality_c_to_litr_met_c
Litter 1 harvest_c_to_litr_met_c
Litter 1 m_c_to_litr_met_fire
Litter 1 phenology_c_to_litr_met_c
Litter 2 gap_mortality_c_to_litr_cel_c
Litter 2 harvest_c_to_litr_cel_c
Litter 2 m_c_to_litr_cel_fire
Litter 2 phenology_c_to_litr_cel_c
Litter 3 gap_mortality_c_to_litr_lig_c
Litter 3 harvest_c_to_litr_lig_c
Litter 3 m_c_to_litr_lig_fire
Litter 3 phenology_c_to_litr_lig_c
CWD CWD_HR_L2_vr
CWD CWD_HR_L3_vr
CWD M_CWDC_TO_FIRE_vr
Litter 1 LITR1_HR_vr
Litter 1 M_LITR1C_TO_FIRE_vr
Litter 2 LITR2_HR_vr
Litter 2 M_LITR2C_TO_FIRE_vr
Litter 3 LITR3_HR_vr
Litter 3 M_LITR3C_TO_FIRE_vr
Soil 1 SOIL1_HR_S2_vr
Soil 1 SOIL1_HR_S3_vr
Soil 2 SOIL2_HR_S1_vr
Soil 2 SOIL2_HR_S3_vr
Soil 3 SOIL3_HR_vr
Out[15]:
dask.dataframe.core.Series

In [16]:
df_dask_2


Out[16]:
Dask Series Structure:
npartitions=3
0    object
1       ...
2       ...
3       ...
Name: A, dtype: object
Dask Name: apply, 6 tasks

In [17]:
#list(df_dask_2)

In [18]:
pd.DataFrame(list(df_dask_2), columns=('cell_nr', 'log', 'xs_12C_data', 'us_12C_data', 'rs_12C_data'))


CWD CWDC_vr
Litter 1 LITR1C_vr
Litter 2 LITR2C_vr
Litter 3 LITR3C_vr
CWDCWD CWDC_vr
 CWDC_vr
Soil 1Litter 1 SOIL1C_vr
 LITR1C_vr
Litter 1 LITR1C_vr
Soil 2 SOIL2C_vr
Litter 2 LITR2C_vr
Litter 2 LITR2C_vr
Litter 3 LITR3C_vr
Soil 3 SOIL3C_vr
CWD fire_mortality_c_to_cwdc
Litter 3 LITR3C_vr
Soil 1 SOIL1C_vr
Soil 1 SOIL1C_vr
CWD gap_mortality_c_to_cwdc
Soil 2Soil 2 SOIL2C_vr
 SOIL2C_vr
CWD harvest_c_to_cwdc
Soil 3 SOIL3C_vr
Soil 3 SOIL3C_vr
Litter 1 gap_mortality_c_to_litr_met_c
Litter 1 harvest_c_to_litr_met_c
CWD fire_mortality_c_to_cwdc
Litter 1 m_c_to_litr_met_fire
CWD fire_mortality_c_to_cwdc
CWD gap_mortality_c_to_cwdc
Litter 1 phenology_c_to_litr_met_c
CWD CWD harvest_c_to_cwdcgap_mortality_c_to_cwdc

Litter 2 gap_mortality_c_to_litr_cel_c
CWD harvest_c_to_cwdc
Litter 1 gap_mortality_c_to_litr_met_c
Litter 1 harvest_c_to_litr_met_c
Litter 1Litter 2 harvest_c_to_litr_cel_c
 gap_mortality_c_to_litr_met_c
Litter 1 m_c_to_litr_met_fire
Litter 1Litter 2 phenology_c_to_litr_met_c
 m_c_to_litr_cel_fire
Litter 2Litter 2Litter 1 phenology_c_to_litr_cel_c
 harvest_c_to_litr_met_c
 gap_mortality_c_to_litr_cel_c
Litter 2Litter 1 harvest_c_to_litr_cel_c
 m_c_to_litr_met_fireLitter 3
 gap_mortality_c_to_litr_lig_c
Litter 2 m_c_to_litr_cel_fire
Litter 3 harvest_c_to_litr_lig_c
Litter 1 phenology_c_to_litr_met_c
Litter 2 phenology_c_to_litr_cel_c
Litter 3 m_c_to_litr_lig_fire
Litter 2 gap_mortality_c_to_litr_cel_c
Litter 3 gap_mortality_c_to_litr_lig_c
Litter 3 phenology_c_to_litr_lig_c
Litter 3 harvest_c_to_litr_lig_c
Litter 2 harvest_c_to_litr_cel_c
CWD CWD_HR_L2_vr
Litter 2 m_c_to_litr_cel_fire
Litter 3 m_c_to_litr_lig_fire
Litter 3 phenology_c_to_litr_lig_c
CWD CWD_HR_L3_vr
Litter 2 phenology_c_to_litr_cel_c
CWD CWD_HR_L2_vr
CWD M_CWDC_TO_FIRE_vr
Litter 3 gap_mortality_c_to_litr_lig_c
CWD CWD_HR_L3_vr
Litter 1 LITR1_HR_vr
CWDLitter 3 harvest_c_to_litr_lig_c
 M_CWDC_TO_FIRE_vr
Litter 1 M_LITR1C_TO_FIRE_vr
Litter 3 m_c_to_litr_lig_fire
Litter 1 LITR1_HR_vr
Litter 2 LITR2_HR_vr
Litter 3 phenology_c_to_litr_lig_c
Litter 1 M_LITR1C_TO_FIRE_vr
Litter 2 M_LITR2C_TO_FIRE_vr
CWD CWD_HR_L2_vr
Litter 2 LITR2_HR_vr
Litter 3 Litter 2LITR3_HR_vr M_LITR2C_TO_FIRE_vr

CWD CWD_HR_L3_vr
Litter 3 LITR3_HR_vr
Litter 3 M_LITR3C_TO_FIRE_vr
CWD M_CWDC_TO_FIRE_vr
Soil 1 SOIL1_HR_S2_vr
Litter 3 M_LITR3C_TO_FIRE_vr
Litter 1 LITR1_HR_vr
Litter 1 M_LITR1C_TO_FIRE_vr
Soil 1 SOIL1_HR_S3_vr
Litter 2 LITR2_HR_vr
Soil 1 SOIL1_HR_S2_vr
Soil 2 SOIL2_HR_S1_vr
Soil 1 SOIL1_HR_S3_vr
Litter 2 M_LITR2C_TO_FIRE_vr
Soil 2 SOIL2_HR_S3_vr
Soil 2 SOIL2_HR_S1_vr
Soil 3 SOIL3_HR_vr
Litter 3 LITR3_HR_vr
Soil 2 SOIL2_HR_S3_vr
Soil 3Litter 3 SOIL3_HR_vr
 M_LITR3C_TO_FIRE_vr
Soil 1 SOIL1_HR_S2_vr
Soil 1 SOIL1_HR_S3_vr
Soil 2 SOIL2_HR_S1_vr
Soil 2 SOIL2_HR_S3_vr
Soil 3 SOIL3_HR_vr
CWD CWDC_vr
Litter 1 LITR1C_vr
Litter 2 LITR2C_vr
Litter 3 LITR3C_vr
Soil 1 SOIL1C_vr
Soil 2 SOIL2C_vr
Soil 3 SOIL3C_vr
CWD fire_mortality_c_to_cwdc
CWD gap_mortality_c_to_cwdc
CWD harvest_c_to_cwdc
Litter 1 gap_mortality_c_to_litr_met_c
Litter 1 harvest_c_to_litr_met_c
Litter 1 m_c_to_litr_met_fire
Litter 1 phenology_c_to_litr_met_c
Litter 2 gap_mortality_c_to_litr_cel_c
Litter 2 harvest_c_to_litr_cel_c
Litter 2 m_c_to_litr_cel_fire
Litter 2 phenology_c_to_litr_cel_c
Litter 3 gap_mortality_c_to_litr_lig_c
Litter 3 harvest_c_to_litr_lig_c
Litter 3 m_c_to_litr_lig_fire
Litter 3 phenology_c_to_litr_lig_c
CWD CWD_HR_L2_vr
CWD CWD_HR_L3_vr
CWD M_CWDC_TO_FIRE_vr
Litter 1 LITR1_HR_vr
Litter 1 M_LITR1C_TO_FIRE_vr
Litter 2 LITR2_HR_vr
Litter 2 M_LITR2C_TO_FIRE_vr
Litter 3 LITR3_HR_vr
Litter 3 M_LITR3C_TO_FIRE_vr
Soil 1 SOIL1_HR_S2_vr
Soil 1 SOIL1_HR_S3_vr
Soil 2 SOIL2_HR_S1_vr
Soil 2 SOIL2_HR_S3_vr
Soil 3 SOIL3_HR_vr
CWD CWDC_vr
Litter 1 LITR1C_vr
Litter 2 LITR2C_vr
Litter 3 LITR3C_vr
Soil 1 SOIL1C_vr
Soil 2 SOIL2C_vr
Soil 3 SOIL3C_vr
CWD fire_mortality_c_to_cwdc
CWD gap_mortality_c_to_cwdc
CWD harvest_c_to_cwdc
Litter 1 gap_mortality_c_to_litr_met_c
Litter 1 harvest_c_to_litr_met_c
Litter 1 m_c_to_litr_met_fire
Litter 1 phenology_c_to_litr_met_c
Litter 2 gap_mortality_c_to_litr_cel_c
Litter 2 harvest_c_to_litr_cel_c
Litter 2 m_c_to_litr_cel_fire
Litter 2 phenology_c_to_litr_cel_c
Litter 3 gap_mortality_c_to_litr_lig_c
Litter 3 harvest_c_to_litr_lig_c
Litter 3 m_c_to_litr_lig_fire
Litter 3 phenology_c_to_litr_lig_c
CWD CWD_HR_L2_vr
CWD CWD_HR_L3_vr
CWD M_CWDC_TO_FIRE_vr
Litter 1 LITR1_HR_vr
Litter 1 M_LITR1C_TO_FIRE_vr
Litter 2 LITR2_HR_vr
Litter 2 M_LITR2C_TO_FIRE_vr
Litter 3 LITR3_HR_vr
Litter 3 M_LITR3C_TO_FIRE_vr
Soil 1 SOIL1_HR_S2_vr
Soil 1 SOIL1_HR_S3_vr
Soil 2 SOIL2_HR_S1_vr
Soil 2 SOIL2_HR_S3_vr
Soil 3 SOIL3_HR_vr
CWD CWDC_vr
Litter 1 LITR1C_vr
Litter 2 LITR2C_vr
Litter 3 LITR3C_vr
Soil 1 SOIL1C_vr
Soil 2 SOIL2C_vr
Soil 3 SOIL3C_vr
CWD fire_mortality_c_to_cwdc
CWD gap_mortality_c_to_cwdc
CWD harvest_c_to_cwdc
Litter 1 gap_mortality_c_to_litr_met_c
Litter 1 harvest_c_to_litr_met_c
Litter 1 m_c_to_litr_met_fire
Litter 1 phenology_c_to_litr_met_c
Litter 2 gap_mortality_c_to_litr_cel_c
Litter 2 harvest_c_to_litr_cel_c
Litter 2 m_c_to_litr_cel_fire
Litter 2 phenology_c_to_litr_cel_c
Litter 3 gap_mortality_c_to_litr_lig_c
Litter 3 harvest_c_to_litr_lig_c
Litter 3 m_c_to_litr_lig_fire
Litter 3 phenology_c_to_litr_lig_c
CWD CWD_HR_L2_vr
CWD CWD_HR_L3_vr
CWD M_CWDC_TO_FIRE_vr
Litter 1 LITR1_HR_vr
Litter 1 M_LITR1C_TO_FIRE_vr
Litter 2 LITR2_HR_vr
Litter 2 M_LITR2C_TO_FIRE_vr
Litter 3 LITR3_HR_vr
Litter 3 M_LITR3C_TO_FIRE_vr
Soil 1 SOIL1_HR_S2_vr
Soil 1 SOIL1_HR_S3_vr
Soil 2 SOIL2_HR_S1_vr
Soil 2 SOIL2_HR_S3_vr
Soil 3 SOIL3_HR_vr
CWD CWDC_vr
Litter 1 LITR1C_vr
Litter 2 LITR2C_vr
Litter 3 LITR3C_vr
Soil 1 SOIL1C_vr
Soil 2 SOIL2C_vr
Soil 3 SOIL3C_vr
CWD fire_mortality_c_to_cwdc
CWD gap_mortality_c_to_cwdc
CWD harvest_c_to_cwdc
Litter 1 gap_mortality_c_to_litr_met_c
Litter 1 harvest_c_to_litr_met_c
Litter 1 m_c_to_litr_met_fire
Litter 1 phenology_c_to_litr_met_c
Litter 2 gap_mortality_c_to_litr_cel_c
Litter 2 harvest_c_to_litr_cel_c
Litter 2 m_c_to_litr_cel_fire
Litter 2 phenology_c_to_litr_cel_c
Litter 3 gap_mortality_c_to_litr_lig_c
Litter 3 harvest_c_to_litr_lig_c
Litter 3 m_c_to_litr_lig_fire
Litter 3 phenology_c_to_litr_lig_c
CWD CWD_HR_L2_vr
CWD CWD_HR_L3_vr
CWD M_CWDC_TO_FIRE_vr
Litter 1 LITR1_HR_vr
Litter 1 M_LITR1C_TO_FIRE_vr
Litter 2 LITR2_HR_vr
Litter 2 M_LITR2C_TO_FIRE_vr
Litter 3 LITR3_HR_vr
Litter 3 M_LITR3C_TO_FIRE_vr
Soil 1 SOIL1_HR_S2_vr
Soil 1 SOIL1_HR_S3_vr
Soil 2 SOIL2_HR_S1_vr
Soil 2 SOIL2_HR_S3_vr
Soil 3 SOIL3_HR_vr
CWD CWDC_vr
Litter 1 LITR1C_vr
Litter 2 LITR2C_vr
Litter 3 LITR3C_vr
Soil 1 SOIL1C_vr
Soil 2 SOIL2C_vr
Soil 3 SOIL3C_vr
CWD fire_mortality_c_to_cwdc
CWD gap_mortality_c_to_cwdc
CWD harvest_c_to_cwdc
Litter 1 gap_mortality_c_to_litr_met_c
Litter 1 harvest_c_to_litr_met_c
Litter 1 m_c_to_litr_met_fire
Litter 1 phenology_c_to_litr_met_c
Litter 2 gap_mortality_c_to_litr_cel_c
Litter 2 harvest_c_to_litr_cel_c
Litter 2 m_c_to_litr_cel_fire
Litter 2 phenology_c_to_litr_cel_c
Litter 3 gap_mortality_c_to_litr_lig_c
Litter 3 harvest_c_to_litr_lig_c
Litter 3 m_c_to_litr_lig_fire
Litter 3 phenology_c_to_litr_lig_c
CWD CWD_HR_L2_vr
CWD CWD_HR_L3_vr
CWD M_CWDC_TO_FIRE_vr
Litter 1 LITR1_HR_vr
Litter 1 M_LITR1C_TO_FIRE_vr
Litter 2 LITR2_HR_vr
Litter 2 M_LITR2C_TO_FIRE_vr
Litter 3 LITR3_HR_vr
Litter 3 M_LITR3C_TO_FIRE_vr
Soil 1 SOIL1_HR_S2_vr
Soil 1 SOIL1_HR_S3_vr
Soil 2 SOIL2_HR_S1_vr
Soil 2 SOIL2_HR_S3_vr
Soil 3 SOIL3_HR_vr
Out[18]:
cell_nr log xs_12C_data us_12C_data rs_12C_data
0 0 None [129.1920166015625, 120.71854400634766, 113.43... [262.2552515402221, 230.77564187323762, 193.91... [186.8644734745739, 166.66575469226342, 143.91...
1 1 None [317.9560546875, 298.3962097167969, 281.823089... [498.859132845131, 434.909804927153, 359.02376... [315.6620625071609, 278.7768207001784, 235.945...
2 2 None [257.4531555175781, 239.01255798339844, 222.14... [333.5813136382974, 289.47885777152004, 236.84... [218.52115978701482, 192.21870878923636, 161.7...
3 3 None [201.70277404785156, 188.2528076171875, 176.23... [279.1358715772878, 245.53976821752275, 206.04... [206.74619933914047, 184.36809844388625, 158.9...

In [ ]: