In [1]:
import xarray as xr
import numpy as np
import pandas as pd
%matplotlib inline
from matplotlib import pyplot as plt
from dask.diagnostics import ProgressBar
import seaborn as sns
from matplotlib.colors import LogNorm
In [84]:
# resampling frequency in number of days
freq=15
In [3]:
from tools.load_GlobColor_dataset import load_dataset
import importlib
importlib.reload(load_dataset)
Out[3]:
<module 'tools.load_GlobColor_dataset.load_dataset' from '/Users/vyan2000/work_linux/2Archive/myproject/20161024xray_oceancolor/ocean_color-master/tools/load_GlobColor_dataset/load_dataset.py'>
In [4]:
############### KD490
ds_daily = load_dataset.load_kd490()
100%|██████████| 5538/5538 [01:12<00:00, 76.09it/s]
<xarray.Dataset>
Dimensions: (lat: 553, lon: 721, time: 5538)
Coordinates:
* lat (lat) float32 28.0208 27.9792 27.9375 27.8958 27.8542 ...
* lon (lon) float32 44.9792 45.0208 45.0625 45.1042 45.1458 ...
* time (time) datetime64[ns] 2002-04-28 2002-04-29 2002-04-30 ...
Data variables:
KD490-LEE_mean (time, lat, lon) float64 nan nan nan nan nan nan nan nan ...
Attributes:
Conventions: CF-1.4
title: GlobColour daily merged MERIS/SeaWiFS product
product_name: L3m_20020428__665648402_4_AVW-MERSWF_KD490-LE...
product_type: day
product_version: 2016.1
product_level: 3
parameter_code: KD490-LEE
parameter: Diffuse attenuation coefficient, LEE algorithm
parameter_algo_list: ,
publication: Zhong-Ping Lee, Ke-Ping Du and Robert Arnone ...
site_name: 665648402
sensor_name: WEIGHTED_AVERAGING
sensor: Merged data - weighted mean
sensor_name_list: MER,SWF
start_time: 20020427T224343Z
end_time: 20020429T004709Z
duration_time: PT93807S
period_start_day: 20020428
period_end_day: 20020428
period_duration_day: P1D
grid_type: Equirectangular
spatial_resolution: 4.63831
nb_equ_bins: 721
registration: 5
lat_step: 0.0416667
lon_step: 0.0416667
earth_radius: 6378.137
max_north_grid: 28.0417
max_south_grid: 5.0
max_west_grid: 44.9583
max_east_grid: 75.0
northernmost_latitude: 26.7917
southernmost_latitude: 5.0
westernmost_longitude: 49.75
easternmost_longitude: 75.0
nb_grid_bins: 398713
nb_bins: 398713
pct_bins: 100.0
nb_valid_bins: 62037
pct_valid_bins: 15.5593
software_name: globcolour_l3_extract
software_version: 2016.1
institution: ACRI
processing_time: 20170721T184114Z
netcdf_version: 4.3.3.1 of Jul 8 2016 18:15:50 $
DPM_reference: GC-UD-ACRI-PUG
IODD_reference: GC-UD-ACRI-PUG
references: http://www.globcolour.info
contact: service@globcolour.info
copyright: Copyright ACRI-ST - GlobColour. GlobColour ha...
history: 20170721T184114Z: globcolour_l3_extract.sh -i...
input_files: S2002117220453.L2_GAC_OC.nc,S2002117234340.L2...
input_files_reprocessings: 2014.0,2014.0,2014.0,2014.0,2014.0,2014.0,201...
In [5]:
ds_daily.kd490.sel(time='2002-04-28').plot()
Out[5]:
<matplotlib.collections.QuadMesh at 0x11fa12390>
In [6]:
freq_resample = str(8) + 'D'
ds_8day = ds_daily.resample(freq_resample, dim='time') # see the above for doc, test case, & default behavior
ds_8day
Out[6]:
<xarray.Dataset>
Dimensions: (lat: 553, lon: 721, time: 695)
Coordinates:
* lat (lat) float32 28.0208 27.9792 27.9375 27.8958 27.8542 27.8125 ...
* lon (lon) float32 44.9792 45.0208 45.0625 45.1042 45.1458 45.1875 ...
* time (time) datetime64[ns] 2002-04-28 2002-05-06 2002-05-14 ...
Data variables:
kd490 (time, lat, lon) float64 nan nan nan nan nan nan nan nan nan ...
In [7]:
# check data quality
both_datasets = [ds_8day, ds_daily]
print([(ds.nbytes / 1e6) for ds in both_datasets])
[2216.854936, 17664.630152]
In [8]:
def fix_bad_data(ds):
# for some reason, the cloud / land mask is backwards on some data
# this is obvious because there are kd490 values less than zero
bad_data = ds.kd490.groupby('time').min() < 0
# loop through and fix
for n in np.nonzero(bad_data.values)[0]:
data = ds.kd490[n].values
ds.kd490.values[n] = np.ma.masked_less(data, 0).filled(np.nan)
In [9]:
[fix_bad_data(ds) for ds in both_datasets]
Out[9]:
[None, None]
In [10]:
# Count the number of ocean data points
(ds_8day.kd490>0).sum(dim='time').plot()
/Users/vyan2000/local/miniconda3/envs/condapython3/lib/python3.5/site-packages/xarray/core/variable.py:1164: RuntimeWarning: invalid value encountered in greater
if not reflexive
Out[10]:
<matplotlib.collections.QuadMesh at 0x307eef2b0>
In [11]:
# find a mask for the land
ocean_mask = (ds_8day.kd490>0).sum(dim='time')>0
num_ocean_points = ocean_mask.sum().values
ocean_mask.plot()
plt.title('%g total ocean points' % num_ocean_points)
/Users/vyan2000/local/miniconda3/envs/condapython3/lib/python3.5/site-packages/xarray/core/variable.py:1164: RuntimeWarning: invalid value encountered in greater
if not reflexive
Out[11]:
<matplotlib.text.Text at 0x1d2327438>
In [12]:
plt.figure(figsize=(8,6))
ds_daily.kd490.sel(time='2002-11-18',method='nearest').plot(norm=LogNorm())
Out[12]:
<matplotlib.collections.QuadMesh at 0x1d01816a0>
/Users/vyan2000/local/miniconda3/envs/condapython3/lib/python3.5/site-packages/matplotlib/colors.py:1022: RuntimeWarning: invalid value encountered in less_equal
mask |= resdat <= 0
In [13]:
ds_daily.groupby('time').count() # information from original data
Out[13]:
<xarray.Dataset>
Dimensions: (time: 5538)
Coordinates:
* time (time) datetime64[ns] 2002-04-28 2002-04-29 2002-04-30 ...
Data variables:
kd490 (time) int64 62037 52324 58714 43103 37104 55636 25303 19619 ...
In [14]:
ds_daily.kd490.groupby('time').count()/float(num_ocean_points)
Out[14]:
<xarray.DataArray 'kd490' (time: 5538)>
array([ 0.247571, 0.208809, 0.23431 , ..., 0.003496, 0.020712, 0.034563])
Coordinates:
* time (time) datetime64[ns] 2002-04-28 2002-04-29 2002-04-30 ...
In [15]:
count_8day,count_daily = [ds.kd490.groupby('time').count()/float(num_ocean_points)
for ds in (ds_8day,ds_daily)]
plt.figure(figsize=(12,4))
count_8day.plot(color='k')
count_daily.plot(color='r')
plt.legend(['8 day','daily'])
Out[15]:
<matplotlib.legend.Legend at 0x1d42f1518>
In [16]:
# Maps of individual days
target_date = '2003-02-15'
plt.figure(figsize=(8,6))
ds_8day.kd490.sel(time=target_date, method='nearest').plot(norm=LogNorm())
Out[16]:
<matplotlib.collections.QuadMesh at 0x90de53518>
/Users/vyan2000/local/miniconda3/envs/condapython3/lib/python3.5/site-packages/matplotlib/colors.py:1022: RuntimeWarning: invalid value encountered in less_equal
mask |= resdat <= 0
In [17]:
plt.figure(figsize=(8,6))
ds_daily.kd490.sel(time=target_date, method='nearest').plot(norm=LogNorm())
Out[17]:
<matplotlib.collections.QuadMesh at 0x12b26eac8>
/Users/vyan2000/local/miniconda3/envs/condapython3/lib/python3.5/site-packages/matplotlib/colors.py:1022: RuntimeWarning: invalid value encountered in less_equal
mask |= resdat <= 0
In [85]:
freq
Out[85]:
15
In [86]:
# next carry out interpolation starts here
freq_resample = str(freq) + 'D'
ds_resample = ds_daily.resample(freq_resample, dim='time') # see the above for doc, test case, & default behavior
ds_resample
Out[86]:
<xarray.Dataset>
Dimensions: (lat: 553, lon: 721, time: 371)
Coordinates:
* lat (lat) float32 28.0208 27.9792 27.9375 27.8958 27.8542 27.8125 ...
* lon (lon) float32 44.9792 45.0208 45.0625 45.1042 45.1458 45.1875 ...
* time (time) datetime64[ns] 2002-04-28 2002-05-13 2002-05-28 ...
Data variables:
kd490 (time, lat, lon) float64 nan nan nan nan nan nan nan nan nan ...
In [87]:
plt.figure(figsize=(8,6))
ds_resample.kd490.sel(time=target_date, method='nearest').plot(norm=LogNorm())
Out[87]:
<matplotlib.collections.QuadMesh at 0x1216fbef0>
/Users/vyan2000/local/miniconda3/envs/condapython3/lib/python3.5/site-packages/matplotlib/colors.py:1022: RuntimeWarning: invalid value encountered in less_equal
mask |= resdat <= 0
In [88]:
# check the range for the longitude
print(ds_resample.lon.min(),'\n' ,ds_resample.lat.min())
<xarray.DataArray 'lon' ()>
array(44.97917175292969)
<xarray.DataArray 'lat' ()>
array(5.020830154418945)
In [89]:
# load preprocessed float data, and start the interpolation right here!!!!
var2 = "cdm"
var1 = "chl"
vardist = "dist"
indir_prefix = "./data_globcolour/output.data.interpolate/" + "df_Globcolor_"
indir = indir_prefix + var1 + vardist + var2 + "_" + str(freq) + "d.csv"
floatDF_tmp = pd.read_csv(indir,index_col=0)
print(floatDF_tmp)
id time lat lon temp ve \
366 10206 2002-07-04 16.265717 66.663800 NaN 6.140233
732 10208 2002-07-04 13.549633 70.195217 NaN 11.373300
1098 11089 2002-07-04 15.657150 65.248067 27.773283 9.376883
1464 15703 2002-07-04 13.611350 70.165200 28.590333 10.194983
2196 27069 2002-07-04 19.969700 70.048350 28.916267 25.855350
2928 28842 2002-07-04 18.350883 60.961600 27.226833 5.825783
3294 34159 2002-07-04 13.394633 60.516650 NaN 31.603317
4026 34210 2002-07-04 5.882953 56.749953 26.354721 -5.144814
4392 34211 2002-07-04 7.797533 69.070367 28.430017 19.858683
4758 34212 2002-07-04 6.519433 66.877317 28.568833 34.703000
5490 34310 2002-07-04 5.023286 70.029000 28.954857 8.074714
5856 34311 2002-07-04 9.730864 69.980455 28.593818 -2.896714
6222 34312 2002-07-04 9.638095 65.167048 28.129857 7.005500
6588 34314 2002-07-04 5.116600 54.903200 26.905500 -6.718778
6954 34315 2002-07-04 5.162294 59.998824 28.303294 -17.022625
7686 34708 2002-07-04 10.333167 61.099483 27.384050 22.553517
8052 34709 2002-07-04 6.396520 52.252040 26.204160 0.462760
8418 34710 2002-07-04 12.618183 49.966633 31.137717 -15.991800
8784 34714 2002-07-04 13.767567 65.507833 27.870250 34.244133
9150 34716 2002-07-04 7.695783 66.931733 28.781367 25.851967
9516 34718 2002-07-04 14.420783 73.395800 28.827417 16.290550
9882 34719 2002-07-04 16.656500 71.694917 28.910950 14.472783
10248 34720 2002-07-04 13.938883 69.771467 28.658633 11.449783
10614 34721 2002-07-04 16.653933 65.645233 27.876133 5.294917
10980 34722 2002-07-04 10.935983 70.908850 28.703450 7.445617
11346 34723 2002-07-04 16.587100 66.599800 28.407783 5.319900
86010 2134712 2002-07-04 8.795967 64.194267 28.258667 12.196000
367 10206 2002-07-19 16.072700 67.886683 NaN 14.527167
733 10208 2002-07-19 12.458050 70.996300 NaN 0.988950
1099 11089 2002-07-19 14.965100 66.350767 27.791150 6.747217
... ... ... ... ... ... ...
85276 147144 2017-06-15 14.572667 70.179567 29.493000 10.190933
99550 62321990 2017-06-15 13.434167 60.508167 28.278083 16.173300
100648 63157510 2017-06-15 7.810517 59.133933 28.565683 27.087900
101380 63158530 2017-06-15 7.241083 57.466850 29.115433 -7.467850
102478 63254860 2017-06-15 6.650636 57.272091 29.173409 -13.571667
102844 63255180 2017-06-15 5.942217 62.901967 29.877033 6.626267
103576 63255860 2017-06-15 8.232364 56.526273 28.334909 0.638300
104674 63258900 2017-06-15 11.394667 66.441133 29.527950 19.424917
105040 63258950 2017-06-15 9.665528 67.081306 29.651694 4.542086
105406 63259180 2017-06-15 10.315883 70.122550 29.755983 11.165400
105772 63259190 2017-06-15 8.182300 71.503133 29.743433 2.719310
106504 63259230 2017-06-15 11.710174 69.002391 29.589565 6.724391
108700 63348720 2017-06-15 7.832875 58.230143 28.273804 31.217091
109066 63348750 2017-06-15 5.647766 66.040532 30.124574 2.377617
112360 64111550 2017-06-15 16.753833 67.083500 29.548500 4.374650
112726 64113560 2017-06-15 14.357517 62.191567 28.389650 13.057833
113458 64115560 2017-06-15 16.855283 72.553533 29.724217 10.948783
113824 64117500 2017-06-15 6.810550 71.186017 29.476617 2.463900
84545 147140 2017-06-30 11.557667 61.136333 28.039000 83.892000
85277 147144 2017-06-30 14.291000 70.923667 29.260333 17.542500
99551 62321990 2017-06-30 12.863000 61.727800 27.980800 24.212500
100649 63157510 2017-06-30 8.170833 60.425000 28.591667 25.888400
101381 63158530 2017-06-30 7.562400 57.591200 28.432600 2.102750
102845 63255180 2017-06-30 6.457000 63.455750 29.492250 26.794667
104675 63258900 2017-06-30 10.611667 67.725333 29.285333 11.653500
105407 63259180 2017-06-30 9.708250 71.031750 29.490500 17.048000
112361 64111550 2017-06-30 15.278385 67.633154 28.739692 14.127583
112727 64113560 2017-06-30 13.827154 62.739077 27.976923 0.487667
113459 64115560 2017-06-30 16.871250 73.274250 28.602000 -0.189000
113825 64117500 2017-06-30 6.149231 71.412077 29.282000 3.511583
vn spd var_lat var_lon var_tmp chlor_a \
366 0.357733 7.394967 0.001451 0.005610 1000.000000 NaN
732 -5.285617 15.006967 0.000063 0.000118 1000.000000 NaN
1098 -14.097033 18.695917 0.000067 0.000129 0.003614 NaN
1464 -4.513033 13.965250 0.000055 0.000102 0.088623 NaN
2196 -5.424417 27.865400 0.000057 0.000106 0.001731 NaN
2928 -9.921900 16.832533 0.000149 0.000362 0.003382 NaN
3294 16.559017 36.755683 0.000061 0.000116 1000.000000 0.321245
4026 -18.675465 26.752744 0.000066 0.000129 0.003705 0.328691
4392 -14.960467 27.234933 0.000053 0.000098 0.003538 0.096233
4758 1.993683 42.610483 0.000055 0.000102 0.003553 0.094689
5490 -0.021000 10.808286 0.000056 0.000103 0.003749 0.109962
5856 -11.504476 13.484762 0.000061 0.000114 0.003594 0.115337
6222 -15.195200 17.403650 0.000075 0.000154 0.003670 0.165898
6588 -5.381333 16.793444 0.000047 0.000087 0.003738 0.338628
6954 5.554625 28.943938 0.000052 0.000094 0.003575 0.177327
7686 4.269317 24.302933 0.000058 0.000110 0.001807 0.412506
8052 46.740240 81.157560 0.000079 0.000162 0.001818 0.501909
8418 -6.248550 46.052550 0.000046 0.000087 0.001837 0.224649
8784 1.335217 35.355967 0.000059 0.000110 0.001815 0.344161
9150 2.107750 34.797783 0.000057 0.000106 0.001772 0.110550
9516 -39.532167 44.367833 0.000055 0.000102 0.001691 0.419638
9882 -20.354250 26.786467 0.000060 0.000112 0.001666 NaN
10248 -12.078667 20.477317 0.000061 0.000114 0.001750 0.236139
10614 -11.221400 13.610417 0.000058 0.000109 0.001740 NaN
10980 -16.828850 20.732050 0.000061 0.000115 0.001779 NaN
11346 -1.815867 10.339867 0.000064 0.000124 0.001786 NaN
86010 -20.248500 29.976417 0.000085 0.000176 0.001942 0.175341
367 -8.716150 19.262550 0.001116 0.004011 1000.000000 0.137352
733 -23.323367 26.479183 0.000065 0.000124 1000.000000 0.099238
1099 6.272917 13.490517 0.000059 0.000109 0.003728 0.324946
... ... ... ... ... ... ...
85276 -2.238083 14.281700 0.000347 0.000176 0.002037 NaN
99550 -6.508667 18.012117 0.002924 0.002439 0.001867 NaN
100648 6.130500 34.657567 0.000010 0.000008 0.001766 0.314935
101380 6.872683 28.816750 0.000076 0.000041 0.001765 0.144886
102478 -13.335286 21.792000 0.002290 0.001733 0.001853 0.191329
102844 6.505583 14.081183 0.000066 0.000034 0.001887 0.078646
103576 0.823200 1.099600 0.018849 0.019519 0.002027 0.260779
104674 -5.209017 26.490717 0.000199 0.000106 0.001803 NaN
105040 -28.119771 31.811457 0.000415 0.000222 0.001851 0.104562
105406 -7.142583 15.643117 0.000918 0.000631 0.001846 0.082649
105772 -7.233862 8.656517 0.007749 0.007112 0.001956 0.112472
106504 -17.843696 21.039565 0.005488 0.004817 0.001931 0.041524
108700 -17.404636 39.430727 0.008580 0.008565 0.001927 0.223120
109066 -11.912617 15.776319 0.000015 0.000010 0.001807 0.093046
112360 -12.741350 17.871400 0.000004 0.000006 0.001684 NaN
112726 -6.607233 16.582283 0.000069 0.000034 0.001684 NaN
113458 -1.390750 16.621250 0.000003 0.000005 0.001686 NaN
113824 -6.642417 15.401850 0.000003 0.000005 0.001684 0.124498
84545 -12.843000 85.019000 0.000032 0.000014 0.001585 0.424789
85277 -15.481500 23.507500 0.000129 0.000055 0.001716 NaN
99551 -13.379500 27.666750 0.008339 0.007574 0.002022 NaN
100649 5.273200 27.029800 0.000015 0.000010 0.001986 0.129658
101381 -33.419500 33.622250 0.000004 0.000006 0.001870 0.149060
102845 -4.388000 27.262333 0.000476 0.000268 0.002092 0.130959
104675 -5.334000 12.912000 0.000794 0.000473 0.002321 NaN
105407 -12.979667 21.434667 0.000538 0.000273 0.002068 NaN
112361 -23.375000 27.887500 0.000003 0.000005 0.001705 NaN
112727 -8.357417 10.514917 0.000107 0.000049 0.001705 NaN
113459 16.893333 17.142667 0.000003 0.000005 0.001754 NaN
113825 -21.195083 22.225833 0.000003 0.000005 0.001705 0.135096
dist cdm
366 644.346876 NaN
732 304.915617 NaN
1098 790.995091 NaN
1464 312.036955 NaN
2196 105.763836 NaN
2928 312.718966 0.073044
3294 637.078686 0.037462
4026 761.139172 0.030116
4392 427.708647 0.014937
4758 644.256384 0.012178
5490 316.339631 0.012372
5856 254.118974 0.014427
6222 777.066104 0.019534
6588 628.158223 0.031178
6954 1012.615255 0.016325
7686 756.055492 0.039333
8052 306.502480 0.042730
8418 103.261978 0.017339
8784 763.459988 0.035670
9150 642.504705 0.014206
9516 85.027550 0.056426
9882 170.756169 NaN
10248 367.784917 0.017688
10614 680.264624 NaN
10980 131.953916 NaN
11346 620.209000 NaN
86010 898.206857 0.019824
367 581.752175 NaN
733 160.061930 0.022579
1099 733.142554 NaN
... ... ...
85276 388.223010 NaN
99550 633.297339 NaN
100648 725.658656 0.031678
101380 667.943026 0.022053
102478 715.916792 0.014561
102844 1085.649383 0.007998
103576 522.599171 0.025497
104674 617.336774 NaN
105040 571.323295 0.011004
105406 232.404475 0.010142
105772 167.192022 0.013368
106504 342.140474 NaN
108700 658.910708 0.018644
109066 742.014936 0.010242
112360 574.311951 NaN
112726 692.122499 NaN
113458 77.225451 NaN
113824 173.402129 0.013759
84545 726.254721 0.089067
85277 326.533641 NaN
99551 772.348381 NaN
100649 805.220865 0.012746
101381 643.899278 0.012251
102845 1022.681205 0.011937
104675 481.403226 NaN
105407 141.373115 NaN
112361 623.755105 NaN
112727 775.259018 NaN
113459 2.169704 NaN
113825 145.314631 0.014212
[2559 rows x 14 columns]
In [90]:
from tools.time_lat_lon_interpolate import interpolate
importlib.reload(interpolate)
result_out5 = interpolate.sel_points_multilinear_time_lat_lon(ds_resample, floatDF_tmp, dims = 'points', col_name ='kd490')
print('\n *** after the interpolation *** \n', result_out5)
# important: keep the id, since the dataframe has been modified in a bound-aware way in the function
print('\n *** this two length should be equal *** %d >= %d?' %(len(floatDF_tmp.index), len(result_out5.index) ) )
*** after the interpolation ***
id time lat lon temp ve \
366 10206 2002-07-04 16.265717 66.663800 NaN 6.140233
732 10208 2002-07-04 13.549633 70.195217 NaN 11.373300
1098 11089 2002-07-04 15.657150 65.248067 27.773283 9.376883
1464 15703 2002-07-04 13.611350 70.165200 28.590333 10.194983
2196 27069 2002-07-04 19.969700 70.048350 28.916267 25.855350
2928 28842 2002-07-04 18.350883 60.961600 27.226833 5.825783
3294 34159 2002-07-04 13.394633 60.516650 NaN 31.603317
4026 34210 2002-07-04 5.882953 56.749953 26.354721 -5.144814
4392 34211 2002-07-04 7.797533 69.070367 28.430017 19.858683
4758 34212 2002-07-04 6.519433 66.877317 28.568833 34.703000
5490 34310 2002-07-04 5.023286 70.029000 28.954857 8.074714
5856 34311 2002-07-04 9.730864 69.980455 28.593818 -2.896714
6222 34312 2002-07-04 9.638095 65.167048 28.129857 7.005500
6588 34314 2002-07-04 5.116600 54.903200 26.905500 -6.718778
6954 34315 2002-07-04 5.162294 59.998824 28.303294 -17.022625
7686 34708 2002-07-04 10.333167 61.099483 27.384050 22.553517
8052 34709 2002-07-04 6.396520 52.252040 26.204160 0.462760
8418 34710 2002-07-04 12.618183 49.966633 31.137717 -15.991800
8784 34714 2002-07-04 13.767567 65.507833 27.870250 34.244133
9150 34716 2002-07-04 7.695783 66.931733 28.781367 25.851967
9516 34718 2002-07-04 14.420783 73.395800 28.827417 16.290550
9882 34719 2002-07-04 16.656500 71.694917 28.910950 14.472783
10248 34720 2002-07-04 13.938883 69.771467 28.658633 11.449783
10614 34721 2002-07-04 16.653933 65.645233 27.876133 5.294917
10980 34722 2002-07-04 10.935983 70.908850 28.703450 7.445617
11346 34723 2002-07-04 16.587100 66.599800 28.407783 5.319900
86010 2134712 2002-07-04 8.795967 64.194267 28.258667 12.196000
367 10206 2002-07-19 16.072700 67.886683 NaN 14.527167
733 10208 2002-07-19 12.458050 70.996300 NaN 0.988950
1099 11089 2002-07-19 14.965100 66.350767 27.791150 6.747217
... ... ... ... ... ... ...
85276 147144 2017-06-15 14.572667 70.179567 29.493000 10.190933
99550 62321990 2017-06-15 13.434167 60.508167 28.278083 16.173300
100648 63157510 2017-06-15 7.810517 59.133933 28.565683 27.087900
101380 63158530 2017-06-15 7.241083 57.466850 29.115433 -7.467850
102478 63254860 2017-06-15 6.650636 57.272091 29.173409 -13.571667
102844 63255180 2017-06-15 5.942217 62.901967 29.877033 6.626267
103576 63255860 2017-06-15 8.232364 56.526273 28.334909 0.638300
104674 63258900 2017-06-15 11.394667 66.441133 29.527950 19.424917
105040 63258950 2017-06-15 9.665528 67.081306 29.651694 4.542086
105406 63259180 2017-06-15 10.315883 70.122550 29.755983 11.165400
105772 63259190 2017-06-15 8.182300 71.503133 29.743433 2.719310
106504 63259230 2017-06-15 11.710174 69.002391 29.589565 6.724391
108700 63348720 2017-06-15 7.832875 58.230143 28.273804 31.217091
109066 63348750 2017-06-15 5.647766 66.040532 30.124574 2.377617
112360 64111550 2017-06-15 16.753833 67.083500 29.548500 4.374650
112726 64113560 2017-06-15 14.357517 62.191567 28.389650 13.057833
113458 64115560 2017-06-15 16.855283 72.553533 29.724217 10.948783
113824 64117500 2017-06-15 6.810550 71.186017 29.476617 2.463900
84545 147140 2017-06-30 11.557667 61.136333 28.039000 83.892000
85277 147144 2017-06-30 14.291000 70.923667 29.260333 17.542500
99551 62321990 2017-06-30 12.863000 61.727800 27.980800 24.212500
100649 63157510 2017-06-30 8.170833 60.425000 28.591667 25.888400
101381 63158530 2017-06-30 7.562400 57.591200 28.432600 2.102750
102845 63255180 2017-06-30 6.457000 63.455750 29.492250 26.794667
104675 63258900 2017-06-30 10.611667 67.725333 29.285333 11.653500
105407 63259180 2017-06-30 9.708250 71.031750 29.490500 17.048000
112361 64111550 2017-06-30 15.278385 67.633154 28.739692 14.127583
112727 64113560 2017-06-30 13.827154 62.739077 27.976923 0.487667
113459 64115560 2017-06-30 16.871250 73.274250 28.602000 -0.189000
113825 64117500 2017-06-30 6.149231 71.412077 29.282000 3.511583
vn spd var_lat var_lon var_tmp chlor_a \
366 0.357733 7.394967 0.001451 0.005610 1000.000000 NaN
732 -5.285617 15.006967 0.000063 0.000118 1000.000000 NaN
1098 -14.097033 18.695917 0.000067 0.000129 0.003614 NaN
1464 -4.513033 13.965250 0.000055 0.000102 0.088623 NaN
2196 -5.424417 27.865400 0.000057 0.000106 0.001731 NaN
2928 -9.921900 16.832533 0.000149 0.000362 0.003382 NaN
3294 16.559017 36.755683 0.000061 0.000116 1000.000000 0.321245
4026 -18.675465 26.752744 0.000066 0.000129 0.003705 0.328691
4392 -14.960467 27.234933 0.000053 0.000098 0.003538 0.096233
4758 1.993683 42.610483 0.000055 0.000102 0.003553 0.094689
5490 -0.021000 10.808286 0.000056 0.000103 0.003749 0.109962
5856 -11.504476 13.484762 0.000061 0.000114 0.003594 0.115337
6222 -15.195200 17.403650 0.000075 0.000154 0.003670 0.165898
6588 -5.381333 16.793444 0.000047 0.000087 0.003738 0.338628
6954 5.554625 28.943938 0.000052 0.000094 0.003575 0.177327
7686 4.269317 24.302933 0.000058 0.000110 0.001807 0.412506
8052 46.740240 81.157560 0.000079 0.000162 0.001818 0.501909
8418 -6.248550 46.052550 0.000046 0.000087 0.001837 0.224649
8784 1.335217 35.355967 0.000059 0.000110 0.001815 0.344161
9150 2.107750 34.797783 0.000057 0.000106 0.001772 0.110550
9516 -39.532167 44.367833 0.000055 0.000102 0.001691 0.419638
9882 -20.354250 26.786467 0.000060 0.000112 0.001666 NaN
10248 -12.078667 20.477317 0.000061 0.000114 0.001750 0.236139
10614 -11.221400 13.610417 0.000058 0.000109 0.001740 NaN
10980 -16.828850 20.732050 0.000061 0.000115 0.001779 NaN
11346 -1.815867 10.339867 0.000064 0.000124 0.001786 NaN
86010 -20.248500 29.976417 0.000085 0.000176 0.001942 0.175341
367 -8.716150 19.262550 0.001116 0.004011 1000.000000 0.137352
733 -23.323367 26.479183 0.000065 0.000124 1000.000000 0.099238
1099 6.272917 13.490517 0.000059 0.000109 0.003728 0.324946
... ... ... ... ... ... ...
85276 -2.238083 14.281700 0.000347 0.000176 0.002037 NaN
99550 -6.508667 18.012117 0.002924 0.002439 0.001867 NaN
100648 6.130500 34.657567 0.000010 0.000008 0.001766 0.314935
101380 6.872683 28.816750 0.000076 0.000041 0.001765 0.144886
102478 -13.335286 21.792000 0.002290 0.001733 0.001853 0.191329
102844 6.505583 14.081183 0.000066 0.000034 0.001887 0.078646
103576 0.823200 1.099600 0.018849 0.019519 0.002027 0.260779
104674 -5.209017 26.490717 0.000199 0.000106 0.001803 NaN
105040 -28.119771 31.811457 0.000415 0.000222 0.001851 0.104562
105406 -7.142583 15.643117 0.000918 0.000631 0.001846 0.082649
105772 -7.233862 8.656517 0.007749 0.007112 0.001956 0.112472
106504 -17.843696 21.039565 0.005488 0.004817 0.001931 0.041524
108700 -17.404636 39.430727 0.008580 0.008565 0.001927 0.223120
109066 -11.912617 15.776319 0.000015 0.000010 0.001807 0.093046
112360 -12.741350 17.871400 0.000004 0.000006 0.001684 NaN
112726 -6.607233 16.582283 0.000069 0.000034 0.001684 NaN
113458 -1.390750 16.621250 0.000003 0.000005 0.001686 NaN
113824 -6.642417 15.401850 0.000003 0.000005 0.001684 0.124498
84545 -12.843000 85.019000 0.000032 0.000014 0.001585 0.424789
85277 -15.481500 23.507500 0.000129 0.000055 0.001716 NaN
99551 -13.379500 27.666750 0.008339 0.007574 0.002022 NaN
100649 5.273200 27.029800 0.000015 0.000010 0.001986 0.129658
101381 -33.419500 33.622250 0.000004 0.000006 0.001870 0.149060
102845 -4.388000 27.262333 0.000476 0.000268 0.002092 0.130959
104675 -5.334000 12.912000 0.000794 0.000473 0.002321 NaN
105407 -12.979667 21.434667 0.000538 0.000273 0.002068 NaN
112361 -23.375000 27.887500 0.000003 0.000005 0.001705 NaN
112727 -8.357417 10.514917 0.000107 0.000049 0.001705 NaN
113459 16.893333 17.142667 0.000003 0.000005 0.001754 NaN
113825 -21.195083 22.225833 0.000003 0.000005 0.001705 0.135096
dist cdm kd490
366 644.346876 NaN NaN
732 304.915617 NaN NaN
1098 790.995091 NaN NaN
1464 312.036955 NaN NaN
2196 105.763836 NaN NaN
2928 312.718966 0.073044 NaN
3294 637.078686 0.037462 0.082638
4026 761.139172 0.030116 0.079110
4392 427.708647 0.014937 0.049980
4758 644.256384 0.012178 0.049653
5490 316.339631 0.012372 0.052449
5856 254.118974 0.014427 0.053984
6222 777.066104 0.019534 0.058026
6588 628.158223 0.031178 0.080774
6954 1012.615255 0.016325 0.061184
7686 756.055492 0.039333 0.091085
8052 306.502480 0.042730 0.099538
8418 103.261978 0.017339 0.070830
8784 763.459988 0.035670 0.084350
9150 642.504705 0.014206 0.050987
9516 85.027550 0.056426 0.090653
9882 170.756169 NaN NaN
10248 367.784917 0.017688 0.070251
10614 680.264624 NaN NaN
10980 131.953916 NaN NaN
11346 620.209000 NaN NaN
86010 898.206857 0.019824 0.058893
367 581.752175 NaN 0.061941
733 160.061930 0.022579 0.050537
1099 733.142554 NaN 0.083765
... ... ... ...
85276 388.223010 NaN NaN
99550 633.297339 NaN NaN
100648 725.658656 0.031678 0.074890
101380 667.943026 0.022053 0.055573
102478 715.916792 0.014561 0.056099
102844 1085.649383 0.007998 0.044243
103576 522.599171 0.025497 0.070162
104674 617.336774 NaN NaN
105040 571.323295 0.011004 0.045220
105406 232.404475 0.010142 0.045343
105772 167.192022 0.013368 0.049120
106504 342.140474 NaN 0.039608
108700 658.910708 0.018644 0.061427
109066 742.014936 0.010242 0.046977
112360 574.311951 NaN NaN
112726 692.122499 NaN NaN
113458 77.225451 NaN NaN
113824 173.402129 0.013759 0.049696
84545 726.254721 0.089067 0.094564
85277 326.533641 NaN NaN
99551 772.348381 NaN NaN
100649 805.220865 0.012746 0.048210
101381 643.899278 0.012251 0.050483
102845 1022.681205 0.011937 0.047742
104675 481.403226 NaN NaN
105407 141.373115 NaN NaN
112361 623.755105 NaN NaN
112727 775.259018 NaN NaN
113459 2.169704 NaN NaN
113825 145.314631 0.014212 0.052090
[2559 rows x 15 columns]
*** this two length should be equal *** 2559 >= 2559?
/Users/vyan2000/local/miniconda3/envs/condapython3/lib/python3.5/site-packages/numpy/lib/nanfunctions.py:703: RuntimeWarning: Mean of empty slice
warnings.warn("Mean of empty slice", RuntimeWarning)
In [91]:
# output the dataframe result_out4
var3 = "kd490"
outdir_prefix = "./data_globcolour/output.data.interpolate/" + "df_Globcolor_"
outdir = outdir_prefix + var1 + vardist + var2 + var3 + "_" + str(freq) + "d.csv"
result_out5.to_csv(outdir)
print(pd.read_csv(outdir,index_col=0))
### plot for id 125776, which will be fit by LDS
plt.figure(figsize=(8,6))
result_out5[result_out5.id == 135776].plot(x='time', y ='kd490', title=('id - %d' % 135776) )
plt.show();
plt.close("all")
id time lat lon temp ve \
366 10206 2002-07-04 16.265717 66.663800 NaN 6.140233
732 10208 2002-07-04 13.549633 70.195217 NaN 11.373300
1098 11089 2002-07-04 15.657150 65.248067 27.773283 9.376883
1464 15703 2002-07-04 13.611350 70.165200 28.590333 10.194983
2196 27069 2002-07-04 19.969700 70.048350 28.916267 25.855350
2928 28842 2002-07-04 18.350883 60.961600 27.226833 5.825783
3294 34159 2002-07-04 13.394633 60.516650 NaN 31.603317
4026 34210 2002-07-04 5.882953 56.749953 26.354721 -5.144814
4392 34211 2002-07-04 7.797533 69.070367 28.430017 19.858683
4758 34212 2002-07-04 6.519433 66.877317 28.568833 34.703000
5490 34310 2002-07-04 5.023286 70.029000 28.954857 8.074714
5856 34311 2002-07-04 9.730864 69.980455 28.593818 -2.896714
6222 34312 2002-07-04 9.638095 65.167048 28.129857 7.005500
6588 34314 2002-07-04 5.116600 54.903200 26.905500 -6.718778
6954 34315 2002-07-04 5.162294 59.998824 28.303294 -17.022625
7686 34708 2002-07-04 10.333167 61.099483 27.384050 22.553517
8052 34709 2002-07-04 6.396520 52.252040 26.204160 0.462760
8418 34710 2002-07-04 12.618183 49.966633 31.137717 -15.991800
8784 34714 2002-07-04 13.767567 65.507833 27.870250 34.244133
9150 34716 2002-07-04 7.695783 66.931733 28.781367 25.851967
9516 34718 2002-07-04 14.420783 73.395800 28.827417 16.290550
9882 34719 2002-07-04 16.656500 71.694917 28.910950 14.472783
10248 34720 2002-07-04 13.938883 69.771467 28.658633 11.449783
10614 34721 2002-07-04 16.653933 65.645233 27.876133 5.294917
10980 34722 2002-07-04 10.935983 70.908850 28.703450 7.445617
11346 34723 2002-07-04 16.587100 66.599800 28.407783 5.319900
86010 2134712 2002-07-04 8.795967 64.194267 28.258667 12.196000
367 10206 2002-07-19 16.072700 67.886683 NaN 14.527167
733 10208 2002-07-19 12.458050 70.996300 NaN 0.988950
1099 11089 2002-07-19 14.965100 66.350767 27.791150 6.747217
... ... ... ... ... ... ...
85276 147144 2017-06-15 14.572667 70.179567 29.493000 10.190933
99550 62321990 2017-06-15 13.434167 60.508167 28.278083 16.173300
100648 63157510 2017-06-15 7.810517 59.133933 28.565683 27.087900
101380 63158530 2017-06-15 7.241083 57.466850 29.115433 -7.467850
102478 63254860 2017-06-15 6.650636 57.272091 29.173409 -13.571667
102844 63255180 2017-06-15 5.942217 62.901967 29.877033 6.626267
103576 63255860 2017-06-15 8.232364 56.526273 28.334909 0.638300
104674 63258900 2017-06-15 11.394667 66.441133 29.527950 19.424917
105040 63258950 2017-06-15 9.665528 67.081306 29.651694 4.542086
105406 63259180 2017-06-15 10.315883 70.122550 29.755983 11.165400
105772 63259190 2017-06-15 8.182300 71.503133 29.743433 2.719310
106504 63259230 2017-06-15 11.710174 69.002391 29.589565 6.724391
108700 63348720 2017-06-15 7.832875 58.230143 28.273804 31.217091
109066 63348750 2017-06-15 5.647766 66.040532 30.124574 2.377617
112360 64111550 2017-06-15 16.753833 67.083500 29.548500 4.374650
112726 64113560 2017-06-15 14.357517 62.191567 28.389650 13.057833
113458 64115560 2017-06-15 16.855283 72.553533 29.724217 10.948783
113824 64117500 2017-06-15 6.810550 71.186017 29.476617 2.463900
84545 147140 2017-06-30 11.557667 61.136333 28.039000 83.892000
85277 147144 2017-06-30 14.291000 70.923667 29.260333 17.542500
99551 62321990 2017-06-30 12.863000 61.727800 27.980800 24.212500
100649 63157510 2017-06-30 8.170833 60.425000 28.591667 25.888400
101381 63158530 2017-06-30 7.562400 57.591200 28.432600 2.102750
102845 63255180 2017-06-30 6.457000 63.455750 29.492250 26.794667
104675 63258900 2017-06-30 10.611667 67.725333 29.285333 11.653500
105407 63259180 2017-06-30 9.708250 71.031750 29.490500 17.048000
112361 64111550 2017-06-30 15.278385 67.633154 28.739692 14.127583
112727 64113560 2017-06-30 13.827154 62.739077 27.976923 0.487667
113459 64115560 2017-06-30 16.871250 73.274250 28.602000 -0.189000
113825 64117500 2017-06-30 6.149231 71.412077 29.282000 3.511583
vn spd var_lat var_lon var_tmp chlor_a \
366 0.357733 7.394967 0.001451 0.005610 1000.000000 NaN
732 -5.285617 15.006967 0.000063 0.000118 1000.000000 NaN
1098 -14.097033 18.695917 0.000067 0.000129 0.003614 NaN
1464 -4.513033 13.965250 0.000055 0.000102 0.088623 NaN
2196 -5.424417 27.865400 0.000057 0.000106 0.001731 NaN
2928 -9.921900 16.832533 0.000149 0.000362 0.003382 NaN
3294 16.559017 36.755683 0.000061 0.000116 1000.000000 0.321245
4026 -18.675465 26.752744 0.000066 0.000129 0.003705 0.328691
4392 -14.960467 27.234933 0.000053 0.000098 0.003538 0.096233
4758 1.993683 42.610483 0.000055 0.000102 0.003553 0.094689
5490 -0.021000 10.808286 0.000056 0.000103 0.003749 0.109962
5856 -11.504476 13.484762 0.000061 0.000114 0.003594 0.115337
6222 -15.195200 17.403650 0.000075 0.000154 0.003670 0.165898
6588 -5.381333 16.793444 0.000047 0.000087 0.003738 0.338628
6954 5.554625 28.943938 0.000052 0.000094 0.003575 0.177327
7686 4.269317 24.302933 0.000058 0.000110 0.001807 0.412506
8052 46.740240 81.157560 0.000079 0.000162 0.001818 0.501909
8418 -6.248550 46.052550 0.000046 0.000087 0.001837 0.224649
8784 1.335217 35.355967 0.000059 0.000110 0.001815 0.344161
9150 2.107750 34.797783 0.000057 0.000106 0.001772 0.110550
9516 -39.532167 44.367833 0.000055 0.000102 0.001691 0.419638
9882 -20.354250 26.786467 0.000060 0.000112 0.001666 NaN
10248 -12.078667 20.477317 0.000061 0.000114 0.001750 0.236139
10614 -11.221400 13.610417 0.000058 0.000109 0.001740 NaN
10980 -16.828850 20.732050 0.000061 0.000115 0.001779 NaN
11346 -1.815867 10.339867 0.000064 0.000124 0.001786 NaN
86010 -20.248500 29.976417 0.000085 0.000176 0.001942 0.175341
367 -8.716150 19.262550 0.001116 0.004011 1000.000000 0.137352
733 -23.323367 26.479183 0.000065 0.000124 1000.000000 0.099238
1099 6.272917 13.490517 0.000059 0.000109 0.003728 0.324946
... ... ... ... ... ... ...
85276 -2.238083 14.281700 0.000347 0.000176 0.002037 NaN
99550 -6.508667 18.012117 0.002924 0.002439 0.001867 NaN
100648 6.130500 34.657567 0.000010 0.000008 0.001766 0.314935
101380 6.872683 28.816750 0.000076 0.000041 0.001765 0.144886
102478 -13.335286 21.792000 0.002290 0.001733 0.001853 0.191329
102844 6.505583 14.081183 0.000066 0.000034 0.001887 0.078646
103576 0.823200 1.099600 0.018849 0.019519 0.002027 0.260779
104674 -5.209017 26.490717 0.000199 0.000106 0.001803 NaN
105040 -28.119771 31.811457 0.000415 0.000222 0.001851 0.104562
105406 -7.142583 15.643117 0.000918 0.000631 0.001846 0.082649
105772 -7.233862 8.656517 0.007749 0.007112 0.001956 0.112472
106504 -17.843696 21.039565 0.005488 0.004817 0.001931 0.041524
108700 -17.404636 39.430727 0.008580 0.008565 0.001927 0.223120
109066 -11.912617 15.776319 0.000015 0.000010 0.001807 0.093046
112360 -12.741350 17.871400 0.000004 0.000006 0.001684 NaN
112726 -6.607233 16.582283 0.000069 0.000034 0.001684 NaN
113458 -1.390750 16.621250 0.000003 0.000005 0.001686 NaN
113824 -6.642417 15.401850 0.000003 0.000005 0.001684 0.124498
84545 -12.843000 85.019000 0.000032 0.000014 0.001585 0.424789
85277 -15.481500 23.507500 0.000129 0.000055 0.001716 NaN
99551 -13.379500 27.666750 0.008339 0.007574 0.002022 NaN
100649 5.273200 27.029800 0.000015 0.000010 0.001986 0.129658
101381 -33.419500 33.622250 0.000004 0.000006 0.001870 0.149060
102845 -4.388000 27.262333 0.000476 0.000268 0.002092 0.130959
104675 -5.334000 12.912000 0.000794 0.000473 0.002321 NaN
105407 -12.979667 21.434667 0.000538 0.000273 0.002068 NaN
112361 -23.375000 27.887500 0.000003 0.000005 0.001705 NaN
112727 -8.357417 10.514917 0.000107 0.000049 0.001705 NaN
113459 16.893333 17.142667 0.000003 0.000005 0.001754 NaN
113825 -21.195083 22.225833 0.000003 0.000005 0.001705 0.135096
dist cdm kd490
366 644.346876 NaN NaN
732 304.915617 NaN NaN
1098 790.995091 NaN NaN
1464 312.036955 NaN NaN
2196 105.763836 NaN NaN
2928 312.718966 0.073044 NaN
3294 637.078686 0.037462 0.082638
4026 761.139172 0.030116 0.079110
4392 427.708647 0.014937 0.049980
4758 644.256384 0.012178 0.049653
5490 316.339631 0.012372 0.052449
5856 254.118974 0.014427 0.053984
6222 777.066104 0.019534 0.058026
6588 628.158223 0.031178 0.080774
6954 1012.615255 0.016325 0.061184
7686 756.055492 0.039333 0.091085
8052 306.502480 0.042730 0.099538
8418 103.261978 0.017339 0.070830
8784 763.459988 0.035670 0.084350
9150 642.504705 0.014206 0.050987
9516 85.027550 0.056426 0.090653
9882 170.756169 NaN NaN
10248 367.784917 0.017688 0.070251
10614 680.264624 NaN NaN
10980 131.953916 NaN NaN
11346 620.209000 NaN NaN
86010 898.206857 0.019824 0.058893
367 581.752175 NaN 0.061941
733 160.061930 0.022579 0.050537
1099 733.142554 NaN 0.083765
... ... ... ...
85276 388.223010 NaN NaN
99550 633.297339 NaN NaN
100648 725.658656 0.031678 0.074890
101380 667.943026 0.022053 0.055573
102478 715.916792 0.014561 0.056099
102844 1085.649383 0.007998 0.044243
103576 522.599171 0.025497 0.070162
104674 617.336774 NaN NaN
105040 571.323295 0.011004 0.045220
105406 232.404475 0.010142 0.045343
105772 167.192022 0.013368 0.049120
106504 342.140474 NaN 0.039608
108700 658.910708 0.018644 0.061427
109066 742.014936 0.010242 0.046977
112360 574.311951 NaN NaN
112726 692.122499 NaN NaN
113458 77.225451 NaN NaN
113824 173.402129 0.013759 0.049696
84545 726.254721 0.089067 0.094564
85277 326.533641 NaN NaN
99551 772.348381 NaN NaN
100649 805.220865 0.012746 0.048210
101381 643.899278 0.012251 0.050483
102845 1022.681205 0.011937 0.047742
104675 481.403226 NaN NaN
105407 141.373115 NaN NaN
112361 623.755105 NaN NaN
112727 775.259018 NaN NaN
113459 2.169704 NaN NaN
113825 145.314631 0.014212 0.052090
[2559 rows x 15 columns]
<matplotlib.figure.Figure at 0x11bda3ba8>
In [ ]:
In [ ]:
In [ ]:
In [ ]:
Content source: vyan2000/ocml-public
Similar notebooks: