In [1]:
# Reload when code changed:
%load_ext autoreload
%autoreload 2
%pwd
import sys
import os
path = "../"
sys.path.append(path)
#os.path.abspath("../")
print(os.path.abspath(path))
In [2]:
import os
import core
import importlib
importlib.reload(core)
try:
logging.shutdown()
importlib.reload(logging)
except:
pass
import pandas as pd
import numpy as np
import json
import time
import re
from event_handler import EventHandler
print(core.__file__)
pd.__version__
Out[2]:
In [3]:
user_id_1 = 'user_1'
user_id_2 = 'user_2'
user_1_ws_1 = 'mw1'
print(path)
paths = {'user_id': user_id_1,
'workspace_directory': path + '/workspaces',
'resource_directory': path + '/resources',
'log_directory': path + '/log',
'test_data_directory': path + '/test_data'}
In [25]:
workspace_uuid = '327bbf4f-f367-4d85-80f4-e9151907eadc'
subset_uuid = 'ac45fef1-3042-44f4-bdca-e121d1d93f45'
In [26]:
# ekos = EventHandler(**paths)
# ekos.action_workspace_load_default_data(workspace_uuid)
In [27]:
ekos = EventHandler(**paths)
ekos.action_load_data(workspace_uuid)
Out[27]:
In [28]:
workspace = ekos.workspaces[workspace_uuid]
subset = workspace.get_subset_object(subset_uuid)
df = workspace.data_handler.all_data
In [ ]:
print(workspace.index_handler.booleans.keys())
print(workspace.index_handler.booleans['step_0'].keys())
print(workspace.index_handler.booleans['step_0'][subset_uuid].keys())
print(workspace.index_handler.booleans['step_0'][subset_uuid]['step_1'].keys())
b0 = workspace.index_handler.booleans['step_0']['boolean']
In [ ]:
print(set(df['MYEAR']))
print(set(df.loc[b0, 'MYEAR']))
In [ ]:
# Check filter
ekos.apply_data_filter(workspace_uuid=workspace_uuid,
step=0)
ekos.apply_data_filter(workspace_uuid=workspace_uuid,
subset_uuid=subset_uuid,
step=1)
In [29]:
ih = core.index_handler.new_IndexHandler(workspace_object=workspace, data_handler_object=workspace.data_handler)
In [30]:
ih._get_filter_level('step_0')
Out[30]:
In [31]:
ih._add_filter_level('step_0')
In [ ]:
class Level(dict):
def __init__(self,
level=None,
name=None,
boolean=None,
parent_object=None):
self.level = level
self.name = name
self.boolean = boolean
self.parent_object = parent_object
In [16]:
levels = ['data_0', 'subset', 'data_1', 'water_body', 'indicator']
parent = dict(zip(levels[1:], levels[:-1]))
def _get_parent_level(key):
return parent.get(key, None)
In [17]:
def get_level_info(level, name):
return {'level': level,
'name': name,
'boolean': None,
'parent_level': _get_parent_level(level)}
In [90]:
all_data = {}
def add_boolean(filter_object=None,
df=None,
**kwargs):
given_levels = []
for level in levels:
if level in kwargs:
given_levels.append(level)
# Check if sufficient information
if given_levels != levels[:len(given_levels)]:
print('NO')
return False
# Fix boolean
boolean = 5
data = all_data
for level in given_levels:
name = kwargs[level]
if name not in data:
data[name] = {}
data = data[name]
if 'booelan' not in data:
data['boolean'] = boolean
# level_dict = {'level': level,
# 'name': name,
# 'boolean': boolean,
# 'next_level': {}}
# if data.get('level') == _get_parent_level(level):
# data = data['next_level']
# elif not data:
# data = level_dict
# data = data['next_level']
In [93]:
all_data = {}
add_boolean(data_1='data_1', data_0='data_0', subset='sub1')
add_boolean(data_1='data_1', data_0='data_0', subset='sub2')
In [94]:
all_data
Out[94]:
In [95]:
print(all_data.keys())
print(all_data['data_0'].keys())
print(all_data['data_0']['sub1'].keys())
print(all_data['data_0']['sub1']['data_1'].keys())
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [21]:
all_data['fgsagfsa']['parent_dict'].keys()
Out[21]:
In [138]:
all_data.keys()
Out[138]:
In [8]:
def set_boolean_info(level=None,
name=None,
boolean=None,
parent_dict=None,
**kwargs):
return
In [9]:
data0 = set_boolean_info(level='data_0',
name='data_0',
boolean=1,
parent_dict=False)
sub1 = set_boolean_info(level='subset',
name='ac45fef1-3042-44f4-bdca-e121d1d93f45',
boolean=2,
parent_dict=data0)
data1 = set_boolean_info(level='data_1',
name='data_1',
boolean=3,
parent_dict=sub1)
wb1 = set_boolean_info(level='water_body',
name='wb1',
boolean=5,
parent_dict=data1)
wb2 = set_boolean_info(level='water_body',
name='wb2',
boolean=10,
parent_dict=data1)
In [ ]:
In [10]:
def get_combined_boolean(info_dict):
boolean = info_dict['boolean']
if info_dict['parent_dict'] == False:
return boolean
else:
# print(info_dict.keys())
boolean += get_combined_boolean(info_dict['parent_dict'])
return boolean
In [11]:
get_combined_boolean(wb1)
Out[11]:
In [ ]:
In [ ]:
In [56]:
indicator_din_winter = 'din_winter'
indicator_bqi = 'bqi'
indicator_oxygen = 'oxygen'
type_area = False
viss_eu_cd = 'SE582000-115270'
In [57]:
sf_din_winter = ekos.get_settings_filter_object(workspace_uuid=workspace_uuid,
subset_uuid=subset_uuid,
indicator=indicator_din_winter,
filter_type='data')
sf_bqi = ekos.get_settings_filter_object(workspace_uuid=workspace_uuid,
subset_uuid=subset_uuid,
indicator=indicator_bqi,
filter_type='data')
sf_oxygen = ekos.get_settings_filter_object(workspace_uuid=workspace_uuid,
subset_uuid=subset_uuid,
indicator=indicator_oxygen,
filter_type='data')
In [15]:
sf_oxygen.get_viss_eu_cd_list()
Out[15]:
In [48]:
sf_bqi.get_value(type_area='2',
variable='DEPH_INTERVAL',
water_body=False,
return_series=False)
Out[48]:
In [49]:
sf_din_winter.get_value(type_area='2',
variable='DEPH_INTERVAL',
water_body=False,
return_series=False)
Out[49]:
In [41]:
df = sf_bqi.settings.df.copy(deep=True)
suf = sf_bqi.settings.suf
num = sf_bqi.settings.num
var = sf_bqi.settings.var
print(suf)
print(num)
print(var)
In [46]:
df.loc[(df['TYPE_AREA_NUMBER']=='2') | \
(df['TYPE_AREA_NUMBER']=='4'), var]
Out[46]:
In [44]:
df.loc[(df['TYPE_AREA_NUMBER']==num) & \
(df['VISS_EU_CD'] == 'unspecified'), var]
Out[44]:
In [25]:
sf_bqi.settings.value_series
Out[25]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [298]:
In [ ]:
In [299]:
In [ ]:
In [ ]:
In [300]:
sf = ekos.get_settings_filter_object(workspace_uuid=workspace_uuid,
subset_uuid=subset_uuid,
indicator=indicator,
filter_type='data')
In [301]:
print(len(sf.settings.df.columns))
print(len(sf.settings.columns_in_file))
In [304]:
sf.settings.df
Out[304]:
In [303]:
set_data = {viss_eu_cd:{'DEPH_INTERVAL': [[10, 20]]}}
sf.set_values(set_data)
In [266]:
len(sf.settings.columns)
Out[266]:
In [265]:
len(sf.settings.columns_in_file)
Out[265]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [165]:
df = pd.read_csv('D:/Utveckling/git/ekostat_calculator/workspaces/40e11bb1-49fb-4eb3-bb2b-4d5b9f520b2c/subsets/a15e6f23-5f78-4e78-b3fa-d9cc14e88f93/step_2/settings/indicator_settings/oxygen_test.set', sep='\t', encoding='cp1252')
df = pd.read_csv('D:/Utveckling/git/ekostat_calculator/workspaces/40e11bb1-49fb-4eb3-bb2b-4d5b9f520b2c/subsets/a15e6f23-5f78-4e78-b3fa-d9cc14e88f93/step_2/settings/indicator_settings/BQI.set', sep='\t', encoding='cp1252')
df = df.fillna('')
In [166]:
df.head()
Out[166]:
In [167]:
viss_eu_cd = 'SE582000-115270'
#viss_eu_cd = 'SE592000-184700'
viss_eu_cd = 'SE570900-121060'
type_area = ekos.mapping_objects['water_body'].get_type_area_for_water_body(viss_eu_cd, include_suffix=True).replace('-', '')
water_body_name = ekos.mapping_objects['water_body'].get_display_name(water_body=viss_eu_cd)
print('type_area:', type_area)
print('viss_eu_cd:', viss_eu_cd)
print('water_body_name:', water_body_name)
In [168]:
s = df.loc[((df['Type_Area_number'].astype(str) + df['Type_Area_suffix'])==type_area) & (df['VISS_EU_CD']=='unspecified')].copy()
if not len(s):
type_area = re.findall('\d+', type_area)[0]
s = df.loc[((df['Type_Area_number'].astype(str) + df['Type_Area_suffix'])==type_area) & (df['VISS_EU_CD']=='unspecified')].copy()
In [169]:
s['VISS_EU_CD'] = viss_eu_cd
s['WATERBODY_NAME'] = water_body_name
In [170]:
type(s)
Out[170]:
In [171]:
df2 = df.append(s)
In [172]:
wbm = ekos.mapping_objects['water_body']
In [173]:
wbm.get_display_name(water_body=viss_eu_cd)
Out[173]:
In [174]:
df2.tail()
Out[174]:
In [195]:
print(viss_eu_cd)
print()
matching_columns = ['WATERBODY_NAME', 'tolerance_MIN_NR_YEARS_int',
'tolerance_BOTTOM_WATER_int', 'MONTH_LIST_int',
'filter_DEPH_INTERVAL_int']
for viss in set(df2['VISS_EU_CD']):
if viss =='unspecified':
continue
type_area = ekos.mapping_objects['water_body'].get_type_area_for_water_body(viss, include_suffix=True).replace('-', '')
type_area_series = df2.loc[((df2['Type_Area_number'].astype(str) + df2['Type_Area_suffix'])==type_area) & (df2['VISS_EU_CD']=='unspecified')].copy()
if not len(type_area_series):
type_area = re.findall('\d+', type_area)[0]
type_area_series = df2.loc[((df2['Type_Area_number'].astype(str) + df2['Type_Area_suffix'])==type_area) & (df2['VISS_EU_CD']=='unspecified')].copy()
viss_series = df2.loc[df2['VISS_EU_CD']==viss_eu_cd, :]
for col in matching_columns:
if col not in viss_series.columns:
continue
if list(viss_series[col].values) != list(type_area_series[col].values):
print('+', viss, type_area)
In [220]:
df2.loc[df2['Type_Area_number']==1, :]
Out[220]:
In [ ]:
In [ ]:
In [217]:
df3 = df2.copy(deep=True)
df3.reset_index(inplace=True)
In [218]:
viss_eu_cd_boolean = df3['VISS_EU_CD']==viss_eu_cd
In [ ]:
In [219]:
import numpy as np
df3.drop(np.where(viss_eu_cd_boolean)[0])
Out[219]:
In [215]:
df3
Out[215]:
In [ ]:
In [ ]:
In [188]:
import codecs
hp = {}
with codecs.open('D:/Utveckling/git/ekostat_calculator/resources/mappings/indicator_settings_homogeneous_parameters.txt') as fid:
for line in fid:
line = line.strip()
if line:
indicator, par = [item.strip() for item in line.split('\t')]
if not hp.get(indicator):
hp[indicator] = []
hp[indicator].append(par)
In [189]:
hp
Out[189]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]: