In [17]:
# Reload when code changed:
%load_ext autoreload
%autoreload 2
%pwd
import os
import sys
path = "../"
sys.path.append(path)
#os.path.abspath("../")
print(os.path.abspath(path))
In [18]:
import pandas as pd
import numpy as np
import json
import timeit
import time
import core
import importlib
import re
importlib.reload(core)
import logging
importlib.reload(core)
try:
logging.shutdown()
importlib.reload(logging)
except:
pass
from event_handler import EventHandler
print(core.__file__)
pd.__version__
Out[18]:
In [19]:
root_directory = 'D:/github/w_vattenstatus/ekostat_calculator'#"../" #os.getcwd()
workspace_directory = root_directory + '/workspaces'
resource_directory = root_directory + '/resources'
user_id = 'test_user'
In [20]:
print(root_directory)
paths = {'user_id': user_id,
'workspace_directory': root_directory + '/workspaces',
'resource_directory': root_directory + '/resources',
'log_directory': 'D:/github' + '/log',
'test_data_directory': 'D:/github' + '/test_data',
'cache_directory': 'D:/github/w_vattenstatus/cache'}
In [21]:
t0 = time.time()
ekos = EventHandler(**paths)
print('-'*50)
print('Time for request: {}'.format(time.time()-t0))
In [22]:
workspace_alias = 'kustzon_selection'
ekos.print_workspaces()
In [23]:
workspace_uuid = ekos.get_unique_id_for_alias(workspace_alias = workspace_alias)
print(workspace_uuid)
In [24]:
workspace_alias = ekos.get_alias_for_unique_id(workspace_uuid = workspace_uuid)
In [25]:
ekos.load_workspace(unique_id = workspace_uuid)
Out[25]:
In [26]:
w = ekos.get_workspace(workspace_uuid = workspace_uuid)
len(w.data_handler.get_all_column_data_df())
Out[26]:
In [27]:
subset_alias = 'SE1_selection'
subset_uuid = ekos.get_unique_id_for_alias(workspace_alias = workspace_alias, subset_alias = subset_alias)
In [28]:
w.get_step_object(step = 3, subset = subset_uuid).calculate_quality_element(quality_element = 'nutrients_sw')
In [29]:
w.get_step_object(step = 3, subset = subset_uuid).quality_element['Nutrients summer-winter'].results.head()
In [17]:
w.get_step_object(step = 3, subset = subset_uuid).calculate_quality_element(subset_unique_id = subset_uuid, quality_element = 'Phytoplankton', class_name = 'QualityElementPhytoplankton')
In [19]:
merge_on = ['VISS_EU_CD', 'WATER_BODY_NAME', 'WATER_TYPE_AREA']
indicator_name = 'qe_nutrients'
# current
df = w.get_step_object(step = 3, subset = subset_uuid).quality_element['Nutrients'].sld.load_df(indicator_name)
kolumner = df.columns
kolumner
Out[19]:
In [21]:
import re
In [22]:
these_cols = [col for col in kolumner if re.search(indicator_name + r'$', col)]
df2 = df[these_cols + merge_on]
df2.rename(columns = {col: col.strip(indicator_name) for col in these_cols})
Out[22]:
In [ ]:
In [23]:
df
Out[23]:
In [66]:
{col: col.strip(indicator_name) for col in these_cols}
Out[66]:
In [77]:
tolset = w.get_step_object(step = 2, subset = subset_uuid).get_indicator_tolerance_settings('indicator_din_winter')
In [96]:
boolean_list = df['STATUS'] == 'HIGH'
df.loc[boolean_list, 'STATUS'] = 'HIGH2'
In [ ]:
w.get_step_object(step = 3, subset = subset_uuid).quality_element['Nutrients'].results[['VISS_EU_CD','WATER_BODY_NAME','WATER_TYPE_AREA','STATUS_NUTRIENTS','mean_EQR','MEAN_N_EQR','EQR_N_winter_mean','global_EQR_ntot_summer','MEAN_P_EQR','EQR_P_winter_mean','global_EQR_ptot_summer'
]].to_csv('D:/Nutrients'+subset_alias+'.txt', float_format='%.3f', header = True, index = None, sep = '\t')
In [ ]:
#ekos.get_workspace(unique_id = workspace_uuid, alias = workspace_alias).delete_alldata_export()
In [ ]:
#%%timeit
ekos.load_data(workspace_uuid = workspace_uuid)
In [ ]:
print('subsetlist', w.get_subset_list())
In [ ]:
w.data_handler.all_data.head()
In [ ]:
w.apply_data_filter(step = 0) # This sets the first level of data filter in the IndexHandler
In [ ]:
#w.copy_subset(source_uuid='default_subset', target_alias='period_2007-2012_refvalues_2017')
In [ ]:
subset_alias = 'period_2007-2012_refvalues_2017'
subset_uuid = ekos.get_unique_id_for_alias(workspace_alias = workspace_alias, subset_alias = subset_alias)
w.set_data_filter(subset = subset_uuid, step=1,
filter_type='include_list',
filter_name='MYEAR',
data=['2007', '2008', '2009', '2010', '2011', '2012'])
w.set_data_filter(subset = subset_uuid, step=1,
filter_type='include_list',
filter_name='viss_eu_cd', data = ['SE584340-174401', 'SE581700-113000', 'SE654470-222700', 'SE633000-195000', 'SE625180-181655'])
#data=['SE584340-174401', 'SE581700-113000', 'SE654470-222700', 'SE633000-195000', 'SE625180-181655'])
#wb with no data for din 'SE591400-182320'
f1 = w.get_data_filter_object(subset = subset_uuid, step=1)
print(f1.include_list_filter)
In [ ]:
print('subset_alias:', subset_alias, '\nsubset uuid:', subset_uuid)
In [ ]:
f1 = w.get_data_filter_object(subset = subset_uuid, step=1)
print(f1.include_list_filter)
In [ ]:
w.apply_data_filter(subset = subset_uuid, step = 1)
In [ ]:
w.get_step_object(step = 2, subset = subset_uuid).load_indicator_settings_filters()
In [ ]:
w.get_available_indicators(subset= subset_uuid, step=2)
In [ ]:
#list(zip(typeA_list, df_step1.WATER_TYPE_AREA.unique()))
#indicator_list = w.get_available_indicators(subset= subset_uuid, step=2)
indicator_list = ['oxygen','din_winter','ntot_summer', 'ntot_winter', 'dip_winter', 'ptot_summer', 'ptot_winter','bqi', 'biov', 'chl', 'secchi']
#indicator_list = ['din_winter','ntot_summer', 'ntot_winter', 'dip_winter', 'ptot_summer', 'ptot_winter']
#indicator_list = ['biov', 'chl']
#indicator_list = ['bqi', 'secchi']
#indicator_list = ['bqi', 'secchi'] + ['biov', 'chl'] + ['din_winter']
indicator_list.remove('din_winter')
# indicator_list = ['dip_winter', 'ntot_winter', 'ptot_winter']
indicator_list = ['indicator_' + indicator for indicator in indicator_list]
In [ ]:
print('apply indicator data filter to {}'.format(indicator_list))
for indicator in indicator_list:
w.apply_indicator_data_filter(step = 2,
subset = subset_uuid,
indicator = indicator)#,
# water_body_list = test_wb)
#print(w.mapping_objects['water_body'][wb])
#print('*************************************')
#df = w.get_filtered_data(subset = subset_uuid, step = 'step_2', water_body = 'SE625180-181655', indicator = 'indicator_din_winter').dropna(subset = ['DIN'])
In [ ]:
w.get_step_object(step = 3, subset = subset_uuid).indicator_setup(subset_unique_id = subset_uuid, indicator_list = indicator_list)
In [ ]:
def concat_df(df, save_df, filename, water_body, indicator_object):
#concatenate results
if type(save_df) is pd.DataFrame:
save_df = pd.concat([save_df, df])
elif os.path.exists(indicator_object.result_directory + filename + '.txt'):
save_df = indicator_object.sld.load_df(file_name = filename)
save_df = pd.concat([save_df, df])
else:
save_df = df
#save_df['new_index'] = [str(ix) +'_' + wb for ix, ind, wb in zip(save_df.index, indicator_object.name, save_df.VISS_EU_CD)]
#save_df.set_index(keys = 'new_index')
return save_df
In [ ]:
w.get_step_object(step = 3, subset = subset_uuid).indicator_objects['indicator_biov'].\
get_water_body_indicator_df(water_body = 'SE584340-174401')
save_df = False
for ind in indicator_list:
for wb in ['SE584340-174401', 'SE581700-113000', 'SE654470-222700', 'SE633000-195000', 'SE625180-181655']:
ind_obj = w.get_step_object(step = 3, subset = subset_uuid).indicator_objects[ind]
df = ind_obj.get_water_body_indicator_df(water_body = wb)
save_df = concat_df(df, save_df, 'test', wb, ind_obj)
In [ ]:
save_df['period'] = str(np.min(save_df['YEAR'])) +'-'+ str(np.max(save_df['YEAR']))
save_df.head()
In [ ]:
str(np.min(save_df['YEAR'])) +'-'+ str(np.max(save_df['YEAR']))
In [ ]:
save_df.to_csv('D:/github/ekostat_calcR-master/data/'+subset_alias+'.txt', float_format='%.3f', header = True, index = None, sep = '\t')
In [ ]:
#w.get_step_object(step = 3, subset = subset_uuid).calculate_status(indicator_list = indicator_list)
In [ ]:
subset_alias
In [ ]:
#%matplotlib inline
#import seaborn as sns
#for name, group in ind_obj.classification_results['status_by_date'].groupby('VISS_EU_CD'):
#group['date'] = pd.to_datetime(group.SDATE)
#group.dropna(subset = ['date', 'DIN'])
#sns.tsplot(data = group.to_dict(), time = 'SDATE', value = 'DIN', condition = 'STATUS', legend = True)
# group.plot('SDATE', ['DIN', 'REFERENCE_VALUE'], title = name + group.WATER_TYPE_AREA.values[0], marker ='*')
In [ ]:
#name + group.WATER_TYPE_AREA.values[0]