In [17]:
# Reload when code changed:
%load_ext autoreload
%autoreload 2
%pwd
import os 
import sys
path = "../"
sys.path.append(path)
#os.path.abspath("../")
print(os.path.abspath(path))


The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload
D:\github\w_vattenstatus\ekostat_calculator

In [18]:
import pandas as pd
import numpy as np
import json
import timeit
import time
import core
import importlib
import re
importlib.reload(core)
import logging
importlib.reload(core) 
try:
    logging.shutdown()
    importlib.reload(logging)
except:
    pass
from event_handler import EventHandler
print(core.__file__)
pd.__version__


..\core\__init__.py
Out[18]:
'0.19.2'

Load directories


In [19]:
root_directory = 'D:/github/w_vattenstatus/ekostat_calculator'#"../" #os.getcwd()
workspace_directory = root_directory + '/workspaces' 
resource_directory = root_directory + '/resources'

user_id = 'test_user'

Initiate EventHandler


In [20]:
print(root_directory)
paths = {'user_id': user_id, 
         'workspace_directory': root_directory + '/workspaces', 
         'resource_directory': root_directory + '/resources', 
         'log_directory': 'D:/github' + '/log', 
         'test_data_directory': 'D:/github' + '/test_data',
         'cache_directory': 'D:/github/w_vattenstatus/cache'}


D:/github/w_vattenstatus/ekostat_calculator

In [21]:
t0 = time.time()
ekos = EventHandler(**paths)
print('-'*50)
print('Time for request: {}'.format(time.time()-t0))


2018-12-12 10:30:41,530	logger.py	85	add_log	DEBUG	
2018-12-12 10:30:41,530	logger.py	86	add_log	DEBUG	========================================================================================================================
2018-12-12 10:30:41,530	logger.py	87	add_log	DEBUG	### Log added for log_id "event_handler" at locaton: D:\github\log\main_event_handler.log
2018-12-12 10:30:41,530	logger.py	88	add_log	DEBUG	------------------------------------------------------------------------------------------------------------------------
2018-12-12 10:30:41,530	event_handler.py	117	__init__	DEBUG	Start EventHandler: event_handler
2018-12-12 10:30:41,531	event_handler.py	157	_load_mapping_objects	DEBUG	Loading mapping files from original files.
====================================================================================================
event_handler
D:/github/log
main
----------------------------------------------------------------------------------------------------
2018-12-12 10:30:45,498	event_handler.py	128	__init__	DEBUG	Time for mapping: 3.9667999744415283
2018-12-12 10:30:45,498	event_handler.py	133	__init__	DEBUG	Time for initiating EventHandler: 3.9678001403808594
--------------------------------------------------
Time for request: 3.9678001403808594

Load existing workspace


In [22]:
workspace_alias = 'kustzon_selection'
ekos.print_workspaces()


====================================================================================================
Current workspaces for user are:

uuid                                    alias                         status                        
----------------------------------------------------------------------------------------------------
default_workspace                       default_workspace             readable                      
e86ae1c5-d241-46a4-9236-59524b44e500    lena_indicator                editable                      
2c27da69-6035-418b-8f5e-bc8ef8e6320b    kuszonsmodellen               editable                      
78bd7584-5de1-45ca-9176-09a998a7e734    kustzonsmodellen_3daydata     editable                      
6f85f2fc-dcce-4bd4-9fc4-26fc14f9ad0c    waters_export                 editable                      
c876d9b9-e68c-476c-88c6-dee685d70334    satellit                      editable                      
09d6b6d4-4177-4562-8893-b2200ffc0472    kustzon_SE1                   editable                      
62b3b0e7-67b8-41f6-b8c1-8e50073480b9    kustzon_selection             editable                      
====================================================================================================

In [23]:
workspace_uuid = ekos.get_unique_id_for_alias(workspace_alias = workspace_alias)
print(workspace_uuid)


62b3b0e7-67b8-41f6-b8c1-8e50073480b9

In [24]:
workspace_alias = ekos.get_alias_for_unique_id(workspace_uuid = workspace_uuid)

In [25]:
ekos.load_workspace(unique_id = workspace_uuid)


2018-12-12 10:31:02,098	event_handler.py	3071	load_workspace	DEBUG	Trying to load new workspace "62b3b0e7-67b8-41f6-b8c1-8e50073480b9" with alias "kustzon_selection"
2018-12-12 10:31:02,235	logger.py	85	add_log	DEBUG	
2018-12-12 10:31:02,235	logger.py	86	add_log	DEBUG	========================================================================================================================
2018-12-12 10:31:02,250	logger.py	87	add_log	DEBUG	### Log added for log_id "7512aff0-aa3a-44c4-83d1-cb24596c4d50" at locaton: D:\github\w_vattenstatus\ekostat_calculator\workspaces\62b3b0e7-67b8-41f6-b8c1-8e50073480b9\log\subset_7512aff0-aa3a-44c4-83d1-cb24596c4d50.log
2018-12-12 10:31:02,250	logger.py	88	add_log	DEBUG	------------------------------------------------------------------------------------------------------------------------
2018-12-12 10:31:02,385	logger.py	85	add_log	DEBUG	
2018-12-12 10:31:02,385	logger.py	86	add_log	DEBUG	========================================================================================================================
2018-12-12 10:31:02,385	logger.py	87	add_log	DEBUG	### Log added for log_id "default_subset" at locaton: D:\github\w_vattenstatus\ekostat_calculator\workspaces\62b3b0e7-67b8-41f6-b8c1-8e50073480b9\log\subset_default_subset.log
2018-12-12 10:31:02,385	logger.py	88	add_log	DEBUG	------------------------------------------------------------------------------------------------------------------------
====================================================================================================
7512aff0-aa3a-44c4-83d1-cb24596c4d50
D:/github/w_vattenstatus/ekostat_calculator/workspaces/62b3b0e7-67b8-41f6-b8c1-8e50073480b9/log
subset
----------------------------------------------------------------------------------------------------
====================================================================================================
default_subset
D:/github/w_vattenstatus/ekostat_calculator/workspaces/62b3b0e7-67b8-41f6-b8c1-8e50073480b9/log
subset
----------------------------------------------------------------------------------------------------
2018-12-12 10:31:02,569	logger.py	85	add_log	DEBUG	
2018-12-12 10:31:02,584	logger.py	86	add_log	DEBUG	========================================================================================================================
2018-12-12 10:31:02,584	logger.py	87	add_log	DEBUG	### Log added for log_id "62b3b0e7-67b8-41f6-b8c1-8e50073480b9" at locaton: D:\github\w_vattenstatus\ekostat_calculator\workspaces\62b3b0e7-67b8-41f6-b8c1-8e50073480b9\log\workspace_62b3b0e7-67b8-41f6-b8c1-8e50073480b9.log
2018-12-12 10:31:02,584	logger.py	88	add_log	DEBUG	------------------------------------------------------------------------------------------------------------------------
2018-12-12 10:31:02,584	event_handler.py	3089	load_workspace	INFO	Workspace "62b3b0e7-67b8-41f6-b8c1-8e50073480b9" with alias "kustzon_selection loaded."
====================================================================================================
62b3b0e7-67b8-41f6-b8c1-8e50073480b9
D:/github/w_vattenstatus/ekostat_calculator/workspaces/62b3b0e7-67b8-41f6-b8c1-8e50073480b9/log
workspace
----------------------------------------------------------------------------------------------------
Out[25]:
True

QUALITY ELEMENTS


In [26]:
w = ekos.get_workspace(workspace_uuid = workspace_uuid)
len(w.data_handler.get_all_column_data_df())


Out[26]:
0

In [27]:
subset_alias = 'SE1_selection'
subset_uuid = ekos.get_unique_id_for_alias(workspace_alias = workspace_alias, subset_alias = subset_alias)

In [28]:
w.get_step_object(step = 3, subset = subset_uuid).calculate_quality_element(quality_element = 'nutrients_sw')


QualityElementBase
********
nutrients_sw

In [29]:
w.get_step_object(step = 3, subset = subset_uuid).quality_element['Nutrients summer-winter'].results.head()


---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
<ipython-input-29-fb94b95aa5a1> in <module>()
----> 1 w.get_step_object(step = 3, subset = subset_uuid).quality_element['Nutrients summer-winter'].results.head()

KeyError: 'Nutrients summer-winter'

In [17]:
w.get_step_object(step = 3, subset = subset_uuid).calculate_quality_element(subset_unique_id = subset_uuid, quality_element = 'Phytoplankton', class_name = 'QualityElementPhytoplankton')


********
phytoplankton
phytoplankton

In [19]:
merge_on = ['VISS_EU_CD', 'WATER_BODY_NAME', 'WATER_TYPE_AREA']
indicator_name = 'qe_nutrients'
# current
df = w.get_step_object(step = 3, subset = subset_uuid).quality_element['Nutrients'].sld.load_df(indicator_name)
kolumner = df.columns
kolumner


Out[19]:
Index(['STATIONS_USED_indicator_dip_winter', 'STATUS_indicator_dip_winter',
       'VISS_EU_CD', 'WATER_BODY_NAME', 'WATER_TYPE_AREA',
       'YEAR_count_indicator_dip_winter', 'global_EQR_indicator_dip_winter',
       'local_EQR_indicator_dip_winter', 'new_index_indicator_dip_winter',
       'ok_indicator_dip_winter', 'p_ges_indicator_dip_winter',
       'variance_indicator_dip_winter', 'index_column_indicator_dip_winter',
       'STATIONS_USED_indicator_ptot_winter', 'STATUS_indicator_ptot_winter',
       'YEAR_count_indicator_ptot_winter', 'global_EQR_indicator_ptot_winter',
       'local_EQR_indicator_ptot_winter', 'new_index_indicator_ptot_winter',
       'ok_indicator_ptot_winter', 'p_ges_indicator_ptot_winter',
       'variance_indicator_ptot_winter', 'index_column_indicator_ptot_winter',
       'EQR_P_winter_mean', 'STATUS_P_winter',
       'STATIONS_USED_indicator_ptot_summer', 'STATUS_indicator_ptot_summer',
       'YEAR_count_indicator_ptot_summer', 'global_EQR_indicator_ptot_summer',
       'local_EQR_indicator_ptot_summer', 'new_index_indicator_ptot_summer',
       'ok_indicator_ptot_summer', 'p_ges_indicator_ptot_summer',
       'variance_indicator_ptot_summer', 'index_column_indicator_ptot_summer',
       'MEAN_P_EQR', 'STATUS_P', 'STATIONS_USED_indicator_din_winter',
       'STATUS_indicator_din_winter', 'YEAR_count_indicator_din_winter',
       'global_EQR_indicator_din_winter', 'local_EQR_indicator_din_winter',
       'new_index_indicator_din_winter', 'ok_indicator_din_winter',
       'p_ges_indicator_din_winter', 'variance_indicator_din_winter',
       'index_column_indicator_din_winter',
       'STATIONS_USED_indicator_ntot_winter', 'STATUS_indicator_ntot_winter',
       'YEAR_count_indicator_ntot_winter', 'global_EQR_indicator_ntot_winter',
       'local_EQR_indicator_ntot_winter', 'new_index_indicator_ntot_winter',
       'ok_indicator_ntot_winter', 'p_ges_indicator_ntot_winter',
       'variance_indicator_ntot_winter', 'index_column_indicator_ntot_winter',
       'EQR_N_winter_mean', 'STATUS_N_winter',
       'STATIONS_USED_indicator_ntot_summer', 'STATUS_indicator_ntot_summer',
       'YEAR_count_indicator_ntot_summer', 'global_EQR_indicator_ntot_summer',
       'local_EQR_indicator_ntot_summer', 'new_index_indicator_ntot_summer',
       'ok_indicator_ntot_summer', 'p_ges_indicator_ntot_summer',
       'variance_indicator_ntot_summer', 'index_column_indicator_ntot_summer',
       'MEAN_N_EQR', 'STATUS_N', 'mean_EQR', 'STATUS_NUTRIENTS'],
      dtype='object')

In [21]:
import re

In [22]:
these_cols = [col for col in kolumner if re.search(indicator_name + r'$', col)]
df2 = df[these_cols + merge_on]
df2.rename(columns = {col: col.strip(indicator_name) for col in these_cols})


Out[22]:
VISS_EU_CD WATER_BODY_NAME WATER_TYPE_AREA
0 SE592000-184700 Kanholmsfjärden 12n - Östergötlands och Stockholms skärgård. M...
1 SE581700-113000 Gullmarn centralbassäng 02 - Västkustens fjordar
2 SE561400-161201 S Kalmarsunds utsjövatten 09 - Blekinge skärgård och Kalmarsund. Yttre k...
3 SE562000-123800 Skälderviken 05 - Södra Hallands och norra Öresunds kustvatten
4 SE633000-195000 Örefjärden 20 - Norra Kvarkens inre kustvatten

In [ ]:


In [23]:
df


Out[23]:
STATIONS_USED_indicator_dip_winter STATUS_indicator_dip_winter VISS_EU_CD WATER_BODY_NAME WATER_TYPE_AREA YEAR_count_indicator_dip_winter global_EQR_indicator_dip_winter local_EQR_indicator_dip_winter new_index_indicator_dip_winter ok_indicator_dip_winter ... local_EQR_indicator_ntot_summer new_index_indicator_ntot_summer ok_indicator_ntot_summer p_ges_indicator_ntot_summer variance_indicator_ntot_summer index_column_indicator_ntot_summer MEAN_N_EQR STATUS_N mean_EQR STATUS_NUTRIENTS
0 KANHOLMSFJÄRDEN POOR SE592000-184700 Kanholmsfjärden 12n - Östergötlands och Stockholms skärgård. M... 6 0.352877 0.404658 0_SE592000-184700 True ... 0.772898 0_SE592000-184700 True NaN NaN 0 0.519586 MODERATE 0.505927 MODERATE
1 SLÄGGÖ, INRE GULLMARN/BJÖRKHOLM, ALSBÄCK HIGH SE581700-113000 Gullmarn centralbassäng 02 - Västkustens fjordar 6 0.806206 0.806206 0_SE581700-113000 True ... 0.698768 0_SE581700-113000 True NaN NaN 0 0.579589 MODERATE 0.633686 GOOD
2 REF M1V1, K11-MV POOR SE561400-161201 S Kalmarsunds utsjövatten 09 - Blekinge skärgård och Kalmarsund. Yttre k... 7 0.240423 0.320317 0_SE561400-161201 True ... 0.712249 0_SE561400-161201 True NaN NaN 0 0.517773 MODERATE 0.396591 POOR
3 SKÄLDERVIKEN (S2), SI-2, SKÄLDERVIKEN, S5, S2 MODERATE SE562000-123800 Skälderviken 05 - Södra Hallands och norra Öresunds kustvatten 6 0.582250 0.649587 0_SE562000-123800 True ... 0.742072 0_SE562000-123800 True NaN NaN 0 0.526185 MODERATE 0.532146 MODERATE
4 B7, NB1 / B3 MODERATE SE633000-195000 Örefjärden 20 - Norra Kvarkens inre kustvatten 7 0.551692 0.614446 0_SE633000-195000 True ... 0.944530 0_SE633000-195000 True NaN NaN 0 0.759925 GOOD 0.704021 GOOD

5 rows × 73 columns


In [66]:
{col: col.strip(indicator_name) for col in these_cols}


Out[66]:
{'STATUS_indicator_n_winter': 'STATUS',
 'global_EQR_indicator_n_winter': 'global_EQR'}

In [77]:
tolset = w.get_step_object(step = 2, subset = subset_uuid).get_indicator_tolerance_settings('indicator_din_winter')

In [96]:
boolean_list = df['STATUS'] == 'HIGH'
df.loc[boolean_list, 'STATUS'] = 'HIGH2'

In [ ]:
w.get_step_object(step = 3, subset = subset_uuid).quality_element['Nutrients'].results[['VISS_EU_CD','WATER_BODY_NAME','WATER_TYPE_AREA','STATUS_NUTRIENTS','mean_EQR','MEAN_N_EQR','EQR_N_winter_mean','global_EQR_ntot_summer','MEAN_P_EQR','EQR_P_winter_mean','global_EQR_ptot_summer'
]].to_csv('D:/Nutrients'+subset_alias+'.txt', float_format='%.3f', header = True, index = None, sep = '\t')

Load all data in workspace


In [ ]:
#ekos.get_workspace(unique_id = workspace_uuid, alias = workspace_alias).delete_alldata_export()

In [ ]:
#%%timeit
ekos.load_data(workspace_uuid = workspace_uuid)

In [ ]:
print('subsetlist', w.get_subset_list())

Step 0


In [ ]:
w.data_handler.all_data.head()

Apply first data filter


In [ ]:
w.apply_data_filter(step = 0) # This sets the first level of data filter in the IndexHandler

Step 1

Set subset filter


In [ ]:
#w.copy_subset(source_uuid='default_subset', target_alias='period_2007-2012_refvalues_2017')

In [ ]:
subset_alias = 'period_2007-2012_refvalues_2017'
subset_uuid = ekos.get_unique_id_for_alias(workspace_alias = workspace_alias, subset_alias = subset_alias)
w.set_data_filter(subset = subset_uuid, step=1, 
                         filter_type='include_list', 
                         filter_name='MYEAR', 
                         data=['2007', '2008', '2009', '2010', '2011', '2012']) 
w.set_data_filter(subset = subset_uuid, step=1, 
                         filter_type='include_list', 
                         filter_name='viss_eu_cd', data = ['SE584340-174401', 'SE581700-113000', 'SE654470-222700', 'SE633000-195000', 'SE625180-181655'])
                         #data=['SE584340-174401', 'SE581700-113000', 'SE654470-222700', 'SE633000-195000', 'SE625180-181655']) 
                         #wb with no data for din 'SE591400-182320'


f1 = w.get_data_filter_object(subset = subset_uuid, step=1) 
print(f1.include_list_filter)

In [ ]:
print('subset_alias:', subset_alias, '\nsubset uuid:', subset_uuid)

In [ ]:
f1 = w.get_data_filter_object(subset = subset_uuid, step=1) 
print(f1.include_list_filter)

Apply step 1 datafilter to subset


In [ ]:
w.apply_data_filter(subset = subset_uuid, step = 1)

Step 2

Load indicator settings filter


In [ ]:
w.get_step_object(step = 2, subset = subset_uuid).load_indicator_settings_filters()

set available indicators


In [ ]:
w.get_available_indicators(subset= subset_uuid, step=2)

Apply indicator data filter


In [ ]:
#list(zip(typeA_list, df_step1.WATER_TYPE_AREA.unique()))
#indicator_list = w.get_available_indicators(subset= subset_uuid, step=2)
indicator_list = ['oxygen','din_winter','ntot_summer', 'ntot_winter', 'dip_winter', 'ptot_summer', 'ptot_winter','bqi', 'biov', 'chl', 'secchi']
#indicator_list = ['din_winter','ntot_summer', 'ntot_winter', 'dip_winter', 'ptot_summer', 'ptot_winter']
#indicator_list = ['biov', 'chl']
#indicator_list = ['bqi', 'secchi']
#indicator_list = ['bqi', 'secchi'] + ['biov', 'chl'] + ['din_winter']
indicator_list.remove('din_winter')
# indicator_list = ['dip_winter', 'ntot_winter', 'ptot_winter']
indicator_list = ['indicator_' + indicator for indicator in indicator_list]

In [ ]:
print('apply indicator data filter to {}'.format(indicator_list))
for indicator in indicator_list:
    w.apply_indicator_data_filter(step = 2, 
                          subset = subset_uuid, 
                          indicator = indicator)#,
                        # water_body_list = test_wb)
    #print(w.mapping_objects['water_body'][wb])
    #print('*************************************')

#df = w.get_filtered_data(subset = subset_uuid, step = 'step_2', water_body = 'SE625180-181655', indicator = 'indicator_din_winter').dropna(subset = ['DIN'])

Step 3

Set up indicator objects


In [ ]:
w.get_step_object(step = 3, subset = subset_uuid).indicator_setup(subset_unique_id = subset_uuid, indicator_list = indicator_list)

In [ ]:
def concat_df(df, save_df, filename, water_body, indicator_object):
            #concatenate results
            if type(save_df) is pd.DataFrame:
                save_df = pd.concat([save_df, df])
            elif os.path.exists(indicator_object.result_directory + filename + '.txt'):
                save_df = indicator_object.sld.load_df(file_name = filename)
                
                save_df = pd.concat([save_df, df])
            else:
                save_df = df
            #save_df['new_index'] = [str(ix) +'_' + wb for ix, ind, wb in zip(save_df.index, indicator_object.name, save_df.VISS_EU_CD)]
            #save_df.set_index(keys = 'new_index')
            return save_df

In [ ]:
w.get_step_object(step = 3, subset = subset_uuid).indicator_objects['indicator_biov'].\
    get_water_body_indicator_df(water_body = 'SE584340-174401')

save_df = False
for ind in indicator_list:
    for wb in ['SE584340-174401', 'SE581700-113000', 'SE654470-222700', 'SE633000-195000', 'SE625180-181655']:
        ind_obj = w.get_step_object(step = 3, subset = subset_uuid).indicator_objects[ind]
        df = ind_obj.get_water_body_indicator_df(water_body = wb)
        save_df = concat_df(df, save_df, 'test', wb, ind_obj)

In [ ]:
save_df['period'] = str(np.min(save_df['YEAR'])) +'-'+ str(np.max(save_df['YEAR']))
save_df.head()

In [ ]:
str(np.min(save_df['YEAR'])) +'-'+ str(np.max(save_df['YEAR']))

In [ ]:
save_df.to_csv('D:/github/ekostat_calcR-master/data/'+subset_alias+'.txt', float_format='%.3f', header = True, index = None, sep = '\t')

CALCULATE STATUS


In [ ]:
#w.get_step_object(step = 3, subset = subset_uuid).calculate_status(indicator_list = indicator_list)

CALCULATE QUALITY ELEMENTS


In [ ]:
subset_alias

Plotting results


In [ ]:
#%matplotlib inline
#import seaborn as sns
#for name, group in ind_obj.classification_results['status_by_date'].groupby('VISS_EU_CD'):
    #group['date'] = pd.to_datetime(group.SDATE)
    #group.dropna(subset = ['date', 'DIN'])
    #sns.tsplot(data = group.to_dict(), time = 'SDATE', value = 'DIN', condition = 'STATUS', legend = True)
#    group.plot('SDATE', ['DIN', 'REFERENCE_VALUE'], title = name + group.WATER_TYPE_AREA.values[0], marker ='*')

In [ ]:
#name + group.WATER_TYPE_AREA.values[0]