notebook.community

Edit and run 





In [1]:



    


# coding: utf-8

# In[1]:


import os 
import sys
path = "../"
path = "D:/github/w_vattenstatus/ekostat_calculator"
sys.path.append(path)
#os.path.abspath("../")
print(os.path.abspath(path))


# In[2]:


import pandas as pd
import numpy as np
import json
import timeit
import time
import core
import importlib
importlib.reload(core)
import logging
importlib.reload(core) 
try:
    logging.shutdown()
    importlib.reload(logging)
except:
    pass
from event_handler import EventHandler
print(core.__file__)
pd.__version__



    


















    






D:\github\w_vattenstatus\ekostat_calculator
D:/github/w_vattenstatus/ekostat_calculator\core\__init__.py










    
Out[1]:








'0.19.2'
#

Load directories





In [21]:



    


root_directory = 'D:/github/w_vattenstatus/ekostat_calculator'#"../" #os.getcwd()
workspace_directory = root_directory + '/workspaces' 
resource_directory = root_directory + '/resources'
#alias = 'lena'
user_id = 'test_user' #kanske ska vara off_line user?
# workspace_alias = 'lena_indicator' # kustzonsmodellen_3daydata
workspace_alias = 'kustzonsmodellen_3daydata'

# ## Initiate EventHandler
print(root_directory)
paths = {'user_id': user_id, 
         'workspace_directory': root_directory + '/workspaces', 
         'resource_directory': root_directory + '/resources', 
         'log_directory': 'D:/github' + '/log', 
         'test_data_directory': 'D:/github' + '/test_data',
         'cache_directory': 'D:/github/w_vattenstatus/cache'}

t0 = time.time()
ekos = EventHandler(**paths)
#request = ekos.test_requests['request_workspace_list']
#response = ekos.request_workspace_list(request) 
#ekos.write_test_response('request_workspace_list', response)
print('-'*50)
print('Time for request: {}'.format(time.time()-t0))



    


















    






2018-10-12 15:38:05,187	event_handler.py	117	__init__	DEBUG	Start EventHandler: event_handler
2018-10-12 15:38:05,189	event_handler.py	152	_load_mapping_objects	DEBUG	Loading mapping files from pickle file.










    






D:/github/w_vattenstatus/ekostat_calculator










    






2018-10-12 15:38:05,871	event_handler.py	128	__init__	DEBUG	Time for mapping: 0.6818232536315918
2018-10-12 15:38:05,871	event_handler.py	133	__init__	DEBUG	Time for initiating EventHandler: 0.6848235130310059










    






--------------------------------------------------
Time for request: 0.6848235130310059

















In [11]:



    


###############################################################################################################################
# ### Make a new workspace



    











In [12]:



    


# ekos.copy_workspace(source_uuid='default_workspace', target_alias='kustzonsmodellen_3daydata')



    











In [22]:



    


# ### See existing workspaces and choose workspace name to load
ekos.print_workspaces()
workspace_uuid = ekos.get_unique_id_for_alias(workspace_alias = workspace_alias) #'kuszonsmodellen' lena_indicator 
print(workspace_uuid)

workspace_alias = ekos.get_alias_for_unique_id(workspace_uuid = workspace_uuid)



    


















    






====================================================================================================
Current workspaces for user are:

uuid                                    alias                         status                        
----------------------------------------------------------------------------------------------------
default_workspace                       default_workspace             readable                      
e86ae1c5-d241-46a4-9236-59524b44e500    lena_indicator                editable                      
2c27da69-6035-418b-8f5e-bc8ef8e6320b    kuszonsmodellen               editable                      
78bd7584-5de1-45ca-9176-09a998a7e734    kustzonsmodellen_3daydata     editable                      
====================================================================================================
78bd7584-5de1-45ca-9176-09a998a7e734

















In [23]:



    


###############################################################################################################################
# ### Load existing workspace
ekos.load_workspace(unique_id = workspace_uuid)



    


















    






2018-10-12 15:38:20,849	event_handler.py	3070	load_workspace	DEBUG	Trying to load new workspace "78bd7584-5de1-45ca-9176-09a998a7e734" with alias "kustzonsmodellen_3daydata"
2018-10-12 15:38:21,167	event_handler.py	3088	load_workspace	INFO	Workspace "78bd7584-5de1-45ca-9176-09a998a7e734" with alias "kustzonsmodellen_3daydata loaded."










    
Out[23]:








True

















In [24]:



    


###############################################################################################################################
# ### import data
# ekos.import_default_data(workspace_alias = workspace_alias)



    











In [25]:



    


###############################################################################################################################
# ### Load all data in workspace
# #### if there is old data that you want to remove
ekos.get_workspace(workspace_uuid = workspace_uuid).delete_alldata_export()
ekos.get_workspace(workspace_uuid = workspace_uuid).delete_all_export_data()



    


















    






2018-10-12 15:38:26,679	workspaces.py	1522	delete_all_export_data	DEBUG	All files in export directory are deleted and all "loaded" in datatype_settings is 0.










    






¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤
D:/github/w_vattenstatus/ekostat_calculator/workspaces/78bd7584-5de1-45ca-9176-09a998a7e734/input_data/exports\all_data.pkl
D:/github/w_vattenstatus/ekostat_calculator/workspaces/78bd7584-5de1-45ca-9176-09a998a7e734/input_data/exports\all_data.txt

















In [26]:



    


###############################################################################################################################
# #### to just load existing data in workspace
ekos.load_data(workspace_uuid = workspace_uuid)



    


















    






2018-10-12 15:38:53,771	workspaces.py	1834	load_all_data	DEBUG	All selected data in (status 1 in datatype_settings.txt) is not loaded.
2018-10-12 15:38:53,806	workspaces.py	1917	load_datatype_data	DEBUG	New data files has been loaded for datatype: physicalchemical










    






¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤
D:/github/w_vattenstatus/ekostat_calculator/workspaces/78bd7584-5de1-45ca-9176-09a998a7e734/input_data/exports\all_data.pkl
D:/github/w_vattenstatus/ekostat_calculator/workspaces/78bd7584-5de1-45ca-9176-09a998a7e734/input_data/exports\all_data.txt










    






2018-10-12 15:38:55,463	workspaces.py	1917	load_datatype_data	DEBUG	New data files has been loaded for datatype: physicalchemicalmodel
2018-10-12 15:38:55,464	workspaces.py	1917	load_datatype_data	DEBUG	New data files has been loaded for datatype: chlorophyll
2018-10-12 15:38:55,501	workspaces.py	1917	load_datatype_data	DEBUG	New data files has been loaded for datatype: phytoplankton
2018-10-12 15:38:55,565	workspaces.py	1917	load_datatype_data	DEBUG	New data files has been loaded for datatype: zoobenthos










    






self.all_data 0
MMMMMMMMM
--------------------------------------------------
Total time: 0.9752404689788818
time_preparations              0.0
time_list_group_data:          0.04780101776123047
time_list_calc_integ:          0.04320263862609863
time_list_add_row:             0.4178156852722168
time_all_calculations:         0.880638837814331
time_iterator:                 0.0
time_add_data:                 0.04680061340332031
Done adding integrated_calc "CPHL_INTEG_CALC" using parameter "CPHL_BTL"
time for integrated_calc "CPHL_INTEG_CALC" using parameter "CPHL_BTL is: 0.9752404689788818
Saving data to: D:/github/w_vattenstatus/ekostat_calculator/workspaces/78bd7584-5de1-45ca-9176-09a998a7e734/input_data/exports/all_data.txt










    
Out[26]:








True

















In [27]:



    


############################################################################################################################### 
# ### check workspace data length
w = ekos.get_workspace(workspace_uuid = workspace_uuid)
len(w.data_handler.get_all_column_data_df())



    


















    
Out[27]:








62780

















In [28]:



    


###############################################################################################################################  
# ### see subsets in data  
for subset_uuid in w.get_subset_list():
    print('uuid {} alias {}'.format(subset_uuid, w.uuid_mapping.get_alias(unique_id=subset_uuid)))



    


















    






uuid default_subset alias default_subset
uuid e2ede510-cd06-44f2-a775-8f0c6840057e alias test_kustzon

















In [29]:



    


###############################################################################################################################  
# # Step 0 
print(w.data_handler.all_data.columns)



    


















    






Index(['LONGI_DD', 'DEPH', 'RLABO', 'WATER_DISTRICT', 'STIME', 'WADEP',
       'STATN', 'MYEAR', 'WATER_TYPE_AREA', 'WATER_BODY_NAME', 'VISS_EU_CD',
       'SDATE', 'LATIT_DD', 'AMON', 'NTRI', 'CPHL_BTL', 'NTRZ', 'PHOS', 'NTOT',
       'SECCHI', 'PTOT', 'NTRA', 'origin_dtype', 'origin_file_path', 'DIN',
       'MONTH', 'YEAR', 'POSITION', 'MNDEP', 'MXDEP', 'visit_id_str', 'date',
       'CPHL_INTEG_CALC', 'CPHL_INTEG_CALC_depths', 'CPHL_INTEG_CALC_values',
       'index_column'],
      dtype='object')

















In [30]:



    


###############################################################################################################################    
# ### Apply first data filter 
w.apply_data_filter(step = 0) # This sets the first level of data filter in the IndexHandler



    











In [31]:



    


###############################################################################################################################  
# # Step 1 
# ### make new subset
# w.copy_subset(source_uuid='default_subset', target_alias='test_kustzon')



    











In [32]:



    


###############################################################################################################################
# ### Choose subset name to load
subset_alias = 'test_kustzon'
# subset_alias = 'period_2007-2012_refvalues_2013'
# subset_alias = 'test_subset'
subset_uuid = ekos.get_unique_id_for_alias(workspace_alias = workspace_alias, subset_alias = subset_alias)
print('subset_alias', subset_alias, 'subset_uuid', subset_uuid)



    


















    






subset_alias test_kustzon subset_uuid e2ede510-cd06-44f2-a775-8f0c6840057e
#

Set subset filters





In [33]:



    


# #### year filter
w.set_data_filter(subset = subset_uuid, step=1, 
                         filter_type='include_list', 
                         filter_name='MYEAR', 
                         data=[2007,2008,2009,2010,2011,2012])#['2011', '2012', '2013']) #, 2014, 2015, 2016



    


















    
Out[33]:








True

















In [34]:



    


###############################################################################################################################
# #### waterbody filter
w.set_data_filter(subset = subset_uuid, step=1, 
                         filter_type='include_list', 
                         filter_name='viss_eu_cd', data = []) #'SE584340-174401', 'SE581700-113000', 'SE654470-222700', 'SE633000-195000', 'SE625180-181655'
#                          data=['SE584340-174401', 'SE581700-113000', 'SE654470-222700', 'SE633000-195000', 'SE625180-181655']) 
#                          wb with no data for din 'SE591400-182320'
  
f1 = w.get_data_filter_object(subset = subset_uuid, step=1) 
print(f1.include_list_filter)

print('subset_alias:', subset_alias, '\nsubset uuid:', subset_uuid)

f1 = w.get_data_filter_object(subset = subset_uuid, step=1) 
print(f1.include_list_filter)



    


















    






{'MYEAR': ['2007', '2008', '2009', '2010', '2011', '2012'], 'STATN': [], 'VISS_EU_CD': []}
subset_alias: test_kustzon 
subset uuid: e2ede510-cd06-44f2-a775-8f0c6840057e
{'MYEAR': ['2007', '2008', '2009', '2010', '2011', '2012'], 'STATN': [], 'VISS_EU_CD': []}

















In [35]:



    


###############################################################################################################################    
# ## Apply step 1 datafilter to subset
w.apply_data_filter(subset = subset_uuid, step = 1)
filtered_data = w.get_filtered_data(step = 1, subset = subset_uuid)
print(filtered_data['VISS_EU_CD'].unique())



    


















    






['SE641250-211751' 'SE648760-213140' 'SE652400-223501' 'SE653840-247900'
 'SE653870-235570']

















In [51]:



    


filtered_data[['AMON','NTRA','DIN','CPHL_INTEG_CALC','DEPH']].head()



    


















    
Out[51]:










  
    

      
      AMON
      NTRA
      DIN
      CPHL_INTEG_CALC
      DEPH
    

  
  
    

      0
      0.180
      6.238
      6.42
      NaN
      0.00
    

    

      1
      0.180
      6.238
      6.42
      NaN
      0.25
    

    

      2
      0.179
      6.237
      6.42
      NaN
      0.75
    

    

      3
      0.175
      6.245
      6.42
      NaN
      1.25
    

    

      4
      0.165
      6.266
      6.43
      NaN
      1.75
#

Step 2





In [36]:



    


### Load indicator settings filter 
w.get_step_object(step = 2, subset = subset_uuid).load_indicator_settings_filters()



    











In [37]:



    


############################################################################################################################### 
### set available indicators  
w.get_available_indicators(subset= subset_uuid, step=2)



    


















    
Out[37]:








[]

















In [38]:



    


###############################################################################################################################
# ### choose indicators
#list(zip(typeA_list, df_step1.WATER_TYPE_AREA.unique()))
# indicator_list = ['oxygen','din_winter','ntot_summer', 'ntot_winter', 'dip_winter', 'ptot_summer', 'ptot_winter','bqi', 'biov', 'chl', 'secchi']
# indicator_list = ['din_winter','ntot_summer', 'ntot_winter', 'dip_winter', 'ptot_summer', 'ptot_winter']
#indicator_list = ['biov', 'chl']
# indicator_list = ['bqi', 'biov', 'chl', 'secchi']
#indicator_list = ['bqi', 'secchi'] + ['biov', 'chl'] + ['din_winter']
# indicator_list = ['din_winter','ntot_summer']
# indicator_list = ['indicator_' + indicator for indicator in indicator_list]
indicator_list = w.available_indicators



    











In [39]:



    


###############################################################################################################################  
# ### Apply indicator data filter
print('apply indicator data filter to {}'.format(indicator_list))
for indicator in indicator_list:
    w.apply_indicator_data_filter(step = 2, 
                          subset = subset_uuid, 
                          indicator = indicator)#,
#                         water_body_list = test_wb)
    #print(w.mapping_objects['water_body'][wb])
    #print('*************************************')

#df = w.get_filtered_data(subset = subset_uuid, step = 'step_2', water_body = 'SE625180-181655', indicator = 'indicator_din_winter').dropna(subset = ['DIN'])



    


















    






apply indicator data filter to []
#

Step 3





In [40]:



    


# ### Set up indicator objects
print('indicator set up to {}'.format(indicator_list))
w.get_step_object(step = 3, subset = subset_uuid).indicator_setup(indicator_list = indicator_list)



    


















    






indicator set up to []

















In [41]:



    


###############################################################################################################################
# ### CALCULATE STATUS
print('CALCULATE STATUS to {}'.format(indicator_list))
w.get_step_object(step = 3, subset = subset_uuid).calculate_status(indicator_list = indicator_list)



    


















    






CALCULATE STATUS to []

















In [ ]:



    


###############################################################################################################################  
# ### CALCULATE QUALITY ELEMENTS
w.get_step_object(step = 3, subset = subset_uuid).calculate_quality_element(quality_element = 'nutrients')
# w.get_step_object(step = 3, subset = subset_uuid).calculate_quality_element(quality_element = 'phytoplankton')
# w.get_step_object(step = 3, subset = subset_uuid).calculate_quality_element(quality_element = 'bottomfauna')
# w.get_step_object(step = 3, subset = subset_uuid).calculate_quality_element(quality_element = 'oxygen')
# w.get_step_object(step = 3, subset = subset_uuid).calculate_quality_element(quality_element = 'secchi')
 
# w.get_step_object(step = 3, subset = subset_uuid).calculate_quality_element(subset_unique_id = subset_uuid, quality_element = 'Phytoplankton')

Content source: ekostat/ekostat_calculator

Similar notebooks:

notebook.community | gallery | about