Step by step workflow test notebook



In [1]:

    
%load_ext autoreload
%autoreload 2



In [2]:

    
import logging as log
import os
from dkrz_forms import form_handler
from test import test_steps
from dkrz_forms.config.project_config import FORM_REPO
vprint = form_handler.vprint









    



Project directory:  /home/stephan/tmp/Repos/form_repo/test

Generate a personal submission form based on a project template



In [3]:

    
from dkrz_forms import utils



In [4]:

    
par = {'a':'b','__doc__':"Test Test"}

tst = utils.Form(par)



In [5]:

    
tst['a']
?tst



In [6]:

    
### form generation
import pprint
test_steps.test_init_form()

sf = test_steps.sf
#sf = form_handler.set_doc(sf,DOCUMENTATION_DICT)

#pprint.pprint(form_handler.form_to_dict(sf))









    



Form Handler: Initialized form for project: test
test
/home/stephan/tmp/Repos/form_repo/test
test_testsuite_1234.ipynb
/home/stephan/Repos/ENES-EUDAT/submission_forms/test/forms/test/test_testsuite_1234.ipynb
/home/stephan/tmp/Repos/form_repo/test/test_testsuite_1234.ipynb
submission form intitialized: sf
(For the curious: the sf object is used in the following to store and manage all your information)



In [7]:

    
?sf.sub.entity_out.form



In [8]:

    
?sf

Retrieve the private template (pwd protected) and make changes form



In [9]:

    
test_steps.test_generate_submission_form()









    



Form Handler: Initialized form for project: test
test
/home/stephan/tmp/Repos/form_repo/test
test_testsuite_1234.ipynb
TTT:  store in keystore /home/stephan/tmp/Repos/form_repo/keystore
Keystore:  {'P3SDHX': {'first_name': 'unit_tester', 'last_name': 'testsuite', 'form_repo': '/home/stephan/tmp/Repos/form_repo/test', 'form_path': '/home/stephan/tmp/Repos/form_repo/test/test_testsuite_1234.ipynb', 'project': 'test', 'pwd': 'P3SDHX', 'key': '1234', 'form_name': 'test_testsuite_1234', 'form_json': '/home/stephan/tmp/Repos/form_repo/test/test_testsuite_1234.json', 'email': 'stephan.kindermann@gmail.com'}}
--- copy from: /home/stephan/Repos/ENES-EUDAT/submission_forms/dkrz_forms/Templates/test_submission_form.ipynb
--- to:   /home/stephan/tmp/Repos/form_repo/test/test_testsuite_1234.ipynb
--------------------------------------------------------------------
   A submission form was created for you, please visit the following link:
http://localhost:8888/notebooks/Repos/ENES-EUDAT/submission_forms/test/forms/test/test_testsuite_1234.ipynb
--------------------------------------------------------------------
/home/stephan/Repos/ENES-EUDAT/submission_forms/test/forms/test/test_testsuite_1234.ipynb
/home/stephan/tmp/Repos/form_repo/test/test_testsuite_1234.ipynb
 
 ......  initial version saved ...
/home/stephan/Repos/ENES-EUDAT/submission_forms/test/forms/test/test_testsuite_1234.ipynb
/home/stephan/tmp/Repos/form_repo/test/test_testsuite_1234.ipynb
 
  !!  current version saved in repository
  !!  the above link is only valid for the next 5 hours
  !!  to retrieve the form after this use the following link: 
       http://localhost:888/notebooks.tst 
       with the password: P3SDHX
id:  P3SDHX

Retrieve a partially filled form and complete it



In [10]:

    
test_steps.test_form_completion()

print test_steps.sf









    




Form Handler - save form status message:
/home/stephan/Repos/ENES-EUDAT/submission_forms/test/forms/test/test_testsuite_1234.ipynb
/home/stephan/tmp/Repos/form_repo/test/test_testsuite_1234.ipynb
 --- form stored in transfer format in: /home/stephan/tmp/Repos/form_repo/test/test_testsuite_1234.json
 
 --- commit message:[master 19fb239] Form Handler: submission form for user testsuite saved using prefix test_testsuite_1234 ## test: formcompletion()
 1 file changed, 5 insertions(+), 3 deletions(-)
DKRZ Form object: {u'project': u'test', u'sub': DKRZ Form object , u'workflow': [[u'sub', u'data_submission'], [u'rev', u'data_submission_review'], [u'ing', u'data_ingest'], [u'qua', u'data_quality_assurance']], u'rev': DKRZ Form object , u'qua': DKRZ Form object , u'form_dir': u'/home/stephan/Repos/ENES-EUDAT/submission_forms/test/forms/test', u'ing': DKRZ Form object , u'__doc__': u'\n             Form object for project test\n            \n             Workflow step related sub-forms (filled by data managers):\n               - sub: data submission form\n               - rev: data review_form\n               - ing: data ingest form\n               - qua: data quality assurance form\n               - pub: data publication form\n               \n            \n             \n             Each workfow step form is structured according to\n               - entity_in : input information for this step\n               - entity_out: output information for this step\n               - agent: information related to responsible party for this step\n               - activity: information related the workflow step execution\n               \n              End user provided form information is stored in:\n             \n             _this_form_object.sub.entity_out.form  \n             \n             The following end user attributes are defined:\n           \n               - project: project this form is related to\n               - ....\n               \n             '}

Submit a completed form



In [11]:

    
test_steps.test_form_submission()









    



No global submission repo !!!
On branch master
Changes to be committed:
  (use "git reset HEAD <file>..." to unstage)

	modified:   test_testsuite_1234.json

[master 79033b6] Form Handler: submission form for user testsuite saved using prefix test_testsuite_1234 ##
 1 file changed, 5 insertions(+), 5 deletions(-)
Push to global submission repo failed !
Please send form: /home/stephan/Repos/ENES-EUDAT/submission_forms/test/forms/test/test_testsuite_1234.ipynb

to data@dkrz.de with subject "DKRZ data submission form for project" test

Admin workflow step 1: form validation



In [12]:

    
test_steps.test_form_review()









    




Form Handler - save form status message:
/home/stephan/Repos/ENES-EUDAT/submission_forms/test/forms/test/test_testsuite_1234.ipynb
/home/stephan/tmp/Repos/form_repo/test/test_testsuite_1234.ipynb
 --- form stored in transfer format in: /home/stephan/tmp/Repos/form_repo/test/test_testsuite_1234.json
 
 --- commit message:[master f44a4c1] Form Handler: submission form for user testsuite saved using prefix test_testsuite_1234 ## test: form_review()
 1 file changed, 5 insertions(+), 2 deletions(-)

step 2: data ingest



In [13]:

    
test_steps.test_data_ingest()









    




Form Handler - save form status message:
/home/stephan/Repos/ENES-EUDAT/submission_forms/test/forms/test/test_testsuite_1234.ipynb
/home/stephan/tmp/Repos/form_repo/test/test_testsuite_1234.ipynb
 --- form stored in transfer format in: /home/stephan/tmp/Repos/form_repo/test/test_testsuite_1234.json
 
 --- commit message:[master 2d240d5] Form Handler: submission form for user testsuite saved using prefix test_testsuite_1234 ## test: data_ingest()
 1 file changed, 7 insertions(+), 4 deletions(-)

step 3: data quality assurance



In [14]:

    
test_steps.test_data_quality_assurance()









    




Form Handler - save form status message:
/home/stephan/Repos/ENES-EUDAT/submission_forms/test/forms/test/test_testsuite_1234.ipynb
/home/stephan/tmp/Repos/form_repo/test/test_testsuite_1234.ipynb
 --- form stored in transfer format in: /home/stephan/tmp/Repos/form_repo/test/test_testsuite_1234.json
 
 --- commit message:[master 3912658] Form Handler: submission form for user testsuite saved using prefix test_testsuite_1234 ## test: quality_assurance()
 1 file changed, 39 insertions(+), 4 deletions(-)

step 4: data publication



In [15]:

    
test_steps.test_data_publication()









    



---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-15-74be24385b0b> in <module>()
----> 1 test_steps.test_data_publication()

/home/stephan/Repos/ENES-EUDAT/submission_forms/test/test_steps.pyc in test_data_publication()
    179     workflow_form = form_handler.load_workflow_form(FORM_JSON)
    180 
--> 181     publication = workflow_form.pub
    182 
    183     publication.activity.status = "published"

AttributeError: 'Form' object has no attribute 'pub'

step 5: data archival



In [ ]:

    
test_steps.test_data_archival()



In [ ]:

    
sf = test_steps.sf



In [ ]:

    
help(sf.sub.entity_in)



In [ ]:

    
print sf.workflow
for (a,b) in sf.workflow:
    print a,b



In [ ]:

    
error 


SUBMISSION_FORMTEMPLATE_ENTITY_DOC = """
        :param source_path: : 
                            a
                            b
                            c
                            d
                            ,
          args:
             form_version: : 
                            x
                            y
                            z
                            
               """
sf = test_steps.sf
print sf.sub.__dict__
sf.sub.__doc__ = "test test TEst"
sf.sub.entity_out.__doc__="test entity_submission out doc"
sf.rev.entity_out.__doc__=SUBMISSION_FORMTEMPLATE_ENTITY_DOC



In [ ]:



In [ ]:

    
#?sf.rev.entity_out
?sf.rev.entity_out



In [ ]:

    
#import abc

def myprop(x, doc):
    def getx(self):
        return getattr(self, '_' + x)

    def setx(self, val):
        setattr(self, '_' + x, val)

    def delx(self):
        delattr(self, '_' + x)

    return property(getx, setx, delx, doc)

class C(object):
    #__metaclass__=abc.ABCMeta
    a = myprop("a", "Hi, I'm A!")
    b = myprop("b", "Hi, I'm B!")
    d = 5
    
C.d = myprop("d","....DDDDDDDDDDDDDDDD......")
#C.__doc__ = "test test"

co.x = 5
    
co = C()



In [ ]:

    
print C.d



In [ ]: