In [25]:
import glob, string, os, re
import yaml, json
from collections import OrderedDict
class UnsortableList(list):
def sort(self, *args, **kwargs):
pass
class UnsortableOrderedDict(OrderedDict):
def items(self, *args, **kwargs):
return UnsortableList(OrderedDict.items(self, *args, **kwargs))
class OrderedDictYAMLLoader(yaml.Loader):
"""
A YAML loader that loads mappings into ordered dictionaries.
"""
def __init__(self, *args, **kwargs):
yaml.Loader.__init__(self, *args, **kwargs)
self.add_constructor(u'tag:yaml.org,2002:map', type(self).construct_yaml_map)
self.add_constructor(u'tag:yaml.org,2002:omap', type(self).construct_yaml_map)
def construct_yaml_map(self, node):
data = OrderedDict()
yield data
value = self.construct_mapping(node)
data.update(value)
def construct_mapping(self, node, deep=False):
if isinstance(node, yaml.MappingNode):
self.flatten_mapping(node)
else:
raise yaml.constructor.ConstructorError(None, None,
'expected a mapping node, but found %s' % node.id, node.start_mark)
mapping = OrderedDict()
for key_node, value_node in node.value:
key = self.construct_object(key_node, deep=deep)
try:
hash(key)
except TypeError, exc:
raise yaml.constructor.ConstructorError('while constructing a mapping',
node.start_mark, 'found unacceptable key (%s)' % exc, key_node.start_mark)
value = self.construct_object(value_node, deep=deep)
mapping[key] = value
return mapping
In [26]:
def create_argv_dot_json(comp_dir):
comp_name = string.split(comp_dir,'/')[-1]
argv = '["' + comp_name + '"]'
o = open(comp_dir + '/db/argv.json', 'w')
o.write(argv)
o.close()
def create_provides_dot_json(comp_dir):
comp_name = string.split(comp_dir,'/')[-1]
argv = '["' + comp_name + '"]'
o = open(comp_dir + '/db/provides.json', 'w')
o.write(argv)
o.close()
def create_uses_dot_json(comp_dir):
comp_name = string.split(comp_dir,'/')[-1]
argv = '["' + comp_name + '"]'
o = open(comp_dir + '/db/uses.json', 'w')
o.write(argv)
o.close()
def create_files_dot_json(comp_dir):
comp_name = string.split(comp_dir,'/')[-1]
cfg_name = comp_name + '.cfg.in'
filesjson_out = open(comp_dir + '/db/files.json','w')
json.dump([cfg_name], filesjson_out)
filesjson_out.close()
In [27]:
def read_extra_info():
itemFile = 'info.txt'
extra_info = UnsortableOrderedDict()
o = open(itemFile, mode = 'r')
for line in o:
line = re.sub('\n$','',line)
ln = re.split('\s*[\:\=]\s*', line)
error_message = 'The file ' + itemFile + ' cannot be read. Please use the format key : value'
assert len(ln) == 2, error_message
extra_info[str(ln[0])] = str(ln[1])
o.close()
return extra_info
def create_info_dot_json(comp_dir):
comp_name = string.split(comp_dir,'/')[-1]
info = UnsortableOrderedDict()
info['id'] = comp_name
info['name'] = comp_name
info['class'] = comp_name
info['initialize_args'] = comp_name + '.cfg'
info['time_step'] = ''
info['summary'] = ''
info['url'] = "http://csdms.colorado.edu/wiki/Model_help:" + comp_name
info['author'] = ''
info['email'] = ''
info['version'] = ''
info['doi'] = ''
info['license'] = ''
extra = read_extra_info()
if len(extra)>0:
for k in extra.keys():
info[k] = extra[k]
o = open(comp_dir + '/db/info.json', 'w')
json.dump(info, o, indent = 2)
o.close()
In [363]:
def parameters_dot_json(code_root,comp_name):
with open(code_root + '/' + comp_name + '/Initialize.c') as p:
k = p.read()
k = re.sub('\n',' ',k)
m = re.search('int Initialize\((.*?)\)', k)
m = m.group(0)
m = re.sub('int Initialize\(','',m)
m = re.sub('\)','',m)
m = re.sub('\*','',m)
m = re.sub('\s{2}','',m)
m = re.sub('double','float',m)
m_ = m.split(', ')
m_ = [m.split(' ') for m in m_]
out_key = [m[1] for m in m_]
out_type = [m[0] for m in m_]
with open(code_root + '/' + comp_name + '/Key for Inputs.txt') as p:
k = p.read()
keys_in = k.split('\n')
kin = [k.split(' ') for k in keys_in]
kin = [k for k in kin if len(k)==2]
out_key_file = [k[0] for k in kin]
out_desc = [k[1] for k in kin]
for i in range(len(out_desc)):
u = re.search('\[.{1,6}\]',out_desc[i])
if u:
out_units.append(u.group(0))
out_desc[i] = re.sub(' \[.{1,6}\]','',out_desc[i])
else:
out_units.append('-')
keys = ['key', 'name', 'description', 'value']
out_all = []
if len(out_key) == len(out_key_file):
for i in range(len(out_key)):
val = OrderedDict()
val['type'] = out_type[i]
val['units'] = out_units[i]
out_obj = OrderedDict()
pairs = zip(keys,[out_key[i],out_key[i],out_desc[i],val])
for key, value in pairs:
out_obj[key] = value
out_all.append(out_obj)
outFile = comp_dir + '/db/parameters.json'
json_out = open(outFile,'w')
json.dump(out_all, json_out, indent = 4)
json_out.close()
else:
print 'not the same length! Input', comp_name
def parameters_out_dot_json(code_root,comp_name):
with open(code_root + '/' + comp_name + '/Finalize.c') as p:
k = p.read()
k = re.sub('\n',' ',k)
m = re.search('int Finalize\((.*?)\)', k)
m = m.group(0)
m = re.sub('int Finalize\(','',m)
m = re.sub('\)','',m)
m = re.sub('\[\]','',m)
m = re.sub('\s{2}','',m)
m = re.sub('double','float',m)
m_ = m.split(', ')
m_ = [m.split(' ') for m in m_]
out_key = [m[1] for m in m_]
out_type = [m[0] for m in m_]
with open(code_root + '/' + comp_name + '/Key for Outputs.txt') as p:
k = p.read()
k = re.sub('\t','@',k)
keys_in = k.split('\n')
keys_in = keys_in[1:]
keys_in = [k for k in keys_in if len(k)>1]
kin = [k.split('@') for k in keys_in]
kin_ = [[k_ for k_ in k if len(k_)>0] for k in kin]
out_key_file = [re.sub('\s','',k[0]) for k in kin_]
out_desc = [k[1] for k in kin_]
out_units = []
for i in range(len(out_desc)):
u = re.search('\[.{1,6}\]',out_desc[i])
if u:
out_units.append(u.group(0))
out_desc[i] = re.sub(' \[.{1,6}\]','',out_desc[i])
else:
out_units.append('-')
keys = ['key', 'name', 'description', 'value']
out_all = []
if len(out_key) == len(out_key_file):
for i in range(len(out_key)):
val = OrderedDict()
val['type'] = out_type[i]
val['units'] = out_units[i]
out_obj = OrderedDict()
pairs = zip(keys,[out_key[i],out_key[i],out_desc[i],val])
for key, value in pairs:
out_obj[key] = value
out_all.append(out_obj)
outFile = comp_dir + '/db/parameters_out.json'
json_out = open(outFile,'w')
json.dump(out_all, json_out, indent = 4)
json_out.close()
else:
print 'not the same length! Output', comp_name
In [94]:
def create_cfg_file(comp_dir, code_root):
comp_name = string.split(comp_dir,'/')[-1]
cfg_name = comp_name + '.cfg.in'
cfg_file = comp_dir + '/files/' + cfg_name
# header
head = ['#' + 79*'=', '# STM Config File for: ' + comp_name]
tablestr = head
try:
# open Key files - input
with open(code_root + '/' + comp_name + '/Key for Inputs.txt') as p:
k = p.read()
keys_in = k.split('\n')
header = ['#' + 79*'=', '# Input']
tablestr = tablestr + header
for k in keys_in:
k_ = k.split(' ')
if len(k_) == 2:
col1 = k_[0]
col2 = '{' + k_[0] + '}'
col3 = 'float'
col4 = k_[1].capitalize()
col5 = ' [-]' if not '[' in k_[1] else ''
table = ['{0:20}| {1:20}| {2:10}| {3}{4}'.format(col1,col2,col3,col4,col5)]
tablestr = tablestr + table
except:
print '%%%%Problem in ' + comp_name + ' Input'
try:
# open Key files - output
with open(code_root + '/' + comp_name + '/Key for Outputs.txt') as p:
k = p.read()
keys_in = k.split('\n')
keys_in = keys_in[1:]
header = ['#' + 79*'=', '# Output']
tablestr = tablestr + header
for k in keys_in:
if len(k)>0:
k_ = re.split('\t*',k)
if len(k_) == 2:
k_[0] = re.sub('\s*','',k_[0])
col1 = k_[0]
col2 = '{' + k_[0] + '}'
col3 = 'float'
col4 = k_[1].capitalize()
col5 = ' [-]' if not '[' in k_[1] else ''
table = ['{0:20}| {1:20}| {2:10}| {3}{4}'.format(col1,col2,col3,col4,col5)]
tablestr = tablestr + table
except:
print '%%%%Problem in ' + comp_name + ' Output'
tablestr = '\n'.join(tablestr)
cfgfile_out = open(cfg_file, 'w')
cfgfile_out.write(tablestr)
cfgfile_out.close()
In [365]:
code_root = '../../stm/'
components_root = 'components/'
code_dirs = glob.iglob(code_root + '*') # original code
for comp in code_dirs:
comp_name = string.split(comp,'/')[-1]
comp_name = comp_name.replace(" ","")
# print 'Creating component for ' + comp_name
# make new dirs
comp_dir = components_root + comp_name
if not os.path.exists(comp_dir):
os.makedirs(comp_dir)
os.makedirs(comp_dir + '/db')
os.makedirs(comp_dir + '/files')
# make a json parameter file
# files.json
# create_files_dot_json(comp_dir)
# argv.json -> name of the component
# create_argv_dot_json(comp_dir)
# info.json
# create_info_dot_json(comp_dir)
# provides.json - empty?
# create_provides_dot_json(comp_dir)
# uses.json - empty?
# create_uses_dot_json(comp_dir)
# cfg.in file
# create_cfg_file(comp_dir, code_root)
parameters_dot_json(code_root,comp_name)
parameters_out_dot_json(code_root,comp_name)
In [382]:
with open(code_root + '/' + comp_name + '/Initialize.c') as p:
k = p.read()
k = re.sub('\n',' ',k)
m = re.search('int Initialize\((.*?)\)', k)
m = m.group(0)
m = re.sub('int Initialize\(','',m)
m = re.sub('\)','',m)
m = re.sub('\*','',m)
m = re.sub('\s{2}','',m)
m = re.sub('double','float',m)
m = re.sub('\[\]','',m)
m_ = m.split(', ')
m_ = [m.split(' ') for m in m_]
out_key = [m[1] for m in m_]
out_type = [m[0] for m in m_]
with open(code_root + '/' + comp_name + '/Key for Inputs.txt') as p:
k = p.read()
keys_in = k.split('\n')
kin = [k.split(' ') for k in keys_in]
kin = [k for k in kin if len(k)==2]
out_key_file = [k[0] for k in kin]
out_desc = [k[1] for k in kin]
for i in range(len(out_desc)):
u = re.search('\[.{1,6}\]',out_desc[i])
if u:
out_units.append(u.group(0))
out_desc[i] = re.sub(' \[.{1,6}\]','',out_desc[i])
else:
out_units.append('-')
keys = ['key', 'name', 'description', 'value']
out_all = []
if len(out_key) == len(out_key_file):
for i in range(len(out_key)):
val = OrderedDict()
val['type'] = out_type[i]
val['units'] = out_units[i]
out_obj = OrderedDict()
pairs = zip(keys,[out_key[i],out_key[i],out_desc[i],val])
for key, value in pairs:
out_obj[key] = value
out_all.append(out_obj)
outFile = comp_dir + '/db/parameters.json'
json_out = open(outFile,'w')
json.dump(out_all, json_out, indent = 4)
json_out.close()
else:
print 'not the same length! Input', comp_name
In [385]:
out_key
Out[385]:
In [386]:
out_key_file
Out[386]:
In [ ]: