In [210]:

    

import os, sys, glob, string, shutil, re
import yaml, json
from collections import OrderedDict



class UnsortableList(list):
    def sort(self, *args, **kwargs):
        pass
    
class UnsortableOrderedDict(OrderedDict):
    def items(self, *args, **kwargs):
        return UnsortableList(OrderedDict.items(self, *args, **kwargs))


    

In [211]:

    

class OrderedDictYAMLLoader(yaml.Loader):
    """
    A YAML loader that loads mappings into ordered dictionaries.
    """
 
    def __init__(self, *args, **kwargs):
        yaml.Loader.__init__(self, *args, **kwargs)
 
        self.add_constructor(u'tag:yaml.org,2002:map', type(self).construct_yaml_map)
        self.add_constructor(u'tag:yaml.org,2002:omap', type(self).construct_yaml_map)
 
    def construct_yaml_map(self, node):
        data = OrderedDict()
        yield data
        value = self.construct_mapping(node)
        data.update(value)
 
    def construct_mapping(self, node, deep=False):
        if isinstance(node, yaml.MappingNode):
            self.flatten_mapping(node)
        else:
            raise yaml.constructor.ConstructorError(None, None,
                'expected a mapping node, but found %s' % node.id, node.start_mark)
 
        mapping = OrderedDict()
        for key_node, value_node in node.value:
            key = self.construct_object(key_node, deep=deep)
            try:
                hash(key)
            except TypeError, exc:
                raise yaml.constructor.ConstructorError('while constructing a mapping',
                    node.start_mark, 'found unacceptable key (%s)' % exc, key_node.start_mark)
            value = self.construct_object(value_node, deep=deep)
            mapping[key] = value
        return mapping


    

In [409]:

    

def find_prov(content):

    prov = re.search('_output_var_names\s*=\s*\[(.*?)\]',content, re.DOTALL)
    if prov:
        prov = re.split('\n.*?',prov.groups()[0])
        prov = [re.sub('\s*##.*','',i) for i in prov]
        prov = [re.sub('\s*\'','',i) for i in prov]
        prov = [re.sub(',\s*','',i) for i in prov]
        prov = [re.sub('#\s*.*','',i) for i in prov]
        prov = [re.sub(' ','',i) for i in prov]
        prov = [re.sub('\\r','',i) for i in prov]
        prov = [i for i in prov if len(i)>0]
        
        return prov

def find_uses(content):

    uses = re.search('_input_var_names\s*=\s*\[(.*?)\]',content, re.DOTALL)
    if uses:
        uses = re.split('\n.*?',uses.groups()[0])
        uses = [re.sub('\s*##.*','',i) for i in uses]
        uses = [re.sub('\s*\'','',i) for i in uses]
        uses = [re.sub(',\s*','',i) for i in uses]
        uses = [re.sub('#\s*.*','',i) for i in uses]
        uses = [re.sub(' ','',i) for i in uses]
        uses = [re.sub('\\r','',i) for i in uses]
        uses = [i for i in uses if len(i)>0]
        
        return uses


    

In [386]:

    

def match_keys(toKey,fromKey):

    allU = {}

    for k in toKey.keys():

        allk = {}
        vals = toKey[k]
        keys = [[name for name in fromKey.keys() for i in fromKey[name] if i == j] for j in vals]
        keys = [i[0].split('_')[0] if len(i)>0 else '' for i in keys]

        allk = {keys[i]:[] for i in range(len(keys))}

        for i in range(len(keys)):
            allk[keys[i]].append(vals[i])

        allU[k] = allk
        
    return allU


    

In [410]:

    

def create_provides_dot_json(comp_dir, prov):
    # where prov is a list of dictionaries of the connections it provides
    
    print prov
    
    
    
    
    o = open(comp_dir + '/db/provides.json', 'w')
    json.dump(prov, o, indent = 2, sort_keys = False)
    o.close()
    

def create_uses_dot_json(comp_dir, uses):
    # where uses is a list of dictionaries of the connections it needs
    
    o = open(comp_dir + '/db/uses.json', 'w')
    json.dump(uses, o, indent = 2, sort_keys = False)
    o.close()


    

In [411]:

    

yaml_root = 'yaml/'
components_root = 'components/'
source_dir = 'topoflow/'

source_dirs = glob.iglob(source_dir + '*.py') # py files

Uses = {}
Provides = {}

for comp in source_dirs:
    
    comp_name = string.split(comp,'/')[-1][:-3]
    
    print 'Updating uses/provides for ' + comp_name

    # make new dirs
    comp_dir = components_root + string.lower(comp_name)
    if not os.path.exists(comp_dir):
        os.makedirs(comp_dir)
        os.makedirs(comp_dir + '/db')
    
    with open(comp, 'r') as content_file:
        content = content_file.read()
    
    # provides.json
    provides = find_prov(content)
    if provides:
        Provides[comp_name] = provides
    
    # uses.json
    uses = find_uses(content)
    if uses:
        Uses[comp_name] = uses
    
keyed_provides = match_keys(Provides,Uses)
keyed_uses = match_keys(Uses,Provides)

source_dirs = glob.iglob(source_dir + '*.py') # py files

for comp in source_dirs:
    
    comp_name = string.split(comp,'/')[-1][:-3]
    
    if any(i == comp_name for i in keyed_provides.keys()):
        create_provides_dot_json(comp_dir, keyed_provides[comp_name])
    if any(i == comp_name for i in keyed_uses.keys()):
        create_uses_dot_json(comp_dir, keyed_uses[comp_name])


# inBoth = set(Uses) & set(Provides)
# inOneButNotOther = set(Uses) | set(Provides)
# inUsesNotProvides = {i for i in Uses if set(i) | set(Provides)}


    

    




Updating uses/provides for channels_base
Updating uses/provides for channels_diffusive_wave
Updating uses/provides for channels_dynamic_wave
Updating uses/provides for channels_kinematic_wave
Updating uses/provides for d8_base
Updating uses/provides for d8_global
Updating uses/provides for d8_local
Updating uses/provides for diversions_base
Updating uses/provides for diversions_fraction_method
Updating uses/provides for diversions_fraction_method_LAST
Updating uses/provides for erode_base
Updating uses/provides for erode_d8_global
Updating uses/provides for erode_d8_local
Updating uses/provides for evap_base
Updating uses/provides for evap_energy_balance
Updating uses/provides for evap_priestley_taylor
Updating uses/provides for evap_read_file
Updating uses/provides for gc2d
Updating uses/provides for HIS_base
Updating uses/provides for HIS_test
Updating uses/provides for ice_base
Updating uses/provides for infil_base
Updating uses/provides for infil_beven
Updating uses/provides for infil_green_ampt
Updating uses/provides for infil_richards_1D
Updating uses/provides for infil_richards_1D_METHOD1
Updating uses/provides for infil_smith_parlange
Updating uses/provides for met_base
Updating uses/provides for OLD_solar_funcs_gui
Updating uses/provides for satzone_base
Updating uses/provides for satzone_darcy_layers
Updating uses/provides for smooth_DEM
Updating uses/provides for snow_base
Updating uses/provides for snow_degree_day
Updating uses/provides for snow_energy_balance
Updating uses/provides for soil_base
Updating uses/provides for soil_funcs
Updating uses/provides for solar_funcs
Updating uses/provides for topoflow_driver
{'': ['canals_entrance_water__volume_flow_rate', 'channel_bottom_surface__slope', 'channel_bottom_water_flow__log_law_roughness_length', 'channel_bottom_water_flow__magnitude_of_shear_stress', 'channel_bottom_water_flow__shear_speed', 'channel_centerline__sinuosity', 'channel_water_flow__froude_number', 'channel_water_flow__half_of_fanning_friction_factor', 'channel_water_flow__manning_n_parameter', 'channel_water_surface__slope', 'channel_water_x-section__hydraulic_radius', 'channel_water_x-section__initial_mean_depth', 'channel_water_x-section__volume_flow_rate', 'channel_water_x-section__volume_flux', 'channel_water_x-section__wetted_area', 'channel_water_x-section__wetted_perimeter', 'channel_x-section_trapezoid_bottom__width', 'channel_x-section_trapezoid_side__flare_angle', 'model__time_step', 'model_grid_cell__area'], 'diversions': ['channel_water__volume'], 'evap': ['channel_water_x-section__mean_depth'], 'topoflow': ['basin_outlet_water_flow__half_of_fanning_friction_factor', 'basin_outlet_water_x-section__mean_depth', 'basin_outlet_water_x-section__peak_time_of_depth', 'basin_outlet_water_x-section__peak_time_of_volume_flow_rate', 'basin_outlet_water_x-section__peak_time_of_volume_flux', 'basin_outlet_water_x-section__time_integral_of_volume_flow_rate', 'basin_outlet_water_x-section__time_max_of_mean_depth', 'basin_outlet_water_x-section__time_max_of_volume_flow_rate', 'basin_outlet_water_x-section__time_max_of_volume_flux', 'basin_outlet_water_x-section__volume_flow_rate', 'basin_outlet_water_x-section__volume_flux', 'channel_bottom_water_flow__domain_max_of_log_law_roughness_length', 'channel_bottom_water_flow__domain_min_of_log_law_roughness_length', 'channel_water_flow__domain_max_of_manning_n_parameter', 'channel_water_flow__domain_min_of_manning_n_parameter', 'channel_water_x-section__domain_max_of_mean_depth', 'channel_water_x-section__domain_min_of_mean_depth', 'channel_water_x-section__domain_max_of_volume_flow_rate', 'channel_water_x-section__domain_min_of_volume_flow_rate', 'channel_water_x-section__domain_max_of_volume_flux', 'channel_water_x-section__domain_min_of_volume_flux', 'land_surface_water__runoff_volume_flux', 'land_surface_water__domain_time_integral_of_runoff_volume_flux']}
{'': ['canals__count', 'canals_entrance_water__volume_fraction', 'canals_entrance__x_coordinate', 'canals_entrance__y_coordinate', 'canals_exit_water__volume_flow_rate', 'canals_exit__x_coordinate', 'canals_exit__y_coordinate', 'model__time_step', 'sinks_water__volume_flow_rate', 'sinks__count', 'sinks__x_coordinate', 'sinks__y_coordinate', 'sources_water__volume_flow_rate', 'sources__count', 'sources__x_coordinate', 'sources__y_coordinate']}
{'': ['model__time_step']}
{'': ['atmosphere_water__geomorphic_precipitation_leq-volume_flux', 'bedrock__uplift_rate', 'channel_water_x-section__volume_flow_rate', 'channel_water_x-section__volume_flow_rate_law_area_exponent', 'channel_water_total-sediment__volume_flow_rate', 'channel_water_total-sediment__volume_flow_rate_law_area_exponent', 'channel_water_total-sediment__volume_flow_rate_law_coefficient', 'channel_water_total-sediment__volume_flow_rate_law_slope_exponent', 'land_surface__increment_of_elevation', 'land_surface__initial_elevation', 'land_surface__domain_max_of_increment_of_elevation', 'land_surface__slope', 'land_surface__time_derivative_of_elevation', 'land_surface_contour-segment__total_contributing_area', 'model__time_step', 'model_domain_boundary__lowering_rate', 'model_grid_cell__area', 'model_grid_cell__d8_flow_width', 'model_grid_cell__d8_flow_length', 'model_grid_cell__diameter', 'model_grid_cell__x_length', 'model_grid_cell__y_length'], 'infil': ['land_surface__elevation']}
{'': ['land_surface_soil__conduction_heat_flux', 'model__time_step'], 'channels': ['land_surface_water__evaporation_volume_flux'], 'topoflow': ['land_surface_water__domain_time_integral_of_evaporation_volume_flux']}
{'': ['land_surface_soil__conduction_heat_flux', 'model__time_step'], 'channels': ['land_surface_water__evaporation_volume_flux'], 'topoflow': ['land_surface_water__domain_time_integral_of_evaporation_volume_flux']}
{'channels': ['land_surface_water__evaporation_volume_flux'], '': ['model_grid_cell__area', 'model__time_step'], 'topoflow': ['land_surface_water__domain_time_integral_of_evaporation_volume_flux']}
{'channels': ['glacier_ice__melt_volume_flux'], '': ['glacier_top_surface__elevation', 'glacier_ice__thickness', 'model_grid_cell__x_length', 'model_grid_cell__y_length', 'model__time_step'], 'topoflow': ['glacier_ice__domain_time_integral_of_melt_volume_flux']}
{'': ['model__time_step', 'soil_surface_water__time_integral_of_infiltration_volume_flux', 'soil_water__green-ampt_capillary_length', 'soil_water__potential_infiltration_volume_flux', 'soil_water__initial_hydraulic_conductivity', 'soil_water__initial_volume_fraction', 'soil_water__saturated_hydraulic_conductivity', 'soil_water__saturated_volume_fraction', 'soil_water_flow__z_component_of_darcy_velocity'], 'channels': ['soil_surface_water__infiltration_volume_flux'], 'topoflow': ['soil_surface_water__domain_time_integral_of_infiltration_volume_flux', 'soil_water_sat-zone_top__domain_time_integral_of_recharge_volume_flux'], 'satzone': ['soil_water_sat-zone_top__recharge_volume_flux']}
{'': ['model__time_step', 'soil_surface_water__volume_fraction', 'soil_water__brooks-corey_eta_parameter', 'soil_water__brooks-corey_lambda_parameter', 'soil_water__brooks-corey-smith_c_parameter', 'soil_water__brooks-corey-smith_pressure_head_offset_parameter', 'soil_water__bubbling_pressure_head', 'soil_water__hydraulic_conductivity', 'soil_water__hygroscopic_volume_fraction', 'soil_water__initial_hydraulic_conductivity', 'soil_water__initial_volume_fraction', 'soil_water__pressure_head', 'soil_water__residual_volume_fraction', 'soil_water__saturated_hydraulic_conductivity', 'soil_water__saturated_volume_fraction', 'soil_water__volume_fraction', 'soil_water_flow__z_component_of_darcy_velocity', 'soil_water_wetting-front__depth'], 'channels': ['soil_surface_water__infiltration_volume_flux'], 'topoflow': ['soil_surface_water__domain_time_integral_of_infiltration_volume_flux', 'soil_water_sat-zone_top__domain_time_integral_of_recharge_volume_flux'], 'satzone': ['soil_water_sat-zone_top__recharge_volume_flux']}
{'': ['model__time_step', 'soil_surface_water__time_integral_of_infiltration_volume_flux', 'soil_water__green-ampt_capillary_length', 'soil_water__potential_infiltration_volume_flux', 'soil_water__initial_hydraulic_conductivity', 'soil_water__initial_volume_fraction', 'soil_water__saturated_hydraulic_conductivity', 'soil_water__saturated_volume_fraction', 'soil_water__smith-parlange_gamma_parameter', 'soil_water_flow__z_component_of_darcy_velocity'], 'channels': ['soil_surface_water__infiltration_volume_flux'], 'topoflow': ['soil_surface_water__domain_time_integral_of_infiltration_volume_flux', 'soil_water_sat-zone_top__domain_time_integral_of_recharge_volume_flux'], 'satzone': ['soil_water_sat-zone_top__recharge_volume_flux']}
{'': ['atmosphere_aerosol_dust__reduction_of_transmittance', 'atmosphere_air-column_water-vapor__liquid-equivalent_depth', 'atmosphere_bottom_air__brutsaert_emissivity_canopy_factor', 'atmosphere_bottom_air__brutsaert_emissivity_cloud_factor', 'atmosphere_bottom_air__bulk_latent_heat_aerodynamic_conductance', 'atmosphere_bottom_air__bulk_sensible_heat_aerodynamic_conductance', 'atmosphere_bottom_air__emissivity', 'atmosphere_bottom_air__neutral_bulk_aerodynamic_conductance', 'atmosphere_bottom_air__pressure', 'atmosphere_bottom_air_flow__bulk_richardson_number', 'atmosphere_bottom_air_flow__log_law_roughness_length', 'atmosphere_bottom_air_flow__reference-height_speed', 'atmosphere_bottom_air_flow__speed_reference_height', 'atmosphere_bottom_air_land_net-sensible-heat__energy_flux', 'atmosphere_bottom_air_water-vapor__dew_point_temperature', 'atmosphere_bottom_air_water-vapor__partial_pressure', 'atmosphere_bottom_air_water-vapor__relative_saturation', 'atmosphere_bottom_air_water-vapor__saturated_partial_pressure', 'atmosphere_water__precipitation_leq-volume_flux', 'earth__standard_gravity_constant', 'land_surface__albedo', 'land_surface__aspect_angle', 'land_surface__emissivity', 'land_surface__latitude', 'land_surface__longitude', 'land_surface__slope_angle', 'land_surface_air_water-vapor__partial_pressure', 'land_surface_air_water-vapor__saturated_partial_pressure', 'model__time_step', 'physics__stefan_boltzmann_constant', 'physics__von_karman_constant', 'water__mass-specific_latent_fusion_heat', 'water__mass-specific_latent_vaporization_heat'], 'channels': ['atmosphere_water__rainfall_volume_flux'], 'evap': ['atmosphere_bottom_air_land_net-latent-heat__energy_flux', 'land_surface_net-longwave-radiation__energy_flux', 'land_surface_net-shortwave-radiation__energy_flux'], 'topoflow': ['atmosphere_water__domain_time_integral_of_precipitation_leq-volume_flux', 'atmosphere_water__domain_time_max_of_precipitation_leq-volume_flux'], 'snow': ['atmosphere_bottom_air__mass-per-volume_density', 'atmosphere_bottom_air__mass-specific_isobaric_heat_capacity', 'atmosphere_bottom_air__temperature', 'atmosphere_water__snowfall_leq-volume_flux', 'land_surface__temperature', 'land_surface_net-total-energy__energy_flux', 'water-liquid__mass-per-volume_density']}
{'channels': ['land_surface_water__baseflow_volume_flux'], '': ['model__time_step', 'model_soil_layer-0__porosity'], 'topoflow': ['land_surface_water__domain_time_integral_of_baseflow_volume_flux'], 'infil': ['land_surface__elevation']}
{'': ['model__time_step', 'snowpack__degree-day_coefficient', 'snowpack__degree-day_threshold_temperature', 'snowpack__initial_depth', 'snowpack__initial_liquid-equivalent_depth'], 'met': ['snowpack__depth', 'snowpack__liquid-equivalent_depth', 'snowpack__z_mean_of_mass-per-volume_density'], 'topoflow': ['snowpack__domain_time_integral_of_melt_volume_flux'], 'channels': ['snowpack__melt_volume_flux']}
{'': ['model__time_step', 'snowpack__energy-per-area_cold_content', 'snowpack__initial_depth', 'snowpack__initial_liquid-equivalent_depth', 'snowpack__z_mean_of_mass-specific_isobaric_heat_capacity'], 'met': ['snowpack__depth', 'snowpack__liquid-equivalent_depth', 'snowpack__z_mean_of_mass-per-volume_density'], 'topoflow': ['snowpack__domain_time_integral_of_melt_volume_flux'], 'channels': ['snowpack__melt_volume_flux']}
{'': ['model__time_step']}

In [407]:

    

any(i == comp_name for i in keyed_provides.keys())


    

    
Out[407]:





True

In [405]:

    

keyed_provides.keys()


    

    
Out[405]:





['snow_energy_balance',
 'diversions_fraction_method_LAST',
 'channels_base',
 'ice_base',
 'evap_read_file',
 'met_base',
 'infil_smith_parlange',
 'snow_degree_day',
 'diversions_fraction_method',
 'topoflow_driver',
 'evap_energy_balance',
 'infil_green_ampt',
 'evap_priestley_taylor',
 'infil_richards_1D',
 'satzone_darcy_layers',
 'erode_d8_global']

In [ ]: