In [9]:
import os
import yaml
import fnmatch
from conda_build.metadata import MetaData
from pprint import pprint
import binstar_client
import matplotlib.pyplot as plt
import copy
%matplotlib inline
import networkx as nx

In [10]:
token = open('/home/edill/dev/dotfiles/tokens/lightsource2-testing.token', 'r').read().strip()

In [11]:
cli = binstar_client.Binstar(domain="https://api.anaconda.org", token=token)

In [12]:
packages_on_conda_forge = set([f['full_name'].split('/')[1] for f in cli.show_channel('main', 'conda-forge')['files']])

In [14]:
def sanitize_names(list_of_names):
    list_of_names = [name.split(' ')[0] for name in list_of_names]
    list_of_names = [name for name in list_of_names]# if name not in packages_on_conda_forge]
    return list_of_names

In [127]:
# Get the list of folders to check
# get the tagged stack
folders = ['/home/edill/dev/conda/lightsource2-recipes/recipes-tag/',
           '/home/edill/dev/conda/lightsource2-recipes/recipes-dev/',
           '/home/edill/dev/conda/lightsource2-recipes/recipes-config/',
           '/home/edill/dev/conda/skbeam-recipes/']

In [142]:
run = {}
build = {}
test = {}
for path_to_recipes in folders:
    for dirpath, dirnames, filenames in os.walk(path_to_recipes):
        if 'meta.yaml' not in filenames:
            continue
            # we are not in the right directory
        print("evaluating", dirpath)
        meta = MetaData(dirpath)
        name = meta.meta['package']['name']
        build[name] = sanitize_names(meta.meta.get('requirements', {}).get('build', []))
        run[name] = sanitize_names(meta.meta.get('requirements', {}).get('run', []))
        test[name] = sanitize_names(meta.meta.get('test', {}).get('requires', []))


evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/analysis
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/trackpy-0.3.0
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/tzlocal-1.1.2
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/datamuxer
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/chxtools
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/ixstools
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/pims-0.3.3
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/csxtools
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/ophyd
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/databroker
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/suitcase
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/h5py-2.6.0
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/super_state_machine-1.0
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/pyolog
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/gobject-introspection-1.47.1
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/sphinx-bootstrap-theme-0.4.5
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/cairo-gobject-1.12.18
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/hdf5-1.10.0
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/pyxrf
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/py3cairo-1.10.0
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/epics-base-R3.14.12.4
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/exceptional
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/xray-vision
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/filestore
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/hdf5-lz4-0.4
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/metadatastore
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/bluesky
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/amostra
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/pygobject3-3.12.0
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/pymongo-3.2.2
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/collection
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/pyepics-3.2.5
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/hkl-v5.0.0.2080
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/fabio-0.3.0
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/channelarchiver-0.0.5-6
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-tag/eiger-io
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-dev/doct
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-dev/datamuxer
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-dev/scikit-beam
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-dev/ixstools
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-dev/csxtools
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-dev/ophyd
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-dev/databroker
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-dev/event-model
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-dev/suitcase
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-dev/historydict
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-dev/pyxrf
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-dev/xray-vision
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-dev/filestore
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-dev/metadatastore
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-dev/bluesky
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-dev/eiger-io
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-config/csx
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-config/srx
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-config/hxn
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-config/lix
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-config/smi
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-config/amx
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-config/chx
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-config/nslsii_dev
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-config/csx2
evaluating /home/edill/dev/conda/lightsource2-recipes/recipes-config/ixs
evaluating /home/edill/dev/conda/skbeam-recipes/recipes/lmfit
evaluating /home/edill/dev/conda/skbeam-recipes/recipes/scikit-beam
evaluating /home/edill/dev/conda/skbeam-recipes/xraylib-3.1.0

In [143]:
union = {k: set(test.get(k, []) + run.get(k, []) + build.get(k, []))
         for k in set(list(test.keys()) + list(build.keys()) + list(run.keys()))}

In [144]:
# drop all extra packages that I do not have conda recipes for
cleaned_union = {}
for name, items in union.items():
    cleaned_items = set()
    print("Checking", name)
    for item in items:
        if item not in union:
            print('\tdropping', item)
        else:
            cleaned_items.add(item)
    print('\n')
    cleaned_union[name] = cleaned_items


Checking pims
	dropping python
	dropping scikit-image
	dropping numpy
	dropping jinja2
	dropping slicerator
	dropping tifffile
	dropping six
	dropping setuptools
	dropping pillow
	dropping libtiff
	dropping nose


Checking super_state_machine
	dropping python
	dropping setuptools
	dropping six


Checking ixstools
	dropping python
	dropping pyyaml
	dropping numpy
	dropping scipy
	dropping coverage
	dropping nose
	dropping matplotlib
	dropping pandas


Checking scikit-beam
	dropping python
	dropping scikit-image
	dropping numpy
	dropping setuptools
	dropping six
	dropping scipy
	dropping coverage
	dropping cython
	dropping nose


Checking ophyd
	dropping python
	dropping numpy
	dropping ipython
	dropping setuptools
	dropping prettytable
	dropping networkx


Checking smi_configuration
	dropping python


Checking xraylib
	dropping python
	dropping numpy
	dropping setuptools
	dropping scipy
	dropping cython
	dropping swig


Checking gobject-introspection
	dropping python
	dropping glib
	dropping libffi


Checking chx_configuration
	dropping python


Checking ixs_configuration
	dropping python


Checking epics-base
	dropping readline


Checking channelarchiver
	dropping python
	dropping matplotlib
	dropping numpy


Checking historydict
	dropping python
	dropping setuptools
	dropping six


Checking hdf5-lz4


Checking bluesky
	dropping python
	dropping boltons
	dropping jsonschema
	dropping numpy
	dropping ipython
	dropping tifffile
	dropping cycler
	dropping prettytable
	dropping traitlets
	dropping matplotlib


Checking cairo-gobject
	dropping cairo
	dropping zlib
	dropping freetype
	dropping pkg-config
	dropping glib
	dropping libffi
	dropping libpng
	dropping pixman


Checking amx_configuration
	dropping python


Checking analysis
	dropping python
	dropping openpyxl
	dropping jupyter
	dropping ipywidgets
	dropping album
	dropping matplotlib


Checking hxn_configuration
	dropping python


Checking tzlocal
	dropping python
	dropping pytz
	dropping setuptools


Checking pygobject3
	dropping python
	dropping glib
	dropping libffi


Checking event-model
	dropping python


Checking pyepics
	dropping python
	dropping setuptools
	dropping numpy


Checking chxtools
	dropping python
	dropping numpy
	dropping setuptools
	dropping prettytable
	dropping matplotlib


Checking srx_configuration
	dropping python


Checking trackpy
	dropping python
	dropping pytables
	dropping pyyaml
	dropping numpy
	dropping setuptools
	dropping six
	dropping scipy
	dropping numexpr
	dropping numba
	dropping matplotlib
	dropping pandas


Checking xray-vision
	dropping python
	dropping numpy
	dropping setuptools
	dropping six
	dropping pyqt
	dropping matplotlib


Checking lix_configuration
	dropping python


Checking filestore
	dropping python
	dropping pyyaml
	dropping jsonschema
	dropping boltons
	dropping tifffile
	dropping six


Checking csx2_configuration
	dropping python


Checking databroker
	dropping python
	dropping boltons
	dropping requests
	dropping six
	dropping setuptools
	dropping cytoolz
	dropping pandas


Checking doct
	dropping python
	dropping prettytable
	dropping setuptools
	dropping six
	dropping humanize


Checking pyxrf
	dropping python
	dropping numpy
	dropping setuptools
	dropping enaml
	dropping matplotlib
	dropping pandas


Checking metadatastore
	dropping python
	dropping pyyaml
	dropping boltons
	dropping numpy
	dropping jinja2
	dropping six
	dropping humanize
	dropping prettytable
	dropping pytz


Checking pymongo
	dropping python
	dropping setuptools


Checking exceptional
	dropping python
	dropping ipython
	dropping requests


Checking py3cairo
	dropping python
	dropping pkg-config
	dropping cairo


Checking collection


Checking lmfit
	dropping python
	dropping setuptools
	dropping scipy
	dropping numpy


Checking hkl
	dropping python
	dropping m4
	dropping numpy
	dropping automake
	dropping pkg-config
	dropping autoconf
	dropping libtool
	dropping gsl


Checking hdf5
	dropping zlib


Checking suitcase
	dropping python
	dropping numpy
	dropping six
	dropping setuptools


Checking fabio
	dropping python
	dropping numpy
	dropping setuptools
	dropping lxml
	dropping pillow
	dropping cython


Checking h5py
	dropping python
	dropping numpy
	dropping six
	dropping pkgconfig
	dropping cython


Checking amostra
	dropping python
	dropping pyyaml
	dropping requests
	dropping jsonschema
	dropping tornado
	dropping setuptools
	dropping mongoquery
	dropping ujson


Checking nslsii_dev_configuration
	dropping python


Checking pyolog
	dropping python
	dropping requests
	dropping keyring
	dropping ipython
	dropping setuptools
	dropping six


Checking csx_configuration
	dropping python


Checking datamuxer
	dropping python
	dropping scipy
	dropping six
	dropping setuptools
	dropping pandas


Checking eiger-io
	dropping python
	dropping numpy
	dropping setuptools


Checking csxtools
	dropping python
	dropping numpy
	dropping ipython
	dropping setuptools
	dropping ipywidgets
	dropping pytest
	dropping matplotlib


Checking sphinx-bootstrap-theme
	dropping python
	dropping setuptools



In [145]:
kw = dict(node_size=5, node_color='w', edge_color='b', alpha=0.25)

def make_network(packages_dict):
    G = nx.DiGraph()
    for package, deps in packages_dict.items():
#         print(package)
        for dep in deps:
#             print('\t{}'.format(dep))
            G.add_edge(package, dep)
    return G

In [146]:
run_G = make_network(run)
build_G = make_network(build)
all_G = make_network(union)

Create an .svg graph of the dependency graph


In [131]:
import graphviz as gv
need_to_build_kwargs = {'color': 'red', 'shape': 'triangle'}
conda_forge_kwargs = {'color': 'blue', 'shape': 'diamond'}
anaconda_kwargs = {'color': 'black', 'shape': 'pentagon'}
# dev_kwargs = {'color': 'cyan3', 'shape': 'hexagon'}

packages_to_ignore = ['numpy', 'python', 'setuptools', 'six']
def make_gv_network(packages_dict):
    graph = gv.Digraph(format='svg')
    all_packages = set()
    for package, deps in packages_dict.items():
        all_packages.add(package)
        all_packages = all_packages.union(deps)
    # add all the nodes
    for pkg in all_packages:
        if pkg in packages_to_ignore:
            continue
        if pkg in packages_on_conda_forge: 
            graph.node(pkg, **conda_forge_kwargs)
        elif pkg not in union:
            graph.node(pkg, **anaconda_kwargs)
        else:
            graph.node(pkg, **need_to_build_kwargs)
    # build the edges
    for package, deps in packages_dict.items():
        for dep in deps:
            if dep in packages_to_ignore:
                continue
            graph.edge(package, dep)
    return graph

gvG = make_gv_network(union)
gvG.render('all_dependencies')


Out[131]:
'all_dependencies.svg'

In [124]:
gvG.view()


Out[124]:
'all_dependencies.svg'

List the build order for a serial build (i.e., one build at a time)


In [154]:
print("This is an attempt to characterize our build stack in terms of serial ordering")
print("As in: start from the top and work your way down to hit all the dependencies\n")
counter = 0
for item in reversed(nx.topological_sort(all_G)):
    if item in packages_on_conda_forge:
        print('##: {}  # Package already on conda-forge'.format(item))
    elif item not in union:
        print('##: {}  # Package not on conda-forge and is not in our recipes'.format(item))
    else:
        counter += 1
        print('{:>2}: {}'.format(counter, item))


This is an attempt to characterize our build stack in terms of serial ordering
As in: start from the top and work your way down to hit all the dependencies

##: six  # Package already on conda-forge
##: setuptools  # Package not on conda-forge and is not in our recipes
##: python  # Package already on conda-forge
 1: super_state_machine
##: requests  # Package already on conda-forge
##: numpy  # Package already on conda-forge
##: slicerator  # Package already on conda-forge
##: tornado  # Package already on conda-forge
 2: smi_configuration
##: coverage  # Package not on conda-forge and is not in our recipes
##: swig  # Package not on conda-forge and is not in our recipes
##: scipy  # Package already on conda-forge
##: cython  # Package already on conda-forge
##: xraylib  # Package already on conda-forge
##: libffi  # Package already on conda-forge
##: glib  # Package already on conda-forge
##: pkg-config  # Package already on conda-forge
##: freetype  # Package already on conda-forge
##: libpng  # Package already on conda-forge
##: pixman  # Package already on conda-forge
##: zlib  # Package already on conda-forge
##: cairo  # Package already on conda-forge
 3: cairo-gobject
 4: gobject-introspection
##: ipywidgets  # Package already on conda-forge
##: nose  # Package already on conda-forge
 5: chx_configuration
##: boltons  # Package already on conda-forge
##: openpyxl  # Package not on conda-forge and is not in our recipes
##: pytest  # Package not on conda-forge and is not in our recipes
 6: srx_configuration
 7: hxn_configuration
 8: py3cairo
 9: pygobject3
##: event-model  # Package already on conda-forge
##: pytz  # Package already on conda-forge
##: pyyaml  # Package already on conda-forge
##: pymongo  # Package already on conda-forge
##: prettytable  # Package already on conda-forge
10: nslsii_dev_configuration
##: humanize  # Package already on conda-forge
##: doct  # Package already on conda-forge
##: jinja2  # Package already on conda-forge
11: metadatastore
##: networkx  # Package not on conda-forge and is not in our recipes
##: ipython  # Package already on conda-forge
##: readline  # Package already on conda-forge
12: epics-base
13: pyepics
##: hdf5  # Package already on conda-forge
##: pkgconfig  # Package not on conda-forge and is not in our recipes
##: h5py  # Package already on conda-forge
##: tifffile  # Package already on conda-forge
##: jsonschema  # Package already on conda-forge
##: libtiff  # Package already on conda-forge
##: pillow  # Package already on conda-forge
##: scikit-image  # Package already on conda-forge
##: pims  # Package already on conda-forge
14: filestore
15: ophyd
##: tzlocal  # Package already on conda-forge
##: pandas  # Package already on conda-forge
##: cytoolz  # Package already on conda-forge
##: matplotlib  # Package already on conda-forge
16: channelarchiver
17: databroker
18: datamuxer
##: album  # Package not on conda-forge and is not in our recipes
##: pyqt  # Package not on conda-forge and is not in our recipes
19: xray-vision
##: ujson  # Package not on conda-forge and is not in our recipes
##: mongoquery  # Package already on conda-forge
20: amostra
##: jupyter  # Package not on conda-forge and is not in our recipes
21: suitcase
##: historydict  # Package already on conda-forge
##: traitlets  # Package already on conda-forge
##: cycler  # Package already on conda-forge
##: lmfit  # Package already on conda-forge
22: bluesky
23: scikit-beam
24: analysis
##: keyring  # Package already on conda-forge
25: pyolog
26: collection
27: lix_configuration
28: csx2_configuration
##: enaml  # Package not on conda-forge and is not in our recipes
##: autoconf  # Package already on conda-forge
##: automake  # Package already on conda-forge
##: gsl  # Package already on conda-forge
##: libtool  # Package already on conda-forge
##: m4  # Package already on conda-forge
29: hkl
##: lxml  # Package not on conda-forge and is not in our recipes
##: fabio  # Package already on conda-forge
30: csx_configuration
31: eiger-io
32: ixstools
##: numexpr  # Package not on conda-forge and is not in our recipes
33: ixs_configuration
34: hdf5-lz4
35: amx_configuration
36: chxtools
##: numba  # Package not on conda-forge and is not in our recipes
##: pytables  # Package not on conda-forge and is not in our recipes
37: trackpy
38: pyxrf
39: exceptional
40: csxtools
##: sphinx-bootstrap-theme  # Package already on conda-forge

List all groups of leaves for a theoretical parallel build


In [151]:
def print_and_trim(graph):
    nodes = [x for x in (G) if G.out_degree(x) == 0]
    for node in nodes:
        G.remove_node(node)
    for idx, node in enumerate(nodes):
        if node in packages_on_conda_forge:
            continue
        elif node not in cleaned_union:
            continue
        else:
            nodes[idx] = node.upper()
    pprint(sorted(nodes, key=lambda x: x.lower()))

G = all_G.copy()
G.reverse()
print("Packges in all CAPS are packages that still need to be built")
print("This is an attempt to characterize our build stack in terms "
      "of which packages can be built in parallel")
print("Intra-group can be built in parallel")
print("Each group should be built serially")
count = 0
while G.nodes():
    print('\nGROUP', count+1)
    print_and_trim(G)
    count += 1
    if count > 50:
        break


Packges in all CAPS are packages that still need to be built
This is an attempt to characterize our build stack in terms of which packages can be built in parallel
Intra-group can be built in parallel
Each group should be built serially

GROUP 1
['album',
 'autoconf',
 'automake',
 'boltons',
 'cairo',
 'coverage',
 'cycler',
 'cython',
 'cytoolz',
 'enaml',
 'freetype',
 'glib',
 'gsl',
 'humanize',
 'ipython',
 'ipywidgets',
 'jinja2',
 'jsonschema',
 'jupyter',
 'keyring',
 'libffi',
 'libpng',
 'libtiff',
 'libtool',
 'lxml',
 'm4',
 'matplotlib',
 'mongoquery',
 'networkx',
 'nose',
 'numba',
 'numexpr',
 'numpy',
 'openpyxl',
 'pandas',
 'pillow',
 'pixman',
 'pkg-config',
 'pkgconfig',
 'prettytable',
 'pyqt',
 'pytables',
 'pytest',
 'python',
 'pytz',
 'pyyaml',
 'readline',
 'requests',
 'scikit-image',
 'scipy',
 'setuptools',
 'six',
 'slicerator',
 'swig',
 'tifffile',
 'tornado',
 'traitlets',
 'ujson',
 'zlib']

GROUP 2
['AMX_CONFIGURATION',
 'CAIRO-GOBJECT',
 'CHX_CONFIGURATION',
 'CSX2_CONFIGURATION',
 'CSX_CONFIGURATION',
 'doct',
 'EPICS-BASE',
 'event-model',
 'EXCEPTIONAL',
 'hdf5',
 'historydict',
 'HXN_CONFIGURATION',
 'IXS_CONFIGURATION',
 'LIX_CONFIGURATION',
 'lmfit',
 'NSLSII_DEV_CONFIGURATION',
 'pims',
 'pymongo',
 'PYOLOG',
 'SMI_CONFIGURATION',
 'sphinx-bootstrap-theme',
 'SRX_CONFIGURATION',
 'SUPER_STATE_MACHINE',
 'tzlocal',
 'XRAY-VISION',
 'xraylib']

GROUP 3
['AMOSTRA',
 'CHANNELARCHIVER',
 'GOBJECT-INTROSPECTION',
 'h5py',
 'HDF5-LZ4',
 'IXSTOOLS',
 'METADATASTORE',
 'PY3CAIRO',
 'PYEPICS',
 'SCIKIT-BEAM',
 'TRACKPY']

GROUP 4
['fabio', 'FILESTORE', 'PYGOBJECT3', 'PYXRF']

GROUP 5
['CHXTOOLS', 'DATABROKER', 'EIGER-IO', 'HKL', 'OPHYD']

GROUP 6
['BLUESKY', 'CSXTOOLS', 'DATAMUXER', 'SUITCASE']

GROUP 7
['ANALYSIS']

GROUP 8
['COLLECTION']