In [1]:
ls
build.sh meta.yaml test_regex.ipynb
conda-skeletor.yml rewrite-setup-dot-py.ipynb Untitled.ipynb
find_imports_to_test.ipynb scrape notebook.ipynb
In [2]:
import json
In [ ]:
json.
In [4]:
nb = json.load(open('find_imports_to_test.ipynb'))
In [5]:
nb
Out[5]:
{'cells': [{'cell_type': 'code',
'execution_count': 1,
'metadata': {'collapsed': False},
'outputs': [{'name': 'stderr',
'output_type': 'stream',
'text': ['The module located at /home/edill/miniconda/lib/python3.4/site-packages/whatsmyversion-0.0.4.post10-py3.4.egg/whatsmyversion.py does not have a git repo in its directory hierarchy\n',
'INFO:whatsmyversion:The module located at /home/edill/miniconda/lib/python3.4/site-packages/whatsmyversion-0.0.4.post10-py3.4.egg/whatsmyversion.py does not have a git repo in its directory hierarchy\n',
'path = /home/edill/miniconda/lib/python3.4/site-packages/whatsmyversion-0.0.4.post10-py3.4.egg/whatsmyversion.py\n',
'DEBUG:whatsmyversion:path = /home/edill/miniconda/lib/python3.4/site-packages/whatsmyversion-0.0.4.post10-py3.4.egg/whatsmyversion.py\n',
"split_path = ['', 'home', 'edill', 'miniconda', 'lib', 'python3.4', 'site-packages', 'whatsmyversion-0.0.4.post10-py3.4.egg', 'whatsmyversion.py']\n",
"DEBUG:whatsmyversion:split_path = ['', 'home', 'edill', 'miniconda', 'lib', 'python3.4', 'site-packages', 'whatsmyversion-0.0.4.post10-py3.4.egg', 'whatsmyversion.py']\n",
'version_info_folder = whatsmyversion-0.0.4.post10-py3.4.egg\n',
'DEBUG:whatsmyversion:version_info_folder = whatsmyversion-0.0.4.post10-py3.4.egg\n',
"split_version_info = ['whatsmyversion', '0.0.4.post10', 'py3.4.egg']\n",
"DEBUG:whatsmyversion:split_version_info = ['whatsmyversion', '0.0.4.post10', 'py3.4.egg']\n"]}],
'source': ['from conda_skeletor.main import split_deps, get_runtime_deps, construct_template_info, package_mapping\n',
'from conda_skeletor import setup_parser\n',
'import depfinder\n',
'import os\n',
'import yaml\n',
'import re\n',
'import pandas as pd']},
{'cell_type': 'code',
'execution_count': 2,
'metadata': {'collapsed': False},
'outputs': [],
'source': ['# Function coped from main() --> conda-skeletor.main:main\n',
'def find_imports(path_to_source):\n',
' # find the dependencies for all modules in the source directory\n',
' repo_deps = depfinder.iterate_over_library(path_to_source)\n',
'\n',
' # Compile the regexers listed in the conda-skeleton.yml\n',
" test_regexers = [re.compile(reg) for reg in skeletor_config.get('test_regex', [])]\n",
" ignore_path_regexers = [re.compile(reg) for reg in skeletor_config.get('ignore_path_regex', [])]\n",
" include_path_regexers = [re.compile(reg) for reg in skeletor_config.get('include_path_regex', [])]\n",
' ignored, without_ignored = split_deps(repo_deps, ignore_path_regexers)\n',
' included, without_included = split_deps(without_ignored, include_path_regexers)\n',
' tests, without_tests = split_deps(included, test_regexers)\n',
' \n',
' return tests, without_tests']},
{'cell_type': 'code',
'execution_count': 3,
'metadata': {'collapsed': False},
'outputs': [],
'source': ["skeletor_config = yaml.load(open('conda-skeletor.yml', 'r'))"]},
{'cell_type': 'code',
'execution_count': 4,
'metadata': {'collapsed': False},
'outputs': [{'data': {'text/plain': ["{'blacklist_packages': ['setupext',\n",
" 'versioneer',\n",
" 'ctrans',\n",
" 'skxray',\n",
" 'src',\n",
" 'netCDF4',\n",
" 'pyFAI'],\n",
" 'ignore_path_regex': ['.*(doc).*', '.*(examples).*', '.*(versioneer).*'],\n",
" 'include_path_regex': ['.*(skxray).*'],\n",
" 'test_regex': ['.*(test).*']}"]},
'execution_count': 4,
'metadata': {},
'output_type': 'execute_result'}],
'source': ['skeletor_config']},
{'cell_type': 'code',
'execution_count': 5,
'metadata': {'collapsed': False},
'outputs': [],
'source': ["path_to_source = os.path.join(os.path.expanduser('~/dev/python/scikit-xray'))"]},
{'cell_type': 'code',
'execution_count': 6,
'metadata': {'collapsed': False},
'outputs': [{'name': 'stdout',
'output_type': 'stream',
'text': ['...................................................................................']}],
'source': ['test, without_test = find_imports(path_to_source)']},
{'cell_type': 'code',
'execution_count': 57,
'metadata': {'collapsed': False},
'outputs': [],
'source': ['def find_lib_name(without_test):\n',
' non_test_paths = [full_path for mod_name, full_path, catcher in without_test]\n',
' common_path = os.path.commonprefix(non_test_paths)\n',
' return common_path.strip(os.sep).split(os.sep)[-1]\n',
'# non_test_paths = [path.split(os.sep) for path in non_test_paths]\n',
'# df = pd.DataFrame(non_test_paths)\n',
'# prev_col_name = None\n',
'# equivalent = True\n',
'# for col_name in df:\n',
'# equivalent = not any([val != df[col_name][0] for val in df[col_name]])\n',
'# if not equivalent:\n',
'# break\n',
'# prev_col_name = col_name\n',
'# return df[prev_col_name][0]']},
{'cell_type': 'code',
'execution_count': 58,
'metadata': {'collapsed': False},
'outputs': [],
'source': ['lib_name = find_lib_name(without_test)']},
{'cell_type': 'code',
'execution_count': 59,
'metadata': {'collapsed': False},
'outputs': [{'data': {'text/plain': ["'skxray'"]},
'execution_count': 59,
'metadata': {},
'output_type': 'execute_result'}],
'source': ['lib_name']},
{'cell_type': 'code',
'execution_count': 89,
'metadata': {'collapsed': False},
'outputs': [{'data': {'text/plain': ['6']},
'execution_count': 89,
'metadata': {},
'output_type': 'execute_result'}],
'source': ['prev_col_name']},
{'cell_type': 'code',
'execution_count': 10,
'metadata': {'collapsed': False},
'outputs': [],
'source': ['path = without_test[0][1]']},
{'cell_type': 'code',
'execution_count': 39,
'metadata': {'collapsed': False},
'outputs': [{'data': {'text/plain': ["'__init__'"]},
'execution_count': 39,
'metadata': {},
'output_type': 'execute_result'}],
'source': ['lib_name']},
{'cell_type': 'code',
'execution_count': 45,
'metadata': {'collapsed': False},
'outputs': [],
'source': ['def into_importable(full_module_path, lib_name):\n',
' relative_module_path = full_module_path.split(lib_name, maxsplit=1)[1]\n',
" # trim the '.py'\n",
' relative_module_path = relative_module_path[:-3]\n',
' # add the library name back in\n',
' relative_module_path = lib_name + relative_module_path\n',
" importable_path = '.'.join(relative_module_path.split(os.sep))\n",
" # turn imports from 'foo.bar.baz.__init__' into 'foo.bar.baz'\n",
" if importable_path.endswith('__init__'):\n",
' importable_path = importable_path[:-9]\n',
' # turn the string from path/to/module to path.to.module\n',
' return importable_path']},
{'cell_type': 'code',
'execution_count': 47,
'metadata': {'collapsed': False},
'outputs': [],
'source': ['all_imports = [into_importable(path_to_module, lib_name) for mod_name, path_to_module, catcher in without_test]']},
{'cell_type': 'code',
'execution_count': 25,
'metadata': {'collapsed': False},
'outputs': [{'data': {'text/plain': ["'skxray.fluorescence'"]},
'execution_count': 25,
'metadata': {},
'output_type': 'execute_result'}],
'source': []},
{'cell_type': 'code',
'execution_count': 50,
'metadata': {'collapsed': False},
'outputs': [{'ename': 'NameError',
'evalue': "name 'df' is not defined",
'output_type': 'error',
'traceback': ['\x1b[1;31m---------------------------------------------------------------------------\x1b[0m',
'\x1b[1;31mNameError\x1b[0m Traceback (most recent call last)',
'\x1b[1;32m<ipython-input-50-7ed0097d7e9e>\x1b[0m in \x1b[0;36m<module>\x1b[1;34m()\x1b[0m\n\x1b[1;32m----> 1\x1b[1;33m \x1b[0mdf\x1b[0m\x1b[1;33m\x1b[0m\x1b[0m\n\x1b[0m',
"\x1b[1;31mNameError\x1b[0m: name 'df' is not defined"]}],
'source': ['df']},
{'cell_type': 'code',
'execution_count': None,
'metadata': {'collapsed': True},
'outputs': [],
'source': []}],
'metadata': {'kernelspec': {'display_name': 'Python 3',
'language': 'python',
'name': 'python3'},
'language_info': {'codemirror_mode': {'name': 'ipython', 'version': 3},
'file_extension': '.py',
'mimetype': 'text/x-python',
'name': 'python',
'nbconvert_exporter': 'python',
'pygments_lexer': 'ipython3',
'version': '3.4.3'}},
'nbformat': 4,
'nbformat_minor': 0}
In [8]:
nb['cells']
Out[8]:
[{'cell_type': 'code',
'execution_count': 1,
'metadata': {'collapsed': False},
'outputs': [{'name': 'stderr',
'output_type': 'stream',
'text': ['The module located at /home/edill/miniconda/lib/python3.4/site-packages/whatsmyversion-0.0.4.post10-py3.4.egg/whatsmyversion.py does not have a git repo in its directory hierarchy\n',
'INFO:whatsmyversion:The module located at /home/edill/miniconda/lib/python3.4/site-packages/whatsmyversion-0.0.4.post10-py3.4.egg/whatsmyversion.py does not have a git repo in its directory hierarchy\n',
'path = /home/edill/miniconda/lib/python3.4/site-packages/whatsmyversion-0.0.4.post10-py3.4.egg/whatsmyversion.py\n',
'DEBUG:whatsmyversion:path = /home/edill/miniconda/lib/python3.4/site-packages/whatsmyversion-0.0.4.post10-py3.4.egg/whatsmyversion.py\n',
"split_path = ['', 'home', 'edill', 'miniconda', 'lib', 'python3.4', 'site-packages', 'whatsmyversion-0.0.4.post10-py3.4.egg', 'whatsmyversion.py']\n",
"DEBUG:whatsmyversion:split_path = ['', 'home', 'edill', 'miniconda', 'lib', 'python3.4', 'site-packages', 'whatsmyversion-0.0.4.post10-py3.4.egg', 'whatsmyversion.py']\n",
'version_info_folder = whatsmyversion-0.0.4.post10-py3.4.egg\n',
'DEBUG:whatsmyversion:version_info_folder = whatsmyversion-0.0.4.post10-py3.4.egg\n',
"split_version_info = ['whatsmyversion', '0.0.4.post10', 'py3.4.egg']\n",
"DEBUG:whatsmyversion:split_version_info = ['whatsmyversion', '0.0.4.post10', 'py3.4.egg']\n"]}],
'source': ['from conda_skeletor.main import split_deps, get_runtime_deps, construct_template_info, package_mapping\n',
'from conda_skeletor import setup_parser\n',
'import depfinder\n',
'import os\n',
'import yaml\n',
'import re\n',
'import pandas as pd']},
{'cell_type': 'code',
'execution_count': 2,
'metadata': {'collapsed': False},
'outputs': [],
'source': ['# Function coped from main() --> conda-skeletor.main:main\n',
'def find_imports(path_to_source):\n',
' # find the dependencies for all modules in the source directory\n',
' repo_deps = depfinder.iterate_over_library(path_to_source)\n',
'\n',
' # Compile the regexers listed in the conda-skeleton.yml\n',
" test_regexers = [re.compile(reg) for reg in skeletor_config.get('test_regex', [])]\n",
" ignore_path_regexers = [re.compile(reg) for reg in skeletor_config.get('ignore_path_regex', [])]\n",
" include_path_regexers = [re.compile(reg) for reg in skeletor_config.get('include_path_regex', [])]\n",
' ignored, without_ignored = split_deps(repo_deps, ignore_path_regexers)\n',
' included, without_included = split_deps(without_ignored, include_path_regexers)\n',
' tests, without_tests = split_deps(included, test_regexers)\n',
' \n',
' return tests, without_tests']},
{'cell_type': 'code',
'execution_count': 3,
'metadata': {'collapsed': False},
'outputs': [],
'source': ["skeletor_config = yaml.load(open('conda-skeletor.yml', 'r'))"]},
{'cell_type': 'code',
'execution_count': 4,
'metadata': {'collapsed': False},
'outputs': [{'data': {'text/plain': ["{'blacklist_packages': ['setupext',\n",
" 'versioneer',\n",
" 'ctrans',\n",
" 'skxray',\n",
" 'src',\n",
" 'netCDF4',\n",
" 'pyFAI'],\n",
" 'ignore_path_regex': ['.*(doc).*', '.*(examples).*', '.*(versioneer).*'],\n",
" 'include_path_regex': ['.*(skxray).*'],\n",
" 'test_regex': ['.*(test).*']}"]},
'execution_count': 4,
'metadata': {},
'output_type': 'execute_result'}],
'source': ['skeletor_config']},
{'cell_type': 'code',
'execution_count': 5,
'metadata': {'collapsed': False},
'outputs': [],
'source': ["path_to_source = os.path.join(os.path.expanduser('~/dev/python/scikit-xray'))"]},
{'cell_type': 'code',
'execution_count': 6,
'metadata': {'collapsed': False},
'outputs': [{'name': 'stdout',
'output_type': 'stream',
'text': ['...................................................................................']}],
'source': ['test, without_test = find_imports(path_to_source)']},
{'cell_type': 'code',
'execution_count': 57,
'metadata': {'collapsed': False},
'outputs': [],
'source': ['def find_lib_name(without_test):\n',
' non_test_paths = [full_path for mod_name, full_path, catcher in without_test]\n',
' common_path = os.path.commonprefix(non_test_paths)\n',
' return common_path.strip(os.sep).split(os.sep)[-1]\n',
'# non_test_paths = [path.split(os.sep) for path in non_test_paths]\n',
'# df = pd.DataFrame(non_test_paths)\n',
'# prev_col_name = None\n',
'# equivalent = True\n',
'# for col_name in df:\n',
'# equivalent = not any([val != df[col_name][0] for val in df[col_name]])\n',
'# if not equivalent:\n',
'# break\n',
'# prev_col_name = col_name\n',
'# return df[prev_col_name][0]']},
{'cell_type': 'code',
'execution_count': 58,
'metadata': {'collapsed': False},
'outputs': [],
'source': ['lib_name = find_lib_name(without_test)']},
{'cell_type': 'code',
'execution_count': 59,
'metadata': {'collapsed': False},
'outputs': [{'data': {'text/plain': ["'skxray'"]},
'execution_count': 59,
'metadata': {},
'output_type': 'execute_result'}],
'source': ['lib_name']},
{'cell_type': 'code',
'execution_count': 89,
'metadata': {'collapsed': False},
'outputs': [{'data': {'text/plain': ['6']},
'execution_count': 89,
'metadata': {},
'output_type': 'execute_result'}],
'source': ['prev_col_name']},
{'cell_type': 'code',
'execution_count': 10,
'metadata': {'collapsed': False},
'outputs': [],
'source': ['path = without_test[0][1]']},
{'cell_type': 'code',
'execution_count': 39,
'metadata': {'collapsed': False},
'outputs': [{'data': {'text/plain': ["'__init__'"]},
'execution_count': 39,
'metadata': {},
'output_type': 'execute_result'}],
'source': ['lib_name']},
{'cell_type': 'code',
'execution_count': 45,
'metadata': {'collapsed': False},
'outputs': [],
'source': ['def into_importable(full_module_path, lib_name):\n',
' relative_module_path = full_module_path.split(lib_name, maxsplit=1)[1]\n',
" # trim the '.py'\n",
' relative_module_path = relative_module_path[:-3]\n',
' # add the library name back in\n',
' relative_module_path = lib_name + relative_module_path\n',
" importable_path = '.'.join(relative_module_path.split(os.sep))\n",
" # turn imports from 'foo.bar.baz.__init__' into 'foo.bar.baz'\n",
" if importable_path.endswith('__init__'):\n",
' importable_path = importable_path[:-9]\n',
' # turn the string from path/to/module to path.to.module\n',
' return importable_path']},
{'cell_type': 'code',
'execution_count': 47,
'metadata': {'collapsed': False},
'outputs': [],
'source': ['all_imports = [into_importable(path_to_module, lib_name) for mod_name, path_to_module, catcher in without_test]']},
{'cell_type': 'code',
'execution_count': 25,
'metadata': {'collapsed': False},
'outputs': [{'data': {'text/plain': ["'skxray.fluorescence'"]},
'execution_count': 25,
'metadata': {},
'output_type': 'execute_result'}],
'source': []},
{'cell_type': 'code',
'execution_count': 50,
'metadata': {'collapsed': False},
'outputs': [{'ename': 'NameError',
'evalue': "name 'df' is not defined",
'output_type': 'error',
'traceback': ['\x1b[1;31m---------------------------------------------------------------------------\x1b[0m',
'\x1b[1;31mNameError\x1b[0m Traceback (most recent call last)',
'\x1b[1;32m<ipython-input-50-7ed0097d7e9e>\x1b[0m in \x1b[0;36m<module>\x1b[1;34m()\x1b[0m\n\x1b[1;32m----> 1\x1b[1;33m \x1b[0mdf\x1b[0m\x1b[1;33m\x1b[0m\x1b[0m\n\x1b[0m',
"\x1b[1;31mNameError\x1b[0m: name 'df' is not defined"]}],
'source': ['df']},
{'cell_type': 'code',
'execution_count': None,
'metadata': {'collapsed': True},
'outputs': [],
'source': []}]
In [12]:
code = [''.join(cell['source']) for cell in nb['cells'] if cell['cell_type'] == 'code']
In [14]:
len(code)
Out[14]:
17
In [16]:
print(code[0])
from conda_skeletor.main import split_deps, get_runtime_deps, construct_template_info, package_mapping
from conda_skeletor import setup_parser
import depfinder
import os
import yaml
import re
import pandas as pd
In [1]:
import depfinder
In [2]:
depfinder.notebook_path_to_dependencies('find_imports_to_test.ipynb')
Out[2]:
defaultdict(set,
{'builtin': {'os', 're'},
'required': {'conda_skeletor', 'depfinder', 'pandas', 'yaml'}})
In [3]:
import stdlib_list
In [8]:
from stdlib_list import fetch
In [9]:
fetch??
In [ ]:
Content source: ericdill/conda-skeletor
Similar notebooks: