In [ ]:
from __future__ import print_function
from github_settings import (ry_username, ry_password,
username, password,
token,
GITENBERG_GITHUB_TOKEN,
RDHYEE_GITHUB_TOKEN)
from itertools import (islice, izip)
import requests
from second_folio import (apply_to_repos)
from gitenberg_utils import (GitenbergJob,
GitenbergTravisJob,
ForkBuildRepo,
BuildRepo,
BuildRepo2,
MetadataWrite,
repo_md)
import pandas as pd
from pandas import (DataFrame, Series)
In [ ]:
from second_folio import (all_repos)
all_repos[:5]
In [ ]:
len(all_repos)
In [ ]:
repo_name = all_repos[0]
In [ ]:
repos = all_repos[:]
def status_for_repo(repo_name):
rs = GitenbergJob(username=username, password=password, repo_name=repo_name,
repo_owner='GITenberg',
update_travis_commit_msg='update travis',
tag_commit_message='update travis')
return rs.status()
results_iter = apply_to_repos(status_for_repo, repos=repos)
In [ ]:
results = []
for (i,result) in enumerate(results_iter):
results.append(result)
print ("\r{}: {}".format(i, result['repo_name']), end="")
In [ ]:
df = DataFrame(results)
In [ ]:
df.head()
In [ ]:
import json
print (json.dumps(results))
In [ ]:
df.columns
In [ ]:
from semantic_version import Version
import re
def my_filter(r):
#return (r['metadata.yaml'] and r['book.asciidoc']) and (r['ebooks_in_release_count'] == 0) and (not r['.travis.yml'])
#return (r['ebooks_in_release_count'] == 0) and (r['book.asciidoc'])
# return (r['ebooks_in_release_count'] > 0)
return len(r['asciidocs'])> 0 and not r['book.asciidoc']
def txts(r):
repo_id = r['repo_name'].split("_")[-1]
return [hash_ for hash_ in r['root_hashes'] if re.match(r'(\d+)\-*(\d*).txt', hash_)]
In [ ]:
# how to find .txt or html files
In [ ]:
len(df[df.apply(my_filter, axis=1)])
In [ ]:
repos = list(df[df.apply(my_filter, axis=1)]['repo_name'])
repos
In [ ]:
repos_to_process = repos[0:1]
def build_repo(repo_name):
rs = BuildRepo(username=username, password=password, repo_name=repo_name,
repo_owner='GITenberg',
update_travis_commit_msg='update travis',
tag_commit_message='update travis',
github_token = GITENBERG_GITHUB_TOKEN)
return rs.run()
results_iter = apply_to_repos(build_repo, repos=repos_to_process)
In [ ]:
results = []
for (i,result) in enumerate(results_iter):
results.append(result)
print ("\r{}: {}".format(i, result[0]), end="")
In [ ]:
results
In [ ]:
repo_name = repos_to_process[0]
rs = BuildRepo(username=username, password=password, repo_name=repo_name,
repo_owner='GITenberg',
update_travis_commit_msg='update travis',
tag_commit_message='update travis',
github_token = GITENBERG_GITHUB_TOKEN)
rs.run()
In [ ]:
repo_md('1079')
In [ ]:
repos_to_process = ['The-Life-and-Opinions-of-Tristram-Shandy-Gentleman_1079']
def write_metadata(repo_name):
rs = MetadataWrite(username=username, password=password, repo_name=repo_name,
repo_owner='GITenberg',
update_travis_commit_msg='update travis',
tag_commit_message='update travis')
return rs.run()
results_iter = apply_to_repos(write_metadata, repos=repos_to_process)
In [ ]:
results = []
for (i,result) in enumerate(results_iter):
results.append(result)
print ("\r{}: {}".format(i, result[0]), end="")
In [ ]:
repos_to_process
In [ ]:
repos[0]
In [ ]:
write_metadata(repos[0])
In [ ]:
yaml.safe_dump(y,default_flow_style=False,
allow_unicode=True)
In [ ]:
# https://gist.githubusercontent.com/rdhyee/9665aa23084a45269cd3c941f702602f/raw/9332e96fa5d3c86137d4a5c49c2ba7b484a4f7c1/repos_status.json
In [ ]:
for repo in repos2_to_build[:]:
try:
bj = BuildRepo2(username=username, password=password, repo_name=repo,
repo_owner='GITenberg',
update_travis_commit_msg='update travis',
tag_commit_message='update travis',
github_token=GITENBERG_GITHUB_TOKEN)
bj.run()
except Exception as e:
print (repo, e)
In [ ]:
from gitenberg import metadata
import os
def local_yaml_file(id_):
fname = "/Users/raymondyee/C/src/gitenberg-dev/giten_site/metadata/{}.yaml".format(id_)
if os.path.exists(fname):
md = metadata.pandata.Pandata(fname)
return md
else:
return None
In [ ]:
for repo in all_repos[:]:
id_ = repo.split("_")[-1]
yaml_file = local_yaml_file(id_)
if yaml_file is None:
print (repo, yaml_file)
covers:
- attribution: Ed Gaither - Modern Electrographic, 2015
cover_type: original
image_path: cover.jpg
rights: Attribution-NonCommercial 4.0 International (CC BY-
rights_url: https://creativecommons.org/licenses/by-nc/4.0/publication_date
publisher: Recovering the Classics
rights: CC BY-NC
rights_url http://creativecommons.org/licenses/by-nc/4.0/
In [ ]:
gj.travis_encrypt(RDHYEE_DON_QUIXOTE_TOKEN)
In [ ]:
u = gj.gh.user()
u.email
In [ ]:
gj.fork_and_build_gitenberg_repo()
In [ ]:
gj.create_or_update_file(path='JUNK.md', message=b'updated junk.md', content=u'hello'.encode('utf-8'))
In [ ]:
print(gj.update_travis_template(write_changes=False,
encrypted_key=gj.travis_encrypt(gj.repo_token()))[0])
In [ ]:
from travispy import TravisPy
travis = TravisPy.github_auth(RDHYEE_GITHUB_TOKEN)
t_user = travis.user()
t_user.login
In [ ]:
travis_repo = travis.repo('rdhyee/Don-Quixote_996')
travis_repo.active
travis_repo.enable()
In [ ]:
travis_encrypt(token_to_encrypt=token.token.encode('utf-8'),
repo_slug="rdhyee/Don-Quixote_996")
https://github.com/menegazzo/travispy/blob/v0.3.4/travispy/_tests/test_settings.example.json
In [ ]:
import json
from travispy import TravisPy
def create_travispy_test_settings(github_token, repo_slug):
settings = {}
travis = TravisPy.github_auth(github_token)
settings['github_token'] = github_token
settings['repo_slug'] = repo_slug
# account
# https://github.com/menegazzo/travispy/blob/v0.3.4/travispy/_tests/test_authenticated.py#L31
accounts = travis.accounts()
account = travis.account(accounts[0].id)
settings['account'] = {
'count': len(accounts),
'id': account.id,
'name': account.name,
'login': account.login,
'type': account.type,
'repos_count': account.repos_count,
'subscribed': hasattr(account, 'subscribed')
}
# hook
# https://github.com/menegazzo/travispy/blob/v0.3.4/travispy/_tests/test_authenticated.py#L73
hooks = travis.hooks()
hook = hooks[0]
settings['hook'] = {
'count': len(hooks),
'name': hook.name,
'description': hook.description,
'owner_name': hook.owner_name,
'active': hook.active,
'private': hook.private,
'admin': hook.admin
}
# user
# https://github.com/menegazzo/travispy/blob/v0.3.4/travispy/_tests/test_authenticated.py#L110
user = travis.user()
settings['user'] = {
'login': user['login'],
'name': user['name']
}
# branch
# https://github.com/menegazzo/travispy/blob/v0.3.4/travispy/_tests/test_not_authenticated.py#L19
branches = travis.branches(slug=repo_slug)
branch = travis.branch('master', repo_slug)
settings['branch'] = {
'count': len(branches),
'id': branch.id,
'repository_id': branch.repository_id,
'pull_request': branch.pull_request,
'config': branch.config,
'number': branch.number
}
# build
# https://github.com/menegazzo/travispy/blob/v0.3.4/travispy/_tests/test_not_authenticated.py#L66
builds = travis.builds(slug=repo_slug)
build = travis.build(builds[0].id)
build_id = builds[0].id
settings['build'] = {
'count': len(builds),
'id': build.id,
'repository_id': build.repository_id,
'number': build.number,
'pull_request': build.pull_request,
'pull_request_title': build.pull_request_title,
'pull_request_number': build.pull_request_number,
'config': build.config
}
# commit
# https://github.com/menegazzo/travispy/blob/v0.3.4/travispy/_tests/test_not_authenticated.py#L115
commit = build.commit
settings['commit'] = {
'count': len(builds),
'id': commit.id,
'sha': commit.sha,
'branch': commit.branch,
'message': commit.message,
'committed_at': commit.committed_at,
'author_name': commit.author_name,
'author_email': commit.author_email,
'commiter_name': commit.committer_name, # sic
'commiter_email': commit.committer_email, # sic
'compare_url': commit.compare_url,
'pull_request_number': None if not hasattr(commit, 'pull_request_number') else commit.pull_request_number
}
# jobs
# https://github.com/menegazzo/travispy/blob/v0.3.4/travispy/_tests/test_not_authenticated.py#L140
jobs = travis.jobs(ids=build.job_ids)
job = travis.job(build.job_ids[0])
settings['job'] = {
'count': len(jobs),
'build_id': job.build_id,
'repository_id': job.repository_id,
'number': job.number,
'config': job.config,
'queue': job.queue,
'allow_failure': job.allow_failure,
'annotation_ids': job.annotation_ids
}
# repo
# https://github.com/menegazzo/travispy/blob/v0.3.4/travispy/_tests/test_not_authenticated.py#L252
# let's add fake stuff for now
settings['repo'] = {
"public_count": 25,
"member_count": 5,
"owner_count": 7,
"github_language": "Python",
"id": 2598876,
"description": "TravisPy test project for Python 2.7",
"active": True
}
return json.dumps(settings, indent=2)
In [ ]:
print(create_travispy_test_settings(RDHYEE_GITHUB_TOKEN, 'rdhyee/hello-travis-ci'))
In [ ]: