Promote repositories to 0.1.x · Issue #24 · gitenberg-dev/Second-Folio
Our process for generating epub for almost all of the Second Folio books is good enough to now promote version to 0.1.x. Specifically:
metadata.yaml to 0.1.0 (or 0.1.x)GITenberg keyword.travis.yml so the repos get loaded to unglue.it
In [ ]:
from __future__ import print_function
import json
import yaml
from itertools import (islice, izip)
import arrow
from gitenberg import metadata
import jinja2
import semantic_version
import requests
from second_folio import (all_repos,
apply_to_repos,
git_pull,
travis_template,
repo_version,
travis_template,
slugify,
TRAVIS_TEMPLATE_URL
)
from github_settings import (username, password, token)
# import github3.py
import github3
GITenberg_ORG = "GITenberg"
# https://github.com/sigmavirus24/github3.py/pull/368/files
# for personal access tokens
gh = github3.login(username, password=token)
In [ ]:
repos = list(islice(all_repos,0,None))
repos
In [ ]:
# git pull to get updated
results = list(izip(repos, apply_to_repos(git_pull,
repos=repos)))
[result for result in results if isinstance(result[1], Exception) ]
In [ ]:
# which books still at 0.0.1
[(repo, version_info) for (repo, version_info) in
izip(repos, apply_to_repos(repo_version, repos=repos, kwargs={'version_type':'minor', 'write_version':False}))
if version_info[0] == '0.0.1']
In [ ]:
# loop over repos and see which ones need to be updated to 0.1.0
list(apply_to_repos(repo_version, repos=repos, kwargs={'version_type':'minor', 'write_version':False}))
In [ ]:
gh = github3.login(username, password=token)
gh.user().name, gh.user().email
Different ways to work with .travis.yml:
In [ ]:
# rewrite function without assuming that there is a local copy of the repo
class GITenbergPromoter(object):
def __init__(self, username, token, repo_name, owner_name='GITenberg',
update_travis_commit_msg = "updated .travis.yml",
promote_commit_message = "promote to 0.1.x; add GITenberg keyword",
keywords=('GITenberg',)):
self.gh = github3.login(username, password=token)
self.repo = self.gh.repository(owner_name, repo_name)
self.repo_name = repo_name
self.owner_name = owner_name
self.update_travis_commit_msg = update_travis_commit_msg
self.promote_commit_message = promote_commit_message
self.keywords = keywords
def promote(self, write_changes=True, update_travis=True, promote_repo=True):
if update_travis:
(template_result, template_written, commit) = self.update_travis_template(write_changes=write_changes)
else:
(template_result, template_written, commit) = (None, None, None)
if promote_repo:
(_version, _next_version, metadata_updated, commit, tag) = self.promote_repo_to_minor_add_kw(write_changes=write_changes)
else:
(_version, _next_version, metadata_updated, commit, tag) = (None, None, None, None, None)
return [(template_result, template_written, commit),
(_version, _next_version, metadata_updated, commit, tag)]
def metadata(self):
"""
returns a metadata.pandata.Pandata object for repo
"""
metadata_url = "https://raw.githubusercontent.com/{owner}/{repo_name}/master/metadata.yaml".format(owner=self.owner_name,
repo_name=self.repo_name)
md = metadata.pandata.Pandata(metadata_url)
return md
def update_travis_template(self, write_changes=True, template=None):
"""
compute (and optionally write) .travis.yml based on the template and current metadata.yaml
repo: github3.py representation of repository
"""
template_written = False
if template is None:
template = travis_template()
md = self.metadata()
repo_name = md.metadata.get("_repo")
epub_title = slugify(md.metadata.get("title"))
# pick from rep
encrypted_key = self.repo.contents(".travis.deploy.api_key.txt", ref='master').decoded.decode('utf-8')
template_vars = {
'epub_title': epub_title,
'encrypted_key': encrypted_key,
'repo_name': repo_name
}
template_result = template.render(**template_vars)
if write_changes:
# how to write to file
content = self.repo.contents('.travis.yml', ref='master')
data = {
'message': self.update_travis_commit_msg,
'content': template_result.encode('utf-8'),
}
commit = content.update(**data)
else:
commit = None
return (template_result, template_written, commit)
def promote_repo_to_minor_add_kw(self, write_changes=True):
"""
github3.py representation of repository
returns current version, next version, whether metadata updated, commit
"""
md = self.metadata()
# promote to 0.1.0 or next patch if current version is at least a minor version
_version = semantic_version.Version(md.metadata.get("_version"))
if _version >= semantic_version.Version("0.1.0"):
_next_version = unicode(_version.next_patch())
else:
_next_version = u"0.1.0"
# add keywords
subjects = md.metadata.get("subjects")
for keyword in self.keywords:
if keyword not in subjects:
subjects.append(keyword)
md.metadata["subjects"] = subjects
if write_changes:
# how to write to file
content = self.repo.contents('metadata.yaml', ref='master')
md.metadata["_version"] = _next_version
data = {
'message': self.promote_commit_message,
'content': yaml.safe_dump(md.metadata,default_flow_style=False,allow_unicode=True)
}
commit = content.update(**data)
# also tag the commit
tag_data = {
'tag': _next_version,
'message': _next_version,
'sha': commit.sha,
'obj_type': 'commit',
'tagger': {
'name': self.gh.user().name,
'email': gh.user().email,
'date': arrow.utcnow().isoformat()
},
'lightweight': False
}
tag = self.repo.create_tag(**tag_data)
metadata_updated = True
else:
commit = None
tag = None
metadata_updated = False
return (_version, _next_version, metadata_updated, commit, tag)
def url_latest_epub(self):
"""
repo is a github3.py repo
"""
md = self.metadata()
epub_title = slugify(md.metadata.get("title"))
tag = md.metadata.get("_version")
url = "https://github.com/GITenberg/{}/releases/download/{}/{}.epub".format(self.repo_name, tag, epub_title)
return url
def status_latest_epub(self):
return requests.head(self.url_latest_epub()).status_code
In [ ]:
def promote_repo(repo_name):
gp = GITenbergPromoter(username, token, repo_name)
try:
result = gp.promote(write_changes=True, update_travis=True, promote_repo=True)
except Exception as e:
result = e
return (result, gp)
def epub_status(repo_name):
gp = GITenbergPromoter(username, token, repo_name)
return gp.status_latest_epub()
In [ ]:
repos = all_repos
for (repo, status) in izip(repos, apply_to_repos(epub_status, repos=repos)):
print (repo, status)
In [ ]:
all_repos[7:10]
In [ ]:
results = list(apply_to_repos(promote_repo, repos=all_repos[10:]))
list(results)
In [ ]:
gp = results[0][-1]
gp.status_latest_epub()
In [ ]:
gp = GITenbergPromoter(username, token, repo_name)
gp.status_latest_epub()
In [ ]:
(template_result, template_written, commit) = new_travis_template_2(repo=repo, template=template, write_template=True)
(template_result, template_written, commit)
In [ ]:
(template_result, template_written, commit)
In [ ]:
def repo_version_2(repo, version_type='patch', write_version=False):
"""
github3.py representation of repository
returns current version, next version, whether metadata updated, commit
"""
assert version_type in ('patch', 'minor', 'major')
metadata_updated = False
metadata_url = "https://raw.githubusercontent.com/{owner}/{repo_name}/master/metadata.yaml".format(owner='GITenberg',
repo_name=repo.name)
md = metadata.pandata.Pandata(metadata_url)
_version = md.metadata.get("_version")
next_func = getattr(semantic_version.Version(_version), "next_{}".format(version_type))
_next_version = unicode(next_func())
if write_version:
# how to write to file
content = repo.contents('metadata.yaml', ref='master')
md.metadata["_version"] = _next_version
data = {
'message': 'updated metadata.yaml',
'content': yaml.safe_dump(md.metadata,default_flow_style=False,allow_unicode=True)
}
commit = content.update(**data)
metadata_updated = True
else:
commit = None
return (_version, _next_version, metadata_updated, commit)
In [ ]:
import arrow
def promote_repo_to_minor_add_kw(repo, user_name, user_email, commit_message,
keywords=('GITenberg',), write_version=False):
"""
github3.py representation of repository
returns current version, next version, whether metadata updated, commit
"""
metadata_updated = False
metadata_url = "https://raw.githubusercontent.com/{owner}/{repo_name}/master/metadata.yaml".format(owner='GITenberg',
repo_name=repo.name)
md = metadata.pandata.Pandata(metadata_url)
# promote to 0.1.0 or next patch if current version is at least a minor version
71 _version = semantic_version.Version(md.metadata.get("_version"))
if _version >= semantic_version.Version("0.1.0"):
_next_version = unicode(_version.next_patch())
else:
_next_version = u"0.1.0"
# add keywords
subjects = md.metadata.get("subjects")
for keyword in keywords:
if keyword not in subjects:
subjects.append(keyword)
md.metadata["subjects"] = subjects
if write_version:
# how to write to file
content = repo.contents('metadata.yaml', ref='master')
md.metadata["_version"] = _next_version
data = {
'message': commit_message,
'content': yaml.safe_dump(md.metadata,default_flow_style=False,allow_unicode=True)
}
commit = content.update(**data)
# also tag the commit
tag_data = {
'tag': _next_version,
'message': _next_version,
'sha': commit.sha,
'obj_type': 'commit',
'tagger': {
'name': user_name,
'email': user_email,
'date': arrow.utcnow().isoformat()
},
'lightweight': False
}
tag = repo.create_tag(**tag_data)
metadata_updated = True
else:
commit = None
tag = None
return (_version, _next_version, metadata_updated, commit, tag)
In [ ]:
user = gh.user()
(_version, _next_version, metadata_updated, commit, tag) = promote_repo_to_minor_add_kw(repo=repo,
user_name=user.name,
user_email=user.email,
write_version=False,
commit_message="promote to 0.1.x; add GITenberg keyword")
(_version, _next_version, metadata_updated, commit, tag)
In [ ]:
def latest_epub_2(repo):
"""
repo is a github3.py repo
"""
metadata_url = "https://raw.githubusercontent.com/{owner}/{repo_name}/master/metadata.yaml".format(owner='GITenberg',
repo_name=repo.name)
md = metadata.pandata.Pandata(metadata_url)
#repo_name = md.metadata.get("_repo")
epub_title = slugify(md.metadata.get("title"))
tag = md.metadata.get("_version")
url = "https://github.com/GITenberg/{}/releases/download/{}/{}.epub".format(repo, tag, epub_title)
return url
In [ ]:
import requests
requests.head(latest_epub_2(repo)).status_code