Jenkins Configuration Notebook

This notebook should be used to update the Jenkins configuration for the Menpo project en-masse. It works by completing three template projects with the state that changes between projects - namely, the name of the project (and hence the github URL) and the Python versions that need to be built.

Dependencies

java
an internet connection (no SSH tunnel etc required!)



In [ ]:

    
# set your jenkins credentials here - and do not commit them!!
USERNAME = 'USER'
PASSWORD = 'PASS'



In [ ]:

    
from functools import partial
from subprocess import check_output, Popen, PIPE, STDOUT, CalledProcessError
from utils import PROJECTS, download_file

# downlad the CLI file for use
download_file('https://jenkins.menpo.org/jnlpJars/jenkins-cli.jar', 'jenkins-cli.jar')

T_OFF = '<disabled>true</disabled>'
T_ON = '<disabled>false</disabled>'
T_VERSIONS = '<string>@PYTHON_VERSIONS@</string>\n'
T_NAME = '@PROJECT_NAME@'

VERSION_TO_STR = {k: '<string>{}</string>\n        '.format(v) for k, v in 
                  {2: '2.7', 34: '3.4', 35: '3.5'}.items()}

JENKINS_CMD = ['java', '-jar', 'jenkins-cli.jar', 
               '-noKeyAuth', '-noCertificateCheck', 
               '-s', 'https://jenkins.menpo.org']
AUTH = [
    '--username', USERNAME,
    '--password', PASSWORD
]

cmd_for_args = lambda args: JENKINS_CMD + list(args) + AUTH

    
def jcall(*args, **kwargs):
    verbose = kwargs.get('verbose', False)
    cmd = cmd_for_args(args)
    if verbose:
        print(' '.join(cmd))
    try:
        output = check_output(cmd).decode()
        return output.split('Skipping HTTPS certificate checks altogether. Note that this is not secure at all.\n')[-1]
    except CalledProcessError as e:
        print(e.returncode)
        print(e.output)


def jinput(stdin, cmd):
    p = Popen(cmd_for_args(cmd), stdout=PIPE, stdin=PIPE, stderr=STDOUT)
    try:
        return p.communicate(input=stdin.encode())[0].decode()
    except CalledProcessError as e:
        print(e.returncode)
        print(e.output)

def fill_template(t, name, versions):
    return t.replace(T_NAME, name).replace(T_VERSIONS, versions).replace(T_OFF, T_ON)


get_job = partial(jcall, 'get-job')
create_job = lambda name, job: jinput(job, ['create-job', name])
update_job = lambda name, job: jinput(job, ['update-job', name])
delete_job = lambda name: jcall('delete-job', name)



In [ ]:

    
# Check we are successfully logged in:
print(jcall('who-am-i'))



In [ ]:

    
print('loading templates...')

def load_text(name):
    with open(name, 'rt') as f:
        content = f.read()
    return content

T, T_PR = (load_text('./jenkins/templates/TEMPLATE.xml'), 
           load_text('./jenkins/templates/TEMPLATE-pr.xml'))

SUFFIX_TO_TEMPLATE = { '': T, '-pr': T_PR }
print('done.')
print('acquiring current jobs...')
JOBS = set(x for x in jcall('list-jobs').split('\n') if not x == '')
print('done.')



In [ ]:

    
non_pr_jobs = {j for j in JOBS if not j.endswith('-pr')}
known_jobs = {p.name for p in PROJECTS}
extra_jobs = non_pr_jobs - known_jobs - { 'TEMPLATE' }
if len(extra_jobs) != 0:
    print(extra_jobs)
    raise ValueError("Warning - a Jenkins jobs is present that isn't managed")



In [ ]:

    
for p in PROJECTS:
    versions = ' '.join([VERSION_TO_STR[v] for v in p.versions])
    for suffix, template in SUFFIX_TO_TEMPLATE.items():
        job = fill_template(template, p.name, versions)
        job_name = p.name + suffix
        if job_name in JOBS:
            print('{} already exists, updating.'.format(job_name))
            update_job(job_name, job)
        else:
            print('creating job: {}'.format(job_name))
            create_job(job_name, job)