In [ ]:

    

import re
import os
import sys
import getopt
import yaml
import collections
import datetime
import bisect
import os
import glob
import json
import jenkinsapi
import itertools
from tabulate import tabulate
from jenkinsapi.jenkins import Jenkins

from collections import abc

class FrozenJSON:
    """A read-only façade for navigating a JSON-like object
       using attribute notation.
       
       Credit: "O'Reilly Fluent Python", Luciano Ramalho
       http://www.amazon.com/Fluent-Python-Luciano-Ramalho/dp/1491946008
    """
    def __init__(self, mapping):
        self.__data = dict(mapping)

    def __getattr__(self, name):
        if hasattr(self.__data, name):
            return getattr(self.__data, name)
        else:
            return FrozenJSON.build(self.__data[name])

    @classmethod
    def build(cls, obj):
        if isinstance(obj, abc.Mapping):
            return cls(obj)
        elif isinstance(obj, abc.MutableSequence):
            return [cls.build(item) for item in obj]
        else:
            return obj


    

In [ ]:

    

class JenkinsServer(object):
    """
    Class representing the Jenkins Server for Branch View
    """

    actionTable = {
        "red" :            { "status" : "Failed"},
        "red_anime" :      { "status" : "InProgress"},
        "yellow" :         { "status" : "Unstable"},
        "yellow_anime" :   { "status" : "InProgress"},
        "blue" :           { "status" : "Success"},
        "blue_anime" :     { "status" : "InProgress"},
        "grey" :           { "status" : "Pending"},
        "grey_anime" :     { "status" : "InProgress"},
        "disabled" :       { "status" : "Disabled"},
        "disabled_anime" : { "status" : "InProgress"},
        "aborted" :        { "status" : "Aborted"},
        "aborted_anime" :  { "status" : "InProgress"},
        "nobuilt" :        { "status" : "NotBuilt"},
        "nobuilt_anime" :  { "status" : "InProgress"}    
    }
    
    commandActor = {
        "build"    : "build",
        "schedule" : "schedule",
        "rerun"    : "runFailedUnstableJobs",
        "failed"   : "failedJobReport",
        "report"   : "jobReport"
    }

    def __init__(self, jkCfg, profile):
        self.jkCfg = jkCfg
        self.profile = profile

        self._jserver = Jenkins(jkCfg.getValue(profile, "jenkinServerUrl"),
                                jkCfg.getValue(profile, "userName"),
                                jkCfg.getValue(profile, "password"))
        self._projectView = self._jserver.views[self.jkCfg.getValue(self.profile, "projectName")]
        self._branchView  = self._projectView.views[self.jkCfg.getValue(self.profile, "branchName")]
    
    def _testConditions(self, rexps, value):
        """
        Test the value against a list of regular expressions.
        Returns True if any of them matches
        """
        tests = [ re.match(m, value) for m in rexps]
        return any(tests)
    
    @property
    def jenkinsServer(self):
        return self._jserver
    
    @property
    def projectView(self):
        return self._projectView
    
    @property
    def branchView(self):
        return self._branchView
    
    def getJobs(self):
        """
        Generator returns all types jobs
        """
        jlist = self._branchView.get_data(self._branchView.python_api_url(self._branchView.baseurl))["jobs"]
        for j in jlist:
            job = FrozenJSON(j)
            yield job
    
    def getRegressionJobs(self, exclude=None):
        """
        Generator returns regressions jobs whose name usually not ends with "Build" or "Scheduler"
        exclude is a list of conditions separated by comma. Specify it to override the value from jenkins.yaml
        """
        if not exclude:
            exclude = self.jkCfg.getValue(self.profile, "regressionJobFilter")

        rexps = exclude.split(",")
        for j in itertools.filterfalse(self._testConditions(rexps, x.name), self.getJobs()):
            yield j
            
    def jobDetails(self, job):
        return (
                job.name, 
                JenkinsServer.actionTable[job.color]["status"], 
                job.lastBuild.number if job.lastBuild is not None else "",
                job.lastStableBuild.number if job.lastStableBuild is not None else "",
                job.healthReport[0].description
               )
    
    def isQueuedOrRunning(self, job):
        j = self._jserver.get_job(job.name)
        return j.is_queued_or_running()
    
    def isFailedOrUnstable(self, job):
        return "red" in job.color or "yellow" in job.color
    
    def isSuccessful(self, job):
        return not self.isFailedOrUnstable(job)
    
    def findJob(self, namePattern):
        """
        Find the first job based on the name pattern in regular expression.
        namePattern is a list of regular expressions separated by comma.
        """
        return next(self.findJobs(namePattern))

    def findJobs(self, namePattern):
        """
        A generator
        Find all the jobs based on the name pattern in regular expression.
        namePattern is a list of regular expressions separated by comma.
        Specify it to override the value from jenkins.yaml
        """
        rexps = namePattern.split(",")
        return (x for x in self.getJobs() if self._testConditions(rexps, x.name))
    
    def getBuildJobs(self, namePattern=None):
        """
        This is to get the job for the Building Job which builds the application.
        
        namePattern is a list of regular expressions separated by comma. By default, the build job should
        have a name like ".*-Build"
        """
        if not namePattern:
            namePattern = self.jkCfg.getValue(self.profile, "buildJob")
        return self.findJobs(namePattern)
    
    def getSchedulerJobs(self, namePattern=None):
        """
        This is to get the job for the Building Job which builds the application.
        
        namePattern is a list of regular expressions separated by comma. By default, the build job should
        have a name like ".*-Scheduler"
        """
        if not namePattern:
            namePattern = self.jkCfg.getValue(self.profile, "schedulerJob")
        return self.findJobs(namePattern)
    
    def getJobsReportShort(self, onlyFailedJobs=False):
        """
        THIS IS FAST.

        Generator returns list of details of jobs. It consists the folloowing data:
            "Name", "Status", "HealthReport"
            
        If parameter onlyFailedJobs=True is specified, only failed jobs will be reported.
        
        Failed jobs are those with color RED (FAILED) or YELLOW (UNSTABLE)
            
        Use the following to print a pretty-formated report:
            print(tabulate(jserver.getJobsReport(), headers=["Name", "Status", "HealthReport"]))
        """
        jobs = self.getJobs()

        for job in jobs:
            healthReport = "-"
            if self.isFailedOrUnstable(job):
                j = self.branchView.get_data(self.branchView.python_api_url(job.url))
                healthReport = j["healthReport"][0]["description"]
            if not onlyFailedJobs:
                yield (job.name, JenkinsServer.actionTable[job.color]["status"], healthReport)
            elif self.isFailedOrUnstable(job):
                yield (job.name, JenkinsServer.actionTable[job.color]["status"], healthReport)
            else:
                continue

    def jobReport(self):
        print(tabulate(self.getJobsReportShort(), headers=["Name", "Status", "HealthReport"]))
        
    def failedJobReport(self):
        print(tabulate(self.getJobsReportShort(onlyFailedJobs=True), headers=["Name", "Status", "HealthReport"]))

    def getJobsSlow(self):
        """
        Generator returns jobs
        """
        for j, url in self._branchView.get_job_dict().items():
            job = FrozenJSON(self._branchView.get_data(self._branchView.python_api_url(url)))
            yield job

    def getJobsReportDetailed(self, onlyFailedJobs=False):
        """
        THIS IS SLOW BECUASE IT CHECKS BUILDS OF EACH JOB

        Generator returns list of details of jobs. It consists the folloowing data:
            "Name", "Status", "Last Build", "Last Stable Build", "Report"
            
        If parameter onlyFailedJobs=True is specified, only failed jobs will be reported.
        
        Failed jobs are those with color RED (FAILED) or YELLOW (UNSTABLE)
            
        Use the following to print a pretty-formated report:
            print(tabulate(jserver.getJobsReport(), headers=["Name", "Status", "Last Build", "Last Stable Build", "Report"]))
        """
        jobs = self.getJobsSlow()

        for job in jobs:
            if not onlyFailedJobs:
                yield self.jobDetails(job)
            elif self.isFailedOrUnstable(job):
                yield self.jobDetails(job)
            else:
                continue

    def startJob(self, job):
        if not self.isQueuedOrRunning(job):
            jobBuild = self.jenkinsServer.get_job(job.name)
            jobBuild.invoke()

    def build(self, verbose=True, namePattern=None):
        """
        Start the building jobs to build the applications.
        verbose=True will print the status.
        """
        for job in self.getBuildJobs(namePattern):
            if verbose:
                print("Starting building job: {}".format(job.name))
            self.startJob(job)

    def schedule(self, verbose=True, namePattern=None):
        """
        Start the scheduling jobs to run the regressions jobs.
        verbose=True will print the status.
        """
        for job in self.getSchedulerJobs(namePattern):
            if verbose:
                print("Starting schedule job: {}".format(job.name))
            self.startJob(job)
        
    def runFailedUnstableJobs(self, verbose=True, skipJob=None):
        """
        Start failed or unstable jobs. Provide regular expressions to exclude any job from being started
        skipJob, regular expressions separated by comma define the jobs to be skipped
        """
        
        if not skipJob:
            skipJob = self.jkCfg.getValue(self.profile, "skipJob")
        rexps = skipJob.split(",")
        jobs = self.getJobsReportShort(onlyFailedJobs=True)
        for job in jobs:
            print(job)
            j = collections.namedtuple("JobTemp", ("name", "status", "healthReport"))(*job)
            if not self._testConditions(rexps, j.name):
                if verbose:
                    print("Starting job: {}".format(j.name))
                self.startJob(j)
                
    def runIt(self, func):
        f = getattr(self, JenkinsServer.commandActor[func], None)
        if f is not None:
            f()
        else:
            raise ValueError("ERROR: Bad function name '{} = {}'".format(func, JenkinsServer.commandActor[func]))


    

In [ ]:

    

"""
The main program
"""

jenkins_yaml = """---
# Jenkins configuraions

# If this value is Yes, this application will not run.
# So change the values below and then change needChange to "No"
needChange: Yes

#------------------------------------------------------
# Values for variables not defined at the profile level
#------------------------------------------------------
# Jenkins' master URL
jenkinServerUrl: "http://jenkinsmasterprod.dovetail.net/jenkins/"

# User name and password to login to Jenkins master server
userName: "<username>"
password: "<password>"

buildJob: ".*-Build"            # The regular expression patterns, separated by comma, of build jobs
schedulerJob: ".*-Scheduler"    # The regular expression patterns, separated by comma,  of scheduler jobs
skipJob: ".*-MOD"               # The regular expression patterns, separated by comma, of jobs to be skipped when rerun

# The false filter for regressions jobs. Any job whose name does not satisfy the regular expression
# is considered as regression jobs.
# The patterns, separated by comma.
regressionJobFilter: ".*Build$,.*Scheduler$"

#------------------------------------------------------
# Values defined at the top level will be overridden by
# the values defined in profile level
#------------------------------------------------------
Release:
    projectName: "<project>"        # Main Jenkins' main view, mostly one per project
    branchName: "Release"           # Jenkins sub-views, mostly one per mercurial branch

Branch:
    projectName: "<project>"        # Main Jenkins' main view, mostly one per project
    branchName: "Branch"            # Jenkins sub-views, mostly one per mercurial branch
..."""

info = """
==============================================================================================
A new Jenkins configuration file ./jenkins.yaml has been generated.

Before you continue, modify the file accordingly first.

Check the jenkins.yaml for details.
==============================================================================================
"""

class JKCfg(object):
    def __init__(self, jkCfg):
        self._jkCfg = jkCfg
        
    def getValue(self, profile, name):
        pd = self._jkCfg[profile]
        defaultValue = self._jkCfg[name] if name in self._jkCfg else None
        return pd.get(name, defaultValue)

def runIt(jkCfg, profile, options):
    jserver = JenkinsServer(jkCfg, profile)
    
    for cmd in options:
        jserver.runIt(cmd)

def main(profile, options):

    generatedNewYaml = False
    if not os.path.exists("./jenkins.yaml"):
        generatedNewYaml = True
        with open("./jenkins.yaml", 'w', encoding='utf-8') as f:
             f.write(jenkins_yaml)

    with open("./jenkins.yaml", 'r') as f:
        jkCfg = JKCfg(yaml.load(f))

    if generatedNewYaml:
        print(info)

    if jkCfg.getValue(profile, "needChange"):
        print("It seems that you've not change the Jenkins configuration jenkins.yaml yet.\nPlease do so and try it again.")
    else:
        runIt(jkCfg, profile, options)


    

In [ ]:

    

def run_from_ipython():
    try:
        __IPYTHON__
        return True
    except NameError:
        return False

def displayHelpAndExit():
    print(
'''
Usage:

    python jenkins_tool.py -p profile_name
    Options:
        -p --profile    profile name
        -r --run        re-run all failed and unstable jobs
        -b --build      build the application
        -s --schedule   schedule all regressions to run
        -f --failed     list failed jobs
        -t --report     list all the jobs
'''
)

argvIPython = ["-t", "-p", "Branch"]
    
if __name__ == '__main__':
    profile = None
    options = []

    args = argvIPython if run_from_ipython() else sys.argv[1:]
    
    try:
        opts, args = getopt.getopt(args,"hbsrftp:",["help", "build", "schedule", "rerun", "failed", "report", "profile="])
    except getopt.GetoptError:
        displayHelpAndExit()
    for opt, arg in opts:
        if opt in ("-h", "--help"):
            profile = None
        elif opt in ("-p", "--profile"):
            profile = arg
        elif opt in ("-b", "--build"):
            options.append("build")
        elif opt in ("-s", "--schedule"):
            options.append("schedule")
        elif opt in ("-r", "--rerun"):
            options.append("rerun")
        elif opt in ("-f", "--failed"):
            options.append("failed")
        elif opt in ("-t", "--report"):
            options.append("report")
            
    if not profile:
        displayHelpAndExit()
    else:
        main(profile, options)


    

In [ ]:

    

%debug


    

In [ ]:

    

%reset


    

In [ ]:

    

with open("./jenkins.yaml", 'r') as f:
    jkCfg = JKCfg(yaml.load(f))

jserver = JenkinsServer(jkCfg, "Branch")