Precommit Job Times

In [ ]:

    

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.dates as md
import requests


    

In [ ]:

    

# Fetch precommit job data from Jenkins.

class Build(dict):
    def __init__(self, job_name, json):
        self['job_name'] = job_name
        self['result'] = json['result']
        self['number'] = json['number']
        self['timestamp'] = pd.Timestamp.utcfromtimestamp(json['timestamp'] / 1000)
        self['queuingDurationMillis'] = -1
        self['totalDurationMillis'] = -1
        for action in json['actions']:
            if action.get('_class', None) == 'jenkins.metrics.impl.TimeInQueueAction':
                self['queuingDurationMinutes'] = action['queuingDurationMillis'] / 60000.
                self['totalDurationMinutes'] = action['totalDurationMillis'] / 60000.
        if self['queuingDurationMinutes'] == -1:
            raise ValueError('could not find queuingDurationMillis in: %s', json)
        if self['totalDurationMinutes'] == -1:
            raise ValueError('could not find totalDurationMillis in: %s', json)
        
# Can be 'builds' (last 50) or 'allBuilds'.
builds_key = 'allBuilds'  

builds = []
job_names = ['beam_PreCommit_Java_Cron', 'beam_PreCommit_Python_Cron', 'beam_PreCommit_Go_Cron']
for job_name in job_names:
    url = 'https://ci-beam.apache.org/job/%s/api/json' % job_name
    params = {
        'tree': '%s[result,number,timestamp,actions[queuingDurationMillis,totalDurationMillis]]' % builds_key}
    r = requests.get(url, params=params)
    data = r.json()
    builds.extend([Build(job_name, build_json)
                         for build_json in data[builds_key]])

df = pd.DataFrame(builds)

timestamp_cutoff = pd.Timestamp.utcnow().tz_convert(None) - pd.Timedelta(weeks=4)
df_4weeks = df[df.timestamp >= timestamp_cutoff]
timestamp_cutoff = pd.Timestamp.utcnow().tz_convert(None) - pd.Timedelta(weeks=1)
df_1week = df[df.timestamp >= timestamp_cutoff]
timestamp_cutoff = pd.Timestamp.utcnow().tz_convert(None) - pd.Timedelta(days=1)
df_1day = df[df.timestamp >= timestamp_cutoff]


    

In [ ]:

    

# Graphs of precommit job times.

for job_name in job_names:
    duration_df = df_4weeks[df_4weeks.job_name == job_name]
    duration_df = duration_df[['timestamp', 'queuingDurationMinutes', 'totalDurationMinutes']]
    ax = duration_df.plot(x='timestamp')
    ax.set_title(job_name)


    

In [ ]:

    

# Get 95th percentile of precommit run times.
test_dfs = {'4 weeks': df_4weeks, '1 week': df_1week, '1 day': df_1day}
metrics = []

for sample_time, test_df in test_dfs.items():
    for job_name in job_names:
        df_times = test_df[test_df.job_name == job_name]
        for percentile in [95]:
            total_all = np.percentile(df_times.totalDurationMinutes, q=percentile)
            total_success = np.percentile(df_times[df_times.result == 'SUCCESS'].totalDurationMinutes,
                                          q=percentile)
            queue = np.percentile(df_times.queuingDurationMinutes, q=percentile)
            metrics.append({'job_name': '%s %s %dth' % (
                                job_name.replace('beam_PreCommit_','').replace('_GradleBuild',''),
                                sample_time, percentile),
                            'totalDurationMinutes_all': total_all,
                            'totalDurationMinutes_success_only': total_success,
                            'queuingDurationMinutes': queue,
                           })

pd.DataFrame(metrics).sort_values('job_name')


    

In [ ]:

    

# Fetch individual test data (precommit) from Jenkins.
MAX_FETCH_PER_JOB_TYPE = 5

test_results_raw = []
for job_name in list(df.job_name.unique()):
    if job_name == 'beam_PreCommit_Go_Cron':
        # TODO: Go builds are missing testReport data on Jenkins.
        continue
    build_nums = list(df.number[df.job_name == job_name].unique())
    num_fetched = 0
    for build_num in build_nums:
        url = 'https://ci-beam.apache.org/job/%s/%s/testReport/api/json?depth=1' % (job_name, build_num)
        print('.', end='')
        r = requests.get(url)
        if not r.ok:
            # Typically a 404 means that the job is still running.
            print('skipping (%s): %s' % (r.status_code, url))
            continue
        raw_result = r.json()
        raw_result['job_name'] = job_name
        raw_result['build_num'] = build_num
        test_results_raw.append(raw_result)
        
        num_fetched += 1
        if num_fetched >= MAX_FETCH_PER_JOB_TYPE:
            break

print(' done')


    

In [ ]:

    

# Analyze individual test results.

class TestResult(dict):
    def __init__(self, job_name, build_num, json):
        self['job_name'] = job_name
        self['build_num'] = build_num
        self['name'] = json['name']
        self['duration'] = json['duration']
        self['className'] = json['className']
        self['status'] = json['status']

test_results = []
for test_result_raw in test_results_raw:
    job_name = test_result_raw['job_name']
    build_num = test_result_raw['build_num']
    for suite in test_result_raw['suites']:
        for case in suite['cases']:
            test_results.append(TestResult(job_name, build_num, case))

df_tests = pd.DataFrame(test_results)
df_tests = df_tests.drop(columns=['build_num'])
df_tests = df_tests.groupby(['className', 'job_name', 'name', 'status'], as_index=False).max()
df_tests = df_tests.sort_values('duration', ascending=False)

def filter_test_results(job_name, status):
    res = df_tests
    if job_name != 'all':
        res = res[res.job_name == job_name]
    if status != 'all':
        res = res[res.status == status]
    return res.head(n=20)

from ipywidgets import interact
interact(filter_test_results,
         job_name=['all'] + list(df_tests.job_name.unique()),
         status=['all'] + list(df_tests.status.unique()))