In [ ]:

    

import requests
import os
import logging
import argparse
import json
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns


    

In [ ]:

    

cache_file = "cache.json"
token = f'Bearer {os.getenv("BUILDKITE_API_TOKEN")}'
builds = []


    

In [ ]:

    

if os.path.exists(cache_file):
    with open(cache_file) as f:
        builds = json.load(f)
        print(f'loaded {len(builds)} entries')


    

In [ ]:

    

# load new jobs from Buildkite API
if True:
    existing = set()
    for j in builds:
        existing.add(j['id'])

    # uncomment to reset
#     builds = []
#     existing = set()
    page = 1
    stop = False
    while not stop:
        print('loading page', page)
        re = requests.get('https://api.buildkite.com/v2/organizations/llvm-project/builds',
                          params={'page': page},
                          headers={'Authorization': token})
        if re.status_code != 200:
            print('response status', re.status_code, re)
            break
        x = re.json()
        if x == []:
            print('empty response')
            break
        for j in x:
            if j['id'] in existing:
                print('found existing job', j['id'])
                stop = True
                break
            # TODO: skip running jobs            
            if (j['state'] == 'running') or (j['state'] == 'scheduled'):
                print(j['web_url'], 'is', j['state'], ', skipping')
                continue
            builds.append(j)
        page += 1
    print(len(builds), 'jobs in total')    
    with open(cache_file, 'w') as f:
        json.dump(builds, f)
    print(f'saved {len(builds)} entries')


    

In [ ]:

    

d = {
    'id': [],
    'number': [],
    'pipeline': [],
}

jobs = {
    'pipeline': [],
    'name': [],
    'step_key': [],
    'state': [],
    'exit_status': [],
    'agent_id': [],
    
    'agent_name': [],
    'runnable_at': [],
    'started_at': [],
    'wait_duration': [],
    'finished_at': [],
    'run_duration': [],
}

sec = np.timedelta64(1, 's')
for b in builds:
    d['id'].append(b['id'])
    d['number'].append(b['number'])
    d['pipeline'].append(b['pipeline']['slug'])
    for x in b['jobs']:
        if x['state'] in ['waiting_failed', 'canceled', 'skipped', 'broken']:
            continue
        try:
            jobs['pipeline'].append(b['pipeline']['slug'])
            jobs['name'].append(x['name'])
            jobs['step_key'].append(x['step_key'] if 'step_key' in x else '')
            jobs['state'].append(x['state'] )
            jobs['exit_status'].append(x['exit_status'] if 'exit_status' in x else -1)
            jobs['agent_id'].append(x['agent']['id'] if 'agent' in x else '')
            jobs['agent_name'].append(x['agent']['name'] if 'agent' in x else '')
            runnable = np.datetime64(x['runnable_at'].replace('Z', ''))
            started = np.datetime64(x['started_at'].replace('Z', ''))
            finished = np.datetime64(x['finished_at'].replace('Z', ''))
            jobs['runnable_at'].append(runnable)
            jobs['started_at'].append(started)
            jobs['wait_duration'].append((started - runnable) / sec)
            jobs['finished_at'].append(finished)
            jobs['run_duration'].append((finished - started) / sec)
        except Exception as e:
            print(x)
            raise e            
jobs = pd.DataFrame(jobs)


    

In [ ]:

    

jobs.pipeline.unique()


    

In [ ]:

    

jobs


    

In [ ]:

    

ds = jobs[jobs['pipeline'] == 'premerge-checks'][jobs['step_key'] == 'windows'][jobs['state']=='passed'][~jobs['agent_name'].str.startswith('buildkite-')]


    

In [ ]:

    

ds


    

In [ ]:

    

fig, ax = plt.subplots(figsize=(10,10)) # size of the plot (width, height)

  sns.violinplot(
        ax=ax,
        x='run_duration',
        y='agent_name',
        split=True,
        data=ds)


    

In [ ]:

    

t = pd.pivot_table(ds, values=['run_duration'], index=['agent_name'],
                    aggfunc=[np.median, np.mean, np.std, np.count_nonzero])
t = t.reindex(t.sort_values(by=('median', 'run_duration'), ascending=False).index)
t