low-resource-jobs

#!/usr/bin/python

# low-resource-jobs - run jobs on low-resource buildbot slaves
# Copyright 2013 Linux Foundation.
# Written by Jeff Licquia <licquia@linuxfoundation.org>.

# This script looks at available jobs on low-resource build slaves,
# and runs one job per low-resource slave.  It decides which job to
# run by looking at the age of the current builds, and whether any
# builds are so old as to never have been run, and picks the oldest.
# For jobs that have been never run, or jobs with the same age, it
# picks randomly.  Beware: it runs one job per builder per run, so
# your low-resource slave can start consuming lots of resources
# fairly quickly if you run this script a lot of times.  For testing
# purposes, use --dry-run.

import sys
import random
import json
import urllib2

toplevel_url = "http://www.linuxbase.org/buildbot/json"
spool_dir = "/opt/buildbot/jobdir"

# Ultimately, read this from the buildbot config.  For now, though,
# it's easier just to hard-code.
low_resource_slave_archs = []

# We use the x86 build list for now, as it should have one of every
# build minus the arch-independent builds.
canonical_slave = "lfbuild-x86"

# Skip certain builders that are only there for building legacy
# stable builds.
skip_builds = ["azov-qt3-tests"]

def get_build_list():
    build_list_source = urllib2.urlopen("%s/slaves/%s" % (toplevel_url,
                                                          canonical_slave))
    build_data = json.loads(build_list_source.read())
    return [x[:x.rindex("-")] for x in build_data["builders"].keys()
            if x not in skip_builds]

def get_low_resource_slave_archs():
    # XXX: Read this from the buildbot config someday.
    return low_resource_slave_archs

def is_slave_building(arch):
    slave_source = urllib2.urlopen("%s/slaves/lfbuild-%s"
                                   % (toplevel_url, arch))
    slave_data = json.loads(slave_source.read())
    return len(slave_data["runningBuilds"]) > 0

def get_last_build_time(arch, build):
    try:
        build_source = urllib2.urlopen("%s/builders/%s/builds/-1" 
                                       % (toplevel_url, build + "-" + arch))
        build_data = json.loads(build_source.read())
        return build_data["times"][-1]
    except urllib2.URLError:
        return 0

def submit_build(arch, build):
    jobfile = open(spool_dir + "/" + arch + "_lowresource_job", "w")
    jobfile.write("""projects=%s
branch_name=devel
architectures=%s
""" % (build, arch))
    jobfile.close()

def main():
    random.seed()
    builds = get_build_list()
    last_run_times = {}
    builds_to_run = []
    for slave_arch in get_low_resource_slave_archs():
        # Don't add jobs if the builder is currently building.
        if is_slave_building(slave_arch):
            continue

        # Find builder with oldest most recent build.
        last_run_times[slave_arch] = {}
        oldest_builds = []
        for build in builds:
            last_run_times[slave_arch][build] = \
                get_last_build_time(slave_arch, build)
            if len(oldest_builds) == 0:
                oldest_builds.append(build)
            elif last_run_times[slave_arch][build] == \
                    last_run_times[slave_arch][oldest_builds[0]]:
                oldest_builds.append(build)
            elif last_run_times[slave_arch][build] < \
                    last_run_times[slave_arch][oldest_builds[0]]:
                oldest_builds = [build]

        # Tack on oldest builder.  If several have the same age,
        # pick one at random.
        if len(oldest_builds) == 1:
            builds_to_run.append((slave_arch, oldest_builds[0]))
        else:
            build_to_run = random.choice(oldest_builds)
            builds_to_run.append((slave_arch, build_to_run))

    if len(sys.argv) > 1 and sys.argv[1] == "--dry-run":
        for (arch, build) in builds_to_run:
            print arch + ": " + build
    else:
        for (arch, build) in builds_to_run:
            submit_build(arch, build)

if __name__ == "__main__":
    main()