debian-iso: add autoupdate script

Because we want to spend less time working out how to update our
upstream dependencies, add a script that can automatically determine
the latest debian ISO, and automatically make a commit to upgrade to
it.

This is as simple as the following command:

    $ ./bump-debian-iso.sh auto

It includes a basic general-purpose bazel code modification tool that
should help us make these kinds of autoupdate scripts in general.

Author: celskeggs

tools/bump-debian-iso.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+set -euo pipefail
+
+cd "$(dirname "$0")"
+
+VERSION="${1:-}"
+TARGET="../platform/spire/debian-iso/deps.bzl"
+
+if [ "$VERSION" = "" ]
+then
+    echo "usage: $0 20190702+deb10u3 [or other new version]" 1>&2
+    echo "usage: $0 auto" 1>&2
+    echo 1>&2
+    echo "this command will try to update debian-iso/deps.bzl and then make a commit" 1>&2
+    exit 1
+fi
+
+## discover the latest version, if relevant
+
+if [ "$VERSION" = "auto" ]
+then
+    # yes, this is a HTTP fetch, and in theory could be MITM'd, but if the version number is
+    # invalid, debian-iso-checksum.sh will just fail later.
+    VERSION="$(curl -s http://debian.csail.mit.edu/debian/dists/buster/Release |
+               grep -E "^ [0-9a-f]{64} [ 0-9]+ main/installer-amd64/[0-9]+[+][0-9a-z]+/images/SHA256SUMS$" |
+               sed 's|^[^/]*/installer-amd64/\([^/]*\)/.*|\1|g')"
+fi
+
+## make sure the git state is clean enough for us
+
+if ! git diff --quiet "${TARGET}" || ! git diff --cached --quiet "${TARGET}"
+then
+    echo "cannot update ${TARGET} automatically; it has already been modified" 1>&2
+    exit 1
+fi
+
+## make the change
+
+./debian-iso-checksum.sh "${VERSION}" | ./update-defs.py "${TARGET}"
+
+## make the commit
+
+if git diff --quiet "${TARGET}"
+then
+    echo "no commit to make; files did not change" 1>&2
+    exit 1
+fi
+
+git commit -e -v "${TARGET}" -m "debian-iso: bump version to $VERSION" -m "This commit was automatically created with bump-debian-iso.sh."

tools/debian-checksum.sh tools/debian-iso-checksum.sh

@@ -1,9 +1,15 @@
 #!/bin/bash
 set -e -u
 
-VERSION="20190702+deb10u2"
+VERSION="${1:-}"
 RELEASE="buster"
 
+if [ "$VERSION" = "" ]
+then
+    echo "usage: debian-iso-checksum.sh <version>" 1>&2
+    exit 1
+fi
+
 echo "finding hash for release ${RELEASE} version ${VERSION}:" >&2
 
 WORK=$(mktemp -d)
@@ -29,6 +35,6 @@ echo >&2
 echo "new version block:" >&2
 
 echo >&2
-echo "VERSION = '${VERSION}'"
-echo "RELEASE = '${RELEASE}'"
-echo "MINI_ISO_HASH = '$(grep './netboot/mini.iso' SHA256SUMS | cut -d " " -f 1)'"
+echo "VERSION = \"${VERSION}\""
+echo "RELEASE = \"${RELEASE}\""
+echo "MINI_ISO_HASH = \"$(grep './netboot/mini.iso' SHA256SUMS | cut -d " " -f 1)\""

tools/update-defs.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""
+update-defs: change a set of Bazel variable declarations in a particular file.
+
+For example, if a file named `debian-iso/deps.bzl` had the following lines in the middle:
+
+    VERSION = '20190702+deb10u2'
+    RELEASE = 'buster'
+    MINI_ISO_HASH = 'fa713fb9ab7853de6aefe6b83c58bcdba19ef79a0310de84d40dd2754a9539d7'
+
+And you ran the following command with the following input:
+
+    $ ./update-defs.py debian-iso/deps.bzl
+    VERSION = '20190702+deb10u3'
+    RELEASE = 'buster'
+    MINI_ISO_HASH = '26b6f6f6bcb24c4e59b965d4a2a6c44af5d79381b9230d69a7d4db415ddcb4cd'
+
+Then those three lines (and only those lines) would be updated in `deps.bzl`.
+"""
+
+import re
+import sys
+
+
+def is_valid_identifier(text):
+    return bool(re.match("^[A-Z0-9_]+$", text))
+
+
+def get_match(line):
+    if "=" in line:
+        match = line.split("=",1)[0].strip()
+        if is_valid_identifier(match):
+            return match
+
+
+def parse_changes(changes):
+    mapping = {}
+    for line in changes.split("\n"):
+        if not line: continue
+        match = get_match(line)
+        if match is None:
+            raise ValueError("could not parse change line: %s" % repr(line))
+        if match in mapping:
+            raise KeyError("duplicate input mapping: %s" % match)
+        mapping[match] = line
+    return mapping
+
+
+def apply_changes(line, lookup, used):
+    match = get_match(line)
+    if match is not None and match in lookup:
+        used[match] += 1
+        line = lookup[match]
+        print("=>", line)
+    return line
+
+
+def perform_changes(filename, changes):
+    with open(filename, "r") as f:
+        lines = f.read().split("\n")
+    lookup = parse_changes(changes)
+    used = {var: 0 for var in lookup}
+    lines = [apply_changes(line, lookup, used) for line in lines]
+    for var, usedcount in used.items():
+        if usedcount < 1:
+            raise ValueError("did not use requested change: %s was not used" % (repr(var)))
+        elif usedcount > 1:
+            raise ValueError("used requested change multiple times: %s was used %d times" % (repr(var), usedcount))
+    with open(filename, "w") as f:
+        f.write("\n".join(lines))
+    print()
+    print("updated", len(lookup), "entries")
+
+
+def main(argv):
+    if len(argv) != 2:
+        print("usage: %s <target file>", file=sys.stderr)
+        sys.exit(1)
+    changes = sys.stdin.read()
+    perform_changes(sys.argv[1], changes)
+
+
+if __name__ == "__main__":
+    main(sys.argv)