diff --git a/docs/checkout.md b/docs/checkout.md index e0e424e..fd66d6e 100644 --- a/docs/checkout.md +++ b/docs/checkout.md @@ -1,4 +1,4 @@ -### checkout +## checkout This command allow to test arbitary commit on the KernelCI Pipeline instance. This might be useful in several cases: - You want to test a specific commit, if it fails or pass test, or introduce any other degradation comparing to the current, or another commit. @@ -18,4 +18,63 @@ Where: - `commit` is the commit hash to test. - `jobfilter` is the job filter to use for the test (optional parameter) + +Other options: + +### --tipoftree + You can also set instead of --commit option --tipoftree which will retrieve the latest commit of the tree. + +### --watch + +Additionally, you can use --watch option to watch the progress of the test. + +After executing the command, you will see the output similar to the following: +```sh +./kci-dev.py checkout --giturl https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git --branch master --tipoftree --jobfilter baseline-nfs-arm64-qualcomm --jobfilter kbuild-gcc-12-arm64-chromeos-qualcomm --watch +api connect: https://staging.kernelci.org:9100/ +Retrieving latest commit on tree: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git branch: master +Commit to checkout: d3d1556696c1a993eec54ac585fe5bf677e07474 +OK +Watching for jobs on treeid: ad137d5a009f685d1c9c964897bcc35d552b031c9f542b433908fa1368b95465 +Current time: 2024-10-10 14:20:11 +Total tree nodes 1 found. +Node 6707b869322a7c560a1a2c69 job checkout State running Result None +Refresh in 30s...Current time: 2024-10-10 14:20:41 +Total tree nodes 1 found. +Node 6707b869322a7c560a1a2c69 job checkout State running Result None +Refresh in 30s...Current time: 2024-10-10 14:21:13 +Total tree nodes 1 found. +Node 6707b869322a7c560a1a2c69 job checkout State running Result None +Refresh in 30s...Current time: 2024-10-10 14:21:43 +Total tree nodes 1 found. +Node 6707b869322a7c560a1a2c69 job checkout State running Result None +Refresh in 30s...Current time: 2024-10-10 14:22:14 +Total tree nodes 1 found. +Node 6707b869322a7c560a1a2c69 job checkout State available Result None +Refresh in 30s...Current time: 2024-10-10 14:22:45 +Total tree nodes 2 found. +Node 6707b869322a7c560a1a2c69 job checkout State available Result None +Node 6707b8ed322a7c560a1a2dc2 job kbuild-gcc-12-arm64-chromeos-qualcomm State running Result None +... +Refresh in 30s...Current time: 2024-10-10 14:41:22 +Total tree nodes 12 found. +Node 6707b869322a7c560a1a2c69 job checkout State closing Result None +Node 6707b8ed322a7c560a1a2dc2 job kbuild-gcc-12-arm64-chromeos-qualcomm State done Result pass +Node 6707bc74322a7c560a1a38f6 job baseline-nfs-arm64-qualcomm State done Result pass +Node 6707bc75322a7c560a1a38f7 job baseline-nfs-arm64-qualcomm State running Result None +Refresh in 30s...Current time: 2024-10-10 14:41:53 +Total tree nodes 12 found. +Node 6707b869322a7c560a1a2c69 job checkout State closing Result None +Node 6707b8ed322a7c560a1a2dc2 job kbuild-gcc-12-arm64-chromeos-qualcomm State done Result pass +Node 6707bc74322a7c560a1a38f6 job baseline-nfs-arm64-qualcomm State done Result pass +Node 6707bc75322a7c560a1a38f7 job baseline-nfs-arm64-qualcomm State running Result None +Refresh in 30s...Current time: 2024-10-10 14:42:23 +Total tree nodes 12 found. +Node 6707b869322a7c560a1a2c69 job checkout State closing Result None +Node 6707b8ed322a7c560a1a2dc2 job kbuild-gcc-12-arm64-chromeos-qualcomm State done Result pass +Node 6707bc74322a7c560a1a38f6 job baseline-nfs-arm64-qualcomm State done Result pass +Node 6707bc75322a7c560a1a38f7 job baseline-nfs-arm64-qualcomm State running Result None +``` + +The command will keep watching the progress of the test until all jobs are done. You can also stop the watching by pressing `Ctrl+C` or command will stop after all jobs are done(or failed). diff --git a/kci-dev/subcommands/checkout.py b/kci-dev/subcommands/checkout.py index e451483..51b7903 100644 --- a/kci-dev/subcommands/checkout.py +++ b/kci-dev/subcommands/checkout.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +import datetime import json import re import subprocess +import time import click import requests @@ -22,6 +24,8 @@ def display_api_error(response): click.secho(response.json(), fg="red") except json.decoder.JSONDecodeError: click.secho(f"No JSON response. Plain text: {response.text}", fg="yellow") + except Exception as e: + click.secho(f"API response error: {e}: {response.text}", fg="red") return @@ -39,7 +43,7 @@ def send_checkout_full(baseurl, token, **kwargs): } jdata = json.dumps(data) try: - response = requests.post(url, headers=headers, data=jdata) + response = requests.post(url, headers=headers, data=jdata, timeout=30) except requests.exceptions.RequestException as e: click.secho(f"API connection error: {e}", fg="red") return @@ -50,6 +54,98 @@ def send_checkout_full(baseurl, token, **kwargs): return response.json() +def retrieve_treeid_nodes(baseurl, token, treeid): + url = baseurl + "latest/nodes/fast?treeid=" + treeid + headers = { + "Content-Type": "application/json; charset=utf-8", + "Authorization": f"{token}", + } + try: + response = requests.get(url, headers=headers, timeout=30) + except requests.exceptions.RequestException as e: + click.secho(f"API connection error: {e}, retrying...", fg="yellow") + return None + except Exception as e: + click.secho(f"API connection error: {e}, retrying...", fg="yellow") + return None + + if response.status_code >= 400: + display_api_error(response) + return None + + return response.json() + + +def check_node(node): + """ + Node can be defined RUNNING/DONE/FAIL based on the state + Simplify, as our current state model suboptimal + """ + name = node["name"] + state = node["state"] + result = node["result"] + if name == "checkout": + if state == "running": + return "RUNNING" + elif state == "available" or state == "closing": + return "DONE" + elif state == "done" and result == "pass": + return "DONE" + else: + return "FAIL" + else: + if state == "running": + return "RUNNING" + elif state == "done" and result == "pass": + return "DONE" + else: + return "FAIL" + + +def watch_jobs(baseurl, token, treeid, jobfilter): + # we need to add to jobfilter "checkout" node + jobfilter = list(jobfilter) + jobfilter.append("checkout") + while True: + inprogress = 0 + joblist = jobfilter.copy() + nodes = retrieve_treeid_nodes(baseurl, token, treeid) + if not nodes: + click.secho("No nodes found. Retrying...", fg="yellow") + time.sleep(5) + continue + time_local = time.localtime() + click.echo(f"Current time: {time.strftime('%Y-%m-%d %H:%M:%S', time_local)}") + click.secho(f"Total tree nodes {len(nodes)} found.", fg="green") + + # Tricky part in watch is that we might have one item in jobfilter (job, test), + # but it might spawn multiple nodes with same name + for node in nodes: + if node["name"] in jobfilter: + result = check_node(node) + if result == "DONE": + if isinstance(joblist, list) and node["name"] in joblist: + joblist.remove(node["name"]) + color = "green" + elif result == "RUNNING": + inprogress += 1 + color = "yellow" + else: + if isinstance(joblist, list) and node["name"] in joblist: + joblist.remove(node["name"]) + color = "red" + click.secho( + f"Node {node['_id']} job {node['name']} State {node['state']} Result {node['result']}", + fg=color, + ) + if len(joblist) == 0 and inprogress == 0: + click.secho("All jobs completed", fg="green") + return + + click.echo(f"\rRefresh in 30s...", nl=False) + time.sleep(30) + + def retrieve_tot_commit(repourl, branch): """ Retrieve the latest commit on a branch @@ -85,6 +181,11 @@ def retrieve_tot_commit(repourl, branch): help="Checkout on latest commit on tree/branch", is_flag=True, ) +@click.option( + "--watch", + help="Interactively watch for a tasks in jobfilter", + is_flag=True, +) # jobfilter is a list, might be one or more jobs @click.option( "--jobfilter", @@ -92,14 +193,18 @@ def retrieve_tot_commit(repourl, branch): multiple=True, ) @click.pass_context -def checkout(ctx, giturl, branch, commit, jobfilter, tipoftree): +def checkout(ctx, giturl, branch, commit, jobfilter, tipoftree, watch): cfg = ctx.obj.get("CFG") instance = ctx.obj.get("INSTANCE") url = api_connection(cfg[instance]["pipeline"]) + apiurl = cfg[instance]["api"] token = cfg[instance]["token"] if not jobfilter: jobfilter = None click.secho("No job filter defined. All jobs will be triggered!", fg="yellow") + if watch and not jobfilter: + click.secho("No job filter defined. Can't watch for a job(s)!", fg="red") + return if not commit and not tipoftree: click.secho("No commit or tree/branch latest commit defined", fg="red") return @@ -110,11 +215,27 @@ def checkout(ctx, giturl, branch, commit, jobfilter, tipoftree): commit = retrieve_tot_commit(giturl, branch) click.secho(f"Commit to checkout: {commit}", fg="green") resp = send_checkout_full( - url, token, giturl=giturl, branch=branch, commit=commit, jobfilter=jobfilter + url, + token, + giturl=giturl, + branch=branch, + commit=commit, + jobfilter=jobfilter, + watch=watch, ) if resp and "message" in resp: click.secho(resp["message"], fg="green") + if watch and isinstance(resp, dict): + node = resp.get("node") + treeid = node.get("treeid") + if not treeid: + click.secho("No treeid returned. Can't watch for a job(s)!", fg="red") + return + click.secho(f"Watching for jobs on treeid: {treeid}", fg="green") + # watch for jobs + watch_jobs(apiurl, token, treeid, jobfilter) + if __name__ == "__main__": main_kcidev()