Merge pull request #31 from nuclearcat/add-watch-feature

feat(checkout): Add --watch option

docs/checkout.md

@@ -1,4 +1,4 @@
-### checkout
+## checkout
 
 This command allow to test arbitary commit on the KernelCI Pipeline instance. This might be useful in several cases:
 - You want to test a specific commit, if it fails or pass test, or introduce any other degradation comparing to the current, or another commit.
@@ -18,4 +18,63 @@ Where:
 - `commit` is the commit hash to test.
 - `jobfilter` is the job filter to use for the test (optional parameter)
 
+
+Other options:
+
+### --tipoftree
+
 You can also set instead of --commit option --tipoftree which will retrieve the latest commit of the tree.
+
+### --watch
+
+Additionally, you can use --watch option to watch the progress of the test.
+
+After executing the command, you will see the output similar to the following:
+```sh
+./kci-dev.py checkout --giturl https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git --branch master --tipoftree --jobfilter baseline-nfs-arm64-qualcomm --jobfilter kbuild-gcc-12-arm64-chromeos-qualcomm --watch
+api connect: https://staging.kernelci.org:9100/
+Retrieving latest commit on tree: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git branch: master
+Commit to checkout: d3d1556696c1a993eec54ac585fe5bf677e07474
+OK
+Watching for jobs on treeid: ad137d5a009f685d1c9c964897bcc35d552b031c9f542b433908fa1368b95465
+Current time: 2024-10-10 14:20:11
+Total tree nodes 1 found.
+Node 6707b869322a7c560a1a2c69 job checkout State running Result None
+Refresh in 30s...Current time: 2024-10-10 14:20:41
+Total tree nodes 1 found.
+Node 6707b869322a7c560a1a2c69 job checkout State running Result None
+Refresh in 30s...Current time: 2024-10-10 14:21:13
+Total tree nodes 1 found.
+Node 6707b869322a7c560a1a2c69 job checkout State running Result None
+Refresh in 30s...Current time: 2024-10-10 14:21:43
+Total tree nodes 1 found.
+Node 6707b869322a7c560a1a2c69 job checkout State running Result None
+Refresh in 30s...Current time: 2024-10-10 14:22:14
+Total tree nodes 1 found.
+Node 6707b869322a7c560a1a2c69 job checkout State available Result None
+Refresh in 30s...Current time: 2024-10-10 14:22:45
+Total tree nodes 2 found.
+Node 6707b869322a7c560a1a2c69 job checkout State available Result None
+Node 6707b8ed322a7c560a1a2dc2 job kbuild-gcc-12-arm64-chromeos-qualcomm State running Result None
+...
+Refresh in 30s...Current time: 2024-10-10 14:41:22
+Total tree nodes 12 found.
+Node 6707b869322a7c560a1a2c69 job checkout State closing Result None
+Node 6707b8ed322a7c560a1a2dc2 job kbuild-gcc-12-arm64-chromeos-qualcomm State done Result pass
+Node 6707bc74322a7c560a1a38f6 job baseline-nfs-arm64-qualcomm State done Result pass
+Node 6707bc75322a7c560a1a38f7 job baseline-nfs-arm64-qualcomm State running Result None
+Refresh in 30s...Current time: 2024-10-10 14:41:53
+Total tree nodes 12 found.
+Node 6707b869322a7c560a1a2c69 job checkout State closing Result None
+Node 6707b8ed322a7c560a1a2dc2 job kbuild-gcc-12-arm64-chromeos-qualcomm State done Result pass
+Node 6707bc74322a7c560a1a38f6 job baseline-nfs-arm64-qualcomm State done Result pass
+Node 6707bc75322a7c560a1a38f7 job baseline-nfs-arm64-qualcomm State running Result None
+Refresh in 30s...Current time: 2024-10-10 14:42:23
+Total tree nodes 12 found.
+Node 6707b869322a7c560a1a2c69 job checkout State closing Result None
+Node 6707b8ed322a7c560a1a2dc2 job kbuild-gcc-12-arm64-chromeos-qualcomm State done Result pass
+Node 6707bc74322a7c560a1a38f6 job baseline-nfs-arm64-qualcomm State done Result pass
+Node 6707bc75322a7c560a1a38f7 job baseline-nfs-arm64-qualcomm State running Result None
+```
+
+The command will keep watching the progress of the test until all jobs are done. You can also stop the watching by pressing `Ctrl+C` or command will stop after all jobs are done(or failed).

kci-dev/subcommands/checkout.py

@@ -1,9 +1,11 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
+import datetime
 import json
 import re
 import subprocess
+import time
 
 import click
 import requests
@@ -22,6 +24,8 @@ def display_api_error(response):
         click.secho(response.json(), fg="red")
     except json.decoder.JSONDecodeError:
         click.secho(f"No JSON response. Plain text: {response.text}", fg="yellow")
+    except Exception as e:
+        click.secho(f"API response error: {e}: {response.text}", fg="red")
     return
 
 
@@ -39,7 +43,7 @@ def send_checkout_full(baseurl, token, **kwargs):
     }
     jdata = json.dumps(data)
     try:
-        response = requests.post(url, headers=headers, data=jdata)
+        response = requests.post(url, headers=headers, data=jdata, timeout=30)
     except requests.exceptions.RequestException as e:
         click.secho(f"API connection error: {e}", fg="red")
         return
@@ -50,6 +54,98 @@ def send_checkout_full(baseurl, token, **kwargs):
     return response.json()
 
 
+def retrieve_treeid_nodes(baseurl, token, treeid):
+    url = baseurl + "latest/nodes/fast?treeid=" + treeid
+    headers = {
+        "Content-Type": "application/json; charset=utf-8",
+        "Authorization": f"{token}",
+    }
+    try:
+        response = requests.get(url, headers=headers, timeout=30)
+    except requests.exceptions.RequestException as e:
+        click.secho(f"API connection error: {e}, retrying...", fg="yellow")
+        return None
+    except Exception as e:
+        click.secho(f"API connection error: {e}, retrying...", fg="yellow")
+        return None
+
+    if response.status_code >= 400:
+        display_api_error(response)
+        return None
+
+    return response.json()
+
+
+def check_node(node):
+    """
+    Node can be defined RUNNING/DONE/FAIL based on the state
+    Simplify, as our current state model suboptimal
+    """
+    name = node["name"]
+    state = node["state"]
+    result = node["result"]
+    if name == "checkout":
+        if state == "running":
+            return "RUNNING"
+        elif state == "available" or state == "closing":
+            return "DONE"
+        elif state == "done" and result == "pass":
+            return "DONE"
+        else:
+            return "FAIL"
+    else:
+        if state == "running":
+            return "RUNNING"
+        elif state == "done" and result == "pass":
+            return "DONE"
+        else:
+            return "FAIL"
+
+
+def watch_jobs(baseurl, token, treeid, jobfilter):
+    # we need to add to jobfilter "checkout" node
+    jobfilter = list(jobfilter)
+    jobfilter.append("checkout")
+    while True:
+        inprogress = 0
+        joblist = jobfilter.copy()
+        nodes = retrieve_treeid_nodes(baseurl, token, treeid)
+        if not nodes:
+            click.secho("No nodes found. Retrying...", fg="yellow")
+            time.sleep(5)
+            continue
+        time_local = time.localtime()
+        click.echo(f"Current time: {time.strftime('%Y-%m-%d %H:%M:%S', time_local)}")
+        click.secho(f"Total tree nodes {len(nodes)} found.", fg="green")
+
+        # Tricky part in watch is that we might have one item in jobfilter (job, test),
+        # but it might spawn multiple nodes with same name
+        for node in nodes:
+            if node["name"] in jobfilter:
+                result = check_node(node)
+                if result == "DONE":
+                    if isinstance(joblist, list) and node["name"] in joblist:
+                        joblist.remove(node["name"])
+                    color = "green"
+                elif result == "RUNNING":
+                    inprogress += 1
+                    color = "yellow"
+                else:
+                    if isinstance(joblist, list) and node["name"] in joblist:
+                        joblist.remove(node["name"])
+                    color = "red"
+                click.secho(
+                    f"Node {node['_id']} job {node['name']} State {node['state']} Result {node['result']}",
+                    fg=color,
+                )
+        if len(joblist) == 0 and inprogress == 0:
+            click.secho("All jobs completed", fg="green")
+            return
+
+        click.echo(f"\rRefresh in 30s...", nl=False)
+        time.sleep(30)
+
+
 def retrieve_tot_commit(repourl, branch):
     """
     Retrieve the latest commit on a branch
@@ -85,21 +181,30 @@ def retrieve_tot_commit(repourl, branch):
     help="Checkout on latest commit on tree/branch",
     is_flag=True,
 )
+@click.option(
+    "--watch",
+    help="Interactively watch for a tasks in jobfilter",
+    is_flag=True,
+)
 # jobfilter is a list, might be one or more jobs
 @click.option(
     "--jobfilter",
     help="Job filter to trigger",
     multiple=True,
 )
 @click.pass_context
-def checkout(ctx, giturl, branch, commit, jobfilter, tipoftree):
+def checkout(ctx, giturl, branch, commit, jobfilter, tipoftree, watch):
     cfg = ctx.obj.get("CFG")
     instance = ctx.obj.get("INSTANCE")
     url = api_connection(cfg[instance]["pipeline"])
+    apiurl = cfg[instance]["api"]
     token = cfg[instance]["token"]
     if not jobfilter:
         jobfilter = None
         click.secho("No job filter defined. All jobs will be triggered!", fg="yellow")
+    if watch and not jobfilter:
+        click.secho("No job filter defined. Can't watch for a job(s)!", fg="red")
+        return
     if not commit and not tipoftree:
         click.secho("No commit or tree/branch latest commit defined", fg="red")
         return
@@ -110,11 +215,27 @@ def checkout(ctx, giturl, branch, commit, jobfilter, tipoftree):
         commit = retrieve_tot_commit(giturl, branch)
         click.secho(f"Commit to checkout: {commit}", fg="green")
     resp = send_checkout_full(
-        url, token, giturl=giturl, branch=branch, commit=commit, jobfilter=jobfilter
+        url,
+        token,
+        giturl=giturl,
+        branch=branch,
+        commit=commit,
+        jobfilter=jobfilter,
+        watch=watch,
     )
     if resp and "message" in resp:
         click.secho(resp["message"], fg="green")
 
+    if watch and isinstance(resp, dict):
+        node = resp.get("node")
+        treeid = node.get("treeid")
+        if not treeid:
+            click.secho("No treeid returned. Can't watch for a job(s)!", fg="red")
+            return
+        click.secho(f"Watching for jobs on treeid: {treeid}", fg="green")
+        # watch for jobs
+        watch_jobs(apiurl, token, treeid, jobfilter)
+
 
 if __name__ == "__main__":
     main_kcidev()