Merge pull request #30 from nuclearcat/add-api-interaction

Unify endpoints and extend summary tool

.kci-dev.toml.example

@@ -1,21 +1,16 @@
 default_instance="local"
 
 [local]
-host="https://127.0.0.1"
+pipeline="https://127.0.0.1"
+api="https://127.0.0.1:8001/"
 token="example"
 
-[staging_pipeline]
-host="https://staging.kernelci.org:9100/"
+[staging]
+pipeline="https://staging.kernelci.org:9100/"
+api="https://staging.kernelci.org:9000/"
 token="example"
 
-[staging_api]
-host="https://staging.kernelci.org:9000/"
-token="example"
-
-[production_pipeline]
-host="https://kernelci-pipeline.westus3.cloudapp.azure.com/"
-token="example"
-
-[production_api]
-host="https://kernelci-api.westus3.cloudapp.azure.com/"
-token="example"
+[production]
+pipeline="https://kernelci-pipeline.westus3.cloudapp.azure.com/"
+api="https://kernelci-api.westus3.cloudapp.azure.com/"
+token="example"

docs/results.md

@@ -71,3 +71,36 @@ Result sample:
 ```
 
 testnodeid is the node id of the test job, which you can get from the KernelCI dashboard. Usually it is hexadecimal string.
+
+Additionally, for --nodes you can provide optional parameters --filter to filter the results by the given key and value. For example:
+```sh
+./kci-dev.py results --nodes --filter treeid=e25266f77837de335edba3c1b8d2a04edc2bfb195b77c44711d81ebea4494140 --filter kind=test
+```
+This command will show the nodes of tests in particular tree checkout. But as you might see, there is a lot of fields you might be not
+interested in.
+
+For this we have additional option --field, that will restrict output only to specified fields. For example:
+```sh
+./kci-dev.py results --nodes --filter treeid=e25266f77837de335edba3c1b8d2a04edc2bfb195b77c44711d81ebea4494140 --filter kind=test --field name --field result
+```
+Example:
+
+```json
+{'name': 'kver', 'result': 'pass'}
+{'name': 'config', 'result': 'pass'}
+{'name': 'build', 'result': 'pass'}
+{'name': 'example_init_test', 'result': 'pass'}
+{'name': 'time64_to_tm_test_date_range', 'result': 'pass'}
+{'name': 'test_one_cpu', 'result': 'skip'}
+{'name': 'test_many_cpus', 'result': 'skip'}
+{'name': 'test_one_task_on_all_cpus', 'result': 'skip'}
+{'name': 'test_two_tasks_on_all_cpus', 'result': 'skip'}
+{'name': 'test_one_task_on_one_cpu', 'result': 'skip'}
+{'name': 'test_one_task_mixed', 'result': 'skip'}
+{'name': 'test_two_tasks_on_one_cpu', 'result': 'skip'}
+{'name': 'test_two_tasks_on_one_all_cpus', 'result': 'skip'}
+{'name': 'test_task_on_all_and_one_cpu', 'result': 'skip'}
+{'name': 'resource_test_union', 'result': 'pass'}
+{'name': 'resource_test_intersection', 'result': 'pass'}
+....
+```

kci-dev/subcommands/checkout.py

@@ -95,7 +95,7 @@ def retrieve_tot_commit(repourl, branch):
 def checkout(ctx, giturl, branch, commit, jobfilter, tipoftree):
     cfg = ctx.obj.get("CFG")
     instance = ctx.obj.get("INSTANCE")
-    url = api_connection(cfg[instance]["host"])
+    url = api_connection(cfg[instance]["pipeline"])
     token = cfg[instance]["token"]
     if not jobfilter:
         jobfilter = None

kci-dev/subcommands/commit.py

@@ -68,7 +68,7 @@ def send_build(url, patch, branch, treeurl, token):
 def commit(ctx, repository, branch, origin, private, path):
     config = ctx.obj.get("CFG")
     instance = ctx.obj.get("INSTANCE")
-    url = api_connection(config[instance]["host"])
+    url = api_connection(config[instance]["pipeline"])
     diff = find_diff(path, branch, origin, repository)
     send_build(url, diff, branch, repository, config[instance]["token"])
 

kci-dev/subcommands/patch.py

@@ -49,7 +49,7 @@ def send_build(url, patch, branch, treeurl, token):
 def patch(ctx, repository, branch, private, patch):
     config = ctx.obj.get("CFG")
     instance = ctx.obj.get("INSTANCE")
-    url = api_connection(config[instance]["host"])
+    url = api_connection(config[instance]["pipeline"])
     patch = open(patch, "rb")
     send_build(url, patch, branch, repository, config[instance]["token"])
 

kci-dev/subcommands/results.py

@@ -15,32 +15,61 @@ def api_connection(host):
     return host
 
 
-def get_node(url, nodeid):
+def print_nodes(nodes, field):
+    if not isinstance(nodes, list):
+        nodes = [nodes]
+    for node in nodes:
+        if field:
+            data = {}
+            for f in field:
+                data[f] = node.get(f)
+            click.secho(pprint.pprint(data), fg="green", nl=False)
+        else:
+            click.secho(pprint.pprint(node), fg="green", nl=False)
+
+
+def get_node(url, nodeid, field):
     headers = {
         "Content-Type": "application/json; charset=utf-8",
     }
-    url = url + "/node/" + nodeid
+    url = url + "latest/node/" + nodeid
     click.secho(url)
     response = requests.get(url, headers=headers)
-    click.secho(response.status_code, fg="green")
     try:
-        click.secho(pprint.pprint(response.json()), fg="green")
-    except:
-        click.secho(pprint.pprint(response.text), fg="green")
+        response.raise_for_status()
+    except requests.exceptions.HTTPError as ex:
+        click.secho(ex.response.json().get("detail"), fg="red")
+        return None
+    except Exception as ex:
+        click.secho(ex, fg="red")
+        return None
+    print_nodes(response.json(), field)
 
 
-def get_nodes(url, limit, offset):
+def get_nodes(url, limit, offset, filter, field):
     headers = {
         "Content-Type": "application/json; charset=utf-8",
     }
-    url = url + "/nodes?limit=" + str(limit) + "&offset=" + str(offset)
+    url = url + "latest/nodes/fast?limit=" + str(limit) + "&offset=" + str(offset)
+    if filter:
+        for f in filter:
+            # TBD: We need to add translate filter to API
+            # if we need anything more complex than eq(=)
+            url = url + "&" + f
+
     click.secho(url)
     response = requests.get(url, headers=headers)
-    click.secho(response.status_code, fg="green")
     try:
-        click.secho(pprint.pprint(response.json()), fg="green")
-    except:
-        click.secho(pprint.pprint(response.text), fg="green")
+        response.raise_for_status()
+    except requests.exceptions.HTTPError as ex:
+        click.secho(ex.response.json().get("detail"), fg="red")
+        return None
+    except Exception as ex:
+        click.secho(ex, fg="red")
+        return None
+
+    nodes = response.json()
+    print_nodes(nodes, field)
 
 
 @click.command(help="Get results")
@@ -67,15 +96,27 @@ def get_nodes(url, limit, offset):
     required=False,
     help="Offset of the pagination",
 )
+@click.option(
+    "--filter",
+    required=False,
+    multiple=True,
+    help="Filter nodes by conditions",
+)
+@click.option(
+    "--field",
+    required=False,
+    multiple=True,
+    help="Print only particular field(s) from node data",
+)
 @click.pass_context
-def results(ctx, nodeid, nodes, limit, offset):
+def results(ctx, nodeid, nodes, limit, offset, filter, field):
     config = ctx.obj.get("CFG")
     instance = ctx.obj.get("INSTANCE")
-    url = api_connection(config[instance]["host"])
+    url = api_connection(config[instance]["api"])
     if nodeid:
-        get_node(url, nodeid)
+        get_node(url, nodeid, field)
     if nodes:
-        get_nodes(url, limit, offset)
+        get_nodes(url, limit, offset, filter, field)
 
 
 if __name__ == "__main__":

kci-dev/subcommands/testretry.py

@@ -53,7 +53,7 @@ def send_jobretry(baseurl, jobid, token):
 def testretry(ctx, nodeid):
     cfg = ctx.obj.get("CFG")
     instance = ctx.obj.get("INSTANCE")
-    url = api_connection(cfg[instance]["host"])
+    url = api_connection(cfg[instance]["pipeline"])
     resp = send_jobretry(url, nodeid, cfg[instance]["token"])
     if resp and "message" in resp:
         click.secho(resp["message"], fg="green")

tests/test_kcidev.py

@@ -84,7 +84,7 @@ def test_kcidev_results_tests():
         "run",
         "kci-dev",
         "--instance",
-        "staging_api",
+        "staging",
         "results",
         "--nodeid",
         "65a1355ee98651d0fe81e41d",
@@ -118,7 +118,7 @@ def test_kcidev_commit():
         "run",
         "kci-dev",
         "--instance",
-        "staging_api",
+        "staging",
         "commit",
         "--repository",
         "linux-next",