From c376630bad6bf6ef74ba3d6c246232c218c6e69b Mon Sep 17 00:00:00 2001
From: Paul Larson <paul.larson@canonical.com>
Date: Thu, 25 Jan 2024 09:37:26 -0600
Subject: [PATCH 1/4] Add support for specifying tags in a job

---
 docs/reference/job-schema.rst | 5 +++++
 server/src/api/schemas.py     | 1 +
 server/tests/test_v1.py       | 2 +-
 3 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/docs/reference/job-schema.rst b/docs/reference/job-schema.rst
index 3633e877..5e4113c0 100644
--- a/docs/reference/job-schema.rst
+++ b/docs/reference/job-schema.rst
@@ -16,6 +16,11 @@ The following table lists the key elements that a job definition file should con
     - String
     - /
     - Name of the job queue to which you want to submit the job. This field is mandatory.
+  * - ``tags``
+    - List of strings
+    - /
+    - | (Optional) List of tags that you want to associate with the job. 
+      | Tags can be used to search for jobs with the search API.
   * - ``global_timeout``
     - integer
     - | 14400
diff --git a/server/src/api/schemas.py b/server/src/api/schemas.py
index 463ac995..75574e90 100644
--- a/server/src/api/schemas.py
+++ b/server/src/api/schemas.py
@@ -55,6 +55,7 @@ class Job(Schema):
     job_id = fields.String(required=False)
     parent_job_id = fields.String(required=False)
     name = fields.String(required=False)
+    tags = fields.List(fields.String(), required=False)
     job_queue = fields.String(required=True)
     global_timeout = fields.Integer(required=False)
     output_timeout = fields.Integer(required=False)
diff --git a/server/tests/test_v1.py b/server/tests/test_v1.py
index 7fd87575..1f7c8c5b 100644
--- a/server/tests/test_v1.py
+++ b/server/tests/test_v1.py
@@ -33,7 +33,7 @@ def test_home(mongo_app):
 
 def test_add_job_good(mongo_app):
     """Test that adding a new job works"""
-    job_data = {"job_queue": "test"}
+    job_data = {"job_queue": "test", "tags": ["foo", "bar"]}
     # Place a job on the queue
     app, _ = mongo_app
     output = app.post("/v1/job", json=job_data)

From b08f35df915ef6cad2c98033ea3d953f5b6e8c58 Mon Sep 17 00:00:00 2001
From: Paul Larson <paul.larson@canonical.com>
Date: Thu, 25 Jan 2024 09:37:53 -0600
Subject: [PATCH 2/4] Add an API to search for jobs by tags and/or state

---
 server/README.rst         | 20 +++++++++++
 server/src/api/schemas.py | 18 ++++++++++
 server/src/api/v1.py      | 41 ++++++++++++++++++++++
 server/tests/test_v1.py   | 71 +++++++++++++++++++++++++++++++++++++++
 4 files changed, 150 insertions(+)

diff --git a/server/README.rst b/server/README.rst
index 1cb08b7b..26bb00f2 100644
--- a/server/README.rst
+++ b/server/README.rst
@@ -136,6 +136,26 @@ server will only return one job.
     $ curl http://localhost:8000/v1/job?queue=foo\&queue=bar
 
 
+** [GET] /v1/job/search ** - Search for jobs by tag(s) and state(s)
+
+Parameters:
+
+tags (array): List of string tags to search for
+match (string): Match mode for tags - "all" or "any" (default "any")
+state (array): List of job states to include (default all states other than cancelled and completed)
+Returns:
+
+Array of matching jobs
+
+Example:
+
+.. code-block:: console
+
+$ curl 'http://localhost:8000/v1/job/search?tags=foo&tags=bar&match=all'
+
+This will find jobs tagged with both "foo" and "bar".
+
+
 **[POST] /v1/result/<job_id>** - post job outcome data for the specified job_id
 
 - Parameters:
diff --git a/server/src/api/schemas.py b/server/src/api/schemas.py
index 75574e90..5c114e6c 100644
--- a/server/src/api/schemas.py
+++ b/server/src/api/schemas.py
@@ -73,6 +73,24 @@ class JobId(Schema):
     job_id = fields.String(required=True)
 
 
+class JobSearchRequest(Schema):
+    """Job search request schema"""
+
+    tags = fields.List(fields.String, description="List of tags to search for")
+    match = fields.String(
+        description="Match mode - 'all' or 'any' (default 'any')"
+    )
+    state = fields.List(
+        fields.String, description="List of job states to include"
+    )
+
+
+class JobSearchResponse(Schema):
+    """Job search response schema"""
+
+    jobs = fields.List(fields.Nested(Job), required=True)
+
+
 class Result(Schema):
     """Result schema"""
 
diff --git a/server/src/api/v1.py b/server/src/api/v1.py
index 42e74d69..254ed1c9 100644
--- a/server/src/api/v1.py
+++ b/server/src/api/v1.py
@@ -146,6 +146,47 @@ def job_get_id(job_id):
     return job_data
 
 
+@v1.get("/job/search")
+@v1.input(schemas.JobSearchRequest, location="query")
+@v1.output(schemas.JobSearchResponse)
+def search_jobs(query_data):
+    """Search for jobs by tags"""
+    tags = query_data.get("tags")
+    match = request.args.get("match", "any")
+    states = request.args.getlist("state")
+
+    if match not in ["any", "all"]:
+        abort(400, "Invalid match mode")
+
+    query = {}
+    if tags and match == "all":
+        query["job_data.tags"] = {"$all": tags}
+    elif tags and match == "any":
+        query["job_data.tags"] = {"$in": tags}
+
+    if states:
+        query["result_data.job_state"] = {"$in": states}
+    else:
+        # Exclude terminal states by default
+        query["result_data.job_state"] = {"$nin": ["cancelled", "complete"]}
+
+    pipeline = [
+        {"$match": query},
+        {
+            "$project": {
+                "job_id": True,
+                "created_at": True,
+                "job_state": "$result_data.job_state",
+                "_id": False,
+            },
+        },
+    ]
+
+    jobs = mongo.db.jobs.aggregate(pipeline)
+
+    return jsonify(list(jobs))
+
+
 @v1.post("/result/<job_id>")
 @v1.input(schemas.Result, location="json")
 def result_post(job_id, json_data):
diff --git a/server/tests/test_v1.py b/server/tests/test_v1.py
index 1f7c8c5b..b27d299a 100644
--- a/server/tests/test_v1.py
+++ b/server/tests/test_v1.py
@@ -421,3 +421,74 @@ def test_get_agents_data(mongo_app):
     assert len(output.json) == 1
     for key, value in agent_data.items():
         assert output.json[0][key] == value
+
+
+def test_search_jobs_by_tags(mongo_app):
+    """Test search_jobs by tags"""
+    app, _ = mongo_app
+
+    # Create some test jobs
+    job1 = {
+        "job_queue": "test",
+        "tags": ["tag1", "tag2"],
+    }
+    job2 = {
+        "job_queue": "test",
+        "tags": ["tag2", "tag3"],
+    }
+    job3 = {
+        "job_queue": "test",
+        "tags": ["tag3", "tag4"],
+    }
+    app.post("/v1/job", json=job1)
+    app.post("/v1/job", json=job2)
+    app.post("/v1/job", json=job3)
+
+    # Match any of the specified tags
+    output = app.get("/v1/job/search?tags=tag1&tags=tag2")
+    assert 200 == output.status_code
+    assert len(output.json) == 2
+
+    # Match all of the specified tags
+    output = app.get("/v1/job/search?tags=tag2&tags=tag3&match=all")
+    assert 200 == output.status_code
+    assert len(output.json) == 1
+
+
+def test_search_jobs_invalid_match(mongo_app):
+    """Test search_jobs with invalid match"""
+    app, _ = mongo_app
+
+    output = app.get("/v1/job/search?match=foo")
+    assert 400 == output.status_code
+    assert "Invalid match mode" in output.text
+
+
+def test_search_jobs_by_state(mongo_app):
+    """Test search jobs by state"""
+    app, _ = mongo_app
+
+    job = {
+        "job_queue": "test",
+        "tags": ["foo"],
+    }
+    # Two jobs that will stay in waiting state
+    app.post("/v1/job", json=job)
+    app.post("/v1/job", json=job)
+
+    # One job that will be cancelled
+    job_response = app.post("/v1/job", json=job)
+    job_id = job_response.json.get("job_id")
+    result_url = f"/v1/result/{job_id}"
+    data = {"job_state": "cancelled"}
+    app.post(result_url, json=data)
+
+    # By default, cancelled and completed jobs are filtered
+    output = app.get("/v1/job/search?tags=foo")
+    assert 200 == output.status_code
+    assert len(output.json) == 2
+
+    # But we can specify searching for one in any state
+    output = app.get("/v1/job/search?state=cancelled")
+    assert 200 == output.status_code
+    assert len(output.json) == 1

From e8f9c491aff305c33bb87afb7802916a7eac3caa Mon Sep 17 00:00:00 2001
From: Paul Larson <paul.larson@canonical.com>
Date: Thu, 25 Jan 2024 14:21:25 -0600
Subject: [PATCH 3/4] Add a howto for using the new search api

---
 docs/how-to/index.rst      |  1 +
 docs/how-to/search-job.rst | 63 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 64 insertions(+)
 create mode 100644 docs/how-to/search-job.rst

diff --git a/docs/how-to/index.rst b/docs/how-to/index.rst
index ecea6efa..78b2bb80 100644
--- a/docs/how-to/index.rst
+++ b/docs/how-to/index.rst
@@ -13,3 +13,4 @@ Work with jobs via Testflinger CLI
     change-server
     submit-job
     cancel-job
+    search-job
diff --git a/docs/how-to/search-job.rst b/docs/how-to/search-job.rst
new file mode 100644
index 00000000..cb72fcda
--- /dev/null
+++ b/docs/how-to/search-job.rst
@@ -0,0 +1,63 @@
+Search Jobs
+=============
+
+You can search for jobs based on the following criteria:
+   * tags
+   * state
+
+
+Searching by Tags
+------------------
+
+Tags can make it easier to find related jobs later. For instance, tools like
+spread have a need to find currently running jobs so that they can be
+cancelled once they are done using those devices for their testing.
+
+To add tags to a job submission, add a section like this with one or more tags
+in your job yaml:
+
+.. code-block:: yaml
+
+      tags:
+        - myproject
+        - client
+
+Testflinger doesn't use any of this information, but it makes it easier for
+you to search for those jobs later.
+
+The search API allows you to search for jobs based on tags and state. To
+search for a jobs by tag, you can provide one or more tags in the query string:
+
+.. code-block:: console
+
+      $ curl 'http://localhost:8000/v1/job/search?tags=foo&tags=bar'
+
+By default, the search API will return jobs that match any of the tags. To
+require that all tags match, you can provide the "match" query parameter with
+the value "all":
+
+.. code-block:: console
+
+      $ curl 'http://localhost:8000/v1/job/search?tags=foo&tags=bar&match=all'
+
+Searching by Job State
+-----------------------
+
+By default, the search API will only return jobs that are not already marked as
+cancelled or completed. To specify searching for jobs in a specific state, you
+can provide the "state" query parameter with one of the
+:doc:`test phases <../reference/test-phases>`:
+
+.. code-block:: console
+
+      $ curl 'http://localhost:8000/v1/job/search?state=provision'
+
+You can also search specify more than one state to match against. Obviously,
+since a job can only be in one state at a given moment, the matching mode
+for this will always be "any".
+
+.. code-block:: console
+
+      $ curl 'http://localhost:8000/v1/job/search?state=cancelled&state=completed'
+
+This can be done with or without providing tags in the search query.

From 4cbbb7a8409c1ac3727318a821a9724e3db4c98b Mon Sep 17 00:00:00 2001
From: Paul Larson <paul.larson@canonical.com>
Date: Mon, 29 Jan 2024 15:26:46 -0600
Subject: [PATCH 4/4] Change search behavior to search all jobs by default

---
 docs/how-to/search-job.rst | 17 ++++++++++++-----
 server/README.rst          |  2 +-
 server/src/api/schemas.py  | 21 +++++++++++++++++++--
 server/src/api/v1.py       | 10 +++-------
 server/tests/test_v1.py    | 20 +++++++++++++++++---
 5 files changed, 52 insertions(+), 18 deletions(-)

diff --git a/docs/how-to/search-job.rst b/docs/how-to/search-job.rst
index cb72fcda..46fa8dce 100644
--- a/docs/how-to/search-job.rst
+++ b/docs/how-to/search-job.rst
@@ -43,10 +43,9 @@ the value "all":
 Searching by Job State
 -----------------------
 
-By default, the search API will only return jobs that are not already marked as
-cancelled or completed. To specify searching for jobs in a specific state, you
-can provide the "state" query parameter with one of the
-:doc:`test phases <../reference/test-phases>`:
+By default, the search API will match jobs in any state.  To specify searching
+for jobs in a specific state, you can provide the "state" query parameter with
+one of the :doc:`test phases <../reference/test-phases>`:
 
 .. code-block:: console
 
@@ -60,4 +59,12 @@ for this will always be "any".
 
       $ curl 'http://localhost:8000/v1/job/search?state=cancelled&state=completed'
 
-This can be done with or without providing tags in the search query.
+To only search for jobs that have not been cancelled or completed, you can
+specify "active" for the state.
+
+.. code-block:: console
+
+      $ curl 'http://localhost:8000/v1/job/search?state=active'
+
+Searching for jobs by state can be done with or without providing tags in the
+search query.
diff --git a/server/README.rst b/server/README.rst
index 26bb00f2..9dc9521d 100644
--- a/server/README.rst
+++ b/server/README.rst
@@ -142,7 +142,7 @@ Parameters:
 
 tags (array): List of string tags to search for
 match (string): Match mode for tags - "all" or "any" (default "any")
-state (array): List of job states to include (default all states other than cancelled and completed)
+state (array): List of job states to include (or "active" to search all states other than cancelled and completed)
 Returns:
 
 Array of matching jobs
diff --git a/server/src/api/schemas.py b/server/src/api/schemas.py
index 5c114e6c..54d2cc81 100644
--- a/server/src/api/schemas.py
+++ b/server/src/api/schemas.py
@@ -21,6 +21,21 @@
 from apiflask.validators import OneOf
 
 
+ValidJobStates = (
+    "setup",
+    "provision",
+    "firmware_update",
+    "test",
+    "allocate",
+    "allocated",
+    "reserve",
+    "cleanup",
+    "cancelled",
+    "completed",
+    "active",  # fake state for jobs that are not completed or cancelled
+)
+
+
 class AgentIn(Schema):
     """Agent data input schema"""
 
@@ -78,10 +93,12 @@ class JobSearchRequest(Schema):
 
     tags = fields.List(fields.String, description="List of tags to search for")
     match = fields.String(
-        description="Match mode - 'all' or 'any' (default 'any')"
+        description="Match mode - 'all' or 'any' (default 'any')",
+        validate=OneOf(["any", "all"]),
     )
     state = fields.List(
-        fields.String, description="List of job states to include"
+        fields.String(validate=OneOf(ValidJobStates)),
+        description="List of job states to include",
     )
 
 
diff --git a/server/src/api/v1.py b/server/src/api/v1.py
index 254ed1c9..381ed0a7 100644
--- a/server/src/api/v1.py
+++ b/server/src/api/v1.py
@@ -155,20 +155,16 @@ def search_jobs(query_data):
     match = request.args.get("match", "any")
     states = request.args.getlist("state")
 
-    if match not in ["any", "all"]:
-        abort(400, "Invalid match mode")
-
     query = {}
     if tags and match == "all":
         query["job_data.tags"] = {"$all": tags}
     elif tags and match == "any":
         query["job_data.tags"] = {"$in": tags}
 
-    if states:
-        query["result_data.job_state"] = {"$in": states}
-    else:
-        # Exclude terminal states by default
+    if "active" in states:
         query["result_data.job_state"] = {"$nin": ["cancelled", "complete"]}
+    elif states:
+        query["result_data.job_state"] = {"$in": states}
 
     pipeline = [
         {"$match": query},
diff --git a/server/tests/test_v1.py b/server/tests/test_v1.py
index b27d299a..784f9dd0 100644
--- a/server/tests/test_v1.py
+++ b/server/tests/test_v1.py
@@ -460,8 +460,8 @@ def test_search_jobs_invalid_match(mongo_app):
     app, _ = mongo_app
 
     output = app.get("/v1/job/search?match=foo")
-    assert 400 == output.status_code
-    assert "Invalid match mode" in output.text
+    assert 422 == output.status_code
+    assert "Must be one of" in output.text
 
 
 def test_search_jobs_by_state(mongo_app):
@@ -483,12 +483,26 @@ def test_search_jobs_by_state(mongo_app):
     data = {"job_state": "cancelled"}
     app.post(result_url, json=data)
 
-    # By default, cancelled and completed jobs are filtered
+    # By default, all jobs are included if we don't specify the state
     output = app.get("/v1/job/search?tags=foo")
     assert 200 == output.status_code
+    assert len(output.json) == 3
+
+    # We can restrict this to active jobs
+    output = app.get("/v1/job/search?tags=foo&state=active")
+    assert 200 == output.status_code
     assert len(output.json) == 2
 
     # But we can specify searching for one in any state
     output = app.get("/v1/job/search?state=cancelled")
     assert 200 == output.status_code
     assert len(output.json) == 1
+
+
+def test_search_jobs_invalid_state(mongo_app):
+    """Test search jobs with invalid state"""
+    app, _ = mongo_app
+
+    output = app.get("/v1/job/search?state=foo")
+    assert 422 == output.status_code
+    assert "Must be one of" in output.text