Tagged job searching

docs/how-to/index.rst

@@ -13,3 +13,4 @@ Work with jobs via Testflinger CLI
     change-server
     submit-job
     cancel-job
+    search-job

docs/how-to/search-job.rst

@@ -0,0 +1,70 @@
+Search Jobs
+=============
+
+You can search for jobs based on the following criteria:
+   * tags
+   * state
+
+
+Searching by Tags
+------------------
+
+Tags can make it easier to find related jobs later. For instance, tools like
+spread have a need to find currently running jobs so that they can be
+cancelled once they are done using those devices for their testing.
+
+To add tags to a job submission, add a section like this with one or more tags
+in your job yaml:
+
+.. code-block:: yaml
+
+      tags:
+        - myproject
+        - client
+
+Testflinger doesn't use any of this information, but it makes it easier for
+you to search for those jobs later.
+
+The search API allows you to search for jobs based on tags and state. To
+search for a jobs by tag, you can provide one or more tags in the query string:
+
+.. code-block:: console
+
+      $ curl 'http://localhost:8000/v1/job/search?tags=foo&tags=bar'
+
+By default, the search API will return jobs that match any of the tags. To
+require that all tags match, you can provide the "match" query parameter with
+the value "all":
+
+.. code-block:: console
+
+      $ curl 'http://localhost:8000/v1/job/search?tags=foo&tags=bar&match=all'
+
+Searching by Job State
+-----------------------
+
+By default, the search API will match jobs in any state.  To specify searching
+for jobs in a specific state, you can provide the "state" query parameter with
+one of the :doc:`test phases <../reference/test-phases>`:
+
+.. code-block:: console
+
+      $ curl 'http://localhost:8000/v1/job/search?state=provision'
+
+You can also search specify more than one state to match against. Obviously,
+since a job can only be in one state at a given moment, the matching mode
+for this will always be "any".
+
+.. code-block:: console
+
+      $ curl 'http://localhost:8000/v1/job/search?state=cancelled&state=completed'
+
+To only search for jobs that have not been cancelled or completed, you can
+specify "active" for the state.
+
+.. code-block:: console
+
+      $ curl 'http://localhost:8000/v1/job/search?state=active'
+
+Searching for jobs by state can be done with or without providing tags in the
+search query.

docs/reference/job-schema.rst

@@ -16,6 +16,11 @@ The following table lists the key elements that a job definition file should con
     - String
     - /
     - Name of the job queue to which you want to submit the job. This field is mandatory.
+  * - ``tags``
+    - List of strings
+    - /
+    - | (Optional) List of tags that you want to associate with the job. 
+      | Tags can be used to search for jobs with the search API.
   * - ``global_timeout``
     - integer
     - | 14400

server/README.rst

@@ -136,6 +136,26 @@ server will only return one job.
     $ curl http://localhost:8000/v1/job?queue=foo\&queue=bar
 
 
+** [GET] /v1/job/search ** - Search for jobs by tag(s) and state(s)
+
+Parameters:
+
+tags (array): List of string tags to search for
+match (string): Match mode for tags - "all" or "any" (default "any")
+state (array): List of job states to include (or "active" to search all states other than cancelled and completed)
+Returns:
+
+Array of matching jobs
+
+Example:
+
+.. code-block:: console
+
+$ curl 'http://localhost:8000/v1/job/search?tags=foo&tags=bar&match=all'
+
+This will find jobs tagged with both "foo" and "bar".
+
+
 **[POST] /v1/result/<job_id>** - post job outcome data for the specified job_id
 
 - Parameters:

server/src/api/schemas.py

@@ -21,6 +21,21 @@
 from apiflask.validators import OneOf
 
 
+ValidJobStates = (
+    "setup",
+    "provision",
+    "firmware_update",
+    "test",
+    "allocate",
+    "allocated",
+    "reserve",
+    "cleanup",
+    "cancelled",
+    "completed",
+    "active",  # fake state for jobs that are not completed or cancelled
+)
+
+
 class AgentIn(Schema):
     """Agent data input schema"""
 
@@ -55,6 +70,7 @@ class Job(Schema):
     job_id = fields.String(required=False)
     parent_job_id = fields.String(required=False)
     name = fields.String(required=False)
+    tags = fields.List(fields.String(), required=False)
     job_queue = fields.String(required=True)
     global_timeout = fields.Integer(required=False)
     output_timeout = fields.Integer(required=False)
@@ -72,6 +88,26 @@ class JobId(Schema):
     job_id = fields.String(required=True)
 
 
+class JobSearchRequest(Schema):
+    """Job search request schema"""
+
+    tags = fields.List(fields.String, description="List of tags to search for")
+    match = fields.String(
+        description="Match mode - 'all' or 'any' (default 'any')",
+        validate=OneOf(["any", "all"]),
+    )
+    state = fields.List(
+        fields.String(validate=OneOf(ValidJobStates)),
+        description="List of job states to include",
+    )
+
+
+class JobSearchResponse(Schema):
+    """Job search response schema"""
+
+    jobs = fields.List(fields.Nested(Job), required=True)
+
+
 class Result(Schema):
     """Result schema"""
 

server/src/api/v1.py

@@ -146,6 +146,43 @@ def job_get_id(job_id):
     return job_data
 
 
+@v1.get("/job/search")
+@v1.input(schemas.JobSearchRequest, location="query")
+@v1.output(schemas.JobSearchResponse)
+def search_jobs(query_data):
+    """Search for jobs by tags"""
+    tags = query_data.get("tags")
+    match = request.args.get("match", "any")
+    states = request.args.getlist("state")
+
+    query = {}
+    if tags and match == "all":
+        query["job_data.tags"] = {"$all": tags}
+    elif tags and match == "any":
+        query["job_data.tags"] = {"$in": tags}
+
+    if "active" in states:
+        query["result_data.job_state"] = {"$nin": ["cancelled", "complete"]}
+    elif states:
+        query["result_data.job_state"] = {"$in": states}
+
+    pipeline = [
+        {"$match": query},
+        {
+            "$project": {
+                "job_id": True,
+                "created_at": True,
+                "job_state": "$result_data.job_state",
+                "_id": False,
+            },
+        },
+    ]
+
+    jobs = mongo.db.jobs.aggregate(pipeline)
+
+    return jsonify(list(jobs))
+
+
 @v1.post("/result/<job_id>")
 @v1.input(schemas.Result, location="json")
 def result_post(job_id, json_data):

server/tests/test_v1.py

@@ -33,7 +33,7 @@ def test_home(mongo_app):
 
 def test_add_job_good(mongo_app):
     """Test that adding a new job works"""
-    job_data = {"job_queue": "test"}
+    job_data = {"job_queue": "test", "tags": ["foo", "bar"]}
     # Place a job on the queue
     app, _ = mongo_app
     output = app.post("/v1/job", json=job_data)
@@ -421,3 +421,88 @@ def test_get_agents_data(mongo_app):
     assert len(output.json) == 1
     for key, value in agent_data.items():
         assert output.json[0][key] == value
+
+
+def test_search_jobs_by_tags(mongo_app):
+    """Test search_jobs by tags"""
+    app, _ = mongo_app
+
+    # Create some test jobs
+    job1 = {
+        "job_queue": "test",
+        "tags": ["tag1", "tag2"],
+    }
+    job2 = {
+        "job_queue": "test",
+        "tags": ["tag2", "tag3"],
+    }
+    job3 = {
+        "job_queue": "test",
+        "tags": ["tag3", "tag4"],
+    }
+    app.post("/v1/job", json=job1)
+    app.post("/v1/job", json=job2)
+    app.post("/v1/job", json=job3)
+
+    # Match any of the specified tags
+    output = app.get("/v1/job/search?tags=tag1&tags=tag2")
+    assert 200 == output.status_code
+    assert len(output.json) == 2
+
+    # Match all of the specified tags
+    output = app.get("/v1/job/search?tags=tag2&tags=tag3&match=all")
+    assert 200 == output.status_code
+    assert len(output.json) == 1
+
+
+def test_search_jobs_invalid_match(mongo_app):
+    """Test search_jobs with invalid match"""
+    app, _ = mongo_app
+
+    output = app.get("/v1/job/search?match=foo")
+    assert 422 == output.status_code
+    assert "Must be one of" in output.text
+
+
+def test_search_jobs_by_state(mongo_app):
+    """Test search jobs by state"""
+    app, _ = mongo_app
+
+    job = {
+        "job_queue": "test",
+        "tags": ["foo"],
+    }
+    # Two jobs that will stay in waiting state
+    app.post("/v1/job", json=job)
+    app.post("/v1/job", json=job)
+
+    # One job that will be cancelled
+    job_response = app.post("/v1/job", json=job)
+    job_id = job_response.json.get("job_id")
+    result_url = f"/v1/result/{job_id}"
+    data = {"job_state": "cancelled"}
+    app.post(result_url, json=data)
+
+    # By default, all jobs are included if we don't specify the state
+    output = app.get("/v1/job/search?tags=foo")
+    assert 200 == output.status_code
+    assert len(output.json) == 3
+
+    # We can restrict this to active jobs
+    output = app.get("/v1/job/search?tags=foo&state=active")
+    assert 200 == output.status_code
+    assert len(output.json) == 2
+
+    # But we can specify searching for one in any state
+    output = app.get("/v1/job/search?state=cancelled")
+    assert 200 == output.status_code
+    assert len(output.json) == 1
+
+
+def test_search_jobs_invalid_state(mongo_app):
+    """Test search jobs with invalid state"""
+    app, _ = mongo_app
+
+    output = app.get("/v1/job/search?state=foo")
+    assert 422 == output.status_code
+    assert "Must be one of" in output.text