From 65dc497e9de70d22fb6312a48e3fc600ff9c4729 Mon Sep 17 00:00:00 2001 From: francisco Date: Thu, 6 Feb 2025 12:08:00 -0300 Subject: [PATCH] feat: add documentation to tree listing endpoint Part of #86 Close #808 --- backend/kernelCI_app/serializers.py | 50 ---------- backend/kernelCI_app/typeModels/databases.py | 1 + .../kernelCI_app/typeModels/treeListing.py | 44 +++++++++ backend/kernelCI_app/views/treeView.py | 92 +++++++++++++++---- backend/schema.yml | 87 +++++++++++++----- 5 files changed, 181 insertions(+), 93 deletions(-) create mode 100644 backend/kernelCI_app/typeModels/treeListing.py diff --git a/backend/kernelCI_app/serializers.py b/backend/kernelCI_app/serializers.py index 9842b900f..3a39d5589 100644 --- a/backend/kernelCI_app/serializers.py +++ b/backend/kernelCI_app/serializers.py @@ -1,4 +1,3 @@ -from typing import Dict from rest_framework import serializers from kernelCI_app.models import Checkouts @@ -15,55 +14,6 @@ class Meta: ] -class TreeSerializer(serializers.Serializer): - build_status = serializers.SerializerMethodField(method_name="get_build_status") - test_status = serializers.SerializerMethodField(method_name="get_test_status") - boot_status = serializers.SerializerMethodField(method_name="get_boot_status") - git_commit_hash = serializers.CharField() - git_commit_name = serializers.CharField() - git_commit_tags = serializers.ListField() - patchset_hash = serializers.CharField() - tree_names = serializers.ListField() - git_repository_branch = serializers.CharField() - git_repository_url = serializers.CharField() - start_time = serializers.DateTimeField() - - class Meta(): - fields = ['build_status', 'test_status', 'git_commit_hash', 'patchset_hash'] - - def get_repository_url(self, obj) -> str: - return obj.id - - def get_build_status(self, obj) -> Dict: - return { - "valid": obj.valid_builds, - "invalid": obj.invalid_builds, - "null": obj.null_builds, - } - - def get_test_status(self, obj) -> Dict: - return { - "fail": obj.fail_tests, - "error": obj.error_tests, - "miss": obj.miss_tests, - "pass": obj.pass_tests, - "done": obj.done_tests, - "skip": obj.skip_tests, - "null": obj.null_tests, - } - - def get_boot_status(self, obj) -> Dict: - return { - "fail": obj.fail_boots, - "error": obj.error_boots, - "miss": obj.miss_boots, - "pass": obj.pass_boots, - "done": obj.done_boots, - "skip": obj.skip_boots, - "null": obj.null_boots, - } - - class TreeDetailsSerializer(serializers.Serializer): id = serializers.CharField() architecture = serializers.CharField() diff --git a/backend/kernelCI_app/typeModels/databases.py b/backend/kernelCI_app/typeModels/databases.py index 530a7e761..88c85058f 100644 --- a/backend/kernelCI_app/typeModels/databases.py +++ b/backend/kernelCI_app/typeModels/databases.py @@ -27,6 +27,7 @@ type Checkout__GitCommitName = Optional[str] type Checkout__GitCommitTags = Optional[List[str]] type Checkout__GitRepositoryBranch = Optional[str] +type Checkout__GitRepositoryUrl = Optional[str] type Build__Id = str type Build__Architecture = Optional[str] diff --git a/backend/kernelCI_app/typeModels/treeListing.py b/backend/kernelCI_app/typeModels/treeListing.py new file mode 100644 index 000000000..c6cd917ce --- /dev/null +++ b/backend/kernelCI_app/typeModels/treeListing.py @@ -0,0 +1,44 @@ +from typing import List +from kernelCI_app.typeModels.commonDetails import BuildStatusCount +from kernelCI_app.typeModels.databases import ( + Checkout__GitCommitHash, + Checkout__GitCommitName, + Checkout__GitCommitTags, + Checkout__TreeName, + Checkout__GitRepositoryBranch, + Checkout__GitRepositoryUrl, + Timestamp +) +from pydantic import BaseModel, Field, RootModel + + +class TestStatusCount(BaseModel): + pass_count: int = Field(default=0, alias='pass') + error_count: int = Field(default=0, alias='error') + fail_count: int = Field(default=0, alias='fail') + skip_count: int = Field(default=0, alias='skip') + miss_count: int = Field(default=0, alias='miss') + done_count: int = Field(default=0, alias='done') + null_count: int = Field(default=0, alias='null') + + +class Checkout(BaseModel): + build_status: BuildStatusCount + test_status: TestStatusCount + boot_status: TestStatusCount + git_commit_hash: Checkout__GitCommitHash + git_commit_name: Checkout__GitCommitName + git_commit_tags: List[Checkout__GitCommitTags] + tree_names: List[Checkout__TreeName] + git_repository_branch: Checkout__GitRepositoryBranch + git_repository_url: Checkout__GitRepositoryUrl + start_time: Timestamp + + +class TreeListingResponse(RootModel): + root: List[Checkout] + + +class TreeListingParameters(BaseModel): + origin: str = "maestro" + intervalInDays: int diff --git a/backend/kernelCI_app/views/treeView.py b/backend/kernelCI_app/views/treeView.py index d6f807793..1f28598df 100644 --- a/backend/kernelCI_app/views/treeView.py +++ b/backend/kernelCI_app/views/treeView.py @@ -1,23 +1,72 @@ -from django.http import JsonResponse +from typing import Dict from drf_spectacular.utils import extend_schema from rest_framework.views import APIView -from kernelCI_app.models import Checkouts -from kernelCI_app.serializers import TreeSerializer +from rest_framework.response import Response from kernelCI_app.utils import getQueryTimeInterval -from kernelCI_app.helpers.errorHandling import ExceptionWithJsonResponse +from kernelCI_app.helpers.errorHandling import ( + ExceptionWithJsonResponse, + create_api_error_response +) from kernelCI_app.helpers.date import parseIntervalInDaysGetParameter from http import HTTPStatus -from kernelCI_app.helpers.errorHandling import create_error_response +from kernelCI_app.typeModels.treeListing import ( + TreeListingResponse, + TreeListingParameters +) +from pydantic import ValidationError +from django.db import connection + DEFAULT_ORIGIN = "maestro" class TreeView(APIView): - # TODO Remove serializer, let's go full pydantic + def _sanitize_tree(self, checkout: Dict) -> Dict: + build_status = { + "valid": checkout[8], + "invalid": checkout[9], + "null": checkout[10], + } + + test_status = { + "fail": checkout[11], + "error": checkout[12], + "miss": checkout[13], + "pass": checkout[14], + "done": checkout[15], + "skip": checkout[16], + "null": checkout[17], + } + + boot_status = { + "fail": checkout[18], + "error": checkout[19], + "miss": checkout[20], + "pass": checkout[21], + "done": checkout[22], + "skip": checkout[23], + "null": checkout[24], + } + + return { + "git_repository_branch": checkout[2], + "git_repository_url": checkout[3], + "git_commit_hash": checkout[4], + "git_commit_tags": checkout[5], + "git_commit_name": checkout[6], + "start_time": checkout[7], + "build_status": {**build_status}, + "test_status": {**test_status}, + "boot_status": {**boot_status}, + "tree_names": checkout[25], + } + @extend_schema( - responses=TreeSerializer(many=True), + responses=TreeListingResponse, + parameters=[TreeListingParameters], + methods=["GET"] ) - def get(self, request): + def get(self, request) -> Response: origin_param = request.GET.get("origin", DEFAULT_ORIGIN) try: interval_days = parseIntervalInDaysGetParameter( @@ -36,8 +85,7 @@ def get(self, request): # '1 as id' is necessary in this case because django raw queries must include the primary key. # In this case we don't need the primary key and adding it would alter the GROUP BY clause, # potentially causing the tree listing page show the same tree multiple times - checkouts = Checkouts.objects.raw( - """ + query = """ SELECT 1 as id, checkouts.tree_name, @@ -138,15 +186,21 @@ def get(self, request): ORDER BY checkouts.git_commit_hash; ; - """, - params, - ) + """ + + checkouts = {} + with connection.cursor() as cursor: + cursor.execute(query, params) + checkouts = cursor.fetchall() if not checkouts: - return create_error_response( - error_message="Trees not found", status_code=HTTPStatus.OK - ) + return create_api_error_response(error_message="Trees not found", status_code=HTTPStatus.OK) + + checkouts = [self._sanitize_tree(checkout) for checkout in checkouts] + + try: + valid_response = TreeListingResponse(checkouts) + except ValidationError as e: + return Response(data=e.json(), status=HTTPStatus.INTERNAL_SERVER_ERROR) - serializer = TreeSerializer(checkouts, many=True) - resp = JsonResponse(serializer.data, safe=False) - return resp + return Response(valid_response.model_dump(by_alias=True)) diff --git a/backend/schema.yml b/backend/schema.yml index f672c60c2..0fc02b569 100644 --- a/backend/schema.yml +++ b/backend/schema.yml @@ -444,7 +444,20 @@ paths: description: '' /api/tree/: get: - operationId: tree_list + operationId: tree_retrieve + parameters: + - in: query + name: intervalInDays + schema: + title: Intervalindays + type: integer + required: true + - in: query + name: origin + schema: + default: maestro + title: Origin + type: string tags: - tree security: @@ -456,9 +469,7 @@ paths: content: application/json: schema: - type: array - items: - $ref: '#/components/schemas/Tree' + $ref: '#/components/schemas/Checkout' description: '' /api/tree/{commit_hash}/boots: get: @@ -975,6 +986,47 @@ components: anyOf: - type: boolean - type: 'null' + Checkout: + properties: + build_status: + $ref: '#/components/schemas/BuildStatusCount' + test_status: + $ref: '#/components/schemas/TestStatusCount' + boot_status: + $ref: '#/components/schemas/TestStatusCount' + git_commit_hash: + $ref: '#/components/schemas/Checkout__GitCommitHash' + git_commit_name: + $ref: '#/components/schemas/Checkout__GitCommitName' + git_commit_tags: + items: + $ref: '#/components/schemas/Checkout__GitCommitTags' + title: Git Commit Tags + type: array + tree_names: + items: + $ref: '#/components/schemas/Checkout__TreeName' + title: Tree Names + type: array + git_repository_branch: + $ref: '#/components/schemas/Checkout__GitRepositoryBranch' + git_repository_url: + $ref: '#/components/schemas/Checkout__GitRepositoryUrl' + start_time: + $ref: '#/components/schemas/Timestamp' + required: + - build_status + - test_status + - boot_status + - git_commit_hash + - git_commit_name + - git_commit_tags + - tree_names + - git_repository_branch + - git_repository_url + - start_time + title: Checkout + type: object Checkout__GitCommitHash: anyOf: - type: string @@ -993,6 +1045,10 @@ components: anyOf: - type: string - type: 'null' + Checkout__GitRepositoryUrl: + anyOf: + - type: string + - type: 'null' Checkout__Id: type: string Checkout__TreeName: @@ -1144,19 +1200,16 @@ components: title: Git Repository Branch tree_name: anyOf: - - type: string + - $ref: '#/components/schemas/Checkout__TreeName' - type: 'null' - title: Tree Name issue_id: anyOf: - - type: string + - $ref: '#/components/schemas/Issue__Id' - type: 'null' - title: Issue Id issue_version: anyOf: - - type: integer + - $ref: '#/components/schemas/Issue__Version' - type: 'null' - title: Issue Version required: - id - architecture @@ -1522,10 +1575,6 @@ components: $ref: '#/components/schemas/Issue__CulpritTool' culprit_harness: $ref: '#/components/schemas/Issue__CulpritHarness' - build_valid: - $ref: '#/components/schemas/Issue__BuildValid' - test_status: - $ref: '#/components/schemas/Issue__TestStatus' comment: $ref: '#/components/schemas/Issue__Comment' misc: @@ -1540,8 +1589,6 @@ components: - culprit_code - culprit_tool - culprit_harness - - build_valid - - test_status - comment - misc title: IssueDetailsResponse @@ -1583,10 +1630,6 @@ components: $ref: '#/components/schemas/IssueTestItem' title: IssueTestsResponse type: array - Issue__BuildValid: - anyOf: - - type: boolean - - type: 'null' Issue__Comment: anyOf: - type: string @@ -1617,10 +1660,6 @@ components: anyOf: - type: string - type: 'null' - Issue__TestStatus: - anyOf: - - type: string - - type: 'null' Issue__Version: type: integer Jsonb: