Updated Marketplace models & log_level settings

Author: SteveMcGrath

pyproject.toml

@@ -75,6 +75,7 @@ unfixable = [ "B" ]
 
 [tool.ruff.format]
 line-ending = "lf"
+#quote-style="single"
 docstring-code-format = false
 docstring-code-line-length = "dynamic"
 

tenint/__init__.py

@@ -2,4 +2,4 @@
 from .models.configuration import Configuration, Settings  # noqa F401
 from .connector import Connector  # noqa F401
 
-__version__ = "1.0.3"
+__version__ = "1.0.4"

tenint/connector.py

@@ -9,7 +9,6 @@
 import json
 import logging
 import os
-import tomllib
 from enum import Enum
 from pathlib import Path
 from typing import Annotated, Callable
@@ -29,10 +28,6 @@
 logger = logging.getLogger("tenint.connector")
 
 
-class ConfigurationError(Exception):
-    pass
-
-
 class LogLevel(str, Enum):
     notset = "notset"
     debug = "debug"
@@ -105,53 +100,63 @@ def log_env_vars(self) -> None:
                 value = "{{ HIDDEN }}"
             logger.debug(f'EnvVar {key}="{value}"')
 
-    def fetch_config(
-        self,
-        data: str | None = None,
-        fn: Path | None = None,
-    ) -> BaseModel:
+    def fetch_config(self, data: str) -> (BaseModel, int):
         """
         Fetch and validate the configuration from either the data string or the filepath
         and return the settings object to the caller.
 
         Args:
-            data:
-                The string object of the
+            data: JSON formatted string of the settings
+
+        Returns:
+            The pydantic settings model and the status code.
+        """
+        try:
+            return self.settings(**json.loads(data)), 0
+        except ValidationError as e:
+            self.console.print(f"JSON String validation failed: {e}")
+        except Exception as _:
+            self.console.print_exception()
+        return None, 2
+
+    def callback(
+        self, url: str | None, resp: dict, job_id: str | None, status_code: int
+    ) -> None:
         """
-        settings = None
-
-        # If a string object is passed, we will handle that first.
-        if data:
-            settings = self.settings(**json.loads(data))
-
-        # If a file has been passed in instead an the suffix appears to be a JSON
-        # suffix, then we will assume a JSON file and handle accordingly.
-        elif fn and fn.is_file() and fn.suffix.lower() in [".json", ".jsn"]:
-            with fn.open("r", encoding="utf-8") as fobj:
-                settings = self.settings(**json.load(fobj))
-
-        # If the file passed has a TOML suffix, we will process as a toml file through
-        # tomllib.
-        elif fn and fn.is_file() and fn.suffix.lower() in [".toml", ".tml"]:
-            with fn.open("rb") as fobj:
-                settings = self.settings(**tomllib.load(fobj))
-
-        # If we processed anything, then return the settings object, otherwise raise a
-        # ConfigurationError
-        if settings:
-            logger.debug(f"Job config={settings.model_dump(mode='json')}")
-            return settings
-        raise ConfigurationError("No valid configurations passed.")
+        Initiate the callback response to the job scheduler
+
+        Args:
+            url: Callback url
+            resp: Dictionary response of the job
+            job_id: The Job UUID
+            status_code: Job status
+        """
+        # Set the Callback payload to the job response if the response is a dictionary
+        try:
+            payload = CallbackResponse(exit_code=status_code, **resp).model_dump(
+                mode="json"
+            )
+        except (ValidationError, TypeError) as _:
+            logger.error(f"Job response format is invalid: {resp=}")
+            payload = CallbackResponse(exit_code=status_code).model_dump(mode="json")
+
+        # If a callback and a job id have been set, then we will initiate a callback
+        # to the job manager with the response payload of the job to inform the manager
+        # that we have finished.
+        if job_id and url:
+            requests.post(url, headers={"X-Job-ID": job_id}, json=payload, verify=False)
+            logger.info(f"Called back to {url=} with {job_id=} and {payload=}")
+        else:
+            logger.warning("Did not initiate a callback!")
+        logger.info(f"callback={payload}")
 
     def cmd_config(
         self, pretty: Annotated[bool, Option(help="Pretty format the response")] = False
     ):
         """
         Return the connector configuration
         """
-        indent = 2
-        if not pretty:
-            indent = None
+        indent = 2 if pretty else None
 
         class Config(Configuration):
             settings_model: type[Settings] = self.settings
@@ -173,23 +178,14 @@ def cmd_validate(self):
     def cmd_run(
         self,
         json_data: Annotated[
-            str | None,
+            str,
             Option(
                 "--json",
                 "-j",
                 envvar="CONFIG_JSON",
                 help="The JSON config object as a string",
             ),
-        ] = None,
-        filename: Annotated[
-            Path | None,
-            Option(
-                "--filename",
-                "-f",
-                envvar="CONFIG_FILENAME",
-                help="Filename of either a json or toml file containing the job config",
-            ),
-        ] = None,
+        ],
         job_id: Annotated[
             str | None,
             Option(
@@ -209,15 +205,15 @@ def cmd_run(
             ),
         ] = None,
         log_level: Annotated[
-            LogLevel,
+            LogLevel | None,
             Option(
                 "--log-level",
                 "-l",
                 envvar="LOG_LEVEL",
                 help="Sets the logging verbosity for the job",
                 case_sensitive=False,
             ),
-        ] = LogLevel.info,
+        ] = None,
         since: Annotated[
             int | None,
             Option(
@@ -231,13 +227,24 @@ def cmd_run(
         """
         Invoke the connector
         """
+        resp = None
+        config, status_code = self.fetch_config(json_data)
+
+        # Set the log level, using local before config and then ultimately setting the
+        # log level to debug if nothing has been set.
+        if log_level:
+            lvl = log_level.upper()
+        elif config:
+            lvl = config.log_level
+        else:
+            lvl = "DEBUG"
+
+        # Configure the logging handlers
         logging.basicConfig(
-            level=log_level.upper(),
-            # format="%(asctime) %(name)s(%(filename)s:%(lineno)d): %(message)s",
+            level=lvl,
             format="%(name)s: %(message)s",
             datefmt="%Y-%m-%d %H:%M:%S",
             handlers=[
-                # logging.FileHandler("job.log"),
                 RichHandler(
                     console=self.console,
                     show_path=False,
@@ -248,46 +255,26 @@ def cmd_run(
                 logging.StreamHandler(),
             ],
         )
+
+        # Log the environment variables and configuration data
         self.log_env_vars()
         logger.info(f"Logging to {self.logfile}")
-        status_code = 0
-        resp = None
+        logger.debug(f"Job {json_data=}")
+        logger.info(f"Job config={config.model_dump_json() if config else None}")
 
         # Attempt to run the connector job and handle any errors that may be thrown
         # in a graceful way.
         try:
-            config = self.fetch_config(json_data, filename)
-            resp = self.main(config=config, since=since)
-        except (ValidationError, ConfigurationError) as e:
-            logging.error(f"Invalid configuration presented: {e}")
-            status_code = 2
+            if status_code == 0:
+                resp = self.main(config=config, since=since)
         except Exception as _:
             logging.exception("Job run failed with error", stack_info=2)
             status_code = 1
 
-        # Set the Callback payload to the job response if the response is a dictionary
-        try:
-            payload = CallbackResponse(exit_code=status_code, **resp).model_dump(
-                mode="json"
-            )
-        except (ValidationError, TypeError) as _:
-            logger.error(f"Job response format is invalid: {resp=}")
-            payload = CallbackResponse(exit_code=status_code).model_dump(mode="json")
-
-        # If a callback and a job id have been set, then we will initiate a callback
-        # to the job manager with the response payload of the job to inform the manager
-        # that we have finished.
-        if job_id and callback_url:
-            requests.post(
-                callback_url,
-                headers={"X-Job-ID": job_id},
-                json=payload,
-                verify=False,
-            )
-            logger.info(f"Called back to {callback_url=} with {job_id=} and {payload=}")
-        else:
-            logger.warning("Did not initiate a callback!")
-        logger.info(f"callback={payload}")
+        # Initiate the callback to the job management system.
+        self.callback(
+            url=callback_url, job_id=job_id, resp=resp, status_code=status_code
+        )
 
         # Exit the connector with the status code from the runner.
         raise Exit(code=status_code)

tenint/models/configuration.py

@@ -1,4 +1,4 @@
-from typing import Any
+from typing import Any, Literal
 
 from pydantic import BaseModel, ConfigDict, Field, computed_field
 
@@ -7,6 +7,7 @@
 
 class Settings(BaseModel):
     model_config = ConfigDict(extra="forbid")
+    log_level: Literal["INFO", "DEBUG"] = "INFO"
 
 
 class Configuration(BaseModel):

tenint/models/marketplace.py

@@ -2,23 +2,38 @@
 
 import tomllib
 from pathlib import Path
+from typing import Literal
 
 from pydantic import AnyHttpUrl, BaseModel, EmailStr
 
 from .pyproject import PyProject
 
 
+class MarketplaceResources(BaseModel):
+    disk: int
+    memory: int
+    cpu_cores: int
+
+
+class MarketplaceTimeout(BaseModel):
+    min: int
+    default: int
+    max: int
+
+
 class MarketplaceConnector(BaseModel):
     name: str
     slug: str
     description: str
     icon_url: AnyHttpUrl
     image_url: str
-    timeout: int
     marketplace_tag: str
     connector_owner: str
     support_contact: EmailStr
     tags: list[str]
+    schedule_types: list[Literal["hourly", "daily"]]
+    resources: MarketplaceResources
+    timeout: MarketplaceTimeout
 
     @classmethod
     def load_from_pyproject(
@@ -55,7 +70,9 @@ def load_from_pyproject(
             description=obj.project.description,
             icon_url=icon_url,
             image_url=image_url,
-            timeout=obj.tool.tenint.connector.timeout,
+            timeout=obj.tool.tenint.connector.timeout.model_dump(),
+            resources=obj.tool.tenint.connector.resources.model_dump(),
+            schedule_types=obj.tool.tenint.connector.schedule_types,
             marketplace_tag=obj.project.version,
             connector_owner=obj.project.authors[0].name,
             support_contact=obj.project.authors[0].email,

tenint/models/pyproject.py

@@ -1,6 +1,7 @@
-from typing import Annotated
+from typing import Annotated, Literal
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, model_validator
+from pydantic.functional_validators import model_validator
 from pydantic.networks import AnyHttpUrl, EmailStr
 
 
@@ -29,10 +30,24 @@ class Project(BaseModel):
     ]
 
 
+class TenintConnectorTimeout(BaseModel):
+    min: int = 3600
+    default: int = 3600
+    max: int = 86400
+
+
+class TenintConnectorResources(BaseModel):
+    disk: int = 1024
+    memory: int = 1024
+    cpu_cores: int = 1
+
+
 class TenintConnector(BaseModel):
     title: str
     tags: list[str]
-    timeout: int = 3600
+    schedule_types: list[Literal["hourly", "daily"]] = ["daily"]
+    timeout: Annotated[TenintConnectorTimeout, Field(validate_default=True)] = {}
+    resources: Annotated[TenintConnectorResources, Field(validate_default=True)] = {}
 
 
 class Tenint(BaseModel):

tenint/templates/pyproject.toml

@@ -42,4 +42,15 @@ title = "Example Connector"
 tags = ['tvm', 'example']
 
 # The default maximum run time in seconds
-timeout = 3600
+[tool.tenint.connector.timeout]
+default = 3600
+
+
+# This section defines the minimum resources necessary to run the
+# connector.  Below are the default values.  Disk and Memory values
+# are in Megabytes, where CPU Cores is is simply the number of cores
+# required to run the connector.
+[tool.tenint.connector.resources]
+disk = 1024
+memory = 1024
+cpu_cores = 1

tests/models/test_configuration.py

@@ -27,7 +27,13 @@ class TestConfiguration(Configuration):
 def test_config(config):
     assert config.settings == {
         "properties": {
-            "is_true": {"default": True, "title": "Is True", "type": "boolean"}
+            "is_true": {"default": True, "title": "Is True", "type": "boolean"},
+            "log_level": {
+                "default": "INFO",
+                "title": "Log Level",
+                "type": "string",
+                "enum": ["INFO", "DEBUG"],
+            },
         },
         "title": "TestSettings",
         "type": "object",