add log forwarding and fetch-lib (#21)

* add log forwarding and fetch-lib

* fix dashboard and add alert rules (#22)

* fix dashboard and add alert rules

* tox fmt

* minor fix alert rule

* add juju >= 3.4 for pebble log forwarding

lib/charms/observability_libs/v0/kubernetes_compute_resources_patch.py

@@ -133,7 +133,7 @@ def setUp(self, *unused):
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 6
+LIBPATCH = 7
 
 
 _Decimal = Union[Decimal, float, str, int]  # types that are potentially convertible to Decimal
@@ -364,7 +364,7 @@ def is_patched(self, resource_reqs: ResourceRequirements) -> bool:
         Returns:
             bool: A boolean indicating if the service patch has been applied.
         """
-        return equals_canonically(self.get_templated(), resource_reqs)
+        return equals_canonically(self.get_templated(), resource_reqs)  # pyright: ignore
 
     def get_templated(self) -> Optional[ResourceRequirements]:
         """Returns the resource limits specified in the StatefulSet template."""
@@ -397,8 +397,8 @@ def is_ready(self, pod_name, resource_reqs: ResourceRequirements):
             self.get_templated(),
             self.get_actual(pod_name),
         )
-        return self.is_patched(resource_reqs) and equals_canonically(
-            resource_reqs, self.get_actual(pod_name)
+        return self.is_patched(resource_reqs) and equals_canonically(  # pyright: ignore
+            resource_reqs, self.get_actual(pod_name)  # pyright: ignore
         )
 
     def apply(self, resource_reqs: ResourceRequirements) -> None:

lib/charms/traefik_k8s/v2/ingress.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Canonical Ltd.
+# Copyright 2024 Canonical Ltd.
 # See LICENSE file for licensing details.
 
 r"""# Interface Library for ingress.
@@ -13,7 +13,7 @@
 
 ```shell
 cd some-charm
-charmcraft fetch-lib charms.traefik_k8s.v1.ingress
+charmcraft fetch-lib charms.traefik_k8s.v2.ingress
 ```
 
 In the `metadata.yaml` of the charm, add the following:
@@ -72,69 +72,142 @@ def _on_ingress_revoked(self, event: IngressPerAppRevokedEvent):
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 8
+LIBPATCH = 11
 
-PYDEPS = ["pydantic<2.0"]
+PYDEPS = ["pydantic"]
 
 DEFAULT_RELATION_NAME = "ingress"
 RELATION_INTERFACE = "ingress"
 
 log = logging.getLogger(__name__)
 BUILTIN_JUJU_KEYS = {"ingress-address", "private-address", "egress-subnets"}
 
+PYDANTIC_IS_V1 = int(pydantic.version.VERSION.split(".")[0]) < 2
+if PYDANTIC_IS_V1:
 
-class DatabagModel(BaseModel):
-    """Base databag model."""
+    class DatabagModel(BaseModel):  # type: ignore
+        """Base databag model."""
 
-    class Config:
-        """Pydantic config."""
-
-        allow_population_by_field_name = True
-        """Allow instantiating this class by field name (instead of forcing alias)."""
-
-    _NEST_UNDER = None
+        class Config:
+            """Pydantic config."""
 
-    @classmethod
-    def load(cls, databag: MutableMapping):
-        """Load this model from a Juju databag."""
-        if cls._NEST_UNDER:
-            return cls.parse_obj(json.loads(databag[cls._NEST_UNDER]))
+            allow_population_by_field_name = True
+            """Allow instantiating this class by field name (instead of forcing alias)."""
 
-        try:
-            data = {k: json.loads(v) for k, v in databag.items() if k not in BUILTIN_JUJU_KEYS}
-        except json.JSONDecodeError as e:
-            msg = f"invalid databag contents: expecting json. {databag}"
-            log.error(msg)
-            raise DataValidationError(msg) from e
+        _NEST_UNDER = None
 
-        try:
-            return cls.parse_raw(json.dumps(data))  # type: ignore
-        except pydantic.ValidationError as e:
-            msg = f"failed to validate databag: {databag}"
-            log.error(msg, exc_info=True)
-            raise DataValidationError(msg) from e
+        @classmethod
+        def load(cls, databag: MutableMapping):
+            """Load this model from a Juju databag."""
+            if cls._NEST_UNDER:
+                return cls.parse_obj(json.loads(databag[cls._NEST_UNDER]))
 
-    def dump(self, databag: Optional[MutableMapping] = None, clear: bool = True):
-        """Write the contents of this model to Juju databag.
+            try:
+                data = {
+                    k: json.loads(v)
+                    for k, v in databag.items()
+                    # Don't attempt to parse model-external values
+                    if k in {f.alias for f in cls.__fields__.values()}
+                }
+            except json.JSONDecodeError as e:
+                msg = f"invalid databag contents: expecting json. {databag}"
+                log.error(msg)
+                raise DataValidationError(msg) from e
 
-        :param databag: the databag to write the data to.
-        :param clear: ensure the databag is cleared before writing it.
-        """
-        if clear and databag:
-            databag.clear()
+            try:
+                return cls.parse_raw(json.dumps(data))  # type: ignore
+            except pydantic.ValidationError as e:
+                msg = f"failed to validate databag: {databag}"
+                log.debug(msg, exc_info=True)
+                raise DataValidationError(msg) from e
+
+        def dump(self, databag: Optional[MutableMapping] = None, clear: bool = True):
+            """Write the contents of this model to Juju databag.
+
+            :param databag: the databag to write the data to.
+            :param clear: ensure the databag is cleared before writing it.
+            """
+            if clear and databag:
+                databag.clear()
+
+            if databag is None:
+                databag = {}
+
+            if self._NEST_UNDER:
+                databag[self._NEST_UNDER] = self.json(by_alias=True, exclude_defaults=True)
+                return databag
+
+            for key, value in self.dict(by_alias=True, exclude_defaults=True).items():  # type: ignore
+                databag[key] = json.dumps(value)
+
+            return databag
+
+else:
+    from pydantic import ConfigDict
+
+    class DatabagModel(BaseModel):
+        """Base databag model."""
+
+        model_config = ConfigDict(
+            # tolerate additional keys in databag
+            extra="ignore",
+            # Allow instantiating this class by field name (instead of forcing alias).
+            populate_by_name=True,
+            # Custom config key: whether to nest the whole datastructure (as json)
+            # under a field or spread it out at the toplevel.
+            _NEST_UNDER=None,
+        )  # type: ignore
+        """Pydantic config."""
 
-        if databag is None:
-            databag = {}
+        @classmethod
+        def load(cls, databag: MutableMapping):
+            """Load this model from a Juju databag."""
+            nest_under = cls.model_config.get("_NEST_UNDER")
+            if nest_under:
+                return cls.model_validate(json.loads(databag[nest_under]))  # type: ignore
 
-        if self._NEST_UNDER:
-            databag[self._NEST_UNDER] = self.json()
+            try:
+                data = {
+                    k: json.loads(v)
+                    for k, v in databag.items()
+                    # Don't attempt to parse model-external values
+                    if k in {(f.alias or n) for n, f in cls.__fields__.items()}
+                }
+            except json.JSONDecodeError as e:
+                msg = f"invalid databag contents: expecting json. {databag}"
+                log.error(msg)
+                raise DataValidationError(msg) from e
 
-        dct = self.dict()
-        for key, field in self.__fields__.items():  # type: ignore
-            value = dct[key]
-            databag[field.alias or key] = json.dumps(value)
+            try:
+                return cls.model_validate_json(json.dumps(data))  # type: ignore
+            except pydantic.ValidationError as e:
+                msg = f"failed to validate databag: {databag}"
+                log.debug(msg, exc_info=True)
+                raise DataValidationError(msg) from e
+
+        def dump(self, databag: Optional[MutableMapping] = None, clear: bool = True):
+            """Write the contents of this model to Juju databag.
+
+            :param databag: the databag to write the data to.
+            :param clear: ensure the databag is cleared before writing it.
+            """
+            if clear and databag:
+                databag.clear()
+
+            if databag is None:
+                databag = {}
+            nest_under = self.model_config.get("_NEST_UNDER")
+            if nest_under:
+                databag[nest_under] = self.model_dump_json(  # type: ignore
+                    by_alias=True,
+                    # skip keys whose values are default
+                    exclude_defaults=True,
+                )
+                return databag
 
-        return databag
+            dct = self.model_dump(mode="json", by_alias=True, exclude_defaults=True)  # type: ignore
+            databag.update({k: json.dumps(v) for k, v in dct.items()})
+            return databag
 
 
 # todo: import these models from charm-relation-interfaces/ingress/v2 instead of redeclaring them
@@ -165,10 +238,14 @@ class IngressRequirerAppData(DatabagModel):
 
     # fields on top of vanilla 'ingress' interface:
     strip_prefix: Optional[bool] = Field(
-        description="Whether to strip the prefix from the ingress url.", alias="strip-prefix"
+        default=False,
+        description="Whether to strip the prefix from the ingress url.",
+        alias="strip-prefix",
     )
     redirect_https: Optional[bool] = Field(
-        description="Whether to redirect http traffic to https.", alias="redirect-https"
+        default=False,
+        description="Whether to redirect http traffic to https.",
+        alias="redirect-https",
     )
 
     scheme: Optional[str] = Field(
@@ -195,8 +272,9 @@ class IngressRequirerUnitData(DatabagModel):
 
     host: str = Field(description="Hostname at which the unit is reachable.")
     ip: Optional[str] = Field(
+        None,
         description="IP at which the unit is reachable, "
-        "IP can only be None if the IP information can't be retrieved from juju."
+        "IP can only be None if the IP information can't be retrieved from juju.",
     )
 
     @validator("host", pre=True)
@@ -356,14 +434,6 @@ class IngressRequirerData:
     units: List["IngressRequirerUnitData"]
 
 
-class TlsProviderType(typing.Protocol):
-    """Placeholder."""
-
-    @property
-    def enabled(self) -> bool:  # type: ignore
-        """Placeholder."""
-
-
 class IngressPerAppProvider(_IngressPerAppBase):
     """Implementation of the provider of ingress."""
 
@@ -479,10 +549,10 @@ def _published_url(self, relation: Relation) -> Optional["IngressProviderAppData
     def publish_url(self, relation: Relation, url: str):
         """Publish to the app databag the ingress url."""
         ingress_url = {"url": url}
-        IngressProviderAppData.parse_obj({"ingress": ingress_url}).dump(relation.data[self.app])
+        IngressProviderAppData(ingress=ingress_url).dump(relation.data[self.app])  # type: ignore
 
     @property
-    def proxied_endpoints(self) -> Dict[str, str]:
+    def proxied_endpoints(self) -> Dict[str, Dict[str, str]]:
         """Returns the ingress settings provided to applications by this IngressPerAppProvider.
 
         For example, when this IngressPerAppProvider has provided the
@@ -497,7 +567,7 @@ def proxied_endpoints(self) -> Dict[str, str]:
         }
         ```
         """
-        results = {}
+        results: Dict[str, Dict[str, str]] = {}
 
         for ingress_relation in self.relations:
             if not ingress_relation.app:
@@ -517,8 +587,10 @@ def proxied_endpoints(self) -> Dict[str, str]:
             if not ingress_data:
                 log.warning(f"relation {ingress_relation} not ready yet: try again in some time.")
                 continue
-
-            results[ingress_relation.app.name] = ingress_data.ingress.dict()
+            if PYDANTIC_IS_V1:
+                results[ingress_relation.app.name] = ingress_data.ingress.dict()
+            else:
+                results[ingress_relation.app.name] = ingress_data.ingress.model_dump(mode=json)  # type: ignore
         return results
 
 
@@ -606,7 +678,6 @@ def __init__(
     def _handle_relation(self, event):
         # created, joined or changed: if we have auto data: publish it
         self._publish_auto_data()
-
         if self.is_ready():
             # Avoid spurious events, emit only when there is a NEW URL available
             new_url = (

metadata.yaml

@@ -4,6 +4,8 @@
 name: blackbox-exporter-k8s
 assumes:
   - k8s-api
+  # Juju >= 3.4 needed for Pebble log forwarding
+  - juju >= 3.4
 
 summary: |
   Kubernetes charm for Blackbox Exporter.

src/blackbox.py

@@ -111,7 +111,7 @@ def _command():
                 f"--config.file={self._config_path} "
                 f"--web.listen-address=:{self._port} "
                 f"--web.external-url={self._web_external_url} "
-                f"2>&1 | tee {self._log_path}'"
+                f"2>&1'"
             )
 
         return Layer(

src/charm.py

@@ -12,7 +12,7 @@
 from blackbox import ConfigUpdateFailure, WorkloadManager
 from charms.catalogue_k8s.v0.catalogue import CatalogueConsumer, CatalogueItem
 from charms.grafana_k8s.v0.grafana_dashboard import GrafanaDashboardProvider
-from charms.loki_k8s.v0.loki_push_api import LogProxyConsumer
+from charms.loki_k8s.v1.loki_push_api import LogForwarder
 from charms.observability_libs.v0.kubernetes_compute_resources_patch import (
     K8sResourcePatchFailedEvent,
     KubernetesComputeResourcesPatch,
@@ -107,13 +107,7 @@ def __init__(self, *args):
             ],
         )
         self._grafana_dashboard_provider = GrafanaDashboardProvider(charm=self)
-        self._log_proxy = LogProxyConsumer(
-            charm=self,
-            relation_name="logging",
-            log_files=[self._log_path],
-            container_name=self._container_name,
-            enable_syslog=False,
-        )
+        self._log_forwarding = LogForwarder(self, relation_name="logging")
 
         self.framework.observe(self.ingress.on.ready, self._handle_ingress)
         self.framework.observe(self.ingress.on.revoked, self._handle_ingress)