[MISC] Update libs and remove warning (#1014)

* Update libs

* Workflow tweaks

* Remove from_environ warning

Author: dragomirp

.github/workflows/cla-check.yml

@@ -2,7 +2,7 @@ name: CLA check
 
 on:
   pull_request:
-    branches: [main]
+    branches: [main, 16/edge]
 
 jobs:
   cla-check:

.github/workflows/lib-check.yaml

@@ -32,4 +32,6 @@ jobs:
         with:
           credentials: "${{ secrets.CHARMHUB_TOKEN }}"
           github-token: "${{ secrets.GITHUB_TOKEN }}"
-
+    permissions:
+      # Add label to prs
+      pull-requests: write

lib/charms/data_platform_libs/v0/data_interfaces.py

@@ -331,7 +331,7 @@ def _on_topic_requested(self, event: TopicRequestedEvent):
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 45
+LIBPATCH = 47
 
 PYDEPS = ["ops>=2.0.0"]
 
@@ -989,11 +989,7 @@ def __init__(
     @property
     def relations(self) -> List[Relation]:
         """The list of Relation instances associated with this relation_name."""
-        return [
-            relation
-            for relation in self._model.relations[self.relation_name]
-            if self._is_relation_active(relation)
-        ]
+        return self._model.relations[self.relation_name]
 
     @property
     def secrets_enabled(self):
@@ -1271,15 +1267,6 @@ def _legacy_apply_on_delete(self, fields: List[str]) -> None:
 
     # Internal helper methods
 
-    @staticmethod
-    def _is_relation_active(relation: Relation):
-        """Whether the relation is active based on contained data."""
-        try:
-            _ = repr(relation.data)
-            return True
-        except (RuntimeError, ModelError):
-            return False
-
     @staticmethod
     def _is_secret_field(field: str) -> bool:
         """Is the field in question a secret reference (URI) field or not?"""
@@ -2582,7 +2569,7 @@ def __init__(
 
 
 ################################################################################
-# Cross-charm Relatoins Data Handling and Evenets
+# Cross-charm Relations Data Handling and Events
 ################################################################################
 
 # Generic events
@@ -3281,7 +3268,7 @@ def __init__(
 # Kafka Events
 
 
-class KafkaProvidesEvent(RelationEvent):
+class KafkaProvidesEvent(RelationEventWithSecret):
     """Base class for Kafka events."""
 
     @property
@@ -3300,6 +3287,40 @@ def consumer_group_prefix(self) -> Optional[str]:
 
         return self.relation.data[self.relation.app].get("consumer-group-prefix")
 
+    @property
+    def mtls_cert(self) -> Optional[str]:
+        """Returns TLS cert of the client."""
+        if not self.relation.app:
+            return None
+
+        if not self.secrets_enabled:
+            raise SecretsUnavailableError("Secrets unavailable on current Juju version")
+
+        secret_field = f"{PROV_SECRET_PREFIX}{SECRET_GROUPS.MTLS}"
+        if secret_uri := self.relation.data[self.app].get(secret_field):
+            secret = self.framework.model.get_secret(id=secret_uri)
+            content = secret.get_content(refresh=True)
+            if content:
+                return content.get("mtls-cert")
+
+
+class KafkaClientMtlsCertUpdatedEvent(KafkaProvidesEvent):
+    """Event emitted when the mtls relation is updated."""
+
+    def __init__(self, handle, relation, old_mtls_cert: Optional[str] = None, app=None, unit=None):
+        super().__init__(handle, relation, app, unit)
+
+        self.old_mtls_cert = old_mtls_cert
+
+    def snapshot(self):
+        """Return a snapshot of the event."""
+        return super().snapshot() | {"old_mtls_cert": self.old_mtls_cert}
+
+    def restore(self, snapshot):
+        """Restore the event from a snapshot."""
+        super().restore(snapshot)
+        self.old_mtls_cert = snapshot["old_mtls_cert"]
+
 
 class TopicRequestedEvent(KafkaProvidesEvent, ExtraRoleEvent):
     """Event emitted when a new topic is requested for use on this relation."""
@@ -3312,6 +3333,7 @@ class KafkaProvidesEvents(CharmEvents):
     """
 
     topic_requested = EventSource(TopicRequestedEvent)
+    mtls_cert_updated = EventSource(KafkaClientMtlsCertUpdatedEvent)
 
 
 class KafkaRequiresEvent(RelationEvent):
@@ -3429,6 +3451,13 @@ def __init__(self, charm: CharmBase, relation_data: KafkaProviderData) -> None:
     def _on_relation_changed_event(self, event: RelationChangedEvent) -> None:
         """Event emitted when the relation has changed."""
         super()._on_relation_changed_event(event)
+
+        new_data_keys = list(event.relation.data[event.app].keys())
+        if any(newval for newval in new_data_keys if self.relation_data._is_secret_field(newval)):
+            self.relation_data._register_secrets_to_relation(event.relation, new_data_keys)
+
+        getattr(self.on, "mtls_cert_updated").emit(event.relation, app=event.app, unit=event.unit)
+
         # Leader only
         if not self.relation_data.local_unit.is_leader():
             return
@@ -3443,6 +3472,33 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None:
                 event.relation, app=event.app, unit=event.unit
             )
 
+    def _on_secret_changed_event(self, event: SecretChangedEvent):
+        """Event notifying about a new value of a secret."""
+        if not event.secret.label:
+            return
+
+        relation = self.relation_data._relation_from_secret_label(event.secret.label)
+        if not relation:
+            logging.info(
+                f"Received secret {event.secret.label} but couldn't parse, seems irrelevant"
+            )
+            return
+
+        if relation.app == self.charm.app:
+            logging.info("Secret changed event ignored for Secret Owner")
+
+        remote_unit = None
+        for unit in relation.units:
+            if unit.app != self.charm.app:
+                remote_unit = unit
+
+        old_mtls_cert = event.secret.get_content().get("mtls-cert")
+        # mtls-cert is the only secret that can be updated
+        logger.info("mtls-cert updated")
+        getattr(self.on, "mtls_cert_updated").emit(
+            relation, app=relation.app, unit=remote_unit, old_mtls_cert=old_mtls_cert
+        )
+
 
 class KafkaProvides(KafkaProviderData, KafkaProviderEventHandlers):
     """Provider-side of the Kafka relation."""
@@ -3463,11 +3519,13 @@ def __init__(
         extra_user_roles: Optional[str] = None,
         consumer_group_prefix: Optional[str] = None,
         additional_secret_fields: Optional[List[str]] = [],
+        mtls_cert: Optional[str] = None,
     ):
         """Manager of Kafka client relations."""
         super().__init__(model, relation_name, extra_user_roles, additional_secret_fields)
         self.topic = topic
         self.consumer_group_prefix = consumer_group_prefix or ""
+        self.mtls_cert = mtls_cert
 
     @property
     def topic(self):
@@ -3481,6 +3539,15 @@ def topic(self, value):
             raise ValueError(f"Error on topic '{value}', cannot be a wildcard.")
         self._topic = value
 
+    def set_mtls_cert(self, relation_id: int, mtls_cert: str) -> None:
+        """Set the mtls cert in the application relation databag / secret.
+
+        Args:
+            relation_id: the identifier for a particular relation.
+            mtls_cert: mtls cert.
+        """
+        self.update_relation_data(relation_id, {"mtls-cert": mtls_cert})
+
 
 class KafkaRequirerEventHandlers(RequirerEventHandlers):
     """Requires-side of the Kafka relation."""
@@ -3502,6 +3569,9 @@ def _on_relation_created_event(self, event: RelationCreatedEvent) -> None:
         # Sets topic, extra user roles, and "consumer-group-prefix" in the relation
         relation_data = {"topic": self.relation_data.topic}
 
+        if self.relation_data.mtls_cert:
+            relation_data["mtls-cert"] = self.relation_data.mtls_cert
+
         if self.relation_data.extra_user_roles:
             relation_data["extra-user-roles"] = self.relation_data.extra_user_roles
 
@@ -3560,15 +3630,17 @@ def __init__(
         extra_user_roles: Optional[str] = None,
         consumer_group_prefix: Optional[str] = None,
         additional_secret_fields: Optional[List[str]] = [],
+        mtls_cert: Optional[str] = None,
     ) -> None:
         KafkaRequirerData.__init__(
             self,
             charm.model,
             relation_name,
             topic,
-            extra_user_roles,
-            consumer_group_prefix,
-            additional_secret_fields,
+            extra_user_roles=extra_user_roles,
+            consumer_group_prefix=consumer_group_prefix,
+            additional_secret_fields=additional_secret_fields,
+            mtls_cert=mtls_cert,
         )
         KafkaRequirerEventHandlers.__init__(self, charm, self)
 

lib/charms/data_platform_libs/v0/upgrade.py

@@ -285,7 +285,7 @@ def restart(self, event) -> None:
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 18
+LIBPATCH = 19
 
 PYDEPS = ["pydantic>=1.10,<2", "poetry-core"]
 
@@ -929,7 +929,7 @@ def _on_upgrade_charm(self, event: UpgradeCharmEvent) -> None:
             # for k8s run version checks only on highest ordinal unit
             if (
                 self.charm.unit.name
-                == f"{self.charm.app.name}/{self.charm.app.planned_units() -1}"
+                == f"{self.charm.app.name}/{self.charm.app.planned_units() - 1}"
             ):
                 try:
                     self._upgrade_supported_check()

lib/charms/grafana_k8s/v0/grafana_dashboard.py

@@ -219,7 +219,7 @@ def __init__(self, *args):
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
 
-LIBPATCH = 43
+LIBPATCH = 44
 
 PYDEPS = ["cosl >= 0.0.50"]
 
@@ -1676,14 +1676,22 @@ def _set_default_data(self) -> None:
 
     def set_peer_data(self, key: str, data: Any) -> None:
         """Put information into the peer data bucket instead of `StoredState`."""
-        self._charm.peers.data[self._charm.app][key] = json.dumps(data)  # type: ignore[attr-defined]
+        peers = self._charm.peers  # type: ignore[attr-defined]
+        if not peers or not peers.data:
+            logger.info("set_peer_data: no peer relation. Is the charm being installed/removed?")
+            return
+        peers.data[self._charm.app][key] = json.dumps(data)  # type: ignore[attr-defined]
 
     def get_peer_data(self, key: str) -> Any:
         """Retrieve information from the peer data bucket instead of `StoredState`."""
-        if rel := self._charm.peers:  # type: ignore[attr-defined]
-            data = rel.data[self._charm.app].get(key, "")
-            return json.loads(data) if data else {}
-        return {}
+        peers = self._charm.peers  # type: ignore[attr-defined]
+        if not peers or not peers.data:
+            logger.warning(
+                "get_peer_data: no peer relation. Is the charm being installed/removed?"
+            )
+            return {}
+        data = peers.data[self._charm.app].get(key, "")
+        return json.loads(data) if data else {}
 
 
 class GrafanaDashboardAggregator(Object):

lib/charms/loki_k8s/v1/loki_push_api.py

@@ -546,7 +546,7 @@ def __init__(self, ...):
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 16
+LIBPATCH = 17
 
 PYDEPS = ["cosl"]
 
@@ -1354,7 +1354,7 @@ def _url(self) -> str:
 
         Return url to loki, including port number, but without the endpoint subpath.
         """
-        return "http://{}:{}".format(socket.getfqdn(), self.port)
+        return f"{self.scheme}://{socket.getfqdn()}:{self.port}"
 
     def _endpoint(self, url) -> dict:
         """Get Loki push API endpoint for a given url.

lib/charms/prometheus_k8s/v0/prometheus_scrape.py

@@ -362,7 +362,7 @@ def _on_scrape_targets_changed(self, event):
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 52
+LIBPATCH = 53
 
 # Version 0.0.53 needed for cosl.rules.generic_alert_groups
 PYDEPS = ["cosl>=0.0.53"]
@@ -1265,6 +1265,15 @@ def _dedupe_job_names(jobs: List[dict]):
     return deduped_jobs
 
 
+def _dedupe_list(items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """Deduplicate items in the list via object identity."""
+    unique_items = []
+    for item in items:
+        if item not in unique_items:
+            unique_items.append(item)
+    return unique_items
+
+
 def _resolve_dir_against_charm_path(charm: CharmBase, *path_elements: str) -> str:
     """Resolve the provided path items against the directory of the main file.
 
@@ -1538,7 +1547,7 @@ def set_scrape_job_spec(self, _=None):
         if self._forward_alert_rules:
             alert_rules.add_path(self._alert_rules_path, recursive=True)
             alert_rules.add(
-                generic_alert_groups.application_rules, group_name_prefix=self.topology.identifier
+                copy.deepcopy(generic_alert_groups.application_rules), group_name_prefix=self.topology.identifier
             )
         alert_rules_as_dict = alert_rules.as_dict()
 
@@ -1889,6 +1898,9 @@ def _set_prometheus_data(self, event: Optional[RelationJoinedEvent] = None):
         )
         groups.extend(alert_rules.as_dict()["groups"])
 
+        groups = _dedupe_list(groups)
+        jobs = _dedupe_list(jobs)
+
         # Set scrape jobs and alert rules in relation data
         relations = [event.relation] if event else self.model.relations[self._prometheus_relation]
         for rel in relations:
@@ -2141,10 +2153,12 @@ def _on_alert_rules_changed(self, event):
         self.set_alert_rule_data(app_name, unit_rules)
 
     def set_alert_rule_data(self, name: str, unit_rules: dict, label_rules: bool = True) -> None:
-        """Update alert rule data.
+        """Consolidate incoming alert rules (from stored-state or event) with those from relation data.
 
-        The unit rules should be a dict, which is has additional Juju topology labels added. For
+        The unit rules should be a dict, which have additional Juju topology labels added. For
         rules generated by the NRPE exporter, they are pre-labeled so lookups can be performed.
+        The unit rules are combined with the alert rules from relation data before being written
+        back to relation data and stored-state.
         """
         if not self._charm.unit.is_leader():
             return
@@ -2166,6 +2180,9 @@ def set_alert_rule_data(self, name: str, unit_rules: dict, label_rules: bool = T
 
             if updated_group["name"] not in [g["name"] for g in groups]:
                 groups.append(updated_group)
+
+            groups = _dedupe_list(groups)
+
             relation.data[self._charm.app]["alert_rules"] = json.dumps(
                 {"groups": groups if self._forward_alert_rules else []}
             )
@@ -2216,6 +2233,8 @@ def remove_alert_rules(self, group_name: str, unit_name: str) -> None:
                 changed_group["rules"] = rules_kept  # type: ignore
                 groups.append(changed_group)
 
+            groups = _dedupe_list(groups)
+
             relation.data[self._charm.app]["alert_rules"] = json.dumps(
                 {"groups": groups if self._forward_alert_rules else []}
             )