feat(nimbus): Add a test that loads all FML manifests (#15090)

Because:

- we are constantly adding and updating feature manifests; 
- we are constantly updating our application-services dependency; and
- we have no tests that validate that existing manifests are valid in
these cases

this commit:

- adds a test case per application that validates the vendored FML files
are loadable.

Fixes #15020

Author: freshstrangemusic

experimenter/experimenter/experiments/constants.py

@@ -298,19 +298,19 @@ class Application(models.TextChoices):
     )
 
     @staticmethod
-    def is_sdk(application):
-        return application != Application.DESKTOP
+    def is_sdk(slug: str):
+        return slug != Application.DESKTOP
 
     @staticmethod
-    def is_mobile(application):
-        return application in (
+    def is_mobile(slug: str):
+        return slug in (
             Application.FENIX,
             Application.IOS,
         )
 
     @staticmethod
-    def is_web(application):
-        return application in (
+    def is_web(slug: str):
+        return slug in (
             Application.DEMO_APP,
             Application.MONITOR,
             Application.FXA,

experimenter/experimenter/experiments/tests/api/v6/test_serializers.py

@@ -27,10 +27,10 @@ class TestNimbusExperimentSerializer(TestCase):
     @classmethod
     def _validate_experiment_schema(
         cls,
-        application: NimbusExperiment.Application,
+        application_slug: str,
         experiment_data: dict[str, Any],
     ):
-        if NimbusExperiment.Application.is_sdk(application):
+        if NimbusExperiment.Application.is_sdk(application_slug):
             schema = SdkNimbusExperiment
         else:
             schema = DesktopAllVersionsNimbusExperiment

experimenter/experimenter/features/manifests/nimbus_fml_loader.py

@@ -55,20 +55,27 @@ def fml_client(self, version: Optional[NimbusFeatureVersion] = None) -> FmlClien
         """
         file_path = self.file_path(version)
         if file_path is not None:
-            try:
-                return FmlClient(
-                    str(file_path),
-                    self.channel,
-                )
-            except FmlError:
-                logger.exception(
-                    f"Nimbus FML Loader: FmlClient failed to parse manifest: {file_path}"
-                )
-                return None
+            return NimbusFmlLoader.get_fml_client_uncached(
+                str(file_path),
+                self.channel,
+            )
         else:
             logger.error("Nimbus FML Loader: Failed to get FmlClient.")
             return None
 
+    @staticmethod
+    def get_fml_client_uncached(file_path: Path, channel: str) -> FmlClient:
+        try:
+            return FmlClient(
+                str(file_path),
+                channel,
+            )
+        except FmlError:
+            logger.exception(
+                f"Nimbus FML Loader: FmlClient failed to parse manifest: {file_path}"
+            )
+            return None
+
     def get_fml_errors(
         self,
         blob: str,

experimenter/experimenter/features/tests/test_nimbus_fml_loader.py

@@ -1,8 +1,11 @@
 import json
+from pathlib import Path
 from unittest.mock import patch
 
+from django.conf import settings
 from django.test import TestCase
 from nimbus_megazord.fml import FmlClient, FmlError
+from parameterized import parameterized
 
 from experimenter.experiments.constants import NimbusConstants
 from experimenter.features.manifests.nimbus_fml_loader import NimbusFmlLoader
@@ -243,3 +246,55 @@ def test_get_fml_errors_returns_empty_when_manifest_fails_to_parse(
                 "Nimbus FML Loader: FmlClient failed to parse manifest",
                 log.output[0],
             )
+
+
+class FeatureManifestTests(TestCase):
+    @classmethod
+    def setUpClass(cls):
+        NimbusFmlLoader.create_loader.cache_clear()
+        NimbusFmlLoader.fml_client.cache_clear()
+
+    @classmethod
+    def tearDownClass(cls):
+        NimbusFmlLoader.create_loader.cache_clear()
+        NimbusFmlLoader.fml_client.cache_clear()
+
+    @staticmethod
+    def _discover_fml_files(application):
+        return [
+            (path, channel)
+            for (path, channel) in (
+                (path, path.name.split(".", maxsplit=1)[0])
+                for path in (
+                    root_path / file_name
+                    for (root_path, _, file_names) in Path.walk(
+                        settings.FEATURE_MANIFESTS_PATH / application.slug
+                    )
+                    for file_name in file_names
+                    if file_name.endswith(".fml.yaml")
+                )
+            )
+            if channel in application.channel_app_id
+        ]
+
+    @parameterized.expand(
+        [
+            application
+            for application in NimbusConstants.APPLICATION_CONFIGS.values()
+            if NimbusConstants.Application.is_sdk(application.slug)
+        ]
+    )
+    def test_fml_loads(self, application):
+        for file_path, channel in FeatureManifestTests._discover_fml_files(application):
+            client = NimbusFmlLoader.get_fml_client_uncached(
+                str(file_path),
+                channel,
+            )
+
+            self.assertIsNotNone(
+                client,
+                (
+                    f"`{file_path}' is a valid manifest for `{application}' on channel "
+                    f"`{channel}'"
+                ),
+            )