test: Add Testcontainers and Gherkin execution for our test-harness

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "providers/openfeature-provider-flagd/test-harness"]
 	path = providers/openfeature-provider-flagd/test-harness
 	url = [email protected]:open-feature/flagd-testbed.git
+[submodule "providers/openfeature-provider-flagd/spec"]
+	path = providers/openfeature-provider-flagd/spec
+	url = https://github.com/open-feature/spec

providers/openfeature-provider-flagd/pyproject.toml

@@ -37,6 +37,9 @@ dependencies = [
   "coverage[toml]>=6.5",
   "pytest",
   "pytest-bdd",
+  "testcontainers",
+  "asserts",
+  "grpcio-health-checking==1.60.0",
 ]
 post-install-commands = [
   "./scripts/gen_protos.sh"

providers/openfeature-provider-flagd/tests/e2e/conftest.py

@@ -1,208 +1,33 @@
 import typing
 
 import pytest
-from pytest_bdd import given, parsers, then, when
-from tests.e2e.parsers import to_bool
+from testcontainers.core.container import DockerContainer
+from tests.e2e.flagd_container import FlagDContainer
+from tests.e2e.steps import *  # noqa: F403
 
 from openfeature import api
-from openfeature.client import OpenFeatureClient
 from openfeature.contrib.provider.flagd import FlagdProvider
-from openfeature.contrib.provider.flagd.config import ResolverType
-from openfeature.evaluation_context import EvaluationContext
 
 JsonPrimitive = typing.Union[str, bool, float, int]
 
 
-@pytest.fixture
-def evaluation_context() -> EvaluationContext:
-    return EvaluationContext()
-
-
-@given("a flagd provider is set", target_fixture="client")
-def setup_provider(flag_file) -> OpenFeatureClient:
+@pytest.fixture(autouse=True, scope="package")
+def setup(request, port, image, resolver_type):
+    container: DockerContainer = FlagDContainer(
+        image=image,
+        port=port,
+    )
+    # Setup code
+    c = container.start()
     api.set_provider(
         FlagdProvider(
-            resolver_type=ResolverType.IN_PROCESS,
-            offline_flag_source_path=flag_file,
-            offline_poll_interval_seconds=0.1,
+            resolver_type=resolver_type,
+            port=int(container.get_exposed_port(port)),
         )
     )
-    return api.get_client()
-
-
-@when(
-    parsers.cfparse(
-        'a zero-value boolean flag with key "{key}" is evaluated with default value "{default:bool}"',
-        extra_types={"bool": to_bool},
-    ),
-    target_fixture="key_and_default",
-)
-@when(
-    parsers.cfparse(
-        'a zero-value string flag with key "{key}" is evaluated with default value "{default}"',
-    ),
-    target_fixture="key_and_default",
-)
-@when(
-    parsers.cfparse(
-        'a string flag with key "{key}" is evaluated with default value "{default}"'
-    ),
-    target_fixture="key_and_default",
-)
-@when(
-    parsers.cfparse(
-        'a zero-value integer flag with key "{key}" is evaluated with default value {default:d}',
-    ),
-    target_fixture="key_and_default",
-)
-@when(
-    parsers.cfparse(
-        'an integer flag with key "{key}" is evaluated with default value {default:d}',
-    ),
-    target_fixture="key_and_default",
-)
-@when(
-    parsers.cfparse(
-        'a zero-value float flag with key "{key}" is evaluated with default value {default:f}',
-    ),
-    target_fixture="key_and_default",
-)
-def setup_key_and_default(
-    key: str, default: JsonPrimitive
-) -> typing.Tuple[str, JsonPrimitive]:
-    return (key, default)
-
-
-@when(
-    parsers.cfparse(
-        'a context containing a targeting key with value "{targeting_key}"'
-    ),
-)
-def assign_targeting_context(evaluation_context: EvaluationContext, targeting_key: str):
-    """a context containing a targeting key with value <targeting key>."""
-    evaluation_context.targeting_key = targeting_key
-
-
-@when(
-    parsers.cfparse('a context containing a key "{key}", with value "{value}"'),
-)
-@when(
-    parsers.cfparse('a context containing a key "{key}", with value {value:d}'),
-)
-def update_context(
-    evaluation_context: EvaluationContext, key: str, value: JsonPrimitive
-):
-    """a context containing a key and value."""
-    evaluation_context.attributes[key] = value
-
-
-@when(
-    parsers.cfparse(
-        'a context containing a nested property with outer key "{outer}" and inner key "{inner}", with value "{value}"'
-    ),
-)
-@when(
-    parsers.cfparse(
-        'a context containing a nested property with outer key "{outer}" and inner key "{inner}", with value {value:d}'
-    ),
-)
-def update_context_nested(
-    evaluation_context: EvaluationContext,
-    outer: str,
-    inner: str,
-    value: typing.Union[str, int],
-):
-    """a context containing a nested property with outer key, and inner key, and value."""
-    if outer not in evaluation_context.attributes:
-        evaluation_context.attributes[outer] = {}
-    evaluation_context.attributes[outer][inner] = value
-
-
-@then(
-    parsers.cfparse(
-        'the resolved boolean zero-value should be "{expected_value:bool}"',
-        extra_types={"bool": to_bool},
-    )
-)
-def assert_boolean_value(
-    client: OpenFeatureClient,
-    key_and_default: tuple,
-    expected_value: bool,
-    evaluation_context: EvaluationContext,
-):
-    key, default = key_and_default
-    evaluation_result = client.get_boolean_value(key, default, evaluation_context)
-    assert evaluation_result == expected_value
-
-
-@then(
-    parsers.cfparse(
-        "the resolved integer zero-value should be {expected_value:d}",
-    )
-)
-@then(parsers.cfparse("the returned value should be {expected_value:d}"))
-def assert_integer_value(
-    client: OpenFeatureClient,
-    key_and_default: tuple,
-    expected_value: bool,
-    evaluation_context: EvaluationContext,
-):
-    key, default = key_and_default
-    evaluation_result = client.get_integer_value(key, default, evaluation_context)
-    assert evaluation_result == expected_value
-
-
-@then(
-    parsers.cfparse(
-        "the resolved float zero-value should be {expected_value:f}",
-    )
-)
-def assert_float_value(
-    client: OpenFeatureClient,
-    key_and_default: tuple,
-    expected_value: bool,
-    evaluation_context: EvaluationContext,
-):
-    key, default = key_and_default
-    evaluation_result = client.get_float_value(key, default, evaluation_context)
-    assert evaluation_result == expected_value
-
-
-@then(parsers.cfparse('the returned value should be "{expected_value}"'))
-def assert_string_value(
-    client: OpenFeatureClient,
-    key_and_default: tuple,
-    expected_value: bool,
-    evaluation_context: EvaluationContext,
-):
-    key, default = key_and_default
-    evaluation_result = client.get_string_value(key, default, evaluation_context)
-    assert evaluation_result == expected_value
-
-
-@then(
-    parsers.cfparse(
-        'the resolved string zero-value should be ""',
-    )
-)
-def assert_empty_string(
-    client: OpenFeatureClient,
-    key_and_default: tuple,
-    evaluation_context: EvaluationContext,
-):
-    key, default = key_and_default
-    evaluation_result = client.get_string_value(key, default, evaluation_context)
-    assert evaluation_result == ""
 
+    def fin():
+        c.stop()
 
-@then(parsers.cfparse('the returned reason should be "{reason}"'))
-def assert_reason(
-    client: OpenFeatureClient,
-    key_and_default: tuple,
-    evaluation_context: EvaluationContext,
-    reason: str,
-):
-    """the returned reason should be <reason>."""
-    key, default = key_and_default
-    evaluation_result = client.get_string_details(key, default, evaluation_context)
-    assert evaluation_result.reason.value == reason
+    # Teardown code
+    request.addfinalizer(fin)

providers/openfeature-provider-flagd/tests/e2e/flagd_container.py

@@ -0,0 +1,59 @@
+import time
+
+import grpc
+from grpc_health.v1 import health_pb2, health_pb2_grpc
+from testcontainers.core.container import DockerContainer
+from testcontainers.core.waiting_utils import wait_container_is_ready, wait_for_logs
+
+HEALTH_CHECK = 8014
+
+
+class FlagDContainer(DockerContainer):
+    def __init__(
+        self,
+        image: str = "ghcr.io/open-feature/flagd-testbed:v0.5.13",
+        port: int = 8013,
+        **kwargs,
+    ) -> None:
+        super().__init__(image, **kwargs)
+        self.port = port
+        self.with_exposed_ports(self.port, HEALTH_CHECK)
+
+    def start(self) -> "FlagDContainer":
+        super().start()
+        self._checker(self.get_container_host_ip(), self.get_exposed_port(HEALTH_CHECK))
+        return self
+
+    @wait_container_is_ready(ConnectionError)
+    def _checker(self, host: str, port: int) -> None:
+        # First we wait for Flagd to say it's listening
+        wait_for_logs(
+            self,
+            "listening",
+            5,
+        )
+
+        time.sleep(1)
+        # Second we use the GRPC health check endpoint
+        with grpc.insecure_channel(host + ":" + port) as channel:
+            health_stub = health_pb2_grpc.HealthStub(channel)
+
+            def health_check_call(stub: health_pb2_grpc.HealthStub):
+                request = health_pb2.HealthCheckRequest()
+                resp = stub.Check(request)
+                if resp.status == health_pb2.HealthCheckResponse.SERVING:
+                    return True
+                elif resp.status == health_pb2.HealthCheckResponse.NOT_SERVING:
+                    return False
+
+            # Should succeed
+            # Check health status every 1 second for 30 seconds
+            ok = False
+            for _ in range(30):
+                ok = health_check_call(health_stub)
+                if ok:
+                    break
+                time.sleep(1)
+
+            if not ok:
+                raise ConnectionError("flagD not ready in time")

providers/openfeature-provider-flagd/tests/e2e/parsers.py

@@ -1,2 +1,7 @@
 def to_bool(s: str) -> bool:
     return s.lower() == "true"
+
+
+def to_list(s: str) -> list:
+    values = s.replace('"', "").split(",")
+    return [s.strip() for s in values]