Feature Flags: dedupe & basic superadmin interface (#3170)

Re-implements #3152 a little more simply with fewer features. See
original #3152 for more context and discussion.

Author: emma-sg

backend/btrixcloud/models.py

@@ -3,7 +3,7 @@
 """
 
 # pylint: disable=invalid-name, too-many-lines
-
+from __future__ import annotations
 from datetime import datetime
 from enum import Enum, IntEnum
 from uuid import UUID
@@ -13,7 +13,7 @@
 import math
 import os
 
-from typing import Optional, List, Dict, Union, Literal, Any, get_args
+from typing import Optional, List, Dict, Self, Union, Literal, Any, get_args, get_origin
 from typing_extensions import Annotated
 
 from pydantic import (
@@ -22,6 +22,8 @@
     HttpUrl as HttpUrlNonStr,
     AnyHttpUrl as AnyHttpUrlNonStr,
     EmailStr as CasedEmailStr,
+    create_model,
+    model_validator,
     validate_email,
     RootModel,
     BeforeValidator,
@@ -2227,6 +2229,110 @@ class UserOut(UserOutNoId):
     is_superuser: bool = False
 
 
+# ============================================================================
+# Feature Flags
+# ============================================================================
+
+
+class ValidatedFeatureFlags(BaseModel):
+    """Base class for feature flags with validation."""
+
+    @model_validator(mode="after")
+    def validate_all_fields(self) -> Self:
+        """Ensure all fields have descriptions and are bools."""
+        missing_descriptions = []
+        non_bool_fields = []
+
+        for field_name, field_info in self.model_fields.items():
+            # Check for missing descriptions
+            if not field_info.description:
+                missing_descriptions.append(field_name)
+
+            # Check if field type is bool (handles Annotated[bool, ...])
+            annotation = field_info.annotation
+            if get_origin(annotation) is Annotated:
+                actual_type = get_args(annotation)[0]
+            else:
+                actual_type = annotation
+
+            if actual_type is not bool:
+                non_bool_fields.append(f"{field_name} (type: {actual_type})")
+
+        if missing_descriptions:
+            raise ValueError(
+                f"The following fields are missing descriptions: {', '.join(missing_descriptions)}"
+            )
+
+        if non_bool_fields:
+            raise ValueError(
+                f"The following fields must be bool type: {', '.join(non_bool_fields)}"
+            )
+
+        return self
+
+
+def make_feature_flags_partial(model_cls: type[BaseModel]) -> type[BaseModel]:
+    """Return a partial model for feature flags without validation inheritance.
+
+    This creates a model where all fields are optional (bool | None) but doesn't
+    inherit from the original model to avoid the ValidatedFeatureFlags validator
+    that checks for exact bool types.
+    """
+    new_fields = {}
+
+    for f_name, f_info in model_cls.model_fields.items():
+        f_dct = f_info.asdict()  # type: ignore
+
+        # Create a new field that's bool | None with the same description
+        new_fields[f_name] = (
+            Annotated[
+                (
+                    bool | None,
+                    Field(description=f_dct.get("description"), default=None),  # type: ignore
+                )
+            ],
+            None,
+        )
+
+    return create_model(  # type: ignore
+        f"{model_cls.__name__}Partial",
+        **new_fields,
+    )
+
+
+# ============================================================================
+# Feature Flags - Edit here
+# ============================================================================
+
+
+class FeatureFlags(ValidatedFeatureFlags):
+    """Feature flags for an organization"""
+
+    dedupeEnabled: bool = Field(
+        description="Enable deduplication options for an org. Intended for beta-testing dedupe.",
+        default=False,
+    )
+
+
+# ============================================================================
+
+
+FeatureFlagsPartial = make_feature_flags_partial(FeatureFlags)
+
+
+class FeatureFlagStats(BaseModel):
+    """Output model for feature flags"""
+
+    model_config = ConfigDict(use_attribute_docstrings=True)
+
+    name: str
+
+    description: str
+
+    count: int
+    """Number of organizations that have this feature flag enabled."""
+
+
 # ============================================================================
 # ORGS
 # ============================================================================
@@ -2354,6 +2460,8 @@ class OrgOut(BaseMongoModel):
     publicDescription: str = ""
     publicUrl: str = ""
 
+    featureFlags: FeatureFlags = FeatureFlags()
+
 
 # ============================================================================
 class Organization(BaseMongoModel):
@@ -2417,6 +2525,8 @@ class Organization(BaseMongoModel):
     publicDescription: Optional[str] = None
     publicUrl: Optional[str] = None
 
+    featureFlags: FeatureFlags = FeatureFlags()
+
     def is_owner(self, user):
         """Check if user is owner"""
         return self._is_auth(user, UserRole.OWNER)

backend/btrixcloud/orgs.py

@@ -22,6 +22,7 @@
     Literal,
     AsyncGenerator,
     Any,
+    cast,
 )
 
 from motor.motor_asyncio import (
@@ -43,6 +44,9 @@
     RUNNING_STATES,
     WAITING_STATES,
     BaseCrawl,
+    FeatureFlagStats,
+    FeatureFlags,
+    FeatureFlagsPartial,
     Organization,
     PlansResponse,
     StorageRef,
@@ -770,6 +774,45 @@ async def update_quotas(
                 print(f"Error updating organization quotas: {e}")
                 raise HTTPException(status_code=500, detail=str(e)) from e
 
+    async def update_feature_flags(
+        self,
+        org: Organization,
+        feature_flags: FeatureFlagsPartial,  # type: ignore
+        session: AsyncIOMotorClientSession | None = None,
+    ):
+        """Update organization feature flag"""
+        update = {"$set": {}}
+        for feature, enabled in feature_flags.model_dump(exclude_none=True).items():  # type: ignore
+            update["$set"][f"featureFlags.{feature}"] = enabled
+        await self.orgs.find_one_and_update({"_id": org.id}, update, session=session)
+
+    async def get_feature_flags(
+        self,
+        session: AsyncIOMotorClientSession | None = None,
+    ):
+        """Get feature flags, with their metadata and org counts."""
+        # get number of organizations that have each feature flag enabled
+        counts = await self.orgs.aggregate(
+            [
+                {"$match": {"featureFlags": {"$exists": True, "$ne": {}}}},
+                {"$project": {"flags": {"$objectToArray": "$featureFlags"}}},
+                {"$unwind": "$flags"},
+                {"$match": {"flags.v": True}},
+                {"$group": {"_id": "$flags.k", "count": {"$sum": 1}}},
+                {"$sort": {"_id": 1}},
+            ],
+            session=session,
+        ).to_list(None)
+        org_counts = {count["_id"]: count["count"] for count in counts}
+        return [
+            FeatureFlagStats(
+                name=name,
+                count=org_counts.get(name, 0),
+                description=cast(str, flag.description),
+            )
+            for name, flag in FeatureFlags.model_fields.items()
+        ]
+
     async def update_event_webhook_urls(
         self, org: Organization, urls: OrgWebhookUrls
     ) -> bool:
@@ -1759,6 +1802,34 @@ async def update_proxies(
 
         return {"updated": True}
 
+    @router.post(
+        "/feature-flags", tags=["organizations"], response_model=UpdatedResponse
+    )
+    async def update_feature_flags(
+        feature_flags: FeatureFlagsPartial,  # type: ignore
+        org: Organization = Depends(org_owner_dep),
+        user: User = Depends(user_dep),
+    ):
+        if not user.is_superuser:
+            raise HTTPException(status_code=403, detail="Not Allowed")
+
+        await ops.update_feature_flags(org, feature_flags)
+
+        return {"updated": True}
+
+    @app.get(
+        "/orgs/feature-flags",
+        tags=["organizations"],
+        response_model=list[FeatureFlagStats],
+    )
+    async def get_feature_flags(
+        user: User = Depends(user_dep),
+    ):
+        if not user.is_superuser:
+            raise HTTPException(status_code=403, detail="Not Allowed")
+
+        return await ops.get_feature_flags()
+
     @router.post("/read-only", tags=["organizations"], response_model=UpdatedResponse)
     async def update_read_only(
         update: OrgReadOnlyUpdate,

backend/test/test_org.py

@@ -56,6 +56,30 @@ def test_get_org_crawler(crawler_auth_headers, default_org_id):
     assert data.get("users") == {}
 
 
+def test_get_org_feature_flags(crawler_auth_headers, default_org_id):
+    """feature flags should be available for all users"""
+    r = requests.get(
+        f"{API_PREFIX}/orgs/{default_org_id}", headers=crawler_auth_headers
+    )
+    assert r.status_code == 200
+    data = r.json()
+    assert data["id"] == default_org_id
+    feature_flags = data["featureFlags"]
+    assert feature_flags == {
+        "dedupeEnabled": False,
+    }
+
+    # List endpoint
+    r = requests.get(f"{API_PREFIX}/orgs", headers=crawler_auth_headers)
+    assert r.status_code == 200
+    data = r.json()
+    for org in data["items"]:
+        feature_flags = org["featureFlags"]
+        assert feature_flags == {
+            "dedupeEnabled": False,
+        }
+
+
 def test_update_org_crawling_defaults(admin_auth_headers, default_org_id):
     r = requests.post(
         f"{API_PREFIX}/orgs/{default_org_id}/defaults/crawling",

frontend/docs/docs/develop/feature-flags.md

@@ -0,0 +1,67 @@
+# Feature Flags
+
+## Introduction
+
+Feature flags are a powerful tool for managing and controlling the release of new features in a software application. They allow developers to enable or disable specific features at runtime without deploying new releases, making it easier to test and deploy new features in a controlled manner.
+
+## Implementation
+
+In Browsertrix, feature flags are implemented as an object stored on the `Organization` model. This object contains a set of key-value pairs, where the key is the name of the feature flag and the value is a boolean indicating whether the feature is enabled or disabled.
+
+### Consuming Feature Flags
+
+On the back-end, feature flags are available through the `featureFlags` property on an `Organization` instance:
+
+```python
+def do_something(org: Organization):
+    if org.featureFlags.newFeature:
+        # do something
+```
+
+On the front-end, you can access feature flags using the `featureFlags.has` and `featureFlags.excludes` method on any element inheriting from `BtrixElement`:
+
+```typescript
+class MyElement extends BtrixElement {
+  render() {
+    if (this.featureFlags.has("newFeature")) {
+      // render new feature
+    }
+    // or
+    if (this.featureFlags.excludes("newFeature")) {
+      // render old feature
+    }
+  }
+}
+```
+
+### Adding a New Feature Flag
+
+There are a few steps to adding a new feature flag:
+
+1. In `backend/btrixcloud/models.py`, add a new field to the `FeatureFlags` model:
+
+```python
+class FeatureFlags(ValidatedFeatureFlags):
+    # ...
+    newFeature: bool = Field(
+        description="Detailed description of the feature flag. It should explain what the flag does and why it is needed.",
+        default=False,
+    )
+```
+
+3. In `frontend/src/types/featureFlags.ts`, add a new value to the `FeatureFlags` type:
+
+```typescript
+export const featureFlagSchema = z.enum([
+  // ...
+  "newFeature",
+]);
+```
+
+## Best Practices
+
+- **Keep feature flags simple**: Feature flags should be simple and easy to understand.
+- **Document feature flags**: Feature flags must be documented using the `description` property in the `Field` definition.
+- **Intend to remove feature flags**: Feature flags should be intended to be removed after a certain period of time. They shouldn't be used for things like gating features based on subscription levels or user roles.
+
+Feature flags as they're currently implemented in Browsertrix are intended to be used for experimental/beta features that are not yet ready for production. They should be used sparingly and only when necessary; they're not designed to be used as circuit breakers or ops/permission toggles at this stage.

frontend/docs/mkdocs.yml

@@ -99,6 +99,7 @@ nav:
       - develop/local-dev-setup.md
       - develop/docs.md
       - develop/emails.md
+      - develop/feature-flags.md
       - UI Development:
         - develop/frontend-dev.md
         - develop/ui/components.md

frontend/src/__mocks__/api/orgs/[id].js

@@ -153,4 +153,5 @@ export default {
   enablePublicProfile: false,
   publicDescription: "This is an example org.",
   publicUrl: "https://example.com",
+  featureFlags: {},
 };

frontend/src/classes/BtrixElement.ts

@@ -39,4 +39,6 @@ export class BtrixElement extends TailwindElement {
   protected get org() {
     return this.appState.org;
   }
+
+  protected readonly featureFlags = this.appState.featureFlags;
 }