[feature] Add usage limits to agenta-provided keys

README.md

@@ -2,12 +2,11 @@
 <p align="center">
   <a href="https://agenta.ai?utm_source=github&utm_medium=referral&utm_campaign=readme">
       <picture >
-        <source width="275" media="(prefers-color-scheme: dark)" srcset="https://github.com/user-attachments/assets/97e31bfc-b1fc-4d19-b443-5aedf6029017"  >
-        <source width="275" media="(prefers-color-scheme: light)" srcset="https://github.com/user-attachments/assets/fdc5f23f-2095-4cfc-9511-14c6851c1262"  >
+        <source width="275" media="(prefers-color-scheme: dark)" srcset="https://github.com/Agenta-AI/agenta/assets/4510758/cdddf5ad-2352-4920-b1d9-ae7f8d9d7735"  >
+        <source width="275" media="(prefers-color-scheme: light)" srcset="https://github.com/Agenta-AI/agenta/assets/4510758/ab75cbac-b807-496f-aab3-57463a33f726"  >
         <img alt="Shows the logo of agenta" src="https://github.com/Agenta-AI/agenta/assets/4510758/68e055d4-d7b8-4943-992f-761558c64253" >
       </picture>
   </a>
-
 <div align="center">
   <strong> <h1> The Open-source LLMOps Platform </h1></strong>
   Build reliable LLM applications faster with integrated prompt management, evaluation, and observability.

api/ee/databases/postgres/migrations/core/versions/79f40f71e912_extend_meters.py

@@ -0,0 +1,70 @@
+"""add CREDITS to meters_type
+
+Revision ID: 79f40f71e912
+Revises: 3b5f5652f611
+Create Date: 2025-11-03 15:00:00.000000
+"""
+
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision: str = "79f40f71e912"
+down_revision: Union[str, None] = "3b5f5652f611"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+ENUM_NAME = "meters_type"
+TEMP_ENUM_NAME = "meters_type_temp"
+TABLE_NAME = "meters"
+COLUMN_NAME = "key"
+
+
+def upgrade() -> None:
+    # 1) Create temp enum including the new value
+    op.execute(
+        sa.text(
+            f"CREATE TYPE {TEMP_ENUM_NAME} AS ENUM ('USERS','APPLICATIONS','EVALUATIONS','TRACES','CREDITS')"
+        )
+    )
+
+    # 2) Alter column to use temp enum
+    op.execute(
+        sa.text(
+            f"ALTER TABLE {TABLE_NAME} "
+            f"ALTER COLUMN {COLUMN_NAME} TYPE {TEMP_ENUM_NAME} "
+            f"USING {COLUMN_NAME}::text::{TEMP_ENUM_NAME}"
+        )
+    )
+
+    # 3) Drop old enum, then 4) rename temp -> original
+    op.execute(sa.text(f"DROP TYPE {ENUM_NAME}"))
+    op.execute(sa.text(f"ALTER TYPE {TEMP_ENUM_NAME} RENAME TO {ENUM_NAME}"))
+
+
+def downgrade() -> None:
+    # Ensure downgrade can proceed (rows with CREDITS would block the type change)
+    op.execute(
+        sa.text(f"DELETE FROM {TABLE_NAME} WHERE {COLUMN_NAME}::text = 'CREDITS'")
+    )
+
+    # 1) Create temp enum WITHOUT CREDITS
+    op.execute(
+        sa.text(
+            f"CREATE TYPE {TEMP_ENUM_NAME} AS ENUM ('USERS','APPLICATIONS','EVALUATIONS','TRACES')"
+        )
+    )
+
+    # 2) Alter column to use temp enum
+    op.execute(
+        sa.text(
+            f"ALTER TABLE {TABLE_NAME} "
+            f"ALTER COLUMN {COLUMN_NAME} TYPE {TEMP_ENUM_NAME} "
+            f"USING {COLUMN_NAME}::text::{TEMP_ENUM_NAME}"
+        )
+    )
+
+    # 3) Drop current enum (which includes CREDITS), then 4) rename temp -> original
+    op.execute(sa.text(f"DROP TYPE {ENUM_NAME}"))
+    op.execute(sa.text(f"ALTER TYPE {TEMP_ENUM_NAME} RENAME TO {ENUM_NAME}"))

api/ee/src/core/entitlements/types.py

@@ -22,6 +22,7 @@ class Counter(str, Enum):
     EVALUATIONS = "evaluations"
     EVALUATORS = "evaluators"
     ANNOTATIONS = "annotations"
+    CREDITS = "credits"
 
 
 class Gauge(str, Enum):
@@ -60,7 +61,7 @@ class Probe(BaseModel):
             },
         },
         "features": [
-            "Unlimited prompts",
+            "2 prompts",
             "20 evaluations/month",
             "5k traces/month",
             "2 seats",
@@ -209,10 +210,11 @@ class Probe(BaseModel):
         Tracker.COUNTERS: {
             Counter.TRACES: Quota(limit=5_000, monthly=True, free=5_000),
             Counter.EVALUATIONS: Quota(limit=20, monthly=True, free=20, strict=True),
+            Counter.CREDITS: Quota(limit=100, monthly=True, free=100, strict=True),
         },
         Tracker.GAUGES: {
             Gauge.USERS: Quota(limit=2, strict=True, free=2),
-            Gauge.APPLICATIONS: Quota(strict=True),
+            Gauge.APPLICATIONS: Quota(limit=2, strict=True, free=2),
         },
     },
     Plan.CLOUD_V0_PRO: {
@@ -223,6 +225,7 @@ class Probe(BaseModel):
         Tracker.COUNTERS: {
             Counter.TRACES: Quota(monthly=True, free=10_000),
             Counter.EVALUATIONS: Quota(monthly=True, strict=True),
+            Counter.CREDITS: Quota(limit=100, monthly=True, free=100, strict=True),
         },
         Tracker.GAUGES: {
             Gauge.USERS: Quota(limit=10, strict=True, free=3),
@@ -237,6 +240,7 @@ class Probe(BaseModel):
         Tracker.COUNTERS: {
             Counter.TRACES: Quota(monthly=True, free=1_000_000),
             Counter.EVALUATIONS: Quota(monthly=True, strict=True),
+            Counter.CREDITS: Quota(limit=100, monthly=True, free=100, strict=True),
         },
         Tracker.GAUGES: {
             Gauge.USERS: Quota(strict=True),
@@ -279,6 +283,12 @@ class Probe(BaseModel):
         Tracker.COUNTERS: {
             Counter.TRACES: Quota(monthly=True),
             Counter.EVALUATIONS: Quota(monthly=True, strict=True),
+            Counter.CREDITS: Quota(
+                limit=100_000,
+                monthly=True,
+                free=100_000,
+                strict=True,
+            ),
         },
         Tracker.GAUGES: {
             Gauge.USERS: Quota(strict=True),

api/ee/src/core/meters/types.py

@@ -13,6 +13,7 @@ class Meters(str, Enum):
     # COUNTERS
     TRACES = Counter.TRACES.value
     EVALUATIONS = Counter.EVALUATIONS.value
+    CREDITS = Counter.CREDITS.value
     # GAUGES
     USERS = Gauge.USERS.value
     APPLICATIONS = Gauge.APPLICATIONS.value

api/ee/src/services/llm_apps_service.py

@@ -202,7 +202,6 @@ async def invoke_app(
     openapi_parameters: List[Dict],
     user_id: str,
     project_id: str,
-    scenario_id: Optional[str] = None,
     **kwargs,
 ) -> InvokationResult:
     """
@@ -248,14 +247,7 @@ async def invoke_app(
         app_response = {}
 
         try:
-            log.info(
-                "Invoking application...",
-                scenario_id=scenario_id,
-                testcase_id=(
-                    datapoint["testcase_id"] if "testcase_id" in datapoint else None
-                ),
-                url=url,
-            )
+            log.info("Invoking workflow...", url=url)
             response = await client.post(
                 url,
                 json=payload,
@@ -276,12 +268,6 @@ async def invoke_app(
             trace_id = app_response.get("trace_id", None)
             span_id = app_response.get("span_id", None)
 
-            log.info(
-                "Invoked application.   ",
-                scenario_id=scenario_id,
-                trace_id=trace_id,
-            )
-
             return InvokationResult(
                 result=Result(
                     type=kind,
@@ -342,7 +328,6 @@ async def run_with_retry(
     openapi_parameters: List[Dict],
     user_id: str,
     project_id: str,
-    scenario_id: Optional[str] = None,
     **kwargs,
 ) -> InvokationResult:
     """
@@ -379,7 +364,6 @@ async def run_with_retry(
                 openapi_parameters,
                 user_id,
                 project_id,
-                scenario_id,
                 **kwargs,
             )
             return result
@@ -419,7 +403,6 @@ async def batch_invoke(
     rate_limit_config: Dict,
     user_id: str,
     project_id: str,
-    scenarios: Optional[List[Dict]] = None,
     **kwargs,
 ) -> List[InvokationResult]:
     """
@@ -514,7 +497,6 @@ async def batch_invoke(
                     openapi_parameters,
                     user_id,
                     project_id,
-                    scenarios[index].get("id") if scenarios else None,
                     **kwargs,
                 )
             )

api/ee/src/tasks/evaluations/legacy.py

@@ -1055,13 +1055,6 @@ def annotate(
                     "application_variant": {"id": str(variant.id)},
                     "application_revision": {"id": str(revision.id)},
                 },
-                scenarios=[
-                    s.model_dump(
-                        mode="json",
-                        exclude_none=True,
-                    )
-                    for s in scenarios
-                ],
             )
         )
         # ----------------------------------------------------------------------
@@ -1111,7 +1104,6 @@ def annotate(
             scenario = scenarios[idx]
             testcase = testcases[idx]
             invocation = invocations[idx]
-            invocation_step_key = invocation_steps_keys[0]
 
             scenario_has_errors = 0
             scenario_status = EvaluationStatus.SUCCESS
@@ -1148,20 +1140,8 @@ def annotate(
                         )
                     )
 
-                if trace:
-                    log.info(
-                        f"Trace found  ",
-                        scenario_id=scenario.id,
-                        step_key=invocation_step_key,
-                        trace_id=invocation.trace_id,
-                    )
-                else:
-                    log.warn(
-                        f"Trace missing",
-                        scenario_id=scenario.id,
-                        step_key=invocation_step_key,
-                        trace_id=invocation.trace_id,
-                    )
+                if not trace:
+                    log.warn(f"Trace with id {invocation.trace_id} not found.")
                     scenario_has_errors += 1
                     scenario_status = EvaluationStatus.ERRORS
                     continue
@@ -1310,13 +1290,6 @@ def annotate(
                         links=links,
                     )
 
-                    log.info(
-                        "Invoking evaluator...  ",
-                        scenario_id=scenario.id,
-                        testcase_id=testcase.id,
-                        trace_id=invocation.trace_id,
-                        uri=interface.get("uri"),
-                    )
                     workflows_service_response = loop.run_until_complete(
                         workflows_service.invoke_workflow(
                             project_id=project_id,
@@ -1327,11 +1300,6 @@ def annotate(
                             annotate=True,
                         )
                     )
-                    log.info(
-                        "Invoked evaluator      ",
-                        scenario_id=scenario.id,
-                        trace_id=workflows_service_response.trace_id,
-                    )
                     # ----------------------------------------------------------
 
                     # run evaluator --------------------------------------------
@@ -1387,20 +1355,8 @@ def annotate(
                                 )
                             )
 
-                        if trace:
-                            log.info(
-                                f"Trace found  ",
-                                scenario_id=scenario.id,
-                                step_key=annotation_step_key,
-                                trace_id=annotation.trace_id,
-                            )
-                        else:
-                            log.warn(
-                                f"Trace missing",
-                                scenario_id=scenario.id,
-                                step_key=annotation_step_key,
-                                trace_id=annotation.trace_id,
-                            )
+                        if not trace:
+                            log.warn(f"Trace with id {annotation.trace_id} not found.")
                             scenario_has_errors += 1
                             scenario_status = EvaluationStatus.ERRORS
                             continue