Merge pull request #25 from PostHog/tom/stale-grants

Publish Cloudwatch metric when stale grants detected

Author: Piccirello

README.md

@@ -225,6 +225,8 @@ output "api_endpoint_url" {
 | <a name="output_config_s3_bucket_arn"></a> [config\_s3\_bucket\_arn](#output\_config\_s3\_bucket\_arn) | The ARN of the S3 bucket for storing configuration and cache data. |
 | <a name="output_config_s3_bucket_name"></a> [config\_s3\_bucket\_name](#output\_config\_s3\_bucket\_name) | The name of the S3 bucket for storing configuration and cache data. |
 | <a name="output_requester_api_endpoint_url"></a> [requester\_api\_endpoint\_url](#output\_requester\_api\_endpoint\_url) | The full URL to invoke the API. Pass this URL into the Slack App manifest as the Request URL. |
+| <a name="output_revoker_lambda_name"></a> [revoker\_lambda\_name](#output\_revoker\_lambda\_name) | The name of the revoker Lambda function. |
+| <a name="output_schedule_group_name"></a> [schedule\_group\_name](#output\_schedule\_group\_name) | The name of the EventBridge Scheduler schedule group. |
 | <a name="output_sso_elevator_bucket_id"></a> [sso\_elevator\_bucket\_id](#output\_sso\_elevator\_bucket\_id) | The name of the SSO elevator bucket. |
 <!-- END_TF_DOCS -->
 

outputs.tf

@@ -34,3 +34,14 @@ output "attribute_sync_schedule_rule_arn" {
   description = "The ARN of the EventBridge rule that triggers the attribute syncer."
   value       = var.attribute_sync_enabled ? aws_cloudwatch_event_rule.attribute_sync_schedule[0].arn : null
 }
+
+# Alerting-related outputs
+output "revoker_lambda_name" {
+  description = "The name of the revoker Lambda function."
+  value       = module.access_revoker.lambda_function_name
+}
+
+output "schedule_group_name" {
+  description = "The name of the EventBridge Scheduler schedule group."
+  value       = aws_scheduler_schedule_group.one_time_schedule_group.name
+}

perm_revoker_lambda.tf

@@ -190,6 +190,19 @@ data "aws_iam_policy_document" "revoker" {
       "${module.config_bucket.s3_bucket_arn}/*"
     ]
   }
+  statement {
+    sid    = "AllowCloudWatchPutMetricData"
+    effect = "Allow"
+    actions = [
+      "cloudwatch:PutMetricData"
+    ]
+    resources = ["*"]
+    condition {
+      test     = "StringEquals"
+      variable = "cloudwatch:namespace"
+      values   = ["SSOElevator"]
+    }
+  }
 
 }
 

src/revoker.py

@@ -1,7 +1,8 @@
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 
 import boto3
 import slack_sdk
+from mypy_boto3_cloudwatch import CloudWatchClient
 from mypy_boto3_events import EventBridgeClient
 from mypy_boto3_identitystore import IdentityStoreClient
 from mypy_boto3_organizations import OrganizationsClient
@@ -38,9 +39,30 @@
 identitystore_client = boto3.client("identitystore")  # type: ignore # noqa: PGH003
 scheduler_client = boto3.client("scheduler")  # type: ignore # noqa: PGH003
 events_client = boto3.client("events")  # type: ignore # noqa: PGH003
+cloudwatch_client: CloudWatchClient = boto3.client("cloudwatch")  # type: ignore # noqa: PGH003
 slack_client = slack_sdk.WebClient(token=cfg.slack_bot_token)
 
 
+def publish_stale_grants_metric(count: int, cloudwatch: CloudWatchClient) -> None:
+    """Publish a CloudWatch metric for stale grants detected.
+
+    This metric can be used to trigger alarms when grants exist that should have been revoked.
+    """
+    if count > 0:
+        logger.info("Publishing stale grants metric", extra={"count": count})
+    cloudwatch.put_metric_data(
+        Namespace="SSOElevator",
+        MetricData=[
+            {
+                "MetricName": "StaleGrantsDetected",
+                "Value": count,
+                "Unit": "Count",
+                "Timestamp": datetime.now(timezone.utc),
+            }
+        ],
+    )
+
+
 def lambda_handler(event: dict, __) -> SlackResponse | None:  # type: ignore # noqa: ANN001, PGH003
     try:
         parsed_event = Event.model_validate(event).root
@@ -80,15 +102,15 @@ def lambda_handler(event: dict, __) -> SlackResponse | None:  # type: ignore # n
 
         case CheckOnInconsistency():
             logger.info("Handling CheckOnInconsistency event", extra={"event": parsed_event})
-            check_on_groups_inconsistency(
+            stale_group_count = check_on_groups_inconsistency(
                 identity_store_client=identitystore_client,
                 sso_client=sso_client,
                 scheduler_client=scheduler_client,
                 events_client=events_client,
                 cfg=cfg,
                 slack_client=slack_client,
             )
-            return handle_check_on_inconsistency(
+            stale_account_count = handle_check_on_inconsistency(
                 sso_client=sso_client,
                 cfg=cfg,
                 scheduler_client=scheduler_client,
@@ -97,6 +119,8 @@ def lambda_handler(event: dict, __) -> SlackResponse | None:  # type: ignore # n
                 identitystore_client=identitystore_client,
                 events_client=events_client,
             )
+            publish_stale_grants_metric(stale_group_count + stale_account_count, cloudwatch_client)
+            return
 
         case SSOElevatorScheduledRevocation():
             logger.info("Handling SSOElevatorScheduledRevocation event", extra={"event": parsed_event})
@@ -687,7 +711,8 @@ def handle_check_on_inconsistency(  # noqa: PLR0913
     slack_client: slack_sdk.WebClient,
     identitystore_client: IdentityStoreClient,
     events_client: EventBridgeClient,
-) -> None:
+) -> int:
+    """Check for inconsistent account assignments and return the count of stale grants found."""
     account_assignments = sso.get_account_assignment_information(sso_client, cfg, org_client)
     scheduled_revoke_events = schedule.get_scheduled_events(scheduler_client)
     account_assignments_from_events = [
@@ -701,8 +726,10 @@ def handle_check_on_inconsistency(  # noqa: PLR0913
         if isinstance(scheduled_event, ScheduledRevokeEvent)
     ]
 
+    stale_count = 0
     for account_assignment in account_assignments:
         if account_assignment not in account_assignments_from_events:
+            stale_count += 1
             try:
                 account = organizations.describe_account(org_client, account_assignment.account_id)
             except Exception:
@@ -739,6 +766,7 @@ def handle_check_on_inconsistency(  # noqa: PLR0913
                     f"The unidentified assignment will be automatically revoked.{time_notice}"
                 ),
             )
+    return stale_count
 
 
 def check_on_groups_inconsistency(  # noqa: PLR0913
@@ -748,7 +776,8 @@ def check_on_groups_inconsistency(  # noqa: PLR0913
     events_client: EventBridgeClient,
     cfg: config.Config,
     slack_client: slack_sdk.WebClient,
-) -> None:
+) -> int:
+    """Check for inconsistent group assignments and return the count of stale grants found."""
     identity_store_id = sso.get_identity_store_id(cfg, sso_client)
     scheduled_revoke_events = schedule.get_scheduled_events(scheduler_client)
     group_assignments = sso.get_group_assignments(identity_store_id, identity_store_client, cfg)
@@ -763,8 +792,10 @@ def check_on_groups_inconsistency(  # noqa: PLR0913
         for scheduled_event in scheduled_revoke_events
         if isinstance(scheduled_event, ScheduledGroupRevokeEvent)
     ]
+    stale_count = 0
     for group_assignment in group_assignments:
         if group_assignment not in group_assignments_from_events:
+            stale_count += 1
             logger.warning("Group assignment is not in the scheduled events", extra={"assignment": group_assignment})
             mention = slack_helpers.create_slack_mention_by_principal_id(
                 sso_user_id=group_assignment.user_principal_id,
@@ -791,6 +822,7 @@ def check_on_groups_inconsistency(  # noqa: PLR0913
                     f"The unidentified assignment will be automatically revoked.{time_notice}"
                 ),
             )
+    return stale_count
 
 
 def handle_sso_elevator_group_scheduled_revocation(  # noqa: PLR0913