Cloud custodian can't count empty or unused S3 buckets

[amckeown-blc]

A Cloud custodian policy can't count empty or unused S3 buckets it seems based on cloudwatch metrics. We are running an unused after 90 day policy for all our resources, it seems S3 it the one with trouble.
If we run a similar Python script on Lambda rather than a Cloud Custodian policy it does pick out buckets which are empty or not used. From the example below it brings just [] back

{
"execution-options": {},
"policies": [
{
"name": "s3-unused-90days",
"resource": "s3",
"filters": [
{
"type": "value",
"key": "Location.LocationConstraint",
"op": "in",
"value": [
"eu-west-2",
"ap-southeast-2",
null
]
},
{
"type": "metrics",
"name": "BucketSizeBytes",
"namespace": "AWS/S3",
"statistics": "Average",
"period": 86400,
"days": 90,
"value": 0,
"op": "eq",
"dimensions": {
"StorageType": "StandardStorage"
}
},
{
"type": "metrics",
"name": "NumberOfObjects",
"namespace": "AWS/S3",
"statistics": "Average",
"period": 86400,
"days": 90,
"value": 0,
"op": "eq",
"dimensions": {
"StorageType": "AllStorageTypes"
}
}
],
"mode": {
"type": "periodic",
"schedule": "rate(1 day)",
"role": "arn:aws:iam::*:role/CloudCustodianRole",
"timeout": 900,
"execution-options": {
"output_dir": "s3://custodian-output-bucket-blc/",
"output_format": "jsonlines"
},
"tags": {
"custodian-info": "mode=periodic:version=0.9.48"
}
}
}
]
}

But the python works fine:

import boto3
import json
import gzip
import io
from datetime import datetime, timedelta

Configuration

THRESHOLD_DAYS = 90
TARGET_REGIONS = ["eu-west-2", "ap-southeast-2"]
OUTPUT_BUCKET = "custodian-output-bucket-blc"
OUTPUT_PREFIX = "s3-unused-python"

s3 = boto3.client("s3")
cloudwatch = boto3.client("cloudwatch")

def lambda_handler(event, context):
now = datetime.utcnow()
start = now - timedelta(days=THRESHOLD_DAYS)

report = []
used_buckets = []
unused_buckets = []
skipped_buckets = []

try:
    buckets = s3.list_buckets()["Buckets"]
except Exception as e:
    return {"status": "error", "message": f"Cannot list buckets: {str(e)}"}

for bucket_info in buckets:
    bucket_name = bucket_info["Name"]
    creation_date = bucket_info["CreationDate"].isoformat()
    bucket_entry = {
        "name": bucket_name,
        "creation_date": creation_date,
        "tags": [],
        "policy_principals": [],
        "status": "skipped",
        "reason": ""
    }

    # Get bucket region
    try:
        region_resp = s3.get_bucket_location(Bucket=bucket_name)
        region = region_resp.get("LocationConstraint") or "us-east-1"
        if region == "None":
            region = "us-east-1"
        bucket_entry["region"] = region
    except Exception as e:
        bucket_entry["reason"] = f"No access to get-bucket-location: {str(e)}"
        skipped_buckets.append(bucket_name)
        report.append(bucket_entry)
        continue

    if region not in TARGET_REGIONS:
        bucket_entry["reason"] = f"Region {region} not in target regions"
        skipped_buckets.append(bucket_name)
        report.append(bucket_entry)
        continue

    # Check CloudWatch metrics for BucketSizeBytes
    cw = boto3.client("cloudwatch", region_name=region)
    try:
        metric_resp = cw.get_metric_statistics(
            Namespace="AWS/S3",
            MetricName="BucketSizeBytes",
            StartTime=start,
            EndTime=now,
            Period=86400,
            Statistics=["Average"],
            Dimensions=[
                {"Name": "BucketName", "Value": bucket_name},
                {"Name": "StorageType", "Value": "StandardStorage"}
            ]
        )
        size_sum = sum(dp["Average"] for dp in metric_resp.get("Datapoints", []))
        if size_sum > 0:
            bucket_entry["status"] = "used"
            used_buckets.append(bucket_name)
        else:
            bucket_entry["status"] = "unused"
            unused_buckets.append(bucket_name)
    except Exception:
        # Fall back: count objects if CloudWatch metrics are unavailable
        try:
            obj_list = s3.list_objects_v2(Bucket=bucket_name, MaxKeys=1)
            if obj_list.get("KeyCount", 0) > 0:
                bucket_entry["status"] = "used"
                used_buckets.append(bucket_name)
            else:
                bucket_entry["status"] = "unused"
                unused_buckets.append(bucket_name)
        except Exception as e:
            bucket_entry["reason"] = f"No access to CloudWatch metrics or list objects: {str(e)}"
            skipped_buckets.append(bucket_name)

    # Get tags
    try:
        tags_resp = s3.get_bucket_tagging(Bucket=bucket_name)
        bucket_entry["tags"] = [{t["Key"]: t["Value"]} for t in tags_resp.get("TagSet", [])]
    except Exception:
        pass  # ignore missing tags

    # Get bucket policy principals
    try:
        policy_resp = s3.get_bucket_policy(Bucket=bucket_name)
        policy_json = json.loads(policy_resp["Policy"])
        principals = []
        for stmt in policy_json.get("Statement", []):
            if "Principal" in stmt:
                principals.append({
                    "Principal": stmt["Principal"],
                    "Action": stmt.get("Action"),
                    "Effect": stmt.get("Effect"),
                    "Resource": stmt.get("Resource")
                })
        bucket_entry["policy_principals"] = principals
    except Exception:
        pass  # ignore missing policies

    report.append(bucket_entry)

# Save to S3
timestamp_prefix = now.strftime("%Y/%m/%d/%H")
output_key = f"{OUTPUT_PREFIX}/{timestamp_prefix}/resources.json.gz"
out_buffer = io.BytesIO()
with gzip.GzipFile(fileobj=out_buffer, mode="w") as f:
    f.write(json.dumps(report).encode("utf-8"))
out_buffer.seek(0)
s3.put_object(
    Bucket=OUTPUT_BUCKET,
    Key=output_key,
    Body=out_buffer,
    ACL="bucket-owner-full-control",
    ServerSideEncryption="AES256"
)

return {
    "status": "success",
    "used": len(used_buckets),
    "unused": len(unused_buckets),
    "skipped": len(skipped_buckets),
    "output_prefix": f"{OUTPUT_PREFIX}/{now.strftime('%Y/%m/%d')}"
}

Apparently

The reason Cloud Custodian is returning an empty list ([]) while your custom Python Lambda finds many unused S3 buckets is a known limitation with how S3 CloudWatch metrics work:

AWS does not emit any datapoints for BucketSizeBytes or NumberOfObjects on truly empty buckets (or buckets with zero size/activity).
When Cloud Custodian's metrics filter queries CloudWatch and gets no datapoints, it treats this as "no match" — the bucket is excluded, even though zero/no datapoints actually indicates an empty/unused bucket.
Your Lambda explicitly falls back to list_objects_v2 (checking KeyCount) when metrics are unavailable or inaccessible, which correctly identifies empty buckets. Cloud Custodian does not do this fallback natively.

This behavior is documented in multiple open Cloud Custodian GitHub issues (e.g., #3798, #363, #1790) and has been a longstanding request — there is still no built-in "empty" or "has-objects" filter for S3 that reliably handles the no-datapoints case.