Initial cross-account log sharing module

.github/scripts/get-pr-reviews.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -euo pipefail
+
+if ! command -v gh >/dev/null 2>&1; then
+  echo "error: GitHub CLI (gh) is required but not installed." >&2
+  exit 1
+fi
+
+if ! command -v jq >/dev/null 2>&1; then
+  echo "error: jq is required but not installed." >&2
+  exit 1
+fi
+
+if ! gh extension list | awk '{print $1}' | grep -qx "agynio/gh-pr-review"; then
+  echo "Installing gh extension agynio/gh-pr-review..." >&2
+  gh extension install agynio/gh-pr-review
+fi
+
+GH_REPO="$(git config --get remote.origin.url | sed 's/.*[:\/]\(.*\)\/\(.*\)\.git/\1\/\2/')"
+GIT_PR="$(gh pr list --head "$(git branch --show-current)" --json number | jq '.[0].number')"
+
+gh pr-review review view "${GIT_PR}" --repo "${GH_REPO}"

.github/workflows/tfvalidate.yml

@@ -76,5 +76,31 @@ jobs:
       - name: Validate ${{ matrix.terraform_dir }}  module (covers all submodules)
         working-directory: ${{ matrix.terraform_dir }}
         run: |
+          # Some modules use aliased AWS providers (for example aws.destination/aws.target)
+          # and fail direct `terraform validate` unless alias configs are present.
+          # Generate a temporary provider file for any discovered aws.<alias> references.
+          AWS_ALIASES="$(grep -RhoE 'provider[[:space:]]*=[[:space:]]*aws\.[A-Za-z0-9_]+' -- *.tf 2>/dev/null | sed -E 's/.*aws\.([A-Za-z0-9_]+)/\1/' | sort -u || true)"
+          if [[ -n "${AWS_ALIASES}" ]]; then
+            # Terraform ignores dot-prefixed files, so this must not start with "."
+            AUTO_PROVIDER_FILE="tfvalidate.providers.auto.tf"
+            {
+              echo "// Generated by CI for terraform validate."
+              echo "// Do not commit."
+              echo
+            } > "${AUTO_PROVIDER_FILE}"
+
+            for alias in ${AWS_ALIASES}; do
+              {
+                echo 'provider "aws" {'
+                echo "  alias  = \"${alias}\""
+                echo '  region = "us-east-2"'
+                echo '}'
+                echo
+              } >> "${AUTO_PROVIDER_FILE}"
+            done
+
+            echo "Generated ${AUTO_PROVIDER_FILE} for aliases: ${AWS_ALIASES}"
+          fi
+
           terraform init -backend=false
           terraform validate

addons/byo-cloudwatch-log-sharing/cloudwatch/.header.md

@@ -0,0 +1,40 @@
+# Fleet CloudWatch Log Group Sharing (Source Account)
+
+## Usage
+
+Use this module in the source Fleet account (and in the same region as the Fleet CloudWatch log group).
+
+### Firehose destination (default)
+
+```hcl
+module "fleet_log_sharing" {
+  source = "github.com/fleetdm/fleet-terraform//addons/byo-cloudwatch-log-sharing/cloudwatch?depth=1&ref=tf-mod-addon-byo-cloudwatch-log-sharing-v1.0.0"
+
+  subscription = {
+    log_group_name  = module.fleet.byo-vpc.byo-db.byo-ecs.logging_config["awslogs-group"]
+    destination_arn = "arn:aws:logs:us-east-2:222222222222:destination:fleet-app-logs-firehose"
+    # destination_type defaults to "firehose"
+    filter_pattern = ""
+  }
+}
+```
+
+### Kinesis destination
+
+```hcl
+module "fleet_log_sharing" {
+  source = "github.com/fleetdm/fleet-terraform//addons/byo-cloudwatch-log-sharing/cloudwatch?depth=1&ref=tf-mod-addon-byo-cloudwatch-log-sharing-v1.0.0"
+
+  subscription = {
+    log_group_name       = module.fleet.byo-vpc.byo-db.byo-ecs.logging_config["awslogs-group"]
+    destination_arn      = "arn:aws:logs:us-east-2:222222222222:destination:fleet-app-logs-kinesis"
+    destination_type     = "kinesis"
+    kinesis_distribution = "ByLogStream"
+    filter_pattern       = ""
+  }
+}
+```
+
+`subscription.destination_arn` is usually provided out-of-band by the organization that runs the target-account module in their environment.
+
+No Fleet environment variable changes are required. This module configures the CloudWatch Logs subscription directly on the Fleet log group.

addons/byo-cloudwatch-log-sharing/cloudwatch/.terraform-docs.yml

@@ -0,0 +1 @@
+header-from: .header.md

addons/byo-cloudwatch-log-sharing/cloudwatch/README.md

@@ -0,0 +1,75 @@
+# Fleet CloudWatch Log Group Sharing (Source Account)
+
+## Usage
+
+Use this module in the source Fleet account (and in the same region as the Fleet CloudWatch log group).
+
+### Firehose destination (default)
+
+```hcl
+module "fleet_log_sharing" {
+  source = "github.com/fleetdm/fleet-terraform//addons/byo-cloudwatch-log-sharing/cloudwatch?depth=1&ref=tf-mod-addon-byo-cloudwatch-log-sharing-v1.0.0"
+
+  subscription = {
+    log_group_name  = module.fleet.byo-vpc.byo-db.byo-ecs.logging_config["awslogs-group"]
+    destination_arn = "arn:aws:logs:us-east-2:222222222222:destination:fleet-app-logs-firehose"
+    # destination_type defaults to "firehose"
+    filter_pattern = ""
+  }
+}
+```
+
+### Kinesis destination
+
+```hcl
+module "fleet_log_sharing" {
+  source = "github.com/fleetdm/fleet-terraform//addons/byo-cloudwatch-log-sharing/cloudwatch?depth=1&ref=tf-mod-addon-byo-cloudwatch-log-sharing-v1.0.0"
+
+  subscription = {
+    log_group_name       = module.fleet.byo-vpc.byo-db.byo-ecs.logging_config["awslogs-group"]
+    destination_arn      = "arn:aws:logs:us-east-2:222222222222:destination:fleet-app-logs-kinesis"
+    destination_type     = "kinesis"
+    kinesis_distribution = "ByLogStream"
+    filter_pattern       = ""
+  }
+}
+```
+
+`subscription.destination_arn` is usually provided out-of-band by the organization that runs the target-account module in their environment.
+
+No Fleet environment variable changes are required. This module configures the CloudWatch Logs subscription directly on the Fleet log group.
+
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.3.7 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 5.29.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 5.29.0 |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [aws_cloudwatch_log_subscription_filter.fleet_log_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_subscription_filter) | resource |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_subscription"></a> [subscription](#input\_subscription) | CloudWatch Logs subscription configuration for sharing a source log group to a cross-account destination. | <pre>object({<br/>    log_group_name       = string<br/>    destination_arn      = string<br/>    filter_name          = optional(string, "fleet-log-sharing")<br/>    filter_pattern       = optional(string, "")<br/>    destination_type     = optional(string, "firehose")<br/>    kinesis_distribution = optional(string, "ByLogStream")<br/>  })</pre> | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_subscription_filter"></a> [subscription\_filter](#output\_subscription\_filter) | CloudWatch Logs subscription filter details. |

addons/byo-cloudwatch-log-sharing/cloudwatch/main.tf

@@ -0,0 +1,7 @@
+resource "aws_cloudwatch_log_subscription_filter" "fleet_log_group" {
+  name            = var.subscription.filter_name
+  log_group_name  = var.subscription.log_group_name
+  filter_pattern  = var.subscription.filter_pattern
+  destination_arn = var.subscription.destination_arn
+  distribution    = var.subscription.destination_type == "kinesis" ? var.subscription.kinesis_distribution : null
+}

addons/byo-cloudwatch-log-sharing/cloudwatch/outputs.tf

@@ -0,0 +1,8 @@
+output "subscription_filter" {
+  description = "CloudWatch Logs subscription filter details."
+  value = {
+    name            = aws_cloudwatch_log_subscription_filter.fleet_log_group.name
+    log_group_name  = aws_cloudwatch_log_subscription_filter.fleet_log_group.log_group_name
+    destination_arn = aws_cloudwatch_log_subscription_filter.fleet_log_group.destination_arn
+  }
+}

addons/byo-cloudwatch-log-sharing/cloudwatch/variables.tf

@@ -0,0 +1,21 @@
+variable "subscription" {
+  description = "CloudWatch Logs subscription configuration for sharing a source log group to a cross-account destination."
+  type = object({
+    log_group_name       = string
+    destination_arn      = string
+    filter_name          = optional(string, "fleet-log-sharing")
+    filter_pattern       = optional(string, "")
+    destination_type     = optional(string, "firehose")
+    kinesis_distribution = optional(string, "ByLogStream")
+  })
+
+  validation {
+    condition     = contains(["firehose", "kinesis"], var.subscription.destination_type)
+    error_message = "subscription.destination_type must be one of: firehose, kinesis."
+  }
+
+  validation {
+    condition     = contains(["ByLogStream", "Random"], var.subscription.kinesis_distribution)
+    error_message = "subscription.kinesis_distribution must be one of: ByLogStream, Random."
+  }
+}

addons/byo-cloudwatch-log-sharing/cloudwatch/version.tf

@@ -0,0 +1,10 @@
+terraform {
+  required_version = ">= 1.3.7"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.29.0"
+    }
+  }
+}

addons/byo-cloudwatch-log-sharing/pubsub-bridge/.header.md

@@ -0,0 +1,105 @@
+# Fleet CloudWatch Logs to GCP Pub/Sub Bridge
+
+This module creates an AWS Lambda subscription target for a Fleet CloudWatch log group and forwards log events to a GCP Pub/Sub topic.
+
+The Lambda function reads Google service account credentials from an existing AWS Secrets Manager secret.
+Deploy this module in the same AWS region as the Fleet CloudWatch log group.
+The bridge Lambda is implemented in Go and compiled during Terraform apply.
+Publishing uses the official `cloud.google.com/go/pubsub/v2` client library.
+The Terraform execution environment must include a working `go` toolchain.
+
+## Usage
+
+```hcl
+provider "aws" {
+  region = "us-east-2"
+}
+
+variable "gcp_pubsub_service_account_credentials_json" {
+  description = "Google service-account credentials JSON shared by the customer (for example from target-account-gcp publisher_credentials_json)."
+  type        = string
+  sensitive   = true
+}
+
+resource "aws_secretsmanager_secret" "gcp_pubsub_credentials" {
+  name = "fleet/gcp/pubsub/service-account"
+}
+
+resource "aws_secretsmanager_secret_version" "gcp_pubsub_credentials" {
+  secret_id = aws_secretsmanager_secret.gcp_pubsub_credentials.id
+  secret_string = jsonencode({
+    service_account_json = var.gcp_pubsub_service_account_credentials_json
+  })
+}
+
+module "fleet_pubsub_bridge" {
+  source = "github.com/fleetdm/fleet-terraform//addons/byo-cloudwatch-log-sharing/pubsub-bridge?depth=1&ref=tf-mod-addon-byo-cloudwatch-log-sharing-v1.0.0"
+
+  subscription = {
+    log_group_name = module.fleet.byo-vpc.byo-db.byo-ecs.logging_config["awslogs-group"]
+    filter_pattern = ""
+  }
+
+  gcp_pubsub = {
+    project_id             = "customer-observability-prod"
+    topic_id               = "fleet-logs"
+    credentials_secret_arn = aws_secretsmanager_secret.gcp_pubsub_credentials.arn
+  }
+
+  lambda = {
+    function_name = "fleet-cloudwatch-pubsub-bridge"
+    role_name     = "fleet-cloudwatch-pubsub-bridge-role"
+    policy_name   = "fleet-cloudwatch-pubsub-bridge-policy"
+    timeout       = 60
+    memory_size   = 256
+    batch_size    = 1000
+  }
+
+  dlq = {
+    enabled                      = true
+    queue_name                   = "fleet-cloudwatch-pubsub-bridge-dlq"
+    maximum_retry_attempts       = 2
+    maximum_event_age_in_seconds = 3600
+  }
+
+  alerting = {
+    enabled                        = true
+    sns_topic_arns                 = ["arn:aws:sns:us-east-2:111111111111:fleet-ops-alerts"]
+    period_seconds                 = 300
+    evaluation_periods             = 1
+    datapoints_to_alarm            = 1
+    lambda_errors_threshold        = 1
+    dlq_visible_messages_threshold = 1
+    enable_ok_notifications        = true
+  }
+
+  replayer = {
+    enabled                            = true
+    function_name                      = "fleet-cloudwatch-pubsub-bridge-replayer"
+    batch_size                         = 10
+    maximum_batching_window_in_seconds = 5
+    maximum_concurrency                = 2
+  }
+}
+
+output "fleet_pubsub_bridge" {
+  value = module.fleet_pubsub_bridge
+}
+```
+
+The referenced secret should contain either:
+- A Google service-account key JSON document directly.
+- A JSON object with a `service_account_json` field that contains the service-account key JSON.
+
+This module uses CloudWatch alarms for notifications so alerts fire on state transitions instead of per-failed event, which avoids high-volume SNS spam during sustained failures.
+
+A built-in SQS replayer Lambda is enabled by default and re-drives failed async events from the DLQ back to the main bridge Lambda with partial-batch failure handling.
+
+## Reprocessing Options
+
+1. Built-in automatic replay:
+Use the module's default `replayer` settings to continuously re-drive failed events from DLQ.
+2. Manual/batched replay:
+Drain DLQ messages to S3 for analysis, then replay selected batches during controlled windows.
+3. Scheduled replay workflow:
+Use EventBridge Scheduler or Step Functions to periodically replay aged DLQ messages with rate limits and stop conditions.

addons/byo-cloudwatch-log-sharing/pubsub-bridge/.terraform-docs.yml

@@ -0,0 +1 @@
+header-from: .header.md