|
| 1 | +# AWS MSK IAM authentication for Burrow |
| 2 | + |
| 3 | +Burrow can now connect to **Amazon MSK clusters that use IAM (SASL/OAUTHBEARER) + TLS** |
| 4 | +without proxies or sidecars. |
| 5 | +The feature is built on: |
| 6 | + |
| 7 | +* [`github.com/aws/aws-msk-iam-sasl-signer-go` v1.0.4] – generates the SigV4 |
| 8 | + bearer token expected by the broker. |
| 9 | +* [`github.com/IBM/sarama` ≥ v1.45.2] – already exposes the generic |
| 10 | + *OAUTHBEARER* mechanism and `AccessTokenProvider` hook we plug into. |
| 11 | + |
| 12 | +Supported credential sources |
| 13 | +---------------------------- |
| 14 | + |
| 15 | +| Credential chain node | Works? | Notes | |
| 16 | +|-----------------------|--------|-------| |
| 17 | +| EC2 / EKS instance profile | ✅ | No extra config needed. | |
| 18 | +| **EKS Pod Identity** | ✅ | Preferred on EKS ≥ 1.29. | |
| 19 | +| **IRSA (IAM Roles for SA)** | ✅ | Works via the same default chain. | |
| 20 | +| Shared credentials / static keys | ✅ | Standard `~/.aws/credentials` or env vars. | |
| 21 | +| STS AssumeRole w/ session tags | ✅ | Add `role-arn` in the TOML (requires `sts:AssumeRole` + `sts:TagSession`). | |
| 22 | + |
| 23 | +> IAM auth **requires** the broker listener `TLS + IAM` (port **9098** or **9198**) to be enabled. |
| 24 | +
|
| 25 | +Quick-start |
| 26 | +----------- |
| 27 | + |
| 28 | +1. Enable the *SASL/IAM* and *TLS* security settings on your MSK cluster. |
| 29 | +2. Make sure Burrow’s container can reach the IAM listener (security groups, NACLs, etc.). |
| 30 | +3. Add the sections shown in `examples/burrow.toml` (below) and restart Burrow. |
| 31 | + |
| 32 | +Example configuration |
| 33 | +--------------------- |
| 34 | + |
| 35 | +```toml |
| 36 | +####################################################################### |
| 37 | +# TLS: trust the Amazon root CAs shipped in most Linux distros |
| 38 | +####################################################################### |
| 39 | +[tls.msk-tls] |
| 40 | +# For Debian/Ubuntu/RHEL/… the CA bundle is already present: |
| 41 | +cacert = "/etc/ssl/certs/ca-certificates.crt" |
| 42 | + |
| 43 | +####################################################################### |
| 44 | +# IAM: pick one of the credential modes below |
| 45 | +####################################################################### |
| 46 | + |
| 47 | +# --- A) Use the pod / instance credentials directly --------------- |
| 48 | +[iam.msk-iam] |
| 49 | +region = "us-west-1" |
| 50 | + |
| 51 | +# --- B) Re-assume a dedicated read-only role ---------------------- |
| 52 | +#[iam.msk-iam] |
| 53 | +#region = "us-west-1" |
| 54 | +#role-arn = "arn:aws:iam::123456789012:role/burrow-readonly" |
| 55 | +#profile = "burrow" # optional named profile |
| 56 | + |
| 57 | +####################################################################### |
| 58 | +[client-profile.msk-iam] |
| 59 | +kafka-version = "2.8.0" |
| 60 | +client-id = "burrow" |
| 61 | +tls = "msk-tls" |
| 62 | +iam = "msk-iam" |
| 63 | + |
| 64 | +####################################################################### |
| 65 | +# Cluster + consumer definitions (use the TLS+IAM listener) |
| 66 | +####################################################################### |
| 67 | +[cluster.prod] |
| 68 | +class-name = "kafka" |
| 69 | +client-profile = "msk-iam" |
| 70 | +servers = [ |
| 71 | + "b-1.prod.abcd.c2.kafka.us-west-1.amazonaws.com:9098", |
| 72 | + "b-2.prod.abcd.c2.kafka.us-west-1.amazonaws.com:9098", |
| 73 | + "b-3.prod.abcd.c2.kafka.us-west-1.amazonaws.com:9098" |
| 74 | +] |
| 75 | + |
| 76 | +[consumer.prod] |
| 77 | +class-name = "kafka" |
| 78 | +client-profile = "msk-iam" |
| 79 | +cluster = "prod" |
| 80 | +start-latest = true |
| 81 | +group-denylist = '^(console-consumer-|.*\s.*).*$' |
| 82 | +servers = ${cluster.prod.servers} # reuse same list |
| 83 | +``` |
0 commit comments