test: point to pr/138-fixes branch for SASL testing

.buildkite/pipeline.yml

@@ -120,6 +120,40 @@ steps:
             - AWS_DEFAULT_REGION
             - REDPANDA_LICENSE
             - CONNECT_RPM_TOKEN
+  - label: aws fedora sasl
+    key: aws-up-fedora-sasl
+    concurrency_group: aws-fd-cn
+    concurrency: 1
+    command: DEPLOYMENT_ID=ci-sl-fd-`tr -dc a-z0-9 </dev/urandom | head -c 4` DISTRO=41 task ci:aws:rp:sasl
+    plugins:
+      - seek-oss/aws-sm#v2.3.2:
+          json-to-env:
+            - json-key: .
+              secret-id: sdlc/prod/buildkite/deployment_automation
+      - docker#v5.8.0:
+          image: glrp/atgt:latest
+          environment:
+            - AWS_ACCESS_KEY_ID
+            - AWS_SECRET_ACCESS_KEY
+            - REDPANDA_LICENSE
+            - AWS_DEFAULT_REGION
+  - label: aws ubuntu sasl
+    key: aws-up-ubuntu-sasl
+    concurrency_group: aws-sl
+    concurrency: 1
+    command: DEPLOYMENT_ID=ci-sl-ub-`tr -dc a-z0-9 </dev/urandom | head -c 4` DISTRO=ubuntu-focal task ci:aws:rp:sasl
+    plugins:
+      - seek-oss/aws-sm#v2.3.2:
+          json-to-env:
+            - json-key: .
+              secret-id: sdlc/prod/buildkite/deployment_automation
+      - docker#v5.8.0:
+          image: glrp/atgt:latest
+          environment:
+            - AWS_ACCESS_KEY_ID
+            - AWS_SECRET_ACCESS_KEY
+            - REDPANDA_LICENSE
+            - AWS_DEFAULT_REGION
   - label: aws fedora tiered large
     key: aws-up-fedora-ts-large
     concurrency_group: aws-fd
@@ -138,6 +172,7 @@ steps:
             - REDPANDA_LICENSE
             - AWS_DEFAULT_REGION
   - label: gcp ubuntu basic
+    skip: true
     key: gcp-up-ubuntu
     concurrency_group: gcp-ub
     concurrency: 1
@@ -152,6 +187,7 @@ steps:
           environment:
             - GCP_CREDS
   - label: gcp ubuntu tiered
+    skip: true
     key: gcp-up-ubuntu-tiered
     concurrency_group: gcp-ub
     concurrency: 1
@@ -168,6 +204,7 @@ steps:
             - GCP_CREDS
             - REDPANDA_LICENSE
   - label: gcp fedora basic
+    skip: true
     key: gcp-up-fedora
     concurrency_group: gcp-fd
     concurrency: 1
@@ -182,6 +219,7 @@ steps:
           environment:
             - GCP_CREDS
   - label: gcp fedora tiered
+    skip: true
     key: gcp-up-fedora-tiered
     concurrency_group: gcp-fd
     concurrency: 1
@@ -196,11 +234,12 @@ steps:
           environment:
             - GCP_CREDS
             - REDPANDA_LICENSE
-  - label: unstable aws fedora tiered
-    key: aws-us-fedora-tiered
-    concurrency_group: unstable
+  - label: gcp ubuntu sasl
+    skip: true
+    key: gcp-up-ubuntu-sasl
+    concurrency_group: gcp-ub
     concurrency: 1
-    command: DEPLOYMENT_ID=ci-ts-fd-us-`tr -dc a-z0-9 </dev/urandom | head -c 4` DISTRO=41 IS_USING_UNSTABLE=true task ci:aws:rp:tiered
+    command: GCP_IMAGE="ubuntu-os-cloud/ubuntu-2204-lts" DEPLOYMENT_ID="ci-sl-ub-`tr -dc a-z0-9 </dev/urandom | head -c 4`" task ci:gcp:rp:sasl
     plugins:
       - seek-oss/aws-sm#v2.3.2:
           json-to-env:
@@ -209,15 +248,14 @@ steps:
       - docker#v5.8.0:
           image: glrp/atgt:latest
           environment:
-            - AWS_ACCESS_KEY_ID
-            - AWS_SECRET_ACCESS_KEY
+            - GCP_CREDS
             - REDPANDA_LICENSE
-            - AWS_DEFAULT_REGION
-  - label: unstable aws fedora tiered large
-    key: aws-us-fedora-ts-large
-    concurrency_group: unstable
+  - label: gcp fedora sasl
+    skip: true
+    key: gcp-up-fedora-sasl
+    concurrency_group: gcp-fd
     concurrency: 1
-    command: DEPLOYMENT_ID=ci-ts-fd-us-lg-`tr -dc a-z0-9 </dev/urandom | head -c 4` DISTRO=41 IS_USING_UNSTABLE=true INSTANCE_TYPE_AWS=is4gen.4xlarge MACHINE_ARCH=arm64 task ci:aws:rp:tiered
+    command: GCP_IMAGE="fedora-cloud/fedora-cloud-39" DEPLOYMENT_ID="ci-sl-fd-`tr -dc a-z0-9 </dev/urandom | head -c 4`" task ci:gcp:rp:sasl
     plugins:
       - seek-oss/aws-sm#v2.3.2:
           json-to-env:
@@ -226,15 +264,13 @@ steps:
       - docker#v5.8.0:
           image: glrp/atgt:latest
           environment:
-            - AWS_ACCESS_KEY_ID
-            - AWS_SECRET_ACCESS_KEY
+            - GCP_CREDS
             - REDPANDA_LICENSE
-            - AWS_DEFAULT_REGION
-  - label: unstable aws ubuntu tiered
-    key: aws-us-ubuntu-tiered
-    concurrency_group: unstable
+  - label: unstable aws fedora sasl
+    key: aws-us-fedora-sasl
+    concurrency_group: aws-sl
     concurrency: 1
-    command: DEPLOYMENT_ID=ci-ts-ub-us-`tr -dc a-z0-9 </dev/urandom | head -c 4` DISTRO=ubuntu-focal IS_USING_UNSTABLE=true task ci:aws:rp:tiered
+    command: DEPLOYMENT_ID=ci-sl-fd-us-`tr -dc a-z0-9 </dev/urandom | head -c 4` DISTRO=41 IS_USING_UNSTABLE=true task ci:aws:rp:sasl
     plugins:
       - seek-oss/aws-sm#v2.3.2:
           json-to-env:
@@ -247,11 +283,11 @@ steps:
             - AWS_SECRET_ACCESS_KEY
             - REDPANDA_LICENSE
             - AWS_DEFAULT_REGION
-  - label: unstable aws ubuntu tiered large
-    key: aws-us-ubuntu-ts-large
-    concurrency_group: unstable
+  - label: unstable aws ubuntu sasl
+    key: aws-us-ubuntu-sasl
+    concurrency_group: aws-sl
     concurrency: 1
-    command: DEPLOYMENT_ID=ci-ts-ub-us-lg-`tr -dc a-z0-9 </dev/urandom | head -c 4` DISTRO=ubuntu-focal IS_USING_UNSTABLE=true INSTANCE_TYPE_AWS=is4gen.4xlarge MACHINE_ARCH=arm64 task ci:aws:rp:tiered
+    command: DEPLOYMENT_ID=ci-sl-ub-us-`tr -dc a-z0-9 </dev/urandom | head -c 4` DISTRO=ubuntu-focal IS_USING_UNSTABLE=true task ci:aws:rp:sasl
     plugins:
       - seek-oss/aws-sm#v2.3.2:
           json-to-env:
@@ -264,6 +300,38 @@ steps:
             - AWS_SECRET_ACCESS_KEY
             - REDPANDA_LICENSE
             - AWS_DEFAULT_REGION
+  - label: unstable gcp ubuntu sasl
+    skip: true
+    key: gcp-us-ubuntu-sasl
+    concurrency_group: gcp-us
+    concurrency: 1
+    command: GCP_IMAGE="ubuntu-os-cloud/ubuntu-2204-lts" DEPLOYMENT_ID="ci-sl-ub-us-`tr -dc a-z0-9 </dev/urandom | head -c 4`" IS_USING_UNSTABLE=true task ci:gcp:rp:sasl
+    plugins:
+      - seek-oss/aws-sm#v2.3.2:
+          json-to-env:
+            - json-key: .
+              secret-id: sdlc/prod/buildkite/deployment_automation
+      - docker#v5.8.0:
+          image: glrp/atgt:latest
+          environment:
+            - GCP_CREDS
+            - REDPANDA_LICENSE
+  - label: unstable gcp fedora sasl
+    skip: true
+    key: gcp-us-fedora-sasl
+    concurrency_group: gcp-us
+    concurrency: 1
+    command: GCP_IMAGE="fedora-cloud/fedora-cloud-39" DEPLOYMENT_ID="ci-sl-fd-us-`tr -dc a-z0-9 </dev/urandom | head -c 4`" IS_USING_UNSTABLE=true task ci:gcp:rp:sasl
+    plugins:
+      - seek-oss/aws-sm#v2.3.2:
+          json-to-env:
+            - json-key: .
+              secret-id: sdlc/prod/buildkite/deployment_automation
+      - docker#v5.8.0:
+          image: glrp/atgt:latest
+          environment:
+            - GCP_CREDS
+            - REDPANDA_LICENSE
 
   - label: cleanup aws resources
     key: cleanup-aws
@@ -289,10 +357,10 @@ steps:
       - aws-up-fedora-tiered
       - aws-up-fed-cts
       - aws-up-fedora-ts-large
-      - aws-us-fedora-tiered
-      - aws-us-fedora-ts-large
-      - aws-us-ubuntu-tiered
-      - aws-us-ubuntu-ts-large
+      - aws-us-fedora-sasl
+      - aws-us-ubuntu-sasl
+      - aws-up-fedora-sasl
+      - aws-up-ubuntu-sasl
     allow_dependency_failure: true
     soft_fail: true
 
@@ -315,5 +383,9 @@ steps:
       - gcp-up-ubuntu-tiered
       - gcp-up-fedora
       - gcp-up-fedora-tiered
+      - gcp-up-ubuntu-sasl
+      - gcp-up-fedora-sasl
+      - gcp-us-ubuntu-sasl
+      - gcp-us-fedora-sasl
     allow_dependency_failure: true
     soft_fail: true

.gitignore

@@ -22,3 +22,4 @@ ansible/tls/clients/client.crt
 /aws-extra/
 ansible/proxy/tls/**
 .ansible
+.claude/

.tasks/ansible.yml

@@ -13,6 +13,16 @@ tasks:
       - :ensure-collections-dir
     cmds:
       - ansible-galaxy collection install -r {{.USER_WORKING_DIR}}/requirements.yml --force -p {{.ANSIBLE_COLLECTIONS_PATH}}
+      - |
+        # Log git-based collection commit SHAs for debugging
+        for collection_dir in {{.ANSIBLE_COLLECTIONS_PATH}}/ansible_collections/*/*; do
+          if [ -d "$collection_dir/.git" ]; then
+            collection_name=$(basename $(dirname $collection_dir))/$(basename $collection_dir)
+            commit_sha=$(git -C "$collection_dir" rev-parse HEAD 2>/dev/null)
+            commit_msg=$(git -C "$collection_dir" log -1 --format="%s" 2>/dev/null)
+            echo "Collection $collection_name at commit: $commit_sha ($commit_msg)"
+          fi
+        done
 
   role:
     desc: Install Ansible roles

.tasks/ci.yml

@@ -17,7 +17,7 @@ tasks:
     cmds:
       - task: :tools:keygen
       - task: :infra:aws:build
-      - task: :cluster:provision
+      - task: :cluster:provision:idempotent
       - task: :monitor:deploy
       - task: :monitor:console:deploy
       - task: :tools:rpk:install
@@ -35,7 +35,7 @@ tasks:
         vars: {ENABLE_CONNECT: "true", DISTRO: "41"}
       - task: :infra:extra:aws:copy
       - task: :infra:extra:aws:build
-      - task: :cluster:provision
+      - task: :cluster:provision:idempotent
       - task: :connect:deploy
       - task: :monitor:deploy
       - task: :monitor:console:deploy
@@ -54,14 +54,30 @@ tasks:
       - task: :tools:keygen
       - task: :infra:aws:build
         vars: {TIERED_STORAGE_ENABLED: "true"}
-      - task: :cluster:tiered
+      - task: :cluster:tiered:idempotent
       - task: :monitor:deploy:tls
       - task: :monitor:console:deploy:tls
       - task: :tools:rpk:install
       - task: :test:cluster:tls
       - task: :test:storage:aws
       - task: :infra:aws:destroy
 
+  aws:rp:sasl:
+    desc: CI workflow - AWS Redpanda with TLS + SASL + tiered storage
+    vars:
+      TIERED_STORAGE_ENABLED: "true"
+    cmds:
+      - task: :tools:keygen
+      - task: :infra:aws:build
+        vars: {TIERED_STORAGE_ENABLED: "true"}
+      - task: :cluster:sasl:idempotent
+      - task: :monitor:deploy:tls
+      - task: :monitor:console:deploy:tls
+      - task: :tools:rpk:install
+      - task: :test:cluster:sasl
+      - task: :test:storage:aws
+      - task: :infra:aws:destroy
+
   aws:rp:ts-connect:
     desc: CI workflow - AWS Redpanda with tiered storage and Connect
     vars:
@@ -72,7 +88,7 @@ tasks:
       - task: :tools:keygen
       - task: :infra:aws:build
         vars: {ENABLE_CONNECT: "true", TIERED_STORAGE_ENABLED: "true", DISTRO: "41"}
-      - task: :cluster:tiered
+      - task: :cluster:tiered:idempotent
       - task: :connect:deploy:tls
       - task: :monitor:deploy:tls
       - task: :monitor:console:deploy:tls
@@ -94,7 +110,7 @@ tasks:
     cmds:
       - task: :tools:keygen
       - task: :infra:gcp:build
-      - task: :cluster:provision
+      - task: :cluster:provision:idempotent
       - task: :monitor:deploy
       - task: :monitor:console:deploy
       - task: :tools:rpk:install
@@ -109,14 +125,30 @@ tasks:
       - task: :tools:keygen
       - task: :infra:gcp:build
         vars: {TIERED_STORAGE_ENABLED: "true"}
-      - task: :cluster:tiered
+      - task: :cluster:tiered:idempotent
       - task: :monitor:deploy:tls
       - task: :monitor:console:deploy:tls
       - task: :tools:rpk:install
       - task: :test:cluster:tls
       - task: :test:storage:gcp
       - task: :infra:gcp:destroy
 
+  gcp:rp:sasl:
+    desc: CI workflow - GCP Redpanda with TLS + SASL + tiered storage
+    vars:
+      TIERED_STORAGE_ENABLED: "true"
+    cmds:
+      - task: :tools:keygen
+      - task: :infra:gcp:build
+        vars: {TIERED_STORAGE_ENABLED: "true"}
+      - task: :cluster:sasl:idempotent
+      - task: :monitor:deploy:tls
+      - task: :monitor:console:deploy:tls
+      - task: :tools:rpk:install
+      - task: :test:cluster:sasl
+      - task: :test:storage:gcp
+      - task: :infra:gcp:destroy
+
   # Extra deployment workflows
   extra:rp:
     desc: Deploy secondary Redpanda cluster

.tasks/cluster.yml

@@ -48,3 +48,39 @@ tasks:
       - :ensure-logs-dir
     cmds:
       - ansible-playbook ansible/provision-cluster-tiered-storage.yml --private-key {{.PRIVATE_KEY}} --extra-vars redpanda_broker_no_log=false --extra-vars development_build=true --extra-vars segment_upload_interval={{.SEGMENT_UPLOAD_INTERVAL}} --extra-vars cloud_storage_credentials_source={{.CLOUD_STORAGE_CREDENTIALS_SOURCE}}
+
+  # Idempotency check variants - run playbook twice to verify no unexpected changes
+  provision:idempotent:
+    desc: Provision Redpanda cluster with idempotency check (runs twice)
+    deps:
+      - :ansible:prereqs
+      - :ensure-logs-dir
+    cmds:
+      - ansible-playbook ansible/provision-cluster.yml --private-key {{.PRIVATE_KEY}} --inventory {{.ANSIBLE_INVENTORY}} --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} --extra-vars redpanda_broker_no_log=false
+      - scripts/check-idempotency.sh ansible-playbook ansible/provision-cluster.yml --private-key {{.PRIVATE_KEY}} --inventory {{.ANSIBLE_INVENTORY}} --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} --extra-vars redpanda_broker_no_log=false
+
+  tiered:idempotent:
+    desc: Provision tiered storage cluster with idempotency check (runs twice)
+    deps:
+      - :ansible:prereqs
+      - :ensure-logs-dir
+    cmds:
+      - ansible-playbook ansible/provision-cluster-tiered-storage.yml --private-key {{.PRIVATE_KEY}} --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} --extra-vars segment_upload_interval={{.SEGMENT_UPLOAD_INTERVAL}} --extra-vars cloud_storage_credentials_source={{.CLOUD_STORAGE_CREDENTIALS_SOURCE}} --extra-vars redpanda_license={{.REDPANDA_LICENSE}} --extra-vars redpanda_broker_no_log=false
+      - scripts/check-idempotency.sh ansible-playbook ansible/provision-cluster-tiered-storage.yml --private-key {{.PRIVATE_KEY}} --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} --extra-vars segment_upload_interval={{.SEGMENT_UPLOAD_INTERVAL}} --extra-vars cloud_storage_credentials_source={{.CLOUD_STORAGE_CREDENTIALS_SOURCE}} --extra-vars redpanda_license={{.REDPANDA_LICENSE}} --extra-vars redpanda_broker_no_log=false
+
+  sasl:
+    desc: Provision Redpanda cluster with TLS + SASL + tiered storage
+    deps:
+      - :ansible:prereqs
+      - :ensure-logs-dir
+    cmds:
+      - ansible-playbook ansible/provision-cluster-tls-sasl.yml --private-key {{.PRIVATE_KEY}} --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} --extra-vars segment_upload_interval={{.SEGMENT_UPLOAD_INTERVAL}} --extra-vars cloud_storage_credentials_source={{.CLOUD_STORAGE_CREDENTIALS_SOURCE}} --extra-vars redpanda_license={{.REDPANDA_LICENSE}} --extra-vars sasl_superuser_password={{.REDPANDA_SASL_PASSWORD}}
+
+  sasl:idempotent:
+    desc: Provision TLS + SASL + tiered storage cluster with idempotency check (runs twice)
+    deps:
+      - :ansible:prereqs
+      - :ensure-logs-dir
+    cmds:
+      - ansible-playbook ansible/provision-cluster-tls-sasl.yml --private-key {{.PRIVATE_KEY}} --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} --extra-vars segment_upload_interval={{.SEGMENT_UPLOAD_INTERVAL}} --extra-vars cloud_storage_credentials_source={{.CLOUD_STORAGE_CREDENTIALS_SOURCE}} --extra-vars redpanda_license={{.REDPANDA_LICENSE}} --extra-vars redpanda_broker_no_log=false --extra-vars sasl_superuser_password={{.REDPANDA_SASL_PASSWORD}}
+      - scripts/check-idempotency.sh ansible-playbook ansible/provision-cluster-tls-sasl.yml --private-key {{.PRIVATE_KEY}} --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} --extra-vars segment_upload_interval={{.SEGMENT_UPLOAD_INTERVAL}} --extra-vars cloud_storage_credentials_source={{.CLOUD_STORAGE_CREDENTIALS_SOURCE}} --extra-vars redpanda_license={{.REDPANDA_LICENSE}} --extra-vars redpanda_broker_no_log=false --extra-vars sasl_superuser_password={{.REDPANDA_SASL_PASSWORD}}