From 0ec4e3308e8b633a9582a010f4220eb7e4935cb1 Mon Sep 17 00:00:00 2001 From: gene-redpanda <123959009+gene-redpanda@users.noreply.github.com> Date: Tue, 7 Oct 2025 22:31:56 -0500 Subject: [PATCH 1/9] chore: bump client dockerfile versions --- Dockerfile_FEDORA | 16 ++++++++-------- Dockerfile_UBUNTU | 18 ++++++++++++------ 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/Dockerfile_FEDORA b/Dockerfile_FEDORA index b49dcf87..5aeba824 100644 --- a/Dockerfile_FEDORA +++ b/Dockerfile_FEDORA @@ -1,4 +1,4 @@ -FROM fedora:36 +FROM fedora:42 ENV DA_AWS_ACCESS_KEY_ID="default" ENV DA_AWS_SECRET_ACCESS_KEY="default" @@ -12,24 +12,24 @@ RUN dnf -y update \ && rm -rf /var/cache/dnf/* # Install Terraform -RUN wget https://releases.hashicorp.com/terraform/1.4.5/terraform_1.4.5_linux_amd64.zip \ - && unzip terraform_1.4.5_linux_amd64.zip -d /usr/local/bin \ - && rm terraform_1.4.5_linux_amd64.zip +RUN wget https://releases.hashicorp.com/terraform/1.13.3/terraform_1.13.3_linux_amd64.zip \ + && unzip terraform_1.13.3_linux_amd64.zip -d /usr/local/bin \ + && rm terraform_1.13.3_linux_amd64.zip # Install aws cli RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip" \ && unzip awscliv2.zip \ && ./aws/install -RUN curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-448.0.0-linux-x86_64.tar.gz && \ - tar -zxvf google-cloud-cli-448.0.0-linux-x86_64.tar.gz && \ +RUN curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-541.0.0-linux-x86_64.tar.gz && \ + tar -zxvf google-cloud-cli-541.0.0-linux-x86_64.tar.gz && \ ./google-cloud-sdk/install.sh --quiet && \ - rm google-cloud-cli-448.0.0-linux-x86_64.tar.gz + rm google-cloud-cli-541.0.0-linux-x86_64.tar.gz ENV PATH="/google-cloud-sdk/bin:${PATH}" # Install task -RUN curl -sSLf "https://github.com/go-task/task/releases/download/v3.21.0/task_linux_amd64.tar.gz" | tar -xz -C /usr/local/bin +RUN curl -sSLf "https://github.com/go-task/task/releases/download/v3.45.4/task_linux_amd64.tar.gz" | tar -xz -C /usr/local/bin ## uncomment for use as a local client #RUN mkdir -p /app diff --git a/Dockerfile_UBUNTU b/Dockerfile_UBUNTU index e695ffd5..7003b31d 100644 --- a/Dockerfile_UBUNTU +++ b/Dockerfile_UBUNTU @@ -1,4 +1,4 @@ -FROM ubuntu:22.04 +FROM ubuntu:24.04 ARG DEBIAN_FRONTEND=noninteractive ENV DA_AWS_ACCESS_KEY_ID="default" @@ -8,7 +8,7 @@ ENV GCP_CREDS="default" # Install required packages RUN apt-get update \ - && apt install -y unzip wget curl vim git software-properties-common awscli jq lsb-release gnupg make default-jdk \ + && apt install -y unzip wget curl vim git software-properties-common jq lsb-release gnupg make default-jdk \ && apt-add-repository --yes --update ppa:ansible/ansible \ && apt-get install -y ansible \ && echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list \ @@ -17,14 +17,20 @@ RUN apt-get update \ && apt-get install -y google-cloud-sdk \ && rm -rf /var/lib/apt/lists/* +# Install AWS CLI v2 +RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip" \ + && unzip awscliv2.zip \ + && ./aws/install \ + && rm -rf awscliv2.zip aws + # Install Terraform -RUN wget https://releases.hashicorp.com/terraform/1.4.5/terraform_1.4.5_linux_amd64.zip \ - && unzip terraform_1.4.5_linux_amd64.zip -d /usr/local/bin \ - && rm terraform_1.4.5_linux_amd64.zip +RUN wget https://releases.hashicorp.com/terraform/1.13.3/terraform_1.13.3_linux_amd64.zip \ + && unzip terraform_1.13.3_linux_amd64.zip -d /usr/local/bin \ + && rm terraform_1.13.3_linux_amd64.zip # Install task -RUN curl -sSLf "https://github.com/go-task/task/releases/download/v3.21.0/task_linux_amd64.tar.gz" | tar -xz -C /usr/local/bin +RUN curl -sSLf "https://github.com/go-task/task/releases/download/v3.45.4/task_linux_amd64.tar.gz" | tar -xz -C /usr/local/bin ## uncomment for use as a local client #RUN mkdir -p /app From 5f6888225def4b751c94b027e3ff4129ba6550a3 Mon Sep 17 00:00:00 2001 From: gene-redpanda <123959009+gene-redpanda@users.noreply.github.com> Date: Tue, 7 Oct 2025 22:53:19 -0500 Subject: [PATCH 2/9] feat: move to task from make --- .tasks/ci.yml | 182 ++++++++++++++++++++++++++++++++++++++ .tasks/cloud.yml | 172 ++++++++++++++++++++++++++++++++++++ .tasks/connect.yml | 150 ++++++++++++++++++++++++++++++++ .tasks/console.yml | 39 +++++++++ .tasks/mirror.yml | 185 +++++++++++++++++++++++++++++++++++++++ .tasks/monitor.yml | 42 +++++++++ .tasks/redpanda.yml | 109 +++++++++++++++++++++++ .tasks/test.yml | 207 ++++++++++++++++++++++++++++++++++++++++++++ .tasks/tools.yml | 129 +++++++++++++++++++++++++++ Taskfile.yml | 22 +++++ 10 files changed, 1237 insertions(+) create mode 100644 .tasks/ci.yml create mode 100644 .tasks/cloud.yml create mode 100644 .tasks/connect.yml create mode 100644 .tasks/console.yml create mode 100644 .tasks/mirror.yml create mode 100644 .tasks/monitor.yml create mode 100644 .tasks/redpanda.yml create mode 100644 .tasks/test.yml create mode 100644 .tasks/tools.yml create mode 100644 Taskfile.yml diff --git a/.tasks/ci.yml b/.tasks/ci.yml new file mode 100644 index 00000000..967c1e4d --- /dev/null +++ b/.tasks/ci.yml @@ -0,0 +1,182 @@ +version: '3' + +vars: + ARTIFACT_DIR: "{{.PWD}}/artifacts" + DEPLOYMENT_ID: '{{.DEPLOYMENT_ID | default "devex-cicd"}}' + +env: + ANSIBLE_COLLECTIONS_PATH: "{{.ARTIFACT_DIR}}/collections" + ANSIBLE_ROLES_PATH: "{{.ARTIFACT_DIR}}/roles" + +tasks: + default: + desc: "Show available CI pipelines" + cmds: + - task --list + silent: true + aws:basic: + desc: "AWS: Basic cluster (provision → test → cleanup)" + cmds: + - task: :cloud:aws:up + - defer: { task: ':cloud:aws:down' } + - task: :redpanda:install:basic + - task: :monitor:deploy + - task: :console:deploy + - task: :test:smoke + + aws:tls: + desc: "AWS: TLS-enabled cluster" + cmds: + - echo "Running AWS TLS pipeline..." + - task: :cloud:aws:up + - defer: { task: ':cloud:aws:down' } + - task: :redpanda:install:tls + - task: :monitor:deploy:tls + - task: :console:deploy:tls + - task: :test:tls + - echo "AWS TLS pipeline complete" + + aws:tiered: + desc: "AWS: Tiered storage cluster" + requires: + vars: [REDPANDA_LICENSE] + cmds: + - echo "Running AWS tiered storage pipeline..." + - task: :cloud:aws:up + vars: + TIERED_STORAGE_ENABLED: true + - defer: { task: ':cloud:aws:down' } + - task: :redpanda:install:tiered + - task: :monitor:deploy:tls + - task: :console:deploy:tls + - task: :test:tls + - task: :test:storage:aws + - echo "AWS tiered storage pipeline complete" + + aws:connect: + desc: "AWS: Cluster with Redpanda Connect and multi-cluster mirroring" + requires: + vars: [CONNECT_RPM_TOKEN] + cmds: + - echo "Running AWS Connect pipeline..." + - task: :cloud:aws:up + vars: + ENABLE_CONNECT: true + - defer: { task: ':cloud:aws:down' } + - task: :mirror:prepare:terraform + - task: :mirror:provision + - defer: { task: ':mirror:infra:down' } + - task: :redpanda:install:basic + - task: :connect:setup + - task: :monitor:deploy + - task: :console:deploy + - task: :test:smoke + - task: :connect:mirror-connector + - task: :test:load + - echo "AWS Connect pipeline complete" + + aws:mirror: + desc: "AWS: Full setup (tiered storage + Connect + TLS + mirroring)" + requires: + vars: [REDPANDA_LICENSE, CONNECT_RPM_TOKEN] + cmds: + - echo "Running AWS pipeline..." + - task: :cloud:aws:up + vars: + ENABLE_CONNECT: true + TIERED_STORAGE_ENABLED: true + - defer: { task: ':cloud:aws:down' } + - task: :redpanda:install:tiered + - task: :connect:setup:tls + - task: :monitor:deploy:tls + - task: :console:deploy:tls + - task: :mirror:prepare:terraform + - task: :mirror:provision + - defer: { task: ':mirror:infra:down' } + - task: :test:tls + - task: :test:storage:aws + - task: :connect:test-endpoints + - task: :connect:mirror-connector-tls + - task: :test:load:tls + - echo "AWS pipeline complete" + + + gcp:basic: + desc: "GCP: Basic cluster (provision → test → cleanup)" + requires: + vars: [GCP_CREDS] + cmds: + - echo "Running GCP basic pipeline..." + - task: :cloud:gcp:up + - defer: { task: ':cloud:gcp:down' } + - task: :redpanda:install:basic + - task: :monitor:deploy + - task: :console:deploy + - task: :test:smoke + - echo "GCP basic pipeline complete" + + gcp:tls: + desc: "GCP: TLS-enabled cluster" + requires: + vars: [GCP_CREDS] + cmds: + - echo "Running GCP TLS pipeline..." + - task: :cloud:gcp:up + - defer: { task: ':cloud:gcp:down' } + - task: :redpanda:install:tls + - task: :monitor:deploy:tls + - task: :console:deploy:tls + - task: :test:tls + - echo "GCP TLS pipeline complete" + + gcp:tiered: + desc: "GCP: Tiered storage cluster" + requires: + vars: [REDPANDA_LICENSE, GCP_CREDS] + cmds: + - echo "Running GCP tiered storage pipeline..." + - task: :cloud:gcp:up + vars: + TIERED_STORAGE_ENABLED: true + - defer: { task: ':cloud:gcp:down' } + - task: :redpanda:install:tiered + - task: :monitor:deploy:tls + - task: :console:deploy:tls + - task: :test:tls + - task: :test:storage:gcp + - echo "GCP tiered storage pipeline complete" + + aws:quick: + desc: "AWS: Quick setup (no testing or cleanup)" + cmds: + - echo "Running AWS quick setup..." + - task: :cloud:aws:up + - task: :redpanda:install:basic + - task: :monitor:deploy + - task: :console:deploy + - echo "AWS quick setup complete (cluster running)" + + gcp:quick: + desc: "GCP: Quick setup (no testing or cleanup)" + cmds: + - echo "Running GCP quick setup..." + - task: :cloud:gcp:up + - task: :redpanda:install:basic + - task: :monitor:deploy + - task: :console:deploy + - echo "GCP quick setup complete (cluster running)" + + + lint: + desc: "Run ansible-lint on all playbooks" + cmds: + - echo "Running ansible-lint..." + - ansible-lint -c .ansible-lint + - echo "Linting complete" + + yamllint: + desc: "Run yamllint on all YAML files" + cmds: + - echo "Running yamllint..." + - yamllint -c .yamllint ansible/ .tasks/ *.yml + - echo "YAML linting complete" diff --git a/.tasks/cloud.yml b/.tasks/cloud.yml new file mode 100644 index 00000000..af490d72 --- /dev/null +++ b/.tasks/cloud.yml @@ -0,0 +1,172 @@ +version: '3' + +vars: + DEPLOYMENT_ID: '{{.DEPLOYMENT_ID | default "devex-cicd"}}' + NUM_NODES: '{{.NUM_NODES | default "3"}}' + ENABLE_MONITORING: '{{.ENABLE_MONITORING | default "true"}}' + TIERED_STORAGE_ENABLED: '{{.TIERED_STORAGE_ENABLED | default "false"}}' + ALLOW_FORCE_DESTROY: '{{.ALLOW_FORCE_DESTROY | default "true"}}' + VPC_ID: '{{.VPC_ID | default ""}}' + DISTRO: '{{.DISTRO | default "ubuntu-focal"}}' + ARTIFACT_DIR: "{{.PWD}}/artifacts" + PUBLIC_KEY: "{{.ARTIFACT_DIR}}/testkey.pub" + ANSIBLE_INVENTORY: "{{.ARTIFACT_DIR}}/hosts_{{.DEPLOYMENT_ID}}.ini" + MACHINE_ARCH: '{{.MACHINE_ARCH | default "x86_64"}}' + ENABLE_CONNECT: '{{.ENABLE_CONNECT | default "false"}}' + INSTANCE_TYPE_AWS: '{{.INSTANCE_TYPE_AWS | default "i3.2xlarge"}}' + GCP_IMAGE: '{{.GCP_IMAGE | default "ubuntu-os-cloud/ubuntu-2204-lts"}}' + GCP_INSTANCE_TYPE: '{{.GCP_INSTANCE_TYPE | default "n2-standard-2"}}' + GCP_CREDS: "{{.GCP_CREDS}}" + TF_DIR: '{{.TF_DIR | default ""}}' + +tasks: + default: + desc: "Show cloud infrastructure commands" + cmds: + - task --list + silent: true + + aws:up: + desc: "Provision AWS infrastructure" + deps: [":tools:ssh-keys"] + dir: aws + cmds: + - echo "Provisioning AWS infrastructure..." + - echo " Deployment {{.DEPLOYMENT_ID}}" + - echo " Nodes {{.NUM_NODES}}" + - echo " Tiered Storage {{.TIERED_STORAGE_ENABLED}}" + - terraform init + - | + terraform apply -auto-approve \ + -var='deployment_prefix={{.DEPLOYMENT_ID}}' \ + -var='public_key_path={{.PUBLIC_KEY}}' \ + -var='broker_count={{.NUM_NODES}}' \ + -var='enable_monitoring={{.ENABLE_MONITORING}}' \ + -var='tiered_storage_enabled={{.TIERED_STORAGE_ENABLED}}' \ + -var='allow_force_destroy={{.ALLOW_FORCE_DESTROY}}' \ + -var='vpc_id={{.VPC_ID}}' \ + -var='distro={{.DISTRO}}' \ + -var='hosts_file={{.ANSIBLE_INVENTORY}}' \ + -var='machine_architecture={{.MACHINE_ARCH}}' \ + -var='enable_connect={{.ENABLE_CONNECT}}' \ + -var='broker_instance_type={{.INSTANCE_TYPE_AWS}}' \ + -var='client_instance_type={{.INSTANCE_TYPE_AWS}}' \ + -var='prometheus_instance_type={{.INSTANCE_TYPE_AWS}}' + - echo "AWS infrastructure ready" + + aws:down: + desc: "Destroy AWS infrastructure" + deps: [":tools:ssh-keys"] + dir: aws + cmds: + - echo "Destroying AWS infrastructure..." + - terraform init + - | + terraform destroy -auto-approve \ + -var='deployment_prefix={{.DEPLOYMENT_ID}}' \ + -var='public_key_path={{.PUBLIC_KEY}}' \ + -var='broker_count={{.NUM_NODES}}' \ + -var='enable_monitoring={{.ENABLE_MONITORING}}' \ + -var='tiered_storage_enabled={{.TIERED_STORAGE_ENABLED}}' \ + -var='allow_force_destroy={{.ALLOW_FORCE_DESTROY}}' \ + -var='vpc_id={{.VPC_ID}}' \ + -var='distro={{.DISTRO}}' \ + -var='hosts_file={{.ANSIBLE_INVENTORY}}' \ + -var='machine_architecture={{.MACHINE_ARCH}}' \ + -var='enable_connect={{.ENABLE_CONNECT}}' \ + -var='broker_instance_type={{.INSTANCE_TYPE_AWS}}' \ + -var='client_instance_type={{.INSTANCE_TYPE_AWS}}' \ + -var='prometheus_instance_type={{.INSTANCE_TYPE_AWS}}' + - echo "AWS infrastructure destroyed" + + aws:proxy-up: + desc: "Provision AWS infrastructure with proxy (for private network testing)" + deps: [":tools:ssh-keys"] + dir: aws/private-test + cmds: + - echo "Provisioning AWS infrastructure with proxy..." + - terraform init + - | + terraform apply -auto-approve \ + -var='deployment_prefix={{.DEPLOYMENT_ID}}' \ + -var='public_key_path={{.PUBLIC_KEY}}' \ + -var='broker_count={{.NUM_NODES}}' \ + -var='enable_monitoring={{.ENABLE_MONITORING}}' \ + -var='tiered_storage_enabled={{.TIERED_STORAGE_ENABLED}}' \ + -var='allow_force_destroy={{.ALLOW_FORCE_DESTROY}}' \ + -var='vpc_id={{.VPC_ID}}' \ + -var='distro={{.DISTRO}}' \ + -var='hosts_file={{.ANSIBLE_INVENTORY}}' \ + -var='machine_architecture={{.MACHINE_ARCH}}' \ + -var='enable_connect={{.ENABLE_CONNECT}}' \ + -var='broker_instance_type={{.INSTANCE_TYPE_AWS}}' \ + -var='client_instance_type={{.INSTANCE_TYPE_AWS}}' \ + -var='prometheus_instance_type={{.INSTANCE_TYPE_AWS}}' + - echo "AWS proxy infrastructure ready" + + aws:proxy-down: + desc: "Destroy AWS proxy infrastructure" + deps: [":tools:ssh-keys"] + dir: aws/private-test + cmds: + - echo "Destroying AWS proxy infrastructure..." + - terraform init + - | + terraform destroy -auto-approve \ + -var='deployment_prefix={{.DEPLOYMENT_ID}}' \ + -var='public_key_path={{.PUBLIC_KEY}}' \ + -var='broker_count={{.NUM_NODES}}' \ + -var='enable_monitoring={{.ENABLE_MONITORING}}' \ + -var='tiered_storage_enabled={{.TIERED_STORAGE_ENABLED}}' \ + -var='allow_force_destroy={{.ALLOW_FORCE_DESTROY}}' \ + -var='vpc_id={{.VPC_ID}}' \ + -var='distro={{.DISTRO}}' \ + -var='hosts_file={{.ANSIBLE_INVENTORY}}' \ + -var='machine_architecture={{.MACHINE_ARCH}}' \ + -var='enable_connect={{.ENABLE_CONNECT}}' \ + -var='broker_instance_type={{.INSTANCE_TYPE_AWS}}' \ + -var='client_instance_type={{.INSTANCE_TYPE_AWS}}' \ + -var='prometheus_instance_type={{.INSTANCE_TYPE_AWS}}' + - echo "AWS proxy infrastructure destroyed" + + gcp:up: + desc: "Provision GCP infrastructure" + deps: [":tools:ssh-keys"] + dir: gcp + cmds: + - echo "Provisioning GCP infrastructure..." + - echo " Deployment {{.DEPLOYMENT_ID}}" + - echo " Nodes {{.NUM_NODES}}" + - terraform init + - | + terraform apply -auto-approve \ + -var='deployment_prefix={{.DEPLOYMENT_ID}}' \ + -var='public_key_path={{.PUBLIC_KEY}}' \ + -var='broker_count={{.NUM_NODES}}' \ + -var='enable_monitoring={{.ENABLE_MONITORING}}' \ + -var='tiered_storage_enabled={{.TIERED_STORAGE_ENABLED}}' \ + -var='image={{.GCP_IMAGE}}' \ + -var='hosts_file={{.ANSIBLE_INVENTORY}}' \ + -var='machine_type={{.GCP_INSTANCE_TYPE}}' \ + -var='gcp_creds={{.GCP_CREDS}}' + - echo "GCP infrastructure ready" + + gcp:down: + desc: "Destroy GCP infrastructure" + deps: [":tools:ssh-keys"] + dir: gcp + cmds: + - echo "Destroying GCP infrastructure..." + - terraform init + - | + terraform destroy -auto-approve \ + -var='deployment_prefix={{.DEPLOYMENT_ID}}' \ + -var='public_key_path={{.PUBLIC_KEY}}' \ + -var='broker_count={{.NUM_NODES}}' \ + -var='enable_monitoring={{.ENABLE_MONITORING}}' \ + -var='tiered_storage_enabled={{.TIERED_STORAGE_ENABLED}}' \ + -var='image={{.GCP_IMAGE}}' \ + -var='hosts_file={{.ANSIBLE_INVENTORY}}' \ + -var='machine_type={{.GCP_INSTANCE_TYPE}}' \ + -var='gcp_creds={{.GCP_CREDS}}' + - echo "GCP infrastructure destroyed" diff --git a/.tasks/connect.yml b/.tasks/connect.yml new file mode 100644 index 00000000..2c3f0ed8 --- /dev/null +++ b/.tasks/connect.yml @@ -0,0 +1,150 @@ +version: '3' + +vars: + ARTIFACT_DIR: "{{.PWD}}/artifacts" + DEPLOYMENT_ID: '{{.DEPLOYMENT_ID | default "devex-cicd"}}' + RPM_VERSION: '{{.RPM_VERSION | default "v1.0.0-7ae9d19"}}' + SERVER_DIR: '{{.SERVER_DIR | default "/tmp"}}' + LOCAL_FILE: "{{.ARTIFACT_DIR}}/redpanda-connect.x86_64.rpm" + TOKEN: + sh: echo "${CONNECT_RPM_TOKEN}" + DL_LINK: "https://dl.redpanda.com/{{.TOKEN}}/connectors-artifacts/raw/names/redpanda-connectors/versions/{{.RPM_VERSION}}/redpanda-connectors-{{.RPM_VERSION}}.x86_64.rpm" + HOSTS_FILE: "{{.ARTIFACT_DIR}}/hosts_{{.DEPLOYMENT_ID}}.ini" + PRIVATE_KEY: "{{.ARTIFACT_DIR}}/testkey" + ANSIBLE_INVENTORY: "{{.ARTIFACT_DIR}}/hosts_{{.DEPLOYMENT_ID}}.ini" + CLIENT_KEY: + sh: echo "${CLIENT_KEY:-{{.PWD}}/ansible/tls/clients/client.key}" + CLIENT_CERT: + sh: echo "${CLIENT_CERT:-{{.PWD}}/ansible/tls/ca/ca.crt}" + EXTRA_INVENTORY: "{{.ARTIFACT_DIR}}/hosts2_{{.DEPLOYMENT_ID}}.ini" + IS_USING_UNSTABLE: '{{.IS_USING_UNSTABLE | default "false"}}' + +env: + OBJC_DISABLE_INITIALIZE_FORK_SAFETY: "YES" + ANSIBLE_LOG_PATH: "{{.ARTIFACT_DIR}}/logs/{{.DEPLOYMENT_ID}}.log" + ANSIBLE_INVENTORY: "{{.ANSIBLE_INVENTORY}}" + ANSIBLE_COLLECTIONS_PATH: "{{.ARTIFACT_DIR}}/collections" + ANSIBLE_ROLES_PATH: "{{.ARTIFACT_DIR}}/roles" + +tasks: + default: + desc: "Show Redpanda Connect commands" + cmds: + - task --list + silent: true + + setup: + desc: "Download, copy, and deploy Redpanda Connect" + cmds: + - task: _download-rpm + - task: _copy-rpm + - task: _deploy-ansible + + setup:tls: + desc: "Download, copy, and deploy Redpanda Connect with TLS" + cmds: + - task: _download-rpm + - task: _copy-rpm + - task: _deploy-ansible-tls + + _download-rpm: + internal: true + desc: "Download Redpanda Connect RPM" + cmds: + - | + if [ -f {{.LOCAL_FILE}} ]; then + echo "RPM already exists at {{.LOCAL_FILE}}" + exit 0 + fi + + echo "Downloading Redpanda Connect RPM..." + echo "URL: {{.DL_LINK}}" + echo "Target: {{.LOCAL_FILE}}" + echo "Token length: ${#CONNECT_RPM_TOKEN}" + + mkdir -p "$(dirname {{.LOCAL_FILE}})" + + if curl -f -L --progress-bar -o {{.LOCAL_FILE}} "{{.DL_LINK}}"; then + echo "RPM downloaded successfully" + ls -lh {{.LOCAL_FILE}} + else + echo "ERROR: Failed to download RPM" + echo "URL was: {{.DL_LINK}}" + echo "Curl exit code: $?" + exit 1 + fi + + _copy-rpm: + internal: true + desc: "Copy Redpanda Connect RPM to connect hosts" + deps: [":tools:ssh-keys"] + cmds: + - echo "Copying RPM to connect hosts..." + - | + IPS_USERS=$(awk '/^\[connect\]/{f=1; next} /^\[/{f=0} f && /^[0-9]/{split($2,a,"="); print a[2] "@" $1}' {{.HOSTS_FILE}}) + for IP_USER in $IPS_USERS; do + scp -o StrictHostKeyChecking=no -i "{{.PRIVATE_KEY}}" "{{.LOCAL_FILE}}" "$IP_USER:{{.SERVER_DIR}}" + done + - echo "RPM copied" + + _deploy-ansible: + internal: true + deps: [":tools:ansible"] + env: + ENABLE_CONNECT: "true" + cmds: + - mkdir -p {{.ARTIFACT_DIR}}/logs + - echo "Deploying Redpanda Connect..." + - ansible-playbook ansible/deploy-connect.yml --private-key {{.PRIVATE_KEY}} --inventory {{.ANSIBLE_INVENTORY}} --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} + - echo "Connect deployed" + + _deploy-ansible-tls: + internal: true + deps: [":tools:ansible"] + env: + ENABLE_CONNECT: "true" + cmds: + - mkdir -p {{.ARTIFACT_DIR}}/logs + - echo "Deploying Redpanda Connect with TLS..." + - ansible-playbook ansible/deploy-connect-tls.yml --private-key {{.PRIVATE_KEY}} --inventory {{.ANSIBLE_INVENTORY}} --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} + - echo "Connect with TLS deployed" + + test-endpoints: + desc: "Test Connect REST API endpoints with TLS" + deps: [":tools:tls-certs"] + cmds: + - | + CONNECT_TARGET=$(awk '/^\[connect\]/{f=1; next} /^$/{f=0} f{print $1}' "{{.HOSTS_FILE}}" | head -n1) + echo "Testing Connect API at https://$CONNECT_TARGET:8083..." + curl -vvvvv -k --cert {{.PWD}}/ansible/tls/clients/client.crt --key {{.CLIENT_KEY}} --cacert {{.CLIENT_CERT}} -X GET https://$CONNECT_TARGET:8083/connectors + + test-prometheus-exporter: + desc: "Test Prometheus metrics endpoint" + deps: [":tools:tls-certs"] + cmds: + - | + PROMETHEUS_EXPORTER_TARGET=$(awk '/^\[connect\]/{f=1; next} /^$/{f=0} f{print $1}' "{{.HOSTS_FILE}}" | head -n1) + echo "Testing Prometheus exporter at https://$PROMETHEUS_EXPORTER_TARGET:9404..." + curl -vvvvv -k --cert {{.PWD}}/ansible/tls/clients/client.crt --key {{.PWD}}/ansible/tls/clients/client.key --cacert {{.PWD}}/ansible/tls/ca/ca.crt -X GET https://$PROMETHEUS_EXPORTER_TARGET:9404/metrics + + mirror-connector: + desc: "Create mirror source connector (plaintext)" + cmds: + - | + REDPANDA_BROKERS=$(awk '/^\[redpanda\]/{f=1; next} /^$/{f=0} f{print $1":9092"}' "{{.HOSTS_FILE}}" | paste -sd ',' -) + EXTRA_BROKERS=$(awk '/^\[redpanda\]/{f=1; next} /^$/{f=0} f{print $1":9092"}' "{{.EXTRA_INVENTORY}}" | paste -sd ',' -) + CONNECT_IP=$(awk '/^\[connect\]/{f=1; next} f{print $1; exit}' {{.HOSTS_FILE}}) + echo "Creating mirror source connector..." + curl -X POST -H 'Content-Type: application/json' -H 'accept: application/json' http://$CONNECT_IP:8083/connectors -d '{"name": "mirror-source-connector","config": {"connector.class": "org.apache.kafka.connect.mirror.MirrorSourceConnector","topics": "testtopic","replication.factor": "1","source.cluster.bootstrap.servers": "'"$REDPANDA_BROKERS"'","source.cluster.security.protocol": "PLAINTEXT","target.cluster.bootstrap.servers": "'"$EXTRA_BROKERS"'","target.cluster.security.protocol": "PLAINTEXT","source.cluster.alias": "source" }}' + echo "Connector created" + + mirror-connector-tls: + desc: "Create mirror source connector with TLS" + cmds: + - | + REDPANDA_BROKERS=$(awk '/^\[redpanda\]/{f=1; next} /^$/{f=0} f{print $1":9092"}' "{{.HOSTS_FILE}}" | paste -sd ',' -) + EXTRA_BROKERS=$(awk '/^\[redpanda\]/{f=1; next} /^$/{f=0} f{print $1":9092"}' "{{.EXTRA_INVENTORY}}" | paste -sd ',' -) + CONNECT_IP=$(awk '/^\[connect\]/{f=1; next} f{print $1; exit}' {{.HOSTS_FILE}}) + echo "Creating mirror source connector with TLS..." + curl -X POST -H 'Content-Type: application/json' -H 'accept: application/json' --key {{.CLIENT_KEY}} --cacert {{.CLIENT_CERT}} https://$CONNECT_IP:8083/connectors -d '{"name": "mirror-source-connector", "config": {"connector.class": "org.apache.kafka.connect.mirror.MirrorSourceConnector", "topics": "testtopic", "replication.factor": "1", "source.cluster.bootstrap.servers": "'"$REDPANDA_BROKERS"'", "source.cluster.security.protocol": "SSL", "source.cluster.ssl.truststore.type": "PKCS12", "source.cluster.ssl.keystore.type": "PKCS12", "target.cluster.bootstrap.servers": "'"$EXTRA_BROKERS"'", "target.cluster.security.protocol": "SSL", "source.cluster.alias": "source", "target.cluster.ssl.truststore.type": "PKCS12", "target.cluster.ssl.keystore.type": "PKCS12"}}' + echo "Connector with TLS created" diff --git a/.tasks/console.yml b/.tasks/console.yml new file mode 100644 index 00000000..9c3e009d --- /dev/null +++ b/.tasks/console.yml @@ -0,0 +1,39 @@ +version: '3' + +vars: + ARTIFACT_DIR: "{{.PWD}}/artifacts" + DEPLOYMENT_ID: '{{.DEPLOYMENT_ID | default "devex-cicd"}}' + PRIVATE_KEY: "{{.ARTIFACT_DIR}}/testkey" + ANSIBLE_INVENTORY: "{{.ARTIFACT_DIR}}/hosts_{{.DEPLOYMENT_ID}}.ini" + IS_USING_UNSTABLE: '{{.IS_USING_UNSTABLE | default "false"}}' + +env: + OBJC_DISABLE_INITIALIZE_FORK_SAFETY: "YES" + ANSIBLE_LOG_PATH: "{{.ARTIFACT_DIR}}/logs/{{.DEPLOYMENT_ID}}.log" + ANSIBLE_INVENTORY: "{{.ANSIBLE_INVENTORY}}" + ANSIBLE_COLLECTIONS_PATH: "{{.ARTIFACT_DIR}}/collections" + ANSIBLE_ROLES_PATH: "{{.ARTIFACT_DIR}}/roles" + +tasks: + default: + desc: "Deploy Redpanda Console" + cmds: + - task: deploy + + deploy: + desc: "Deploy Redpanda Console" + deps: [":tools:ansible"] + cmds: + - mkdir -p {{.ARTIFACT_DIR}}/logs + - echo "Deploying Redpanda Console..." + - ansible-playbook ansible/deploy-console.yml --private-key {{.PRIVATE_KEY}} --inventory {{.ANSIBLE_INVENTORY}} --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} + - echo "Console deployed" + + deploy:tls: + desc: "Deploy Redpanda Console with TLS" + deps: [":tools:ansible"] + cmds: + - mkdir -p {{.ARTIFACT_DIR}}/logs + - echo "Deploying Redpanda Console with TLS..." + - ansible-playbook ansible/deploy-console-tls.yml --private-key {{.PRIVATE_KEY}} --inventory {{.ANSIBLE_INVENTORY}} --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} + - echo "Console with TLS deployed" diff --git a/.tasks/mirror.yml b/.tasks/mirror.yml new file mode 100644 index 00000000..f37e9790 --- /dev/null +++ b/.tasks/mirror.yml @@ -0,0 +1,185 @@ +version: '3' + +vars: + ARTIFACT_DIR: "{{.PWD}}/artifacts" + DEPLOYMENT_ID: '{{.DEPLOYMENT_ID | default "devex-cicd"}}' + NUM_NODES: '{{.NUM_NODES | default "3"}}' + ENABLE_MONITORING: '{{.ENABLE_MONITORING | default "true"}}' + TIERED_STORAGE_ENABLED: '{{.TIERED_STORAGE_ENABLED | default "false"}}' + ALLOW_FORCE_DESTROY: '{{.ALLOW_FORCE_DESTROY | default "true"}}' + VPC_ID: '{{.VPC_ID | default ""}}' + DISTRO: '{{.DISTRO | default "ubuntu-focal"}}' + PUBLIC_KEY: "{{.ARTIFACT_DIR}}/testkey.pub" + PRIVATE_KEY: "{{.ARTIFACT_DIR}}/testkey" + EXTRA_INVENTORY: "{{.ARTIFACT_DIR}}/hosts2_{{.DEPLOYMENT_ID}}.ini" + MACHINE_ARCH: '{{.MACHINE_ARCH | default "x86_64"}}' + INSTANCE_TYPE_AWS: '{{.INSTANCE_TYPE_AWS | default "i3.2xlarge"}}' + IS_USING_UNSTABLE: '{{.IS_USING_UNSTABLE | default "false"}}' + TF_DIR: '{{.TF_DIR | default ""}}' + LOCAL_FILE: "{{.ARTIFACT_DIR}}/redpanda-connect.x86_64.rpm" + SERVER_DIR: '{{.SERVER_DIR | default "/tmp"}}' + +env: + ANSIBLE_COLLECTIONS_PATH: "{{.ARTIFACT_DIR}}/collections" + ANSIBLE_ROLES_PATH: "{{.ARTIFACT_DIR}}/roles" + +tasks: + default: + desc: "Show mirror cluster commands (for multi-cluster testing)" + cmds: + - task --list + silent: true + + prepare:terraform: + desc: "Copy AWS Terraform for secondary deployment" + cmds: + - echo "Preparing Terraform for secondary cluster..." + - | + cp -r aws aws-extra && \ + rm -rf aws-extra/terraform.tfstate && \ + rm -rf aws-extra/terraform.tfstate.backup && \ + rm -rf aws-extra/.terraform && \ + rm -rf aws-extra/.terraform.lock.hcl + - echo "Terraform prepared" + + cleanup:terraform: + desc: "Remove copied Terraform directory" + cmds: + - rm -rf aws-extra + - echo "Terraform cleanup complete" + + provision: + desc: "Provision complete secondary cluster (infra + Redpanda)" + cmds: + - task: infra:up + - task: install + + provision:tls: + desc: "Provision complete secondary cluster with TLS" + cmds: + - task: infra:up + - task: install:tls + - task: deploy:observability:tls + + infra:up: + desc: "Create secondary cluster infrastructure on AWS" + deps: [":tools:ssh-keys"] + dir: aws-extra/{{.TF_DIR}} + cmds: + - echo "Provisioning secondary cluster infrastructure..." + - echo " Deployment {{.DEPLOYMENT_ID}}2" + - terraform init + - | + terraform apply -auto-approve \ + -var='deployment_prefix={{.DEPLOYMENT_ID}}2' \ + -var='public_key_path={{.PUBLIC_KEY}}' \ + -var='broker_count={{.NUM_NODES}}' \ + -var='allow_force_destroy={{.ALLOW_FORCE_DESTROY}}' \ + -var='vpc_id={{.VPC_ID}}' \ + -var='distro={{.DISTRO}}' \ + -var='hosts_file={{.EXTRA_INVENTORY}}' \ + -var='machine_architecture={{.MACHINE_ARCH}}' \ + -var='enable_connect=false' \ + -var='broker_instance_type={{.INSTANCE_TYPE_AWS}}' \ + -var='client_instance_type={{.INSTANCE_TYPE_AWS}}' \ + -var='prometheus_instance_type={{.INSTANCE_TYPE_AWS}}' + - echo "Secondary infrastructure ready" + + infra:down: + desc: "Destroy secondary cluster infrastructure" + deps: [":tools:ssh-keys"] + dir: aws-extra/{{.TF_DIR}} + cmds: + - echo "Destroying secondary cluster infrastructure..." + - terraform init + - | + terraform destroy -auto-approve \ + -var='deployment_prefix={{.DEPLOYMENT_ID}}2' \ + -var='public_key_path={{.PUBLIC_KEY}}' \ + -var='broker_count={{.NUM_NODES}}' \ + -var='enable_monitoring={{.ENABLE_MONITORING}}' \ + -var='tiered_storage_enabled={{.TIERED_STORAGE_ENABLED}}' \ + -var='allow_force_destroy={{.ALLOW_FORCE_DESTROY}}' \ + -var='vpc_id={{.VPC_ID}}' \ + -var='distro={{.DISTRO}}' \ + -var='hosts_file={{.EXTRA_INVENTORY}}' \ + -var='machine_architecture={{.MACHINE_ARCH}}' \ + -var='enable_connect=false' \ + -var='broker_instance_type={{.INSTANCE_TYPE_AWS}}' \ + -var='client_instance_type={{.INSTANCE_TYPE_AWS}}' \ + -var='prometheus_instance_type={{.INSTANCE_TYPE_AWS}}' + - echo "Secondary infrastructure destroyed" + + install: + desc: "Install Redpanda on secondary cluster" + deps: [":tools:ansible"] + cmds: + - mkdir -p {{.ARTIFACT_DIR}}/logs + - echo "Installing Redpanda on secondary cluster..." + - ansible-playbook ansible/provision-cluster.yml --private-key {{.PRIVATE_KEY}} --inventory {{.EXTRA_INVENTORY}} --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} + - echo "Secondary cluster Redpanda installed" + + install:tls: + desc: "Install Redpanda with TLS on secondary cluster" + deps: [":tools:ansible"] + cmds: + - mkdir -p {{.ARTIFACT_DIR}}/logs + - echo "🚀Installing Redpanda with TLS on secondary cluster..." + - ansible-playbook ansible/provision-cluster-tls.yml --private-key {{.PRIVATE_KEY}} --inventory {{.EXTRA_INVENTORY}} --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} + - echo "Secondary cluster Redpanda with TLS installed" + + deploy:observability: + desc: "Deploy monitoring and console on secondary cluster" + cmds: + - task: _monitoring + - task: _console + + deploy:observability:tls: + desc: "Deploy monitoring and console with TLS on secondary cluster" + cmds: + - task: _monitoring-tls + - task: _console-tls + + _monitoring: + internal: true + deps: [":tools:ansible"] + cmds: + - mkdir -p {{.ARTIFACT_DIR}}/logs + - echo "Deploying monitoring on secondary cluster..." + - ansible-playbook ansible/deploy-monitor.yml --private-key {{.PRIVATE_KEY}} --inventory {{.EXTRA_INVENTORY}} --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} + + _monitoring-tls: + internal: true + deps: [":tools:ansible"] + cmds: + - mkdir -p {{.ARTIFACT_DIR}}/logs + - echo "Deploying monitoring with TLS on secondary cluster..." + - ansible-playbook ansible/deploy-monitor-tls.yml --private-key {{.PRIVATE_KEY}} --inventory {{.EXTRA_INVENTORY}} --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} + + _console: + internal: true + deps: [":tools:ansible"] + cmds: + - mkdir -p {{.ARTIFACT_DIR}}/logs + - echo "Deploying console on secondary cluster..." + - ansible-playbook ansible/deploy-console.yml --private-key {{.PRIVATE_KEY}} --inventory {{.EXTRA_INVENTORY}} --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} + + _console-tls: + internal: true + deps: [":tools:ansible"] + cmds: + - mkdir -p {{.ARTIFACT_DIR}}/logs + - echo "Deploying console with TLS on secondary cluster..." + - ansible-playbook ansible/deploy-console-tls.yml --private-key {{.PRIVATE_KEY}} --inventory {{.EXTRA_INVENTORY}} --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} + + copy:connect-rpm: + desc: "Copy Redpanda Connect RPM to secondary cluster" + deps: [":tools:ssh-keys"] + cmds: + - echo "Copying Connect RPM to secondary cluster..." + - | + IPS_USERS=$(awk '/^\[connect\]/{f=1; next} /^\[/{f=0} f && /^[0-9]/{split($2,a,"="); print a[2] "@" $1}' {{.EXTRA_INVENTORY}}) + for IP_USER in $IPS_USERS; do + scp -o StrictHostKeyChecking=no -i "{{.PRIVATE_KEY}}" "{{.LOCAL_FILE}}" "$IP_USER:{{.SERVER_DIR}}" + done + - echo "RPM copied to secondary cluster" diff --git a/.tasks/monitor.yml b/.tasks/monitor.yml new file mode 100644 index 00000000..46ab1403 --- /dev/null +++ b/.tasks/monitor.yml @@ -0,0 +1,42 @@ +version: '3' + +vars: + ARTIFACT_DIR: "{{.PWD}}/artifacts" + DEPLOYMENT_ID: '{{.DEPLOYMENT_ID | default "devex-cicd"}}' + PRIVATE_KEY: "{{.ARTIFACT_DIR}}/testkey" + ANSIBLE_INVENTORY: "{{.ARTIFACT_DIR}}/hosts_{{.DEPLOYMENT_ID}}.ini" + IS_USING_UNSTABLE: '{{.IS_USING_UNSTABLE | default "false"}}' + DISABLE_GRAFANA_GPG_CHECK: '{{.DISABLE_GRAFANA_GPG_CHECK | default "false"}}' + +env: + OBJC_DISABLE_INITIALIZE_FORK_SAFETY: "YES" + ANSIBLE_LOG_PATH: "{{.ARTIFACT_DIR}}/logs/{{.DEPLOYMENT_ID}}.log" + ANSIBLE_INVENTORY: "{{.ANSIBLE_INVENTORY}}" + ANSIBLE_COLLECTIONS_PATH: "{{.ARTIFACT_DIR}}/collections" + ANSIBLE_ROLES_PATH: "{{.ARTIFACT_DIR}}/roles" + +tasks: + default: + desc: "Deploy monitoring stack (Prometheus, Grafana)" + cmds: + - task: deploy + + deploy: + desc: "Deploy monitoring stack (Prometheus, Grafana)" + deps: [":tools:ansible"] + env: + OBJC_DISABLE_INITIALIZE_FORK_SAFETY: "YES" + cmds: + - mkdir -p {{.ARTIFACT_DIR}}/logs + - echo "Deploying monitoring stack..." + - ansible-playbook ansible/deploy-monitor.yml --private-key {{.PRIVATE_KEY}} --inventory {{.ANSIBLE_INVENTORY}} --extra-vars "is_using_unstable={{.IS_USING_UNSTABLE}} disable_grafana_gpg_check={{.DISABLE_GRAFANA_GPG_CHECK}}" + - echo "Monitoring deployed" + + deploy:tls: + desc: "Deploy monitoring stack with TLS" + deps: [":tools:ansible"] + cmds: + - mkdir -p {{.ARTIFACT_DIR}}/logs + - echo "Deploying monitoring stack with TLS..." + - ansible-playbook ansible/deploy-monitor-tls.yml --private-key {{.PRIVATE_KEY}} --inventory {{.ANSIBLE_INVENTORY}} --extra-vars "is_using_unstable={{.IS_USING_UNSTABLE}} disable_grafana_gpg_check={{.DISABLE_GRAFANA_GPG_CHECK}}" + - echo "Monitoring with TLS deployed" diff --git a/.tasks/redpanda.yml b/.tasks/redpanda.yml new file mode 100644 index 00000000..e0500d82 --- /dev/null +++ b/.tasks/redpanda.yml @@ -0,0 +1,109 @@ +version: '3' + +vars: + ARTIFACT_DIR: "{{.PWD}}/artifacts" + DEPLOYMENT_ID: '{{.DEPLOYMENT_ID | default "devex-cicd"}}' + PRIVATE_KEY: "{{.ARTIFACT_DIR}}/testkey" + ANSIBLE_INVENTORY: "{{.ARTIFACT_DIR}}/hosts_{{.DEPLOYMENT_ID}}.ini" + IS_USING_UNSTABLE: '{{.IS_USING_UNSTABLE | default "false"}}' + SEGMENT_UPLOAD_INTERVAL: '{{.SEGMENT_UPLOAD_INTERVAL | default "1"}}' + CLOUD_STORAGE_CREDENTIALS_SOURCE: '{{.CLOUD_STORAGE_CREDENTIALS_SOURCE | default "aws_instance_metadata"}}' + SQUID_ACL_LOCALNET: '{{.SQUID_ACL_LOCALNET | default ""}}' + SKIP_TAGS: '{{.SKIP_TAGS | default ""}}' + CLI_ARGS: '{{.CLI_ARGS | default ""}}' + +env: + OBJC_DISABLE_INITIALIZE_FORK_SAFETY: "YES" + ANSIBLE_LOG_PATH: "{{.ARTIFACT_DIR}}/logs/{{.DEPLOYMENT_ID}}.log" + ANSIBLE_INVENTORY: "{{.ANSIBLE_INVENTORY}}" + ANSIBLE_COLLECTIONS_PATH: "{{.ARTIFACT_DIR}}/collections" + ANSIBLE_ROLES_PATH: "{{.ARTIFACT_DIR}}/roles" + +tasks: + default: + desc: "Show Redpanda installation commands" + cmds: + - task --list + silent: true + + install:basic: + desc: "Install Redpanda cluster (standard configuration)" + deps: [":tools:ansible"] + cmds: + - mkdir -p {{.ARTIFACT_DIR}}/logs + - echo "Installing Redpanda cluster..." + - ansible-playbook ansible/provision-cluster.yml --private-key {{.PRIVATE_KEY}} --inventory {{.ANSIBLE_INVENTORY}} --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} + - echo "Redpanda cluster installed" + + install:tls: + desc: "Install Redpanda cluster with TLS enabled" + deps: [":tools:ansible"] + cmds: + - mkdir -p {{.ARTIFACT_DIR}}/logs + - echo "Installing Redpanda cluster with TLS..." + - ansible-playbook ansible/provision-cluster-tls.yml --private-key {{.PRIVATE_KEY}} --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} + - echo "Redpanda cluster with TLS installed" + + install:tiered: + desc: "Install Redpanda cluster with tiered storage" + deps: [":tools:ansible"] + requires: + vars: [REDPANDA_LICENSE] + cmds: + - mkdir -p {{.ARTIFACT_DIR}}/logs + - echo "Installing Redpanda cluster with tiered storage..." + - | + ansible-playbook ansible/provision-cluster-tiered-storage.yml \ + --private-key {{.PRIVATE_KEY}} \ + --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} \ + --extra-vars segment_upload_interval={{.SEGMENT_UPLOAD_INTERVAL}} \ + --extra-vars cloud_storage_credentials_source={{.CLOUD_STORAGE_CREDENTIALS_SOURCE}} \ + --extra-vars redpanda_license="${REDPANDA_LICENSE}" + - echo "Redpanda cluster with tiered storage installed" + + install:proxy: + desc: "Install Redpanda cluster behind proxy (for private network testing)" + deps: [":tools:ansible"] + cmds: + - mkdir -p {{.ARTIFACT_DIR}}/logs + - echo "Installing Redpanda cluster with proxy..." + - | + ansible-playbook ansible/proxy/provision-private-proxied-cluster.yml \ + --private-key {{.PRIVATE_KEY}} \ + --inventory {{.ANSIBLE_INVENTORY}} \ + --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} \ + --extra-vars '{"squid_acl_localnet": ["{{.SQUID_ACL_LOCALNET}}"]}' \ + --extra-vars redpanda='{"cluster":{"cloud_storage_segment_max_upload_interval_sec":"{{.SEGMENT_UPLOAD_INTERVAL}}"}}' \ + {{.SKIP_TAGS}} {{.CLI_ARGS}} + - echo "Redpanda proxy cluster installed" + + install:dev-tiered: + desc: "Install development build with tiered storage (verbose logging)" + deps: [":tools:ansible"] + cmds: + - mkdir -p {{.ARTIFACT_DIR}}/logs + - echo "Installing Redpanda dev build with tiered storage..." + - | + ansible-playbook ansible/provision-cluster-tiered-storage.yml \ + --private-key {{.PRIVATE_KEY}} \ + --extra-vars redpanda_broker_no_log=false \ + --extra-vars development_build=true \ + --extra-vars segment_upload_interval={{.SEGMENT_UPLOAD_INTERVAL}} \ + --extra-vars cloud_storage_credentials_source={{.CLOUD_STORAGE_CREDENTIALS_SOURCE}} + - echo "Development cluster with tiered storage installed" + + configure:logging: + desc: "Configure logging settings on Redpanda cluster" + deps: [":tools:ansible"] + vars: + REDPANDA_LOGGING_LOG_FILE: '{{.REDPANDA_LOGGING_LOG_FILE | default "/var/log/redpanda2/redpanda.log"}}' + cmds: + - mkdir -p {{.ARTIFACT_DIR}}/logs + - echo "Configuring Redpanda logging..." + - | + ansible-playbook ansible/operation-configure-logging.yml \ + --private-key {{.PRIVATE_KEY}} \ + --inventory {{.ANSIBLE_INVENTORY}} \ + --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} \ + --extra-vars redpanda_logging_log_file={{.REDPANDA_LOGGING_LOG_FILE}} + - echo "Logging configured" diff --git a/.tasks/test.yml b/.tasks/test.yml new file mode 100644 index 00000000..5d419a82 --- /dev/null +++ b/.tasks/test.yml @@ -0,0 +1,207 @@ +version: '3' + +vars: + ARTIFACT_DIR: "{{.PWD}}/artifacts" + DEPLOYMENT_ID: '{{.DEPLOYMENT_ID | default "devex-cicd"}}' + RPK_PATH: "{{.ARTIFACT_DIR}}/bin/rpk" + HOSTS_FILE: "{{.ARTIFACT_DIR}}/hosts_{{.DEPLOYMENT_ID}}.ini" + TEST_TOPIC_NAME: '{{.TEST_TOPIC_NAME | default "testtopic"}}' + PARTITION_COUNT: '{{.PARTITION_COUNT | default "3"}}' + CA_CRT: + sh: echo "${CA_CRT:-{{.PWD}}/ansible/tls/ca/ca.crt}" + AWS_DEFAULT_REGION: '{{.AWS_DEFAULT_REGION | default "us-west-2"}}' + BUCKET_NAME: + sh: echo "$(echo {{.DEPLOYMENT_ID}} | sed 's/_/-/g')-bucket" + GOOGLE_PROJECT_ID: '{{.GOOGLE_PROJECT_ID | default "hallowed-ray-376320"}}' + GOOGLE_APPLICATION_CREDENTIALS: "/tmp/gcp_creds.json" + GCP_CREDS: "{{.GCP_CREDS}}" + SPAM_MESSAGE_COUNT: '{{.SPAM_MESSAGE_COUNT | default "10"}}' + PRIVATE_KEY: "{{.ARTIFACT_DIR}}/testkey" + SSHKEY: + sh: echo "${SSHKEY:-{{.ARTIFACT_DIR}}/testkey}" + PATH_TO_CA_CRT: '{{.PATH_TO_CA_CRT | default ""}}' + +env: + OBJC_DISABLE_INITIALIZE_FORK_SAFETY: "YES" + ANSIBLE_LOG_PATH: "{{.ARTIFACT_DIR}}/logs/{{.DEPLOYMENT_ID}}.log" + ANSIBLE_INVENTORY: "{{.ANSIBLE_INVENTORY}}" + ANSIBLE_COLLECTIONS_PATH: "{{.ARTIFACT_DIR}}/collections" + ANSIBLE_ROLES_PATH: "{{.ARTIFACT_DIR}}/roles" + +tasks: + default: + desc: "Show test commands" + cmds: + - task --list + silent: true + + smoke: + desc: "Basic cluster health test (status, produce, consume)" + deps: [":tools:rpk"] + cmds: + - echo "Running smoke test..." + - | + REDPANDA_BROKERS=$(awk '/^\[redpanda\]/{f=1; next} /^$/{f=0} f{print $1":9092"}' "{{.HOSTS_FILE}}" | paste -sd ',' -) + echo " Brokers: $REDPANDA_BROKERS" + + echo " Checking cluster status..." + {{.RPK_PATH}} cluster status -X brokers=$REDPANDA_BROKERS -v || exit 1 + + echo " Creating test topic..." + {{.RPK_PATH}} topic create {{.TEST_TOPIC_NAME}} -p {{.PARTITION_COUNT}} -X brokers=$REDPANDA_BROKERS -v || exit 1 + + echo " Producing test message..." + echo squirrel | {{.RPK_PATH}} topic produce {{.TEST_TOPIC_NAME}} -X brokers=$REDPANDA_BROKERS -v || exit 1 + + echo " Consuming test message..." + {{.RPK_PATH}} topic consume {{.TEST_TOPIC_NAME}} -X brokers=$REDPANDA_BROKERS -v -o :end | grep squirrel || exit 1 + - echo "Smoke test passed" + + tls: + desc: "Test cluster with TLS encryption" + deps: [":tools:rpk"] + cmds: + - echo "Running TLS test..." + - | + REDPANDA_BROKERS=$(awk '/^\[redpanda\]/{f=1; next} /^$/{f=0} f{print $1":9092"}' "{{.HOSTS_FILE}}" | paste -sd ',' -) + + echo "Redpanda Brokers: $REDPANDA_BROKERS" + echo "TLS Truststore: {{.CA_CRT}}" + + echo "checking TLS cluster status" + {{.ARTIFACT_DIR}}/bin/rpk cluster status -X brokers="$REDPANDA_BROKERS" -X tls.ca="{{.CA_CRT}}" -v || exit 1 + + echo "creating topic" + {{.ARTIFACT_DIR}}/bin/rpk topic create {{.TEST_TOPIC_NAME}} -p {{.PARTITION_COUNT}} -X brokers="$REDPANDA_BROKERS" -X tls.ca="{{.CA_CRT}}" -v || exit 1 + + echo "producing to topic" + echo squirrels | {{.ARTIFACT_DIR}}/bin/rpk topic produce {{.TEST_TOPIC_NAME}} -X brokers="$REDPANDA_BROKERS" -X tls.ca="{{.CA_CRT}}" -v || exit 1 + + echo "consuming from topic" + {{.ARTIFACT_DIR}}/bin/rpk topic consume {{.TEST_TOPIC_NAME}} -X brokers="$REDPANDA_BROKERS" -X tls.ca="{{.CA_CRT}}" -v -o :end | grep squirrels || exit 1 + - echo "TLS test passed" + + schema: + desc: "Test schema registry (plaintext)" + cmds: + - echo "Testing schema registry..." + - | + REDPANDA_REGISTRY=$(awk '/^\[redpanda\]/{f=1; next} /^$/{f=0} f{print $1}' "{{.HOSTS_FILE}}" | paste -sd ',' - | awk '{gsub(/,/,":8081,"); sub(/,$/,":8081")}1') + echo " Registry endpoints: $REDPANDA_REGISTRY" + for ip_port in $(echo $REDPANDA_REGISTRY | tr ',' ' '); do + echo " Testing $ip_port..." + curl $ip_port/subjects + done + - echo "Schema registry test passed" + + schema:tls: + desc: "Test schema registry with TLS" + cmds: + - echo "Testing schema registry with TLS..." + - | + REDPANDA_REGISTRY=$(sed -n '/^\[redpanda\]/,/^$$/p' "{{.HOSTS_FILE}}" | \ + grep 'private_ip=' | \ + cut -d' ' -f1 | \ + sed 's/$/:8081/' | \ + tr '\n' ',' | \ + sed 's/,$/\n/') + echo " Registry endpoints: $REDPANDA_REGISTRY" + for ip_port in $(echo $REDPANDA_REGISTRY | tr ',' ' '); do \ + echo " Testing $ip_port..." + curl $ip_port/subjects -k --cacert "{{.CA_CRT}}"; \ + done + - echo "Schema registry TLS test passed" + + storage:aws: + desc: "Verify data uploaded to AWS S3 tiered storage" + cmds: + - echo "Testing AWS S3 tiered storage..." + - | + echo " Bucket: {{.BUCKET_NAME}}" + echo " Region: {{.AWS_DEFAULT_REGION}}" + count=$(aws s3api list-objects-v2 --bucket "{{.BUCKET_NAME}}" --region {{.AWS_DEFAULT_REGION}} --query "Contents[?contains(Key, 'testtopic/')]" --output text | wc -l | xargs) + if [ "$count" -ge 1 ]; then + echo " Found $count objects in tiered storage" + exit 0 + else + echo " No testtopic data found in tiered storage" + exit 1 + fi + - echo "AWS tiered storage test passed" + + storage:gcp: + desc: "Verify data uploaded to GCP Storage tiered storage" + cmds: + - echo "Testing GCP Storage tiered storage..." + - | + echo "{{.GCP_CREDS}}" | base64 -d > /tmp/gcp_creds.json + export CLOUDSDK_CORE_PROJECT={{.GOOGLE_PROJECT_ID}} + gcloud auth activate-service-account --key-file={{.GOOGLE_APPLICATION_CREDENTIALS}} --project={{.GOOGLE_PROJECT_ID}} + SIMPLE_BUCKET_NAME=$(echo {{.BUCKET_NAME}} | sed 's/-bucket$//') + echo " Bucket pattern: $SIMPLE_BUCKET_NAME" + gcloud storage --project {{.GOOGLE_PROJECT_ID}} ls | grep $SIMPLE_BUCKET_NAME + - echo "GCP tiered storage test passed" + + proxy: + desc: "Test cluster connectivity through proxy" + deps: [":tools:ssh-keys"] + cmds: + - echo "Testing cluster through proxy..." + - | + REDPANDA_BROKERS=$(sed -n '/^\[redpanda\]/,/^$$/p' "{{.HOSTS_FILE}}" | \ + grep 'private_ip=' | \ + cut -d' ' -f1 | \ + sed 's/$/:9092/' | \ + tr '\n' ',' | \ + sed 's/,$/\n/') + CLIENT_SSH_USER=$(sed -n '/\[redpanda\]/,/\[/p' "{{.HOSTS_FILE}}" | \ + grep ansible_user | \ + head -n1 | \ + tr ' ' '\n' | \ + grep ansible_user | \ + cut -d'=' -f2) + CLIENT_PUBLIC_IP=$(sed -n '/^\[client\]/,/^$$/p' "{{.HOSTS_FILE}}" | \ + grep 'private_ip=' | \ + cut -f1 -d' ') + + echo " Checking cluster status..." + ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 -i "{{.SSHKEY}}" "$CLIENT_SSH_USER@$CLIENT_PUBLIC_IP" "rpk cluster status -X brokers=\"$REDPANDA_BROKERS\" -X tls.ca=\"{{.PATH_TO_CA_CRT}}\" -v" || exit 1 + + echo " Creating topic..." + ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 -i "{{.SSHKEY}}" "$CLIENT_SSH_USER@$CLIENT_PUBLIC_IP" "rpk topic create {{.TEST_TOPIC_NAME}} -p {{.PARTITION_COUNT}} -X brokers=\"$REDPANDA_BROKERS\" -X tls.ca=\"{{.PATH_TO_CA_CRT}}\" -v" || exit 1 + + echo " Producing to topic..." + ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 -i "{{.SSHKEY}}" "$CLIENT_SSH_USER@$CLIENT_PUBLIC_IP" "echo squirrels | rpk topic produce {{.TEST_TOPIC_NAME}} -X brokers=\"$REDPANDA_BROKERS\" -X tls.ca=\"{{.PATH_TO_CA_CRT}}\" -v" || exit 1 + + sleep 30 + + echo " Consuming from topic..." + testoutput=$(ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 -i "{{.SSHKEY}}" "$CLIENT_SSH_USER@$CLIENT_PUBLIC_IP" "rpk topic consume {{.TEST_TOPIC_NAME}} -X brokers=\"$REDPANDA_BROKERS\" -X tls.ca=\"{{.PATH_TO_CA_CRT}}\" -v -o :end") + echo "$testoutput" | grep squirrels || exit 1 + - echo "Proxy test passed" + + load: + desc: "Produce multiple messages to test topic (load testing)" + deps: [":tools:rpk"] + cmds: + - echo "Running load test ({{.SPAM_MESSAGE_COUNT}} messages)..." + - | + REDPANDA_BROKERS=$(awk '/^\[redpanda\]/{f=1; next} /^$/{f=0} f{print $1":9092"}' "{{.HOSTS_FILE}}" | paste -sd ',' -) + echo " Producing {{.SPAM_MESSAGE_COUNT}} messages..." + for i in $(seq 1 {{.SPAM_MESSAGE_COUNT}}); do + echo "squirrel$i" | {{.RPK_PATH}} topic produce {{.TEST_TOPIC_NAME}} -X brokers=$REDPANDA_BROKERS -v || exit 1 + done + - echo "Load test completed" + + load:tls: + desc: "Produce multiple messages with TLS (load testing)" + deps: [":tools:rpk"] + cmds: + - echo "Running TLS load test ({{.SPAM_MESSAGE_COUNT}} messages)..." + - | + REDPANDA_BROKERS=$(awk '/^\[redpanda\]/{f=1; next} /^$/{f=0} f{print $1":9092"}' "{{.HOSTS_FILE}}" | paste -sd ',' -) + echo " Producing {{.SPAM_MESSAGE_COUNT}} messages..." + for i in $(seq 1 {{.SPAM_MESSAGE_COUNT}}); do + echo "squirrel$i" | {{.RPK_PATH}} topic produce {{.TEST_TOPIC_NAME}} -X brokers=$REDPANDA_BROKERS -X tls.ca="{{.CA_CRT}}" -v || exit 1 + done + - echo "TLS load test completed" diff --git a/.tasks/tools.yml b/.tasks/tools.yml new file mode 100644 index 00000000..86e74455 --- /dev/null +++ b/.tasks/tools.yml @@ -0,0 +1,129 @@ +version: '3' + +vars: + ARTIFACT_DIR: "{{.PWD}}/artifacts" + ANSIBLE_COLLECTIONS_PATH: "{{.ARTIFACT_DIR}}/collections" + ANSIBLE_ROLES_PATH: "{{.ARTIFACT_DIR}}/roles" + MAC_RPK: "https://github.com/redpanda-data/redpanda/releases/latest/download/rpk-darwin-amd64.zip" + LINUX_RPK: "https://github.com/redpanda-data/redpanda/releases/latest/download/rpk-linux-amd64.zip" + SSH_EMAIL: '{{.SSH_EMAIL | default "test@test.com"}}' + CLIENT_NAME: '{{.CLIENT_NAME | default "client"}}' + CLIENT_DIR: "{{.PWD}}/ansible/tls/clients" + CA_DIR: "{{.PWD}}/ansible/tls/ca" + CERT_DIR: "{{.PWD}}/ansible/tls/certs" + +tasks: + default: + desc: "Install all tools" + cmds: + - task: ansible + - task: rpk + - task: ssh-keys + + ansible: + desc: "Install Ansible collections and roles" + cmds: + - task: _ansible-collections + - task: _ansible-roles + - echo "Ansible dependencies installed" + + _ansible-collections: + internal: true + desc: "Install Ansible collections from requirements.yml" + cmds: + - mkdir -p {{.ANSIBLE_COLLECTIONS_PATH}} + - ansible-galaxy collection install -r {{.PWD}}/requirements.yml --force -p {{.ANSIBLE_COLLECTIONS_PATH}} + status: + - test -d {{.ANSIBLE_COLLECTIONS_PATH}}/ansible_collections + + _ansible-roles: + internal: true + desc: "Install Ansible roles from requirements.yml" + cmds: + - mkdir -p {{.ANSIBLE_ROLES_PATH}} + - ansible-galaxy role install -r {{.PWD}}/requirements.yml --force -p {{.ANSIBLE_ROLES_PATH}} + status: + - test -d {{.ANSIBLE_ROLES_PATH}} + + rpk: + desc: "Install rpk CLI tool" + cmds: + - mkdir -p {{.ARTIFACT_DIR}}/tmp + - mkdir -p {{.ARTIFACT_DIR}}/bin + - | + OS=$(uname) + if [ "$OS" = "Darwin" ]; then + curl -L {{.MAC_RPK}} -o {{.ARTIFACT_DIR}}/tmp/rpk.zip + else + curl -L {{.LINUX_RPK}} -o {{.ARTIFACT_DIR}}/tmp/rpk.zip + fi + - unzip -o {{.ARTIFACT_DIR}}/tmp/rpk.zip -d {{.ARTIFACT_DIR}}/bin/ + - chmod 755 {{.ARTIFACT_DIR}}/bin/rpk + - rm {{.ARTIFACT_DIR}}/tmp/rpk.zip + - echo "rpk installed to {{.ARTIFACT_DIR}}/bin/rpk" + status: + - test -f {{.ARTIFACT_DIR}}/bin/rpk + + ssh-keys: + desc: "Generate SSH keypair for deployment" + cmds: + - | + if [ ! -f artifacts/testkey ]; then + printf 'y\n' | ssh-keygen -t rsa -b 4096 -C "{{.SSH_EMAIL}}" -N "" -f artifacts/testkey && chmod 0700 artifacts/testkey + echo "SSH keypair generated" + else + echo "SSH keypair already exists" + fi + status: + - test -f artifacts/testkey + + tls-certs: + desc: "Generate client TLS certificates" + cmds: + - task: _cert-client-dir + - task: _cert-client-key + - task: _cert-client-csr + - task: _cert-client-crt + - echo "Client TLS certificates generated" + + _cert-client-dir: + internal: true + cmds: + - mkdir -p {{.CLIENT_DIR}} + status: + - test -d {{.CLIENT_DIR}} + + _cert-client-key: + internal: true + cmds: + - openssl genrsa -out {{.CLIENT_DIR}}/{{.CLIENT_NAME}}.key 2048 + status: + - test -f {{.CLIENT_DIR}}/{{.CLIENT_NAME}}.key + + _cert-client-csr: + internal: true + cmds: + - openssl req -new -key {{.CLIENT_DIR}}/{{.CLIENT_NAME}}.key -out {{.CLIENT_DIR}}/{{.CLIENT_NAME}}.csr -subj "/CN={{.CLIENT_NAME}}" + status: + - test -f {{.CLIENT_DIR}}/{{.CLIENT_NAME}}.csr + + _cert-client-crt: + internal: true + cmds: + - openssl x509 -req -in {{.CLIENT_DIR}}/{{.CLIENT_NAME}}.csr -CA {{.CA_DIR}}/ca.crt -CAkey {{.CA_DIR}}/ca.key -CAcreateserial -out {{.CLIENT_DIR}}/{{.CLIENT_NAME}}.crt -days 365 -sha256 + status: + - test -f {{.CLIENT_DIR}}/{{.CLIENT_NAME}}.crt + + clean-certs: + desc: "Remove all TLS certificates" + cmds: + - rm -rf {{.CA_DIR}} + - rm -rf {{.CERT_DIR}} + - rm -rf {{.CLIENT_DIR}} + - echo "All certificates removed" + + clean-client-certs: + desc: "Remove client TLS certificates only" + cmds: + - rm -rf {{.CLIENT_DIR}} + - echo "Client certificates removed" diff --git a/Taskfile.yml b/Taskfile.yml new file mode 100644 index 00000000..1feabd9e --- /dev/null +++ b/Taskfile.yml @@ -0,0 +1,22 @@ +--- +version: '3' + +dotenv: ['.env'] + +includes: + tools: .tasks/tools.yml + cloud: .tasks/cloud.yml + redpanda: .tasks/redpanda.yml + monitor: .tasks/monitor.yml + console: .tasks/console.yml + connect: .tasks/connect.yml + test: .tasks/test.yml + mirror: .tasks/mirror.yml + ci: .tasks/ci.yml + +tasks: + default: + desc: "Show available tasks" + cmds: + - task --list-all + silent: true From bb57f91e66107082b76dee70b6354f8f24bb7c47 Mon Sep 17 00:00:00 2001 From: gene-redpanda <123959009+gene-redpanda@users.noreply.github.com> Date: Tue, 7 Oct 2025 22:53:51 -0500 Subject: [PATCH 3/9] ci: pipeline to use taskfile --- .buildkite/pipeline.yml | 70 ++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 33 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 56026e03..e5eca9e6 100755 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -6,14 +6,14 @@ steps: key: aws-up-ubuntu concurrency_group: aws-ub concurrency: 1 - command: make ci-aws-rp -e DEPLOYMENT_ID=ci-bs-ub-`tr -dc a-z0-9 Date: Wed, 8 Oct 2025 13:22:32 -0500 Subject: [PATCH 4/9] ci: yamllint config --- .yamllint | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .yamllint diff --git a/.yamllint b/.yamllint new file mode 100644 index 00000000..3b503025 --- /dev/null +++ b/.yamllint @@ -0,0 +1,31 @@ +--- +extends: default + +rules: + document-start: disable + line-length: + max: 200 + level: warning + indentation: + spaces: 2 + indent-sequences: true + comments: + min-spaces-from-content: 1 + braces: + max-spaces-inside: 1 + brackets: + max-spaces-inside: 1 + truthy: + allowed-values: ['true', 'false'] + +ignore: | + artifacts/** + .github/** + aws/** + azure/** + gcp/** + ibm/** + templates/** + node_modules/** + .task/** + ssh-bootstrap.yml From 7a1dce71715dcd929c06d401f3b2d2d9985ce3db Mon Sep 17 00:00:00 2001 From: gene-redpanda <123959009+gene-redpanda@users.noreply.github.com> Date: Tue, 7 Oct 2025 22:48:10 -0500 Subject: [PATCH 5/9] chore: update grafana role --- ansible/deploy-monitor-tls.yml | 10 ++++++---- ansible/deploy-monitor.yml | 10 ++++++---- requirements.yml | 3 ++- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/ansible/deploy-monitor-tls.yml b/ansible/deploy-monitor-tls.yml index c79f8e63..72bdf4b1 100644 --- a/ansible/deploy-monitor-tls.yml +++ b/ansible/deploy-monitor-tls.yml @@ -68,10 +68,12 @@ roles: - grafana.grafana.grafana vars: - grafana_version: 10.4.1 - grafana_security: - admin_user: admin - admin_password: "{{ grafana_admin_pass | default('enter_your_secure_password', true) }}" + grafana_version: "latest" + grafana_yum_key: "https://rpm.grafana.com/gpg.key" + grafana_ini: + security: + admin_user: admin + admin_password: "{{ grafana_admin_pass | default('enter_your_secure_password', true) }}" grafana_datasources: - name: prometheus type: prometheus diff --git a/ansible/deploy-monitor.yml b/ansible/deploy-monitor.yml index 812fb963..74ab76dd 100644 --- a/ansible/deploy-monitor.yml +++ b/ansible/deploy-monitor.yml @@ -43,10 +43,12 @@ roles: - grafana.grafana.grafana vars: - grafana_version: 10.4.1 - grafana_security: - admin_user: admin - admin_password: "{{ grafana_admin_pass | default('enter_your_secure_password', true) }}" + grafana_version: "latest" + grafana_yum_key: "https://rpm.grafana.com/gpg.key" + grafana_ini: + security: + admin_user: admin + admin_password: "{{ grafana_admin_pass | default('enter_your_secure_password', true) }}" grafana_datasources: - name: prometheus type: prometheus diff --git a/requirements.yml b/requirements.yml index ed394291..73d12c26 100644 --- a/requirements.yml +++ b/requirements.yml @@ -1,10 +1,11 @@ +--- collections: - name: community.general - name: redpanda.cluster type: galaxy - name: ansible.posix - name: grafana.grafana - version: 5.6.0 + version: 6.0.4 - name: prometheus.prometheus roles: From 0c42971e53bee0944f9c841b0b5a2c81f93e8bae Mon Sep 17 00:00:00 2001 From: gene-redpanda <123959009+gene-redpanda@users.noreply.github.com> Date: Wed, 8 Oct 2025 14:41:33 -0500 Subject: [PATCH 6/9] chore: delete makefile --- Makefile | 640 ------------------------------------------------------- 1 file changed, 640 deletions(-) delete mode 100644 Makefile diff --git a/Makefile b/Makefile deleted file mode 100644 index 259ccf93..00000000 --- a/Makefile +++ /dev/null @@ -1,640 +0,0 @@ -ARTIFACT_DIR := $(PWD)/artifacts -DEPLOYMENT_ID ?= devex-cicd -NUM_NODES ?= 3 -ENABLE_MONITORING ?= true -TIERED_STORAGE_ENABLED ?= false -ALLOW_FORCE_DESTROY ?= true -VPC_ID ?= -BUCKET_NAME := $(subst _,-,$(DEPLOYMENT_ID))-bucket -DISTRO ?= ubuntu-focal -IS_USING_UNSTABLE ?= false -CA_CRT ?= $(PWD)/ansible/tls/ca/ca.crt -REDPANDA_LICENSE ?= empty - -# RPK -RPK_PATH ?= $(ARTIFACT_DIR)/bin/rpk - -# Terraform environment values -TERRAFORM_VERSION := 1.7.4 -TERRAFORM_INSTALL_DIR := $(ARTIFACT_DIR)/terraform/$(TERRAFORM_VERSION) -ENABLE_CONNECT ?= false - -# Ansible environment values -export ANSIBLE_VERSION := 2.16.4 -export ANSIBLE_INSTALL_DIR := $(ARTIFACT_DIR)/ansible/$(ANSIBLE_VERSION) -export ANSIBLE_LOG_PATH := $(ARTIFACT_DIR)/logs/$(DEPLOYMENT_ID).log -export ANSIBLE_INVENTORY := $(ARTIFACT_DIR)/hosts_$(DEPLOYMENT_ID).ini -export ANSIBLE_COLLECTIONS_PATH := $(ARTIFACT_DIR)/collections -export ANSIBLE_ROLES_PATH := $(ARTIFACT_DIR)/roles - -# hosts and keys -HOSTS_FILE ?= $(ARTIFACT_DIR)/hosts_$(DEPLOYMENT_ID).ini -PUBLIC_KEY ?= $(ARTIFACT_DIR)/testkey.pub -PRIVATE_KEY ?= $(ARTIFACT_DIR)/testkey - -# gcp env -GOOGLE_PROJECT_ID ?= "hallowed-ray-376320" - -# copy_file environment values -RPM_VERSION ?= v1.0.0-7ae9d19 -SERVER_DIR ?= /tmp -LOCAL_FILE := $(ARTIFACT_DIR)/redpanda-connect.x86_64.rpm -TOKEN := ${CONNECT_RPM_TOKEN} -DL_LINK := https://dl.redpanda.com/$(TOKEN)/connectors-artifacts/raw/names/redpanda-connectors/versions/$(RPM_VERSION)/redpanda-connectors-$(RPM_VERSION).x86_64.rpm - -INSTANCE_TYPE_AWS ?= i3.2xlarge -MACHINE_ARCH ?= x86_64 - -export TF_IN_AUTOMATION := $(CI) -export AWS_ACCESS_KEY_ID := $(if $(AWS_ACCESS_KEY_ID),$(AWS_ACCESS_KEY_ID),$(DA_AWS_ACCESS_KEY_ID)) -export AWS_SECRET_ACCESS_KEY := $(if $(AWS_SECRET_ACCESS_KEY),$(AWS_SECRET_ACCESS_KEY),$(DA_AWS_SECRET_ACCESS_KEY)) -export AWS_DEFAULT_REGION ?= us-west-2 - -.PHONY: ansible-prereqs -ansible-prereqs: collection role - @echo "Ansible prereqs installed" - -.PHONY: ci-aws-rp -ci-aws-rp: aws-rp install-rpk test-cluster destroy-aws - -.PHONY: aws-rp -aws-rp: keygen build-aws cluster monitor console - -.PHONY: ci-aws-rp-connect -ci-aws-rp-connect: ENABLE_CONNECT := true -ci-aws-rp-connect: keygen build-aws extra-aws-copy deploy-extra-rp cluster deploy-connect monitor console install-rpk test-cluster create-connector test-cluster-spam-messages extra-aws-destroy destroy-aws - -.PHONY: ci-aws-rp-tls -ci-aws-rp-tls: keygen build-aws cluster-tls monitor-tls console-tls install-rpk test-cluster-tls destroy-aws - -.PHONY: ci-aws-rp-tiered -ci-aws-rp-tiered: TIERED_STORAGE_ENABLED := true -ci-aws-rp-tiered: keygen build-aws cluster-tiered-storage monitor-tls console-tls install-rpk test-cluster-tls test-aws-storage destroy-aws - -.PHONY: ci-aws-rp-ts-connect -ci-aws-rp-ts-connect: ENABLE_CONNECT := true -ci-aws-rp-ts-connect: TIERED_STORAGE_ENABLED := true -ci-aws-rp-ts-connect: keygen build-aws cluster-tiered-storage deploy-connect-tls monitor-tls console-tls extra-aws-copy deploy-extra-rp install-rpk test-cluster-tls test-aws-storage test-connect-tls-client create-connector-tls test-cluster-spam-messages-tls destroy-aws extra-aws-destroy - -.PHONY: ci-gcp-rp -ci-gcp-rp: keygen build-gcp cluster monitor console install-rpk test-cluster destroy-gcp - -.PHONY: ci-gcp-rp-tls -ci-gcp-rp-tls: keygen build-gcp cluster-tls monitor-tls console-tls install-rpk test-cluster-tls destroy-gcp - -.PHONY: ci-gcp-rp-tiered -ci-gcp-rp-tiered: TIERED_STORAGE_ENABLED := true -ci-gcp-rp-tiered: keygen build-gcp cluster-tiered-storage monitor-tls console-tls install-rpk test-cluster-tls test-gcp-storage destroy-gcp - -.PHONY: deploy-connect -deploy-connect: get-rpm copy-rpm connect - -.PHONY: deploy-connect-tls -deploy-connect-tls: get-rpm copy-rpm connect-tls - -.PHONY: get-rpm -get-rpm: - @if [ ! -f $(LOCAL_FILE) ]; then \ - echo "Downloading $(LOCAL_FILE)..."; \ - curl -o $(LOCAL_FILE) $(DL_LINK); \ - else \ - echo "$(LOCAL_FILE) already exists. Skipping download."; \ - fi - -.PHONY: copy-rpm -copy-rpm: - @echo "Copying $(LOCAL_FILE).tar.gz to $(SERVER_DIR)" - $(eval IPS_USERS=$(shell awk '/^\[connect\]/{f=1; next} /^\[/{f=0} f && /^[0-9]/{split($$2,a,"="); print a[2] "@" $$1}' $(HOSTS_FILE))) - @echo $(IPS_USERS) - @for IP_USER in $(IPS_USERS); do \ - scp -o StrictHostKeyChecking=no -i "$(PRIVATE_KEY)" "$(LOCAL_FILE)" "$$IP_USER:$(SERVER_DIR)"; \ - done - -SSH_EMAIL ?= test@test.com -.PHONY: keygen -keygen: - @if [ ! -f artifacts/testkey ]; then \ - printf 'y\n' | ssh-keygen -t rsa -b 4096 -C "$(SSH_EMAIL)" -N "" -f artifacts/testkey && chmod 0700 artifacts/testkey; \ - else \ - echo "artifacts/testkey already exists. Skipping key generation."; \ - fi - -.PHONY: build-aws -build-aws: - @echo $(TIERED_STORAGE_ENABLED) - @cd aws && \ - terraform init && \ - terraform apply -auto-approve \ - -var='deployment_prefix=$(DEPLOYMENT_ID)' \ - -var='public_key_path=$(PUBLIC_KEY)' \ - -var='broker_count=$(NUM_NODES)' \ - -var='enable_monitoring=$(ENABLE_MONITORING)' \ - -var='tiered_storage_enabled=$(TIERED_STORAGE_ENABLED)' \ - -var='allow_force_destroy=$(ALLOW_FORCE_DESTROY)' \ - -var='vpc_id=$(VPC_ID)' \ - -var='distro=$(DISTRO)' \ - -var='hosts_file=$(ANSIBLE_INVENTORY)' \ - -var='machine_architecture=$(MACHINE_ARCH)' \ - -var='enable_connect=$(ENABLE_CONNECT)' \ - -var='broker_instance_type=$(INSTANCE_TYPE_AWS)' \ - -var='client_instance_type=$(INSTANCE_TYPE_AWS)' \ - -var='prometheus_instance_type=$(INSTANCE_TYPE_AWS)' - -GCP_IMAGE ?= ubuntu-os-cloud/ubuntu-2204-lts -GCP_INSTANCE_TYPE ?= n2-standard-2 -GCP_CREDS ?= $(shell echo $$GCP_CREDS) -.PHONY: build-gcp -build-gcp: - @cd gcp && \ - terraform init && \ - terraform apply -auto-approve \ - -var='deployment_prefix=$(DEPLOYMENT_ID)' \ - -var='public_key_path=$(PUBLIC_KEY)' \ - -var='broker_count=$(NUM_NODES)' \ - -var='enable_monitoring=$(ENABLE_MONITORING)' \ - -var='tiered_storage_enabled=$(TIERED_STORAGE_ENABLED)' \ - -var='image=$(GCP_IMAGE)' \ - -var='hosts_file=$(ANSIBLE_INVENTORY)' \ - -var='machine_type=$(GCP_INSTANCE_TYPE)' \ - -var='gcp_creds=$(GCP_CREDS)' - -.PHONY: destroy-aws -destroy-aws: - @cd aws/$(TF_DIR) && \ - terraform init && \ - terraform destroy -auto-approve \ - -var='deployment_prefix=$(DEPLOYMENT_ID)' \ - -var='public_key_path=$(PUBLIC_KEY)' \ - -var='broker_count=$(NUM_NODES)' \ - -var='enable_monitoring=$(ENABLE_MONITORING)' \ - -var='tiered_storage_enabled=$(TIERED_STORAGE_ENABLED)' \ - -var='allow_force_destroy=$(ALLOW_FORCE_DESTROY)' \ - -var='vpc_id=$(VPC_ID)' \ - -var='distro=$(DISTRO)' \ - -var='hosts_file=$(ANSIBLE_INVENTORY)' \ - -var='machine_architecture=$(MACHINE_ARCH)' \ - -var='enable_connect=$(ENABLE_CONNECT)' \ - -var='broker_instance_type=$(INSTANCE_TYPE_AWS)' \ - -var='client_instance_type=$(INSTANCE_TYPE_AWS)' \ - -var='prometheus_instance_type=$(INSTANCE_TYPE_AWS)' - -.PHONY: build-aws-proxy -build-aws-proxy: - @echo $(TIERED_STORAGE_ENABLED) - @cd aws/private-test && \ - terraform init && \ - terraform apply -auto-approve \ - -var='deployment_prefix=$(DEPLOYMENT_ID)' \ - -var='public_key_path=$(PUBLIC_KEY)' \ - -var='broker_count=$(NUM_NODES)' \ - -var='enable_monitoring=$(ENABLE_MONITORING)' \ - -var='tiered_storage_enabled=$(TIERED_STORAGE_ENABLED)' \ - -var='allow_force_destroy=$(ALLOW_FORCE_DESTROY)' \ - -var='vpc_id=$(VPC_ID)' \ - -var='distro=$(DISTRO)' \ - -var='hosts_file=$(ANSIBLE_INVENTORY)' \ - -var='machine_architecture=$(MACHINE_ARCH)' \ - -var='enable_connect=$(ENABLE_CONNECT)' \ - -var='broker_instance_type=$(INSTANCE_TYPE_AWS)' \ - -var='client_instance_type=$(INSTANCE_TYPE_AWS)' \ - -var='prometheus_instance_type=$(INSTANCE_TYPE_AWS)' - -destroy-aws-proxy: - @echo $(TIERED_STORAGE_ENABLED) - @cd aws/private-test && \ - terraform init && \ - terraform destroy -auto-approve \ - -var='deployment_prefix=$(DEPLOYMENT_ID)' \ - -var='public_key_path=$(PUBLIC_KEY)' \ - -var='broker_count=$(NUM_NODES)' \ - -var='enable_monitoring=$(ENABLE_MONITORING)' \ - -var='tiered_storage_enabled=$(TIERED_STORAGE_ENABLED)' \ - -var='allow_force_destroy=$(ALLOW_FORCE_DESTROY)' \ - -var='vpc_id=$(VPC_ID)' \ - -var='distro=$(DISTRO)' \ - -var='hosts_file=$(ANSIBLE_INVENTORY)' \ - -var='machine_architecture=$(MACHINE_ARCH)' \ - -var='enable_connect=$(ENABLE_CONNECT)' \ - -var='broker_instance_type=$(INSTANCE_TYPE_AWS)' \ - -var='client_instance_type=$(INSTANCE_TYPE_AWS)' \ - -var='prometheus_instance_type=$(INSTANCE_TYPE_AWS)' - - -.PHONY: destroy-gcp -destroy-gcp: - @cd gcp/$(TF_DIR) && \ - terraform init && \ - terraform destroy -auto-approve \ - -var='deployment_prefix=$(DEPLOYMENT_ID)' \ - -var='public_key_path=$(PUBLIC_KEY)' \ - -var='broker_count=$(NUM_NODES)' \ - -var='enable_monitoring=$(ENABLE_MONITORING)' \ - -var='tiered_storage_enabled=$(TIERED_STORAGE_ENABLED)' \ - -var='image=$(GCP_IMAGE)' \ - -var='hosts_file=$(ANSIBLE_INVENTORY)' \ - -var='machine_type=$(GCP_INSTANCE_TYPE)' \ - -var='gcp_creds=$(GCP_CREDS)' - -.PHONY: collection -collection: - @mkdir -p $(ANSIBLE_COLLECTIONS_PATH) - @ansible-galaxy collection install -r $(PWD)/requirements.yml --force -p $(ANSIBLE_COLLECTIONS_PATH) - -.PHONY: role -role: - @mkdir -p $(ANSIBLE_ROLES_PATH) - @ansible-galaxy role install -r $(PWD)/requirements.yml --force -p $(ANSIBLE_ROLES_PATH) - -.PHONY: monitor -monitor: ansible-prereqs - @mkdir -p $(ARTIFACT_DIR)/logs - export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES - @ansible-playbook ansible/deploy-monitor.yml --private-key $(PRIVATE_KEY) --inventory $(ANSIBLE_INVENTORY) --extra-vars is_using_unstable=$(IS_USING_UNSTABLE) - -.PHONY: monitor-tls -monitor-tls: ansible-prereqs - @mkdir -p $(ARTIFACT_DIR)/logs - @ansible-playbook ansible/deploy-monitor-tls.yml --private-key $(PRIVATE_KEY) --inventory $(ANSIBLE_INVENTORY) --extra-vars is_using_unstable=$(IS_USING_UNSTABLE) - -.PHONY: console -console: ansible-prereqs - @mkdir -p $(ARTIFACT_DIR)/logs - @ansible-playbook ansible/deploy-console.yml --private-key $(PRIVATE_KEY) --inventory $(ANSIBLE_INVENTORY) --extra-vars is_using_unstable=$(IS_USING_UNSTABLE) - -.PHONY: console-tls -console-tls: ansible-prereqs - @mkdir -p $(ARTIFACT_DIR)/logs - @ansible-playbook ansible/deploy-console-tls.yml --private-key $(PRIVATE_KEY) --inventory $(ANSIBLE_INVENTORY) --extra-vars is_using_unstable=$(IS_USING_UNSTABLE) - -.PHONY: connect -connect: ENABLE_CONNECT := true -connect: ansible-prereqs - @mkdir -p $(ARTIFACT_DIR)/logs - @ansible-playbook ansible/deploy-connect.yml --private-key $(PRIVATE_KEY) --inventory $(ANSIBLE_INVENTORY) --extra-vars is_using_unstable=$(IS_USING_UNSTABLE) - -.PHONY: connect-tls -connect-tls: ENABLE_CONNECT := true -connect-tls: ansible-prereqs - @mkdir -p $(ARTIFACT_DIR)/logs - @ansible-playbook ansible/deploy-connect-tls.yml --private-key $(PRIVATE_KEY) --inventory $(ANSIBLE_INVENTORY) --extra-vars is_using_unstable=$(IS_USING_UNSTABLE) - - -MAC_RPK := "https://github.com/redpanda-data/redpanda/releases/latest/download/rpk-darwin-amd64.zip" -LINUX_RPK := "https://github.com/redpanda-data/redpanda/releases/latest/download/rpk-linux-amd64.zip" - -.PHONY: install-rpk -install-rpk: - @mkdir -p $(ARTIFACT_DIR)/tmp - @mkdir -p $(ARTIFACT_DIR)/bin -ifeq ($(shell uname),Darwin) - @curl -L $(MAC_RPK) -o $(ARTIFACT_DIR)/tmp/rpk.zip -else - @curl -L $(LINUX_RPK) -o $(ARTIFACT_DIR)/tmp/rpk.zip -endif - @unzip -o $(ARTIFACT_DIR)/tmp/rpk.zip -d $(ARTIFACT_DIR)/bin/ - @chmod 755 $(ARTIFACT_DIR)/bin/rpk - @rm $(ARTIFACT_DIR)/tmp/rpk.zip - -.PHONY: cluster -cluster: ansible-prereqs - @mkdir -p $(ARTIFACT_DIR)/logs - @ansible-playbook ansible/provision-cluster.yml --private-key $(PRIVATE_KEY) --inventory $(ANSIBLE_INVENTORY) --extra-vars is_using_unstable=$(IS_USING_UNSTABLE) - -# Logging configuration -REDPANDA_LOGGING_LOG_FILE ?= /var/log/redpanda2/redpanda.log - -.PHONY: operation-configure-logging -operation-configure-logging: ansible-prereqs - @mkdir -p $(ARTIFACT_DIR)/logs - @ansible-playbook ansible/operation-configure-logging.yml --private-key $(PRIVATE_KEY) --inventory $(ANSIBLE_INVENTORY) --extra-vars is_using_unstable=$(IS_USING_UNSTABLE) --extra-vars redpanda_logging_log_file=$(REDPANDA_LOGGING_LOG_FILE) - -TEST_TOPIC_NAME ?= testtopic -PARTITION_COUNT ?= 3 -.PHONY: test-cluster -test-cluster: - @# Assemble the redpanda brokers by chopping up the hosts file - chmod 775 $(RPK_PATH) - echo $(RPK_PATH) - $(eval REDPANDA_BROKERS := $(shell awk '/^\[redpanda\]/{f=1; next} /^$$/{f=0} f{print $$1}' "$(HOSTS_FILE)" | paste -sd ',' - | awk '{gsub(/,/,":9092,"); sub(/,$$/,":9092")}1')) - - @echo $(REDPANDA_BROKERS) - @echo "checking cluster status" - @echo rpk cluster status -X brokers=$(REDPANDA_BROKERS) -v || exit 1 - @$(RPK_PATH) cluster status -X brokers=$(REDPANDA_BROKERS) -v || exit 1 - - @echo "creating topic" - @$(RPK_PATH) topic create $(TEST_TOPIC_NAME) -p $(PARTITION_COUNT) -X brokers=$(REDPANDA_BROKERS) -v || exit 1 - - @echo "producing to topic" - @echo squirrel | $(RPK_PATH) topic produce $(TEST_TOPIC_NAME) -X brokers=$(REDPANDA_BROKERS) -v || exit 1 - - @echo "consuming from topic" - @$(RPK_PATH) topic consume $(TEST_TOPIC_NAME) -X brokers=$(REDPANDA_BROKERS) -v -o :end | grep squirrel || exit 1 - -.PHONY: test-schema -test-schema: - $(eval REDPANDA_REGISTRY := $(shell awk '/^\[redpanda\]/{f=1; next} /^$$/{f=0} f{print $$1}' "$(HOSTS_FILE)" | paste -sd ',' - | awk '{gsub(/,/,":8081,"); sub(/,$$/,":8081")}1')) - - @echo "testing schema registry" - @echo $(REDPANDA_REGISTRY) - - @for ip_port in $$(echo $(REDPANDA_REGISTRY) | tr ',' ' '); do curl $$ip_port/subjects ; done - -.PHONY: cluster-tls -cluster-tls: ansible-prereqs - @mkdir -p $(ARTIFACT_DIR)/logs - ansible-playbook ansible/provision-cluster-tls.yml --private-key $(PRIVATE_KEY) --extra-vars is_using_unstable=$(IS_USING_UNSTABLE) - -.PHONY: test-cluster-tls -test-cluster-tls: - $(eval REDPANDA_BROKERS := $(shell sed -n '/^\[redpanda\]/,/^$$/p' "$(ANSIBLE_INVENTORY)" | \ - grep 'private_ip=' | \ - cut -d' ' -f1 | \ - sed 's/$$/:9092/' | \ - tr '\n' ',' | \ - sed 's/,$$/\n/')) - - @echo "Redpanda Brokers: $(REDPANDA_BROKERS)" - @echo "TLS Truststore: $(CA_CRT)" - - @echo "checking TLS cluster status" - @$(ARTIFACT_DIR)/bin/rpk cluster status -X brokers="$(REDPANDA_BROKERS)" -X tls.ca="$(CA_CRT)" -v || exit 1 - - @echo "creating topic" - @$(ARTIFACT_DIR)/bin/rpk topic create $(TEST_TOPIC_NAME) -p $(PARTITION_COUNT) -X brokers="$(REDPANDA_BROKERS)" -X tls.ca="$(CA_CRT)" -v || exit 1 - - @echo "producing to topic" - @echo squirrels | $(ARTIFACT_DIR)/bin/rpk topic produce $(TEST_TOPIC_NAME) -X brokers="$(REDPANDA_BROKERS)" -X tls.ca="$(CA_CRT)" -v || exit 1 - - @echo "consuming from topic" - @$(ARTIFACT_DIR)/bin/rpk topic consume $(TEST_TOPIC_NAME) -X brokers="$(REDPANDA_BROKERS)" -X tls.ca="$(CA_CRT)" -v -o :end | grep squirrels || exit 1 - -.PHONY: test-schema-tls -test-schema-tls: - $(eval REDPANDA_REGISTRY := $(shell sed -n '/^\[redpanda\]/,/^$$/p' "$(ANSIBLE_INVENTORY)" | \ - grep 'private_ip=' | \ - cut -d' ' -f1 | \ - sed 's/$$/:8081/' | \ - tr '\n' ',' | \ - sed 's/,$$/\n/')) - - @echo "testing schema registry" - @for ip_port in $$(echo $(REDPANDA_REGISTRY) | tr ',' ' '); do \ - curl $$ip_port/subjects -k --cacert "$(CA_CRT)"; \ - done - -SEGMENT_UPLOAD_INTERVAL ?= "1" -CLOUD_STORAGE_CREDENTIALS_SOURCE ?= "aws_instance_metadata" - -.PHONY: cluster-tiered-storage -cluster-tiered-storage: ansible-prereqs - @mkdir -p $(ARTIFACT_DIR)/logs - ansible-playbook ansible/provision-cluster-tiered-storage.yml --private-key $(PRIVATE_KEY) --extra-vars is_using_unstable=$(IS_USING_UNSTABLE) --extra-vars segment_upload_interval=$(SEGMENT_UPLOAD_INTERVAL) --extra-vars cloud_storage_credentials_source=$(CLOUD_STORAGE_CREDENTIALS_SOURCE) --extra-vars redpanda_license=$(REDPANDA_LICENSE) - -GOOGLE_APPLICATION_CREDENTIALS ?= "/tmp/gcp_creds.json" -SIMPLE_BUCKET_NAME=$(shell echo $(BUCKET_NAME) | sed 's/-bucket$$//') -.PHONY: test-gcp-storage -test-gcp-storage: - @echo "$(GCP_CREDS)" | base64 -d > /tmp/gcp_creds.json - export CLOUDSDK_CORE_PROJECT=$(GOOGLE_PROJECT_ID) - @gcloud auth activate-service-account --key-file=$(GOOGLE_APPLICATION_CREDENTIALS) --project=$(GOOGLE_PROJECT_ID) - @gcloud storage --project $(GOOGLE_PROJECT_ID) ls | grep $(SIMPLE_BUCKET_NAME) - -.PHONY: test-aws-storage -test-aws-storage: - @aws s3api list-objects-v2 --bucket "$(BUCKET_NAME)" --region $(AWS_DEFAULT_REGION) --query "Contents[?contains(Key, 'testtopic/')]" --output text | wc -l | xargs -I {} sh -c 'if [ "{}" -ge 1 ]; then exit 0; else echo "testtopic folder not found" && exit 1; fi' - -.PHONY: cluster-proxy -cluster-proxy: ansible-prereqs - @mkdir -p $(ARTIFACT_DIR)/logs - @ansible-playbook ansible/proxy/provision-private-proxied-cluster.yml --private-key $(PRIVATE_KEY) --inventory $(ANSIBLE_INVENTORY) --extra-vars is_using_unstable=$(IS_USING_UNSTABLE) --extra-vars '{\"squid_acl_localnet\": [\"$(SQUID_ACL_LOCALNET)\"]}' --extra-vars redpanda='{\"cluster\":{\"cloud_storage_segment_max_upload_interval_sec\":\"$(SEGMENT_UPLOAD_INTERVAL)\"}}' $(SKIP_TAGS) $(CLI_ARGS) - -.PHONY: test-cluster-proxy -test-cluster-proxy: - $(eval REDPANDA_BROKERS := $(shell sed -n '/^\[redpanda\]/,/^$$/p' "$(HOSTS_FILE)" | \ - grep 'private_ip=' | \ - cut -d' ' -f1 | \ - sed 's/$$/:9092/' | \ - tr '\n' ',' | \ - sed 's/,$$/\n/')) - - $(eval CLIENT_SSH_USER := $(shell sed -n '/\[redpanda\]/,/\[/p' "$(HOSTS_FILE)" | \ - grep ansible_user | \ - head -n1 | \ - tr ' ' '\n' | \ - grep ansible_user | \ - cut -d'=' -f2)) - - $(eval CLIENT_PUBLIC_IP := $(shell sed -n '/^\[client\]/,/^$$/p' "$(HOSTS_FILE)" | \ - grep 'private_ip=' | \ - cut -f1 -d' ')) - - @echo "checking proxy cluster status" - @ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 -i "$(SSHKEY)" "$(CLIENT_SSH_USER)@$(CLIENT_PUBLIC_IP)" 'rpk cluster status -X brokers="$(REDPANDA_BROKERS)" -X tls.ca="$(PATH_TO_CA_CRT)" -v' || exit 1 - - @echo "creating topic" - @ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 -i "$(SSHKEY)" "$(CLIENT_SSH_USER)@$(CLIENT_PUBLIC_IP)" 'rpk topic create $(TEST_TOPIC_NAME) -p $(PARTITION_COUNT) -X brokers="$(REDPANDA_BROKERS)" -X tls.ca="$(PATH_TO_CA_CRT)" -v' || exit 1 - - @echo "producing to topic" - @ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 -i "$(SSHKEY)" "$(CLIENT_SSH_USER)@$(CLIENT_PUBLIC_IP)" 'echo squirrels | rpk topic produce $(TEST_TOPIC_NAME) -X brokers="$(REDPANDA_BROKERS)" -X tls.ca="$(PATH_TO_CA_CRT)" -v' || exit 1 - - @sleep 30 - - @echo "consuming from topic" - $(eval testoutput := $(shell ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 -i "$(SSHKEY)" "$(CLIENT_SSH_USER)@$(CLIENT_PUBLIC_IP)" 'rpk topic consume $(TEST_TOPIC_NAME) -X brokers="$(REDPANDA_BROKERS)" -X tls.ca="$(PATH_TO_CA_CRT)" -v -o :end')) - @echo "$(testoutput)" | grep squirrels || exit 1 - -CLIENT_NAME ?= client -CLIENT_DIR := ansible/tls/clients -CA_DIR := ansible/tls/ca -CERT_DIR := ansible/tls/certs -CLIENT_KEY ?= ansible/tls/clients/client.key -CLIENT_CERT ?= ansible/tls/ca/ca.crt - -$(CLIENT_DIR): - mkdir -p $@ - -$(CLIENT_DIR)/$(CLIENT_NAME).key: - openssl genrsa -out $@ 2048 - -$(CLIENT_DIR)/$(CLIENT_NAME).csr: $(CLIENT_DIR)/$(CLIENT_NAME).key - openssl req -new -key $< -out $@ -subj "/CN=$(CLIENT_NAME)" - -$(CLIENT_DIR)/$(CLIENT_NAME).crt: $(CLIENT_DIR)/$(CLIENT_NAME).csr - openssl x509 -req -in $< -CA $(CA_DIR)/ca.crt -CAkey $(CA_DIR)/ca.key -CAcreateserial -out $@ -days 365 -sha256 - -.PHONY: cert-client -cert-client: $(CLIENT_DIR) $(CLIENT_DIR)/$(CLIENT_NAME).key $(CLIENT_DIR)/$(CLIENT_NAME).csr $(CLIENT_DIR)/$(CLIENT_NAME).crt - -.PHONY: cert-clean -cert-clean: - rm -rf $(CA_DIR) - rm -rf $(CERT_DIR) - rm -rf $(CLIENT_DIR) - -.PHONY: cert-clean-client -cert-clean-client: - rm -rf $(CLIENT_DIR) - -.PHONY: test-connect-tls-client -test-connect-tls-client: cert-client - $(eval CONNECT_TARGET := $(shell awk '/^\[connect\]/{f=1; next} /^$$/{f=0} f{print $$1}' "$(HOSTS_FILE)" | head -n1)) - curl -vvvvv -k --cert ansible/tls/clients/client.crt --key $(CLIENT_KEY) --cacert $(CLIENT_CERT) -X GET https://$(CONNECT_TARGET):8083/connectors - -test-prometheus-exporter: cert-client - $(eval PROMETHEUS_EXPORTER_TARGET := $(shell awk '/^\[connect\]/{f=1; next} /^$$/{f=0} f{print $$1}' "$(HOSTS_FILE)" | head -n1)) - curl -vvvvv -k --cert ansible/tls/clients/client.crt --key ansible/tls/clients/client.key --cacert ansible/tls/ca/ca.crt -X GET https://$(PROMETHEUS_EXPORTER_TARGET):9404/metrics - -# Extra deployment enables deploying a second cluster -EXTRA_INVENTORY = $(ARTIFACT_DIR)/hosts2_$(DEPLOYMENT_ID).ini - -.PHONY: deploy-extra-rp -deploy-extra-rp: extra-aws extra-cluster - -.PHONY: extra-aws-copy -extra-aws-copy: - cp -r aws aws-extra && \ - rm -rf aws-extra/terraform.tfstate && \ - rm -rf aws-extra/terraform.tfstate.backup && \ - rm -rf aws-extra/.terraform && \ - rm -rf aws-extra/.terraform.lock.hcl - - -.PHONY: extra-aws-cleanup -extra-aws-cleanup: - rm -rf aws-extra - -.PHONY: extra-aws -extra-aws: - @cd aws-extra/$(TF_DIR) && \ - terraform init && \ - terraform apply -auto-approve \ - -var='deployment_prefix=$(DEPLOYMENT_ID)2' \ - -var='public_key_path=$(PUBLIC_KEY)' \ - -var='broker_count=$(NUM_NODES)' \ - -var='allow_force_destroy=$(ALLOW_FORCE_DESTROY)' \ - -var='vpc_id=$(VPC_ID)' \ - -var='distro=$(DISTRO)' \ - -var='hosts_file=$(EXTRA_INVENTORY)' \ - -var='machine_architecture=$(MACHINE_ARCH)' \ - -var='enable_connect=false' \ - -var='broker_instance_type=$(INSTANCE_TYPE_AWS)' \ - -var='client_instance_type=$(INSTANCE_TYPE_AWS)' \ - -var='prometheus_instance_type=$(INSTANCE_TYPE_AWS)' - -.PHONY: extra-aws-destroy -extra-aws-destroy: - @cd aws-extra/$(TF_DIR) && \ - terraform init && \ - terraform destroy -auto-approve \ - -var='deployment_prefix=$(DEPLOYMENT_ID)2' \ - -var='public_key_path=$(PUBLIC_KEY)' \ - -var='broker_count=$(NUM_NODES)' \ - -var='enable_monitoring=$(ENABLE_MONITORING)' \ - -var='tiered_storage_enabled=$(TIERED_STORAGE_ENABLED)' \ - -var='allow_force_destroy=$(ALLOW_FORCE_DESTROY)' \ - -var='vpc_id=$(VPC_ID)' \ - -var='distro=$(DISTRO)' \ - -var='hosts_file=$(EXTRA_INVENTORY)' \ - -var='machine_architecture=$(MACHINE_ARCH)' \ - -var='enable_connect=false' \ - -var='broker_instance_type=$(INSTANCE_TYPE_AWS)' \ - -var='client_instance_type=$(INSTANCE_TYPE_AWS)' \ - -var='prometheus_instance_type=$(INSTANCE_TYPE_AWS)' - - -.PHONY: extra-cluster -extra-cluster: ansible-prereqs - @mkdir -p $(ARTIFACT_DIR)/logs - @ansible-playbook ansible/provision-cluster.yml --private-key $(PRIVATE_KEY) --inventory $(EXTRA_INVENTORY) --extra-vars is_using_unstable=$(IS_USING_UNSTABLE) - -.PHONY: extra-monitor -extra-monitor: ansible-prereqs - @mkdir -p $(ARTIFACT_DIR)/logs - @ansible-playbook ansible/deploy-monitor.yml --private-key $(PRIVATE_KEY) --inventory $(EXTRA_INVENTORY) --extra-vars is_using_unstable=$(IS_USING_UNSTABLE) - -.PHONY: extra-console -extra-console: ansible-prereqs - @mkdir -p $(ARTIFACT_DIR)/logs - @ansible-playbook ansible/deploy-console.yml --private-key $(PRIVATE_KEY) --inventory $(EXTRA_INVENTORY) --extra-vars is_using_unstable=$(IS_USING_UNSTABLE) - -.PHONY: deploy-rp-tls-extra -deploy-extra-rp-tls: extra-aws extra-cluster extra-monitor-tls extra-console-tls - -.PHONY: extra-monitor-tls -extra-monitor-tls: ansible-prereqs - @mkdir -p $(ARTIFACT_DIR)/logs - @ansible-playbook ansible/deploy-monitor-tls.yml --private-key $(PRIVATE_KEY) --inventory $(EXTRA_INVENTORY) --extra-vars is_using_unstable=$(IS_USING_UNSTABLE) - -.PHONY: extra-console-tls -extra-console-tls: ansible-prereqs - @mkdir -p $(ARTIFACT_DIR)/logs - @ansible-playbook ansible/deploy-console-tls.yml --private-key $(PRIVATE_KEY) --inventory $(EXTRA_INVENTORY) --extra-vars is_using_unstable=$(IS_USING_UNSTABLE) - -.PHONY: extra-cluster-tls -extra-cluster-tls: ansible-prereqs - @mkdir -p $(ARTIFACT_DIR)/logs - @ansible-playbook ansible/provision-cluster-tls.yml --private-key $(PRIVATE_KEY) --inventory $(EXTRA_INVENTORY) --extra-vars is_using_unstable=$(IS_USING_UNSTABLE) - -.PHONY: extra-copy-rpm -extra-copy-rpm: - @echo "Copying $(LOCAL_FILE).tar.gz to $(SERVER_DIR)" - $(eval IPS_USERS=$(shell awk '/^\[connect\]/{f=1; next} /^\[/{f=0} f && /^[0-9]/{split($$2,a,"="); print a[2] "@" $$1}' $(EXTRA_INVENTORY))) - @echo $(IPS_USERS) - @for IP_USER in $(IPS_USERS); do \ - scp -o StrictHostKeyChecking=no -i "$(PRIVATE_KEY)" "$(LOCAL_FILE)" "$$IP_USER:$(SERVER_DIR)"; \ - done - -# spam messages at an existing topic -SPAM_MESSAGE_COUNT ?= 10 -.PHONY: test-cluster-spam-messages -test-cluster-spam-messages: - @# Assemble the redpanda brokers by chopping up the hosts file - chmod 775 $(RPK_PATH) - echo $(RPK_PATH) - $(eval REDPANDA_BROKERS := $(shell awk '/^\[redpanda\]/{f=1; next} /^$$/{f=0} f{print $$1}' "$(HOSTS_FILE)" | paste -sd ',' - | awk '{gsub(/,/,":9092,"); sub(/,$$/,":9092")}1')) - - @echo "producing to topic" - $(foreach i,$(shell seq 1 $(SPAM_MESSAGE_COUNT)), \ - echo "squirrel$i" | $(RPK_PATH) topic produce $(TEST_TOPIC_NAME) -X brokers=$(REDPANDA_BROKERS) -v || exit 1; \ - ) - -# spam messages at an existing topic -.PHONY: test-cluster-spam-messages-tls -test-cluster-spam-messages-tls: - @# Assemble the redpanda brokers by chopping up the hosts file - chmod 775 $(RPK_PATH) - echo $(RPK_PATH) - $(eval REDPANDA_BROKERS := $(shell awk '/^\[redpanda\]/{f=1; next} /^$$/{f=0} f{print $$1}' "$(HOSTS_FILE)" | paste -sd ',' - | awk '{gsub(/,/,":9092,"); sub(/,$$/,":9092")}1')) - - @echo "producing to topic" - $(foreach i,$(shell seq 1 $(SPAM_MESSAGE_COUNT)), \ - echo "squirrel$i" | $(RPK_PATH) topic produce $(TEST_TOPIC_NAME) -X brokers=$(REDPANDA_BROKERS) -X tls.ca="$(CA_CRT)" -v || exit 1; \ - ) - - -.PHONY: create-connector -create-connector: - $(eval REDPANDA_BROKERS := $(shell awk '/^\[redpanda\]/{f=1; next} /^$$/{f=0} f{print $$1":9092"}' "$(HOSTS_FILE)" | paste -sd ',' -)) - $(eval EXTRA_BROKERS := $(shell awk '/^\[redpanda\]/{f=1; next} /^$$/{f=0} f{print $$1":9092"}' "$(EXTRA_INVENTORY)" | paste -sd ',' -)) - $(eval CONNECT_IP := $(shell awk '/^\[connect\]/{f=1; next} f{print $$1; exit}' $(HOSTS_FILE))) - - curl -X POST -H 'Content-Type: application/json' -H 'accept: application/json' http://$(CONNECT_IP):8083/connectors -d '{"name": "mirror-source-connector","config": {"connector.class": "org.apache.kafka.connect.mirror.MirrorSourceConnector","topics": "testtopic","replication.factor": "1","source.cluster.bootstrap.servers": "$(REDPANDA_BROKERS)","source.cluster.security.protocol": "PLAINTEXT","target.cluster.bootstrap.servers": "$(EXTRA_BROKERS)","target.cluster.security.protocol": "PLAINTEXT","source.cluster.alias": "source" }}' - -.PHONY: create-connector-tls -create-connector-tls: - $(eval REDPANDA_BROKERS := $(shell awk '/^\[redpanda\]/{f=1; next} /^$$/{f=0} f{print $$1":9092"}' "$(HOSTS_FILE)" | paste -sd ',' -)) - $(eval EXTRA_BROKERS := $(shell awk '/^\[redpanda\]/{f=1; next} /^$$/{f=0} f{print $$1":9092"}' "$(EXTRA_INVENTORY)" | paste -sd ',' -)) - $(eval CONNECT_IP := $(shell awk '/^\[connect\]/{f=1; next} f{print $$1; exit}' $(HOSTS_FILE))) - - curl -X POST -H 'Content-Type: application/json' -H 'accept: application/json' --key $(CLIENT_KEY) --cacert $(CLIENT_CERT) https://$(CONNECT_IP):8083/connectors -d '{"name": "mirror-source-connector", "config": {"connector.class": "org.apache.kafka.connect.mirror.MirrorSourceConnector", "topics": "testtopic", "replication.factor": "1", "source.cluster.bootstrap.servers": "$(REDPANDA_BROKERS)", "source.cluster.security.protocol": "SSL", "source.cluster.ssl.truststore.type": "PKCS12", "source.cluster.ssl.keystore.type": "PKCS12", "target.cluster.bootstrap.servers": "$(EXTRA_BROKERS)", "target.cluster.security.protocol": "SSL", "source.cluster.alias": "source", "target.cluster.ssl.truststore.type": "PKCS12", "target.cluster.ssl.keystore.type": "PKCS12"}}' - -.PHONY: lint -lint: - @echo "Running ansible-lint" - @ansible-lint -c .ansible-lint - -.PHONY: dev-tiered-storage -dev-tiered-storage: ansible-prereqs - @mkdir -p $(ARTIFACT_DIR)/logs - ansible-playbook ansible/provision-cluster-tiered-storage.yml --private-key $(PRIVATE_KEY) --extra-vars redpanda_broker_no_log=false --extra-vars development_build=true --extra-vars segment_upload_interval=$(SEGMENT_UPLOAD_INTERVAL) --extra-vars cloud_storage_credentials_source=$(CLOUD_STORAGE_CREDENTIALS_SOURCE) From 882a1f631d7e9811938b8a20ea7a966902e31401 Mon Sep 17 00:00:00 2001 From: gene-redpanda <123959009+gene-redpanda@users.noreply.github.com> Date: Wed, 8 Oct 2025 22:17:19 -0500 Subject: [PATCH 7/9] test --- .tasks/ci.yml | 5 ++-- .tasks/cloud.yml | 1 - .tasks/console.yml | 5 ++-- .tasks/mirror.yml | 1 - .tasks/monitor.yml | 5 ++-- .tasks/redpanda.yml | 5 ++-- .tasks/test.yml | 3 +- .tasks/tools.yml | 73 ++++++++++++++++++++++++--------------------- Taskfile.yml | 5 ++++ 9 files changed, 53 insertions(+), 50 deletions(-) diff --git a/.tasks/ci.yml b/.tasks/ci.yml index 967c1e4d..a59dadd6 100644 --- a/.tasks/ci.yml +++ b/.tasks/ci.yml @@ -1,12 +1,11 @@ version: '3' vars: - ARTIFACT_DIR: "{{.PWD}}/artifacts" DEPLOYMENT_ID: '{{.DEPLOYMENT_ID | default "devex-cicd"}}' env: - ANSIBLE_COLLECTIONS_PATH: "{{.ARTIFACT_DIR}}/collections" - ANSIBLE_ROLES_PATH: "{{.ARTIFACT_DIR}}/roles" + ANSIBLE_COLLECTIONS_PATH: "{{.ANSIBLE_COLLECTIONS_PATH}}" + ANSIBLE_ROLES_PATH: "{{.ANSIBLE_ROLES_PATH}}" tasks: default: diff --git a/.tasks/cloud.yml b/.tasks/cloud.yml index af490d72..4504007b 100644 --- a/.tasks/cloud.yml +++ b/.tasks/cloud.yml @@ -8,7 +8,6 @@ vars: ALLOW_FORCE_DESTROY: '{{.ALLOW_FORCE_DESTROY | default "true"}}' VPC_ID: '{{.VPC_ID | default ""}}' DISTRO: '{{.DISTRO | default "ubuntu-focal"}}' - ARTIFACT_DIR: "{{.PWD}}/artifacts" PUBLIC_KEY: "{{.ARTIFACT_DIR}}/testkey.pub" ANSIBLE_INVENTORY: "{{.ARTIFACT_DIR}}/hosts_{{.DEPLOYMENT_ID}}.ini" MACHINE_ARCH: '{{.MACHINE_ARCH | default "x86_64"}}' diff --git a/.tasks/console.yml b/.tasks/console.yml index 9c3e009d..35a7a065 100644 --- a/.tasks/console.yml +++ b/.tasks/console.yml @@ -1,7 +1,6 @@ version: '3' vars: - ARTIFACT_DIR: "{{.PWD}}/artifacts" DEPLOYMENT_ID: '{{.DEPLOYMENT_ID | default "devex-cicd"}}' PRIVATE_KEY: "{{.ARTIFACT_DIR}}/testkey" ANSIBLE_INVENTORY: "{{.ARTIFACT_DIR}}/hosts_{{.DEPLOYMENT_ID}}.ini" @@ -11,8 +10,8 @@ env: OBJC_DISABLE_INITIALIZE_FORK_SAFETY: "YES" ANSIBLE_LOG_PATH: "{{.ARTIFACT_DIR}}/logs/{{.DEPLOYMENT_ID}}.log" ANSIBLE_INVENTORY: "{{.ANSIBLE_INVENTORY}}" - ANSIBLE_COLLECTIONS_PATH: "{{.ARTIFACT_DIR}}/collections" - ANSIBLE_ROLES_PATH: "{{.ARTIFACT_DIR}}/roles" + ANSIBLE_COLLECTIONS_PATH: "{{.ANSIBLE_COLLECTIONS_PATH}}" + ANSIBLE_ROLES_PATH: "{{.ANSIBLE_ROLES_PATH}}" tasks: default: diff --git a/.tasks/mirror.yml b/.tasks/mirror.yml index f37e9790..c4874bef 100644 --- a/.tasks/mirror.yml +++ b/.tasks/mirror.yml @@ -1,7 +1,6 @@ version: '3' vars: - ARTIFACT_DIR: "{{.PWD}}/artifacts" DEPLOYMENT_ID: '{{.DEPLOYMENT_ID | default "devex-cicd"}}' NUM_NODES: '{{.NUM_NODES | default "3"}}' ENABLE_MONITORING: '{{.ENABLE_MONITORING | default "true"}}' diff --git a/.tasks/monitor.yml b/.tasks/monitor.yml index 46ab1403..2728e2e9 100644 --- a/.tasks/monitor.yml +++ b/.tasks/monitor.yml @@ -1,7 +1,6 @@ version: '3' vars: - ARTIFACT_DIR: "{{.PWD}}/artifacts" DEPLOYMENT_ID: '{{.DEPLOYMENT_ID | default "devex-cicd"}}' PRIVATE_KEY: "{{.ARTIFACT_DIR}}/testkey" ANSIBLE_INVENTORY: "{{.ARTIFACT_DIR}}/hosts_{{.DEPLOYMENT_ID}}.ini" @@ -12,8 +11,8 @@ env: OBJC_DISABLE_INITIALIZE_FORK_SAFETY: "YES" ANSIBLE_LOG_PATH: "{{.ARTIFACT_DIR}}/logs/{{.DEPLOYMENT_ID}}.log" ANSIBLE_INVENTORY: "{{.ANSIBLE_INVENTORY}}" - ANSIBLE_COLLECTIONS_PATH: "{{.ARTIFACT_DIR}}/collections" - ANSIBLE_ROLES_PATH: "{{.ARTIFACT_DIR}}/roles" + ANSIBLE_COLLECTIONS_PATH: "{{.ANSIBLE_COLLECTIONS_PATH}}" + ANSIBLE_ROLES_PATH: "{{.ANSIBLE_ROLES_PATH}}" tasks: default: diff --git a/.tasks/redpanda.yml b/.tasks/redpanda.yml index e0500d82..65a8df5a 100644 --- a/.tasks/redpanda.yml +++ b/.tasks/redpanda.yml @@ -1,7 +1,6 @@ version: '3' vars: - ARTIFACT_DIR: "{{.PWD}}/artifacts" DEPLOYMENT_ID: '{{.DEPLOYMENT_ID | default "devex-cicd"}}' PRIVATE_KEY: "{{.ARTIFACT_DIR}}/testkey" ANSIBLE_INVENTORY: "{{.ARTIFACT_DIR}}/hosts_{{.DEPLOYMENT_ID}}.ini" @@ -16,8 +15,8 @@ env: OBJC_DISABLE_INITIALIZE_FORK_SAFETY: "YES" ANSIBLE_LOG_PATH: "{{.ARTIFACT_DIR}}/logs/{{.DEPLOYMENT_ID}}.log" ANSIBLE_INVENTORY: "{{.ANSIBLE_INVENTORY}}" - ANSIBLE_COLLECTIONS_PATH: "{{.ARTIFACT_DIR}}/collections" - ANSIBLE_ROLES_PATH: "{{.ARTIFACT_DIR}}/roles" + ANSIBLE_COLLECTIONS_PATH: "{{.ANSIBLE_COLLECTIONS_PATH}}" + ANSIBLE_ROLES_PATH: "{{.ANSIBLE_ROLES_PATH}}" tasks: default: diff --git a/.tasks/test.yml b/.tasks/test.yml index 5d419a82..710c2fa3 100644 --- a/.tasks/test.yml +++ b/.tasks/test.yml @@ -1,14 +1,13 @@ version: '3' vars: - ARTIFACT_DIR: "{{.PWD}}/artifacts" DEPLOYMENT_ID: '{{.DEPLOYMENT_ID | default "devex-cicd"}}' RPK_PATH: "{{.ARTIFACT_DIR}}/bin/rpk" HOSTS_FILE: "{{.ARTIFACT_DIR}}/hosts_{{.DEPLOYMENT_ID}}.ini" TEST_TOPIC_NAME: '{{.TEST_TOPIC_NAME | default "testtopic"}}' PARTITION_COUNT: '{{.PARTITION_COUNT | default "3"}}' CA_CRT: - sh: echo "${CA_CRT:-{{.PWD}}/ansible/tls/ca/ca.crt}" + sh: echo "${CA_CRT:-{{.ROOT_DIR}}/ansible/tls/ca/ca.crt}" AWS_DEFAULT_REGION: '{{.AWS_DEFAULT_REGION | default "us-west-2"}}' BUCKET_NAME: sh: echo "$(echo {{.DEPLOYMENT_ID}} | sed 's/_/-/g')-bucket" diff --git a/.tasks/tools.yml b/.tasks/tools.yml index 86e74455..7304a982 100644 --- a/.tasks/tools.yml +++ b/.tasks/tools.yml @@ -1,16 +1,10 @@ version: '3' vars: - ARTIFACT_DIR: "{{.PWD}}/artifacts" - ANSIBLE_COLLECTIONS_PATH: "{{.ARTIFACT_DIR}}/collections" - ANSIBLE_ROLES_PATH: "{{.ARTIFACT_DIR}}/roles" - MAC_RPK: "https://github.com/redpanda-data/redpanda/releases/latest/download/rpk-darwin-amd64.zip" - LINUX_RPK: "https://github.com/redpanda-data/redpanda/releases/latest/download/rpk-linux-amd64.zip" - SSH_EMAIL: '{{.SSH_EMAIL | default "test@test.com"}}' CLIENT_NAME: '{{.CLIENT_NAME | default "client"}}' - CLIENT_DIR: "{{.PWD}}/ansible/tls/clients" - CA_DIR: "{{.PWD}}/ansible/tls/ca" - CERT_DIR: "{{.PWD}}/ansible/tls/certs" + CLIENT_DIR: "{{.ROOT_DIR}}/ansible/tls/clients" + CA_DIR: "{{.ROOT_DIR}}/ansible/tls/ca" + CERT_DIR: "{{.ROOT_DIR}}/ansible/tls/certs" tasks: default: @@ -30,23 +24,32 @@ tasks: _ansible-collections: internal: true desc: "Install Ansible collections from requirements.yml" + sources: + - "{{.ROOT_DIR}}/requirements.yml" + generates: + - "{{.ANSIBLE_COLLECTIONS_PATH}}/ansible_collections" cmds: - mkdir -p {{.ANSIBLE_COLLECTIONS_PATH}} - - ansible-galaxy collection install -r {{.PWD}}/requirements.yml --force -p {{.ANSIBLE_COLLECTIONS_PATH}} - status: - - test -d {{.ANSIBLE_COLLECTIONS_PATH}}/ansible_collections + - ansible-galaxy collection install -r {{.ROOT_DIR}}/requirements.yml --force -p {{.ANSIBLE_COLLECTIONS_PATH}} _ansible-roles: internal: true desc: "Install Ansible roles from requirements.yml" + sources: + - "{{.ROOT_DIR}}/requirements.yml" + generates: + - "{{.ANSIBLE_ROLES_PATH}}" cmds: - mkdir -p {{.ANSIBLE_ROLES_PATH}} - - ansible-galaxy role install -r {{.PWD}}/requirements.yml --force -p {{.ANSIBLE_ROLES_PATH}} - status: - - test -d {{.ANSIBLE_ROLES_PATH}} + - ansible-galaxy role install -r {{.ROOT_DIR}}/requirements.yml --force -p {{.ANSIBLE_ROLES_PATH}} rpk: desc: "Install rpk CLI tool" + vars: + MAC_RPK: "https://github.com/redpanda-data/redpanda/releases/latest/download/rpk-darwin-amd64.zip" + LINUX_RPK: "https://github.com/redpanda-data/redpanda/releases/latest/download/rpk-linux-amd64.zip" + generates: + - "{{.ARTIFACT_DIR}}/bin/rpk" cmds: - mkdir -p {{.ARTIFACT_DIR}}/tmp - mkdir -p {{.ARTIFACT_DIR}}/bin @@ -61,21 +64,17 @@ tasks: - chmod 755 {{.ARTIFACT_DIR}}/bin/rpk - rm {{.ARTIFACT_DIR}}/tmp/rpk.zip - echo "rpk installed to {{.ARTIFACT_DIR}}/bin/rpk" - status: - - test -f {{.ARTIFACT_DIR}}/bin/rpk ssh-keys: desc: "Generate SSH keypair for deployment" + vars: + SSH_EMAIL: '{{.SSH_EMAIL | default "test@test.com"}}' + generates: + - "{{.ARTIFACT_DIR}}/testkey" + - "{{.ARTIFACT_DIR}}/testkey.pub" cmds: - - | - if [ ! -f artifacts/testkey ]; then - printf 'y\n' | ssh-keygen -t rsa -b 4096 -C "{{.SSH_EMAIL}}" -N "" -f artifacts/testkey && chmod 0700 artifacts/testkey - echo "SSH keypair generated" - else - echo "SSH keypair already exists" - fi - status: - - test -f artifacts/testkey + - printf 'y\n' | ssh-keygen -t rsa -b 4096 -C "{{.SSH_EMAIL}}" -N "" -f {{.ARTIFACT_DIR}}/testkey && chmod 0700 {{.ARTIFACT_DIR}}/testkey + - echo "SSH keypair generated" tls-certs: desc: "Generate client TLS certificates" @@ -88,31 +87,37 @@ tasks: _cert-client-dir: internal: true + generates: + - "{{.CLIENT_DIR}}" cmds: - mkdir -p {{.CLIENT_DIR}} - status: - - test -d {{.CLIENT_DIR}} _cert-client-key: internal: true + generates: + - "{{.CLIENT_DIR}}/{{.CLIENT_NAME}}.key" cmds: - openssl genrsa -out {{.CLIENT_DIR}}/{{.CLIENT_NAME}}.key 2048 - status: - - test -f {{.CLIENT_DIR}}/{{.CLIENT_NAME}}.key _cert-client-csr: internal: true + sources: + - "{{.CLIENT_DIR}}/{{.CLIENT_NAME}}.key" + generates: + - "{{.CLIENT_DIR}}/{{.CLIENT_NAME}}.csr" cmds: - openssl req -new -key {{.CLIENT_DIR}}/{{.CLIENT_NAME}}.key -out {{.CLIENT_DIR}}/{{.CLIENT_NAME}}.csr -subj "/CN={{.CLIENT_NAME}}" - status: - - test -f {{.CLIENT_DIR}}/{{.CLIENT_NAME}}.csr _cert-client-crt: internal: true + sources: + - "{{.CLIENT_DIR}}/{{.CLIENT_NAME}}.csr" + - "{{.CA_DIR}}/ca.crt" + - "{{.CA_DIR}}/ca.key" + generates: + - "{{.CLIENT_DIR}}/{{.CLIENT_NAME}}.crt" cmds: - openssl x509 -req -in {{.CLIENT_DIR}}/{{.CLIENT_NAME}}.csr -CA {{.CA_DIR}}/ca.crt -CAkey {{.CA_DIR}}/ca.key -CAcreateserial -out {{.CLIENT_DIR}}/{{.CLIENT_NAME}}.crt -days 365 -sha256 - status: - - test -f {{.CLIENT_DIR}}/{{.CLIENT_NAME}}.crt clean-certs: desc: "Remove all TLS certificates" diff --git a/Taskfile.yml b/Taskfile.yml index 1feabd9e..a127432d 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -3,6 +3,11 @@ version: '3' dotenv: ['.env'] +vars: + ARTIFACT_DIR: "{{.ROOT_DIR}}/artifacts" + ANSIBLE_COLLECTIONS_PATH: "{{.ARTIFACT_DIR}}/collections" + ANSIBLE_ROLES_PATH: "{{.ARTIFACT_DIR}}/roles" + includes: tools: .tasks/tools.yml cloud: .tasks/cloud.yml From 29e97432187bca380e875c2adf8cff4106c87ce0 Mon Sep 17 00:00:00 2001 From: gene-redpanda <123959009+gene-redpanda@users.noreply.github.com> Date: Thu, 9 Oct 2025 10:17:56 -0500 Subject: [PATCH 8/9] fix: ensures no ssh regen When run during the connect test without this particular method of testing for the existence of keys the connect test fails. --- .tasks/tools.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.tasks/tools.yml b/.tasks/tools.yml index 7304a982..fcabfed5 100644 --- a/.tasks/tools.yml +++ b/.tasks/tools.yml @@ -75,6 +75,9 @@ tasks: cmds: - printf 'y\n' | ssh-keygen -t rsa -b 4096 -C "{{.SSH_EMAIL}}" -N "" -f {{.ARTIFACT_DIR}}/testkey && chmod 0700 {{.ARTIFACT_DIR}}/testkey - echo "SSH keypair generated" + status: + - test -f {{.ARTIFACT_DIR}}/testkey + - test -f {{.ARTIFACT_DIR}}/testkey.pub tls-certs: desc: "Generate client TLS certificates" From 7d3f336d6e4ac7db727385b05de05638bf9b1d57 Mon Sep 17 00:00:00 2001 From: gene-redpanda <123959009+gene-redpanda@users.noreply.github.com> Date: Thu, 9 Oct 2025 13:05:42 -0500 Subject: [PATCH 9/9] ci: move citsubus to new concurrency group to reduce test time --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index e5eca9e6..d68bcd3b 100755 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -253,7 +253,7 @@ steps: - AWS_DEFAULT_REGION - label: unstable aws ubuntu tiered large key: aws-us-ubuntu-ts-large - concurrency_group: unstable + concurrency_group: aws-ub concurrency: 1 command: DEPLOYMENT_ID=ci-ts-ub-us-lg-`tr -dc a-z0-9