scale-test-100-gce.yaml

name: 100 Nodes Scale Test (scale-100)

on:
  schedule:
    - cron: '39 0 * * 1-5'

  workflow_dispatch:
    inputs:
      PR-number:
        description: "Pull request number."
        required: true
      context-ref:
        description: "Context in which the workflow runs. If PR is from a fork, will be the PR target branch (general case). If PR is NOT from a fork, will be the PR branch itself (this allows committers to test changes to workflows directly from PRs)."
        required: true
      SHA:
        description: "SHA under test (head of the PR branch)."
        required: true
      base-SHA:
        description: "SHA of the base branch (target branch of the PR)."
        required: false
      extra-args:
        description: "[JSON object] Arbitrary arguments passed from the trigger comment via regex capture group. Parse with 'fromJson(inputs.extra-args).argName' in workflow."
        required: false
        default: '{}'

# For testing uncomment following lines:
#  push:
#    branches:
#      - your_branch_name

permissions:
  # To be able to retrieve artifacts information
  actions: read
  # To be able to access the repository with actions/checkout
  contents: read
  # To be able to request the JWT from GitHub's OIDC provider
  id-token: write
  # To allow retrieving information from the PR API
  pull-requests: read
  # To be able to set commit status
  statuses: write

concurrency:
  # Structure:
  # - Workflow name
  # - Event type
  # - A unique identifier depending on event type:
  #   - schedule: SHA
  #   - workflow_dispatch: PR number
  #
  # This structure ensures a unique concurrency group name is generated for each
  # type of testing, such that re-runs will cancel the previous run.
  group: |
    ${{ github.workflow }}
    ${{ github.event_name }}
    ${{
      (github.event_name == 'schedule' && github.sha) ||
      (github.event_name == 'workflow_dispatch' && github.event.inputs.PR-number)
    }}
  cancel-in-progress: true

env:
  # renovate: datasource=golang-version depName=go
  go_version: 1.25.7
  test_name: scale-100
  cluster_name: ${{ github.run_id }}-${{ github.run_attempt }}
  # renovate: datasource=docker depName=google/cloud-sdk
  gcloud_version: 555.0.0

jobs:
  echo-inputs:
    if: ${{ github.event_name == 'workflow_dispatch' }}
    name: Echo Workflow Dispatch Inputs
    runs-on: ubuntu-24.04
    steps:
      - name: Echo Workflow Dispatch Inputs
        run: |
          echo '${{ tojson(inputs) }}'
  commit-status-start:
    name: Commit Status Start
    runs-on: ubuntu-24.04
    steps:
      - name: Set initial commit status
        uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1
        with:
          sha: ${{ inputs.SHA || github.sha }}

  install-and-scaletest:
    runs-on: ubuntu-24.04
    name: Install and Scale Test
    timeout-minutes: 300
    env:
      job_name: "Install and Scale Test"
    steps:
      - name: Set commit status to pending
        uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1
        with:
          sha: ${{ inputs.SHA || github.sha }}
          status: pending

      - name: Checkout context ref (trusted)
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ inputs.context-ref || github.sha }}
          persist-credentials: false

      - name: Set Environment Variables
        uses: ./.github/actions/set-env-variables

      - name: Set up job variables
        id: vars
        run: |
          if [ "${{ github.event_name }}" = "workflow_dispatch" ] ; then
            SHA="${{ inputs.SHA }}"
          else
            SHA="${{ github.sha }}"
          fi

          # Run test against specific commit SHA if provided
          if [[ '${{ inputs.extra-args }}' =~ sha=([a-f0-9]+) ]]; then
            SHA=${BASH_REMATCH[1]}
          fi

          # Retrieve the desired version from the arguments.
          if [[ '${{ inputs.extra-args }}' =~ version=([-+.0-9a-z]+) ]]; then
            VERSION=${BASH_REMATCH[1]}
          else
            VERSION=""
          fi

          # Retrive number of workload nodes from the arguments.
          if [[ '${{ inputs.extra-args }}' =~ nodes=([0-9+]+) ]]; then
            WORKLOAD_NODES=${BASH_REMATCH[1]}
          else
            WORKLOAD_NODES=100
          fi

          if [ "${WORKLOAD_NODES}" -gt 1000 ]; then
            echo "WORKLOAD_NODES cannot be larger than 1000, got ${WORKLOAD_NODES}"
            exit 1
          fi

          # Adding k8s.local to the end makes kops happy
          # has stricter DNS naming requirements.
          CLUSTER_NAME="${{ env.test_name }}-${{ env.cluster_name }}.k8s.local"

          CILIUM_INSTALL_DEFAULTS=" \
            --set=pprof.enabled=true \
            --set=prometheus.enabled=true \
            --set=cluster.name=${{ env.cluster_name }} \
            --set=k8sServiceHost=api.internal.${CLUSTER_NAME} \
            --set=k8sServicePort=443 \
            --set=kubeProxyReplacement=true \
            --set=operator.replicas=1 \
            --set=updateStrategy.rollingUpdate.maxUnavailable=100% \
            --set=ipam.mode=\"cluster-pool\" \
            --set=ipam.operator.clusterPoolIPv4PodCIDRList[0]=\"10.0.0.0/9\" \
            --wait=false"

          # only add SHA to the image tags if VERSION is not set
          if [ -z "${VERSION}" ]; then
            CILIUM_INSTALL_DEFAULTS+=" --set=image.repository=quay.io/${{ env.QUAY_ORGANIZATION_DEV }}/cilium-ci \
            --set=image.useDigest=false \
            --set=image.tag=${SHA} \
            --set=operator.image.repository=quay.io/${{ env.QUAY_ORGANIZATION_DEV }}/operator \
            --set=operator.image.suffix=-ci \
            --set=operator.image.tag=${SHA} \
            --set=operator.image.useDigest=false \
            --set=clustermesh.apiserver.image.repository=quay.io/${{ env.QUAY_ORGANIZATION_DEV }}/clustermesh-apiserver-ci \
            --set=clustermesh.apiserver.image.tag=${SHA} \
            --set=clustermesh.apiserver.image.useDigest=false \
            --set=hubble.relay.image.repository=quay.io/${{ env.QUAY_ORGANIZATION_DEV }}/hubble-relay-ci \
            --set=hubble.relay.image.tag=${SHA} \
            --set=hubble.relay.image.useDigest=false"
          fi

          echo SHA=${SHA} >> $GITHUB_OUTPUT
          echo cilium_install_defaults=${CILIUM_INSTALL_DEFAULTS} >> $GITHUB_OUTPUT
          echo CLUSTER_NAME=${CLUSTER_NAME} >> $GITHUB_OUTPUT
          echo VERSION=${VERSION} >> $GITHUB_OUTPUT
          echo WORKLOAD_NODES=${WORKLOAD_NODES} >> $GITHUB_OUTPUT

      - name: Checkout pull request branch (NOT TRUSTED)
        if: ${{ steps.vars.outputs.version == '' }} # We don't need to checkout the PR if it's released version
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ steps.vars.outputs.sha }}
          persist-credentials: false
          path: untrusted
          sparse-checkout: |
            install/kubernetes/cilium

      - name: Wait for images
        if: ${{ steps.vars.outputs.version == '' }} # We don't need to wait for images if it's released version
        uses: ./.github/actions/wait-for-images
        with:
          SHA: ${{ steps.vars.outputs.SHA }}
          images: cilium-ci operator-generic-ci hubble-relay-ci

      - name: Install Go
        uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0
        with:
          go-version: ${{ env.go_version }}

      - name: Install Kops
        uses: cilium/scale-tests-action/install-kops@8535f579142fa598c3b2a895cfb248cd492d7c5a # main

      - name: Setup gcloud credentials
        uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0
        with:
          workload_identity_provider: ${{ secrets.GCP_PERF_WORKLOAD_IDENTITY_PROVIDER }}
          service_account: ${{ secrets.GCP_PERF_SA }}
          project_id: ${{ secrets.GCP_PERF_PROJECT_ID }}
          create_credentials_file: true
          export_environment_variables: true

      - name: Setup gcloud CLI
        uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1
        with:
          version: ${{ env.gcloud_version }}

      - name: Clone ClusterLoader2
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          repository: kubernetes/perf-tests
          # Avoid using renovate to update this dependency because: (1)
          # perf-tests does not tag or release, so renovate will pull
          # all updates to the default branch and (2) continually
          # updating CL2 may impact the stability of the scale test
          # results.
          ref: 6eb52ac89d5de15a0ad13cfeb2b2026e57ce4f64
          persist-credentials: false
          sparse-checkout: clusterloader2
          path: perf-tests

      - name: Setup CL2
        run: |
          # CL2 needs ssh access to control plane nodes
          gcloud compute config-ssh

          # Copy the custom configs to the folder where CL2 expects them.
          cp -r .github/actions/cl2-modules ./perf-tests/clusterloader2/testing/custom

          cd ./perf-tests/clusterloader2

          # CL2 hardcodes module paths to live in ./testing/load, even
          # if the path given is relative.
          cp ../../.github/actions/cl2-modules/cilium-agent-pprofs.yaml ./testing/load/
          cp ../../.github/actions/cl2-modules/cilium-metrics.yaml ./testing/load/
          echo \
            '{"CL2_ADDITIONAL_MEASUREMENT_MODULES": ["./cilium-agent-pprofs.yaml", "./cilium-metrics.yaml"]}' \
            > modules.yaml

          go build ./cmd/clusterloader.go

      - name: Deploy cluster
        id: deploy-cluster
        uses: cilium/scale-tests-action/create-cluster@8535f579142fa598c3b2a895cfb248cd492d7c5a # main
        timeout-minutes: 30
        with:
          cluster_name: ${{ steps.vars.outputs.cluster_name }}
          control_plane_size: ${{ steps.vars.outputs.WORKLOAD_NODES <= 100 && 'n2-standard-8' || 'n2-standard-16' }}
          control_plane_count: ${{ steps.vars.outputs.WORKLOAD_NODES <= 100 && 1 || 3 }}
          node_size: ${{ steps.vars.outputs.WORKLOAD_NODES <= 100 && 'n2-standard-8' || 'n2-standard-16' }}
          node_count: 1
          kops_state: ${{ secrets.GCP_PERF_KOPS_STATE_STORE }}
          project_id: ${{ secrets.GCP_PERF_PROJECT_ID }}
          node_cidr: 10.255.0.0/16
          kube_proxy_enabled: false
          sync_cloud_routes: false
          etcd_volume_size: 250
          max_in_flight: 200

      - name: Setup firewall rules
        uses: cilium/scale-tests-action/setup-firewall@8535f579142fa598c3b2a895cfb248cd492d7c5a  # main
        with:
          cluster_name: ${{ steps.vars.outputs.cluster_name }}

      - name: Install Cilium CLI
        uses: cilium/cilium-cli@eb32911890466ddec93da63cf83f5bcb2ae23e26 # v0.19.0
        with:
          skip-build: ${{ env.CILIUM_CLI_SKIP_BUILD }}
          image-repo: ${{ env.CILIUM_CLI_IMAGE_REPO }}
          image-tag: ${{ inputs.SHA || github.sha }}
          repository: ${{ env.CILIUM_CLI_RELEASE_REPO }}
          release-version: ${{ env.CILIUM_CLI_VERSION }}

      - name: Display version info of installed tools
        run: |
          echo "--- go ---"
          go version
          echo "--- cilium-cli ---"
          cilium version --client
          echo "--- kops ---"
          ./kops version
          echo "--- gcloud ---"
          gcloud version

      - name: Install Cilium
        id: install-cilium
        run: |
          cilium_install_args="${{ steps.vars.outputs.cilium_install_defaults }}"
          if [[ -z "${{ steps.vars.outputs.version }}" ]]; then
            helm install -n kube-system cilium ./untrusted/install/kubernetes/cilium $cilium_install_args
          else
            helm repo add cilium https://helm.cilium.io/
            helm install -n kube-system cilium cilium/cilium --version ${{ steps.vars.outputs.version }} $cilium_install_args
          fi

      - name: Wait for cluster to be ready
        uses: cilium/scale-tests-action/validate-cluster@8535f579142fa598c3b2a895cfb248cd492d7c5a # main
        timeout-minutes: 20
        with:
          cluster_name: ${{ steps.vars.outputs.cluster_name }}
          kops_state: ${{ secrets.GCP_PERF_KOPS_STATE_STORE }}
          interval: 10s

      - name: Run CL2 to setup prometheus
        shell: bash
        working-directory: ./perf-tests/clusterloader2
        env:
          CL2_PROMETHEUS_PVC_ENABLED: "false"
          CL2_PROMETHEUS_SCRAPE_CILIUM_OPERATOR: "true"
          CL2_PROMETHEUS_SCRAPE_CILIUM_AGENT: "true"
          CL2_PROMETHEUS_MEMORY_SCALE_FACTOR: ${{ steps.vars.outputs.WORKLOAD_NODES <= 100 && 12.0 || 24.0 }}
        timeout-minutes: 10
        run: |
          # Don't run any tasks at this point, just setup the monitoring stack
          ./clusterloader \
            -v=2 \
            --testconfig=./testing/custom/common/setup.yaml \
            --testoverrides=./testing/prometheus/not-scrape-kube-proxy.yaml \
            --nodes=${{ steps.vars.outputs.workload_nodes }} \
            --provider=gce \
            --enable-exec-service=false \
            --enable-prometheus-server \
            --tear-down-prometheus-server=false \
            --kubeconfig=$HOME/.kube/config \
            2>&1 | tee cl2-setup.txt

      - name: Create Instance Group for workload deployments
        uses: cilium/scale-tests-action/create-instance-group@8535f579142fa598c3b2a895cfb248cd492d7c5a # main
        timeout-minutes: 30
        with:
          cluster_name: ${{ steps.vars.outputs.cluster_name }}
          node_size: e2-medium
          node_count: ${{ steps.vars.outputs.workload_nodes }}
          ig_name: workloads
          kops_state: ${{ secrets.GCP_PERF_KOPS_STATE_STORE }}

      - name: Wait for workloads nodes to be ready
        timeout-minutes: 20
        run: |
          cnt=0

          # Allow for a 2 per thousand toleration in the expected nodes count, to
          # continue with the test even if a tiny fraction of the workers failed
          # bootstrapping and joining the cluster.
          desired=$(( ${{ steps.vars.outputs.WORKLOAD_NODES}} - ${{ steps.vars.outputs.WORKLOAD_NODES}} * 2 / 1000 ))

          # A more idiomatic way would be using the kops.k8s.io/instancegroup=workloads
          # label selector, but kops appears to be significantly slow at reconciling
          # the labels to the node objects. Hence, let's just grep for the node names.
          while [[ "$cnt" -lt "$desired" ]]; do
            # shellcheck disable=SC2196
            cnt=$(kubectl get nodes --no-headers | egrep -c 'workloads-[-a-z0-9]+\s+Ready\s' || true)
            echo "workloads nodes count: $cnt"
            sleep 10
          done

      - name: Setup firewall rules
        uses: cilium/scale-tests-action/setup-firewall@8535f579142fa598c3b2a895cfb248cd492d7c5a # main
        with:
          cluster_name: ${{ steps.vars.outputs.cluster_name }}
          create_native_routing_firewall: 'false'

      - name: Wait for Cilium status to be ready
        run: |
          kubectl rollout status -n kube-system ds/cilium
          kubectl rollout status -n kube-system ds/cilium-envoy

      - name: Run CL2
        id: run-cl2
        working-directory: ./perf-tests/clusterloader2
        shell: bash
        timeout-minutes: ${{ steps.vars.outputs.WORKLOAD_NODES <= 100 && 60 || 180 }}
        env:
          CL2_ENABLE_PVS: "false"
          CL2_ENABLE_NETWORKPOLICIES: "true"
          CL2_ALLOWED_SLOW_API_CALLS: 1
          CL2_SCHEDULER_THROUGHPUT_THRESHOLD: 0
          CL2_PROMETHEUS_PVC_ENABLED: "false"
          CL2_PROMETHEUS_SCRAPE_CILIUM_OPERATOR: "true"
          CL2_PROMETHEUS_SCRAPE_CILIUM_AGENT: "true"
          CL2_PROMETHEUS_MEMORY_SCALE_FACTOR: ${{ steps.vars.outputs.WORKLOAD_NODES <= 100 && 12.0 || 24.0 }}
          CL2_PPROF_INTERVAL_SECONDS: ${{ steps.vars.outputs.WORKLOAD_NODES <= 100 && 60 || 300 }}
          CL2_ENABLE_VIOLATIONS: ${{ steps.vars.outputs.WORKLOAD_NODES <= 100 && 'true' || 'false' }}
        run: |
          ./clusterloader \
            -v=2 \
            ${{ steps.vars.outputs.WORKLOAD_NODES <= 100 && '--testconfig=./testing/load/config.yaml' || '' }}  \
            ${{ steps.vars.outputs.WORKLOAD_NODES <= 100 && '--testconfig=./testing/custom/common/restart.yaml' || '' }}  \
            --testconfig=./testing/custom/netpol/config.yaml \
            --testconfig=./testing/custom/common/restart.yaml \
            --testconfig=./testing/custom/servicechurn/config.yaml \
            --provider=gce \
            --enable-prometheus-server \
            --tear-down-prometheus-server=false \
            --nodes=${{ steps.vars.outputs.workload_nodes }} \
            --report-dir=./report \
            --experimental-prometheus-snapshot-to-report-dir=true \
            --kubeconfig=$HOME/.kube/config \
            --testoverrides=./testing/overrides/load_throughput.yaml \
            --testoverrides=./testing/experiments/use_simple_latency_query.yaml \
            --testoverrides=./testing/prometheus/not-scrape-kube-proxy.yaml \
            --testoverrides=./modules.yaml \
            2>&1 | tee cl2-output.txt

      - name: Run common post steps
        if: ${{ always() && steps.install-cilium.outcome != 'skipped' }}
        uses: ./.github/actions/post-logic
        with:
          capture_sysdump: ${{ steps.vars.outputs.WORKLOAD_NODES <= 100 && true || false }}
          always_capture_sysdump: ${{ steps.vars.outputs.WORKLOAD_NODES <= 100 && true || false }}
          artifacts_suffix: "final"
          job_status: "${{ job.status }}"
          capture_features_tested: ${{ steps.run-cl2.outcome != 'skipped' }}

      - name: Cleanup cluster
        if: ${{ always() && steps.deploy-cluster.outcome != 'skipped' }}
        uses: cilium/scale-tests-action/cleanup-cluster@8535f579142fa598c3b2a895cfb248cd492d7c5a # main
        with:
          cluster_name: ${{ steps.vars.outputs.cluster_name }}
          kops_state: ${{ secrets.GCP_PERF_KOPS_STATE_STORE }}

      - name: Export results and sysdump to GS bucket
        if: ${{ always() && steps.run-cl2.outcome != 'skipped' && steps.run-cl2.outcome != 'cancelled' }}
        uses: cilium/scale-tests-action/export-results@8535f579142fa598c3b2a895cfb248cd492d7c5a # main
        with:
          test_name: ${{ env.test_name }}
          tested_version: ${{ steps.vars.outputs.version }}
          tested_sha: ${{ steps.vars.outputs.sha }}
          results_bucket: ${{ env.GCP_PERF_RESULTS_BUCKET }}
          artifacts: ./perf-tests/clusterloader2/report/
          other_files: ${{ steps.vars.outputs.WORKLOAD_NODES <= 100 && 'cilium-sysdump-final.zip' || '' }} ./perf-tests/clusterloader2/cl2-output.txt


  merge-upload-and-status:
    name: Merge Upload and Status
    if: ${{ always() }}
    needs: install-and-scaletest
    uses: ./.github/workflows/common-post-jobs.yaml
    secrets: inherit
    with:
      context-ref: ${{ inputs.context-ref || github.sha }}
      sha: ${{ inputs.SHA || github.sha }}
      success: ${{ needs.install-and-scaletest.result == 'success' }}