gke-managed-hyperdisk.yaml

# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
---
blueprint_name: gke-managed-hyperdisk
vars:
  project_id:  ## Set GCP Project ID Here ##
  deployment_name: gke-managed-hd
  region: europe-west1
  zone: europe-west1-b

  # Cidr block containing the IP of the machine calling terraform.
  # The following line must be updated for this example to work.
  authorized_cidr: <your-ip-address>/32

deployment_groups:
- group: primary
  modules:
  - id: network
    source: modules/network/vpc
    settings:
      subnetwork_name: gke-subnet-hyperdisk
      secondary_ranges:
        gke-subnet-hyperdisk:
        - range_name: pods
          ip_cidr_range: 10.4.0.0/14
        - range_name: services
          ip_cidr_range: 10.0.32.0/20

  - id: node_pool_service_account
    source: community/modules/project/service-account
    settings:
      name: gke-np-sa
      project_roles:
      - logging.logWriter
      - monitoring.metricWriter
      - monitoring.viewer
      - stackdriver.resourceMetadata.writer
      - storage.objectViewer
      - artifactregistry.reader

  - id: workload_service_account
    source: community/modules/project/service-account
    settings:
      name: gke-wl-sa
      project_roles:
      - logging.logWriter
      - monitoring.metricWriter
      - monitoring.viewer
      - stackdriver.resourceMetadata.writer
      - storage.objectAdmin
      - artifactregistry.reader

  - id: gke_cluster
    source: modules/scheduler/gke-cluster
    use: [network, workload_service_account]
    settings:
      release_channel: RAPID
      enable_persistent_disk_csi: true # enable Hyperdisk for the cluster
      configure_workload_identity_sa: true
      enable_private_endpoint: false  # Allows for access from authorized public IPs
      master_authorized_networks:
      - display_name: deployment-machine
        cidr_block: $(vars.authorized_cidr)
      maintenance_exclusions:
      - name: no-minor-or-node-upgrades-indefinite
        start_time: "2024-12-01T00:00:00Z"
        end_time: "2025-12-22T00:00:00Z"
        exclusion_scope: NO_MINOR_OR_NODE_UPGRADES
    outputs: [instructions]

  ### Set up storage class and persistent volume claim for Hyperdisk ###
  - id: hyperdisk-balanced-setup
    source: modules/file-system/gke-storage
    use: [gke_cluster]
    settings:
      storage_type: Hyperdisk-balanced
      access_mode: ReadWriteOnce
      sc_volume_binding_mode: Immediate
      sc_reclaim_policy: Delete
      sc_topology_zones: [$(vars.zone)]
      pvc_count: 1
      capacity_gb: 100

  - id: hyperdisk-throughput-setup
    source: modules/file-system/gke-storage
    use: [gke_cluster]
    settings:
      storage_type: Hyperdisk-throughput
      access_mode: ReadWriteOnce
      sc_volume_binding_mode: Immediate
      sc_reclaim_policy: Delete
      sc_topology_zones: [$(vars.zone)]
      pvc_count: 1
      capacity_gb: 5000

  - id: hyperdisk-extreme-setup
    source: modules/file-system/gke-storage
    use: [gke_cluster]
    settings:
      storage_type: Hyperdisk-extreme
      access_mode: ReadWriteOnce
      sc_volume_binding_mode: Immediate
      sc_reclaim_policy: Delete
      sc_topology_zones: [$(vars.zone)]
      pvc_count: 1
      capacity_gb: 100

  - id: sample-pool
    source: modules/compute/gke-node-pool
    use: [gke_cluster, node_pool_service_account]
    settings:
      name: sample-pool
      zones: [$(vars.zone)]
      machine_type: c3-standard-88 # Hyperdisk-extreme requires C3 machine with 88 or more vCPUs
      auto_upgrade: true

  # This is an example job that will install and run an `fio`benchmark against the hyperdisk volumes.
  # For more FIO tests, see https://cloud.google.com/compute/docs/disks/benchmark-hyperdisk-performance
  - id: fio-bench-job-template
    source: modules/compute/gke-job-template
    use:
    - gke_cluster
    - hyperdisk-balanced-setup
    - hyperdisk-extreme-setup
    - hyperdisk-throughput-setup
    settings:
      name: fio-benchmark
      image: ubuntu:latest
      security_context:  # to make sure the job have enough access to install the fio packages
      - key: runAsUser
        value: 0
      - key: runAsGroup
        value: 100
      - key: fsGroup
        value: 100
      command:
      - bash
      - -c
      - |

        set -eux

        cleanup() {
          # This function will be called on script exit
          if [ -n "${TAG:-}" ]; then
            echo "--- Cleaning up temporary directories for tag ${TAG} ---"
            rm -rf "/data/hyperdisk-balanced-pvc-0/fio-benchmarks-${TAG}"
            rm -rf "/data/hyperdisk-extreme-pvc-0/fio-benchmarks-${TAG}"
            rm -rf "/data/hyperdisk-throughput-pvc-0/fio-benchmarks-${TAG}"
          fi
          }
        trap cleanup EXIT

        export DEBIAN_FRONTEND=noninteractive

        # Install fio
        apt update -y && apt install -y fio

        # Use a tag to create a unique path for tests
        TAG=`date +%s`

        # Verify mountpoints
        df -h
        mountpoint /data/hyperdisk-balanced-pvc-0
        mountpoint /data/hyperdisk-extreme-pvc-0
        mountpoint /data/hyperdisk-throughput-pvc-0

        # Create temporary directory for fio benchmarks
        mkdir -p "/data/hyperdisk-balanced-pvc-0/fio-benchmarks-${TAG}"
        mkdir -p "/data/hyperdisk-extreme-pvc-0/fio-benchmarks-${TAG}"
        mkdir -p "/data/hyperdisk-throughput-pvc-0/fio-benchmarks-${TAG}"

        # Perform hyperdisk balanced performance (Mixed IOPS) test
        fio --name=hyperdisk-balanced-iops --ioengine=libaio --iodepth=256 --rw=randrw \
        --bs=4k --direct=1 --size=10G --numjobs=16 --group_reporting --time_based --runtime=300s \
        --ramp_time=10s --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 \
        --directory="/data/hyperdisk-balanced-pvc-0/fio-benchmarks-${TAG}" --filename_format=fiotest-balanced-iops

        # Perform hyperdisk extreme performance test (Max IOPS)
        fio --name=hyperdisk-extreme-iops --ioengine=libaio --iodepth=256 --rw=randwrite \
        --bs=4k --direct=1 --size=10G --numjobs=32 --group_reporting --time_based --runtime=300s --ramp_time=10s \
        --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 \
        --directory="/data/hyperdisk-extreme-pvc-0/fio-benchmarks-${TAG}" --filename_format=fiotest-extreme-iops

        # Perform hyperdisk throughput performance test
        fio --name=hyperdisk-throughput-bw --ioengine=libaio --iodepth=64 --rw=write --bs=1M \
        --direct=1 --size=10G --numjobs=32 --group_reporting --time_based --runtime=300s --ramp_time=10s \
        --iodepth_batch_submit=64 --iodepth_batch_complete_max=64 \
        --directory="/data/hyperdisk-throughput-pvc-0/fio-benchmarks-${TAG}" --filename_format=fiotest-throughput-bw
      node_count: 1

    outputs: [instructions]