diff --git a/k8s-training/README.md b/k8s-training/README.md index a0249cf24..28d9a3f1a 100644 --- a/k8s-training/README.md +++ b/k8s-training/README.md @@ -127,6 +127,24 @@ You can use Filestore to add external storage to K8s clusters, this allows you t For more information on how to access storage in K8s, refer [here](#accessing-storage). +### Shared filesystem CSI automation + +When a shared filesystem is present, either because this stack created it or because `existing_filestore` was provided, Terraform can also install the Nebius Shared Filesystem CSI driver and promote its StorageClass to the cluster default. + +```hcl +enable_filestore = true +existing_filestore = "" # or an existing filesystem ID +filestore_mount_path = "/mnt/data" +filesystem_csi = { + chart_version = "0.1.5" + namespace = "kube-system" + make_default_storage_class = true + previous_default_storage_class_name = "compute-csi-default-sc" +} +``` + +This Terraform automation installs the CSI driver and configures the StorageClass only. Verification, pod-level validation, and cleanup remain in `filesystem-csi-validation/` as an explicit opt-in workflow. + ## Connecting to the cluster ### Preparing the environment diff --git a/k8s-training/filesystem-csi-validation/.gitignore b/k8s-training/filesystem-csi-validation/.gitignore new file mode 100644 index 000000000..95f7491ee --- /dev/null +++ b/k8s-training/filesystem-csi-validation/.gitignore @@ -0,0 +1 @@ +.state/ diff --git a/k8s-training/filesystem-csi-validation/01-verify-node-filesystem-mounts.sh b/k8s-training/filesystem-csi-validation/01-verify-node-filesystem-mounts.sh new file mode 100755 index 000000000..4433a24ba --- /dev/null +++ b/k8s-training/filesystem-csi-validation/01-verify-node-filesystem-mounts.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +# ----------------------------------------------------------------------------- +# File: 01-verify-node-filesystem-mounts.sh +# Purpose: +# Verify that the Nebius Shared Filesystem is mounted on every Kubernetes +# node at the expected host path before any pod-level storage testing begins. +# +# Why We Run This: +# The Nebius CSI workflow in this repo depends on the shared filesystem +# already being attached and mounted on each node. If a node is missing the +# host mount, later PVC or pod checks can fail in ways that are harder to +# diagnose. +# +# Reference Docs: +# https://docs.nebius.com/kubernetes/storage/filesystem-over-csi +# +# Repo Sources of Truth: +# - ../../modules/cloud-init/k8s-cloud-init.tftpl +# - ../main.tf +# +# What This Script Checks: +# - The mount exists at /mnt/data (or the value of MOUNT_POINT) +# - The mount is present in /etc/fstab +# - The mounted filesystem reports capacity via df +# - The target directory exists on the host +# +# Usage: +# ./01-verify-node-filesystem-mounts.sh +# +# Optional Environment Variables: +# TEST_NAMESPACE Namespace used for the temporary node-debugger pods. +# Defaults to the current kubectl namespace or default. +# MOUNT_POINT Host path to validate. Defaults to the Terraform mount. +# DEBUG_IMAGE Image used by kubectl debug. Defaults to ubuntu. +# VERIFY_ALL_NODES When true, validates every node in the cluster. Defaults +# to false. +# TARGET_NODE Specific node to validate. Accepts either +# node/ or . Overrides VERIFY_ALL_NODES. +# +# Created By: Aaron Fagan +# Created On: 2026-03-17 +# Version: 0.1.0 +# ----------------------------------------------------------------------------- +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/common.sh" + +DEBUG_IMAGE="${DEBUG_IMAGE:-ubuntu}" +VERIFY_ALL_NODES="${VERIFY_ALL_NODES:-false}" +TARGET_NODE="${TARGET_NODE:-}" +FAILED=0 + +normalize_node_name() { + local node_name="$1" + if [[ "${node_name}" == node/* ]]; then + printf '%s\n' "${node_name}" + else + printf 'node/%s\n' "${node_name}" + fi +} + +log_step "Starting Nebius Shared Filesystem mount verification" +log_info "Namespace for temporary debug pods: ${TEST_NAMESPACE}" +log_info "Expected mount point: ${MOUNT_POINT}" +log_info "Debug image: ${DEBUG_IMAGE}" + +log_step "Checking required local dependencies" +require_command kubectl +require_command awk +require_command mktemp +log_pass "Required local commands for node mount verification are available" + +log_step "Preparing local state for debugger pod cleanup" +ensure_state_dir +touch "${DEBUG_POD_RECORD_FILE}" +log_info "Debugger pod record file: ${DEBUG_POD_RECORD_FILE}" +log_info "New debugger pods from this run will be appended for later cleanup" + +log_step "Selecting which nodes to validate" +ALL_NODES=() +while IFS= read -r node; do + [[ -n "${node}" ]] && ALL_NODES+=("${node}") +done < <(kubectl get nodes -o name) + +if [[ "${#ALL_NODES[@]}" -eq 0 ]]; then + log_fail "No Kubernetes nodes were returned by kubectl" + exit 1 +fi + +if [[ -n "${TARGET_NODE}" ]]; then + TARGET_NODE="$(normalize_node_name "${TARGET_NODE}")" + NODES_TO_CHECK=("${TARGET_NODE}") + log_info "Using explicitly requested node: ${TARGET_NODE}" +elif [[ "${VERIFY_ALL_NODES}" == "true" ]]; then + NODES_TO_CHECK=("${ALL_NODES[@]}") + log_info "VERIFY_ALL_NODES=true, so every node will be checked" +else + NODES_TO_CHECK=("${ALL_NODES[0]}") + log_info "Defaulting to a single-node validation using: ${NODES_TO_CHECK[0]}" +fi + +log_pass "Selected ${#NODES_TO_CHECK[@]} node(s) for shared filesystem mount validation" + +log_step "Checking Nebius Shared Filesystem mounts on the selected Kubernetes nodes" +for node in "${NODES_TO_CHECK[@]}"; do + echo + echo "------------------------------------------------------------" + echo "=== ${node} ===" + output_file="$(mktemp)" + if ! kubectl debug -n "${TEST_NAMESPACE}" "${node}" \ + --attach=true \ + --quiet \ + --image="${DEBUG_IMAGE}" \ + --profile=sysadmin -- \ + chroot /host sh -lc " + set -eu + echo '[check] Verifying that the Nebius Shared Filesystem is actively mounted at ${MOUNT_POINT}' + mount | awk '\$3 == \"${MOUNT_POINT}\" { print; found=1 } END { exit found ? 0 : 1 }' + echo '[check] Verifying that the mount is persisted in /etc/fstab for node reboot safety' + awk '\$2 == \"${MOUNT_POINT}\" { print; found=1 } END { exit found ? 0 : 1 }' /etc/fstab + echo '[check] Verifying that the mounted filesystem reports capacity and is readable' + df -h ${MOUNT_POINT} + echo '[check] Verifying that the target directory exists on the host' + test -d ${MOUNT_POINT} + echo '[result] PASS: shared filesystem host mount is active and healthy at ${MOUNT_POINT} on this node' + " 2>&1 | tee "${output_file}"; then + FAILED=1 + echo "[result] FAIL: ${node} does not have a healthy shared filesystem mount at ${MOUNT_POINT}" >&2 + fi + + debug_pod_name="$(awk '/Creating debugging pod / { print $4 }' "${output_file}" | tail -n 1)" + if [[ -n "${debug_pod_name}" ]]; then + printf '%s %s\n' "${TEST_NAMESPACE}" "${debug_pod_name}" >> "${DEBUG_POD_RECORD_FILE}" + fi + rm -f "${output_file}" +done + +if [[ "${FAILED}" -eq 0 ]]; then + log_step "Shared filesystem mount verification completed successfully" + log_info "All checked nodes reported a healthy mount at ${MOUNT_POINT}" +else + log_step "Shared filesystem mount verification completed with failures" + log_info "Review the node output above for the failing mount checks" +fi + +exit "${FAILED}" diff --git a/k8s-training/filesystem-csi-validation/02-run-csi-smoke-test.sh b/k8s-training/filesystem-csi-validation/02-run-csi-smoke-test.sh new file mode 100755 index 000000000..07e87a5c3 --- /dev/null +++ b/k8s-training/filesystem-csi-validation/02-run-csi-smoke-test.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# ----------------------------------------------------------------------------- +# File: 02-run-csi-smoke-test.sh +# Purpose: +# Run a minimal end-to-end validation using one PVC and one pod that mounts +# the shared volume at /data. +# +# Why We Run This: +# This is the fastest proof that the Terraform-managed default StorageClass +# works, the PVC binds, and a pod can read and write data through the +# shared filesystem exposed through CSI. +# +# Reference Docs: +# https://docs.nebius.com/kubernetes/storage/filesystem-over-csi +# +# What This Script Does: +# - Applies the single-pod smoke test manifest +# - Waits for the PVC to bind +# - Verifies that the PVC inherited the expected default StorageClass +# - Waits for the pod to become ready +# - Writes and reads a small probe file inside /data +# +# Usage: +# ./02-run-csi-smoke-test.sh +# +# Optional Environment Variables: +# TEST_NAMESPACE Namespace where the validation resources should be created. +# Defaults to the current kubectl namespace or default. +# +# Manifest Used: +# manifests/01-csi-smoke-test.yaml +# +# Created By: Aaron Fagan +# Created On: 2026-03-17 +# Version: 0.1.0 +# ----------------------------------------------------------------------------- +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/common.sh" + +log_step "Starting single-pod shared filesystem smoke test" +log_info "Namespace: ${TEST_NAMESPACE}" +log_info "Manifest: ${FILESYSTEM_SMOKE_MANIFEST_PATH}" +log_info "PVC name: ${FILESYSTEM_SMOKE_PVC_NAME}" +log_info "Pod name: ${FILESYSTEM_SMOKE_POD_NAME}" +log_info "Expected default StorageClass: ${FILESYSTEM_DEFAULT_STORAGE_CLASS_NAME}" + +log_step "Checking required local dependencies" +require_command kubectl +log_pass "Required local commands for the smoke test are available" + +log_step "Applying the smoke test manifest" +kubectl apply -n "${TEST_NAMESPACE}" -f "${FILESYSTEM_SMOKE_MANIFEST_PATH}" +log_pass "Smoke test manifest applied in namespace '${TEST_NAMESPACE}'" + +log_step "Waiting for the smoke test PVC to bind" +kubectl wait -n "${TEST_NAMESPACE}" \ + --for=jsonpath='{.status.phase}'=Bound \ + "pvc/${FILESYSTEM_SMOKE_PVC_NAME}" \ + --timeout=120s +log_info "PVC '${FILESYSTEM_SMOKE_PVC_NAME}' is bound" +log_pass "Smoke test PVC '${FILESYSTEM_SMOKE_PVC_NAME}' bound successfully" + +log_step "Verifying that the smoke test PVC inherited the default StorageClass" +SMOKE_STORAGE_CLASS_NAME="$(kubectl get pvc -n "${TEST_NAMESPACE}" "${FILESYSTEM_SMOKE_PVC_NAME}" -o jsonpath='{.spec.storageClassName}')" +if [[ -z "${SMOKE_STORAGE_CLASS_NAME}" ]]; then + log_fail "Smoke test PVC '${FILESYSTEM_SMOKE_PVC_NAME}' did not receive a StorageClass from the cluster default" + exit 1 +fi +if [[ "${SMOKE_STORAGE_CLASS_NAME}" != "${FILESYSTEM_DEFAULT_STORAGE_CLASS_NAME}" ]]; then + log_fail "Smoke test PVC '${FILESYSTEM_SMOKE_PVC_NAME}' used StorageClass '${SMOKE_STORAGE_CLASS_NAME}', expected '${FILESYSTEM_DEFAULT_STORAGE_CLASS_NAME}'" + exit 1 +fi +log_info "PVC '${FILESYSTEM_SMOKE_PVC_NAME}' was assigned StorageClass '${SMOKE_STORAGE_CLASS_NAME}'" +log_pass "Smoke test PVC '${FILESYSTEM_SMOKE_PVC_NAME}' inherited the expected default StorageClass" + +log_step "Waiting for the smoke test pod to become ready" +kubectl wait -n "${TEST_NAMESPACE}" \ + --for=condition=Ready \ + "pod/${FILESYSTEM_SMOKE_POD_NAME}" \ + --timeout=120s +log_info "Pod '${FILESYSTEM_SMOKE_POD_NAME}' is ready" +log_pass "Smoke test pod '${FILESYSTEM_SMOKE_POD_NAME}' reached Ready state" + +log_step "Writing and reading a probe file through the mounted volume" +kubectl exec -n "${TEST_NAMESPACE}" "${FILESYSTEM_SMOKE_POD_NAME}" -- sh -lc ' + set -eu + df -h /data + echo ok > /data/probe.txt + ls -l /data + cat /data/probe.txt +' +log_pass "Pod '${FILESYSTEM_SMOKE_POD_NAME}' successfully wrote and read the probe file on the shared volume" + +log_step "Smoke test completed successfully" +log_info "The PVC inherited the cluster default StorageClass and the mounted shared filesystem accepted a write and returned the probe file" +log_pass "Single-pod shared filesystem smoke test confirmed default StorageClass behavior and working storage access" diff --git a/k8s-training/filesystem-csi-validation/03-run-csi-rwx-cross-node-test.sh b/k8s-training/filesystem-csi-validation/03-run-csi-rwx-cross-node-test.sh new file mode 100755 index 000000000..a6caf10d5 --- /dev/null +++ b/k8s-training/filesystem-csi-validation/03-run-csi-rwx-cross-node-test.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +# ----------------------------------------------------------------------------- +# File: 03-run-csi-rwx-cross-node-test.sh +# Purpose: +# Validate ReadWriteMany behavior across nodes by mounting the same PVC into +# two pods scheduled onto different hosts. +# +# Why We Run This: +# A single-pod test proves basic functionality, but shared filesystems are +# most valuable when data written from one node can be read from another. This +# script confirms that cross-node sharing works in practice. +# +# Reference Docs: +# https://docs.nebius.com/kubernetes/storage/filesystem-over-csi +# +# What This Script Does: +# - Applies a RWX PVC plus reader/writer pod manifest +# - Uses pod anti-affinity to encourage placement on different nodes +# - Waits for the PVC and both pods to become ready +# - Verifies that the PVC inherited the expected default StorageClass +# - Writes a file from one pod and reads it from the other +# +# Usage: +# ./03-run-csi-rwx-cross-node-test.sh +# +# Optional Environment Variables: +# TEST_NAMESPACE Namespace where the validation resources should be created. +# Defaults to the current kubectl namespace or default. +# +# Manifest Used: +# manifests/02-csi-rwx-cross-node.yaml +# +# Created By: Aaron Fagan +# Created On: 2026-03-17 +# Version: 0.1.0 +# ----------------------------------------------------------------------------- +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/common.sh" + +log_step "Starting cross-node RWX validation" +log_info "Namespace: ${TEST_NAMESPACE}" +log_info "Manifest: ${FILESYSTEM_RWX_MANIFEST_PATH}" +log_info "PVC name: ${FILESYSTEM_RWX_PVC_NAME}" +log_info "Writer pod: ${FILESYSTEM_RWX_WRITER_POD_NAME}" +log_info "Reader pod: ${FILESYSTEM_RWX_READER_POD_NAME}" +log_info "Expected default StorageClass: ${FILESYSTEM_DEFAULT_STORAGE_CLASS_NAME}" + +log_step "Checking required local dependencies" +require_command kubectl +log_pass "Required local commands for the RWX validation are available" + +log_step "Applying the RWX validation manifest" +kubectl apply -n "${TEST_NAMESPACE}" -f "${FILESYSTEM_RWX_MANIFEST_PATH}" +log_pass "RWX validation manifest applied in namespace '${TEST_NAMESPACE}'" + +log_step "Waiting for the RWX PVC to bind" +kubectl wait -n "${TEST_NAMESPACE}" \ + --for=jsonpath='{.status.phase}'=Bound \ + "pvc/${FILESYSTEM_RWX_PVC_NAME}" \ + --timeout=120s +log_info "PVC '${FILESYSTEM_RWX_PVC_NAME}' is bound" +log_pass "RWX PVC '${FILESYSTEM_RWX_PVC_NAME}' bound successfully" + +log_step "Verifying that the RWX PVC inherited the default StorageClass" +RWX_STORAGE_CLASS_NAME="$(kubectl get pvc -n "${TEST_NAMESPACE}" "${FILESYSTEM_RWX_PVC_NAME}" -o jsonpath='{.spec.storageClassName}')" +if [[ -z "${RWX_STORAGE_CLASS_NAME}" ]]; then + log_fail "RWX PVC '${FILESYSTEM_RWX_PVC_NAME}' did not receive a StorageClass from the cluster default" + exit 1 +fi +if [[ "${RWX_STORAGE_CLASS_NAME}" != "${FILESYSTEM_DEFAULT_STORAGE_CLASS_NAME}" ]]; then + log_fail "RWX PVC '${FILESYSTEM_RWX_PVC_NAME}' used StorageClass '${RWX_STORAGE_CLASS_NAME}', expected '${FILESYSTEM_DEFAULT_STORAGE_CLASS_NAME}'" + exit 1 +fi +log_info "PVC '${FILESYSTEM_RWX_PVC_NAME}' was assigned StorageClass '${RWX_STORAGE_CLASS_NAME}'" +log_pass "RWX PVC '${FILESYSTEM_RWX_PVC_NAME}' inherited the expected default StorageClass" + +log_step "Waiting for both RWX test pods to become ready" +kubectl wait -n "${TEST_NAMESPACE}" \ + --for=condition=Ready \ + "pod/${FILESYSTEM_RWX_WRITER_POD_NAME}" \ + --timeout=180s +kubectl wait -n "${TEST_NAMESPACE}" \ + --for=condition=Ready \ + "pod/${FILESYSTEM_RWX_READER_POD_NAME}" \ + --timeout=180s +log_info "Both RWX test pods are ready" +log_pass "RWX writer and reader pods both reached Ready state" + +log_step "Checking the node placement for the reader and writer pods" +WRITER_NODE="$(kubectl get pod -n "${TEST_NAMESPACE}" "${FILESYSTEM_RWX_WRITER_POD_NAME}" -o jsonpath='{.spec.nodeName}')" +READER_NODE="$(kubectl get pod -n "${TEST_NAMESPACE}" "${FILESYSTEM_RWX_READER_POD_NAME}" -o jsonpath='{.spec.nodeName}')" + +echo "writer node: ${WRITER_NODE}" +echo "reader node: ${READER_NODE}" + +kubectl get pods -n "${TEST_NAMESPACE}" "${FILESYSTEM_RWX_WRITER_POD_NAME}" "${FILESYSTEM_RWX_READER_POD_NAME}" -o wide +log_pass "RWX pod placement details collected for both nodes" + +log_step "Writing shared data from the writer pod" +kubectl exec -n "${TEST_NAMESPACE}" "${FILESYSTEM_RWX_WRITER_POD_NAME}" -- sh -lc ' + set -eu + echo "shared-check" > /data/shared.txt + cat /data/shared.txt +' +log_pass "Writer pod '${FILESYSTEM_RWX_WRITER_POD_NAME}' wrote shared data to the mounted volume" + +log_step "Reading the same shared data from the reader pod" +kubectl exec -n "${TEST_NAMESPACE}" "${FILESYSTEM_RWX_READER_POD_NAME}" -- sh -lc ' + set -eu + ls -l /data + cat /data/shared.txt +' +log_pass "Reader pod '${FILESYSTEM_RWX_READER_POD_NAME}' read the shared file created by the writer pod" + +log_step "Cross-node RWX validation completed successfully" +log_info "The PVC inherited the cluster default StorageClass and the same file was visible from both pods through the shared volume" +log_pass "Cross-node ReadWriteMany storage behavior and default StorageClass inheritance confirmed" diff --git a/k8s-training/filesystem-csi-validation/04-cleanup-csi-test-resources.sh b/k8s-training/filesystem-csi-validation/04-cleanup-csi-test-resources.sh new file mode 100755 index 000000000..dcdfe2367 --- /dev/null +++ b/k8s-training/filesystem-csi-validation/04-cleanup-csi-test-resources.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +# ----------------------------------------------------------------------------- +# File: 04-cleanup-csi-test-resources.sh +# Purpose: +# Remove only the temporary validation resources created by the Nebius Shared +# Filesystem CSI test workflow after testing is complete. +# +# Why We Run This: +# The smoke and RWX tests are disposable validation resources. Cleaning them +# up keeps the cluster tidy while leaving the CSI driver and host mounts in +# place for ongoing cluster use. +# +# Reference Docs: +# https://docs.nebius.com/kubernetes/storage/filesystem-over-csi +# +# What This Script Does: +# - Removes the known test files written by the validation workflow +# - Deletes the cross-node RWX validation manifest resources +# - Deletes the single-pod smoke test manifest resources +# - Deletes any PVs still associated with the test PVCs +# - Deletes any remaining labeled validation pods and PVCs as a cleanup +# fallback +# - Deletes leftover node-debugger pods created during node verification +# even if they finished in an error state +# - Ignores already-deleted resources so reruns stay safe +# +# What This Script Does Not Do: +# - It does not uninstall the CSI Helm release +# - It does not delete the CSI StorageClass +# - It does not destroy the Terraform-managed cluster, node groups, or shared +# filesystem +# - It does not remove the host-level shared filesystem mount configured on +# nodes +# +# Usage: +# ./04-cleanup-csi-test-resources.sh +# +# Optional Environment Variables: +# TEST_NAMESPACE Namespace where the validation resources were created. +# Defaults to the current kubectl namespace or default. +# +# Created By: Aaron Fagan +# Created On: 2026-03-17 +# Version: 0.1.0 +# ----------------------------------------------------------------------------- +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/common.sh" + +best_effort_exec() { + local pod_name="$1" + local command="$2" + + if kubectl get pod "${pod_name}" -n "${TEST_NAMESPACE}" >/dev/null 2>&1; then + kubectl exec -n "${TEST_NAMESPACE}" "${pod_name}" -- sh -lc "${command}" || true + fi +} + +log_step "Starting Nebius Shared Filesystem CSI validation cleanup" +log_info "Namespace: ${TEST_NAMESPACE}" +log_info "Smoke test pod: ${FILESYSTEM_SMOKE_POD_NAME}" +log_info "RWX writer pod: ${FILESYSTEM_RWX_WRITER_POD_NAME}" +log_info "RWX reader pod: ${FILESYSTEM_RWX_READER_POD_NAME}" +log_info "Validation label selector: ${FILESYSTEM_VALIDATION_LABEL_SELECTOR}" + +log_step "Checking required local dependencies" +require_command kubectl +log_pass "Required local commands for cleanup are available" + +log_step "Removing validation test files from mounted volumes" +best_effort_exec "${FILESYSTEM_SMOKE_POD_NAME}" 'rm -f /data/probe.txt' +best_effort_exec "${FILESYSTEM_RWX_WRITER_POD_NAME}" 'rm -f /data/shared.txt' +best_effort_exec "${FILESYSTEM_RWX_READER_POD_NAME}" 'rm -f /data/shared.txt' +log_pass "Validation probe files were removed or already absent" + +log_step "Deleting the validation manifests" +kubectl delete -n "${TEST_NAMESPACE}" -f "${FILESYSTEM_RWX_MANIFEST_PATH}" --ignore-not-found=true || true +kubectl delete -n "${TEST_NAMESPACE}" -f "${FILESYSTEM_SMOKE_MANIFEST_PATH}" --ignore-not-found=true || true +log_pass "Validation manifests were deleted or already absent" + +log_step "Deleting any remaining validation pods and PVCs" +kubectl delete pod -n "${TEST_NAMESPACE}" \ + "${FILESYSTEM_SMOKE_POD_NAME}" \ + "${FILESYSTEM_RWX_WRITER_POD_NAME}" \ + "${FILESYSTEM_RWX_READER_POD_NAME}" \ + --ignore-not-found=true || true +kubectl delete pvc -n "${TEST_NAMESPACE}" \ + "${FILESYSTEM_SMOKE_PVC_NAME}" \ + "${FILESYSTEM_RWX_PVC_NAME}" \ + --ignore-not-found=true || true +log_pass "Validation pods and PVCs were deleted or already absent" + +log_step "Deleting any remaining labeled validation pods and PVCs as a fallback" +kubectl delete pod,pvc -n "${TEST_NAMESPACE}" \ + -l "${FILESYSTEM_VALIDATION_LABEL_SELECTOR}" \ + --ignore-not-found=true || true +log_pass "Labeled validation pods and PVCs were deleted or already absent" + +log_step "Deleting any PVs still associated with the validation PVCs" +while read -r pv_name claim_namespace claim_name; do + [[ -z "${pv_name}" ]] && continue + if [[ "${claim_namespace}" == "${TEST_NAMESPACE}" ]] && \ + [[ "${claim_name}" == "${FILESYSTEM_SMOKE_PVC_NAME}" || "${claim_name}" == "${FILESYSTEM_RWX_PVC_NAME}" ]]; then + kubectl delete pv "${pv_name}" --ignore-not-found=true || true + fi +done < <( + kubectl get pv -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.spec.claimRef.namespace}{" "}{.spec.claimRef.name}{"\n"}{end}' \ + 2>/dev/null || true +) +log_pass "Validation PV cleanup completed" + +log_step "Deleting recorded node debugger pods from the verification step" +if [[ -f "${DEBUG_POD_RECORD_FILE}" ]]; then + while read -r namespace pod_name; do + [[ -z "${namespace}" || -z "${pod_name}" ]] && continue + kubectl delete pod -n "${namespace}" "${pod_name}" --ignore-not-found=true || true + done < "${DEBUG_POD_RECORD_FILE}" + rm -f "${DEBUG_POD_RECORD_FILE}" + log_pass "Recorded node debugger pods were deleted or already absent" +else + log_info "No recorded node debugger pods found at ${DEBUG_POD_RECORD_FILE}; skipping" + log_pass "No recorded node debugger pods required cleanup" +fi + +log_step "Deleting any remaining node debugger pods in the test namespace as a fallback" +while read -r pod_name phase; do + [[ -z "${pod_name}" ]] && continue + if [[ "${pod_name}" == node-debugger-* ]]; then + kubectl delete pod -n "${TEST_NAMESPACE}" "${pod_name}" --ignore-not-found=true || true + fi +done < <( + kubectl get pods -n "${TEST_NAMESPACE}" \ + -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.status.phase}{"\n"}{end}' \ + 2>/dev/null || true +) +log_pass "Node debugger pod fallback cleanup finished for namespace '${TEST_NAMESPACE}'" + +log_step "Validation cleanup completed" +log_info "CSI driver, StorageClass, cluster, and host mounts were left in place" +log_pass "Validation cleanup finished without removing the installed CSI platform components" diff --git a/k8s-training/filesystem-csi-validation/README.md b/k8s-training/filesystem-csi-validation/README.md new file mode 100644 index 000000000..aa897e854 --- /dev/null +++ b/k8s-training/filesystem-csi-validation/README.md @@ -0,0 +1,78 @@ +# Filesystem CSI Validation + +## Document Metadata + +- Created By: Aaron Fagan +- Created On: 2026-03-17 +- Version: 0.1.0 + +## Purpose + +This README explains what each helper file in this folder does, why it exists, +and the order in which to use the validation workflow. + +## Reference Docs + +- https://docs.nebius.com/kubernetes/storage/filesystem-over-csi + +This folder contains the commands and manifests used to validate the +Terraform-managed Nebius Shared Filesystem over CSI workflow for this +`k8s-training` deployment. + +These files are not run automatically. + +Important notes: + +- This repo mounts the Nebius Shared Filesystem on nodes at `/mnt/data`. +- The Terraform already attaches the shared filesystem to the node groups and + mounts it with cloud-init. +- Terraform now installs the CSI driver and patches the default `StorageClass` + when a shared filesystem is present. +- The remaining purpose of this folder is host-mount verification, pod-level + validation, and cleanup of temporary validation resources. +- The scripts default to the current `kubectl` namespace. Set + `TEST_NAMESPACE=` if you want to keep the validation resources + somewhere explicit. +- Steps `02` and `03` intentionally omit `storageClassName` from their test + PVCs so they can verify that the cluster default `StorageClass` is applied + automatically. +- Step `01` records the temporary node-debugger pod names in `.state/` so step + `04` can clean up only the debugger pods created by this workflow. +- Step `01` now defaults to a quick single-node shared filesystem spot check. + Set + `VERIFY_ALL_NODES=true` to validate every node, or `TARGET_NODE=` + to validate one specific node. +- The smoke and RWX manifests now use workflow-specific resource names to make + reruns and cleanup easier to understand. + +Suggested order: + +1. Run `./01-verify-node-filesystem-mounts.sh` +2. Run `./02-run-csi-smoke-test.sh` +3. Run `./03-run-csi-rwx-cross-node-test.sh` +4. Run `./04-cleanup-csi-test-resources.sh` to remove only the temporary test + resources when you are done testing + +Prerequisites: + +- `kubectl` points to the target cluster. +- The cluster nodes are already provisioned by this Terraform stack. +- Terraform apply has already completed successfully, including any automatic + shared filesystem CSI installation and default `StorageClass` patching. + +## Shared Defaults + +- `TEST_NAMESPACE` defaults to the current `kubectl` namespace, then falls back + to `default`. +- `MOUNT_POINT` defaults to the mount path discovered from the Terraform + cloud-init template, then falls back to `/mnt/data`. +- `FILESYSTEM_DEFAULT_STORAGE_CLASS_NAME` defaults to + `csi-mounted-fs-path-sc`. +- `VERIFY_ALL_NODES` defaults to `false` for step `01`. +- `TARGET_NODE` can be used in step `01` to test one explicit node. +- The validation resources use the following fixed names: + - `filesystem-csi-smoke-pvc` + - `filesystem-csi-smoke-pod` + - `filesystem-csi-rwx-pvc` + - `filesystem-csi-rwx-writer` + - `filesystem-csi-rwx-reader` diff --git a/k8s-training/filesystem-csi-validation/common.sh b/k8s-training/filesystem-csi-validation/common.sh new file mode 100644 index 000000000..babd82576 --- /dev/null +++ b/k8s-training/filesystem-csi-validation/common.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# ----------------------------------------------------------------------------- +# File: common.sh +# Purpose: +# Provide shared configuration and helper functions for the Nebius Shared +# Filesystem CSI validation workflow so the scripts behave consistently across +# environments. +# +# Why We Run This: +# Reusing the same namespace, mount path, naming, and state logic across all +# validation scripts reduces accidental drift and keeps the workflow easier +# to maintain. +# +# Reference Docs: +# https://docs.nebius.com/kubernetes/storage/filesystem-over-csi +# +# Created By: Aaron Fagan +# Created On: 2026-03-17 +# Version: 0.1.0 +# ----------------------------------------------------------------------------- + +COMMON_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +FILESYSTEM_MOUNT_TEMPLATE="${COMMON_DIR}/../../modules/cloud-init/k8s-cloud-init.tftpl" +STATE_DIR="${STATE_DIR:-${COMMON_DIR}/.state}" + +FILESYSTEM_SMOKE_MANIFEST_PATH="${COMMON_DIR}/manifests/01-csi-smoke-test.yaml" +FILESYSTEM_RWX_MANIFEST_PATH="${COMMON_DIR}/manifests/02-csi-rwx-cross-node.yaml" + +FILESYSTEM_SMOKE_PVC_NAME="filesystem-csi-smoke-pvc" +FILESYSTEM_SMOKE_POD_NAME="filesystem-csi-smoke-pod" +FILESYSTEM_RWX_PVC_NAME="filesystem-csi-rwx-pvc" +FILESYSTEM_RWX_WRITER_POD_NAME="filesystem-csi-rwx-writer" +FILESYSTEM_RWX_READER_POD_NAME="filesystem-csi-rwx-reader" +FILESYSTEM_VALIDATION_LABEL_SELECTOR="app.kubernetes.io/part-of=filesystem-csi-validation" +FILESYSTEM_DEFAULT_STORAGE_CLASS_NAME="${FILESYSTEM_DEFAULT_STORAGE_CLASS_NAME:-csi-mounted-fs-path-sc}" + +DEBUG_POD_RECORD_FILE="${DEBUG_POD_RECORD_FILE:-${STATE_DIR}/verify-node-debugger-pods.txt}" + +default_mount_point() { + if [[ -f "${FILESYSTEM_MOUNT_TEMPLATE}" ]]; then + awk -F'[][]' ' + /virtiofs/ && $2 != "" { + count = split($2, fields, ",") + if (count >= 2) { + gsub(/^[[:space:]]+|[[:space:]]+$/, "", fields[2]) + print fields[2] + exit + } + } + ' "${FILESYSTEM_MOUNT_TEMPLATE}" + fi +} + +default_namespace() { + kubectl config view --minify --output 'jsonpath={..namespace}' 2>/dev/null || true +} + +ensure_state_dir() { + mkdir -p "${STATE_DIR}" +} + +require_command() { + local command_name="$1" + if ! command -v "${command_name}" >/dev/null 2>&1; then + log_fail "Required command '${command_name}' is not available in PATH" + exit 1 + fi +} + +log_step() { + echo + echo "==> $*" +} + +log_info() { + echo " - $*" +} + +log_pass() { + echo "[result] PASS: $*" +} + +log_fail() { + echo "[result] FAIL: $*" >&2 +} + +MOUNT_POINT="${MOUNT_POINT:-$(default_mount_point)}" +MOUNT_POINT="${MOUNT_POINT:-/mnt/data}" + +TEST_NAMESPACE="${TEST_NAMESPACE:-$(default_namespace)}" +TEST_NAMESPACE="${TEST_NAMESPACE:-default}" diff --git a/k8s-training/filesystem-csi-validation/manifests/01-csi-smoke-test.yaml b/k8s-training/filesystem-csi-validation/manifests/01-csi-smoke-test.yaml new file mode 100644 index 000000000..1eb13ff51 --- /dev/null +++ b/k8s-training/filesystem-csi-validation/manifests/01-csi-smoke-test.yaml @@ -0,0 +1,65 @@ +# ----------------------------------------------------------------------------- +# File: manifests/01-csi-smoke-test.yaml +# Purpose: +# Create a minimal PVC and pod pair for validating that the Nebius +# csi-mounted-fs-path StorageClass can mount the shared filesystem into a pod. +# +# Why We Run This: +# This manifest provides the fastest end-to-end confirmation that the CSI +# driver is working, that the cluster default StorageClass is being applied +# to new PVCs, and that a pod can perform simple read/write operations on the +# shared storage. +# +# Reference Docs: +# https://docs.nebius.com/kubernetes/storage/filesystem-over-csi +# +# Applied By: +# ./02-run-csi-smoke-test.sh +# +# Created By: Aaron Fagan +# Created On: 2026-03-17 +# Version: 0.1.0 +# ----------------------------------------------------------------------------- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: filesystem-csi-smoke-pvc + labels: + app.kubernetes.io/part-of: filesystem-csi-validation + app.kubernetes.io/component: smoke-test +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 +kind: Pod +metadata: + name: filesystem-csi-smoke-pod + labels: + app.kubernetes.io/part-of: filesystem-csi-validation + app.kubernetes.io/component: smoke-test +spec: + containers: + - name: filesystem-csi-smoke-pod + image: busybox + command: ["sleep", "1000000"] + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 100m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + privileged: false + volumeMounts: + - mountPath: /data + name: my-csi-volume + volumes: + - name: my-csi-volume + persistentVolumeClaim: + claimName: filesystem-csi-smoke-pvc diff --git a/k8s-training/filesystem-csi-validation/manifests/02-csi-rwx-cross-node.yaml b/k8s-training/filesystem-csi-validation/manifests/02-csi-rwx-cross-node.yaml new file mode 100644 index 000000000..b2392d461 --- /dev/null +++ b/k8s-training/filesystem-csi-validation/manifests/02-csi-rwx-cross-node.yaml @@ -0,0 +1,117 @@ +# ----------------------------------------------------------------------------- +# File: manifests/02-csi-rwx-cross-node.yaml +# Purpose: +# Create a shared RWX PVC and two pods that use the same claim from different +# nodes so we can validate cross-node data sharing. +# +# Why We Run This: +# The Nebius Shared Filesystem is most valuable when multiple pods on +# different nodes can observe the same files. This manifest is the practical +# proof for that behavior while also verifying that a PVC with no explicit +# StorageClass inherits the cluster default. +# +# Reference Docs: +# https://docs.nebius.com/kubernetes/storage/filesystem-over-csi +# +# Applied By: +# ./03-run-csi-rwx-cross-node-test.sh +# +# Created By: Aaron Fagan +# Created On: 2026-03-17 +# Version: 0.1.0 +# ----------------------------------------------------------------------------- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: filesystem-csi-rwx-pvc + labels: + app.kubernetes.io/part-of: filesystem-csi-validation + app.kubernetes.io/component: rwx-test +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 2Gi +--- +apiVersion: v1 +kind: Pod +metadata: + name: filesystem-csi-rwx-writer + labels: + app: filesystem-csi-rwx-check + app.kubernetes.io/part-of: filesystem-csi-validation + app.kubernetes.io/component: rwx-writer +spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - filesystem-csi-rwx-check + topologyKey: kubernetes.io/hostname + containers: + - name: writer + image: busybox + command: ["sleep", "1000000"] + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 100m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + privileged: false + volumeMounts: + - mountPath: /data + name: my-csi-volume + volumes: + - name: my-csi-volume + persistentVolumeClaim: + claimName: filesystem-csi-rwx-pvc +--- +apiVersion: v1 +kind: Pod +metadata: + name: filesystem-csi-rwx-reader + labels: + app: filesystem-csi-rwx-check + app.kubernetes.io/part-of: filesystem-csi-validation + app.kubernetes.io/component: rwx-reader +spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - filesystem-csi-rwx-check + topologyKey: kubernetes.io/hostname + containers: + - name: reader + image: busybox + command: ["sleep", "1000000"] + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 100m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + privileged: false + volumeMounts: + - mountPath: /data + name: my-csi-volume + volumes: + - name: my-csi-volume + persistentVolumeClaim: + claimName: filesystem-csi-rwx-pvc diff --git a/k8s-training/filesystem.tf b/k8s-training/filesystem.tf index df9bd229c..4ca0f122d 100644 --- a/k8s-training/filesystem.tf +++ b/k8s-training/filesystem.tf @@ -31,3 +31,69 @@ locals { mount_tag = local.filestore.mount_tag } : null } + +resource "helm_release" "filesystem_csi" { + count = local.filesystem_csi_enabled ? 1 : 0 + + name = local.filesystem_csi_chart_name + repository = "oci://cr.eu-north1.nebius.cloud/mk8s/helm" + chart = local.filesystem_csi_chart_name + version = var.filesystem_csi.chart_version + namespace = var.filesystem_csi.namespace + create_namespace = true + atomic = true + wait = true + + set = [ + { + name = "dataDir" + value = local.filesystem_csi_data_dir + }, + ] + + depends_on = [ + nebius_mk8s_v1_node_group.cpu-only, + nebius_mk8s_v1_node_group.gpu, + ] +} + +resource "kubernetes_annotations" "filesystem_csi_demote_previous_default_storage_class" { + count = local.filesystem_csi_enabled && var.filesystem_csi.make_default_storage_class && local.filesystem_csi_previous_default_sc != null && local.filesystem_csi_previous_default_sc != "" && local.filesystem_csi_previous_default_sc != local.filesystem_csi_storage_class_name ? 1 : 0 + + api_version = "storage.k8s.io/v1" + kind = "StorageClass" + force = true + + metadata { + name = local.filesystem_csi_previous_default_sc + } + + annotations = { + "storageclass.kubernetes.io/is-default-class" = "false" + } + + depends_on = [ + helm_release.filesystem_csi, + ] +} + +resource "kubernetes_annotations" "filesystem_csi_promote_default_storage_class" { + count = local.filesystem_csi_enabled && var.filesystem_csi.make_default_storage_class ? 1 : 0 + + api_version = "storage.k8s.io/v1" + kind = "StorageClass" + force = true + + metadata { + name = local.filesystem_csi_storage_class_name + } + + annotations = { + "storageclass.kubernetes.io/is-default-class" = "true" + } + + depends_on = [ + helm_release.filesystem_csi, + kubernetes_annotations.filesystem_csi_demote_previous_default_storage_class, + ] +} diff --git a/k8s-training/locals.tf b/k8s-training/locals.tf index 874ec817d..33eb36411 100644 --- a/k8s-training/locals.tf +++ b/k8s-training/locals.tf @@ -4,9 +4,16 @@ locals { fileexists(var.ssh_public_key.path) ? file(var.ssh_public_key.path) : null) filestore = { - mount_tag = "data" + mount_tag = "data" + mount_path = var.filestore_mount_path } + filesystem_csi_chart_name = "csi-mounted-fs-path" + filesystem_csi_storage_class_name = "csi-mounted-fs-path-sc" + filesystem_csi_enabled = local.shared-filesystem != null + filesystem_csi_data_dir = "${trimsuffix(local.filestore.mount_path, "/")}/csi-mounted-fs-path-data/" + filesystem_csi_previous_default_sc = var.filesystem_csi.previous_default_storage_class_name + regions_default = { eu-west1 = { cpu_nodes_platform = "cpu-d3" diff --git a/k8s-training/main.tf b/k8s-training/main.tf index 995489828..dda25103a 100644 --- a/k8s-training/main.tf +++ b/k8s-training/main.tf @@ -102,9 +102,10 @@ resource "nebius_mk8s_v1_node_group" "cpu-only" { priority = 3 } : null cloud_init_user_data = templatefile("${path.module}/../modules/cloud-init/k8s-cloud-init.tftpl", { - enable_filestore = var.enable_filestore ? "true" : "false", - ssh_user_name = var.ssh_user_name, - ssh_public_key = local.ssh_public_key + enable_filestore = var.enable_filestore ? "true" : "false", + filestore_mount_path = local.filestore.mount_path, + ssh_user_name = var.ssh_user_name, + ssh_public_key = local.ssh_public_key }) } } @@ -173,9 +174,10 @@ resource "nebius_mk8s_v1_node_group" "gpu" { underlay_required = false cloud_init_user_data = templatefile("${path.module}/../modules/cloud-init/k8s-cloud-init.tftpl", { - enable_filestore = var.enable_filestore ? "true" : "false", - ssh_user_name = var.ssh_user_name, - ssh_public_key = local.ssh_public_key + enable_filestore = var.enable_filestore ? "true" : "false", + filestore_mount_path = local.filestore.mount_path, + ssh_user_name = var.ssh_user_name, + ssh_public_key = local.ssh_public_key }) } } diff --git a/k8s-training/output.tf b/k8s-training/output.tf index b6e60d68f..593d50e29 100644 --- a/k8s-training/output.tf +++ b/k8s-training/output.tf @@ -26,4 +26,15 @@ output "kube_cluster_ca_certificate" { output "shared-filesystem" { description = "Shared-filesystem." value = local.shared-filesystem -} \ No newline at end of file +} + +output "filesystem_csi" { + description = "Nebius Shared Filesystem CSI installation details." + value = local.filesystem_csi_enabled ? { + release_name = nonsensitive(one(helm_release.filesystem_csi).name) + namespace = nonsensitive(one(helm_release.filesystem_csi).namespace) + status = nonsensitive(one(helm_release.filesystem_csi).status) + storage_class_name = local.filesystem_csi_storage_class_name + default_storage_class_patch = var.filesystem_csi.make_default_storage_class + } : null +} diff --git a/k8s-training/variables.tf b/k8s-training/variables.tf index 0cd2a1fa4..4429d69f3 100644 --- a/k8s-training/variables.tf +++ b/k8s-training/variables.tf @@ -77,6 +77,12 @@ variable "filestore_block_size_kibibytes" { default = 4 # 4kb } +variable "filestore_mount_path" { + description = "Mount path for the shared filesystem on Kubernetes nodes." + type = string + default = "/mnt/data" +} + # K8s access variable "ssh_user_name" { description = "SSH username." @@ -190,9 +196,14 @@ variable "gpu_disk_size" { } variable "enable_gpu_cluster" { - description = "Infiniband's fabric name." + description = "Enable GPU clustering and InfiniBand for the GPU node group." type = bool default = true + + validation { + condition = !var.enable_gpu_cluster || startswith(local.gpu_nodes_preset, "8gpu-") + error_message = "GPU clustering requires an 8-GPU preset. Set 'enable_gpu_cluster = false' for single-GPU presets such as '${local.gpu_nodes_preset}'." + } } variable "infiniband_fabric" { @@ -231,8 +242,11 @@ variable "mig_parted_config" { default = null validation { - condition = var.mig_parted_config == null || contains(local.valid_mig_parted_configs[local.gpu_nodes_platform], coalesce(var.mig_parted_config, "null")) - error_message = "Invalid MIG config '${coalesce(var.mig_parted_config, "null")}' for the selected GPU platform '${local.gpu_nodes_platform}'. Must be one of ${join(", ", local.valid_mig_parted_configs[local.gpu_nodes_platform])} or left unset." + condition = var.mig_parted_config == null || contains( + lookup(local.valid_mig_parted_configs, local.gpu_nodes_platform, []), + var.mig_parted_config, + ) + error_message = length(lookup(local.valid_mig_parted_configs, local.gpu_nodes_platform, [])) > 0 ? "Invalid MIG config '${coalesce(var.mig_parted_config, "null")}' for the selected GPU platform '${local.gpu_nodes_platform}'. Must be one of ${join(", ", lookup(local.valid_mig_parted_configs, local.gpu_nodes_platform, []))} or left unset." : "GPU platform '${local.gpu_nodes_platform}' does not support MIG partitioning. Leave 'mig_parted_config' unset." } } @@ -398,3 +412,14 @@ variable "custom_driver" { } } + +variable "filesystem_csi" { + description = "Configuration for Nebius Shared Filesystem CSI installation when a shared filesystem is present. Set previous_default_storage_class_name to an empty string to skip demoting another StorageClass." + type = object({ + chart_version = optional(string, "0.1.5") + namespace = optional(string, "kube-system") + make_default_storage_class = optional(bool, true) + previous_default_storage_class_name = optional(string, "compute-csi-default-sc") + }) + default = {} +} diff --git a/modules/cilium-egress-gateway/main.tf b/modules/cilium-egress-gateway/main.tf index 207d3bc4e..2ab40fa41 100644 --- a/modules/cilium-egress-gateway/main.tf +++ b/modules/cilium-egress-gateway/main.tf @@ -79,9 +79,10 @@ resource "nebius_mk8s_v1_node_group" "egress-gateway" { type = var.nodes_disk_type } cloud_init_user_data = templatefile("${path.module}/../cloud-init/k8s-cloud-init.tftpl", { - enable_filestore = "false", - ssh_user_name = var.ssh_user_name, - ssh_public_key = var.ssh_public_key + enable_filestore = "false", + filestore_mount_path = "/mnt/data", + ssh_user_name = var.ssh_user_name, + ssh_public_key = var.ssh_public_key }) network_interfaces = [ { diff --git a/modules/cloud-init/k8s-cloud-init.tftpl b/modules/cloud-init/k8s-cloud-init.tftpl index 1f7830fe6..a7ba0b9ab 100644 --- a/modules/cloud-init/k8s-cloud-init.tftpl +++ b/modules/cloud-init/k8s-cloud-init.tftpl @@ -3,11 +3,11 @@ package_upgrade: false %{ if enable_filestore != "false" } mounts: -- [ data, /mnt/data, virtiofs, "defaults", 0, 2 ] +- [ data, ${filestore_mount_path}, virtiofs, "defaults", 0, 2 ] runcmd: - [ modprobe, fuse ] - - [ mkdir, '-p', /mnt/data ] + - [ mkdir, '-p', ${filestore_mount_path} ] - [ mount, '-a' ] %{ endif }