Merge branch 'kubeflow:master' into test-release

Author: Krishna-kg732

.github/workflows/code-quality-check.yaml

@@ -0,0 +1,26 @@
+name: Code Quality Checks
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  code-quality:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Check KubeLinter
+        run: make lint-manifests
+
+      - name: Check Helm Charts
+        run: make helm-lint TARGET_BRANCH=${{ github.event.pull_request.base.ref || github.event.repository.default_branch || 'master' }}

.github/workflows/test-helm.yaml

@@ -0,0 +1,24 @@
+name: Unit and Integration Test - Helm
+
+on:
+  - pull_request
+  - push
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Setup Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: go.mod
+
+      - name: Run Helm Unit Tests
+        run: make helm-unittest

.github/workflows/trivy-scan.yaml

@@ -13,7 +13,7 @@ jobs:
         uses: actions/checkout@v6
 
       - name: Run Trivy vulnerability scanner in repo mode
-        uses: aquasecurity/trivy-action@0.33.1
+        uses: aquasecurity/trivy-action@0.34.0
         with:
           scan-type: "fs"
           ignore-unfixed: true

.kube-linter.yaml

@@ -0,0 +1,24 @@
+# KubeLinter configuration file
+# For more information, see https://docs.kubelinter.io/#/configuring-kubelinter
+
+# checks section configures which checks to run.
+checks:
+  # Do not include any of the default checks.
+  # If this is false, all default checks are included,
+  # and the "include" and "exclude" lists are applied on top of them.
+  doNotIncludeBuiltIn: false
+
+  # include is a list of check names to include.
+  # include:
+  #   - "check-name-1"
+
+  # exclude is a list of check names to exclude.
+  exclude:
+    - "unset-cpu-requirements"
+    - "unset-memory-requirements"
+    - "mismatching-selector"
+    - "no-read-only-root-fs"
+    - "run-as-non-root"
+    - "no-anti-affinity"
+    - "non-existent-service-account" # Often false positive in kustomize
+    - "latest-tag"

Makefile

@@ -28,6 +28,7 @@ HELM_UNITTEST_VERSION ?= 0.5.1
 HELM_CHART_TESTING_VERSION ?= v3.12.0
 HELM_DOCS_VERSION ?= v1.14.2
 YQ_VERSION ?= v4.45.1
+KUBE_LINTER_VERSION ?= v0.7.1
 
 # Container runtime (docker or podman)
 CONTAINER_RUNTIME ?=
@@ -42,6 +43,7 @@ HELM_DOCS ?= $(LOCALBIN)/helm-docs
 YQ ?= $(LOCALBIN)/yq
 GOLANGCI_LINT ?= $(LOCALBIN)/golangci-lint
 GOLANGCI_LINT_KAL ?= $(LOCALBIN)/golangci-lint-kube-api-linter
+KUBE_LINTER ?= $(LOCALBIN)/kube-linter
 
 ##@ General
 
@@ -106,6 +108,15 @@ helm-docs-plugin: ## Download helm-docs plugin locally if required.
 yq: # Download yq locally if required.
 	GOBIN=$(LOCALBIN) go install github.com/mikefarah/yq/v4@$(YQ_VERSION)
 
+.PHONY: kube-linter
+kube-linter: ## Download kube-linter locally if required.
+	GOBIN=$(LOCALBIN) go install golang.stackrox.io/kube-linter/cmd/kube-linter@$(KUBE_LINTER_VERSION)
+
+.PHONY: lint-manifests
+lint-manifests: kube-linter ## Run kube-linter on manifests and helm charts.
+	$(KUBE_LINTER) lint manifests/base --config .kube-linter.yaml
+	$(KUBE_LINTER) lint charts/kubeflow-trainer --config .kube-linter.yaml
+
 # Download external CRDs for Go integration testings.
 EXTERNAL_CRDS_DIR ?= $(PROJECT_DIR)/manifests/external-crds
 
@@ -217,13 +228,15 @@ test-e2e-notebook: ## Run Jupyter Notebook with Papermill.
 
 ##@ Helm
 
+TARGET_BRANCH ?= master
+
 .PHONY: helm-unittest
 helm-unittest: helm-unittest-plugin ## Run Helm chart unittests.
 	$(HELM) unittest $(TRAINER_CHART_DIR) --strict --file "tests/**/*_test.yaml"
 
 .PHONY: helm-lint
 helm-lint: ## Run Helm chart lint test.
-	docker run --rm --workdir /workspace --user "$(shell id -u):$(shell id -g)" --volume "$$(pwd):/workspace" quay.io/helmpack/chart-testing:$(HELM_CHART_TESTING_VERSION) ct lint --target-branch master --validate-maintainers=false
+	docker run --rm --workdir /workspace --user "$(shell id -u):$(shell id -g)" --volume "$$(pwd):/workspace" quay.io/helmpack/chart-testing:$(HELM_CHART_TESTING_VERSION) ct lint --target-branch $(TARGET_BRANCH) --validate-maintainers=false --check-version-increment=false
 
 .PHONY: helm-docs
 helm-docs: helm-docs-plugin ## Generates markdown documentation for helm charts from requirements and values files.

api/openapi-spec/swagger.json

@@ -70,7 +70,7 @@ dataCache:
   cacheImage:
     tag: "v2.0.0"
   runtimes:
-    torchDistributed:
+    torchDistributedWithCache:
       enabled: true
 ```
 
@@ -133,8 +133,8 @@ See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall) for command docum
 | dataCache.cacheImage.registry | string | `"ghcr.io"` | Data cache image registry |
 | dataCache.cacheImage.repository | string | `"kubeflow/trainer/data-cache"` | Data cache image repository |
 | dataCache.cacheImage.tag | string | `""` | Data cache image tag. Defaults to chart version if empty. |
-| dataCache.runtimes.torchDistributed | object | `{"enabled":false}` | PyTorch distributed training with data cache support |
-| dataCache.runtimes.torchDistributed.enabled | bool | `false` | Enable deployment of torch-distributed-with-cache runtime |
+| dataCache.runtimes.torchDistributedWithCache | object | `{"enabled":false}` | PyTorch distributed training with data cache support |
+| dataCache.runtimes.torchDistributedWithCache.enabled | bool | `false` | Enable deployment of torch-distributed-with-cache runtime |
 | runtimes | object | `{"deepspeedDistributed":{"enabled":false,"image":{"registry":"ghcr.io","repository":"kubeflow/trainer/deepspeed-runtime","tag":""}},"defaultEnabled":false,"jaxDistributed":{"enabled":false},"mlxDistributed":{"enabled":false,"image":{"registry":"ghcr.io","repository":"kubeflow/trainer/mlx-runtime","tag":""}},"torchDistributed":{"enabled":false},"torchtuneDistributed":{"image":{"registry":"ghcr.io","repository":"kubeflow/trainer/torchtune-trainer","tag":""},"llama3_2_1B":{"enabled":false},"llama3_2_3B":{"enabled":false},"qwen2_5_1_5B":{"enabled":false}}}` | ClusterTrainingRuntimes configuration These are optional runtime templates that can be deployed with the Helm chart. Each runtime provides a blueprint for different ML frameworks and configurations. |
 | runtimes.defaultEnabled | bool | `false` | Enable all default runtimes (torch, deepspeed, mlx, jax, torchtune) when set to true. Individual runtime settings will be ignored if this is enabled. |
 | runtimes.torchDistributed | object | `{"enabled":false}` | PyTorch distributed training runtime (no custom images required) |

api/python_api/kubeflow_trainer_api/models/__init__.py

@@ -88,7 +88,7 @@ dataCache:
   cacheImage:
     tag: "v2.0.0"
   runtimes:
-    torchDistributed:
+    torchDistributedWithCache:
       enabled: true
 ```