diff --git a/kubernetes/Dockerfile b/kubernetes/Dockerfile new file mode 100644 index 00000000..9d858c05 --- /dev/null +++ b/kubernetes/Dockerfile @@ -0,0 +1,49 @@ +# Copyright 2025 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build the manager binary +FROM golang:1.25 AS builder +ARG TARGETOS +ARG TARGETARCH + +WORKDIR /workspace +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum +# cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN GOPROXY=https://goproxy.cn,direct go mod download + +# Copy the go source +COPY cmd/ cmd/ +COPY api/ api/ +COPY pkg/ pkg/ +COPY internal/ internal/ + +# Build +# the GOARCH has not a default value to allow the binary be built according to the host where the command +# was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO +# the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, +# by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. +ARG PACKAGE=cmd/controller/main.go +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o server ${PACKAGE} + +# Use golang image as base to ensure nsenter (util-linux) is available +# distroless does not contain shell or nsenter +FROM golang:1.25 +ARG USERID=65532 +WORKDIR /workspace +COPY --from=builder /workspace/server . +USER $USERID +ENTRYPOINT ["/workspace/server"] \ No newline at end of file diff --git a/kubernetes/Dockerfile.debug b/kubernetes/Dockerfile.debug new file mode 100644 index 00000000..4df7b24b --- /dev/null +++ b/kubernetes/Dockerfile.debug @@ -0,0 +1,16 @@ +FROM golang:1.25 + +# Install Delve (debugger) and Reflex (file watcher) +RUN go install github.com/go-delve/delve/cmd/dlv@latest && \ + go install github.com/cespare/reflex@latest + +WORKDIR /workspace + +# Set cache env vars to ensuring they are targeted by our volume mounts +ENV GOCACHE=/go/.cache/go-build +ENV GOMODCACHE=/go/pkg/mod +# Expose ports +EXPOSE 5758 2345 + +# The default command will be overridden by the script, but we can set a safe default +CMD ["bash"] diff --git a/kubernetes/Makefile b/kubernetes/Makefile new file mode 100644 index 00000000..970ebe17 --- /dev/null +++ b/kubernetes/Makefile @@ -0,0 +1,413 @@ +# VERSION defines the project version for the bundle. +# Update this value when you upgrade the version of your project. +# To re-generate a bundle for another specific version without changing the standard setup, you can: +# - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) +# - use environment variables to overwrite this value (e.g export VERSION=0.0.2) +VERSION ?= 0.0.1 + +# CHANNELS define the bundle channels used in the bundle. +# Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") +# To re-generate a bundle for other specific channels without changing the standard setup, you can: +# - use the CHANNELS as arg of the bundle target (e.g make bundle CHANNELS=candidate,fast,stable) +# - use environment variables to overwrite this value (e.g export CHANNELS="candidate,fast,stable") +ifneq ($(origin CHANNELS), undefined) +BUNDLE_CHANNELS := --channels=$(CHANNELS) +endif + +# DEFAULT_CHANNEL defines the default channel used in the bundle. +# Add a new line here if you would like to change its default config. (E.g DEFAULT_CHANNEL = "stable") +# To re-generate a bundle for any other default channel without changing the default setup, you can: +# - use the DEFAULT_CHANNEL as arg of the bundle target (e.g make bundle DEFAULT_CHANNEL=stable) +# - use environment variables to overwrite this value (e.g export DEFAULT_CHANNEL="stable") +ifneq ($(origin DEFAULT_CHANNEL), undefined) +BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL) +endif +BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) + +# IMAGE_TAG_BASE defines the docker.io namespace and part of the image name for remote images. +# This variable is used to construct full image tags for bundle and catalog images. +# +# For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both +# opensandbox.io/sandbox-k8s-bundle:$VERSION and opensandbox.io/sandbox-k8s-catalog:$VERSION. +IMAGE_TAG_BASE ?= opensandbox.io/sandbox-k8s + +# BUNDLE_IMG defines the image:tag used for the bundle. +# You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) +BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION) + +# BUNDLE_GEN_FLAGS are the flags passed to the operator-sdk generate bundle command +BUNDLE_GEN_FLAGS ?= -q --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS) + +# USE_IMAGE_DIGESTS defines if images are resolved via tags or digests +# You can enable this value if you would like to use SHA Based Digests +# To enable set flag to true +USE_IMAGE_DIGESTS ?= false +ifeq ($(USE_IMAGE_DIGESTS), true) + BUNDLE_GEN_FLAGS += --use-image-digests +endif + +# Set the Operator SDK version to use. By default, what is installed on the system is used. +# This is useful for CI or a project to utilize a specific version of the operator-sdk toolkit. +OPERATOR_SDK_VERSION ?= v1.42.0 +# Image URL to use all building/pushing image targets +# IMG defines the image for the controller manager. +IMG ?= controller:latest +# TASK_EXECUTOR_IMG defines the image for the task-executor service. +TASK_EXECUTOR_IMG ?= task-executor:latest + +# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) +ifeq (,$(shell go env GOBIN)) +GOBIN=$(shell go env GOPATH)/bin +else +GOBIN=$(shell go env GOBIN) +endif + +# CONTAINER_TOOL defines the container tool to be used for building images. +# Be aware that the target commands are only tested with Docker which is +# scaffolded by default. However, you might want to replace it to use other +# tools. (i.e. podman) +CONTAINER_TOOL ?= docker + +# DOCKER_BUILD_ARGS defines additional arguments to pass to docker build. +# For example, in some environments you may need: DOCKER_BUILD_ARGS=--network=host +DOCKER_BUILD_ARGS ?= + +# Setting SHELL to bash allows bash commands to be executed by recipes. +# Options are set to exit when a recipe line exits non-zero or a piped command fails. +SHELL = /usr/bin/env bash -o pipefail +.SHELLFLAGS = -ec + +.PHONY: all +all: build + +##@ General + +# The help target prints out all targets with their descriptions organized +# beneath their categories. The categories are represented by '##@' and the +# target descriptions by '##'. The awk command is responsible for reading the +# entire set of makefiles included in this invocation, looking for lines of the +# file as xyz: ## something, and then pretty-format the target and help. Then, +# if there's a line with ##@ something, that gets pretty-printed as a category. +# More info on the usage of ANSI control characters for terminal formatting: +# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters +# More info on the awk command: +# http://linuxcommand.org/lc3_adv_awk.php + +.PHONY: help +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +##@ Development + +.PHONY: manifests +manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. + $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases + +.PHONY: generate +generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. + $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." + +.PHONY: fmt +fmt: ## Run go fmt against code. + go fmt ./... + +.PHONY: vet +vet: ## Run go vet against code. + go vet ./... + +.PHONY: test +test: manifests generate fmt vet setup-envtest ## Run tests. + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out + +# To use a different vendor for e2e tests, modify the setup under 'tests/e2e'. +# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally. +KIND_CLUSTER ?= sandbox-k8s-test-e2e +GINKGO_ARGS ?= + +.PHONY: install-kind +install-kind: ## Install Kind using go install if not already installed + @if command -v kind >/dev/null 2>&1; then \ + echo "Kind is already installed: $$(kind version)"; \ + else \ + echo "Installing Kind..."; \ + go install sigs.k8s.io/kind@v0.20.0 && \ + echo "Kind installed successfully: $$(kind version)"; \ + fi + +.PHONY: setup-test-e2e +setup-test-e2e: install-kind ## Set up a Kind cluster for e2e tests if it does not exist + @case "$$($(KIND) get clusters 2>/dev/null || echo '')" in \ + *"$(KIND_CLUSTER)"*) \ + echo "Kind cluster '$(KIND_CLUSTER)' already exists. Skipping creation." ;; \ + *) \ + echo "Creating Kind cluster '$(KIND_CLUSTER)'..."; \ + $(KIND) create cluster --name $(KIND_CLUSTER) ;; \ + esac + +.PHONY: test-e2e +test-e2e: setup-test-e2e manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind. Use GINKGO_ARGS to pass additional arguments. + KIND_CLUSTER=$(KIND_CLUSTER) go test ./test/e2e/ -v -ginkgo.v $(GINKGO_ARGS) + $(MAKE) cleanup-test-e2e + +.PHONY: cleanup-test-e2e +cleanup-test-e2e: ## Tear down the Kind cluster used for e2e tests + @$(KIND) delete cluster --name $(KIND_CLUSTER) + +.PHONY: lint +lint: golangci-lint ## Run golangci-lint linter + $(GOLANGCI_LINT) run + +.PHONY: lint-fix +lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes + $(GOLANGCI_LINT) run --fix + +.PHONY: lint-config +lint-config: golangci-lint ## Verify golangci-lint linter configuration + $(GOLANGCI_LINT) config verify + +##@ Build + +.PHONY: build +build: manifests generate fmt vet ## Build manager binary. + go build -o bin/manager cmd/controller/main.go + +.PHONY: run +run: manifests generate fmt vet ## Run a controller from your host. + go run ./cmd/controller/main.go + +.PHONY: task-executor-build +task-executor-build: ## Build task-executor binary. + go build -o bin/task-executor ./cmd/task-executor + +.PHONY: task-executor-run +task-executor-run: ## Run task-executor from your host. + go run ./cmd/task-executor + +# If you wish to build the manager image targeting other platforms you can use the --platform flag. +# (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it. +# More info: https://docs.docker.com/develop/develop-images/build_enhancements/ +.PHONY: docker-build +# docker-build: ## Build docker image with the manager. +# $(CONTAINER_TOOL) build -t ${IMG} . + +docker-build: docker-build-controller + +.PHONY: docker-build-controller +docker-build-controller: ## Build docker image with the manager. + $(CONTAINER_TOOL) build $(DOCKER_BUILD_ARGS) --build-arg PACKAGE=cmd/controller/main.go -t ${IMG} . + +.PHONY: docker-build-task-executor +docker-build-task-executor: ## Build docker image with task-executor. + $(CONTAINER_TOOL) build $(DOCKER_BUILD_ARGS) --build-arg PACKAGE=cmd/task-executor/main.go --build-arg USERID=0 -t ${TASK_EXECUTOR_IMG} . + +.PHONY: docker-push +# docker-push: ## Push docker image with the manager. +# $(CONTAINER_TOOL) push ${IMG} + +docker-push: docker-push-controller + +.PHONY: docker-push-controller +docker-push-controller: ## Push docker image with the manager. + $(CONTAINER_TOOL) push ${IMG} + +.PHONY: docker-push-task-executor +docker-push-task-executor: ## Push docker image with task-executor. + $(CONTAINER_TOOL) push ${TASK_EXECUTOR_IMG} + +.PHONY: docker-run-task-executor +docker-run-task-executor: docker-build-task-executor ## Run task-executor docker image. + @echo "Running task-executor image: $(TASK_EXECUTOR_IMG) on port 8080" + @$(CONTAINER_TOOL) run --rm -d -p 8080:8080 --name task-executor-local $(TASK_EXECUTOR_IMG) + +.PHONY: docker-stop-task-executor +docker-stop-task-executor: ## Stop task-executor docker container. + @echo "Stopping task-executor container: task-executor-local" + -@$(CONTAINER_TOOL) stop task-executor-local || true + -@$(CONTAINER_TOOL) rm task-executor-local || true + +# PLATFORMS defines the target platforms for the manager image be built to provide support to multiple +# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to: +# - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/ +# - have enabled BuildKit. More info: https://docs.docker.com/develop/develop-images/build_enhancements/ +# - be able to push the image to your registry (i.e. if you do not set a valid value via IMG=> then the export will fail) +# To adequately provide solutions that are compatible with multiple platforms, you should consider using this option. +PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le +.PHONY: docker-buildx +docker-buildx: ## Build and push docker image for the manager for cross-platform support + # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile + sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross + - $(CONTAINER_TOOL) buildx create --name sandbox-k8s-builder + $(CONTAINER_TOOL) buildx use sandbox-k8s-builder + - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross . + - $(CONTAINER_TOOL) buildx rm sandbox-k8s-builder + rm Dockerfile.cross + +.PHONY: build-installer +build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment. + mkdir -p dist + cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + $(KUSTOMIZE) build config/default > dist/install.yaml + +##@ Deployment + +ifndef ignore-not-found + ignore-not-found = false +endif + +.PHONY: install +install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/crd | $(KUBECTL) apply -f - + +.PHONY: uninstall +uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - + +.PHONY: deploy +deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. + cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + $(KUSTOMIZE) build config/default | $(KUBECTL) apply -f - + +.PHONY: undeploy +undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - + +##@ Dependencies + +## Location to install dependencies to +LOCALBIN ?= $(shell pwd)/bin +$(LOCALBIN): + mkdir -p $(LOCALBIN) + +## Tool Binaries +KUBECTL ?= kubectl +KIND ?= kind +KUSTOMIZE ?= $(LOCALBIN)/kustomize +CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen +ENVTEST ?= $(LOCALBIN)/setup-envtest +GOLANGCI_LINT = $(LOCALBIN)/golangci-lint + +## Tool Versions +KUSTOMIZE_VERSION ?= v5.6.0 +CONTROLLER_TOOLS_VERSION ?= v0.18.0 +#ENVTEST_VERSION is the version of controller-runtime release branch to fetch the envtest setup script (i.e. release-0.20) +ENVTEST_VERSION ?= $(shell go list -m -f "{{ .Version }}" sigs.k8s.io/controller-runtime | awk -F'[v.]' '{printf "release-%d.%d", $$2, $$3}') +#ENVTEST_K8S_VERSION is the version of Kubernetes to use for setting up ENVTEST binaries (i.e. 1.31) +ENVTEST_K8S_VERSION ?= $(shell go list -m -f "{{ .Version }}" k8s.io/api | awk -F'[v.]' '{printf "1.%d", $$3}') +GOLANGCI_LINT_VERSION ?= v2.1.0 + +.PHONY: kustomize +kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. +$(KUSTOMIZE): $(LOCALBIN) + $(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v5,$(KUSTOMIZE_VERSION)) + +.PHONY: controller-gen +controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. +$(CONTROLLER_GEN): $(LOCALBIN) + $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen,$(CONTROLLER_TOOLS_VERSION)) + +.PHONY: setup-envtest +setup-envtest: envtest ## Download the binaries required for ENVTEST in the local bin directory. + @echo "Setting up envtest binaries for Kubernetes version $(ENVTEST_K8S_VERSION)..." + @$(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path || { \ + echo "Error: Failed to set up envtest binaries for version $(ENVTEST_K8S_VERSION)."; \ + exit 1; \ + } + +.PHONY: envtest +envtest: $(ENVTEST) ## Download setup-envtest locally if necessary. +$(ENVTEST): $(LOCALBIN) + $(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest,$(ENVTEST_VERSION)) + +.PHONY: golangci-lint +golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. +$(GOLANGCI_LINT): $(LOCALBIN) + $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/v2/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION)) + +# go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist +# $1 - target path with name of binary +# $2 - package url which can be installed +# $3 - specific version of package +define go-install-tool +@[ -f "$(1)-$(3)" ] || { \ +set -e; \ +package=$(2)@$(3) ;\ +echo "Downloading $${package}" ;\ +rm -f $(1) || true ;\ +GOBIN=$(LOCALBIN) go install $${package} ;\ +mv $(1) $(1)-$(3) ;\ +} ;\ +ln -sf $(1)-$(3) $(1) +endef + +.PHONY: operator-sdk +OPERATOR_SDK ?= $(LOCALBIN)/operator-sdk +operator-sdk: ## Download operator-sdk locally if necessary. +ifeq (,$(wildcard $(OPERATOR_SDK))) +ifeq (, $(shell which operator-sdk 2>/dev/null)) + @{ \ + set -e ;\ + mkdir -p $(dir $(OPERATOR_SDK)) ;\ + OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \ + curl -sSLo $(OPERATOR_SDK) https://github.com/operator-framework/operator-sdk/releases/download/$(OPERATOR_SDK_VERSION)/operator-sdk_$${OS}_$${ARCH} ;\ + chmod +x $(OPERATOR_SDK) ;\ + } +else +OPERATOR_SDK = $(shell which operator-sdk) +endif +endif + +.PHONY: bundle +bundle: manifests kustomize operator-sdk ## Generate bundle manifests and metadata, then validate generated files. + $(OPERATOR_SDK) generate kustomize manifests -q + cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) + $(KUSTOMIZE) build config/manifests | $(OPERATOR_SDK) generate bundle $(BUNDLE_GEN_FLAGS) + $(OPERATOR_SDK) bundle validate ./bundle + +.PHONY: bundle-build +bundle-build: ## Build the bundle image. + $(CONTAINER_TOOL) build $(DOCKER_BUILD_ARGS) -f bundle.Dockerfile -t $(BUNDLE_IMG) . + +.PHONY: bundle-push +bundle-push: ## Push the bundle image. + $(MAKE) docker-push IMG=$(BUNDLE_IMG) + +.PHONY: opm +OPM = $(LOCALBIN)/opm +opm: ## Download opm locally if necessary. +ifeq (,$(wildcard $(OPM))) +ifeq (,$(shell which opm 2>/dev/null)) + @{ \ + set -e ;\ + mkdir -p $(dir $(OPM)) ;\ + OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \ + curl -sSLo $(OPM) https://github.com/operator-framework/operator-registry/releases/download/v1.55.0/$${OS}-$${ARCH}-opm ;\ + chmod +x $(OPM) ;\ + } +else +OPM = $(shell which opm) +endif +endif + +# A comma-separated list of bundle images (e.g. make catalog-build BUNDLE_IMGS=example.com/operator-bundle:v0.1.0,example.com/operator-bundle:v0.2.0). +# These images MUST exist in a registry and be pull-able. +BUNDLE_IMGS ?= $(BUNDLE_IMG) + +# The image tag given to the resulting catalog image (e.g. make catalog-build CATALOG_IMG=example.com/operator-catalog:v0.2.0). +CATALOG_IMG ?= $(IMAGE_TAG_BASE)-catalog:v$(VERSION) + +# Set CATALOG_BASE_IMG to an existing catalog image tag to add $BUNDLE_IMGS to that image. +ifneq ($(origin CATALOG_BASE_IMG), undefined) +FROM_INDEX_OPT := --from-index $(CATALOG_BASE_IMG) +endif + +# Build a catalog image by adding bundle images to an empty catalog using the operator package manager tool, 'opm'. +# This recipe invokes 'opm' in 'semver' bundle add mode. For more information on add modes, see: +# https://github.com/operator-framework/community-operators/blob/7f1438c/docs/packaging-operator.md#updating-your-existing-operator +.PHONY: catalog-build +catalog-build: opm ## Build a catalog image. + $(OPM) index add --container-tool $(CONTAINER_TOOL) --mode semver --tag $(CATALOG_IMG) --bundles $(BUNDLE_IMGS) $(FROM_INDEX_OPT) + +# Push the catalog image. +.PHONY: catalog-push +catalog-push: ## Push a catalog image. + $(MAKE) docker-push IMG=$(CATALOG_IMG) \ No newline at end of file diff --git a/kubernetes/PROJECT b/kubernetes/PROJECT new file mode 100644 index 00000000..a3de3e65 --- /dev/null +++ b/kubernetes/PROJECT @@ -0,0 +1,41 @@ +# Code generated by tool. DO NOT EDIT. +# This file is used to track the info used to scaffold your project +# and allow the plugins properly work. +# More info: https://book.kubebuilder.io/reference/project-config.html +domain: opensandbox.io +layout: +- go.kubebuilder.io/v4 +plugins: + manifests.sdk.operatorframework.io/v2: {} + scorecard.sdk.operatorframework.io/v2: {} +projectName: sandbox-k8s +repo: github.com/alibaba/OpenSandbox/sandbox-k8s +resources: +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: opensandbox.io + group: sandbox + kind: Sandbox + path: github.com/alibaba/OpenSandbox/sandbox-k8s/api/v1alpha1 + version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: opensandbox.io + group: sandbox + kind: BatchSandbox + path: github.com/alibaba/OpenSandbox/sandbox-k8s/api/v1alpha1 + version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: opensandbox.io + group: sandbox + kind: Pool + path: github.com/alibaba/OpenSandbox/sandbox-k8s/api/v1alpha1 + version: v1alpha1 +version: "3" diff --git a/kubernetes/README-ZH.md b/kubernetes/README-ZH.md new file mode 100644 index 00000000..00b82b6c --- /dev/null +++ b/kubernetes/README-ZH.md @@ -0,0 +1,377 @@ +# OpenSandbox Kubernetes控制器 + +[English](README.md) | [中文](README-ZH.md) + +OpenSandbox Kubernetes控制器是一个Kubernetes操作器，通过自定义资源管理沙箱环境。它在Kubernetes集群中提供**自动化沙箱生命周期管理**、**资源池化以实现快速供应**、**批处理沙箱创建**和**可选的任务编排**功能。 + +## 关键特性 + +- **灵活的沙箱创建**：在池化和非池化沙箱创建模式之间选择 +- **批处理和单个交付**：支持单个沙箱（用于真实用户交互）和批处理沙箱交付（用于高吞吐量智能体强化学习场景） +- **可选任务调度**：集成任务编排，支持可选的分片任务模板以实现异构任务分发和定制化沙箱交付（例如，进程注入或动态容器创建） +- **资源池化**：维护预热的资源池以实现快速沙箱供应 +- **全面监控**：实时跟踪沙箱和任务状态 + +## 功能特性 + +### 批处理沙箱管理 +BatchSandbox自定义资源允许您创建和管理多个相同的沙箱环境。主要功能包括： +- **灵活的创建模式**：支持池化（使用资源池）和非池化沙箱创建 +- **单个和批处理交付**：根据需要创建单个沙箱（replicas=1）或批处理沙箱（replicas=N） +- **可扩展的副本管理**：通过副本配置轻松控制沙箱实例数量 +- **自动过期**：设置TTL（生存时间）以自动清理过期沙箱 +- **可选任务调度**：内置任务执行引擎，支持可选任务模板 +- **详细状态报告**：关于副本、分配和任务状态的综合指标 + +### 资源池化 +Pool自定义资源维护一个预热的计算资源池，以实现快速沙箱供应： +- 可配置的缓冲区大小（最小和最大）以平衡资源可用性和成本 +- 池容量限制以控制总体资源消耗 +- 基于需求的自动资源分配和释放 +- 实时状态监控，显示总数、已分配和可用资源 + +### 任务编排 +集成的任务管理系统，在沙箱内执行自定义工作负载： +- **可选执行**：任务调度完全可选 - 可以在不带任务的情况下创建沙箱 +- **容器和进程任务**：支持基于容器和基于进程的任务 +- **异构任务分发**：使用shardTaskPatches为批处理中的每个沙箱定制单独的任务 + +### 高级调度 +智能资源管理功能： +- 最小和最大缓冲区设置，以确保资源可用性同时控制成本 +- 池范围的容量限制，防止资源耗尽 +- 基于需求的自动扩展 + +## 入门指南 + +### 先决条件 +- go版本v1.24.0+ +- docker版本17.03+。 +- kubectl版本v1.11.3+。 +- 访问Kubernetes v1.33.1+集群。 + +如果您没有Kubernetes集群的访问权限，可以使用[kind](https://kind.sigs.k8s.io/)创建一个本地Kubernetes集群进行测试。Kind在Docker容器中运行Kubernetes节点，使得设置本地开发环境变得容易。 + +安装kind： +- 从[发布页面](https://github.com/kubernetes-sigs/kind/releases)下载适用于您操作系统的发布二进制文件并将其移动到您的`$PATH`中 +- 或使用包管理器： + - macOS (Homebrew): `brew install kind` + - Windows (winget): `winget install Kubernetes.kind` + +安装kind后，使用以下命令创建集群： +```sh +kind create cluster +``` + +此命令默认创建单节点集群。要与其交互，请使用生成的kubeconfig运行`kubectl`。 + +**Kind用户的重要说明**：如果您使用的是kind集群，在使用`make docker-build`构建镜像后，需要将控制器和任务执行器镜像加载到kind节点中。这是因为kind在Docker容器中运行Kubernetes节点，无法直接访问本地Docker守护进程中的镜像。 + +使用以下命令将镜像加载到kind集群中： +```sh +kind load docker-image : +kind load docker-image : +``` + +例如，如果您使用`make docker-build IMG=my-controller:latest`构建镜像，则使用以下命令加载： +```sh +kind load docker-image my-controller:latest +``` + +完成后使用以下命令删除集群： +```sh +kind delete cluster +``` + +有关使用kind的更多详细说明，请参阅[官方kind文档](https://kind.sigs.k8s.io/docs/user/quick-start/)。 + +### 部署 + +此项目需要两个独立的镜像 - 一个用于控制器，另一个用于任务执行器组件。 + +1. **构建和推送您的镜像：** + ```sh + # 构建和推送控制器镜像 + make docker-build docker-push IMG=/opensandbox-controller:tag + + # 构建和推送任务执行器镜像 + make docker-build-task-executor docker-push-task-executor IMG=/opensandbox-task-executor:tag + ``` + + **注意：** 这些镜像应该发布在您指定的个人注册表中。需要能够从工作环境中拉取镜像。如果上述命令不起作用，请确保您对注册表具有适当的权限。 + +2. **将CRD安装到集群中：** + ```sh + make install + ``` + +3. **将管理器部署到集群：** + ```sh + make deploy IMG=/opensandbox-controller:tag TASK_EXECUTOR_IMG=/opensandbox-task-executor:tag + ``` + + **注意**：您可能需要授予自己集群管理员权限或以管理员身份登录以确保您在运行命令之前具有集群管理员权限。 + +**Kind用户的重要说明**：如果您使用的是kind集群，需要在构建镜像后将两个镜像都加载到kind节点中： +```sh +kind load docker-image : +kind load docker-image : +``` + +### 创建BatchSandbox和Pool资源 + +#### 基础示例 +创建一个简单的非池化沙箱，不带任务调度： + +```yaml +apiVersion: sandbox.opensandbox.io/v1alpha1 +kind: BatchSandbox +metadata: + name: basic-batch-sandbox +spec: + replicas: 2 + template: + spec: + containers: + - name: sandbox-container + image: nginx:latest + ports: + - containerPort: 80 +``` + +应用批处理沙箱配置： +```sh +kubectl apply -f basic-batch-sandbox.yaml +``` + +检查批处理沙箱状态： +```sh +kubectl get batchsandbox basic-batch-sandbox -o wide +``` + +示例输出： +```sh +NAME DESIRED TOTAL ALLOCATED READY EXPIRE AGE +basic-batch-sandbox 2 2 2 2 5m +``` + +状态字段说明： +- **DESIRED**：请求的沙箱数量 +- **TOTAL**：创建的沙箱总数 +- **ALLOCATED**：成功分配的沙箱数量 +- **READY**：准备使用的沙箱数量 +- **EXPIRE**：过期时间（未设置时为空） +- **AGE**：资源创建以来的时间 + +沙箱准备好后，您可以在注解中找到端点信息： +```sh +kubectl get batchsandbox basic-batch-sandbox -o jsonpath='{.metadata.annotations.sandbox\.opensandbox\.io/endpoints}' +``` + +这将显示交付沙箱的IP地址。 + +#### 高级示例 + +##### 不带任务的池化沙箱 +首先，创建一个资源池： + +```yaml +apiVersion: sandbox.opensandbox.io/v1alpha1 +kind: Pool +metadata: + name: example-pool +spec: + template: + spec: + containers: + - name: sandbox-container + image: nginx:latest + ports: + - containerPort: 80 + capacitySpec: + bufferMax: 10 + bufferMin: 2 + poolMax: 20 + poolMin: 5 +``` + +应用资源池配置： +```sh +kubectl apply -f pool-example.yaml +``` + +使用资源池创建一批沙箱： + +```yaml +apiVersion: sandbox.opensandbox.io/v1alpha1 +kind: BatchSandbox +metadata: + name: pooled-batch-sandbox +spec: + replicas: 3 + poolRef: example-pool + template: + spec: + containers: + - name: sandbox-container + image: nginx:latest + ports: + - containerPort: 80 +``` + +应用批处理沙箱配置： +```sh +kubectl apply -f pooled-batch-sandbox.yaml +``` + +##### 带异构任务的池化沙箱 +创建一批带有基于进程的异构任务的沙箱。为了使任务执行正常工作，任务执行器必须作为sidecar容器部署在资源池模板中，并与沙箱容器共享进程命名空间： + +首先，创建一个带有任务执行器sidecar的资源池： + +```yaml +apiVersion: sandbox.opensandbox.io/v1alpha1 +kind: Pool +metadata: + name: task-example-pool +spec: + template: + spec: + shareProcessNamespace: true + containers: + - name: sandbox-container + image: ubuntu:latest + command: ["sleep", "3600"] + - name: task-executor + image: : + securityContext: + capabilities: + add: ["SYS_PTRACE"] + capacitySpec: + bufferMax: 10 + bufferMin: 2 + poolMax: 20 + poolMin: 5 +``` + +使用我们刚刚创建的资源池创建一批带有基于进程的异构任务的沙箱： + +```yaml +apiVersion: sandbox.opensandbox.io/v1alpha1 +kind: BatchSandbox +metadata: + name: task-batch-sandbox +spec: + replicas: 2 + poolRef: task-example-pool + template: + spec: + shareProcessNamespace: true + containers: + - name: sandbox-container + image: ubuntu:latest + command: ["sleep", "3600"] + - name: task-executor + image: : + securityContext: + capabilities: + add: ["SYS_PTRACE"] + taskTemplate: + spec: + process: + command: ["echo", "Default task"] + shardTaskPatches: + - spec: + process: + command: ["echo", "Custom task for sandbox 1"] + - spec: + process: + command: ["echo", "Custom task for sandbox 2"] + args: ["with", "additional", "arguments"] +``` + +应用批处理沙箱配置： +```sh +kubectl apply -f task-batch-sandbox.yaml +``` + +检查带任务的批处理沙箱状态： +```sh +kubectl get batchsandbox task-batch-sandbox -o wide +``` + +示例输出： +```sh +NAME DESIRED TOTAL ALLOCATED READY TASK_RUNNING TASK_SUCCEED TASK_FAILED TASK_UNKNOWN EXPIRE AGE +task-batch-sandbox 2 2 2 2 0 2 0 0 5m +``` + +任务状态字段说明： +- **TASK_RUNNING**：当前正在执行的任务数 +- **TASK_SUCCEED**：成功完成的任务数 +- **TASK_FAILED**：失败的任务数 +- **TASK_UNKNOWN**：状态未知的任务数 + +当您删除带有运行任务的BatchSandbox时，控制器将首先停止所有任务，然后删除BatchSandbox资源。一旦所有任务都成功终止，BatchSandbox将被完全删除，沙箱将返回到资源池中以供重用。 + +删除BatchSandbox： +```sh +kubectl delete batchsandbox task-batch-sandbox +``` + +您可以通过观察BatchSandbox状态来监控删除过程： +```sh +kubectl get batchsandbox task-batch-sandbox -w +``` + +### 监控资源 +检查资源池和批处理沙箱的状态： +```sh +# 查看资源池状态 +kubectl get pools + +# 查看批处理沙箱状态 +kubectl get batchsandboxes + +# 获取特定资源的详细信息 +kubectl describe pool example-pool +kubectl describe batchsandbox example-batch-sandbox +``` + +## 项目结构 + +``` +├── api/ +│ └── v1alpha1/ # 自定义资源定义（BatchSandbox, Pool） +├── cmd/ +│ ├── controller/ # 主控制器管理器入口点 +│ └── task-executor/ # 任务执行器二进制文件 +├── config/ +│ ├── crd/ # 自定义资源定义清单 +│ ├── default/ # 控制器部署的默认配置 +│ ├── manager/ # 控制器管理器配置 +│ ├── rbac/ # 基于角色的访问控制清单 +│ └── samples/ # 资源的示例YAML清单 +├── hack/ # 开发脚本和工具 +├── images/ # 文档图片 +├── internal/ +│ ├── controller/ # 核心控制器实现 +│ ├── scheduler/ # 资源分配和调度逻辑 +│ ├── task-executor/ # 任务执行引擎内部实现 +│ └── utils/ # 实用函数和助手 +├── pkg/ +│ └── task-executor/ # 共享的任务执行器包 +└── test/ # 测试套件 +``` + +## 贡献 +欢迎为OpenSandbox Kubernetes控制器项目做出贡献。请随时提交问题、功能请求和拉取请求。 + +**注意：** 运行`make help`以获取所有潜在`make`目标的更多信息 + +更多信息请参见[Kubebuilder文档](https://book.kubebuilder.io/introduction.html) + +## 许可证 +此项目在Apache 2.0许可证下开源。 + +您可以将OpenSandbox用于个人或商业项目，只要遵守许可证条款即可。 \ No newline at end of file diff --git a/kubernetes/README.md b/kubernetes/README.md new file mode 100644 index 00000000..a85f55c3 --- /dev/null +++ b/kubernetes/README.md @@ -0,0 +1,378 @@ +# OpenSandbox Kubernetes Controller + +[English](README.md) | [中文](README-ZH.md) + +OpenSandbox Kubernetes Controller is a Kubernetes operator that manages sandbox environments through custom resources. It provides **automated sandbox lifecycle management**, **resource pooling for fast provisioning**, **batch sandbox creation**, and **optional task orchestration** capabilities in Kubernetes clusters. + +## Key Features + +- **Flexible Sandbox Creation**: Choose between pooled and non-pooled sandbox creation modes +- **Batch and Individual Delivery**: Support both single sandbox (for real-user interactions) and batch sandbox delivery (for high-throughput agentic-RL scenarios) +- **Optional Task Scheduling**: Integrated task orchestration with optional shard task templates for heterogeneous task distribution and customized sandbox delivery (e.g., process injection or dynamic container creation) +- **Resource Pooling**: Maintain pre-warmed resource pools for rapid sandbox provisioning +- **Comprehensive Monitoring**: Real-time status tracking of sandboxes and tasks + +## Features + +### Batch Sandbox Management +The BatchSandbox custom resource allows you to create and manage multiple identical sandbox environments. Key capabilities include: +- **Flexible Creation Modes**: Support both pooled (using resource pools) and non-pooled sandbox creation +- **Single and Batch Delivery**: Create single sandboxes (replicas=1) or batches of sandboxes (replicas=N) as needed +- **Scalable Replica Management**: Easily control the number of sandbox instances through replica configuration +- **Automatic Expiration**: Set TTL (time-to-live) for automatic cleanup of expired sandboxes +- **Optional Task Scheduling**: Built-in task execution engine with support for optional task templates +- **Detailed Status Reporting**: Comprehensive metrics on replicas, allocations, and task states + +### Resource Pooling +The Pool custom resource maintains a pool of pre-warmed compute resources to enable rapid sandbox provisioning: +- Configurable buffer sizes (minimum and maximum) to balance resource availability and cost +- Pool capacity limits to control overall resource consumption +- Automatic resource allocation and deallocation based on demand +- Real-time status monitoring showing total, allocated, and available resources + +### Task Orchestration +Integrated task management system that executes custom workloads within sandboxes: +- **Optional Execution**: Task scheduling is completely optional - sandboxes can be created without tasks +- **Container and Process Tasks**: Support for both container-based and process-based tasks +- **Heterogeneous Task Distribution**: Customize individual tasks for each sandbox in a batch using shardTaskPatches + +### Advanced Scheduling +Intelligent resource management features: +- Minimum and maximum buffer settings to ensure resource availability while controlling costs +- Pool-wide capacity limits to prevent resource exhaustion +- Automatic scaling based on demand + +## Getting Started + +### Prerequisites +- go version v1.24.0+ +- docker version 17.03+. +- kubectl version v1.11.3+. +- Access to a Kubernetes v1.33.1+ cluster. + +If you don't have access to a Kubernetes cluster, you can use [kind](https://kind.sigs.k8s.io/) to create a local Kubernetes cluster for testing purposes. Kind runs Kubernetes nodes in Docker containers, making it easy to set up a local development environment. + +To install kind: +- Download the release binary for your OS from the [releases page](https://github.com/kubernetes-sigs/kind/releases) and move it into your `$PATH` +- Or use a package manager: + - macOS (Homebrew): `brew install kind` + - Windows (winget): `winget install Kubernetes.kind` + +After installing kind, create a cluster with: +```sh +kind create cluster +``` + +This command creates a single-node cluster by default. To interact with it, use `kubectl` with the generated kubeconfig. + +**Important Note for Kind Users**: If you're using a kind cluster, you need to load the controller and task-executor images into the kind node after building them with `make docker-build`. This is because kind runs Kubernetes nodes in Docker containers and cannot directly access images from your local Docker daemon. + +Load the images into the kind cluster with: +```sh +kind load docker-image : +kind load docker-image : +``` + +For example, if you built your images with `make docker-build IMG=my-controller:latest`, you would load them with: +```sh +kind load docker-image my-controller:latest +``` + +Delete the cluster when you're done with: +```sh +kind delete cluster +``` + +For more detailed instructions on using kind, please refer to the [official kind documentation](https://kind.sigs.k8s.io/docs/user/quick-start/). + +### Deployment + +This project requires two separate images - one for the controller and another for the task-executor component. + +1. **Build and push your images:** + ```sh + # Build and push the controller image + make docker-build docker-push IMG=/opensandbox-controller:tag + + # Build and push the task-executor image + make docker-build-task-executor docker-push-task-executor IMG=/opensandbox-task-executor:tag + ``` + + **NOTE:** These images ought to be published in the personal registry you specified. And it is required to have access to pull the images from the working environment. Make sure you have the proper permission to the registry if the above commands don't work. + +2. **Install the CRDs into the cluster:** + ```sh + make install + ``` + +3. **Deploy the Manager to the cluster:** + ```sh + make deploy IMG=/opensandbox-controller:tag TASK_EXECUTOR_IMG=/opensandbox-task-executor:tag + ``` + + **NOTE**: you may need to grant yourself cluster-admin privileges or be logged in as admin to ensure you have cluster-admin privileges before running the commands. + +**Important Note for Kind Users**: If you're using a kind cluster, you need to load both images into the kind node after building them: +```sh +kind load docker-image : +kind load docker-image : +``` + +### Creating BatchSandbox and Pool Resources + +#### Basic Example +Create a simple non-pooled sandbox without task scheduling: + +```yaml +apiVersion: sandbox.opensandbox.io/v1alpha1 +kind: BatchSandbox +metadata: + name: basic-batch-sandbox +spec: + replicas: 2 + template: + spec: + containers: + - name: sandbox-container + image: nginx:latest + ports: + - containerPort: 80 +``` + +Apply the batch sandbox configuration: +```sh +kubectl apply -f basic-batch-sandbox.yaml +``` + +Check the status of your batch sandbox: +```sh +kubectl get batchsandbox basic-batch-sandbox -o wide +``` + +Example output: +```sh +NAME DESIRED TOTAL ALLOCATED READY EXPIRE AGE +basic-batch-sandbox 2 2 2 2 5m +``` + +Status field explanations: +- **DESIRED**: The number of sandboxes requested +- **TOTAL**: The total number of sandboxes created +- **ALLOCATED**: The number of sandboxes successfully allocated +- **READY**: The number of sandboxes ready for use +- **EXPIRE**: Expiration time (empty if not set) +- **AGE**: Time since the resource was created + +After the sandboxes are ready, you can find the endpoint information in the annotations: +```sh +kubectl get batchsandbox basic-batch-sandbox -o jsonpath='{.metadata.annotations.sandbox\.opensandbox\.io/endpoints}' +``` + +This will show the IP addresses of the delivered sandboxes. + +#### Advanced Examples + +##### Pooled Sandbox Without Task +First, create a resource pool: + +```yaml +apiVersion: sandbox.opensandbox.io/v1alpha1 +kind: Pool +metadata: + name: example-pool +spec: + template: + spec: + containers: + - name: sandbox-container + image: nginx:latest + ports: + - containerPort: 80 + capacitySpec: + bufferMax: 10 + bufferMin: 2 + poolMax: 20 + poolMin: 5 +``` + +Apply the pool configuration: +```sh +kubectl apply -f pool-example.yaml +``` + +Create a batch of sandboxes using the pool: + +```yaml +apiVersion: sandbox.opensandbox.io/v1alpha1 +kind: BatchSandbox +metadata: + name: pooled-batch-sandbox +spec: + replicas: 3 + poolRef: example-pool + template: + spec: + containers: + - name: sandbox-container + image: nginx:latest + ports: + - containerPort: 80 +``` + +Apply the batch sandbox configuration: +```sh +kubectl apply -f pooled-batch-sandbox.yaml +``` + +##### Pooled Sandbox With Heterogeneous Tasks +Create a batch of sandboxes with process-based heterogeneous tasks. For task execution to work properly, the task-executor must be deployed as a sidecar container in the pool template and share the process namespace with the sandbox container: + +First, create a resource pool with the task-executor sidecar: + +```yaml +apiVersion: sandbox.opensandbox.io/v1alpha1 +kind: Pool +metadata: + name: task-example-pool +spec: + template: + spec: + shareProcessNamespace: true + containers: + - name: sandbox-container + image: ubuntu:latest + command: ["sleep", "3600"] + - name: task-executor + image: : + securityContext: + capabilities: + add: ["SYS_PTRACE"] + capacitySpec: + bufferMax: 10 + bufferMin: 2 + poolMax: 20 + poolMin: 5 +``` + +Create a batch of sandboxes with process-based heterogeneous tasks using the pool we just created: + +```yaml +apiVersion: sandbox.opensandbox.io/v1alpha1 +kind: BatchSandbox +metadata: + name: task-batch-sandbox +spec: + replicas: 2 + poolRef: task-example-pool + template: + spec: + shareProcessNamespace: true + containers: + - name: sandbox-container + image: ubuntu:latest + command: ["sleep", "3600"] + - name: task-executor + image: : + securityContext: + capabilities: + add: ["SYS_PTRACE"] + taskTemplate: + spec: + process: + command: ["echo", "Default task"] + shardTaskPatches: + - spec: + process: + command: ["echo", "Custom task for sandbox 1"] + - spec: + process: + command: ["echo", "Custom task for sandbox 2"] + args: ["with", "additional", "arguments"] +``` + +Apply the batch sandbox configuration: +```sh +kubectl apply -f task-batch-sandbox.yaml +``` + +Check the status of your batch sandbox with tasks: +```sh +kubectl get batchsandbox task-batch-sandbox -o wide +``` + +Example output: +```sh +NAME DESIRED TOTAL ALLOCATED READY TASK_RUNNING TASK_SUCCEED TASK_FAILED TASK_UNKNOWN EXPIRE AGE +task-batch-sandbox 2 2 2 2 0 2 0 0 5m +``` + +Task status field explanations: +- **TASK_RUNNING**: The number of tasks currently executing +- **TASK_SUCCEED**: The number of tasks that have completed successfully +- **TASK_FAILED**: The number of tasks that have failed +- **TASK_UNKNOWN**: The number of tasks with unknown status + +When you delete a BatchSandbox with running tasks, the controller will first stop all tasks before deleting the BatchSandbox resource. Once all tasks are successfully terminated, the BatchSandbox will be completely removed, and the sandboxes will be returned to the pool for reuse. + +To delete the BatchSandbox: +```sh +kubectl delete batchsandbox task-batch-sandbox +``` + +You can monitor the deletion process by watching the BatchSandbox status: +```sh +kubectl get batchsandbox task-batch-sandbox -w +``` + +### Monitoring Resources +Check the status of your pools and batch sandboxes: + +```sh +# View pool status +kubectl get pools + +# View batch sandbox status +kubectl get batchsandboxes + +# Get detailed information about a specific resource +kubectl describe pool example-pool +kubectl describe batchsandbox example-batch-sandbox +``` + +## Project Structure + +``` +├── api/ +│ └── v1alpha1/ # Custom resource definitions (BatchSandbox, Pool) +├── cmd/ +│ ├── controller/ # Main controller manager entry point +│ └── task-executor/ # Task executor binary +├── config/ +│ ├── crd/ # Custom resource definitions manifests +│ ├── default/ # Default configuration for controller deployment +│ ├── manager/ # Controller manager configuration +│ ├── rbac/ # Role-based access control manifests +│ └── samples/ # Sample YAML manifests for resources +├── hack/ # Development scripts and tools +├── images/ # Documentation images +├── internal/ +│ ├── controller/ # Core controller implementations +│ ├── scheduler/ # Resource allocation and scheduling logic +│ ├── task-executor/ # Task execution engine internals +│ └── utils/ # Utility functions and helpers +├── pkg/ +│ └── task-executor/ # Shared task executor packages +└── test/ # Test suites and utilities +``` + +## Contributing +We welcome contributions to the OpenSandbox Kubernetes Controller project. Please feel free to submit issues, feature requests, and pull requests. + +**NOTE:** Run `make help` for more information on all potential `make` targets + +More information can be found via the [Kubebuilder Documentation](https://book.kubebuilder.io/introduction.html) + +## License +This project is open source under the Apache 2.0 License. + +You can use OpenSandbox for personal or commercial projects in compliance with the license terms. \ No newline at end of file diff --git a/kubernetes/api/v1alpha1/batchsandbox_types.go b/kubernetes/api/v1alpha1/batchsandbox_types.go new file mode 100644 index 00000000..1c2cc12e --- /dev/null +++ b/kubernetes/api/v1alpha1/batchsandbox_types.go @@ -0,0 +1,263 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// BatchSandboxSpec defines the desired state of BatchSandbox. +type BatchSandboxSpec struct { + // Replicas is the number of desired replicas. + // +kubebuilder:validation:Required + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:default=1 + Replicas *int32 `json:"replicas,omitempty"` + // PoolRef Pool name + // +optional + // +kubebuilder:validation:Optional + PoolRef string `json:"poolRef,omitempty"` + // +optional + // Template describes the pods that will be created. + // +kubebuilder:pruning:PreserveUnknownFields + // +kubebuilder:validation:Schemaless + // +kubebuilder:validation:Optional + Template *corev1.PodTemplateSpec `json:"template"` + // ExpireTime - Absolute time when the batch-sandbox is deleted. + // If a time in the past is provided, the sandbox will be deleted immediately. + // +optional + // +kubebuilder:validation:Format="date-time" + // +kubebuilder:validation:Optional + ExpireTime *metav1.Time `json:"expireTime,omitempty"` + // Task is a custom task spec that is automatically dispatched after the sandbox is successfully created. + // The Sandbox is responsible for managing the lifecycle of the task. + // +optional + // +kubebuilder:pruning:PreserveUnknownFields + // +kubebuilder:validation:Schemaless + // +kubebuilder:validation:Optional + TaskTemplate *TaskTemplateSpec `json:"taskTemplate,omitempty"` + // ShardTaskPatches indicates patching to the TaskTemplate for individual Task. + // +kubebuilder:pruning:PreserveUnknownFields + // +kubebuilder:validation:Schemaless + // +optional + // +kubebuilder:validation:Optional + ShardTaskPatches []runtime.RawExtension `json:"shardTaskPatches,omitempty"` + + // TaskResourcePolicyWhenCompleted specifies how resources should be handled once a task reaches a completed state (SUCCEEDED or FAILED). + // - Retain: Keep the resources until the BatchSandbox is deleted. + // - Release: Free the resources immediately when the task completes. + // +optional + // +kubebuilder:default=Retain + // +kubebuilder:validation:Optional + TaskResourcePolicyWhenCompleted *TaskResourcePolicy `json:"taskResourcePolicyWhenCompleted,omitempty"` +} + +type TaskResourcePolicy string + +const ( + TaskResourcePolicyRetain TaskResourcePolicy = "Retain" + TaskResourcePolicyRelease TaskResourcePolicy = "Release" +) + +// BatchSandboxStatus defines the observed state of BatchSandbox. +type BatchSandboxStatus struct { + // ObservedGeneration is the most recent generation observed for this BatchSandbox. It corresponds to the + // BatchSandbox's generation, which is updated on mutation by the API Server. + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + // Replicas is the number of actual Pods + Replicas int32 `json:"replicas"` + // Allocated is the number of actual scheduled Pod + Allocated int32 `json:"allocated"` + // Ready is the number of actual Ready Pod + Ready int32 `json:"ready"` + // TaskRunning is the number of Running task + TaskRunning int32 `json:"taskRunning"` + // TaskSucceed is the number of Succeed task + TaskSucceed int32 `json:"taskSucceed"` + // TaskFailed is the number of Failed task + TaskFailed int32 `json:"taskFailed"` + // TaskPending is the number of Pending task which is unassigned + TaskPending int32 `json:"taskPending"` + // TaskUnknown is the number of Unknown task + TaskUnknown int32 `json:"taskUnknown"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:shortName=bsbx +// +kubebuilder:printcolumn:name="DESIRED",type="integer",JSONPath=".spec.replicas",description="The desired number of pods." +// +kubebuilder:printcolumn:name="TOTAL",type="integer",JSONPath=".status.replicas",description="The number of currently all pods." +// +kubebuilder:printcolumn:name="ALLOCATED",type="integer",JSONPath=".status.allocated",description="The number of currently all allocated pods." +// +kubebuilder:printcolumn:name="Ready",type="integer",JSONPath=".status.ready",description="The number of currently all ready pods." +// +kubebuilder:printcolumn:name="TASK_RUNNING",type="integer",priority=1,JSONPath=".status.taskRunning",description="The number of currently all running tasks." +// +kubebuilder:printcolumn:name="TASK_SUCCEED",type="integer",priority=1,JSONPath=".status.taskSucceed",description="The number of currently all succeed tasks." +// +kubebuilder:printcolumn:name="TASK_FAILED",type="integer",priority=1,JSONPath=".status.taskFailed",description="The number of currently all failed tasks." +// +kubebuilder:printcolumn:name="TASK_UNKNOWN",type="integer",priority=1,JSONPath=".status.taskUnknown",description="The number of currently all unknown tasks." +// +kubebuilder:printcolumn:name="EXPIRE",type="string",JSONPath=".spec.expireTime",description="sandbox expire time" +// +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp",description="CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC." +// BatchSandbox is the Schema for the batchsandboxes API. +type BatchSandbox struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec BatchSandboxSpec `json:"spec,omitempty"` + Status BatchSandboxStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// BatchSandboxList contains a list of BatchSandbox. +type BatchSandboxList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []BatchSandbox `json:"items"` +} + +func init() { + SchemeBuilder.Register(&BatchSandbox{}, &BatchSandboxList{}) +} + +// TaskTemplateSpec task spec +type TaskTemplateSpec struct { + // Standard object's metadata. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + // +optional + metav1.ObjectMeta `json:"metadata,omitempty"` + // +optional + Spec TaskSpec `json:"spec,omitempty"` +} + +type Task struct { + // +optional + metav1.ObjectMeta `json:"metadata,omitempty"` + // +optional + Spec TaskSpec `json:"spec,omitempty"` +} + +type TaskSpec struct { + // +optional + Container *ContainerTask `json:"container,omitempty"` + // +optional + Process *ProcessTask `json:"process,omitempty"` + // TimeoutSeconds specifies the maximum duration in seconds for task execution. + // If exceeded, the task executor should terminate the task. + // +optional + TimeoutSeconds *int64 `json:"timeoutSeconds,omitempty"` +} + +type ContainerTask struct { + // +kubebuilder:validation:Required + Image string `json:"image"` + // Command command + // +optional + Command []string `json:"command"` + // Arguments to the entrypoint. + // +optional + Args []string `json:"args,omitempty"` + // List of environment variables to set in the task. + // +optional + // +patchMergeKey=name + // +patchStrategy=merge + Env []corev1.EnvVar `json:"env,omitempty"` + // WorkingDir task working directory. + // +optional + WorkingDir string `json:"workingDir,omitempty"` + // more container fields add HERE +} + +type ProcessTask struct { + // Command command + // +kubebuilder:validation:Required + Command []string `json:"command"` + // Arguments to the entrypoint. + // +optional + Args []string `json:"args,omitempty"` + // List of environment variables to set in the task. + // +optional + // +patchMergeKey=name + // +patchStrategy=merge + Env []corev1.EnvVar `json:"env,omitempty"` + // WorkingDir task working directory. + // +optional + WorkingDir string `json:"workingDir,omitempty"` +} + +// TaskStatus task status +type TaskStatus struct { + // Details about the task's current condition. + // +optional + State TaskState `json:"state,omitempty"` + // Details about the task's last termination condition. + // +optional + LastTerminationState TaskState `json:"lastState,omitempty"` +} + +// TaskState holds a possible state of task. +// Only one of its members may be specified. +// If none of them is specified, the default one is TaskStateWaiting. +type TaskState struct { + // Details about a waiting task + // +optional + Waiting *TaskStateWaiting `json:"waiting,omitempty"` + // Details about a running task + // +optional + Running *TaskStateRunning `json:"running,omitempty"` + // Details about a terminated task + // +optional + Terminated *TaskStateTerminated `json:"terminated,omitempty"` +} + +// TaskStateWaiting is a waiting state of a task. +type TaskStateWaiting struct { + // (brief) reason the task is not yet running. + // +optional + Reason string `json:"reason,omitempty"` + // Message regarding why the task is not yet running. + // +optional + Message string `json:"message,omitempty"` +} + +// TaskStateRunning is a running state of a task. +type TaskStateRunning struct { + // Time at which the task was last (re-)started + // +optional + StartedAt metav1.Time `json:"startedAt,omitempty"` +} + +// TaskStateTerminated is a terminated state of a task. +type TaskStateTerminated struct { + // Exit status from the last termination of the task + ExitCode int32 `json:"exitCode"` + // Signal from the last termination of the task + // +optional + Signal int32 `json:"signal,omitempty"` + // (brief) reason from the last termination of the task + // +optional + Reason string `json:"reason,omitempty"` + // Message regarding the last termination of the task + // +optional + Message string `json:"message,omitempty"` + // Time at which previous execution of the task started + // +optional + StartedAt metav1.Time `json:"startedAt,omitempty"` + // Time at which the task last terminated + // +optional + FinishedAt metav1.Time `json:"finishedAt,omitempty"` +} diff --git a/kubernetes/api/v1alpha1/groupversion_info.go b/kubernetes/api/v1alpha1/groupversion_info.go new file mode 100644 index 00000000..55ac3279 --- /dev/null +++ b/kubernetes/api/v1alpha1/groupversion_info.go @@ -0,0 +1,34 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package v1alpha1 contains API Schema definitions for the sandbox v1alpha1 API group. +// +kubebuilder:object:generate=true +// +groupName=sandbox.opensandbox.io +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects. + GroupVersion = schema.GroupVersion{Group: "sandbox.opensandbox.io", Version: "v1alpha1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme. + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/kubernetes/api/v1alpha1/pool_types.go b/kubernetes/api/v1alpha1/pool_types.go new file mode 100644 index 00000000..6238afa0 --- /dev/null +++ b/kubernetes/api/v1alpha1/pool_types.go @@ -0,0 +1,96 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// PoolSpec defines the desired state of Pool. +type PoolSpec struct { + // Pod Template used to create pre-warmed nodes in the pool. + // +kubebuilder:pruning:PreserveUnknownFields + // +kubebuilder:validation:Schemaless + // +kubebuilder:validation:Optional + Template *corev1.PodTemplateSpec `json:"template"` + // CapacitySpec controls the size of the resource pool. + // +kubebuilder:validation:Required + CapacitySpec CapacitySpec `json:"capacitySpec"` +} + +type CapacitySpec struct { + // BufferMax is the maximum number of nodes kept in the warm buffer. + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Required + BufferMax int32 `json:"bufferMax"` + // BufferMin is the minimum number of nodes that must remain in the buffer. + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Required + BufferMin int32 `json:"bufferMin"` + // PoolMax is the maximum total number of nodes allowed in the entire pool. + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Required + PoolMax int32 `json:"poolMax"` + // PoolMin is the minimum total size of the pool. + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Required + PoolMin int32 `json:"poolMin"` +} + +// PoolStatus defines the observed state of Pool. +type PoolStatus struct { + // ObservedGeneration is the most recent generation observed for this BatchSandbox. It corresponds to the + // BatchSandbox's generation, which is updated on mutation by the API Server. + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + // Revision is the latest version of pool + Revision string `json:"revision"` + // Total is the total number of nodes in the pool. + Total int32 `json:"total"` + // Allocated is the number of nodes currently allocated to sandboxes. + Allocated int32 `json:"allocated"` + // Available is the number of nodes currently available in the pool. + Available int32 `json:"available"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="TOTAL",type="integer",JSONPath=".status.total",description="The number of all nodes in pool." +// +kubebuilder:printcolumn:name="ALLOCATED",type="integer",JSONPath=".status.allocated",description="The number of allocated nodes in pool." +// +kubebuilder:printcolumn:name="AVAILABLE",type="integer",JSONPath=".status.available",description="The number of available nodes in pool." +// Pool is the Schema for the pools API. +type Pool struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec PoolSpec `json:"spec,omitempty"` + Status PoolStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// PoolList contains a list of Pool. +type PoolList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Pool `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Pool{}, &PoolList{}) +} diff --git a/kubernetes/api/v1alpha1/zz_generated.deepcopy.go b/kubernetes/api/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 00000000..fa268567 --- /dev/null +++ b/kubernetes/api/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,479 @@ +//go:build !ignore_autogenerated + +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BatchSandbox) DeepCopyInto(out *BatchSandbox) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BatchSandbox. +func (in *BatchSandbox) DeepCopy() *BatchSandbox { + if in == nil { + return nil + } + out := new(BatchSandbox) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BatchSandbox) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BatchSandboxList) DeepCopyInto(out *BatchSandboxList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]BatchSandbox, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BatchSandboxList. +func (in *BatchSandboxList) DeepCopy() *BatchSandboxList { + if in == nil { + return nil + } + out := new(BatchSandboxList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BatchSandboxList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BatchSandboxSpec) DeepCopyInto(out *BatchSandboxSpec) { + *out = *in + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } + if in.Template != nil { + in, out := &in.Template, &out.Template + *out = new(v1.PodTemplateSpec) + (*in).DeepCopyInto(*out) + } + if in.ExpireTime != nil { + in, out := &in.ExpireTime, &out.ExpireTime + *out = (*in).DeepCopy() + } + if in.TaskTemplate != nil { + in, out := &in.TaskTemplate, &out.TaskTemplate + *out = new(TaskTemplateSpec) + (*in).DeepCopyInto(*out) + } + if in.ShardTaskPatches != nil { + in, out := &in.ShardTaskPatches, &out.ShardTaskPatches + *out = make([]runtime.RawExtension, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.TaskResourcePolicyWhenCompleted != nil { + in, out := &in.TaskResourcePolicyWhenCompleted, &out.TaskResourcePolicyWhenCompleted + *out = new(TaskResourcePolicy) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BatchSandboxSpec. +func (in *BatchSandboxSpec) DeepCopy() *BatchSandboxSpec { + if in == nil { + return nil + } + out := new(BatchSandboxSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BatchSandboxStatus) DeepCopyInto(out *BatchSandboxStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BatchSandboxStatus. +func (in *BatchSandboxStatus) DeepCopy() *BatchSandboxStatus { + if in == nil { + return nil + } + out := new(BatchSandboxStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CapacitySpec) DeepCopyInto(out *CapacitySpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CapacitySpec. +func (in *CapacitySpec) DeepCopy() *CapacitySpec { + if in == nil { + return nil + } + out := new(CapacitySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContainerTask) DeepCopyInto(out *ContainerTask) { + *out = *in + if in.Command != nil { + in, out := &in.Command, &out.Command + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Args != nil { + in, out := &in.Args, &out.Args + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Env != nil { + in, out := &in.Env, &out.Env + *out = make([]v1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContainerTask. +func (in *ContainerTask) DeepCopy() *ContainerTask { + if in == nil { + return nil + } + out := new(ContainerTask) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Pool) DeepCopyInto(out *Pool) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Pool. +func (in *Pool) DeepCopy() *Pool { + if in == nil { + return nil + } + out := new(Pool) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Pool) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PoolList) DeepCopyInto(out *PoolList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Pool, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PoolList. +func (in *PoolList) DeepCopy() *PoolList { + if in == nil { + return nil + } + out := new(PoolList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *PoolList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PoolSpec) DeepCopyInto(out *PoolSpec) { + *out = *in + if in.Template != nil { + in, out := &in.Template, &out.Template + *out = new(v1.PodTemplateSpec) + (*in).DeepCopyInto(*out) + } + out.CapacitySpec = in.CapacitySpec +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PoolSpec. +func (in *PoolSpec) DeepCopy() *PoolSpec { + if in == nil { + return nil + } + out := new(PoolSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PoolStatus) DeepCopyInto(out *PoolStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PoolStatus. +func (in *PoolStatus) DeepCopy() *PoolStatus { + if in == nil { + return nil + } + out := new(PoolStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProcessTask) DeepCopyInto(out *ProcessTask) { + *out = *in + if in.Command != nil { + in, out := &in.Command, &out.Command + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Args != nil { + in, out := &in.Args, &out.Args + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Env != nil { + in, out := &in.Env, &out.Env + *out = make([]v1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProcessTask. +func (in *ProcessTask) DeepCopy() *ProcessTask { + if in == nil { + return nil + } + out := new(ProcessTask) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Task) DeepCopyInto(out *Task) { + *out = *in + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Task. +func (in *Task) DeepCopy() *Task { + if in == nil { + return nil + } + out := new(Task) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TaskSpec) DeepCopyInto(out *TaskSpec) { + *out = *in + if in.Container != nil { + in, out := &in.Container, &out.Container + *out = new(ContainerTask) + (*in).DeepCopyInto(*out) + } + if in.Process != nil { + in, out := &in.Process, &out.Process + *out = new(ProcessTask) + (*in).DeepCopyInto(*out) + } + if in.TimeoutSeconds != nil { + in, out := &in.TimeoutSeconds, &out.TimeoutSeconds + *out = new(int64) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TaskSpec. +func (in *TaskSpec) DeepCopy() *TaskSpec { + if in == nil { + return nil + } + out := new(TaskSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TaskState) DeepCopyInto(out *TaskState) { + *out = *in + if in.Waiting != nil { + in, out := &in.Waiting, &out.Waiting + *out = new(TaskStateWaiting) + **out = **in + } + if in.Running != nil { + in, out := &in.Running, &out.Running + *out = new(TaskStateRunning) + (*in).DeepCopyInto(*out) + } + if in.Terminated != nil { + in, out := &in.Terminated, &out.Terminated + *out = new(TaskStateTerminated) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TaskState. +func (in *TaskState) DeepCopy() *TaskState { + if in == nil { + return nil + } + out := new(TaskState) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TaskStateRunning) DeepCopyInto(out *TaskStateRunning) { + *out = *in + in.StartedAt.DeepCopyInto(&out.StartedAt) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TaskStateRunning. +func (in *TaskStateRunning) DeepCopy() *TaskStateRunning { + if in == nil { + return nil + } + out := new(TaskStateRunning) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TaskStateTerminated) DeepCopyInto(out *TaskStateTerminated) { + *out = *in + in.StartedAt.DeepCopyInto(&out.StartedAt) + in.FinishedAt.DeepCopyInto(&out.FinishedAt) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TaskStateTerminated. +func (in *TaskStateTerminated) DeepCopy() *TaskStateTerminated { + if in == nil { + return nil + } + out := new(TaskStateTerminated) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TaskStateWaiting) DeepCopyInto(out *TaskStateWaiting) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TaskStateWaiting. +func (in *TaskStateWaiting) DeepCopy() *TaskStateWaiting { + if in == nil { + return nil + } + out := new(TaskStateWaiting) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TaskStatus) DeepCopyInto(out *TaskStatus) { + *out = *in + in.State.DeepCopyInto(&out.State) + in.LastTerminationState.DeepCopyInto(&out.LastTerminationState) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TaskStatus. +func (in *TaskStatus) DeepCopy() *TaskStatus { + if in == nil { + return nil + } + out := new(TaskStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TaskTemplateSpec) DeepCopyInto(out *TaskTemplateSpec) { + *out = *in + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TaskTemplateSpec. +func (in *TaskTemplateSpec) DeepCopy() *TaskTemplateSpec { + if in == nil { + return nil + } + out := new(TaskTemplateSpec) + in.DeepCopyInto(out) + return out +} diff --git a/kubernetes/cmd/controller/main.go b/kubernetes/cmd/controller/main.go new file mode 100644 index 00000000..e6ab938b --- /dev/null +++ b/kubernetes/cmd/controller/main.go @@ -0,0 +1,261 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "crypto/tls" + "flag" + "os" + "path/filepath" + + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + // to ensure that exec-entrypoint and run can make use of them. + _ "k8s.io/client-go/plugin/pkg/client/auth" + "k8s.io/klog/v2" + + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + sandboxv1alpha1 "github.com/alibaba/OpenSandbox/sandbox-k8s/api/v1alpha1" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/controller" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/utils/fieldindex" + // +kubebuilder:scaffold:imports +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + + utilruntime.Must(sandboxv1alpha1.AddToScheme(scheme)) + // +kubebuilder:scaffold:scheme +} + +// nolint:gocyclo +func main() { + var metricsAddr string + var metricsCertPath, metricsCertName, metricsCertKey string + var webhookCertPath, webhookCertName, webhookCertKey string + var enableLeaderElection bool + var probeAddr string + var secureMetrics bool + var enableHTTP2 bool + var tlsOpts []func(*tls.Config) + flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ + "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") + flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + flag.BoolVar(&enableLeaderElection, "leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + flag.BoolVar(&secureMetrics, "metrics-secure", true, + "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&webhookCertPath, "webhook-cert-path", "", "The directory that contains the webhook certificate.") + flag.StringVar(&webhookCertName, "webhook-cert-name", "tls.crt", "The name of the webhook certificate file.") + flag.StringVar(&webhookCertKey, "webhook-cert-key", "tls.key", "The name of the webhook key file.") + flag.StringVar(&metricsCertPath, "metrics-cert-path", "", + "The directory that contains the metrics server certificate.") + flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") + flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") + flag.BoolVar(&enableHTTP2, "enable-http2", false, + "If set, HTTP/2 will be enabled for the metrics and webhook servers") + opts := zap.Options{ + Development: true, + } + opts.BindFlags(flag.CommandLine) + + klog.InitFlags(flag.CommandLine) + + flag.Parse() + + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + + // if the enable-http2 flag is false (the default), http/2 should be disabled + // due to its vulnerabilities. More specifically, disabling http/2 will + // prevent from being vulnerable to the HTTP/2 Stream Cancellation and + // Rapid Reset CVEs. For more information see: + // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 + // - https://github.com/advisories/GHSA-4374-p667-p6c8 + disableHTTP2 := func(c *tls.Config) { + setupLog.Info("disabling http/2") + c.NextProtos = []string{"http/1.1"} + } + + if !enableHTTP2 { + tlsOpts = append(tlsOpts, disableHTTP2) + } + + // Create watchers for metrics and webhooks certificates + var metricsCertWatcher, webhookCertWatcher *certwatcher.CertWatcher + + // Initial webhook TLS options + webhookTLSOpts := tlsOpts + + if len(webhookCertPath) > 0 { + setupLog.Info("Initializing webhook certificate watcher using provided certificates", + "webhook-cert-path", webhookCertPath, "webhook-cert-name", webhookCertName, "webhook-cert-key", webhookCertKey) + + var err error + webhookCertWatcher, err = certwatcher.New( + filepath.Join(webhookCertPath, webhookCertName), + filepath.Join(webhookCertPath, webhookCertKey), + ) + if err != nil { + setupLog.Error(err, "Failed to initialize webhook certificate watcher") + os.Exit(1) + } + + webhookTLSOpts = append(webhookTLSOpts, func(config *tls.Config) { + config.GetCertificate = webhookCertWatcher.GetCertificate + }) + } + + webhookServer := webhook.NewServer(webhook.Options{ + TLSOpts: webhookTLSOpts, + }) + + // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. + // More info: + // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.21.0/pkg/metrics/server + // - https://book.kubebuilder.io/reference/metrics.html + metricsServerOptions := metricsserver.Options{ + BindAddress: metricsAddr, + SecureServing: secureMetrics, + TLSOpts: tlsOpts, + } + + if secureMetrics { + // FilterProvider is used to protect the metrics endpoint with authn/authz. + // These configurations ensure that only authorized users and service accounts + // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: + // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.21.0/pkg/metrics/filters#WithAuthenticationAndAuthorization + metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization + } + + // If the certificate is not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + // + // TODO(user): If you enable certManager, uncomment the following lines: + // - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates + // managed by cert-manager for the metrics server. + // - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification. + if len(metricsCertPath) > 0 { + setupLog.Info("Initializing metrics certificate watcher using provided certificates", + "metrics-cert-path", metricsCertPath, "metrics-cert-name", metricsCertName, "metrics-cert-key", metricsCertKey) + + var err error + metricsCertWatcher, err = certwatcher.New( + filepath.Join(metricsCertPath, metricsCertName), + filepath.Join(metricsCertPath, metricsCertKey), + ) + if err != nil { + setupLog.Error(err, "to initialize metrics certificate watcher", "error", err) + os.Exit(1) + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = metricsCertWatcher.GetCertificate + }) + } + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + Metrics: metricsServerOptions, + WebhookServer: webhookServer, + HealthProbeBindAddress: probeAddr, + LeaderElection: enableLeaderElection, + LeaderElectionID: "2fa1c467.opensandbox.io", + // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily + // when the Manager ends. This requires the binary to immediately end when the + // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly + // speeds up voluntary leader transitions as the new leader don't have to wait + // LeaseDuration time first. + // + // In the default scaffold provided, the program ends immediately after + // the manager stops, so would be fine to enable this option. However, + // if you are doing or is intended to do any operation such as perform cleanups + // after the manager stops then its usage might be unsafe. + // LeaderElectionReleaseOnCancel: true, + }) + if err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } + setupLog.Info("register field index") + if err := fieldindex.RegisterFieldIndexes(mgr.GetCache()); err != nil { + setupLog.Error(err, "failed to register field index") + os.Exit(1) + } + if err := (&controller.BatchSandboxReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("batchsandbox-controller"), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "BatchSandbox") + os.Exit(1) + } + if err := (&controller.PoolReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("pool-controller"), + Allocator: controller.NewDefaultAllocator(mgr.GetClient()), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Pool") + os.Exit(1) + } + // +kubebuilder:scaffold:builder + + if metricsCertWatcher != nil { + setupLog.Info("Adding metrics certificate watcher to manager") + if err := mgr.Add(metricsCertWatcher); err != nil { + setupLog.Error(err, "unable to add metrics certificate watcher to manager") + os.Exit(1) + } + } + + if webhookCertWatcher != nil { + setupLog.Info("Adding webhook certificate watcher to manager") + if err := mgr.Add(webhookCertWatcher); err != nil { + setupLog.Error(err, "unable to add webhook certificate watcher to manager") + os.Exit(1) + } + } + + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up health check") + os.Exit(1) + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up ready check") + os.Exit(1) + } + + setupLog.Info("starting manager") + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } +} diff --git a/kubernetes/cmd/task-executor/main.go b/kubernetes/cmd/task-executor/main.go new file mode 100644 index 00000000..add1ce8e --- /dev/null +++ b/kubernetes/cmd/task-executor/main.go @@ -0,0 +1,116 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/config" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/manager" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/runtime" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/server" + store "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/storage" + "k8s.io/klog/v2" +) + +func main() { + // Load configuration + cfg := config.NewConfig() + cfg.LoadFromEnv() + cfg.LoadFromFlags() + + klog.InfoS("task-executor starting", "dataDir", cfg.DataDir, "listenAddr", cfg.ListenAddr, "sidecarMode", cfg.EnableSidecarMode) + + // Initialize TaskStore + taskStore, err := store.NewFileStore(cfg.DataDir) + if err != nil { + klog.ErrorS(err, "failed to create task store") + os.Exit(1) + } + klog.InfoS("task store initialized", "dataDir", cfg.DataDir) + + // Initialize Executor + exec, err := runtime.NewExecutor(cfg) + if err != nil { + klog.ErrorS(err, "failed to create executor") + os.Exit(1) + } + mode := "process" + if cfg.EnableContainerMode { + mode = "container" + } + klog.InfoS("executor initialized", "mode", mode) + + // Initialize TaskManager + taskManager, err := manager.NewTaskManager(cfg, taskStore, exec) + if err != nil { + klog.ErrorS(err, "failed to create task manager") + os.Exit(1) + } + + // Start TaskManager + taskManager.Start(context.Background()) + klog.InfoS("task manager started") + + // Initialize HTTP Handler and Router + handler := server.NewHandler(taskManager, cfg) + router := server.NewRouter(handler) + + // Create HTTP Server + svr := &http.Server{ + Addr: cfg.ListenAddr, + Handler: router, + ReadTimeout: cfg.ReadTimeout, + WriteTimeout: cfg.WriteTimeout, + } + + // Start HTTP server in goroutine + go func() { + klog.InfoS("HTTP server listening", "address", cfg.ListenAddr) + if err := svr.ListenAndServe(); err != nil && err != http.ErrServerClosed { + klog.ErrorS(err, "HTTP server error") + os.Exit(1) + } + }() + + // Wait for interrupt signal + quit := make(chan os.Signal, 1) + signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) + <-quit + + klog.InfoS("shutting down task-executor gracefully...") + + // Shutdown context with timeout + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second) + defer shutdownCancel() + + // 1. Stop HTTP server first + if err := svr.Shutdown(shutdownCtx); err != nil { + klog.ErrorS(err, "HTTP server shutdown error") + } else { + klog.InfoS("HTTP server stopped") + } + + // 2. Stop TaskManager + taskManager.Stop() + klog.InfoS("task manager stopped") + + klog.InfoS("task-executor stopped successfully") +} diff --git a/kubernetes/config/crd/bases/sandbox.opensandbox.io_batchsandboxes.yaml b/kubernetes/config/crd/bases/sandbox.opensandbox.io_batchsandboxes.yaml new file mode 100644 index 00000000..675b7088 --- /dev/null +++ b/kubernetes/config/crd/bases/sandbox.opensandbox.io_batchsandboxes.yaml @@ -0,0 +1,184 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + name: batchsandboxes.sandbox.opensandbox.io +spec: + group: sandbox.opensandbox.io + names: + kind: BatchSandbox + listKind: BatchSandboxList + plural: batchsandboxes + shortNames: + - bsbx + singular: batchsandbox + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The desired number of pods. + jsonPath: .spec.replicas + name: DESIRED + type: integer + - description: The number of currently all pods. + jsonPath: .status.replicas + name: TOTAL + type: integer + - description: The number of currently all allocated pods. + jsonPath: .status.allocated + name: ALLOCATED + type: integer + - description: The number of currently all ready pods. + jsonPath: .status.ready + name: Ready + type: integer + - description: The number of currently all running tasks. + jsonPath: .status.taskRunning + name: TASK_RUNNING + priority: 1 + type: integer + - description: The number of currently all succeed tasks. + jsonPath: .status.taskSucceed + name: TASK_SUCCEED + priority: 1 + type: integer + - description: The number of currently all failed tasks. + jsonPath: .status.taskFailed + name: TASK_FAILED + priority: 1 + type: integer + - description: The number of currently all unknown tasks. + jsonPath: .status.taskUnknown + name: TASK_UNKNOWN + priority: 1 + type: integer + - description: sandbox expire time + jsonPath: .spec.expireTime + name: EXPIRE + type: string + - description: CreationTimestamp is a timestamp representing the server time when + this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. + jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: BatchSandbox is the Schema for the batchsandboxes API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: BatchSandboxSpec defines the desired state of BatchSandbox. + properties: + expireTime: + description: |- + ExpireTime - Absolute time when the batch-sandbox is deleted. + If a time in the past is provided, the sandbox will be deleted immediately. + format: date-time + type: string + poolRef: + description: PoolRef Pool name + type: string + replicas: + default: 1 + description: Replicas is the number of desired replicas. + format: int32 + minimum: 0 + type: integer + shardTaskPatches: + description: ShardTaskPatches indicates patching to the TaskTemplate + for individual Task. + x-kubernetes-preserve-unknown-fields: true + taskResourcePolicyWhenCompleted: + default: Retain + description: |- + TaskResourcePolicyWhenCompleted specifies how resources should be handled once a task reaches a completed state (SUCCEEDED or FAILED). + - Retain: Keep the resources until the BatchSandbox is deleted. + - Release: Free the resources immediately when the task completes. + type: string + taskTemplate: + description: |- + Task is a custom task spec that is automatically dispatched after the sandbox is successfully created. + The Sandbox is responsible for managing the lifecycle of the task. + x-kubernetes-preserve-unknown-fields: true + template: + description: Template describes the pods that will be created. + x-kubernetes-preserve-unknown-fields: true + required: + - replicas + type: object + status: + description: BatchSandboxStatus defines the observed state of BatchSandbox. + properties: + allocated: + description: "\tAllocated is the number of actual scheduled Pod" + format: int32 + type: integer + observedGeneration: + description: |- + ObservedGeneration is the most recent generation observed for this BatchSandbox. It corresponds to the + BatchSandbox's generation, which is updated on mutation by the API Server. + format: int64 + type: integer + ready: + description: "\tReady is the number of actual Ready Pod" + format: int32 + type: integer + replicas: + description: Replicas is the number of actual Pods + format: int32 + type: integer + taskFailed: + description: TaskFailed is the number of Failed task + format: int32 + type: integer + taskPending: + description: TaskPending is the number of Pending task which is unassigned + format: int32 + type: integer + taskRunning: + description: TaskRunning is the number of Running task + format: int32 + type: integer + taskSucceed: + description: TaskSucceed is the number of Succeed task + format: int32 + type: integer + taskUnknown: + description: TaskUnknown is the number of Unknown task + format: int32 + type: integer + required: + - allocated + - ready + - replicas + - taskFailed + - taskPending + - taskRunning + - taskSucceed + - taskUnknown + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/kubernetes/config/crd/bases/sandbox.opensandbox.io_pools.yaml b/kubernetes/config/crd/bases/sandbox.opensandbox.io_pools.yaml new file mode 100644 index 00000000..8b987cad --- /dev/null +++ b/kubernetes/config/crd/bases/sandbox.opensandbox.io_pools.yaml @@ -0,0 +1,129 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + name: pools.sandbox.opensandbox.io +spec: + group: sandbox.opensandbox.io + names: + kind: Pool + listKind: PoolList + plural: pools + singular: pool + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The number of all nodes in pool. + jsonPath: .status.total + name: TOTAL + type: integer + - description: The number of allocated nodes in pool. + jsonPath: .status.allocated + name: ALLOCATED + type: integer + - description: The number of available nodes in pool. + jsonPath: .status.available + name: AVAILABLE + type: integer + name: v1alpha1 + schema: + openAPIV3Schema: + description: Pool is the Schema for the pools API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: PoolSpec defines the desired state of Pool. + properties: + capacitySpec: + description: CapacitySpec controls the size of the resource pool. + properties: + bufferMax: + description: BufferMax is the maximum number of nodes kept in + the warm buffer. + format: int32 + minimum: 0 + type: integer + bufferMin: + description: BufferMin is the minimum number of nodes that must + remain in the buffer. + format: int32 + minimum: 0 + type: integer + poolMax: + description: PoolMax is the maximum total number of nodes allowed + in the entire pool. + format: int32 + minimum: 0 + type: integer + poolMin: + description: PoolMin is the minimum total size of the pool. + format: int32 + minimum: 0 + type: integer + required: + - bufferMax + - bufferMin + - poolMax + - poolMin + type: object + template: + description: Pod Template used to create pre-warmed nodes in the pool. + x-kubernetes-preserve-unknown-fields: true + required: + - capacitySpec + type: object + status: + description: PoolStatus defines the observed state of Pool. + properties: + allocated: + description: Allocated is the number of nodes currently allocated + to sandboxes. + format: int32 + type: integer + available: + description: Available is the number of nodes currently available + in the pool. + format: int32 + type: integer + observedGeneration: + description: |- + ObservedGeneration is the most recent generation observed for this BatchSandbox. It corresponds to the + BatchSandbox's generation, which is updated on mutation by the API Server. + format: int64 + type: integer + revision: + description: Revision is the latest version of pool + type: string + total: + description: Total is the total number of nodes in the pool. + format: int32 + type: integer + required: + - allocated + - available + - revision + - total + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/kubernetes/config/crd/kustomization.yaml b/kubernetes/config/crd/kustomization.yaml new file mode 100644 index 00000000..158214bc --- /dev/null +++ b/kubernetes/config/crd/kustomization.yaml @@ -0,0 +1,17 @@ +# This kustomization.yaml is not intended to be run by itself, +# since it depends on service name and namespace that are out of this kustomize package. +# It should be run by config/default +resources: +- bases/sandbox.opensandbox.io_batchsandboxes.yaml +- bases/sandbox.opensandbox.io_pools.yaml +# +kubebuilder:scaffold:crdkustomizeresource + +patches: +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. +# patches here are for enabling the conversion webhook for each CRD +# +kubebuilder:scaffold:crdkustomizewebhookpatch + +# [WEBHOOK] To enable webhook, uncomment the following section +# the following config is for teaching kustomize how to do kustomization for CRDs. +#configurations: +#- kustomizeconfig.yaml diff --git a/kubernetes/config/crd/kustomizeconfig.yaml b/kubernetes/config/crd/kustomizeconfig.yaml new file mode 100644 index 00000000..ec5c150a --- /dev/null +++ b/kubernetes/config/crd/kustomizeconfig.yaml @@ -0,0 +1,19 @@ +# This file is for teaching kustomize how to substitute name and namespace reference in CRD +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/name + +namespace: +- kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/namespace + create: false + +varReference: +- path: metadata/annotations diff --git a/kubernetes/config/default/cert_metrics_manager_patch.yaml b/kubernetes/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 00000000..d9750155 --- /dev/null +++ b/kubernetes/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,30 @@ +# This patch adds the args, volumes, and ports to allow the manager to use the metrics-server certs. + +# Add the volumeMount for the metrics-server certs +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true + +# Add the --metrics-cert-path argument for the metrics server +- op: add + path: /spec/template/spec/containers/0/args/- + value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs + +# Add the metrics-server certs volume configuration +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/kubernetes/config/default/kustomization.yaml b/kubernetes/config/default/kustomization.yaml new file mode 100644 index 00000000..eb84d905 --- /dev/null +++ b/kubernetes/config/default/kustomization.yaml @@ -0,0 +1,234 @@ +# Adds namespace to all resources. +namespace: sandbox-k8s-system + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +namePrefix: sandbox-k8s- + +# Labels to add to all resources and selectors. +#labels: +#- includeSelectors: true +# pairs: +# someName: someValue + +resources: +- ../crd +- ../rbac +- ../manager +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +#- ../webhook +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. +#- ../certmanager +# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. +#- ../prometheus +# [METRICS] Expose the controller manager metrics service. +- metrics_service.yaml +# [NETWORK POLICY] Protect the /metrics endpoint and Webhook Server with NetworkPolicy. +# Only Pod(s) running a namespace labeled with 'metrics: enabled' will be able to gather the metrics. +# Only CR(s) which requires webhooks and are applied on namespaces labeled with 'webhooks: enabled' will +# be able to communicate with the Webhook Server. +#- ../network-policy + +# Uncomment the patches line if you enable Metrics +patches: +# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. +# More info: https://book.kubebuilder.io/reference/metrics +- path: manager_metrics_patch.yaml + target: + kind: Deployment + +# Uncomment the patches line if you enable Metrics and CertManager +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment + +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +#- path: manager_webhook_patch.yaml +# target: +# kind: Deployment + +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. +# Uncomment the following replacements to add the cert-manager CA injection annotations +#replacements: +# - source: # Uncomment the following block to enable certificates for metrics +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - select: # Uncomment the following to set the Service name for TLS config in Prometheus ServiceMonitor +# kind: ServiceMonitor +# group: monitoring.coreos.com +# version: v1 +# name: controller-manager-metrics-monitor +# fieldPaths: +# - spec.endpoints.0.tlsConfig.serverName +# options: +# delimiter: '.' +# index: 0 +# create: true +# +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# - select: # Uncomment the following to set the Service namespace for TLS in Prometheus ServiceMonitor +# kind: ServiceMonitor +# group: monitoring.coreos.com +# version: v1 +# name: controller-manager-metrics-monitor +# fieldPaths: +# - spec.endpoints.0.tlsConfig.serverName +# options: +# delimiter: '.' +# index: 1 +# create: true +# +# - source: # Uncomment the following block if you have any webhook +# kind: Service +# version: v1 +# name: webhook-service +# fieldPath: .metadata.name # Name of the service +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPaths: +# - .spec.dnsNames.0 +# - .spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - source: +# kind: Service +# version: v1 +# name: webhook-service +# fieldPath: .metadata.namespace # Namespace of the service +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPaths: +# - .spec.dnsNames.0 +# - .spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# +# - source: # Uncomment the following block if you have a ValidatingWebhook (--programmatic-validation) +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # This name should match the one in certificate.yaml +# fieldPath: .metadata.namespace # Namespace of the certificate CR +# targets: +# - select: +# kind: ValidatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.name +# targets: +# - select: +# kind: ValidatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true +# +# - source: # Uncomment the following block if you have a DefaultingWebhook (--defaulting ) +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.namespace # Namespace of the certificate CR +# targets: +# - select: +# kind: MutatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.name +# targets: +# - select: +# kind: MutatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true +# +# - source: # Uncomment the following block if you have a ConversionWebhook (--conversion) +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.namespace # Namespace of the certificate CR +# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. +# +kubebuilder:scaffold:crdkustomizecainjectionns +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.name +# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. +# +kubebuilder:scaffold:crdkustomizecainjectionname diff --git a/kubernetes/config/default/manager_metrics_patch.yaml b/kubernetes/config/default/manager_metrics_patch.yaml new file mode 100644 index 00000000..2aaef653 --- /dev/null +++ b/kubernetes/config/default/manager_metrics_patch.yaml @@ -0,0 +1,4 @@ +# This patch adds the args to allow exposing the metrics endpoint using HTTPS +- op: add + path: /spec/template/spec/containers/0/args/0 + value: --metrics-bind-address=:8443 diff --git a/kubernetes/config/default/metrics_service.yaml b/kubernetes/config/default/metrics_service.yaml new file mode 100644 index 00000000..c65f4324 --- /dev/null +++ b/kubernetes/config/default/metrics_service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: sandbox-k8s + app.kubernetes.io/managed-by: kustomize + name: controller-manager-metrics-service + namespace: system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: 8443 + selector: + control-plane: controller-manager + app.kubernetes.io/name: sandbox-k8s diff --git a/kubernetes/config/manager/kustomization.yaml b/kubernetes/config/manager/kustomization.yaml new file mode 100644 index 00000000..3b71d99d --- /dev/null +++ b/kubernetes/config/manager/kustomization.yaml @@ -0,0 +1,8 @@ +resources: +- manager.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: example.com/sandbox-k8s + newTag: v0.0.1 diff --git a/kubernetes/config/manager/manager.yaml b/kubernetes/config/manager/manager.yaml new file mode 100644 index 00000000..4001a945 --- /dev/null +++ b/kubernetes/config/manager/manager.yaml @@ -0,0 +1,99 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: sandbox-k8s + app.kubernetes.io/managed-by: kustomize + name: system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system + labels: + control-plane: controller-manager + app.kubernetes.io/name: sandbox-k8s + app.kubernetes.io/managed-by: kustomize +spec: + selector: + matchLabels: + control-plane: controller-manager + app.kubernetes.io/name: sandbox-k8s + replicas: 1 + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + labels: + control-plane: controller-manager + app.kubernetes.io/name: sandbox-k8s + spec: + # TODO(user): Uncomment the following code to configure the nodeAffinity expression + # according to the platforms which are supported by your solution. + # It is considered best practice to support multiple architectures. You can + # build your manager image using the makefile target docker-buildx. + # affinity: + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/arch + # operator: In + # values: + # - amd64 + # - arm64 + # - ppc64le + # - s390x + # - key: kubernetes.io/os + # operator: In + # values: + # - linux + securityContext: + # Projects are configured by default to adhere to the "restricted" Pod Security Standards. + # This ensures that deployments meet the highest security requirements for Kubernetes. + # For more details, see: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - command: + - /workspace/server + args: + - --leader-elect + - --health-probe-bind-address=:8081 + - --v=3 + image: controller:latest + name: manager + ports: [] + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + # TODO(user): Configure the resources accordingly based on the project requirements. + # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 10m + memory: 64Mi + volumeMounts: [] + volumes: [] + serviceAccountName: controller-manager + terminationGracePeriodSeconds: 10 diff --git a/kubernetes/config/manifests/kustomization.yaml b/kubernetes/config/manifests/kustomization.yaml new file mode 100644 index 00000000..58e2ce25 --- /dev/null +++ b/kubernetes/config/manifests/kustomization.yaml @@ -0,0 +1,28 @@ +# These resources constitute the fully configured set of manifests +# used to generate the 'manifests/' directory in a bundle. +resources: +- bases/sandbox-k8s.clusterserviceversion.yaml +- ../default +- ../samples +- ../scorecard + +# [WEBHOOK] To enable webhooks, uncomment all the sections with [WEBHOOK] prefix. +# Do NOT uncomment sections with prefix [CERTMANAGER], as OLM does not support cert-manager. +# These patches remove the unnecessary "cert" volume and its manager container volumeMount. +#patches: +#- target: +# group: apps +# version: v1 +# kind: Deployment +# name: controller-manager +# namespace: system +# patch: |- +# # Remove the manager container's "cert" volumeMount, since OLM will create and mount a set of certs. +# # Update the indices in this path if adding or removing containers/volumeMounts in the manager's Deployment. +# - op: remove + +# path: /spec/template/spec/containers/0/volumeMounts/0 +# # Remove the "cert" volume, since OLM will create and mount a set of certs. +# # Update the indices in this path if adding or removing volumes in the manager's Deployment. +# - op: remove +# path: /spec/template/spec/volumes/0 diff --git a/kubernetes/config/network-policy/allow-metrics-traffic.yaml b/kubernetes/config/network-policy/allow-metrics-traffic.yaml new file mode 100644 index 00000000..150fc5d9 --- /dev/null +++ b/kubernetes/config/network-policy/allow-metrics-traffic.yaml @@ -0,0 +1,27 @@ +# This NetworkPolicy allows ingress traffic +# with Pods running on namespaces labeled with 'metrics: enabled'. Only Pods on those +# namespaces are able to gather data from the metrics endpoint. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + app.kubernetes.io/name: sandbox-k8s + app.kubernetes.io/managed-by: kustomize + name: allow-metrics-traffic + namespace: system +spec: + podSelector: + matchLabels: + control-plane: controller-manager + app.kubernetes.io/name: sandbox-k8s + policyTypes: + - Ingress + ingress: + # This allows ingress traffic from any namespace with the label metrics: enabled + - from: + - namespaceSelector: + matchLabels: + metrics: enabled # Only from namespaces with this label + ports: + - port: 8443 + protocol: TCP diff --git a/kubernetes/config/network-policy/kustomization.yaml b/kubernetes/config/network-policy/kustomization.yaml new file mode 100644 index 00000000..ec0fb5e5 --- /dev/null +++ b/kubernetes/config/network-policy/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- allow-metrics-traffic.yaml diff --git a/kubernetes/config/prometheus/kustomization.yaml b/kubernetes/config/prometheus/kustomization.yaml new file mode 100644 index 00000000..fdc5481b --- /dev/null +++ b/kubernetes/config/prometheus/kustomization.yaml @@ -0,0 +1,11 @@ +resources: +- monitor.yaml + +# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus +# to securely reference certificates created and managed by cert-manager. +# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml +# to mount the "metrics-server-cert" secret in the Manager Deployment. +#patches: +# - path: monitor_tls_patch.yaml +# target: +# kind: ServiceMonitor diff --git a/kubernetes/config/prometheus/monitor.yaml b/kubernetes/config/prometheus/monitor.yaml new file mode 100644 index 00000000..6e6ee304 --- /dev/null +++ b/kubernetes/config/prometheus/monitor.yaml @@ -0,0 +1,27 @@ +# Prometheus Monitor Service (Metrics) +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: sandbox-k8s + app.kubernetes.io/managed-by: kustomize + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - path: /metrics + port: https # Ensure this is the name of the port that exposes HTTPS metrics + scheme: https + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables + # certificate verification, exposing the system to potential man-in-the-middle attacks. + # For production environments, it is recommended to use cert-manager for automatic TLS certificate management. + # To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml, + # which securely references the certificate from the 'metrics-server-cert' secret. + insecureSkipVerify: true + selector: + matchLabels: + control-plane: controller-manager + app.kubernetes.io/name: sandbox-k8s diff --git a/kubernetes/config/prometheus/monitor_tls_patch.yaml b/kubernetes/config/prometheus/monitor_tls_patch.yaml new file mode 100644 index 00000000..5bf84ce0 --- /dev/null +++ b/kubernetes/config/prometheus/monitor_tls_patch.yaml @@ -0,0 +1,19 @@ +# Patch for Prometheus ServiceMonitor to enable secure TLS configuration +# using certificates managed by cert-manager +- op: replace + path: /spec/endpoints/0/tlsConfig + value: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + serverName: SERVICE_NAME.SERVICE_NAMESPACE.svc + insecureSkipVerify: false + ca: + secret: + name: metrics-server-cert + key: ca.crt + cert: + secret: + name: metrics-server-cert + key: tls.crt + keySecret: + name: metrics-server-cert + key: tls.key diff --git a/kubernetes/config/rbac/batchsandbox_admin_role.yaml b/kubernetes/config/rbac/batchsandbox_admin_role.yaml new file mode 100644 index 00000000..d7ea994d --- /dev/null +++ b/kubernetes/config/rbac/batchsandbox_admin_role.yaml @@ -0,0 +1,27 @@ +# This rule is not used by the project sandbox-k8s itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants full permissions ('*') over sandbox.opensandbox.io. +# This role is intended for users authorized to modify roles and bindings within the cluster, +# enabling them to delegate specific permissions to other users or groups as needed. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: sandbox-k8s + app.kubernetes.io/managed-by: kustomize + name: batchsandbox-admin-role +rules: +- apiGroups: + - sandbox.opensandbox.io + resources: + - batchsandboxes + verbs: + - '*' +- apiGroups: + - sandbox.opensandbox.io + resources: + - batchsandboxes/status + verbs: + - get diff --git a/kubernetes/config/rbac/batchsandbox_editor_role.yaml b/kubernetes/config/rbac/batchsandbox_editor_role.yaml new file mode 100644 index 00000000..40e7ef7f --- /dev/null +++ b/kubernetes/config/rbac/batchsandbox_editor_role.yaml @@ -0,0 +1,33 @@ +# This rule is not used by the project sandbox-k8s itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants permissions to create, update, and delete resources within the sandbox.opensandbox.io. +# This role is intended for users who need to manage these resources +# but should not control RBAC or manage permissions for others. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: sandbox-k8s + app.kubernetes.io/managed-by: kustomize + name: batchsandbox-editor-role +rules: +- apiGroups: + - sandbox.opensandbox.io + resources: + - batchsandboxes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - sandbox.opensandbox.io + resources: + - batchsandboxes/status + verbs: + - get diff --git a/kubernetes/config/rbac/batchsandbox_viewer_role.yaml b/kubernetes/config/rbac/batchsandbox_viewer_role.yaml new file mode 100644 index 00000000..24460902 --- /dev/null +++ b/kubernetes/config/rbac/batchsandbox_viewer_role.yaml @@ -0,0 +1,29 @@ +# This rule is not used by the project sandbox-k8s itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants read-only access to sandbox.opensandbox.io resources. +# This role is intended for users who need visibility into these resources +# without permissions to modify them. It is ideal for monitoring purposes and limited-access viewing. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: sandbox-k8s + app.kubernetes.io/managed-by: kustomize + name: batchsandbox-viewer-role +rules: +- apiGroups: + - sandbox.opensandbox.io + resources: + - batchsandboxes + verbs: + - get + - list + - watch +- apiGroups: + - sandbox.opensandbox.io + resources: + - batchsandboxes/status + verbs: + - get diff --git a/kubernetes/config/rbac/kustomization.yaml b/kubernetes/config/rbac/kustomization.yaml new file mode 100644 index 00000000..6b9c2410 --- /dev/null +++ b/kubernetes/config/rbac/kustomization.yaml @@ -0,0 +1,31 @@ +resources: +# All RBAC will be applied under this service account in +# the deployment namespace. You may comment out this resource +# if your manager will use a service account that exists at +# runtime. Be sure to update RoleBinding and ClusterRoleBinding +# subjects if changing service account names. +- service_account.yaml +- role.yaml +- role_binding.yaml +- leader_election_role.yaml +- leader_election_role_binding.yaml +# The following RBAC configurations are used to protect +# the metrics endpoint with authn/authz. These configurations +# ensure that only authorized users and service accounts +# can access the metrics endpoint. Comment the following +# permissions if you want to disable this protection. +# More info: https://book.kubebuilder.io/reference/metrics.html +- metrics_auth_role.yaml +- metrics_auth_role_binding.yaml +- metrics_reader_role.yaml +# For each CRD, "Admin", "Editor" and "Viewer" roles are scaffolded by +# default, aiding admins in cluster management. Those roles are +# not used by the sandbox-k8s itself. You can comment the following lines +# if you do not want those helpers be installed with your Project. +- pool_admin_role.yaml +- pool_editor_role.yaml +- pool_viewer_role.yaml +- batchsandbox_admin_role.yaml +- batchsandbox_editor_role.yaml +- batchsandbox_viewer_role.yaml + diff --git a/kubernetes/config/rbac/leader_election_role.yaml b/kubernetes/config/rbac/leader_election_role.yaml new file mode 100644 index 00000000..01d15198 --- /dev/null +++ b/kubernetes/config/rbac/leader_election_role.yaml @@ -0,0 +1,40 @@ +# permissions to do leader election. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/name: sandbox-k8s + app.kubernetes.io/managed-by: kustomize + name: leader-election-role +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch diff --git a/kubernetes/config/rbac/leader_election_role_binding.yaml b/kubernetes/config/rbac/leader_election_role_binding.yaml new file mode 100644 index 00000000..d0c2ad16 --- /dev/null +++ b/kubernetes/config/rbac/leader_election_role_binding.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/name: sandbox-k8s + app.kubernetes.io/managed-by: kustomize + name: leader-election-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: leader-election-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/kubernetes/config/rbac/metrics_auth_role.yaml b/kubernetes/config/rbac/metrics_auth_role.yaml new file mode 100644 index 00000000..32d2e4ec --- /dev/null +++ b/kubernetes/config/rbac/metrics_auth_role.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-auth-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/kubernetes/config/rbac/metrics_auth_role_binding.yaml b/kubernetes/config/rbac/metrics_auth_role_binding.yaml new file mode 100644 index 00000000..e775d67f --- /dev/null +++ b/kubernetes/config/rbac/metrics_auth_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-auth-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metrics-auth-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/kubernetes/config/rbac/metrics_reader_role.yaml b/kubernetes/config/rbac/metrics_reader_role.yaml new file mode 100644 index 00000000..51a75db4 --- /dev/null +++ b/kubernetes/config/rbac/metrics_reader_role.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-reader +rules: +- nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/kubernetes/config/rbac/pool_admin_role.yaml b/kubernetes/config/rbac/pool_admin_role.yaml new file mode 100644 index 00000000..e9dd6d01 --- /dev/null +++ b/kubernetes/config/rbac/pool_admin_role.yaml @@ -0,0 +1,27 @@ +# This rule is not used by the project sandbox-k8s itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants full permissions ('*') over sandbox.opensandbox.io. +# This role is intended for users authorized to modify roles and bindings within the cluster, +# enabling them to delegate specific permissions to other users or groups as needed. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: sandbox-k8s + app.kubernetes.io/managed-by: kustomize + name: pool-admin-role +rules: +- apiGroups: + - sandbox.opensandbox.io + resources: + - pools + verbs: + - '*' +- apiGroups: + - sandbox.opensandbox.io + resources: + - pools/status + verbs: + - get diff --git a/kubernetes/config/rbac/pool_editor_role.yaml b/kubernetes/config/rbac/pool_editor_role.yaml new file mode 100644 index 00000000..fbba1957 --- /dev/null +++ b/kubernetes/config/rbac/pool_editor_role.yaml @@ -0,0 +1,33 @@ +# This rule is not used by the project sandbox-k8s itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants permissions to create, update, and delete resources within the sandbox.opensandbox.io. +# This role is intended for users who need to manage these resources +# but should not control RBAC or manage permissions for others. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: sandbox-k8s + app.kubernetes.io/managed-by: kustomize + name: pool-editor-role +rules: +- apiGroups: + - sandbox.opensandbox.io + resources: + - pools + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - sandbox.opensandbox.io + resources: + - pools/status + verbs: + - get diff --git a/kubernetes/config/rbac/pool_viewer_role.yaml b/kubernetes/config/rbac/pool_viewer_role.yaml new file mode 100644 index 00000000..477f9a47 --- /dev/null +++ b/kubernetes/config/rbac/pool_viewer_role.yaml @@ -0,0 +1,29 @@ +# This rule is not used by the project sandbox-k8s itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants read-only access to sandbox.opensandbox.io resources. +# This role is intended for users who need visibility into these resources +# without permissions to modify them. It is ideal for monitoring purposes and limited-access viewing. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: sandbox-k8s + app.kubernetes.io/managed-by: kustomize + name: pool-viewer-role +rules: +- apiGroups: + - sandbox.opensandbox.io + resources: + - pools + verbs: + - get + - list + - watch +- apiGroups: + - sandbox.opensandbox.io + resources: + - pools/status + verbs: + - get diff --git a/kubernetes/config/rbac/role.yaml b/kubernetes/config/rbac/role.yaml new file mode 100644 index 00000000..87fb9602 --- /dev/null +++ b/kubernetes/config/rbac/role.yaml @@ -0,0 +1,56 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: manager-role +rules: +- apiGroups: + - "" + resources: + - events + - pods + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - pods/status + verbs: + - get + - patch + - update +- apiGroups: + - sandbox.opensandbox.io + resources: + - batchsandboxes + - pools + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - sandbox.opensandbox.io + resources: + - batchsandboxes/finalizers + - pools/finalizers + verbs: + - update +- apiGroups: + - sandbox.opensandbox.io + resources: + - batchsandboxes/status + - pools/status + verbs: + - get + - patch + - update diff --git a/kubernetes/config/rbac/role_binding.yaml b/kubernetes/config/rbac/role_binding.yaml new file mode 100644 index 00000000..29e9790e --- /dev/null +++ b/kubernetes/config/rbac/role_binding.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/name: sandbox-k8s + app.kubernetes.io/managed-by: kustomize + name: manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: manager-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/kubernetes/config/rbac/service_account.yaml b/kubernetes/config/rbac/service_account.yaml new file mode 100644 index 00000000..9e28dc41 --- /dev/null +++ b/kubernetes/config/rbac/service_account.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/name: sandbox-k8s + app.kubernetes.io/managed-by: kustomize + name: controller-manager + namespace: system diff --git a/kubernetes/config/samples/kustomization.yaml b/kubernetes/config/samples/kustomization.yaml new file mode 100644 index 00000000..d447cce4 --- /dev/null +++ b/kubernetes/config/samples/kustomization.yaml @@ -0,0 +1,6 @@ +## Append samples of your project ## +resources: +- sandbox_v1alpha1_sandbox.yaml +- sandbox_v1alpha1_batchsandbox.yaml +- sandbox_v1alpha1_pool.yaml +# +kubebuilder:scaffold:manifestskustomizesamples diff --git a/kubernetes/config/samples/sandbox_v1alpha1_batchsandbox-with-task.yaml b/kubernetes/config/samples/sandbox_v1alpha1_batchsandbox-with-task.yaml new file mode 100644 index 00000000..5a52cf59 --- /dev/null +++ b/kubernetes/config/samples/sandbox_v1alpha1_batchsandbox-with-task.yaml @@ -0,0 +1,47 @@ +apiVersion: sandbox.opensandbox.io/v1alpha1 +kind: BatchSandbox +metadata: + labels: + app.kubernetes.io/name: sandbox-k8s + app.kubernetes.io/managed-by: kustomize + name: batchsandbox-sample + namespace: sandbox-k8s +spec: + replicas: 2 + template: + metadata: + labels: + app: example + spec: + containers: + - name: main + image: registry.k8s.io/e2e-test-images/httpd:2.4.38-4 + command: + - tail + - -f + - /dev/null + expireTime: "2025-12-03T12:55:41Z" + taskTemplate: + metadata: + labels: + app.task: task + spec: + container: # container mode + image: example.com/agent-task:latest + command: + - /usr/bin/sleep + args: + - "1" + shardTaskPatches: # patch list for container tasks + - spec: + container: + command: # patch command and args, the final command is `python -m http.server 8080` + - python + args: + - -m + - http.server + - "8080" + - spec: + container: + args: # patch args only, the final command is `/usr/bin/sleep 456` + - "456" diff --git a/kubernetes/config/samples/sandbox_v1alpha1_batchsandbox.yaml b/kubernetes/config/samples/sandbox_v1alpha1_batchsandbox.yaml new file mode 100644 index 00000000..8d060412 --- /dev/null +++ b/kubernetes/config/samples/sandbox_v1alpha1_batchsandbox.yaml @@ -0,0 +1,23 @@ +apiVersion: sandbox.opensandbox.io/v1alpha1 +kind: BatchSandbox +metadata: + labels: + app.kubernetes.io/name: sandbox-k8s + app.kubernetes.io/managed-by: kustomize + name: batchsandbox-sample-11 + namespace: sandbox-k8s +spec: + replicas: 1 + poolRef: pool-sample + expireTime: "2026-12-03T12:55:41Z" + taskTemplate: + metadata: + labels: + app.task: task + name: task-sample + spec: + process: + command: + - sleep + args: + - "10" \ No newline at end of file diff --git a/kubernetes/config/samples/sandbox_v1alpha1_pool.yaml b/kubernetes/config/samples/sandbox_v1alpha1_pool.yaml new file mode 100644 index 00000000..1b30ef7c --- /dev/null +++ b/kubernetes/config/samples/sandbox_v1alpha1_pool.yaml @@ -0,0 +1,49 @@ +apiVersion: sandbox.opensandbox.io/v1alpha1 +kind: Pool +metadata: + labels: + app.kubernetes.io/name: sandbox-k8s + app.kubernetes.io/managed-by: kustomize + name: pool-sample + namespace: sandbox-k8s +spec: + template: + metadata: + labels: + app: example + spec: + shareProcessNamespace: true + volumes: + - name: sandbox-storage + emptyDir: { } + containers: + - name: main + image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/code-interpreter:latest + command: + - /opt/opensandbox/code-interpreter.sh + env: + - name: SANDBOX_MAIN_CONTAINER + value: main + volumeMounts: + - name: sandbox-storage + mountPath: /var/lib/sandbox + - command: + - /workspace/server + - -listen-addr=0.0.0.0:5758 + - -enable-sidecar-mode=true + image: reg.docker.alibaba-inc.com/sandbox-k8s/task-executor:1.0.16 + name: task-executor + securityContext: + capabilities: + add: + - SYS_PTRACE + volumeMounts: + - name: sandbox-storage + mountPath: /var/lib/sandbox + tolerations: + - operator: "Exists" + capacitySpec: + bufferMax: 3 + bufferMin: 1 + poolMax: 5 + poolMin: 0 diff --git a/kubernetes/config/samples/sandbox_v1alpha1_pooled_batchsandbox.yaml b/kubernetes/config/samples/sandbox_v1alpha1_pooled_batchsandbox.yaml new file mode 100644 index 00000000..e57927fd --- /dev/null +++ b/kubernetes/config/samples/sandbox_v1alpha1_pooled_batchsandbox.yaml @@ -0,0 +1,12 @@ +apiVersion: sandbox.opensandbox.io/v1alpha1 +kind: BatchSandbox +metadata: + labels: + app.kubernetes.io/name: sandbox-k8s + app.kubernetes.io/managed-by: kustomize + name: batchsandbox-pool-sample + namespace: sandbox-k8s +spec: + poolRef: pool-sample + replicas: 2 + expireTime: "2026-12-03T12:55:41Z" diff --git a/kubernetes/config/scorecard/bases/config.yaml b/kubernetes/config/scorecard/bases/config.yaml new file mode 100644 index 00000000..c7704784 --- /dev/null +++ b/kubernetes/config/scorecard/bases/config.yaml @@ -0,0 +1,7 @@ +apiVersion: scorecard.operatorframework.io/v1alpha3 +kind: Configuration +metadata: + name: config +stages: +- parallel: true + tests: [] diff --git a/kubernetes/config/scorecard/kustomization.yaml b/kubernetes/config/scorecard/kustomization.yaml new file mode 100644 index 00000000..54e8aa50 --- /dev/null +++ b/kubernetes/config/scorecard/kustomization.yaml @@ -0,0 +1,18 @@ +resources: +- bases/config.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +patches: +- path: patches/basic.config.yaml + target: + group: scorecard.operatorframework.io + kind: Configuration + name: config + version: v1alpha3 +- path: patches/olm.config.yaml + target: + group: scorecard.operatorframework.io + kind: Configuration + name: config + version: v1alpha3 +# +kubebuilder:scaffold:patches diff --git a/kubernetes/config/scorecard/patches/basic.config.yaml b/kubernetes/config/scorecard/patches/basic.config.yaml new file mode 100644 index 00000000..0b45f2fe --- /dev/null +++ b/kubernetes/config/scorecard/patches/basic.config.yaml @@ -0,0 +1,10 @@ +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - basic-check-spec + image: quay.io/operator-framework/scorecard-test:v1.42.0 + labels: + suite: basic + test: basic-check-spec-test diff --git a/kubernetes/config/scorecard/patches/olm.config.yaml b/kubernetes/config/scorecard/patches/olm.config.yaml new file mode 100644 index 00000000..8cc12589 --- /dev/null +++ b/kubernetes/config/scorecard/patches/olm.config.yaml @@ -0,0 +1,50 @@ +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-bundle-validation + image: quay.io/operator-framework/scorecard-test:v1.42.0 + labels: + suite: olm + test: olm-bundle-validation-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-crds-have-validation + image: quay.io/operator-framework/scorecard-test:v1.42.0 + labels: + suite: olm + test: olm-crds-have-validation-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-crds-have-resources + image: quay.io/operator-framework/scorecard-test:v1.42.0 + labels: + suite: olm + test: olm-crds-have-resources-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-spec-descriptors + image: quay.io/operator-framework/scorecard-test:v1.42.0 + labels: + suite: olm + test: olm-spec-descriptors-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-status-descriptors + image: quay.io/operator-framework/scorecard-test:v1.42.0 + labels: + suite: olm + test: olm-status-descriptors-test diff --git a/kubernetes/examples/task-executor/README.md b/kubernetes/examples/task-executor/README.md new file mode 100644 index 00000000..95430b04 --- /dev/null +++ b/kubernetes/examples/task-executor/README.md @@ -0,0 +1,395 @@ +# Task Executor Usage Guide + +## Introduction + +The `task-executor` is a lightweight component designed to run and manage short-lived tasks (processes or containers) within a Kubernetes Pod context. It acts as a local agent, receiving task specifications from a Kubernetes Controller (e.g., `BatchSandboxController`) and executing them on the node where it runs. It exposes a simple HTTP API for task creation, status inquiry, and management. + +## Running the Task Executor + +The `task-executor` can be started using the `cmd/task-executor/main.go` entry point. It supports various command-line flags and environment variables for configuration. + +**Basic Startup:** + +```bash +/path/to/cmd/task-executor/main --data-dir=/var/lib/sandbox/tasks --listen-addr=0.0.0.0:5758 +``` + +**Key Configuration Parameters:** + +| Flag / Environment Variable | Description | Default Value | +| :-------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :---------------------------- | +| `--data-dir` (DATA_DIR) | Directory for persisting task state and logs. | `/var/lib/sandbox/tasks` | +| `--listen-addr` (LISTEN_ADDR)| Address and port for the HTTP API server. | `0.0.0.0:5758` | +| `--enable-sidecar-mode` (ENABLE_SIDECAR_MODE) | If `true`, enables sidecar mode execution, where tasks are run within the PID namespace of a specified main container. Requires `nsenter` and appropriate privileges. | `false` | +| `--main-container-name` (MAIN_CONTAINER_NAME)| When `enable-sidecar-mode` is `true`, specifies the name of the main container whose PID namespace should be used. | `main` | +| `--enable-container-mode` (ENABLE_CONTAINER_MODE) | If `true`, enables container mode execution using the CRI runtime. (Note: Current implementation may be a placeholder). | `false` | +| `--cri-socket` (CRI_SOCKET) | Path to the CRI socket (e.g., `containerd.sock`) when `enable-container-mode` is `true`. | `/var/run/containerd/containerd.sock` | +| `--reconcile-interval` | The interval at which the internal task manager reconciles task states. | `500ms` | + +## HTTP API Endpoints + +The `task-executor` exposes a RESTful HTTP API. All API calls expect JSON request bodies (where applicable) and return JSON responses. + +### 1. `POST /tasks` - Create a new task + +Creates and starts a single task. + +* **Method:** `POST` +* **Path:** `/tasks` +* **Request Body (application/json):** An object representing the desired task. + + ```json + { + "name": "my-first-task", + "spec": { + "process": { + "command": ["sh", "-c"], + "args": ["echo 'Hello from my task!' && sleep 5 && echo 'Task finished.'"] + } + } + } + ``` + +* **Response Body (application/json):** The created task object with its initial status. + + ```json + { + "name": "my-first-task", + "spec": { + "process": { + "command": ["sh", "-c"], + "args": ["echo 'Hello from my task!' && sleep 5 && echo 'Task finished.'"] + } + }, + "status": { + "state": { + "waiting": { + "reason": "Initialized" + } + } + } + } + ``` + +**Example (using `curl`):** + +```bash +curl -X POST -H "Content-Type: application/json" -d '{ + "name": "my-first-task", + "spec": { + "process": { + "command": ["sh", "-c"], + "args": ["echo \"Hello from my task!\" && sleep 5 && echo \"Task finished.\""] + } + } +}' http://localhost:5758/tasks +``` + +### 2. `GET /tasks/{id}` - Get task status + +Retrieves the current status of a specific task by its name. + +* **Method:** `GET` +* **Path:** `/tasks/{taskName}` +* **Response Body (application/json):** The task object, including its current status. + + ```json + { + "name": "my-first-task", + "spec": { + "process": { + "command": ["sh", "-c"], + "args": ["echo 'Hello from my task!' && sleep 5 && echo 'Task finished.'"] + } + }, + "status": { + "state": { + "running": { + "startedAt": "2025-12-17T10:00:00Z" + } + } + } + } + ``` + +**Example (using `curl`):** + +```bash +curl http://localhost:5758/tasks/my-first-task +``` + +### 3. `DELETE /tasks/{id}` - Delete a task + +Marks a task for deletion. The `task-executor` will attempt to gracefully stop the task and then remove its state. + +* **Method:** `DELETE` +* **Path:** `/tasks/{taskName}` +* **Response:** `204 No Content` on successful marking for deletion. + +**Example (using `curl`):** + +```bash +curl -X DELETE http://localhost:5758/tasks/my-first-task +``` + +### 4. `POST /setTasks` - Synchronize tasks + +This endpoint is typically used by controllers to synchronize a desired set of tasks. Tasks not present in the desired list will be marked for deletion; new tasks will be created. + +* **Method:** `POST` +* **Path:** `/setTasks` +* **Request Body (application/json):** An array of task objects representing the desired state. + + ```json + [ + { + "name": "task-alpha", + "spec": { + "process": { + "command": ["sleep", "10"] + } + } + }, + { + "name": "task-beta", + "spec": { + "process": { + "command": ["ls", "-l", "/tmp"] + } + } + } + ] + ``` + +* **Response Body (application/json):** The current list of tasks managed by the executor after synchronization. + + ```json + [ + { + "name": "task-alpha", + "spec": { + "process": { + "command": ["sleep", "10"] + } + }, + "status": { + "state": { + "waiting": { + "reason": "Initialized" + } + } + } + }, + { + "name": "task-beta", + "spec": { + "process": { + "command": ["ls", "-l", "/tmp"] + } + }, + "status": { + "state": { + "waiting": { + "reason": "Initialized" + } + } + } + } + ] + ``` + +**Example (using `curl`):** + +```bash +curl -X POST -H "Content-Type: application/json" -d \ +'[ + { + "name": "task-alpha", + "spec": { "process": { "command": ["sleep", "10"] } } + }, + { + "name": "task-beta", + "spec": { "process": { "command": ["ls", "-l", "/tmp"] } } + } +]' http://localhost:5758/setTasks +``` + +### 5. `GET /getTasks` - List all tasks + +Retrieves a list of all tasks currently managed by the `task-executor`. + +* **Method:** `GET` +* **Path:** `/getTasks` +* **Response Body (application/json):** An array of task objects. + + ```json + [ + { + "name": "task-alpha", + "spec": { + "process": { + "command": ["sleep", "10"] + } + }, + "status": { + "state": { + "running": { + "startedAt": "2025-12-17T10:05:00Z" + } + } + } + }, + { + "name": "task-beta", + "spec": { + "process": { + "command": ["ls", "-l", "/tmp"] + } + }, + "status": { + "state": { + "terminated": { + "exitCode": 0, + "reason": "Succeeded", + "startedAt": "2025-12-17T10:06:00Z", + "finishedAt": "2025-12-17T10:06:01Z" + } + } + } + } + ] + ``` + +**Example (using `curl`):** + +```bash +curl http://localhost:5758/getTasks +``` + +### 6. `GET /health` - Health check + +Returns the health status of the `task-executor`. + +* **Method:** `GET` +* **Path:** `/health` +* **Response Body (application/json):** + + ```json + { + "status": "healthy" + } + ``` + +**Example (using `curl`):** + +```bash +curl http://localhost:5758/health +``` + +## Task Specification (`TaskSpec`) Structure + +The `spec` field within a task object (`api/v1alpha1.TaskSpec`) defines how the task should be executed. It currently supports `process` and `container` execution modes. + +### Process Task Example + +This mode executes a command directly as a process. + +```json +{ + "name": "my-process-task", + "spec": { + "process": { + "command": ["python3", "my_script.py"], + "args": ["--config", "/etc/app/config.yaml"], + "env": [ + { "name": "DEBUG_MODE", "value": "true" } + ], + "workingDir": "/app" + } + } +} +``` + +### Container Task Example (Placeholder/Future Feature) + +This mode is intended for executing tasks within containers managed by the CRI runtime. Note that as per `internal/task-executor/runtime/container.go`, this mode might still be a placeholder. + +```json +{ + "name": "my-container-task", + "spec": { + "container": { + "image": "ubuntu:latest", + "command": ["/bin/bash", "-c"], + "args": ["apt update && apt install -y curl"], + "env": [ + { "name": "http_proxy", "value": "http://myproxy.com:5758" } + ], + "volumeMounts": [ + { + "name": "data-volume", + "mountPath": "/data" + } + ] + } + } +} +``` + +## Task Status (`TaskStatus`) Structure + +The `status` field within a task object (`internal/task-executor/types/Status` mapped to `api/v1alpha1.TaskStatus` for external API) provides details about the task's current execution state. + +```json +{ + "name": "my-task", + "spec": { ... }, + "status": { + "state": { + "waiting": { + "reason": "Initialized" + } + }, + // or + "state": { + "running": { + "startedAt": "2025-12-17T10:00:00Z" + } + }, + // or + "state": { + "terminated": { + "exitCode": 0, + "reason": "Succeeded", + "message": "Task completed successfully", + "startedAt": "2025-12-17T10:00:00Z", + "finishedAt": "2025-12-17T10:00:05Z" + } + } + } +} +``` + +**State Types:** + +* `waiting`: Task is pending execution. +* `running`: Task is currently executing. +* `terminated`: Task has finished (succeeded or failed). + +## Example Scenario: Running a Sidecar Task + +If `task-executor` is configured with `--enable-sidecar-mode=true` and `--main-container-name=my-main-app`, it can execute tasks within the PID namespace of `my-main-app`. + +```bash +# Assume task-executor is running in sidecar mode on a pod with 'my-main-app' +# This task will execute 'ls /proc/self/ns' from within the main container's namespace +curl -X POST -H "Content-Type: application/json" -d '{ + "name": "sidecar-namespace-check", + "spec": { + "process": { + "command": ["ls", "/proc/self/ns"] + } + } +}' http://localhost:5758/tasks +``` + diff --git a/kubernetes/examples/task-executor/README_zh-CN.md b/kubernetes/examples/task-executor/README_zh-CN.md new file mode 100644 index 00000000..59d50e41 --- /dev/null +++ b/kubernetes/examples/task-executor/README_zh-CN.md @@ -0,0 +1,396 @@ +# Task Executor 使用指南 + +## 简介 + +`task-executor` 是一个轻量级组件，旨在 Kubernetes Pod 环境中运行和管理短期任务（进程或容器）。它充当本地代理，从 Kubernetes 控制器（例如 `BatchSandboxController`）接收任务规范，并在其运行的节点上执行这些任务。它暴露了一个简单的 HTTP API 用于任务创建、状态查询和管理。 + +## 运行 Task Executor + +可以使用 `cmd/task-executor/main.go` 入口点启动 `task-executor`。它支持各种命令行标志和环境变量进行配置。 + +**基本启动：** + +```bash +/path/to/cmd/task-executor/main --data-dir=/var/lib/sandbox/tasks --listen-addr=0.0.0.0:5758 +``` + +**关键配置参数：** + +| 标志 / 环境变量 | 描述 | 默认值 | +| :--- | :--- | :--- | +| `--data-dir` (DATA_DIR) | 用于持久化任务状态和日志的目录。 | `/var/lib/sandbox/tasks` | +| `--listen-addr` (LISTEN_ADDR) | HTTP API 服务器的地址和端口。 | `0.0.0.0:5758` | +| `--enable-sidecar-mode` (ENABLE_SIDECAR_MODE) | 如果为 `true`，则启用 sidecar 模式执行，任务将在指定主容器的 PID 命名空间内运行。需要 `nsenter` 和适当的权限。 | `false` | +| `--main-container-name` (MAIN_CONTAINER_NAME) | 当 `enable-sidecar-mode` 为 `true` 时，指定应使用其 PID 命名空间的主容器的名称。 | `main` | +| `--enable-container-mode` (ENABLE_CONTAINER_MODE) | 如果为 `true`，则启用使用 CRI 运行时的容器模式执行。（注意：当前实现可能只是占位符）。 | `false` | +| `--cri-socket` (CRI_SOCKET) | 当 `enable-container-mode` 为 `true` 时，CRI 套接字的路径（例如 `containerd.sock`）。 | `/var/run/containerd/containerd.sock` | +| `--reconcile-interval` | 内部任务管理器协调任务状态的间隔。 | `500ms` | + +## HTTP API 端点 + +`task-executor` 暴露了一个 RESTful HTTP API。所有 API 调用都期望 JSON 请求体（如适用）并返回 JSON 响应。 + +### 1. `POST /tasks` - 创建新任务 + +创建并启动单个任务。 + +* **方法：** `POST` +* **路径：** `/tasks` +* **请求体 (application/json)：** 代表所需任务的对象。 + + ```json + { + "name": "my-first-task", + "spec": { + "process": { + "command": ["sh", "-c"], + "args": ["echo 'Hello from my task!' && sleep 5 && echo 'Task finished.'"] + } + } + } + ``` + +* **响应体 (application/json)：** 创建的任务对象及其初始状态。 + + ```json + { + "name": "my-first-task", + "spec": { + "process": { + "command": ["sh", "-c"], + "args": ["echo 'Hello from my task!' && sleep 5 && echo 'Task finished.'"] + } + }, + "status": { + "state": { + "waiting": { + "reason": "Initialized" + } + } + } + } + ``` + +**示例 (使用 `curl`)：** + +```bash +curl -X POST -H "Content-Type: application/json" -d +'{ + "name": "my-first-task", + "spec": { + "process": { + "command": ["sh", "-c"], + "args": ["echo \"Hello from my task!\" && sleep 5 && echo \"Task finished.\""] + } + } +}' http://localhost:5758/tasks +``` + +### 2. `GET /tasks/{id}` - 获取任务状态 + +通过名称检索特定任务的当前状态。 + +* **方法：** `GET` +* **路径：** `/tasks/{taskName}` +* **响应体 (application/json)：** 任务对象，包括其当前状态。 + + ```json + { + "name": "my-first-task", + "spec": { + "process": { + "command": ["sh", "-c"], + "args": ["echo 'Hello from my task!' && sleep 5 && echo 'Task finished.'"] + } + }, + "status": { + "state": { + "running": { + "startedAt": "2025-12-17T10:00:00Z" + } + } + } + } + ``` + +**示例 (使用 `curl`)：** + +```bash +curl http://localhost:5758/tasks/my-first-task +``` + +### 3. `DELETE /tasks/{id}` - 删除任务 + +标记要删除的任务。`task-executor` 将尝试优雅地停止任务，然后删除其状态。 + +* **方法：** `DELETE` +* **路径：** `/tasks/{taskName}` +* **响应：** 成功标记删除时返回 `204 No Content`。 + +**示例 (使用 `curl`)：** + +```bash +curl -X DELETE http://localhost:5758/tasks/my-first-task +``` + +### 4. `POST /setTasks` - 同步任务 + +此端点通常由控制器用于同步所需的任务集。不在所需列表中的任务将被标记为删除；新任务将被创建。 + +* **方法：** `POST` +* **路径：** `/setTasks` +* **请求体 (application/json)：** 代表所需状态的任务对象数组。 + + ```json + [ + { + "name": "task-alpha", + "spec": { + "process": { + "command": ["sleep", "10"] + } + } + }, + { + "name": "task-beta", + "spec": { + "process": { + "command": ["ls", "-l", "/tmp"] + } + } + } + ] + ``` + +* **响应体 (application/json)：** 同步后执行器管理的当前任务列表。 + + ```json + [ + { + "name": "task-alpha", + "spec": { + "process": { + "command": ["sleep", "10"] + } + }, + "status": { + "state": { + "waiting": { + "reason": "Initialized" + } + } + } + }, + { + "name": "task-beta", + "spec": { + "process": { + "command": ["ls", "-l", "/tmp"] + } + }, + "status": { + "state": { + "waiting": { + "reason": "Initialized" + } + } + } + } + ] + ``` + +**示例 (使用 `curl`)：** + +```bash +curl -X POST -H "Content-Type: application/json" -d \ +'[ + { + "name": "task-alpha", + "spec": { "process": { "command": ["sleep", "10"] } } + }, + { + "name": "task-beta", + "spec": { "process": { "command": ["ls", "-l", "/tmp"] } } + } +]' http://localhost:5758/setTasks +``` + +### 5. `GET /getTasks` - 列出所有任务 + +检索 `task-executor` 当前管理的所有任务的列表。 + +* **方法：** `GET` +* **路径：** `/getTasks` +* **响应体 (application/json)：** 任务对象数组。 + + ```json + [ + { + "name": "task-alpha", + "spec": { + "process": { + "command": ["sleep", "10"] + } + }, + "status": { + "state": { + "running": { + "startedAt": "2025-12-17T10:05:00Z" + } + } + } + }, + { + "name": "task-beta", + "spec": { + "process": { + "command": ["ls", "-l", "/tmp"] + } + }, + "status": { + "state": { + "terminated": { + "exitCode": 0, + "reason": "Succeeded", + "startedAt": "2025-12-17T10:06:00Z", + "finishedAt": "2025-12-17T10:06:01Z" + } + } + } + } + ] + ``` + +**示例 (使用 `curl`)：** + +```bash +curl http://localhost:5758/getTasks +``` + +### 6. `GET /health` - 健康检查 + +返回 `task-executor` 的健康状态。 + +* **方法：** `GET` +* **路径：** `/health` +* **响应体 (application/json)：** + + ```json + { + "status": "healthy" + } + ``` + +**示例 (使用 `curl`)：** + +```bash +curl http://localhost:5758/health +``` + +## 任务规范 (`TaskSpec`) 结构 + +任务对象中的 `spec` 字段 (`api/v1alpha1.TaskSpec`) 定义了应如何执行任务。它目前支持 `process` 和 `container` 执行模式。 + +### 进程任务示例 + +此模式直接作为进程执行命令。 + +```json +{ + "name": "my-process-task", + "spec": { + "process": { + "command": ["python3", "my_script.py"], + "args": ["--config", "/etc/app/config.yaml"], + "env": [ + { "name": "DEBUG_MODE", "value": "true" } + ], + "workingDir": "/app" + } + } +} +``` + +### 容器任务示例（占位符/未来特性） + +此模式旨在执行由 CRI 运行时管理的容器中的任务。请注意，根据 `internal/task-executor/runtime/container.go`，此模式可能仍是一个占位符。 + +```json +{ + "name": "my-container-task", + "spec": { + "container": { + "image": "ubuntu:latest", + "command": ["/bin/bash", "-c"], + "args": ["apt update && apt install -y curl"], + "env": [ + { "name": "http_proxy", "value": "http://myproxy.com:5758" } + ], + "volumeMounts": [ + { + "name": "data-volume", + "mountPath": "/data" + } + ] + } + } +} +``` + +## 任务状态 (`TaskStatus`) 结构 + +任务对象中的 `status` 字段 (`internal/task-executor/types/Status` 映射到 `api/v1alpha1.TaskStatus` 用于外部 API) 提供了有关任务当前执行状态的详细信息。 + +```json +{ + "name": "my-task", + "spec": { ... }, + "status": { + "state": { + "waiting": { + "reason": "Initialized" + } + }, + // 或者 + "state": { + "running": { + "startedAt": "2025-12-17T10:00:00Z" + } + }, + // 或者 + "state": { + "terminated": { + "exitCode": 0, + "reason": "Succeeded", + "message": "Task completed successfully", + "startedAt": "2025-12-17T10:00:00Z", + "finishedAt": "2025-12-17T10:00:05Z" + } + } + } +} +``` + +**状态类型：** + +* `waiting`：任务正在等待执行。 +* `running`：任务当前正在执行。 +* `terminated`：任务已完成（成功或失败）。 + +## 示例场景：运行 Sidecar 任务 + +如果 `task-executor` 配置了 `--enable-sidecar-mode=true` 和 `--main-container-name=my-main-app`，它可以在 `my-main-app` 的 PID 命名空间内执行任务。 + +```bash +# 假设 task-executor 在 sidecar 模式下运行在一个包含 'my-main-app' 的 pod 上 +# 此任务将从主容器的命名空间内执行 'ls /proc/self/ns' +curl -X POST -H "Content-Type: application/json" -d +'{ + "name": "sidecar-namespace-check", + "spec": { + "process": { + "command": ["ls", "/proc/self/ns"] + } + } +}' http://localhost:5758/tasks +``` diff --git a/kubernetes/examples/task-executor/main.go b/kubernetes/examples/task-executor/main.go new file mode 100644 index 00000000..93ce40fc --- /dev/null +++ b/kubernetes/examples/task-executor/main.go @@ -0,0 +1,102 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "log" + "time" + + "github.com/alibaba/OpenSandbox/sandbox-k8s/api/v1alpha1" + taskexecutor "github.com/alibaba/OpenSandbox/sandbox-k8s/pkg/task-executor" +) + +func main() { + baseURL := "http://localhost:5758" + client := taskexecutor.NewClient(baseURL) + ctx := context.Background() + + fmt.Printf("Connecting to Task Executor at %s...\n", baseURL) + + taskName := "example-task" + newTask := &taskexecutor.Task{ + Name: taskName, + Spec: v1alpha1.TaskSpec{ + Process: &v1alpha1.ProcessTask{ + Command: []string{"sh", "-c"}, + Args: []string{"echo 'Hello from SDK example!' && sleep 2 && echo 'Task done.'"}, + }, + }, + } + + fmt.Printf("Submitting task '%s'...\n", taskName) + createdTask, err := client.Set(ctx, newTask) + if err != nil { + log.Fatalf("Failed to set task: %v", err) + } + fmt.Printf("Task submitted successfully. Initial state: %v\n", getTaskState(createdTask)) + + fmt.Println("Polling task status...") + for i := 0; i < 10; i++ { + currentTask, err := client.Get(ctx) + if err != nil { + log.Printf("Error getting task: %v", err) + continue + } + + if currentTask == nil { + fmt.Println("No task found.") + break + } + + state := getTaskState(currentTask) + fmt.Printf("Current state: %s\n", state) + + // Check if task is finished + if currentTask.Status.State.Terminated != nil { + fmt.Printf("Task finished with exit code: %d\n", currentTask.Status.State.Terminated.ExitCode) + break + } + + time.Sleep(500 * time.Millisecond) + } + + // Clean up (pass nil to clear tasks) + fmt.Println("Cleaning up...") + _, err = client.Set(ctx, nil) + if err != nil { + log.Printf("Failed to clear tasks: %v", err) + } else { + fmt.Println("Tasks cleared.") + } +} + +// getTaskState returns a string representation of the task state +func getTaskState(task *taskexecutor.Task) string { + if task == nil { + return "Unknown" + } + if task.Status.State.Running != nil { + return "Running" + } + if task.Status.State.Terminated != nil { + return "Terminated" + } + if task.Status.State.Waiting != nil { + return fmt.Sprintf("Waiting (%s)", task.Status.State.Waiting.Reason) + } + return "Pending" +} diff --git a/kubernetes/go.mod b/kubernetes/go.mod new file mode 100644 index 00000000..4b20f831 --- /dev/null +++ b/kubernetes/go.mod @@ -0,0 +1,101 @@ +module github.com/alibaba/OpenSandbox/sandbox-k8s + +go 1.24.0 + +require ( + github.com/golang/mock v1.6.0 + github.com/onsi/ginkgo/v2 v2.22.0 + github.com/onsi/gomega v1.36.1 + github.com/stretchr/testify v1.10.0 + k8s.io/api v0.33.0 + k8s.io/apimachinery v0.33.0 + k8s.io/client-go v0.33.0 + k8s.io/klog/v2 v2.130.1 + k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 + sigs.k8s.io/controller-runtime v0.21.0 +) + +require ( + cel.dev/expr v0.19.1 // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/evanphx/json-patch v4.12.0+incompatible // indirect + github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-logr/zapr v1.3.0 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/btree v1.1.3 // indirect + github.com/google/cel-go v0.23.2 // indirect + github.com/google/gnostic-models v0.6.9 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.22.0 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/spf13/cobra v1.8.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/stoewer/go-strcase v1.3.0 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect + go.opentelemetry.io/otel v1.33.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 // indirect + go.opentelemetry.io/otel/metric v1.33.0 // indirect + go.opentelemetry.io/otel/sdk v1.33.0 // indirect + go.opentelemetry.io/otel/trace v1.33.0 // indirect + go.opentelemetry.io/proto/otlp v1.4.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/oauth2 v0.27.0 // indirect + golang.org/x/sync v0.12.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/term v0.30.0 // indirect + golang.org/x/text v0.23.0 // indirect + golang.org/x/time v0.9.0 // indirect + golang.org/x/tools v0.26.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect + google.golang.org/grpc v1.68.1 // indirect + google.golang.org/protobuf v1.36.5 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.33.0 // indirect + k8s.io/apiserver v0.33.0 // indirect + k8s.io/component-base v0.33.0 // indirect + k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/kubernetes/go.sum b/kubernetes/go.sum new file mode 100644 index 00000000..b1d7d9ae --- /dev/null +++ b/kubernetes/go.sum @@ -0,0 +1,265 @@ +cel.dev/expr v0.19.1 h1:NciYrtDRIR0lNCnH1LFJegdjspNx9fI59O7TWcua/W4= +cel.dev/expr v0.19.1/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/cel-go v0.23.2 h1:UdEe3CvQh3Nv+E/j9r1Y//WO0K0cSyD7/y0bzyLIMI4= +github.com/google/cel-go v0.23.2/go.mod h1:52Pb6QsDbC5kvgxvZhiL9QX1oZEkcUF/ZqaPx1J5Wwo= +github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= +github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= +github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= +github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= +go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= +go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 h1:5pojmb1U1AogINhN3SurB+zm/nIcusopeBNp42f45QM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0/go.mod h1:57gTHJSE5S1tqg+EKsLPlTWhpHMsWlVmer+LA926XiA= +go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ= +go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M= +go.opentelemetry.io/otel/sdk v1.33.0 h1:iax7M131HuAm9QkZotNHEfstof92xM+N8sr3uHXc2IM= +go.opentelemetry.io/otel/sdk v1.33.0/go.mod h1:A1Q5oi7/9XaMlIWzPSxLRWOI8nG3FnzHJNbiENQuihM= +go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s= +go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck= +go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg= +go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 h1:CkkIfIt50+lT6NHAVoRYEyAvQGFM7xEwXUUywFvEb3Q= +google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 h1:8ZmaLZE4XWrtU3MyClkYqqtl6Oegr3235h7jxsDyqCY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= +google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= +google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.33.0 h1:yTgZVn1XEe6opVpP1FylmNrIFWuDqe2H0V8CT5gxfIU= +k8s.io/api v0.33.0/go.mod h1:CTO61ECK/KU7haa3qq8sarQ0biLq2ju405IZAd9zsiM= +k8s.io/apiextensions-apiserver v0.33.0 h1:d2qpYL7Mngbsc1taA4IjJPRJ9ilnsXIrndH+r9IimOs= +k8s.io/apiextensions-apiserver v0.33.0/go.mod h1:VeJ8u9dEEN+tbETo+lFkwaaZPg6uFKLGj5vyNEwwSzc= +k8s.io/apimachinery v0.33.0 h1:1a6kHrJxb2hs4t8EE5wuR/WxKDwGN1FKH3JvDtA0CIQ= +k8s.io/apimachinery v0.33.0/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= +k8s.io/apiserver v0.33.0 h1:QqcM6c+qEEjkOODHppFXRiw/cE2zP85704YrQ9YaBbc= +k8s.io/apiserver v0.33.0/go.mod h1:EixYOit0YTxt8zrO2kBU7ixAtxFce9gKGq367nFmqI8= +k8s.io/client-go v0.33.0 h1:UASR0sAYVUzs2kYuKn/ZakZlcs2bEHaizrrHUZg0G98= +k8s.io/client-go v0.33.0/go.mod h1:kGkd+l/gNGg8GYWAPr0xF1rRKvVWvzh9vmZAMXtaKOg= +k8s.io/component-base v0.33.0 h1:Ot4PyJI+0JAD9covDhwLp9UNkUja209OzsJ4FzScBNk= +k8s.io/component-base v0.33.0/go.mod h1:aXYZLbw3kihdkOPMDhWbjGCO6sg+luw554KP51t8qCU= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= +sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8= +sigs.k8s.io/controller-runtime v0.21.0/go.mod h1:OSg14+F65eWqIu4DceX7k/+QRAbTTvxeQSNSOQpukWM= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= +sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/kubernetes/hack/boilerplate.go.txt b/kubernetes/hack/boilerplate.go.txt new file mode 100644 index 00000000..221dcbe0 --- /dev/null +++ b/kubernetes/hack/boilerplate.go.txt @@ -0,0 +1,15 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ \ No newline at end of file diff --git a/kubernetes/hack/debug-task.sh b/kubernetes/hack/debug-task.sh new file mode 100755 index 00000000..2dbe0d0b --- /dev/null +++ b/kubernetes/hack/debug-task.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +# Copyright 2025 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +echo "Stopping any running debug containers..." +docker stop task-executor-debug > /dev/null || true +echo "Building debug docker image (dev environment)..." +docker build -t task-executor-debug -f Dockerfile.debug . + +echo "Starting debug container with Auto-Sync and Hot-Reload..." +echo "---------------------------------------------------------" +echo " App URL: http://localhost:8080" +echo " Debugger: localhost:2345" +echo " Source Code: Mounted from $(pwd)" +echo "---------------------------------------------------------" +echo "Usage:" +echo " 1. Connect GoLand to localhost:2345" +echo " 2. Edit code locally -> Container auto-recompiles (watch the logs)" +echo " 3. Re-connect Debugger in GoLand" +echo "---------------------------------------------------------" + +# Create docker volumes for cache if they don't exist +docker volume create sandbox-k8s-gomod > /dev/null +docker volume create sandbox-k8s-gocache > /dev/null + +# Run the container +# --rm: remove container after exit +# -v $(pwd):/workspace: Mount local code +# -v ...: Mount caches for speed +# reflex command: +# -r '\.go$': Watch all .go files recursively +# -s: Service mode (kill old process before starting new one) +# --: Delimiter +# dlv debug: Compile and run ./cmd/task +# --headless: No terminal UI +# --listen=:2345: Debugger port +# --api-version=2: API v2 +# --accept-multiclient: Allow multiple connections +# --continue: Start running immediately (Optional, remove if you want to hit 'Resume' first) +# --output /tmp/debug_bin: Put binary in tmp to avoid clutter/loops + +docker run --rm -it \ + --privileged \ + -p 5758:5758 \ + -p 2345:2345 \ + --security-opt seccomp=unconfined \ + --cap-add=SYS_PTRACE \ + -v "$(pwd):/workspace" \ + -v sandbox-k8s-gomod:/go/pkg/mod \ + -v sandbox-k8s-gocache:/go/.cache/go-build \ + --name task-executor-debug \ + -e SANDBOX_MAIN_CONTAINER=task-executor \ + task-executor-debug \ + reflex -r '\.go$' -s -- \ + dlv debug ./cmd/task-executor \ + --headless \ + --listen=:2345 \ + --api-version=2 \ + --accept-multiclient \ + --output /tmp/debug_bin \ + -- \ + -enable-sidecar-mode=true -main-container-name=task-executor \ No newline at end of file diff --git a/kubernetes/images/process_executor.png b/kubernetes/images/process_executor.png new file mode 100644 index 00000000..cf4d8624 Binary files /dev/null and b/kubernetes/images/process_executor.png differ diff --git a/kubernetes/internal/controller/allocator.go b/kubernetes/internal/controller/allocator.go new file mode 100644 index 00000000..d4861987 --- /dev/null +++ b/kubernetes/internal/controller/allocator.go @@ -0,0 +1,424 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "context" + "encoding/json" + gerrors "errors" + "fmt" + "slices" + "strconv" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + sandboxv1alpha1 "github.com/alibaba/OpenSandbox/sandbox-k8s/api/v1alpha1" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/utils/expectations" +) + +var ( + poolResExpectations = expectations.NewResourceVersionExpectation() +) + +type AllocationStore interface { + GetAllocation(ctx context.Context, pool *sandboxv1alpha1.Pool) (*PoolAllocation, error) + SetAllocation(ctx context.Context, pool *sandboxv1alpha1.Pool, allocation *PoolAllocation) error +} + +type annoAllocationStore struct { + client client.Client +} + +func NewAnnoAllocationStore(client client.Client) AllocationStore { + return &annoAllocationStore{ + client: client, + } +} + +func (store *annoAllocationStore) GetAllocation(ctx context.Context, pool *sandboxv1alpha1.Pool) (*PoolAllocation, error) { + alloc := &PoolAllocation{ + PodAllocation: make(map[string]string), + } + poolResExpectations.Observe(pool) + anno := pool.GetAnnotations() + if anno == nil { + return alloc, nil + } + js, ok := anno[AnnoPoolAllocStatusKey] + if !ok { + return alloc, nil + } + err := json.Unmarshal([]byte(js), alloc) + if err != nil { + return nil, err + } + return alloc, nil +} + +func (store *annoAllocationStore) SetAllocation(ctx context.Context, pool *sandboxv1alpha1.Pool, alloc *PoolAllocation) error { + if satisfied, unsatisfiedDuration := poolResExpectations.IsSatisfied(pool); !satisfied { + return fmt.Errorf("pool allocation is not ready, unsatisfiedDuration:%v", unsatisfiedDuration) + } + js, err := json.Marshal(alloc) + if err != nil { + return err + } + old := pool.DeepCopy() + oldGen := int64(0) + anno := pool.GetAnnotations() + if anno == nil { + anno = map[string]string{} + } + str, ok := anno[AnnoPoolAllocGenerationKey] + if ok { + oldGen, err = strconv.ParseInt(str, 10, 64) + if err != nil { + return err + } + } + gen := strconv.FormatInt(oldGen+1, 10) + anno[AnnoPoolAllocStatusKey] = string(js) + anno[AnnoPoolAllocGenerationKey] = gen + pool.SetAnnotations(anno) + patch := client.MergeFrom(old) + if err := store.client.Patch(ctx, pool, patch); err != nil { + return err + } + poolResExpectations.Expect(pool) + return nil +} + +type AllocationSyncer interface { + SetAllocation(ctx context.Context, sandbox *sandboxv1alpha1.BatchSandbox, allocation *SandboxAllocation) error + GetAllocation(ctx context.Context, sandbox *sandboxv1alpha1.BatchSandbox) (*SandboxAllocation, error) + GetRelease(ctx context.Context, sandbox *sandboxv1alpha1.BatchSandbox) (*AllocationRelease, error) +} +type annoAllocationSyncer struct { + client client.Client +} + +func NewAnnoAllocationSyncer(client client.Client) AllocationSyncer { + return &annoAllocationSyncer{ + client: client, + } +} + +func (syncer *annoAllocationSyncer) SetAllocation(ctx context.Context, sandbox *sandboxv1alpha1.BatchSandbox, allocation *SandboxAllocation) error { + old, ok := sandbox.DeepCopyObject().(*sandboxv1alpha1.BatchSandbox) + if !ok { + return fmt.Errorf("invalid object") + } + anno := sandbox.GetAnnotations() + if anno == nil { + anno = make(map[string]string) + } + js, err := json.Marshal(allocation) + if err != nil { + return err + } + anno[AnnoAllocStatusKey] = string(js) + sandbox.SetAnnotations(anno) + patch := client.MergeFrom(old) + return syncer.client.Patch(ctx, sandbox, patch) +} + +func (syncer *annoAllocationSyncer) GetAllocation(ctx context.Context, sandbox *sandboxv1alpha1.BatchSandbox) (*SandboxAllocation, error) { + allocation := &SandboxAllocation{ + Pods: make([]string, 0), + } + anno := sandbox.GetAnnotations() + if anno == nil { + return allocation, nil + } + if raw := anno[AnnoAllocStatusKey]; raw != "" { + err := json.Unmarshal([]byte(raw), allocation) + if err != nil { + return nil, err + } + } + return allocation, nil +} + +func (syncer *annoAllocationSyncer) GetRelease(ctx context.Context, sandbox *sandboxv1alpha1.BatchSandbox) (*AllocationRelease, error) { + release := &AllocationRelease{ + Pods: make([]string, 0), + } + anno := sandbox.GetAnnotations() + if anno == nil { + return release, nil + } + if raw := anno[AnnoAllocReleaseKey]; raw != "" { + err := json.Unmarshal([]byte(raw), release) + if err != nil { + return nil, err + } + } + return release, nil +} + +type AllocSpec struct { + // sandboxes need to allocate + Sandboxes []*sandboxv1alpha1.BatchSandbox + // pool + Pool *sandboxv1alpha1.Pool + // all pods of pool + Pods []*corev1.Pod +} + +type AllocStatus struct { + // pod allocated to sandbox + PodAllocation map[string]string + // pod request count + PodSupplement int32 +} + +type Allocator interface { + Schedule(ctx context.Context, spec *AllocSpec) (*AllocStatus, error) +} + +type defaultAllocator struct { + store AllocationStore + syncer AllocationSyncer +} + +func NewDefaultAllocator(client client.Client) Allocator { + return &defaultAllocator{ + store: NewAnnoAllocationStore(client), + syncer: NewAnnoAllocationSyncer(client), + } +} + +func (allocator *defaultAllocator) Schedule(ctx context.Context, spec *AllocSpec) (*AllocStatus, error) { + log := logf.FromContext(ctx) + pool := spec.Pool + status, err := allocator.initAllocation(ctx, spec) + if err != nil { + return nil, err + } + availablePods := make([]string, 0) + for _, pod := range spec.Pods { + if _, ok := status.PodAllocation[pod.Name]; ok { // allocated + continue + } + if pod.Status.Phase != corev1.PodRunning { // not running + continue + } + availablePods = append(availablePods, pod.Name) + } + sandboxToPods := make(map[string][]string) + for podName, sandboxName := range status.PodAllocation { + sandboxToPods[sandboxName] = append(sandboxToPods[sandboxName], podName) + } + sandboxAlloc, dirtySandboxes, poolAllocate, err := allocator.allocate(ctx, status, sandboxToPods, availablePods, spec.Sandboxes, spec.Pods) + if err != nil { + log.Error(err, "allocate failed") + } + poolDeallocate, err := allocator.deallocate(ctx, status, sandboxToPods, spec.Sandboxes) + if err != nil { + log.Error(err, "deallocate failed") + } + if poolDeallocate || poolAllocate { + if err := allocator.updateAllocStatus(ctx, status, pool); err != nil { + log.Error(err, "update alloc status failed") + return nil, err // Do not push the allocation to the sandbox and batch sandbox if allocation persist failed. + } + } + if err := allocator.syncAllocResult(ctx, dirtySandboxes, sandboxAlloc, spec.Sandboxes); err != nil { + log.Error(err, "sync alloc result failed") + } + return status, nil // Do not return the error of sandboxes witch will block pool schedule. +} + +func (allocator *defaultAllocator) initAllocation(ctx context.Context, spec *AllocSpec) (*AllocStatus, error) { + var err error + status := &AllocStatus{ + PodAllocation: make(map[string]string), + } + status.PodAllocation, err = allocator.getPodAllocation(ctx, spec.Pool) + if err != nil { + return nil, err + } + return status, nil +} + +func (allocator *defaultAllocator) allocate(ctx context.Context, status *AllocStatus, sandboxToPods map[string][]string, availablePods []string, sandboxes []*sandboxv1alpha1.BatchSandbox, pods []*corev1.Pod) (map[string][]string, []string, bool, error) { + errs := make([]error, 0) + sandboxAlloc := make(map[string][]string) + dirtySandboxes := make([]string, 0) + poolDirty := false + for _, sbx := range sandboxes { + alloc, remainAvailablePods, sandboxDirty, poolAllocate, err := allocator.doAllocate(ctx, status, sandboxToPods, availablePods, sbx, *sbx.Spec.Replicas) + availablePods = remainAvailablePods + if err != nil { + errs = append(errs, err) + } else { + sandboxAlloc[sbx.Name] = alloc + if sandboxDirty { + dirtySandboxes = append(dirtySandboxes, sbx.Name) + } + if poolAllocate { + poolDirty = true + } + } + } + return sandboxAlloc, dirtySandboxes, poolDirty, gerrors.Join(errs...) +} + +func (allocator *defaultAllocator) doAllocate(ctx context.Context, status *AllocStatus, sandboxToPods map[string][]string, availablePods []string, sbx *sandboxv1alpha1.BatchSandbox, cnt int32) ([]string, []string, bool, bool, error) { + sandboxDirty := false + poolAllocate := false + sandboxAlloc := make([]string, 0) + remainAvailablePods := availablePods + if sbx.DeletionTimestamp != nil { + return sandboxAlloc, remainAvailablePods, false, false, nil + } + sbxAlloc, err := allocator.syncer.GetAllocation(ctx, sbx) + if err != nil { + return nil, remainAvailablePods, false, false, err + } + remoteAlloc := sbxAlloc.Pods + allocatedPod := make([]string, 0) + allocatedPod = append(allocatedPod, remoteAlloc...) + name := sbx.Name + if localAlloc, ok := sandboxToPods[name]; ok { + for _, localPod := range localAlloc { + if !slices.Contains(remoteAlloc, localPod) { + sandboxDirty = true + allocatedPod = append(allocatedPod, localPod) + } + } + } + sandboxAlloc = append(sandboxAlloc, allocatedPod...) // old allocation + needAllocateCnt := cnt - int32(len(allocatedPod)) + canAllocateCnt := needAllocateCnt + if int32(len(availablePods)) < canAllocateCnt { + canAllocateCnt = int32(len(availablePods)) + } + pods := availablePods[:canAllocateCnt] + remainAvailablePods = availablePods[canAllocateCnt:] + sandboxToPods[name] = pods + for _, pod := range pods { + sandboxDirty = true + status.PodAllocation[pod] = name + poolAllocate = true + sandboxAlloc = append(sandboxAlloc, pod) // new allocation + } + if canAllocateCnt < needAllocateCnt { + status.PodSupplement += needAllocateCnt - canAllocateCnt + } + return sandboxAlloc, remainAvailablePods, sandboxDirty, poolAllocate, nil +} + +func (allocator *defaultAllocator) deallocate(ctx context.Context, status *AllocStatus, sandboxToPods map[string][]string, sandboxes []*sandboxv1alpha1.BatchSandbox) (bool, error) { + poolDeallocate := false + errs := make([]error, 0) + sbxMap := make(map[string]*sandboxv1alpha1.BatchSandbox) + for _, sandbox := range sandboxes { + sbxMap[sandbox.Name] = sandbox + deallocate, err := allocator.doDeallocate(ctx, status, sandboxToPods, sandbox) + if err != nil { + errs = append(errs, err) + } else { + if deallocate { + poolDeallocate = true + } + } + } + // gc deleted sandbox and batch sandbox + SandboxGC := make([]string, 0) + for name := range sandboxToPods { + if _, ok := sbxMap[name]; !ok { + SandboxGC = append(SandboxGC, name) + } + } + for _, name := range SandboxGC { + pods := sandboxToPods[name] + for _, pod := range pods { + delete(status.PodAllocation, pod) + poolDeallocate = true + } + delete(sandboxToPods, name) + } + return poolDeallocate, gerrors.Join(errs...) +} + +func (allocator *defaultAllocator) doDeallocate(ctx context.Context, status *AllocStatus, sandboxToPods map[string][]string, sbx *sandboxv1alpha1.BatchSandbox) (bool, error) { + deallocate := false + name := sbx.Name + allocatedPods, ok := sandboxToPods[name] + if !ok { // pods is already release to pool + return false, nil + } + toRelease, err := allocator.syncer.GetRelease(ctx, sbx) + if err != nil { + return false, err + } + for _, pod := range toRelease.Pods { + delete(status.PodAllocation, pod) + deallocate = true + } + pods := make([]string, 0) + for _, pod := range allocatedPods { + if slices.Contains(toRelease.Pods, pod) { + continue + } + pods = append(pods, pod) + } + sandboxToPods[name] = pods + return deallocate, nil +} + +func (allocator *defaultAllocator) getPodAllocation(ctx context.Context, pool *sandboxv1alpha1.Pool) (map[string]string, error) { + alloc, err := allocator.store.GetAllocation(ctx, pool) + if err != nil { + return nil, err + } + if alloc == nil { + return map[string]string{}, nil + } + return alloc.PodAllocation, nil +} + +func (allocator *defaultAllocator) updateAllocStatus(ctx context.Context, status *AllocStatus, pool *sandboxv1alpha1.Pool) error { + alloc := &PoolAllocation{} + alloc.PodAllocation = status.PodAllocation + return allocator.store.SetAllocation(ctx, pool, alloc) +} + +func (allocator *defaultAllocator) syncAllocResult(ctx context.Context, dirtySandboxes []string, sandboxAlloc map[string][]string, sandboxes []*sandboxv1alpha1.BatchSandbox) error { + if len(dirtySandboxes) == 0 { + return nil + } + errs := make([]error, 0) + sbxMap := make(map[string]*sandboxv1alpha1.BatchSandbox) + for _, sbx := range sandboxes { + sbxMap[sbx.Name] = sbx + } + for _, name := range dirtySandboxes { + err := allocator.doSyncAllocResult(ctx, sandboxAlloc[name], sbxMap[name]) + if err != nil { + errs = append(errs, err) + } + } + return gerrors.Join(errs...) +} + +func (allocator *defaultAllocator) doSyncAllocResult(ctx context.Context, allocatedPods []string, sbx *sandboxv1alpha1.BatchSandbox) error { + allocation := &SandboxAllocation{} + allocation.Pods = allocatedPods + return allocator.syncer.SetAllocation(ctx, sbx, allocation) +} diff --git a/kubernetes/internal/controller/allocator_mock.go b/kubernetes/internal/controller/allocator_mock.go new file mode 100644 index 00000000..cf12f845 --- /dev/null +++ b/kubernetes/internal/controller/allocator_mock.go @@ -0,0 +1,171 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: allocator.go + +// Package controller is a generated GoMock package. +package controller + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + + v1alpha1 "github.com/alibaba/OpenSandbox/sandbox-k8s/api/v1alpha1" +) + +// MockAllocationStore is a mock of AllocationStore interface. +type MockAllocationStore struct { + ctrl *gomock.Controller + recorder *MockAllocationStoreMockRecorder +} + +// MockAllocationStoreMockRecorder is the mock recorder for MockAllocationStore. +type MockAllocationStoreMockRecorder struct { + mock *MockAllocationStore +} + +// NewMockAllocationStore creates a new mock instance. +func NewMockAllocationStore(ctrl *gomock.Controller) *MockAllocationStore { + mock := &MockAllocationStore{ctrl: ctrl} + mock.recorder = &MockAllocationStoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockAllocationStore) EXPECT() *MockAllocationStoreMockRecorder { + return m.recorder +} + +// GetAllocation mocks base method. +func (m *MockAllocationStore) GetAllocation(ctx context.Context, pool *v1alpha1.Pool) (*PoolAllocation, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAllocation", ctx, pool) + ret0, _ := ret[0].(*PoolAllocation) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAllocation indicates an expected call of GetAllocation. +func (mr *MockAllocationStoreMockRecorder) GetAllocation(ctx, pool interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllocation", reflect.TypeOf((*MockAllocationStore)(nil).GetAllocation), ctx, pool) +} + +// SetAllocation mocks base method. +func (m *MockAllocationStore) SetAllocation(ctx context.Context, pool *v1alpha1.Pool, allocation *PoolAllocation) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetAllocation", ctx, pool, allocation) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetAllocation indicates an expected call of SetAllocation. +func (mr *MockAllocationStoreMockRecorder) SetAllocation(ctx, pool, allocation interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetAllocation", reflect.TypeOf((*MockAllocationStore)(nil).SetAllocation), ctx, pool, allocation) +} + +// MockAllocationSyncer is a mock of AllocationSyncer interface. +type MockAllocationSyncer struct { + ctrl *gomock.Controller + recorder *MockAllocationSyncerMockRecorder +} + +// MockAllocationSyncerMockRecorder is the mock recorder for MockAllocationSyncer. +type MockAllocationSyncerMockRecorder struct { + mock *MockAllocationSyncer +} + +// NewMockAllocationSyncer creates a new mock instance. +func NewMockAllocationSyncer(ctrl *gomock.Controller) *MockAllocationSyncer { + mock := &MockAllocationSyncer{ctrl: ctrl} + mock.recorder = &MockAllocationSyncerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockAllocationSyncer) EXPECT() *MockAllocationSyncerMockRecorder { + return m.recorder +} + +// GetAllocation mocks base method. +func (m *MockAllocationSyncer) GetAllocation(ctx context.Context, sandbox *v1alpha1.BatchSandbox) (*SandboxAllocation, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAllocation", ctx, sandbox) + ret0, _ := ret[0].(*SandboxAllocation) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAllocation indicates an expected call of GetAllocation. +func (mr *MockAllocationSyncerMockRecorder) GetAllocation(ctx, sandbox interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllocation", reflect.TypeOf((*MockAllocationSyncer)(nil).GetAllocation), ctx, sandbox) +} + +// GetRelease mocks base method. +func (m *MockAllocationSyncer) GetRelease(ctx context.Context, sandbox *v1alpha1.BatchSandbox) (*AllocationRelease, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetRelease", ctx, sandbox) + ret0, _ := ret[0].(*AllocationRelease) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetRelease indicates an expected call of GetRelease. +func (mr *MockAllocationSyncerMockRecorder) GetRelease(ctx, sandbox interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRelease", reflect.TypeOf((*MockAllocationSyncer)(nil).GetRelease), ctx, sandbox) +} + +// SetAllocation mocks base method. +func (m *MockAllocationSyncer) SetAllocation(ctx context.Context, sandbox *v1alpha1.BatchSandbox, allocation *SandboxAllocation) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetAllocation", ctx, sandbox, allocation) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetAllocation indicates an expected call of SetAllocation. +func (mr *MockAllocationSyncerMockRecorder) SetAllocation(ctx, sandbox, allocation interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetAllocation", reflect.TypeOf((*MockAllocationSyncer)(nil).SetAllocation), ctx, sandbox, allocation) +} + +// MockAllocator is a mock of Allocator interface. +type MockAllocator struct { + ctrl *gomock.Controller + recorder *MockAllocatorMockRecorder +} + +// MockAllocatorMockRecorder is the mock recorder for MockAllocator. +type MockAllocatorMockRecorder struct { + mock *MockAllocator +} + +// NewMockAllocator creates a new mock instance. +func NewMockAllocator(ctrl *gomock.Controller) *MockAllocator { + mock := &MockAllocator{ctrl: ctrl} + mock.recorder = &MockAllocatorMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockAllocator) EXPECT() *MockAllocatorMockRecorder { + return m.recorder +} + +// Schedule mocks base method. +func (m *MockAllocator) Schedule(ctx context.Context, spec *AllocSpec) (*AllocStatus, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Schedule", ctx, spec) + ret0, _ := ret[0].(*AllocStatus) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Schedule indicates an expected call of Schedule. +func (mr *MockAllocatorMockRecorder) Schedule(ctx, spec interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Schedule", reflect.TypeOf((*MockAllocator)(nil).Schedule), ctx, spec) +} diff --git a/kubernetes/internal/controller/allocator_test.go b/kubernetes/internal/controller/allocator_test.go new file mode 100644 index 00000000..904eea90 --- /dev/null +++ b/kubernetes/internal/controller/allocator_test.go @@ -0,0 +1,305 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "context" + "reflect" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + sandboxv1alpha1 "github.com/alibaba/OpenSandbox/sandbox-k8s/api/v1alpha1" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" +) + +func TestAllocatorSchedule(t *testing.T) { + ctrl := gomock.NewController(t) + store := NewMockAllocationStore(ctrl) + syncer := NewMockAllocationSyncer(ctrl) + allocator := &defaultAllocator{ + store: store, + syncer: syncer, + } + replica1 := int32(1) + replica2 := int32(2) + type TestCase struct { + name string + spec *AllocSpec + poolAlloc *PoolAllocation + sandboxAlloc *SandboxAllocation + release *AllocationRelease + wantStatus *AllocStatus + } + cases := []TestCase{ + { + name: "normal", + spec: &AllocSpec{ + Pods: []*corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + }, + }, + }, + Pool: &sandboxv1alpha1.Pool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pool1", + }, + }, + Sandboxes: []*sandboxv1alpha1.BatchSandbox{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "sbx1", + }, + Spec: sandboxv1alpha1.BatchSandboxSpec{ + PoolRef: "pool1", + Replicas: &replica1, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "sbx2", + }, + Spec: sandboxv1alpha1.BatchSandboxSpec{ + PoolRef: "pool1", + Replicas: &replica1, + }, + }, + }, + }, + poolAlloc: &PoolAllocation{ + PodAllocation: map[string]string{}, + }, + sandboxAlloc: &SandboxAllocation{ + Pods: []string{}, + }, + release: &AllocationRelease{ + Pods: []string{}, + }, + wantStatus: &AllocStatus{ + PodAllocation: map[string]string{ + "pod1": "sbx1", + "pod2": "sbx2", + }, + PodSupplement: 0, + }, + }, + { + name: "pod not running", + spec: &AllocSpec{ + Pods: []*corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodPending, + }, + }, + }, + Pool: &sandboxv1alpha1.Pool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pool1", + }, + }, + Sandboxes: []*sandboxv1alpha1.BatchSandbox{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "sbx1", + }, + Spec: sandboxv1alpha1.BatchSandboxSpec{ + PoolRef: "pool1", + Replicas: &replica1, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "sbx2", + }, + Spec: sandboxv1alpha1.BatchSandboxSpec{ + PoolRef: "pool1", + Replicas: &replica1, + }, + }, + }, + }, + poolAlloc: &PoolAllocation{ + PodAllocation: map[string]string{}, + }, + sandboxAlloc: &SandboxAllocation{ + Pods: []string{}, + }, + release: &AllocationRelease{ + Pods: []string{}, + }, + wantStatus: &AllocStatus{ + PodAllocation: map[string]string{ + "pod1": "sbx1", + }, + PodSupplement: 1, + }, + }, + { + name: "already partial allocated", + spec: &AllocSpec{ + Pods: []*corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pod3", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + }, + }, + }, + Pool: &sandboxv1alpha1.Pool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pool1", + }, + }, + Sandboxes: []*sandboxv1alpha1.BatchSandbox{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "sbx1", + }, + Spec: sandboxv1alpha1.BatchSandboxSpec{ + PoolRef: "pool1", + Replicas: &replica2, + }, + }, + }, + }, + poolAlloc: &PoolAllocation{ + PodAllocation: map[string]string{ + "pod1": "sbx1", + }, + }, + sandboxAlloc: &SandboxAllocation{ + Pods: []string{ + "pod1", + }, + }, + release: &AllocationRelease{ + Pods: []string{}, + }, + wantStatus: &AllocStatus{ + PodAllocation: map[string]string{ + "pod1": "sbx1", + "pod2": "sbx1", + }, + PodSupplement: 0, + }, + }, + { + name: "no need allocated with release", + spec: &AllocSpec{ + Pods: []*corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + }, + }, + }, + Pool: &sandboxv1alpha1.Pool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pool1", + }, + }, + Sandboxes: []*sandboxv1alpha1.BatchSandbox{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "sbx1", + }, + Spec: sandboxv1alpha1.BatchSandboxSpec{ + PoolRef: "pool1", + Replicas: &replica1, + }, + }, + }, + }, + poolAlloc: &PoolAllocation{ + PodAllocation: map[string]string{}, + }, + sandboxAlloc: &SandboxAllocation{ + Pods: []string{ + "pod1", + }, + }, + release: &AllocationRelease{ + Pods: []string{ + "pod1", "sbx1", + }, + }, + wantStatus: &AllocStatus{ + PodAllocation: map[string]string{}, + PodSupplement: 0, + }, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + store.EXPECT().GetAllocation(gomock.Any(), gomock.Any()).Return(c.poolAlloc, nil).Times(1) + store.EXPECT().SetAllocation(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + syncer.EXPECT().GetAllocation(gomock.Any(), gomock.Any()).Return(c.sandboxAlloc, nil).Times(len(c.spec.Sandboxes)) + syncer.EXPECT().SetAllocation(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + syncer.EXPECT().GetRelease(gomock.Any(), gomock.Any()).Return(c.release, nil).Times(len(c.spec.Sandboxes)) + status, err := allocator.Schedule(context.Background(), c.spec) + assert.NoError(t, err) + assert.True(t, reflect.DeepEqual(c.wantStatus, status)) + }) + } + +} diff --git a/kubernetes/internal/controller/apis.go b/kubernetes/internal/controller/apis.go new file mode 100644 index 00000000..2eb3775f --- /dev/null +++ b/kubernetes/internal/controller/apis.go @@ -0,0 +1,75 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "encoding/json" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/utils" +) + +const ( + AnnoAllocStatusKey = "sandbox.opensandbox.io/alloc-status" + AnnoAllocReleaseKey = "sandbox.opensandbox.io/alloc-release" + LabelBatchSandboxPodIndexKey = "batch-sandbox.sandbox.opensandbox.io/pod-index" + + AnnoPoolAllocStatusKey = "pool.opensandbox.io/alloc-status" + AnnoPoolAllocGenerationKey = "pool.opensandbox.io/alloc-generation" + + FinalizerTaskCleanup = "batch-sandbox.sandbox.opensandbox.io/task-cleanup" + + AnnotationSandboxEndpoints = "sandbox.opensandbox.io/endpoints" +) + +type SandboxAllocation struct { + Pods []string `json:"pods"` +} + +type AllocationRelease struct { + Pods []string `json:"pods"` +} + +type PoolAllocation struct { + PodAllocation map[string]string `json:"podAllocation"` +} + +func parseSandboxAllocation(obj metav1.Object) (SandboxAllocation, error) { + ret := SandboxAllocation{} + if raw := obj.GetAnnotations()[AnnoAllocStatusKey]; raw != "" { + if err := json.Unmarshal([]byte(raw), &ret); err != nil { + return ret, err + } + } + return ret, nil +} + +func setSandboxAllocation(obj metav1.Object, alloc SandboxAllocation) { + if obj.GetAnnotations() == nil { + obj.SetAnnotations(map[string]string{}) + } + obj.GetAnnotations()[AnnoAllocStatusKey] = utils.DumpJSON(alloc) +} + +func parseSandboxReleased(obj metav1.Object) (AllocationRelease, error) { + ret := AllocationRelease{} + if raw := obj.GetAnnotations()[AnnoAllocReleaseKey]; raw != "" { + if err := json.Unmarshal([]byte(raw), &ret); err != nil { + return ret, err + } + } + return ret, nil +} diff --git a/kubernetes/internal/controller/batchsandbox_controller.go b/kubernetes/internal/controller/batchsandbox_controller.go new file mode 100644 index 00000000..b0671856 --- /dev/null +++ b/kubernetes/internal/controller/batchsandbox_controller.go @@ -0,0 +1,545 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "context" + "encoding/json" + gerrors "errors" + "fmt" + "reflect" + "slices" + "strconv" + "strings" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/strategicpatch" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + sandboxv1alpha1 "github.com/alibaba/OpenSandbox/sandbox-k8s/api/v1alpha1" + taskscheduler "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/scheduler" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/utils" + controllerutils "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/utils/controller" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/utils/expectations" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/utils/fieldindex" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/utils/requeueduration" +) + +var ( + BatchSandboxScaleExpectations = expectations.NewScaleExpectations() + DurationStore = requeueduration.DurationStore{} +) + +// BatchSandboxReconciler reconciles a BatchSandbox object +type BatchSandboxReconciler struct { + client.Client + Scheme *runtime.Scheme + Recorder record.EventRecorder + taskSchedulers sync.Map +} + +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=events,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=sandbox.opensandbox.io,resources=batchsandboxes,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=sandbox.opensandbox.io,resources=batchsandboxes/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=sandbox.opensandbox.io,resources=batchsandboxes/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the BatchSandbox object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.21.0/pkg/reconcile +func (r *BatchSandboxReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + defer func() { + _ = DurationStore.Pop(req.String()) + }() + batchSbx := &sandboxv1alpha1.BatchSandbox{} + if err := r.Get(ctx, client.ObjectKey{ + Namespace: req.Namespace, + Name: req.Name, + }, batchSbx); err != nil { + if errors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + // handle expire + if expireAt := batchSbx.Spec.ExpireTime; expireAt != nil { + now := time.Now() + if expireAt.Time.Before(now) { + if batchSbx.DeletionTimestamp == nil { + klog.Infof("batch sandbox %s expired, expire at %v, delete", klog.KObj(batchSbx), expireAt) + if err := r.Delete(ctx, batchSbx); err != nil { + if errors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + } + } else { + DurationStore.Push(types.NamespacedName{Namespace: batchSbx.Namespace, Name: batchSbx.Name}.String(), expireAt.Time.Sub(now)) + } + } + + // handle finalizers + if batchSbx.DeletionTimestamp == nil { + if batchSbx.Spec.TaskTemplate != nil { + if !controllerutil.ContainsFinalizer(batchSbx, FinalizerTaskCleanup) { + err := utils.UpdateFinalizer(r.Client, batchSbx, utils.AddFinalizerOpType, FinalizerTaskCleanup) + if err != nil { + klog.Errorf("failed to add finalizer %s %s, err %v", FinalizerTaskCleanup, klog.KObj(batchSbx), err) + } else { + klog.Infof("batchsandbox %s add finalizer %s", klog.KObj(batchSbx), FinalizerTaskCleanup) + } + return ctrl.Result{}, err + } + } + } else { + if batchSbx.Spec.TaskTemplate == nil { + return ctrl.Result{}, nil + } + } + + var aggErrors []error + + pods, err := r.listPods(ctx, batchSbx) + if err != nil { + aggErrors = append(aggErrors, err) + } else { + slices.SortStableFunc[[]*corev1.Pod, *corev1.Pod](pods, utils.PodNameSorter) + var err error + // Normal Mode need scale Pods + if batchSbx.Spec.Template != nil { + err = r.scaleBatchSandbox(ctx, batchSbx, batchSbx.Spec.Template, pods) + } + if err != nil { + aggErrors = append(aggErrors, err) + } + + // TODO merge task status update + newStatus := batchSbx.Status.DeepCopy() + newStatus.ObservedGeneration = batchSbx.Generation + newStatus.Replicas = 0 + newStatus.Allocated = 0 + newStatus.Ready = 0 + ipList := []string{} + for _, pod := range pods { + newStatus.Replicas++ + if utils.IsAssigned(pod) { + newStatus.Allocated++ + ipList = append(ipList, pod.Status.PodIP) + } + if pod.Status.Phase == corev1.PodRunning && utils.IsPodReady(pod) { + newStatus.Ready++ + } + } + raw, _ := json.Marshal(ipList) + if batchSbx.Annotations[AnnotationSandboxEndpoints] != string(raw) { + patchData, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "annotations": map[string]string{ + AnnotationSandboxEndpoints: string(raw), + }, + }, + }) + obj := &sandboxv1alpha1.BatchSandbox{ObjectMeta: metav1.ObjectMeta{Namespace: batchSbx.Namespace, Name: batchSbx.Name}} + if err := r.Patch(ctx, obj, client.RawPatch(types.MergePatchType, []byte(patchData))); err != nil { + klog.Errorf("failed to patch annotation %s, %s, body %s", AnnotationSandboxEndpoints, klog.KObj(batchSbx), patchData) + aggErrors = append(aggErrors, err) + } + } + if !reflect.DeepEqual(newStatus, batchSbx.Status) { + klog.Infof("To update BatchSandbox status for %s, replicas=%d allocated=%d ready=%d", klog.KObj(batchSbx), newStatus.Replicas, newStatus.Allocated, newStatus.Ready) + if err := r.updateStatus(batchSbx, newStatus); err != nil { + aggErrors = append(aggErrors, err) + } + } + } + + // task schedule + if batchSbx.Spec.TaskTemplate != nil { + // Because tasks are in-memory and there is no event mechanism, periodic reconciliation is required. + DurationStore.Push(types.NamespacedName{Namespace: batchSbx.Namespace, Name: batchSbx.Name}.String(), 3*time.Second) + sch, err := r.getTaskScheduler(batchSbx, pods) + if err != nil { + return ctrl.Result{}, err + } + if batchSbx.DeletionTimestamp != nil { + stoppingTasks := sch.StopTask() + if len(stoppingTasks) > 0 { + klog.Infof("BatchSandbox %s is stopping %d tasks this round", klog.KObj(batchSbx), len(stoppingTasks)) + } + } + now := time.Now() + if err = r.scheduleTasks(ctx, sch, batchSbx); err != nil { + klog.Errorf("BatchSandbox %s failed to schedule tasks, err %v", klog.KObj(batchSbx), err) + aggErrors = append(aggErrors, err) + } else { + klog.Infof("BatchSandbox %s schedule tasks cost %d ms", klog.KObj(batchSbx), time.Since(now).Milliseconds()) + } + // check task cleanup is finished + if batchSbx.DeletionTimestamp != nil { + unfinishedTasks := r.getTasksCleanupUnfinished(batchSbx, sch) + if len(unfinishedTasks) > 0 { + klog.Infof("BatchSandbox %s is terminating, tasks cleanup is unfinished, unfinished tasks %v", klog.KObj(batchSbx), unfinishedTasks) + } else { + var err error + if controllerutil.ContainsFinalizer(batchSbx, FinalizerTaskCleanup) { + err = utils.UpdateFinalizer(r.Client, batchSbx, utils.RemoveFinalizerOpType, FinalizerTaskCleanup) + if err != nil { + if errors.IsNotFound(err) { + err = nil + } else { + klog.Errorf("failed to remove finalizer %s %s, err %v", FinalizerTaskCleanup, klog.KObj(batchSbx), err) + } + } + } + if err == nil { + r.deleteTaskScheduler(batchSbx) + klog.Infof("BatchSandbox %s is terminating, task cleanup is finished, remove finalizer %s %s", klog.KObj(batchSbx), FinalizerTaskCleanup, klog.KObj(batchSbx)) + } + return ctrl.Result{}, err + } + } + } + + return reconcile.Result{RequeueAfter: DurationStore.Pop(req.String())}, gerrors.Join(aggErrors...) +} + +func (r *BatchSandboxReconciler) listPods(ctx context.Context, batchSbx *sandboxv1alpha1.BatchSandbox) ([]*corev1.Pod, error) { + var ret []*corev1.Pod + if batchSbx.Spec.PoolRef != "" { + var ( + allocSet = make(sets.Set[string]) + releasedSet = make(sets.Set[string]) + ) + alloc, err := parseSandboxAllocation(batchSbx) + if err != nil { + return nil, err + } + allocSet.Insert(alloc.Pods...) + + released, err := parseSandboxReleased(batchSbx) + if err != nil { + return nil, err + } + releasedSet.Insert(released.Pods...) + + activePods := allocSet.Difference(releasedSet) + for name := range activePods { + pod := &corev1.Pod{} + // TODO maybe performance is problem + if err := r.Client.Get(ctx, types.NamespacedName{Namespace: batchSbx.Namespace, Name: name}, pod); err != nil { + if errors.IsNotFound(err) { + continue + } + return nil, err + } + ret = append(ret, pod) + } + } else { + podList := &corev1.PodList{} + if err := r.Client.List(ctx, podList, &client.ListOptions{ + Namespace: batchSbx.Namespace, + FieldSelector: fields.SelectorFromSet(fields.Set{fieldindex.IndexNameForOwnerRefUID: string(batchSbx.UID)}), + }); err != nil { + return nil, err + } + for i := range podList.Items { + ret = append(ret, &podList.Items[i]) + } + } + return ret, nil +} + +func (r *BatchSandboxReconciler) getTaskScheduler(batchSbx *sandboxv1alpha1.BatchSandbox, pods []*corev1.Pod) (taskscheduler.TaskScheduler, error) { + var tSch taskscheduler.TaskScheduler + key := types.NamespacedName{Namespace: batchSbx.Namespace, Name: batchSbx.Name}.String() + val, ok := r.taskSchedulers.Load(key) + // The reconciler guarantees that it will not concurrently reconcile the same BatchSandbox. + if !ok { + policy := sandboxv1alpha1.TaskResourcePolicyRetain + if batchSbx.Spec.TaskResourcePolicyWhenCompleted != nil { + policy = *batchSbx.Spec.TaskResourcePolicyWhenCompleted + } + taskSpecs, err := generaTaskSpec(batchSbx) + if err != nil { + return nil, err + } + sc, err := taskscheduler.NewTaskScheduler(key, taskSpecs, pods, policy) + if err != nil { + return nil, fmt.Errorf("new task scheduler err %w", err) + } + klog.Infof("successfully new task scheduler for batch sandbox %s", klog.KObj(batchSbx)) + tSch = sc + r.taskSchedulers.Store(key, sc) + } else { + tSch, ok = (val.(taskscheduler.TaskScheduler)) + if !ok { + return nil, gerrors.New("invalid scheduler type stored") + } + // Update the pods list for this scheduler + tSch.UpdatePods(pods) + } + return tSch, nil +} + +func (r *BatchSandboxReconciler) deleteTaskScheduler(batchSbx *sandboxv1alpha1.BatchSandbox) { + klog.Infof("delete task scheduler for batch sandbox %s", klog.KObj(batchSbx)) + key := types.NamespacedName{Namespace: batchSbx.Namespace, Name: batchSbx.Name}.String() + r.taskSchedulers.Delete(key) +} + +func generaTaskSpec(batchSbx *sandboxv1alpha1.BatchSandbox) ([]*sandboxv1alpha1.Task, error) { + ret := make([]*sandboxv1alpha1.Task, *batchSbx.Spec.Replicas) + for idx := range int(*batchSbx.Spec.Replicas) { + task, err := getTaskSpec(batchSbx, idx) + if err != nil { + return ret, err + } + ret[idx] = task + } + return ret, nil +} + +func getTaskSpec(batchSbx *sandboxv1alpha1.BatchSandbox, idx int) (*sandboxv1alpha1.Task, error) { + task := &sandboxv1alpha1.Task{ + ObjectMeta: batchSbx.Spec.TaskTemplate.ObjectMeta, + Spec: *batchSbx.Spec.TaskTemplate.Spec.DeepCopy(), + } + if task.Name == "" { + task.Name = fmt.Sprintf("%s-%d", batchSbx.Name, idx) + } + if task.Namespace == "" { + task.Namespace = batchSbx.Namespace + } + if len(batchSbx.Spec.ShardTaskPatches) > 0 && idx < len(batchSbx.Spec.ShardTaskPatches) { + patch := batchSbx.Spec.ShardTaskPatches[idx] + cloneBytes, _ := json.Marshal(task) + modified, err := strategicpatch.StrategicMergePatch(cloneBytes, patch.Raw, &sandboxv1alpha1.Task{}) + if err != nil { + return nil, fmt.Errorf("batchsandbox: failed to merge patch raw %s, idx %d, err %w", patch.Raw, idx, err) + } + newTask := &sandboxv1alpha1.Task{} + if err = json.Unmarshal(modified, newTask); err != nil { + return nil, fmt.Errorf("batchsandbox: failed to unmarshal %s to Task, idx %d, err %w", modified, idx, err) + } + *task = *newTask + } + return task, nil +} + +func (r *BatchSandboxReconciler) scheduleTasks(ctx context.Context, tSch taskscheduler.TaskScheduler, batchSbx *sandboxv1alpha1.BatchSandbox) error { + if err := tSch.Schedule(); err != nil { + return err + } + tasks := tSch.ListTask() + toReleasedPods := []string{} + var ( + running, failed, succeed, unknown int32 + pending int32 + ) + for i := 0; i < len(tasks); i++ { + task := tasks[i] + if task.GetPodName() == "" { + pending++ + } else { + state := task.GetState() + if task.IsResourceReleased() { + toReleasedPods = append(toReleasedPods, task.GetPodName()) + } + switch state { + case taskscheduler.RunningTaskState: + running++ + case taskscheduler.SucceedTaskState: + succeed++ + case taskscheduler.FailedTaskState: + failed++ + case taskscheduler.UnknownTaskState: + unknown++ + } + } + } + if len(toReleasedPods) > 0 { + klog.Infof("batch sandbox %s try to release %d Pod", klog.KObj(batchSbx), len(toReleasedPods)) + if err := r.releasePods(ctx, batchSbx, toReleasedPods); err != nil { + return err + } + klog.Infof("batch sandbox %s successfully released %d Pods", klog.KObj(batchSbx), len(toReleasedPods)) + } + oldStatus := batchSbx.Status + newStatus := oldStatus.DeepCopy() + newStatus.ObservedGeneration = batchSbx.Generation + newStatus.TaskRunning = running + newStatus.TaskFailed = failed + newStatus.TaskSucceed = succeed + newStatus.TaskUnknown = unknown + newStatus.TaskPending = pending + if !reflect.DeepEqual(newStatus, oldStatus) { + klog.Infof("To update BatchSandbox status for %s, replicas=%d task_running=%d task_succeed=%d, task_failed=%d, task_unknown=%d, task_pending=%d", klog.KObj(batchSbx), newStatus.Replicas, + newStatus.TaskRunning, newStatus.TaskSucceed, newStatus.TaskFailed, newStatus.TaskUnknown, newStatus.TaskPending) + if err := r.updateStatus(batchSbx, newStatus); err != nil { + return err + } + } + return nil +} + +func (r *BatchSandboxReconciler) getTasksCleanupUnfinished(batchSbx *sandboxv1alpha1.BatchSandbox, tSch taskscheduler.TaskScheduler) []taskscheduler.Task { + var notReleased []taskscheduler.Task + for _, task := range tSch.ListTask() { + if !task.IsResourceReleased() { + notReleased = append(notReleased, task) + } + } + return notReleased +} + +func (r *BatchSandboxReconciler) releasePods(ctx context.Context, batchSbx *sandboxv1alpha1.BatchSandbox, toReleasePods []string) error { + releasedSet := make(sets.Set[string]) + released, err := parseSandboxReleased(batchSbx) + if err != nil { + return err + } + releasedSet.Insert(released.Pods...) + releasedSet.Insert(toReleasePods...) + newRelease := AllocationRelease{ + Pods: sets.List(releasedSet), + } + raw, err := json.Marshal(newRelease) + if err != nil { + return fmt.Errorf("Failed to marshal released pod names: %v", err) + } + body := utils.DumpJSON(struct { + MetaData metav1.ObjectMeta `json:"metadata"` + }{ + MetaData: metav1.ObjectMeta{ + Annotations: map[string]string{ + AnnoAllocReleaseKey: string(raw), + }, + }, + }) + b := &sandboxv1alpha1.BatchSandbox{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: batchSbx.Namespace, + Name: batchSbx.Name, + }, + } + return r.Client.Patch(ctx, b, client.RawPatch(types.MergePatchType, []byte(body))) +} + +// Normal Mode +func (r *BatchSandboxReconciler) scaleBatchSandbox(ctx context.Context, batchSandbox *sandboxv1alpha1.BatchSandbox, podTemplateSpec *corev1.PodTemplateSpec, pods []*corev1.Pod) error { + indexedPodMap := map[int]*corev1.Pod{} + for i := range pods { + pod := pods[i] + BatchSandboxScaleExpectations.ObserveScale(controllerutils.GetControllerKey(batchSandbox), expectations.Create, pod.Name) + pods = append(pods, pod) + idx, err := parseIndex(pod) + if err != nil { + return fmt.Errorf("failed to parse idx Pod %s, err %w", pod.Name, err) + } + indexedPodMap[idx] = pod + } + if satisfied, unsatisfiedDuration, dirtyPods := BatchSandboxScaleExpectations.SatisfiedExpectations(controllerutils.GetControllerKey(batchSandbox)); !satisfied { + klog.Infof("BatchSandbox %s scale expectation is not satisfied overtime=%v, dirty pods=%v", klog.KObj(batchSandbox), unsatisfiedDuration, dirtyPods) + DurationStore.Push(types.NamespacedName{Namespace: batchSandbox.Namespace, Name: batchSandbox.Name}.String(), expectations.ExpectationTimeout-unsatisfiedDuration) + return nil + } + // TODO consider supply Pods if Pods is deleted unexpectedly + var needCreateIndex []int + // TODO var needDeleteIndex []int + for i := 0; i < int(*batchSandbox.Spec.Replicas); i++ { + _, ok := indexedPodMap[i] + if !ok { + needCreateIndex = append(needCreateIndex, i) + } + } + // scale + if len(needCreateIndex) > 0 { + klog.Infof("BatchSandbox %s try to create %d Pod, idx %v", klog.KObj(batchSandbox), len(needCreateIndex), needCreateIndex) + } + for _, idx := range needCreateIndex { + pod, err := utils.GetPodFromTemplate(podTemplateSpec, batchSandbox, metav1.NewControllerRef(batchSandbox, sandboxv1alpha1.SchemeBuilder.GroupVersion.WithKind("BatchSandbox"))) + if err != nil { + return err + } + if err := ctrl.SetControllerReference(pod, batchSandbox, r.Scheme); err != nil { + return err + } + pod.Labels[LabelBatchSandboxPodIndexKey] = strconv.Itoa(idx) + pod.Namespace = batchSandbox.Namespace + pod.Name = fmt.Sprintf("%s-%d", batchSandbox.Name, idx) + BatchSandboxScaleExpectations.ExpectScale(controllerutils.GetControllerKey(batchSandbox), expectations.Create, pod.Name) + if err := r.Create(ctx, pod); err != nil { + BatchSandboxScaleExpectations.ObserveScale(controllerutils.GetControllerKey(batchSandbox), expectations.Create, pod.Name) + r.Recorder.Eventf(batchSandbox, corev1.EventTypeWarning, "FailedCreate", "failed to create pod: %v, pod: %v", err, utils.DumpJSON(pod)) + return err + } + r.Recorder.Eventf(batchSandbox, corev1.EventTypeNormal, "SuccessfulCreate", "succeed to create pod %s", pod.Name) + } + return nil +} + +func parseIndex(pod *corev1.Pod) (int, error) { + if v := pod.Labels[LabelBatchSandboxPodIndexKey]; v != "" { + return strconv.Atoi(v) + } + idx := strings.LastIndex(pod.Name, "-") + if idx == -1 { + return -1, gerrors.New("batchsandbox: Invalid pod Name") + } + return strconv.Atoi(pod.Name[idx+1:]) +} + +func (r *BatchSandboxReconciler) updateStatus(batchSandbox *sandboxv1alpha1.BatchSandbox, newStatus *sandboxv1alpha1.BatchSandboxStatus) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + clone := &sandboxv1alpha1.BatchSandbox{} + if err := r.Get(context.TODO(), types.NamespacedName{Namespace: batchSandbox.Namespace, Name: batchSandbox.Name}, clone); err != nil { + return err + } + clone.Status = *newStatus + return r.Status().Update(context.TODO(), clone) + }) +} + +// SetupWithManager sets up the controller with the Manager. +func (r *BatchSandboxReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&sandboxv1alpha1.BatchSandbox{}). + Named("batchsandbox"). + Owns(&corev1.Pod{}). + WithOptions(controller.Options{MaxConcurrentReconciles: 32}). + Complete(r) +} diff --git a/kubernetes/internal/controller/batchsandbox_controller_test.go b/kubernetes/internal/controller/batchsandbox_controller_test.go new file mode 100644 index 00000000..62b74b11 --- /dev/null +++ b/kubernetes/internal/controller/batchsandbox_controller_test.go @@ -0,0 +1,799 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "context" + "encoding/json" + gerrors "errors" + "fmt" + "net" + "reflect" + "sync" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + k8sruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/rand" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/retry" + "k8s.io/utils/ptr" + "k8s.io/utils/set" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + sandboxv1alpha1 "github.com/alibaba/OpenSandbox/sandbox-k8s/api/v1alpha1" + taskscheduler "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/scheduler" + mock_scheduler "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/scheduler/mock" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/utils/fieldindex" +) + +func init() { + testscheme = k8sruntime.NewScheme() + utilruntime.Must(corev1.AddToScheme(testscheme)) + utilruntime.Must(sandboxv1alpha1.AddToScheme(testscheme)) +} + +var testscheme *k8sruntime.Scheme + +var _ = Describe("BatchSandbox Controller", func() { + var ( + timeout = 30 * time.Second + interval = 5 * time.Second + ) + // None Pooling Mode + Context("When create new batch sandbox, create pod base on pod template", func() { + const resourceBaseName = "test-batch-sandbox" + + ctx := context.Background() + + typeNamespacedName := types.NamespacedName{ + Name: resourceBaseName, + Namespace: "default", + } + + BeforeEach(func() { + typeNamespacedName.Name = fmt.Sprintf("%s-%s", resourceBaseName, rand.String(5)) + By(fmt.Sprintf("creating the custom resource %s for the Kind BatchSandbox", typeNamespacedName)) + resource := &sandboxv1alpha1.BatchSandbox{ + ObjectMeta: metav1.ObjectMeta{ + Name: typeNamespacedName.Name, + Namespace: typeNamespacedName.Namespace, + }, + Spec: sandboxv1alpha1.BatchSandboxSpec{ + Replicas: ptr.To(int32(1)), + Template: &v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "main", + Image: "example.com", + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, resource)).Should(Succeed()) + bs := &sandboxv1alpha1.BatchSandbox{} + Eventually(func(g Gomega) { + g.Expect(k8sClient.Get(ctx, typeNamespacedName, bs)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + By(fmt.Sprintf("wait the custom resource %s created", typeNamespacedName)) + }) + + AfterEach(func() { + resource := &sandboxv1alpha1.BatchSandbox{} + err := k8sClient.Get(ctx, typeNamespacedName, resource) + if !errors.IsNotFound(err) { + Expect(err).NotTo(HaveOccurred()) + } else { + return + } + By(fmt.Sprintf("Cleanup the specific resource instance BatchSandbox %s", typeNamespacedName)) + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + }) + It("should successfully create pod, update batch sandbox status, endpoints info", func() { + wantIPSet := make(set.Set[string]) + Eventually(func(g Gomega) { + bs := &sandboxv1alpha1.BatchSandbox{} + if err := k8sClient.Get(ctx, typeNamespacedName, bs); err != nil { + return + } + allPods := &corev1.PodList{} + g.Expect(k8sClient.List(ctx, allPods, &client.ListOptions{Namespace: bs.Namespace})).Should(Succeed()) + pods := []*corev1.Pod{} + for i := range allPods.Items { + po := &allPods.Items[i] + if metav1.IsControlledBy(po, bs) { + pods = append(pods, po) + if po.Status.PodIP != "" { + continue + } + // patch status to make pod Scheduled + mockIP := randomIPv4().String() + wantIPSet.Insert(mockIP) + po.Status.PodIP = mockIP + po.Status.Phase = corev1.PodRunning + po.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + Expect(k8sClient.Status().Update(context.Background(), po)).To(Succeed()) + } + } + g.Expect(len(pods)).To(Equal(int(*bs.Spec.Replicas))) + g.Expect(bs.Status.ObservedGeneration).To(Equal(bs.Generation)) + g.Expect(bs.Status.Replicas).To(Equal(*bs.Spec.Replicas)) + g.Expect(bs.Status.Allocated).To(Equal(*bs.Spec.Replicas)) + g.Expect(bs.Status.Ready).To(Equal(*bs.Spec.Replicas)) + + gotIPs := []string{} + if raw := bs.Annotations[AnnotationSandboxEndpoints]; raw != "" { + json.Unmarshal([]byte(raw), &gotIPs) + } + g.Expect(wantIPSet.Equal(set.New(gotIPs...))).To(BeTrue(), fmt.Sprintf("wantIPSet %v, gotIPs %v", wantIPSet.SortedList(), gotIPs)) + }, timeout, interval).Should(Succeed()) + }) + It("should successfully correctly create new Pod and update batch sandbox status when user scale out", func() { + bs := &sandboxv1alpha1.BatchSandbox{} + Expect(k8sClient.Get(ctx, typeNamespacedName, bs)).Should(Succeed()) + *bs.Spec.Replicas = *bs.Spec.Replicas + 1 // scale out + Expect(k8sClient.Update(ctx, bs)).Should(Succeed()) + Eventually(func(g Gomega) { + batchsandbox := &sandboxv1alpha1.BatchSandbox{} + if err := k8sClient.Get(ctx, typeNamespacedName, batchsandbox); err != nil { + return + } + g.Expect(batchsandbox.Status.ObservedGeneration).To(Equal(batchsandbox.Generation)) + g.Expect(batchsandbox.Status.Replicas).To(Equal(*batchsandbox.Spec.Replicas)) + }, timeout, interval).Should(Succeed()) + Eventually(func(g Gomega) { + pods := &v1.PodList{} + g.Expect(k8sClient.List(ctx, pods, &client.ListOptions{ + Namespace: bs.Namespace, + FieldSelector: fields.SelectorFromSet(fields.Set{fieldindex.IndexNameForOwnerRefUID: string(bs.UID)}), + })).Should(Succeed()) + g.Expect(int32(len(pods.Items))).To(Equal(*bs.Spec.Replicas)) + }, timeout, interval).Should(Succeed()) + }) + It("should successfully correctly supply Pod when pod is deleted unexpectedly", func() { + Eventually(func(g Gomega) { + bs := &sandboxv1alpha1.BatchSandbox{} + if err := k8sClient.Get(ctx, typeNamespacedName, bs); err != nil { + return + } + g.Expect(bs.Status.ObservedGeneration).To(Equal(bs.Generation)) + g.Expect(bs.Status.Replicas).To(Equal(*bs.Spec.Replicas)) + }, timeout, interval).Should(Succeed()) + bs := &sandboxv1alpha1.BatchSandbox{} + Expect(k8sClient.Get(ctx, typeNamespacedName, bs)).Should(Succeed()) + pods := &v1.PodList{} + Expect(k8sClient.List(ctx, pods, &client.ListOptions{ + Namespace: bs.Namespace, + FieldSelector: fields.SelectorFromSet(fields.Set{fieldindex.IndexNameForOwnerRefUID: string(bs.UID)}), + })).Should(Succeed()) + Expect(int32(len(pods.Items))).To(Equal(*bs.Spec.Replicas)) + // delete first pod + oldPod := pods.Items[0] + Expect(k8sClient.Delete(ctx, &oldPod)).Should(Succeed()) + // wait supply pod + Eventually(func(g Gomega) { + newPod := &corev1.Pod{} + if err := k8sClient.Get(ctx, types.NamespacedName{ + Namespace: bs.Namespace, + Name: oldPod.Name, + }, newPod); err != nil { + return + } + g.Expect(newPod.CreationTimestamp).NotTo(Equal(oldPod.CreationTimestamp)) + }, timeout, interval).Should(Succeed()) + }) + It("should delete batch sandbox and related Pods for expired batch sandbox", func() { + Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { + bs := &sandboxv1alpha1.BatchSandbox{} + if err := k8sClient.Get(ctx, typeNamespacedName, bs); err != nil { + return err + } + bs.Spec.ExpireTime = &metav1.Time{Time: time.Now().Add(3 * time.Second)} + return k8sClient.Update(ctx, bs) + })).Should(Succeed()) + + Eventually( + func(g Gomega) { + bs := &sandboxv1alpha1.BatchSandbox{} + g.Expect(errors.IsNotFound(k8sClient.Get(ctx, typeNamespacedName, bs))).To(BeTrue()) + allPods := &corev1.PodList{} + g.Expect(k8sClient.List(ctx, allPods, &client.ListOptions{Namespace: bs.Namespace})).Should(Succeed()) + pods := []*corev1.Pod{} + for i := range allPods.Items { + po := &allPods.Items[i] + if metav1.IsControlledBy(po, bs) { + pods = append(pods, po) + } + } + g.Expect(len(pods)).To(BeZero()) + }, + timeout, interval).Should(Succeed()) + }) + }) + + // Pooling Mode + Context("When create new batch sandbox, get pod from pool", func() { + const resourceBaseName = "test-batch-sandbox-pooling-mode" + var replicas int32 = 1 + ctx := context.Background() + + typeNamespacedName := types.NamespacedName{ + Name: resourceBaseName, + Namespace: "default", + } + BeforeEach(func() { + typeNamespacedName.Name = fmt.Sprintf("%s-%s", resourceBaseName, rand.String(5)) + By(fmt.Sprintf("creating the custom resource %s for the Kind BatchSandbox", typeNamespacedName)) + resource := &sandboxv1alpha1.BatchSandbox{ + ObjectMeta: metav1.ObjectMeta{ + Name: typeNamespacedName.Name, + Namespace: typeNamespacedName.Namespace, + }, + Spec: sandboxv1alpha1.BatchSandboxSpec{ + Replicas: ptr.To(replicas), + PoolRef: "test-pool", + }, + } + Expect(k8sClient.Create(ctx, resource)).Should(Succeed()) + bs := &sandboxv1alpha1.BatchSandbox{} + Eventually(func(g Gomega) { + g.Expect(k8sClient.Get(ctx, typeNamespacedName, bs)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + By(fmt.Sprintf("wait the custom resource %s created", typeNamespacedName)) + }) + + AfterEach(func() { + resource := &sandboxv1alpha1.BatchSandbox{} + err := k8sClient.Get(ctx, typeNamespacedName, resource) + if !errors.IsNotFound(err) { + Expect(err).NotTo(HaveOccurred()) + } + By(fmt.Sprintf("Cleanup the specific resource instance BatchSandbox %s", typeNamespacedName)) + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + }) + + It("should successfully update batch sandbox status, sbx endpoints info when get pod from pool alloc", func() { + // mock pool allocation + mockPods := []string{} + for i := range replicas { + po := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: typeNamespacedName.Namespace, + Name: fmt.Sprintf("test-pod-%d", i), + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + {Name: "main", Image: "test", Command: []string{"hello"}}, + }, + NodeName: "node-1.2.3.4", + }, + } + mockPods = append(mockPods, po.Name) + Expect(k8sClient.Create(context.Background(), po)).To(Succeed()) + po.Status.PodIP = "1.2.3.4" + po.Status.Phase = corev1.PodRunning + po.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + Expect(k8sClient.Status().Update(context.Background(), po)).To(Succeed()) + } + Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { + bs := &sandboxv1alpha1.BatchSandbox{} + if err := k8sClient.Get(ctx, typeNamespacedName, bs); err != nil { + return err + } + setSandboxAllocation(bs, SandboxAllocation{Pods: mockPods}) + return k8sClient.Update(ctx, bs) + })).Should(Succeed()) + By(fmt.Sprintf("Mock pool allocate Pod %v for BatchSandbox %s", mockPods, typeNamespacedName)) + + Eventually(func(g Gomega) { + bs := &sandboxv1alpha1.BatchSandbox{} + if err := k8sClient.Get(ctx, typeNamespacedName, bs); err != nil { + return + } + g.Expect(bs.Status.ObservedGeneration).To(Equal(bs.Generation)) + g.Expect(bs.Status.Replicas).To(Equal(*bs.Spec.Replicas)) + g.Expect(bs.Status.Allocated).To(Equal(*bs.Spec.Replicas)) + g.Expect(bs.Status.Ready).To(Equal(*bs.Spec.Replicas)) + + g.Expect(bs.Annotations[AnnotationSandboxEndpoints]).To(Equal("[\"1.2.3.4\"]")) + }, timeout, interval).Should(Succeed()) + }) + }) +}) + +func randomIPv4() net.IP { + rand.Seed(time.Now().UnixNano()) + ip := make(net.IP, 4) + for i := range ip { + ip[i] = byte(rand.Intn(256)) + } + return ip +} + +var _ = Describe("BatchSandbox Task Scheduler", func() { + var ( + timeout = 30 * time.Second + interval = 5 * time.Second + ) + // None Pooling mode + Context("When create new batch sandbox, create pod base on pod template", func() { + const resourceBaseName = "test-task-batch-sandbox" + + ctx := context.Background() + + typeNamespacedName := types.NamespacedName{ + Name: resourceBaseName, + Namespace: "default", + } + + BeforeEach(func() { + typeNamespacedName.Name = fmt.Sprintf("%s-%s", resourceBaseName, rand.String(5)) + By(fmt.Sprintf("creating the custom resource %s for the Kind BatchSandbox", typeNamespacedName)) + resource := &sandboxv1alpha1.BatchSandbox{ + ObjectMeta: metav1.ObjectMeta{ + Name: typeNamespacedName.Name, + Namespace: typeNamespacedName.Namespace, + }, + Spec: sandboxv1alpha1.BatchSandboxSpec{ + Replicas: ptr.To(int32(1)), + Template: &v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "main", + Image: "example.com", + }, + }, + }, + }, + TaskTemplate: &sandboxv1alpha1.TaskTemplateSpec{ + Spec: sandboxv1alpha1.TaskSpec{ + Process: &sandboxv1alpha1.ProcessTask{ + Command: []string{"echo", "hello"}, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, resource)).Should(Succeed()) + bs := &sandboxv1alpha1.BatchSandbox{} + Eventually(func(g Gomega) { + g.Expect(k8sClient.Get(ctx, typeNamespacedName, bs)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + By(fmt.Sprintf("wait the custom resource %s created", typeNamespacedName)) + }) + + AfterEach(func() { + resource := &sandboxv1alpha1.BatchSandbox{} + err := k8sClient.Get(ctx, typeNamespacedName, resource) + if !errors.IsNotFound(err) { + Expect(err).NotTo(HaveOccurred()) + } else { + // resource is already deleted + return + } + By(fmt.Sprintf("Cleanup the specific resource instance BatchSandbox %s", typeNamespacedName)) + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + }) + + It("should successfully add task cleanup finalizer", func() { + Eventually(func(g Gomega) { + bs := &sandboxv1alpha1.BatchSandbox{} + if err := k8sClient.Get(ctx, typeNamespacedName, bs); err != nil { + return + } + g.Expect(controllerutil.ContainsFinalizer(bs, FinalizerTaskCleanup)).To(BeTrue()) + }, timeout, interval).Should(Succeed()) + }) + + It("should successfully update task status(task_pending=1), because all pods is unassigned", func() { + Eventually(func(g Gomega) { + bs := &sandboxv1alpha1.BatchSandbox{} + if err := k8sClient.Get(ctx, typeNamespacedName, bs); err != nil { + return + } + g.Expect(bs.Status.ObservedGeneration).To(Equal(bs.Generation)) + g.Expect(bs.Status.Replicas).To(Equal(*bs.Spec.Replicas)) + + g.Expect(bs.Status.TaskPending).To(Equal(*bs.Spec.Replicas)) + g.Expect(bs.Status.TaskRunning).To(Equal(int32(0))) + g.Expect(bs.Status.TaskSucceed).To(Equal(int32(0))) + g.Expect(bs.Status.TaskFailed).To(Equal(int32(0))) + g.Expect(bs.Status.TaskUnknown).To(Equal(int32(0))) + }, timeout, interval).Should(Succeed()) + }) + + It("should successfully delete BatchSandbox when all tasks(including pending task) cleanup is finished", func() { + bs := &sandboxv1alpha1.BatchSandbox{} + Expect(k8sClient.Get(ctx, typeNamespacedName, bs)).To(Succeed()) + Eventually(func(g Gomega) { + bs := &sandboxv1alpha1.BatchSandbox{} + Expect(k8sClient.Get(ctx, typeNamespacedName, bs)).To(Succeed()) + g.Expect(controllerutil.ContainsFinalizer(bs, FinalizerTaskCleanup)).To(BeTrue()) + }, timeout, interval).Should(Succeed()) + + By(fmt.Sprintf("try to Delete BatchSandbox %s", typeNamespacedName)) + Expect(k8sClient.Delete(ctx, bs)).To(Succeed()) + + Eventually(func(g Gomega) { + bs := &sandboxv1alpha1.BatchSandbox{} + err := k8sClient.Get(ctx, typeNamespacedName, bs) + g.Expect(errors.IsNotFound(err)).To(BeTrue()) + }, timeout, interval).Should(Succeed()) + }) + }) +}) + +func TestBatchSandboxReconciler_scheduleTasks(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + var ( + fakeBatchSandbox = &sandboxv1alpha1.BatchSandbox{ + TypeMeta: metav1.TypeMeta{ + APIVersion: sandboxv1alpha1.GroupVersion.String(), + Kind: "BatchSandbox", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-batch-sandbox", + }, + Spec: sandboxv1alpha1.BatchSandboxSpec{}, + Status: sandboxv1alpha1.BatchSandboxStatus{}, + } + ) + type fields struct { + Client client.Client + Scheme *runtime.Scheme + Recorder record.EventRecorder + taskSchedulers sync.Map + } + type args struct { + ctx context.Context + tSch taskscheduler.TaskScheduler + batchSbx *sandboxv1alpha1.BatchSandbox + } + tests := []struct { + name string + fields fields + args args + wantErr bool + batchSandboxChecker func(bsbx *sandboxv1alpha1.BatchSandbox) error + }{ + { + name: "schedule err", + args: args{ + tSch: func() taskscheduler.TaskScheduler { + mockSche := mock_scheduler.NewMockTaskScheduler(ctrl) + mockSche.EXPECT().Schedule().Return(gerrors.New("err")).Times(1) + return mockSche + }(), + }, + wantErr: true, + }, + { + name: "tasks, succeed=1; releasedPod=1", + fields: fields{ + Client: fake.NewClientBuilder().WithScheme(testscheme).WithObjects(fakeBatchSandbox).WithStatusSubresource(fakeBatchSandbox).Build(), + }, + args: args{ + tSch: func() taskscheduler.TaskScheduler { + mockSche := mock_scheduler.NewMockTaskScheduler(ctrl) + mockSche.EXPECT().Schedule().Return(nil).Times(1) + mockTask := mock_scheduler.NewMockTask(ctrl) + mockTask.EXPECT().GetState().Return(taskscheduler.SucceedTaskState).Times(1) + mockTask.EXPECT().IsResourceReleased().Return(true).Times(1) + mockTask.EXPECT().GetPodName().Return("pod-0").AnyTimes() + mockSche.EXPECT().ListTask().Return([]taskscheduler.Task{mockTask}).Times(1) + return mockSche + }(), + batchSbx: fakeBatchSandbox.DeepCopy(), + }, + batchSandboxChecker: func(bsbx *sandboxv1alpha1.BatchSandbox) error { + release, err := parseSandboxReleased(bsbx) + if err != nil { + return err + } + if len(release.Pods) != 1 || release.Pods[0] != "pod-0" { + return fmt.Errorf("expect pod-0, actual %v", release.Pods) + } + // check status + if bsbx.Status.TaskSucceed != 1 { + return fmt.Errorf("expect status.succeed=1, actual %d", bsbx.Status.TaskRunning) + } + if bsbx.Status.TaskRunning != 0 || bsbx.Status.TaskFailed != 0 || bsbx.Status.TaskUnknown != 0 { + return fmt.Errorf("expect status.running=0,failed=0,unknown=0, actual %v", bsbx.Status) + } + return nil + }, + }, + } + for i := range tests { + tt := &tests[i] + t.Run(tt.name, func(t *testing.T) { + r := &BatchSandboxReconciler{ + Client: tt.fields.Client, + Scheme: tt.fields.Scheme, + Recorder: tt.fields.Recorder, + } + if err := r.scheduleTasks(tt.args.ctx, tt.args.tSch, tt.args.batchSbx); (err != nil) != tt.wantErr { + t.Errorf("BatchSandboxReconciler.scheduleTasks() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.batchSandboxChecker != nil { + bsbx := &sandboxv1alpha1.BatchSandbox{} + if err := tt.fields.Client.Get(ctx, types.NamespacedName{Namespace: tt.args.batchSbx.Namespace, Name: tt.args.batchSbx.Name}, bsbx); err != nil { + t.Errorf("BatchSandboxReconciler Get() error = %v, wantErr %v", err, nil) + } + if err := tt.batchSandboxChecker(bsbx); err != nil { + t.Errorf("BatchSandboxReconciler batchSandboxChecker() error = %v, wantErr %v", err, nil) + } + } + }) + } +} + +func Test_getTaskSpec(t *testing.T) { + type args struct { + batchSbx *sandboxv1alpha1.BatchSandbox + idx int + } + tests := []struct { + name string + args args + want *sandboxv1alpha1.Task + wantErr bool + }{ + { + name: "basic task spec without patches", + args: args{ + batchSbx: &sandboxv1alpha1.BatchSandbox{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-bs", + Namespace: "default", + }, + Spec: sandboxv1alpha1.BatchSandboxSpec{ + TaskTemplate: &sandboxv1alpha1.TaskTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "test", + }, + }, + Spec: sandboxv1alpha1.TaskSpec{ + Process: &sandboxv1alpha1.ProcessTask{ + Command: []string{"echo", "hello"}, + }, + }, + }, + }, + }, + idx: 0, + }, + want: &sandboxv1alpha1.Task{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "test", + }, + }, + Spec: sandboxv1alpha1.TaskSpec{ + Process: &sandboxv1alpha1.ProcessTask{ + Command: []string{"echo", "hello"}, + }, + }, + }, + wantErr: false, + }, + { + name: "task spec with shard patch", + args: args{ + batchSbx: &sandboxv1alpha1.BatchSandbox{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-bs", + Namespace: "default", + }, + Spec: sandboxv1alpha1.BatchSandboxSpec{ + TaskTemplate: &sandboxv1alpha1.TaskTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "test", + }, + }, + Spec: sandboxv1alpha1.TaskSpec{ + Process: &sandboxv1alpha1.ProcessTask{ + Command: []string{"echo", "hello"}, + }, + }, + }, + ShardTaskPatches: []runtime.RawExtension{ + { + Raw: []byte(`{"spec":{"process":{"command":["echo","world"]}}}`), + }, + }, + }, + }, + idx: 0, + }, + want: &sandboxv1alpha1.Task{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "test", + }, + }, + Spec: sandboxv1alpha1.TaskSpec{ + Process: &sandboxv1alpha1.ProcessTask{ + Command: []string{"echo", "world"}, + }, + }, + }, + wantErr: false, + }, + { + name: "task spec with invalid patch", + args: args{ + batchSbx: &sandboxv1alpha1.BatchSandbox{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-bs", + Namespace: "default", + }, + Spec: sandboxv1alpha1.BatchSandboxSpec{ + TaskTemplate: &sandboxv1alpha1.TaskTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "test", + }, + }, + Spec: sandboxv1alpha1.TaskSpec{ + Process: &sandboxv1alpha1.ProcessTask{ + Command: []string{"echo", "hello"}, + }, + }, + }, + ShardTaskPatches: []runtime.RawExtension{ + { + Raw: []byte(`{"invalid json`), + }, + }, + }, + }, + idx: 0, + }, + want: nil, + wantErr: true, + }, + { + name: "task spec with index out of range patch", + args: args{ + batchSbx: &sandboxv1alpha1.BatchSandbox{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-bs", + Namespace: "default", + }, + Spec: sandboxv1alpha1.BatchSandboxSpec{ + TaskTemplate: &sandboxv1alpha1.TaskTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "test", + }, + }, + Spec: sandboxv1alpha1.TaskSpec{ + Process: &sandboxv1alpha1.ProcessTask{ + Command: []string{"echo", "hello"}, + }, + }, + }, + ShardTaskPatches: []runtime.RawExtension{ + { + Raw: []byte(`{"spec":{"process":{"command":["echo","world"]}}}`), + }, + }, + }, + }, + idx: 1, // Index out of range, should use base template + }, + want: &sandboxv1alpha1.Task{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "test", + }, + }, + Spec: sandboxv1alpha1.TaskSpec{ + Process: &sandboxv1alpha1.ProcessTask{ + Command: []string{"echo", "hello"}, + }, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getTaskSpec(tt.args.batchSbx, tt.args.idx) + if (err != nil) != tt.wantErr { + t.Errorf("getTaskSpec() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !tt.wantErr { + if !reflect.DeepEqual(got.ObjectMeta.Labels, tt.want.ObjectMeta.Labels) { + t.Errorf("getTaskSpec() labels = %v, want %v", got.ObjectMeta.Labels, tt.want.ObjectMeta.Labels) + } + if !reflect.DeepEqual(got.Spec, tt.want.Spec) { + t.Errorf("getTaskSpec() spec = %v, want %v", got.Spec, tt.want.Spec) + } + } + }) + } +} + +func Test_parseIndex(t *testing.T) { + type args struct { + pod *corev1.Pod + } + tests := []struct { + name string + args args + want int + wantErr bool + }{ + { + name: "from label", + args: args{ + pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{LabelBatchSandboxPodIndexKey: "1"}, + Name: "sbx-0"}}, + }, + want: 1, + }, + { + name: "from name", + args: args{ + pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "sbx-0"}}, + }, + want: 0, + }, + { + name: "invalid name", + args: args{ + pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "sbx"}}, + }, + want: -1, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseIndex(tt.args.pod) + if (err != nil) != tt.wantErr { + t.Errorf("parseIndex() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("parseIndex() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/kubernetes/internal/controller/pool_controller.go b/kubernetes/internal/controller/pool_controller.go new file mode 100644 index 00000000..ff92694b --- /dev/null +++ b/kubernetes/internal/controller/pool_controller.go @@ -0,0 +1,463 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + gerrors "errors" + "fmt" + "sort" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/json" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + sandboxv1alpha1 "github.com/alibaba/OpenSandbox/sandbox-k8s/api/v1alpha1" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/utils" + controllerutils "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/utils/controller" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/utils/expectations" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/utils/fieldindex" +) + +const ( + defaultRetryTime = 5 * time.Second +) + +const ( + LabelPoolName = "sandbox.opensandbox.io/pool-name" + LabelPoolRevision = "sandbox.opensandbox.io/pool-revision" +) + +var ( + PoolScaleExpectations = expectations.NewScaleExpectations() +) + +// PoolReconciler reconciles a Pool object +type PoolReconciler struct { + client.Client + Scheme *runtime.Scheme + Recorder record.EventRecorder + Allocator Allocator +} + +// +kubebuilder:rbac:groups=sandbox.opensandbox.io,resources=pools,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=sandbox.opensandbox.io,resources=pools/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=sandbox.opensandbox.io,resources=pools/finalizers,verbs=update +// +kubebuilder:rbac:groups=sandbox.opensandbox.io,resources=batchsandboxes,verbs=get;list;watch;patch +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=pods/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=core,resources=events,verbs=get;list;watch;create;update;patch;delete + +func (r *PoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := logf.FromContext(ctx) + // Fetch the Pool instance + pool := &sandboxv1alpha1.Pool{} + if err := r.Get(ctx, req.NamespacedName, pool); err != nil { + if errors.IsNotFound(err) { + // Pool resource not found, could have been deleted + log.Info("Pool resource not found, ignoring since object must be deleted") + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request + log.Error(err, "Failed to get Pool") + return ctrl.Result{}, err + } + if !pool.DeletionTimestamp.IsZero() { + log.Info("Pool resource is being deleted, ignoring") + return ctrl.Result{}, nil + } + + // List all pods of the pool + podList := &corev1.PodList{} + if err := r.List(ctx, podList, &client.ListOptions{ + Namespace: pool.Namespace, + FieldSelector: fields.SelectorFromSet(fields.Set{fieldindex.IndexNameForOwnerRefUID: string(pool.UID)}), + }); err != nil { + log.Error(err, "Failed to list pods") + return reconcile.Result{}, err + } + pods := make([]*corev1.Pod, 0, len(podList.Items)) + for i := range podList.Items { + pod := podList.Items[i] + PoolScaleExpectations.ObserveScale(controllerutils.GetControllerKey(pool), expectations.Create, pod.Name) + if pod.DeletionTimestamp.IsZero() { + pods = append(pods, &pod) + } + } + + // List all batch sandboxes ref to the pool + batchSandboxList := &sandboxv1alpha1.BatchSandboxList{} + if err := r.List(ctx, batchSandboxList, &client.ListOptions{ + Namespace: pool.Namespace, + FieldSelector: fields.SelectorFromSet(fields.Set{fieldindex.IndexNameForPoolRef: pool.Name}), + }); err != nil { + log.Error(err, "Failed to list batch sandboxes") + return reconcile.Result{}, err + } + batchSandboxes := make([]*sandboxv1alpha1.BatchSandbox, 0, len(batchSandboxList.Items)) + for i := range batchSandboxList.Items { + batchSandbox := batchSandboxList.Items[i] + if batchSandbox.Spec.Template != nil { + continue + } + batchSandboxes = append(batchSandboxes, &batchSandbox) + } // Main reconciliation logic + return r.reconcilePool(ctx, pool, batchSandboxes, pods) +} + +// reconcilePool contains the main reconciliation logic +func (r *PoolReconciler) reconcilePool(ctx context.Context, pool *sandboxv1alpha1.Pool, batchSandboxes []*sandboxv1alpha1.BatchSandbox, pods []*corev1.Pod) (ctrl.Result, error) { + needReconcile := false + delay := time.Duration(0) + // allocate + podAllocation, idlePods, supplySandbox, err := r.scheduleSandbox(ctx, pool, batchSandboxes, pods) + if err != nil { + return ctrl.Result{}, err + } + if supplySandbox > 0 && len(idlePods) > 0 { // Some idle pods may be pending, retry schedule later. + needReconcile = true + delay = defaultRetryTime + } + if int32(len(idlePods)) >= supplySandbox { // Some pods may be pending, no need to create again. + supplySandbox = 0 + } else { + supplySandbox -= int32(len(idlePods)) + } + + // update + latestRevision, err := r.calculateRevision(pool) + if err != nil { + return ctrl.Result{}, err + } + latestIdlePods, deleteOld, supplyNew := r.updatePool(latestRevision, pods, idlePods) + + // scale + args := &scaleArgs{ + latestRevision: latestRevision, + pool: pool, + pods: pods, + allocatedCnt: int32(len(podAllocation)), + idlePods: latestIdlePods, + redundantPods: deleteOld, + supplyCnt: supplySandbox + supplyNew, + } + if err := r.scalePool(ctx, args); err != nil { + return ctrl.Result{}, err + } + + // update status + if err := r.updatePoolStatus(ctx, latestRevision, pool, pods, podAllocation); err != nil { + return ctrl.Result{}, err + } + + if needReconcile { + return ctrl.Result{RequeueAfter: delay}, nil + } + return ctrl.Result{}, nil +} + +func (r *PoolReconciler) calculateRevision(pool *sandboxv1alpha1.Pool) (string, error) { + template, err := json.Marshal(pool.Spec.Template) + if err != nil { + return "", err + } + revision := sha256.Sum256(template) + return hex.EncodeToString(revision[:8]), nil +} + +// SetupWithManager sets up the controller with the Manager. +// Todo pod deletion expectations +func (r *PoolReconciler) SetupWithManager(mgr ctrl.Manager) error { + filterBatchSandbox := predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + bsb, ok := e.Object.(*sandboxv1alpha1.BatchSandbox) + if !ok { + return false + } + return bsb.Spec.PoolRef != "" + }, + UpdateFunc: func(e event.UpdateEvent) bool { + oldObj, okOld := e.ObjectOld.(*sandboxv1alpha1.BatchSandbox) + newObj, okNew := e.ObjectNew.(*sandboxv1alpha1.BatchSandbox) + if !okOld || !okNew { + return false + } + if newObj.Spec.PoolRef == "" { + return false + } + oldVal := oldObj.Annotations[AnnoAllocReleaseKey] + newVal := newObj.Annotations[AnnoAllocReleaseKey] + if oldVal != newVal { + return true + } + if oldObj.Spec.Replicas != newObj.Spec.Replicas { + return true + } + return false + }, + DeleteFunc: func(e event.DeleteEvent) bool { + bsb, ok := e.Object.(*sandboxv1alpha1.BatchSandbox) + if !ok { + return false + } + return bsb.Spec.PoolRef != "" + }, + GenericFunc: func(e event.GenericEvent) bool { + bsb, ok := e.Object.(*sandboxv1alpha1.BatchSandbox) + if !ok { + return false + } + return bsb.Spec.PoolRef != "" + }, + } + + findPoolForBatchSandbox := func(ctx context.Context, obj client.Object) []reconcile.Request { + log := logf.FromContext(ctx) + batchSandbox, ok := obj.(*sandboxv1alpha1.BatchSandbox) + if !ok { + log.Error(nil, "Invalid object type, expected BatchSandbox") + return nil + } + return []reconcile.Request{ + { + NamespacedName: types.NamespacedName{ + Namespace: batchSandbox.Namespace, + Name: batchSandbox.Spec.PoolRef, + }, + }, + } + } + + return ctrl.NewControllerManagedBy(mgr). + For(&sandboxv1alpha1.Pool{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). + Owns(&corev1.Pod{}). + Watches( + &sandboxv1alpha1.BatchSandbox{}, + handler.EnqueueRequestsFromMapFunc(findPoolForBatchSandbox), + builder.WithPredicates(filterBatchSandbox), + ). + Named("pool"). + Complete(r) +} + +func (r *PoolReconciler) scheduleSandbox(ctx context.Context, pool *sandboxv1alpha1.Pool, batchSandboxes []*sandboxv1alpha1.BatchSandbox, pods []*corev1.Pod) (map[string]string, []string, int32, error) { + spec := &AllocSpec{ + Sandboxes: batchSandboxes, + Pool: pool, + Pods: pods, + } + status, err := r.Allocator.Schedule(ctx, spec) + if err != nil { + return nil, nil, 0, err + } + idlePods := make([]string, 0) + for _, pod := range pods { + if _, ok := status.PodAllocation[pod.Name]; !ok { + idlePods = append(idlePods, pod.Name) + } + } + return status.PodAllocation, idlePods, status.PodSupplement, nil +} + +func (r *PoolReconciler) updatePool(latestRevision string, pods []*corev1.Pod, idlePods []string) ([]string, []string, int32) { + podMap := make(map[string]*corev1.Pod) + for _, pod := range pods { + podMap[pod.Name] = pod + } + latestIdlePods := make([]string, 0) + deleteOld := make([]string, 0) + supplyNew := int32(0) + + for _, name := range idlePods { + pod, ok := podMap[name] + if !ok { + continue + } + revision := pod.Labels[LabelPoolRevision] + if revision == latestRevision { + latestIdlePods = append(latestIdlePods, name) + } else { + // Rolling: (1) delete old idle pods (2) create latest pods + deleteOld = append(deleteOld, name) + supplyNew++ + } + } + return latestIdlePods, deleteOld, supplyNew +} + +type scaleArgs struct { + latestRevision string + pool *sandboxv1alpha1.Pool + pods []*corev1.Pod + allocatedCnt int32 + supplyCnt int32 // to create + idlePods []string + redundantPods []string +} + +func (r *PoolReconciler) scalePool(ctx context.Context, args *scaleArgs) error { + log := logf.FromContext(ctx) + errs := make([]error, 0) + pool := args.pool + pods := args.pods + if satisfied, unsatisfiedDuration, dirtyPods := PoolScaleExpectations.SatisfiedExpectations(controllerutils.GetControllerKey(pool)); !satisfied { + log.Info("Pool scale is not ready, requeue", "unsatisfiedDuration", unsatisfiedDuration, "dirtyPods", dirtyPods) + return fmt.Errorf("pool scale is not ready, %v", pool.Name) + } + totalCnt := int32(len(args.pods)) + allocatedCnt := args.allocatedCnt + supplyCnt := args.supplyCnt + redundantPods := args.redundantPods + bufferCnt := totalCnt - allocatedCnt + + // Calculate desired buffer cnt. + desiredBufferCnt := bufferCnt + if bufferCnt < pool.Spec.CapacitySpec.BufferMin || bufferCnt > pool.Spec.CapacitySpec.BufferMax { + desiredBufferCnt = (pool.Spec.CapacitySpec.BufferMin + pool.Spec.CapacitySpec.BufferMax) / 2 + } + + // Calculate desired total cnt. + desiredTotalCnt := allocatedCnt + supplyCnt + desiredBufferCnt + if desiredTotalCnt < pool.Spec.CapacitySpec.PoolMin { + desiredTotalCnt = pool.Spec.CapacitySpec.PoolMin + } else if desiredTotalCnt > pool.Spec.CapacitySpec.PoolMax { + desiredTotalCnt = pool.Spec.CapacitySpec.PoolMax + } + + if desiredTotalCnt > totalCnt { // Need to create pod + createCnt := desiredTotalCnt - totalCnt + for i := int32(0); i < createCnt; i++ { + if err := r.createPoolPod(ctx, pool, args.latestRevision); err != nil { + log.Error(err, "Failed to create pool pod") + errs = append(errs, err) + } + } + } else if desiredTotalCnt < totalCnt || len(redundantPods) > 0 { // Need to delete pod + scaleIn := int32(0) + if desiredTotalCnt < totalCnt { + scaleIn = totalCnt - desiredTotalCnt + } + podsToDelete := r.pickPodsToDelete(pods, args.idlePods, args.redundantPods, scaleIn) + for _, pod := range podsToDelete { + if err := r.Delete(ctx, pod); err != nil { + log.Error(err, "Failed to delete pool pod") + errs = append(errs, err) + } + } + } + return gerrors.Join(errs...) +} + +func (r *PoolReconciler) updatePoolStatus(ctx context.Context, latestRevision string, pool *sandboxv1alpha1.Pool, pods []*corev1.Pod, podAllocation map[string]string) error { + oldStatus := pool.Status.DeepCopy() + availableCnt := int32(0) + for _, pod := range pods { + if _, ok := podAllocation[pod.Name]; ok { + continue + } + if pod.Status.Phase != corev1.PodRunning { + continue + } + availableCnt++ + } + pool.Status.ObservedGeneration = pool.Generation + pool.Status.Total = int32(len(pods)) + pool.Status.Allocated = int32(len(podAllocation)) + pool.Status.Available = availableCnt + pool.Status.Revision = latestRevision + if equality.Semantic.DeepEqual(oldStatus, pool.Status) { + return nil + } + if err := r.Status().Update(ctx, pool); err != nil { + return err + } + return nil +} + +func (r *PoolReconciler) pickPodsToDelete(pods []*corev1.Pod, idlePodNames []string, redundantPodNames []string, scaleIn int32) []*corev1.Pod { + var idlePods []*corev1.Pod + podMap := make(map[string]*corev1.Pod) + for _, pod := range pods { + podMap[pod.Name] = pod + } + for _, name := range idlePodNames { + pod, ok := podMap[name] + if !ok { + continue + } + idlePods = append(idlePods, pod) + } + + sort.Slice(idlePods, func(i, j int) bool { + return idlePods[i].CreationTimestamp.Before(&idlePods[j].CreationTimestamp) + }) + var podsToDelete []*corev1.Pod + for _, name := range redundantPodNames { // delete pod from pool update + pod, ok := podMap[name] + if !ok { + continue + } + podsToDelete = append(podsToDelete, pod) + } + for _, pod := range idlePods { // delete pod from pool scale + if scaleIn <= 0 { + break + } + if pod.DeletionTimestamp == nil { + podsToDelete = append(podsToDelete, pod) + } + scaleIn -= 1 + } + return podsToDelete +} + +func (r *PoolReconciler) createPoolPod(ctx context.Context, pool *sandboxv1alpha1.Pool, latestRevision string) error { + pod, err := utils.GetPodFromTemplate(pool.Spec.Template, pool, metav1.NewControllerRef(pool, sandboxv1alpha1.SchemeBuilder.GroupVersion.WithKind("Pool"))) + if err != nil { + return err + } + pod.Namespace = pool.Namespace + pod.Name = "" + pod.GenerateName = pool.Name + "-" + pod.Labels[LabelPoolName] = pool.Name + pod.Labels[LabelPoolRevision] = latestRevision + if err := ctrl.SetControllerReference(pool, pod, r.Scheme); err != nil { + return err + } + if err := r.Create(ctx, pod); err != nil { + r.Recorder.Eventf(pool, corev1.EventTypeWarning, "FailedCreate", "Failed to create pool pod: %v", err) + return err + } + PoolScaleExpectations.ExpectScale(controllerutils.GetControllerKey(pool), expectations.Create, pod.Name) + r.Recorder.Eventf(pool, corev1.EventTypeNormal, "SuccessfulCreate", "Created pool pod: %v", pod.Name) + return nil +} diff --git a/kubernetes/internal/controller/pool_controller_test.go b/kubernetes/internal/controller/pool_controller_test.go new file mode 100644 index 00000000..ae9c3bf1 --- /dev/null +++ b/kubernetes/internal/controller/pool_controller_test.go @@ -0,0 +1,523 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "context" + "encoding/json" + "time" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/client-go/util/retry" + "k8s.io/utils/ptr" + kclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/utils/fieldindex" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + sandboxv1alpha1 "github.com/alibaba/OpenSandbox/sandbox-k8s/api/v1alpha1" +) + +var _ = Describe("Pool scale", func() { + var ( + timeout = 10 * time.Second + interval = 1 * time.Second + ) + Context("When reconciling a resource", func() { + const resourceName = "pool-scale-test" + + ctx := context.Background() + + typeNamespacedName := types.NamespacedName{ + Name: resourceName, + Namespace: "default", + } + BeforeEach(func() { + By("creating the custom resource for the Kind Pool") + typeNamespacedName.Name = resourceName + "-" + rand.String(8) + resource := &sandboxv1alpha1.Pool{ + ObjectMeta: metav1.ObjectMeta{ + Name: typeNamespacedName.Name, + Namespace: typeNamespacedName.Namespace, + }, + Spec: sandboxv1alpha1.PoolSpec{ + Template: &v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "main", + Image: "example.com", + }, + }, + }, + }, + CapacitySpec: sandboxv1alpha1.CapacitySpec{ + PoolMin: 0, + PoolMax: 2, + BufferMin: 1, + BufferMax: 1, + }, + }, + } + Expect(k8sClient.Create(ctx, resource)).To(Succeed()) + Eventually(func(g Gomega) { + pool := &sandboxv1alpha1.Pool{} + err := k8sClient.Get(ctx, typeNamespacedName, pool) + g.Expect(err).NotTo(HaveOccurred()) + cnt := min(pool.Spec.CapacitySpec.PoolMax, pool.Spec.CapacitySpec.BufferMin) + g.Expect(pool.Status.ObservedGeneration).To(Equal(pool.Generation)) + g.Expect(pool.Status.Total).To(Equal(cnt)) + }, timeout, interval).Should(Succeed()) + }) + + AfterEach(func() { + resource := &sandboxv1alpha1.Pool{} + err := k8sClient.Get(ctx, typeNamespacedName, resource) + if err != nil { + if !errors.IsNotFound(err) { + Expect(err).NotTo(HaveOccurred()) + } else { + By("The specific resource instance Pool already deleted") + return + } + } + By("Cleanup the specific resource instance Pool") + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + }) + It("should successfully update pool status", func() { + pool := &sandboxv1alpha1.Pool{} + Eventually(func(g Gomega) { + if err := k8sClient.Get(ctx, typeNamespacedName, pool); err != nil { + return + } + cnt := min(pool.Spec.CapacitySpec.PoolMax, pool.Spec.CapacitySpec.BufferMin) + g.Expect(pool.Status.ObservedGeneration).To(Equal(pool.Generation)) + g.Expect(pool.Status.Total).To(Equal(cnt)) + }, timeout, interval).Should(Succeed()) + }) + It("should successfully scale out pool buffer size", func() { + pool := &sandboxv1alpha1.Pool{} + Expect(k8sClient.Get(ctx, typeNamespacedName, pool)).To(Succeed()) + pool.Spec.CapacitySpec.BufferMin = 2 + pool.Spec.CapacitySpec.BufferMax = 2 + Expect(k8sClient.Update(ctx, pool)).To(Succeed()) + Eventually(func(g Gomega) { + if err := k8sClient.Get(ctx, typeNamespacedName, pool); err != nil { + return + } + cnt := int32(2) + g.Expect(pool.Status.ObservedGeneration).To(Equal(pool.Generation)) + g.Expect(pool.Status.Total).To(Equal(cnt)) + }, timeout, interval).Should(Succeed()) + }) + It("should successfully scale out buffer limit by pool max", func() { + pool := &sandboxv1alpha1.Pool{} + Expect(k8sClient.Get(ctx, typeNamespacedName, pool)).To(Succeed()) + pool.Spec.CapacitySpec.PoolMax = 2 + pool.Spec.CapacitySpec.BufferMin = 3 + pool.Spec.CapacitySpec.BufferMax = 3 + Expect(k8sClient.Update(ctx, pool)).To(Succeed()) + Eventually(func(g Gomega) { + if err := k8sClient.Get(ctx, typeNamespacedName, pool); err != nil { + return + } + cnt := int32(2) + g.Expect(pool.Status.ObservedGeneration).To(Equal(pool.Generation)) + g.Expect(pool.Status.Total).To(Equal(cnt)) + }, timeout, interval).Should(Succeed()) + }) + It("should successfully scale in pool buffer size", func() { + pool := &sandboxv1alpha1.Pool{} + Expect(k8sClient.Get(ctx, typeNamespacedName, pool)).To(Succeed()) + pool.Spec.CapacitySpec.BufferMin = 0 + pool.Spec.CapacitySpec.BufferMax = 0 + Expect(k8sClient.Update(ctx, pool)).To(Succeed()) + Eventually(func(g Gomega) { + pool := &sandboxv1alpha1.Pool{} + if err := k8sClient.Get(ctx, typeNamespacedName, pool); err != nil { + return + } + cnt := int32(0) + g.Expect(pool.Status.ObservedGeneration).To(Equal(pool.Generation)) + g.Expect(pool.Status.Total).To(Equal(cnt)) + }, timeout, interval).Should(Succeed()) + }) + It("should successfully scale in buffer limit by pool min", func() { + pool := &sandboxv1alpha1.Pool{} + Expect(k8sClient.Get(ctx, typeNamespacedName, pool)).To(Succeed()) + pool.Spec.CapacitySpec.PoolMax = 1 + pool.Spec.CapacitySpec.PoolMin = 1 + pool.Spec.CapacitySpec.BufferMin = 0 + pool.Spec.CapacitySpec.BufferMax = 0 + Expect(k8sClient.Update(ctx, pool)).To(Succeed()) + Eventually(func(g Gomega) { + if err := k8sClient.Get(ctx, typeNamespacedName, pool); err != nil { + return + } + cnt := int32(1) + g.Expect(pool.Status.ObservedGeneration).To(Equal(pool.Generation)) + g.Expect(pool.Status.Total).To(Equal(cnt)) + }, timeout, interval).Should(Succeed()) + }) + }) +}) + +var _ = Describe("Pool update", func() { + var ( + timeout = 10 * time.Second + interval = 1 * time.Second + ) + Context("When reconciling a resource", func() { + const resourceName = "pool-update-test" + + ctx := context.Background() + + typeNamespacedName := types.NamespacedName{ + Name: resourceName, + Namespace: "default", + } + + BeforeEach(func() { + By("creating the custom resource for the Kind Pool") + typeNamespacedName.Name = resourceName + "-" + rand.String(8) + resource := &sandboxv1alpha1.Pool{ + ObjectMeta: metav1.ObjectMeta{ + Name: typeNamespacedName.Name, + Namespace: typeNamespacedName.Namespace, + }, + Spec: sandboxv1alpha1.PoolSpec{ + Template: &v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "main", + Image: "example.com", + }, + }, + }, + }, + CapacitySpec: sandboxv1alpha1.CapacitySpec{ + PoolMin: 0, + PoolMax: 2, + BufferMin: 1, + BufferMax: 1, + }, + }, + } + Expect(k8sClient.Create(ctx, resource)).To(Succeed()) + Eventually(func(g Gomega) { + pool := &sandboxv1alpha1.Pool{} + err := k8sClient.Get(ctx, typeNamespacedName, pool) + g.Expect(err).NotTo(HaveOccurred()) + cnt := min(pool.Spec.CapacitySpec.PoolMax, pool.Spec.CapacitySpec.BufferMin) + g.Expect(pool.Status.ObservedGeneration).To(Equal(pool.Generation)) + g.Expect(pool.Status.Total).To(Equal(cnt)) + }, timeout, interval).Should(Succeed()) + pool := &sandboxv1alpha1.Pool{} + err := k8sClient.Get(ctx, typeNamespacedName, pool) + Expect(err).NotTo(HaveOccurred()) + pods := &v1.PodList{} + Expect(k8sClient.List(ctx, pods, &kclient.ListOptions{ + Namespace: typeNamespacedName.Namespace, + FieldSelector: fields.SelectorFromSet(fields.Set{fieldindex.IndexNameForOwnerRefUID: string(pool.UID)}), + })).To(Succeed()) + // Mock pod running + for _, pod := range pods.Items { + pod.Status.Phase = v1.PodRunning + Expect(k8sClient.Status().Update(ctx, &pod)).To(Succeed()) + } + }) + + AfterEach(func() { + resource := &sandboxv1alpha1.Pool{} + err := k8sClient.Get(ctx, typeNamespacedName, resource) + if err != nil { + if !errors.IsNotFound(err) { + Expect(err).NotTo(HaveOccurred()) + } else { + By("The specific resource instance Pool already deleted") + return + } + } + By("Cleanup the specific resource instance Pool") + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + }) + It("should successfully update pool revision", func() { + var oldRevision string + Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { + pool := &sandboxv1alpha1.Pool{} + if err := k8sClient.Get(ctx, typeNamespacedName, pool); err != nil { + return err + } + if oldRevision == "" { + oldRevision = pool.Status.Revision + } + pool.Spec.Template.Labels = map[string]string{ + "test.pool.update": "v1", + } + return k8sClient.Update(ctx, pool) + })).Should(Succeed()) + Eventually(func(g Gomega) { + pool := &sandboxv1alpha1.Pool{} + Expect(k8sClient.Get(ctx, typeNamespacedName, pool)).To(Succeed()) + cnt := int32(1) + g.Expect(pool.Status.Revision).NotTo(Equal(oldRevision)) + g.Expect(pool.Status.Total).To(Equal(cnt)) + }, timeout, interval).Should(Succeed()) + }) + It("should successfully update pool with allocated pod", func() { + pool := &sandboxv1alpha1.Pool{} + sbxNamespaceName := types.NamespacedName{ + Name: "sandbox-test-" + rand.String(8), + Namespace: typeNamespacedName.Namespace, + } + sandbox := &sandboxv1alpha1.BatchSandbox{ + ObjectMeta: metav1.ObjectMeta{ + Name: sbxNamespaceName.Name, + Namespace: sbxNamespaceName.Namespace, + }, + Spec: sandboxv1alpha1.BatchSandboxSpec{ + PoolRef: typeNamespacedName.Name, + }, + } + Expect(k8sClient.Create(ctx, sandbox)).To(Succeed()) + // wait allocation + Eventually(func(g Gomega) { + g.Expect(k8sClient.Get(ctx, sbxNamespaceName, sandbox)).To(Succeed()) + alloc, err := getSandboxAllocation(sandbox) + Expect(err).NotTo(HaveOccurred()) + g.Expect(alloc.Pods).NotTo(BeEmpty()) + }, timeout, interval).Should(Succeed()) + Expect(k8sClient.Get(ctx, sbxNamespaceName, sandbox)).To(Succeed()) + sbxAlloc, err := getSandboxAllocation(sandbox) + Expect(err).NotTo(HaveOccurred()) + Expect(len(sbxAlloc.Pods)).To(Equal(1)) + // check pool allocation + err = k8sClient.Get(ctx, typeNamespacedName, pool) + Expect(err).NotTo(HaveOccurred()) + allocation, err := getPoolAllocation(pool) + Expect(err).NotTo(HaveOccurred()) + Expect(len(allocation.PodAllocation)).To(Equal(1)) + Expect(allocation.PodAllocation[sbxAlloc.Pods[0]]).To(Equal(sandbox.Name)) + // update pool + Expect(k8sClient.Get(ctx, typeNamespacedName, pool)).To(Succeed()) + oldRevision := pool.Status.Revision + pool.Spec.Template.Labels = map[string]string{ + "test.pool.update": "v1", + } + Expect(k8sClient.Update(ctx, pool)).To(Succeed()) + Eventually(func(g Gomega) { + Expect(k8sClient.Get(ctx, typeNamespacedName, pool)).To(Succeed()) + cnt := int32(2) + g.Expect(pool.Status.Revision).NotTo(Equal(oldRevision)) + g.Expect(pool.Status.Total).To(Equal(cnt)) + pods := &v1.PodList{} + Expect(k8sClient.List(ctx, pods, &kclient.ListOptions{ + Namespace: typeNamespacedName.Namespace, + FieldSelector: fields.SelectorFromSet(fields.Set{fieldindex.IndexNameForOwnerRefUID: string(pool.UID)}), + })).To(Succeed()) + for _, pod := range pods.Items { + if pod.Name == sbxAlloc.Pods[0] { + g.Expect(pod.DeletionTimestamp).To(BeNil()) + g.Expect(pod.Labels[LabelPoolRevision]).To(Equal(oldRevision)) + continue + } + if pod.DeletionTimestamp != nil { + continue + } + g.Expect(pod.Labels[LabelPoolRevision]).NotTo(Equal(oldRevision)) + } + }, timeout, interval).Should(Succeed()) + Expect(k8sClient.Delete(ctx, sandbox)).To(Succeed()) + }) + }) +}) + +var _ = Describe("Pool allocate", func() { + var ( + timeout = 10 * time.Second + interval = 1 * time.Second + ) + Context("When reconciling a resource", func() { + const resourceName = "pool-allocate-test" + + ctx := context.Background() + + typeNamespacedName := types.NamespacedName{ + Name: resourceName, + Namespace: "default", + } + + BeforeEach(func() { + By("creating the custom resource for the Kind Pool") + typeNamespacedName.Name = resourceName + "-" + rand.String(8) + resource := &sandboxv1alpha1.Pool{ + ObjectMeta: metav1.ObjectMeta{ + Name: typeNamespacedName.Name, + Namespace: typeNamespacedName.Namespace, + }, + Spec: sandboxv1alpha1.PoolSpec{ + Template: &v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "main", + Image: "example.com", + }, + }, + }, + }, + CapacitySpec: sandboxv1alpha1.CapacitySpec{ + PoolMin: 0, + PoolMax: 2, + BufferMin: 1, + BufferMax: 1, + }, + }, + } + Expect(k8sClient.Create(ctx, resource)).To(Succeed()) + Eventually(func(g Gomega) { + pool := &sandboxv1alpha1.Pool{} + err := k8sClient.Get(ctx, typeNamespacedName, pool) + g.Expect(err).NotTo(HaveOccurred()) + cnt := min(pool.Spec.CapacitySpec.PoolMax, pool.Spec.CapacitySpec.BufferMin) + g.Expect(pool.Status.ObservedGeneration).To(Equal(pool.Generation)) + g.Expect(pool.Status.Total).To(Equal(cnt)) + }, timeout, interval).Should(Succeed()) + pool := &sandboxv1alpha1.Pool{} + err := k8sClient.Get(ctx, typeNamespacedName, pool) + Expect(err).NotTo(HaveOccurred()) + pods := &v1.PodList{} + Expect(k8sClient.List(ctx, pods, &kclient.ListOptions{ + Namespace: typeNamespacedName.Namespace, + FieldSelector: fields.SelectorFromSet(fields.Set{fieldindex.IndexNameForOwnerRefUID: string(pool.UID)}), + })).To(Succeed()) + // Mock pod running + for _, pod := range pods.Items { + pod.Status.Phase = v1.PodRunning + Expect(k8sClient.Status().Update(ctx, &pod)).To(Succeed()) + } + }) + + AfterEach(func() { + resource := &sandboxv1alpha1.Pool{} + err := k8sClient.Get(ctx, typeNamespacedName, resource) + if err != nil { + if !errors.IsNotFound(err) { + Expect(err).NotTo(HaveOccurred()) + } else { + By("The specific resource instance Pool already deleted") + return + } + } + By("Cleanup the specific resource instance Pool") + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + }) + It("should successfully allocate pool pod to batch sandbox and release", func() { + pool := &sandboxv1alpha1.Pool{} + bsbxNamespaceName := types.NamespacedName{ + Name: "batch-sandbox-test-" + rand.String(8), + Namespace: typeNamespacedName.Namespace, + } + batchSandbox := &sandboxv1alpha1.BatchSandbox{ + ObjectMeta: metav1.ObjectMeta{ + Name: bsbxNamespaceName.Name, + Namespace: bsbxNamespaceName.Namespace, + }, + Spec: sandboxv1alpha1.BatchSandboxSpec{ + Replicas: ptr.To(int32(1)), + PoolRef: typeNamespacedName.Name, + }, + } + Expect(k8sClient.Create(ctx, batchSandbox)).To(Succeed()) + // wait allocation + Eventually(func(g Gomega) { + g.Expect(k8sClient.Get(ctx, bsbxNamespaceName, batchSandbox)).To(Succeed()) + alloc, err := getSandboxAllocation(batchSandbox) + Expect(err).NotTo(HaveOccurred()) + g.Expect(alloc.Pods).NotTo(BeEmpty()) + }, timeout, interval).Should(Succeed()) + Expect(k8sClient.Get(ctx, bsbxNamespaceName, batchSandbox)).To(Succeed()) + sbxAlloc, err := getSandboxAllocation(batchSandbox) + Expect(err).NotTo(HaveOccurred()) + Expect(len(sbxAlloc.Pods)).To(Equal(1)) + // check pool allocation + err = k8sClient.Get(ctx, typeNamespacedName, pool) + Expect(err).NotTo(HaveOccurred()) + allocation, err := getPoolAllocation(pool) + Expect(err).NotTo(HaveOccurred()) + Expect(len(allocation.PodAllocation)).To(Equal(1)) + Expect(allocation.PodAllocation[sbxAlloc.Pods[0]]).To(Equal(batchSandbox.Name)) + // release + release := AllocationRelease{ + Pods: sbxAlloc.Pods, + } + js, err := json.Marshal(release) + Expect(err).NotTo(HaveOccurred()) + batchSandbox.Annotations[AnnoAllocReleaseKey] = string(js) + err = k8sClient.Update(ctx, batchSandbox) + Expect(err).NotTo(HaveOccurred()) + // wait release + Eventually(func(g Gomega) { + err = k8sClient.Get(ctx, typeNamespacedName, pool) + Expect(err).NotTo(HaveOccurred()) + allocation, err = getPoolAllocation(pool) + Expect(err).NotTo(HaveOccurred()) + g.Expect(len(allocation.PodAllocation)).To(Equal(0)) + }, timeout, interval).Should(Succeed()) + Expect(k8sClient.Delete(ctx, batchSandbox)).To(Succeed()) + }) + }) +}) + +func getSandboxAllocation(obj kclient.Object) (*SandboxAllocation, error) { + allocation := &SandboxAllocation{} + anno := obj.GetAnnotations() + if anno == nil { + return allocation, nil + } + str, ok := anno[AnnoAllocStatusKey] + if !ok { + return allocation, nil + } + err := json.Unmarshal([]byte(str), allocation) + if err != nil { + return nil, err + } + return allocation, nil +} + +func getPoolAllocation(pool *sandboxv1alpha1.Pool) (*PoolAllocation, error) { + allocation := &PoolAllocation{} + anno := pool.GetAnnotations() + if anno == nil { + return allocation, nil + } + str, ok := anno[AnnoPoolAllocStatusKey] + if !ok { + return allocation, nil + } + err := json.Unmarshal([]byte(str), allocation) + if err != nil { + return nil, err + } + return allocation, nil +} diff --git a/kubernetes/internal/controller/suite_test.go b/kubernetes/internal/controller/suite_test.go new file mode 100644 index 00000000..6249a391 --- /dev/null +++ b/kubernetes/internal/controller/suite_test.go @@ -0,0 +1,160 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "context" + "os" + "path/filepath" + "sync" + "testing" + + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + sandboxv1alpha1 "github.com/alibaba/OpenSandbox/sandbox-k8s/api/v1alpha1" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/utils/fieldindex" + // +kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var ( + ctx context.Context + cancel context.CancelFunc + testEnv *envtest.Environment + cfg *rest.Config + k8sClient client.Client + k8sManager ctrl.Manager + mgrStopped *sync.WaitGroup +) + +func TestControllers(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Controller Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + var err error + err = sandboxv1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + // +kubebuilder:scaffold:scheme + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + + // Retrieve the first found binary directory to allow running tests from IDEs + if getFirstFoundEnvTestBinaryDir() != "" { + testEnv.BinaryAssetsDirectory = getFirstFoundEnvTestBinaryDir() + } + + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + k8sManager, err = ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme.Scheme, + }) + Expect(err).ToNot(HaveOccurred()) + By("register field index") + Expect(fieldindex.RegisterFieldIndexes(k8sManager.GetCache())).Should(Succeed(), "failed to register fieldindex") + + By("setup reconciler") + Expect((&BatchSandboxReconciler{ + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + Recorder: k8sManager.GetEventRecorderFor("test-batch-sandbox-controller"), + }).SetupWithManager(k8sManager)).Should(Succeed()) + Expect((&PoolReconciler{ + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + Recorder: k8sManager.GetEventRecorderFor("test-pool-controller"), + Allocator: NewDefaultAllocator(k8sManager.GetClient()), + }).SetupWithManager(k8sManager)).Should(Succeed()) + // TODO more reconciler goes HERE + + By("try to start manager") + mgrStopped = startTestManager(ctx, k8sManager) + + k8sManager.GetCache().WaitForCacheSync(ctx) + By("waiting for manager cache synced") + + k8sClient = k8sManager.GetClient() + Expect(k8sClient).NotTo(BeNil()) +}) + +func startTestManager(ctx context.Context, mgr manager.Manager) *sync.WaitGroup { + wg := &sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + Expect(mgr.Start(ctx)).Should(Succeed(), "failed to start manager") + }() + return wg +} + +var _ = AfterSuite(func() { + By("tearing down the test environment") + cancel() + if mgrStopped != nil { + By("waiting manager exit") + mgrStopped.Wait() + } + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) + +// getFirstFoundEnvTestBinaryDir locates the first binary in the specified path. +// ENVTEST-based tests depend on specific binaries, usually located in paths set by +// controller-runtime. When running tests directly (e.g., via an IDE) without using +// Makefile targets, the 'BinaryAssetsDirectory' must be explicitly configured. +// +// This function streamlines the process by finding the required binaries, similar to +// setting the 'KUBEBUILDER_ASSETS' environment variable. To ensure the binaries are +// properly set up, run 'make setup-envtest' beforehand. +func getFirstFoundEnvTestBinaryDir() string { + basePath := filepath.Join("..", "..", "bin", "k8s") + entries, err := os.ReadDir(basePath) + if err != nil { + logf.Log.Error(err, "Failed to read directory", "path", basePath) + return "" + } + for _, entry := range entries { + if entry.IsDir() { + return filepath.Join(basePath, entry.Name()) + } + } + return "" +} diff --git a/kubernetes/internal/scheduler/default_scheduler.go b/kubernetes/internal/scheduler/default_scheduler.go new file mode 100644 index 00000000..ea3b8e38 --- /dev/null +++ b/kubernetes/internal/scheduler/default_scheduler.go @@ -0,0 +1,402 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scheduler + +import ( + "context" + "fmt" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" + "k8s.io/utils/ptr" + + "github.com/alibaba/OpenSandbox/sandbox-k8s/api/v1alpha1" + sandboxv1alpha1 "github.com/alibaba/OpenSandbox/sandbox-k8s/api/v1alpha1" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/utils" + api "github.com/alibaba/OpenSandbox/sandbox-k8s/pkg/task-executor" +) + +var _ Task = &taskNode{} + +var ( + timeNow = func() time.Time { + return time.Now() + } +) + +type taskNode struct { + metav1.ObjectMeta + Spec v1alpha1.TaskSpec + + // status + Status *api.Task + IP string + PodName string + + // collect from endpoints + tState TaskState + tStateLastTransTime *time.Time + + // inner sch state + sStateLastTransTime *time.Time + sState string +} + +func (t *taskNode) GetPodName() string { + return t.PodName +} + +func (t *taskNode) GetState() TaskState { + return t.tState +} + +func (t *taskNode) IsResourceReleased() bool { + return t.sState == stateReleased +} + +func (t *taskNode) isTaskCompleted() bool { + return t.tState == SucceedTaskState || t.tState == FailedTaskState +} + +func (t *taskNode) isTaskDeleted() bool { + return t.Status == nil +} + +func (t *taskNode) transSchState(to string) { + if t.sState == to { + return + } + from := t.sState + t.sState = to + var lat time.Duration + now := timeNow() + if t.sStateLastTransTime != nil { + lat = now.Sub(*t.sStateLastTransTime) + } + t.sStateLastTransTime = ptr.To[time.Time](now) + klog.Infof("task node %s trans sch state %s -> %s, latency=%dms", klog.KObj(t), from, to, lat.Milliseconds()) +} + +func (t *taskNode) transTaskState(to TaskState) { + if t.tState == to { + return + } + from := t.tState + t.tState = to + var lat time.Duration + now := timeNow() + if t.tStateLastTransTime != nil { + lat = now.Sub(*t.tStateLastTransTime) + } + t.tStateLastTransTime = ptr.To[time.Time](now) + klog.Infof("task node %s trans task state %s -> %s, latency=%dms", klog.KObj(t), from, to, lat.Milliseconds()) +} + +const ( + // FSM: TaskNode Sch State Machine + /* + $start --> pending + + pending -- "when task is assigned to Pod" --> assigned + pending -- "when BatchSandbox's deletion timestamp != 0" --> released + + assigned -- "when BatchSandbox's deletion timestamp != 0" --> releasing + assigned -- "when task state is SUCCEED && policy is allowed" --> releasing + assigned -- "when task state is FAILED && policy is allowed" --> releasing + assigned -- "set Task" + + releasing -- "when endpoint returns nil task or endpoint lost too many times (e.g., force-deleted), endpoint is nil(unassigned)" --> released + + released --> $end + */ + //statePending = "pending", endpoint is empty means pending, otherwise means assigned + //stateAssigned = "assigned" + stateReleasing = "releasing" + stateReleased = "released" + stateUnknown = "unknown" +) + +type taskClient interface { + Set(ctx context.Context, task *api.Task) (*api.Task, error) + Get(ctx context.Context) (*api.Task, error) +} + +const ( + defaultTimeout time.Duration = 3 * time.Second + defaultTaskPort = "5758" + defaultSchConcurrency int = 10 +) + +func newTaskClient(ip string) taskClient { + return api.NewClient(fmtEndpoint(ip)) +} + +func fmtEndpoint(podIP string) string { + return fmt.Sprintf("http://%s:%s", podIP, defaultTaskPort) +} + +type defaultTaskScheduler struct { + freePods []*corev1.Pod + allPods []*corev1.Pod + + taskNodes []*taskNode + taskNodeByNameIndex map[string]*taskNode + + maxConcurrency int + once sync.Once + + taskStatusCollector taskStatusCollector + taskClientCreator taskClientCreator + resPolicyWhenTaskComplete sandboxv1alpha1.TaskResourcePolicy + name string +} + +func newTaskScheduler(name string, tasks []*sandboxv1alpha1.Task, pods []*corev1.Pod, resPolicyWhenTaskComplete sandboxv1alpha1.TaskResourcePolicy) (*defaultTaskScheduler, error) { + sch := &defaultTaskScheduler{ + allPods: pods, + maxConcurrency: defaultSchConcurrency, + taskClientCreator: newTaskClient, + taskStatusCollector: newTaskStatusCollector(newTaskClient), + resPolicyWhenTaskComplete: resPolicyWhenTaskComplete, + name: name, + } + taskNodes, err := initTaskNodes(tasks) + if err != nil { + return nil, fmt.Errorf("scheduler: failed to init task node err %w", err) + } + sch.taskNodes = taskNodes + sch.taskNodeByNameIndex = indexByName(taskNodes) + klog.Infof("task scheduler %s successfully init task nodes, size=%d", name, len(taskNodes)) + // TODO: Optimization – skip recovery for a brand-new scheduler. + // Recovery is unnecessary in this case and incurs significant overhead. + if err := sch.recover(); err != nil { + return nil, fmt.Errorf("scheduler: failed to recover, err %w", err) + } + klog.Infof("task scheduler %s successfully recover", name) + return sch, nil +} + +func indexByName(taskNodes []*taskNode) map[string]*taskNode { + ret := make(map[string]*taskNode, len(taskNodes)) + for i := range taskNodes { + ret[taskNodes[i].Name] = taskNodes[i] + } + return ret +} + +func (sch *defaultTaskScheduler) Schedule() error { + sch.refreshFreePods() + sch.collectTaskStatus(sch.taskNodes) + return sch.scheduleTaskNodes() +} + +func (sch *defaultTaskScheduler) UpdatePods(pods []*corev1.Pod) { + sch.allPods = pods +} + +func (sch *defaultTaskScheduler) ListTask() []Task { + ret := make([]Task, len(sch.taskNodes), len(sch.taskNodes)) + for i := range sch.taskNodes { + ret[i] = sch.taskNodes[i] + } + return ret +} + +func (sch *defaultTaskScheduler) StopTask() []Task { + deletedTask := make([]Task, len(sch.taskNodes), len(sch.taskNodes)) + for i := range sch.taskNodes { + if sch.taskNodes[i].DeletionTimestamp != nil { + continue + } + sch.taskNodes[i].DeletionTimestamp = &metav1.Time{Time: timeNow()} + deletedTask[i] = sch.taskNodes[i] + } + return deletedTask +} + +func initTaskNodes(tasks []*sandboxv1alpha1.Task) ([]*taskNode, error) { + size := len(tasks) + taskNodes := make([]*taskNode, size) + for idx := 0; idx < size; idx++ { + task := tasks[idx] + tNode := &taskNode{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: task.Namespace, + Name: task.Name, + Labels: task.Labels, + Annotations: task.Annotations, + }, + Spec: task.Spec, + } + taskNodes[idx] = tNode + } + return taskNodes, nil +} + +// collectTaskStatus from Pod via endpoint +func (sch *defaultTaskScheduler) collectTaskStatus(taskNodes []*taskNode) { + ips := []string{} + for _, tNode := range taskNodes { + // unassigned no need to collect task status + if tNode.IP == "" { + continue + } + ips = append(ips, tNode.IP) + } + if len(ips) == 0 { + return + } + tasks := sch.taskStatusCollector.Collect(context.Background(), ips) + for _, tNode := range taskNodes { + task, ok := tasks[tNode.IP] + tNode.Status = task + if ok && task != nil { + tNode.transTaskState(parseTaskState(&task.Status)) + } + } +} + +func parseTaskState(taskStatus *v1alpha1.TaskStatus) TaskState { + if taskStatus.State.Running != nil { + return RunningTaskState + } else if taskStatus.State.Terminated != nil { + if taskStatus.State.Terminated.ExitCode == 0 { + return SucceedTaskState + } else { + return FailedTaskState + } + } + return UnknownTaskState +} + +func (sch *defaultTaskScheduler) scheduleTaskNodes() error { + sch.freePods = assignTaskNodes(sch.taskNodes, sch.freePods) + semaphore := make(chan struct{}, sch.maxConcurrency) + var wg sync.WaitGroup + for idx := range sch.taskNodes { + tNode := sch.taskNodes[idx] + semaphore <- struct{}{} + wg.Add(1) + go func(node *taskNode) { + defer func() { + <-semaphore + wg.Done() + }() + scheduleSingleTaskNode(node, sch.taskClientCreator, sch.resPolicyWhenTaskComplete) + }(tNode) + } + wg.Wait() + return nil +} + +// refreshFreePods updates the freePods slice based on allPods and currently assigned pods +// This ensures that each pod is only assigned to one taskNode +// Only pods with IP addresses are considered free for assignment +func (sch *defaultTaskScheduler) refreshFreePods() { + // Create a map of assigned pod names for quick lookup + assignedPods := make(map[string]bool, len(sch.allPods)/2) + for _, tNode := range sch.taskNodes { + if tNode.IP != "" && tNode.PodName != "" { + assignedPods[tNode.PodName] = true + } + } + // Rebuild freePods list with only unassigned pods that have IP addresses + sch.freePods = make([]*corev1.Pod, 0, len(sch.allPods)/2) + for _, pod := range sch.allPods { + // Only consider pods with IP addresses as free for assignment + if !assignedPods[pod.Name] && pod.Status.PodIP != "" { + sch.freePods = append(sch.freePods, pod) + } + } +} + +// assignTaskNodes handles all unassigned tasks in batch +func assignTaskNodes(taskNodes []*taskNode, freePods []*corev1.Pod) []*corev1.Pod { + for _, tNode := range taskNodes { + if len(freePods) == 0 { + break + } + if tNode.IP != "" { + continue + } + pod := freePods[0] + klog.Infof("assign Pod %s:%s to task node %s", klog.KObj(pod), pod.Status.PodIP, tNode.Name) + tNode.IP = pod.Status.PodIP + tNode.PodName = pod.Name + freePods = freePods[1:] + } + return freePods +} + +func needRelease(tNode *taskNode, policy sandboxv1alpha1.TaskResourcePolicy) bool { + if tNode.DeletionTimestamp != nil { + return true + } + if policy == sandboxv1alpha1.TaskResourcePolicyRelease && tNode.isTaskCompleted() { + return true + } + return false +} + +// scheduleSingleTaskNode handles scheduling for a single task node based on its state +func scheduleSingleTaskNode(tNode *taskNode, taskClientCreator func(endpoint string) taskClient, resPolicyWhenTaskComplete sandboxv1alpha1.TaskResourcePolicy) { + // pending + if tNode.IP == "" { + if tNode.DeletionTimestamp != nil { + tNode.transSchState(stateReleased) + } + } else { + // assigned + if needRelease(tNode, resPolicyWhenTaskComplete) { + tNode.transSchState(stateReleasing) + } else { + // no need to setTask if task is completed to avoid unnecessary network overhead + if !tNode.isTaskCompleted() { + task := &api.Task{ + Name: tNode.Name, + Spec: tNode.Spec, + } + _, err := setTask(taskClientCreator(tNode.IP), task) + if err != nil { + klog.Errorf("Failed to set task %s, endpoint %s, err %v", klog.KObj(tNode), tNode.IP, err) + } + } + } + } + if tNode.sState == stateReleasing { + if tNode.isTaskDeleted() { + tNode.transSchState(stateReleased) + } else { + _, err := setTask(taskClientCreator(tNode.IP), nil) + if err != nil { + klog.Errorf("Failed to notify executor about releasing task %s, endpoint %s, err %v", klog.KObj(tNode), tNode.IP, err) + } else { + klog.Infof("Successfully to notify client to release task %s, endpoint %s", klog.KObj(tNode), tNode.IP) + } + } + } +} + +func setTask(client taskClient, task *api.Task) (*api.Task, error) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) + defer cancel() + if klog.V(3).Enabled() { + klog.Infof("client set task %s", utils.DumpJSON(task)) + } + return client.Set(ctx, task) +} diff --git a/kubernetes/internal/scheduler/default_scheduler_mock.go b/kubernetes/internal/scheduler/default_scheduler_mock.go new file mode 100644 index 00000000..59ce3d14 --- /dev/null +++ b/kubernetes/internal/scheduler/default_scheduler_mock.go @@ -0,0 +1,68 @@ +package scheduler + +// Code generated by MockGen. DO NOT EDIT. +// Source: internal/task/scheduler/default_scheduler.go + +// Package mock_scheduler is a generated GoMock package. + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + + api "github.com/alibaba/OpenSandbox/sandbox-k8s/pkg/task-executor" +) + +// MocktaskClient is a mock of taskClient interface. +type MocktaskClient struct { + ctrl *gomock.Controller + recorder *MocktaskClientMockRecorder +} + +// MocktaskClientMockRecorder is the mock recorder for MocktaskClient. +type MocktaskClientMockRecorder struct { + mock *MocktaskClient +} + +// NewMocktaskClient creates a new mock instance. +func NewMocktaskClient(ctrl *gomock.Controller) *MocktaskClient { + mock := &MocktaskClient{ctrl: ctrl} + mock.recorder = &MocktaskClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MocktaskClient) EXPECT() *MocktaskClientMockRecorder { + return m.recorder +} + +// Get mocks base method. +func (m *MocktaskClient) Get(ctx context.Context) (*api.Task, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", ctx) + ret0, _ := ret[0].(*api.Task) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MocktaskClientMockRecorder) Get(ctx interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MocktaskClient)(nil).Get), ctx) +} + +// Set mocks base method. +func (m *MocktaskClient) Set(ctx context.Context, task *api.Task) (*api.Task, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Set", ctx, task) + ret0, _ := ret[0].(*api.Task) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Set indicates an expected call of Set. +func (mr *MocktaskClientMockRecorder) Set(ctx, task interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MocktaskClient)(nil).Set), ctx, task) +} diff --git a/kubernetes/internal/scheduler/default_scheduler_test.go b/kubernetes/internal/scheduler/default_scheduler_test.go new file mode 100644 index 00000000..28767240 --- /dev/null +++ b/kubernetes/internal/scheduler/default_scheduler_test.go @@ -0,0 +1,1336 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scheduler + +import ( + "reflect" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/golang/mock/gomock" + + sandboxv1alpha1 "github.com/alibaba/OpenSandbox/sandbox-k8s/api/v1alpha1" + api "github.com/alibaba/OpenSandbox/sandbox-k8s/pkg/task-executor" +) + +func Test_scheduleSingleTaskNode(t *testing.T) { + ctl := gomock.NewController(t) + defer ctl.Finish() + mockTimeNow := time.Now() + o := timeNow + timeNow = func() time.Time { + return mockTimeNow + } + defer func() { + timeNow = o + }() + type args struct { + tNode *taskNode + taskClientCreator func(endpoint string) taskClient + } + tests := []struct { + name string + args args + expectTaskNode *taskNode + }{ + { + name: "pending task node, deleting ", + args: args{ + tNode: &taskNode{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-batch-sandbox-0", + DeletionTimestamp: &metav1.Time{Time: mockTimeNow}, + }, + }, + }, + expectTaskNode: &taskNode{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-batch-sandbox-0", + DeletionTimestamp: &metav1.Time{Time: mockTimeNow}, + }, + sState: stateReleased, + sStateLastTransTime: &mockTimeNow, + }, + }, + { + name: "assigned task node, task state=Running, deleting; setTask(nil)", + args: args{ + tNode: &taskNode{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-batch-sandbox-0", + DeletionTimestamp: &metav1.Time{Time: mockTimeNow}, + }, + IP: "1.2.3.4", + Status: &api.Task{ + Status: sandboxv1alpha1.TaskStatus{ + State: sandboxv1alpha1.TaskState{ + Running: &sandboxv1alpha1.TaskStateRunning{ + StartedAt: metav1.NewTime(mockTimeNow), + }, + }, + }, + }, + tState: RunningTaskState, + }, + taskClientCreator: func(endpoint string) taskClient { + mock := NewMocktaskClient(ctl) + mock.EXPECT().Set(gomock.Any(), nil).Return(nil, nil).Times(1) + return mock + }, + }, + expectTaskNode: &taskNode{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-batch-sandbox-0", + DeletionTimestamp: &metav1.Time{Time: mockTimeNow}, + }, + IP: "1.2.3.4", + Status: &api.Task{ + Status: sandboxv1alpha1.TaskStatus{ + State: sandboxv1alpha1.TaskState{ + Running: &sandboxv1alpha1.TaskStateRunning{ + StartedAt: metav1.NewTime(mockTimeNow), + }, + }, + }, + }, + tState: RunningTaskState, + sState: stateReleasing, + sStateLastTransTime: &mockTimeNow, + }, + }, + { + name: "assigned task node, task state=Running; setTask(task)", + args: args{ + tNode: &taskNode{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-batch-sandbox-0", + }, + IP: "1.2.3.4", + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"hello"}, + }, + }, + Status: &api.Task{ + Status: sandboxv1alpha1.TaskStatus{ + State: sandboxv1alpha1.TaskState{ + Running: &sandboxv1alpha1.TaskStateRunning{ + StartedAt: metav1.NewTime(mockTimeNow), + }, + }, + }, + }, + tState: RunningTaskState, + }, + taskClientCreator: func(endpoint string) taskClient { + mock := NewMocktaskClient(ctl) + mock.EXPECT().Set(gomock.Any(), &api.Task{ + Name: "test-batch-sandbox-0", + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"hello"}, + }, + }, + }).Return(nil, nil).Times(1) + return mock + }, + }, + expectTaskNode: &taskNode{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-batch-sandbox-0", + }, + IP: "1.2.3.4", + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"hello"}, + }, + }, + Status: &api.Task{ + Status: sandboxv1alpha1.TaskStatus{ + State: sandboxv1alpha1.TaskState{ + Running: &sandboxv1alpha1.TaskStateRunning{ + StartedAt: metav1.NewTime(mockTimeNow), + }, + }, + }, + }, + tState: RunningTaskState, + }, + }, + { + name: "assigned task node, task state=Succeed, endpoint return nil task; sState trans from releasing -> released ", + args: args{ + tNode: &taskNode{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-batch-sandbox-0", + }, + IP: "1.2.3.4", + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"hello"}, + }, + }, + Status: nil, + tState: SucceedTaskState, + sState: stateReleasing, + }, + }, + expectTaskNode: &taskNode{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-batch-sandbox-0", + }, + IP: "1.2.3.4", + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"hello"}, + }, + }, + Status: nil, + tState: SucceedTaskState, + sState: stateReleased, + sStateLastTransTime: &mockTimeNow, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheduleSingleTaskNode(tt.args.tNode, tt.args.taskClientCreator, "") + if !reflect.DeepEqual(tt.expectTaskNode, tt.args.tNode) { + t.Errorf("scheduleSingleTaskNode, want %+v, got %+v", tt.expectTaskNode, tt.args.tNode) + } + }) + } +} + +func Test_assignTaskNodes(t *testing.T) { + type args struct { + taskNodes []*taskNode + freePods []*corev1.Pod + } + tests := []struct { + name string + args args + want []*corev1.Pod + expectTaskNodes []*taskNode + }{ + { + name: "empty free pods, no assignment", + args: args{ + taskNodes: []*taskNode{ + { + ObjectMeta: v1.ObjectMeta{Name: "test-0"}, + }, + }, + }, + expectTaskNodes: []*taskNode{ + { + ObjectMeta: v1.ObjectMeta{Name: "test-0"}, + }, + }, + }, + { + name: "free pods, assign", + args: args{ + taskNodes: []*taskNode{ + { + ObjectMeta: v1.ObjectMeta{Name: "test-0"}, + }, + }, + freePods: []*corev1.Pod{ + { + ObjectMeta: v1.ObjectMeta{Name: "pod-hello-world"}, + Status: corev1.PodStatus{PodIP: "1.2.3.4"}, + }, + }, + }, + want: []*corev1.Pod{}, + expectTaskNodes: []*taskNode{ + { + ObjectMeta: v1.ObjectMeta{Name: "test-0"}, + IP: "1.2.3.4", + PodName: "pod-hello-world", + }, + }, + }, + { + name: "free pods, no unassigned task nodes, no assignment", + args: args{ + taskNodes: []*taskNode{ + { + ObjectMeta: v1.ObjectMeta{Name: "test-0"}, + IP: "4.3.2.1", + PodName: "pod-foo-bar", + }, + }, + freePods: []*corev1.Pod{ + { + ObjectMeta: v1.ObjectMeta{Name: "pod-hello-world"}, + Status: corev1.PodStatus{PodIP: "1.2.3.4"}, + }, + }, + }, + want: []*corev1.Pod{ + { + ObjectMeta: v1.ObjectMeta{Name: "pod-hello-world"}, + Status: corev1.PodStatus{PodIP: "1.2.3.4"}, + }, + }, + expectTaskNodes: []*taskNode{ + { + ObjectMeta: v1.ObjectMeta{Name: "test-0"}, + IP: "4.3.2.1", + PodName: "pod-foo-bar", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := assignTaskNodes(tt.args.taskNodes, tt.args.freePods); !reflect.DeepEqual(got, tt.want) { + t.Errorf("assignTaskNodes() = %v, want %v", got, tt.want) + } + if !reflect.DeepEqual(tt.expectTaskNodes, tt.args.taskNodes) { + t.Errorf("assignTaskNodes() = %v, want %v", tt.expectTaskNodes, tt.args.taskNodes) + } + }) + } +} + +func Test_refreshFreePods(t *testing.T) { + tests := []struct { + name string + allPods []*corev1.Pod + taskNodes []*taskNode + expectedFree int + expectedNames []string + }{ + { + name: "no assigned pods", + allPods: []*corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{Name: "pod-1"}, + Status: corev1.PodStatus{PodIP: "1.1.1.1"}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "pod-2"}, + Status: corev1.PodStatus{PodIP: "1.1.1.2"}, + }, + }, + taskNodes: []*taskNode{ + {ObjectMeta: metav1.ObjectMeta{Name: "task-1"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "task-2"}}, + }, + expectedFree: 2, + expectedNames: []string{"pod-1", "pod-2"}, + }, + { + name: "some assigned pods", + allPods: []*corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{Name: "pod-1"}, + Status: corev1.PodStatus{PodIP: "1.1.1.1"}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "pod-2"}, + Status: corev1.PodStatus{PodIP: "1.1.1.2"}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "pod-3"}, + Status: corev1.PodStatus{PodIP: "1.1.1.3"}, + }, + }, + taskNodes: []*taskNode{ + { + ObjectMeta: metav1.ObjectMeta{Name: "task-1"}, + IP: "1.1.1.1", + PodName: "pod-1", + }, + {ObjectMeta: metav1.ObjectMeta{Name: "task-2"}}, + }, + expectedFree: 2, + expectedNames: []string{"pod-2", "pod-3"}, + }, + { + name: "all pods assigned", + allPods: []*corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{Name: "pod-1"}, + Status: corev1.PodStatus{PodIP: "1.1.1.1"}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "pod-2"}, + Status: corev1.PodStatus{PodIP: "1.1.1.2"}, + }, + }, + taskNodes: []*taskNode{ + { + ObjectMeta: metav1.ObjectMeta{Name: "task-1"}, + IP: "1.1.1.1", + PodName: "pod-1", + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "task-2"}, + IP: "1.1.1.2", + PodName: "pod-2", + }, + }, + expectedFree: 0, + expectedNames: []string{}, + }, + { + name: "pods without IP addresses", + allPods: []*corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{Name: "pod-1"}, + Status: corev1.PodStatus{PodIP: "1.1.1.1"}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "pod-2"}, + Status: corev1.PodStatus{PodIP: ""}, + }, + }, + taskNodes: []*taskNode{ + {ObjectMeta: metav1.ObjectMeta{Name: "task-1"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "task-2"}}, + }, + expectedFree: 1, + expectedNames: []string{"pod-1"}, + }, + { + name: "empty pods list", + allPods: []*corev1.Pod{}, + taskNodes: []*taskNode{ + {ObjectMeta: metav1.ObjectMeta{Name: "task-1"}}, + }, + expectedFree: 0, + expectedNames: []string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sch := &defaultTaskScheduler{ + allPods: tt.allPods, + taskNodes: tt.taskNodes, + } + + sch.refreshFreePods() + + if len(sch.freePods) != tt.expectedFree { + t.Errorf("refreshFreePods() freePods length = %v, want %v", len(sch.freePods), tt.expectedFree) + } + + actualNames := make([]string, len(sch.freePods)) + for i, pod := range sch.freePods { + actualNames[i] = pod.Name + } + + if !reflect.DeepEqual(actualNames, tt.expectedNames) { + t.Errorf("refreshFreePods() freePods names = %v, want %v", actualNames, tt.expectedNames) + } + }) + } +} + +func Test_collectTaskStatus(t *testing.T) { + ctl := gomock.NewController(t) + defer ctl.Finish() + + mockTimeNow := time.Now() + o := timeNow + timeNow = func() time.Time { + return mockTimeNow + } + defer func() { + timeNow = o + }() + + tests := []struct { + name string + taskNodes []*taskNode + expectedCollectIPs []string + mockReturnTasks map[string]*api.Task + expectedTaskNodes []*taskNode + }{ + { + name: "no assigned task nodes", + taskNodes: []*taskNode{ + { + ObjectMeta: metav1.ObjectMeta{Name: "task-1"}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "task-2"}, + }, + }, + expectedCollectIPs: []string{}, + mockReturnTasks: map[string]*api.Task{}, + expectedTaskNodes: []*taskNode{ + { + ObjectMeta: metav1.ObjectMeta{Name: "task-1"}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "task-2"}, + }, + }, + }, + { + name: "assigned task nodes with task status", + taskNodes: []*taskNode{ + { + ObjectMeta: metav1.ObjectMeta{Name: "task-1"}, + IP: "1.1.1.1", + PodName: "pod-1", + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "task-2"}, + IP: "1.1.1.2", + PodName: "pod-2", + }, + }, + expectedCollectIPs: []string{"1.1.1.1", "1.1.1.2"}, + mockReturnTasks: map[string]*api.Task{ + "1.1.1.1": { + Name: "task-1", + Status: sandboxv1alpha1.TaskStatus{ + State: sandboxv1alpha1.TaskState{ + Running: &sandboxv1alpha1.TaskStateRunning{ + StartedAt: metav1.NewTime(mockTimeNow), + }, + }, + }, + }, + "1.1.1.2": { + Name: "task-2", + Status: sandboxv1alpha1.TaskStatus{ + State: sandboxv1alpha1.TaskState{ + Terminated: &sandboxv1alpha1.TaskStateTerminated{ + ExitCode: 0, + FinishedAt: metav1.NewTime(mockTimeNow), + }, + }, + }, + }, + }, + expectedTaskNodes: []*taskNode{ + { + ObjectMeta: metav1.ObjectMeta{Name: "task-1"}, + IP: "1.1.1.1", + PodName: "pod-1", + Status: &api.Task{ + Name: "task-1", + Status: sandboxv1alpha1.TaskStatus{ + State: sandboxv1alpha1.TaskState{ + Running: &sandboxv1alpha1.TaskStateRunning{ + StartedAt: metav1.NewTime(mockTimeNow), + }, + }, + }, + }, + tState: RunningTaskState, + tStateLastTransTime: &mockTimeNow, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "task-2"}, + IP: "1.1.1.2", + PodName: "pod-2", + Status: &api.Task{ + Name: "task-2", + Status: sandboxv1alpha1.TaskStatus{ + State: sandboxv1alpha1.TaskState{ + Terminated: &sandboxv1alpha1.TaskStateTerminated{ + ExitCode: 0, + FinishedAt: metav1.NewTime(mockTimeNow), + }, + }, + }, + }, + tState: SucceedTaskState, + tStateLastTransTime: &mockTimeNow, + }, + }, + }, + { + name: "assigned task nodes with nil task status", + taskNodes: []*taskNode{ + { + ObjectMeta: metav1.ObjectMeta{Name: "task-1"}, + IP: "1.1.1.1", + PodName: "pod-1", + }, + }, + expectedCollectIPs: []string{"1.1.1.1"}, + mockReturnTasks: map[string]*api.Task{ + "1.1.1.1": nil, + }, + expectedTaskNodes: []*taskNode{ + { + ObjectMeta: metav1.ObjectMeta{Name: "task-1"}, + IP: "1.1.1.1", + PodName: "pod-1", + }, + }, + }, + { + name: "mixed assigned and unassigned task nodes", + taskNodes: []*taskNode{ + { + ObjectMeta: metav1.ObjectMeta{Name: "task-1"}, + IP: "1.1.1.1", + PodName: "pod-1", + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "task-2"}, + }, + }, + expectedCollectIPs: []string{"1.1.1.1"}, + mockReturnTasks: map[string]*api.Task{ + "1.1.1.1": { + Name: "task-1", + Status: sandboxv1alpha1.TaskStatus{ + State: sandboxv1alpha1.TaskState{ + Running: &sandboxv1alpha1.TaskStateRunning{ + StartedAt: metav1.NewTime(mockTimeNow), + }, + }, + }, + }, + }, + expectedTaskNodes: []*taskNode{ + { + ObjectMeta: metav1.ObjectMeta{Name: "task-1"}, + IP: "1.1.1.1", + PodName: "pod-1", + Status: &api.Task{ + Name: "task-1", + Status: sandboxv1alpha1.TaskStatus{ + State: sandboxv1alpha1.TaskState{ + Running: &sandboxv1alpha1.TaskStateRunning{ + StartedAt: metav1.NewTime(mockTimeNow), + }, + }, + }, + }, + tState: RunningTaskState, + tStateLastTransTime: &mockTimeNow, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "task-2"}, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create mock task status collector + mockCollector := NewMocktaskStatusCollector(ctl) + if len(tt.expectedCollectIPs) > 0 { + mockCollector.EXPECT().Collect(gomock.Any(), tt.expectedCollectIPs).Return(tt.mockReturnTasks).Times(1) + } + + // Create scheduler with mock collector + sch := &defaultTaskScheduler{ + taskNodes: tt.taskNodes, + taskStatusCollector: mockCollector, + } + + // Call collectTaskStatus + sch.collectTaskStatus(tt.taskNodes) + + // Verify results + for i, expectedNode := range tt.expectedTaskNodes { + actualNode := tt.taskNodes[i] + + if actualNode.Name != expectedNode.Name { + t.Errorf("taskNode[%d].Name = %v, want %v", i, actualNode.Name, expectedNode.Name) + } + + if actualNode.IP != expectedNode.IP { + t.Errorf("taskNode[%d].IP = %v, want %v", i, actualNode.IP, expectedNode.IP) + } + + if actualNode.PodName != expectedNode.PodName { + t.Errorf("taskNode[%d].PodName = %v, want %v", i, actualNode.PodName, expectedNode.PodName) + } + + if expectedNode.Status == nil { + if actualNode.Status != nil { + t.Errorf("taskNode[%d].Status = %v, want nil", i, actualNode.Status) + } + } else { + if actualNode.Status == nil { + t.Errorf("taskNode[%d].Status = nil, want %v", i, expectedNode.Status) + } else if actualNode.Status.Name != expectedNode.Status.Name { + t.Errorf("taskNode[%d].Status.Name = %v, want %v", i, actualNode.Status.Name, expectedNode.Status.Name) + } + } + + if actualNode.tState != expectedNode.tState { + t.Errorf("taskNode[%d].tState = %v, want %v", i, actualNode.tState, expectedNode.tState) + } + + // Compare time pointers + if expectedNode.tStateLastTransTime == nil { + if actualNode.tStateLastTransTime != nil { + t.Errorf("taskNode[%d].tStateLastTransTime = %v, want nil", i, actualNode.tStateLastTransTime) + } + } else { + if actualNode.tStateLastTransTime == nil { + t.Errorf("taskNode[%d].tStateLastTransTime = nil, want %v", i, expectedNode.tStateLastTransTime) + } else if !actualNode.tStateLastTransTime.Equal(*expectedNode.tStateLastTransTime) { + t.Errorf("taskNode[%d].tStateLastTransTime = %v, want %v", i, actualNode.tStateLastTransTime, expectedNode.tStateLastTransTime) + } + } + } + }) + } +} + +func Test_indexByName(t *testing.T) { + tests := []struct { + name string + taskNodes []*taskNode + expected map[string]*taskNode + }{ + { + name: "empty task nodes", + taskNodes: []*taskNode{}, + expected: map[string]*taskNode{}, + }, + { + name: "single task node", + taskNodes: []*taskNode{ + { + ObjectMeta: metav1.ObjectMeta{Name: "task-1"}, + }, + }, + expected: map[string]*taskNode{ + "task-1": { + ObjectMeta: metav1.ObjectMeta{Name: "task-1"}, + }, + }, + }, + { + name: "multiple task nodes", + taskNodes: []*taskNode{ + { + ObjectMeta: metav1.ObjectMeta{Name: "task-1"}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "task-2"}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "task-3"}, + }, + }, + expected: map[string]*taskNode{ + "task-1": { + ObjectMeta: metav1.ObjectMeta{Name: "task-1"}, + }, + "task-2": { + ObjectMeta: metav1.ObjectMeta{Name: "task-2"}, + }, + "task-3": { + ObjectMeta: metav1.ObjectMeta{Name: "task-3"}, + }, + }, + }, + { + name: "duplicate task node names", + taskNodes: []*taskNode{ + { + ObjectMeta: metav1.ObjectMeta{Name: "task-1"}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "task-1"}, + }, + }, + expected: map[string]*taskNode{ + "task-1": { + ObjectMeta: metav1.ObjectMeta{Name: "task-1"}, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := indexByName(tt.taskNodes) + + if len(result) != len(tt.expected) { + t.Errorf("indexByName() map length = %v, want %v", len(result), len(tt.expected)) + } + + for key, expectedNode := range tt.expected { + actualNode, ok := result[key] + if !ok { + t.Errorf("indexByName() missing key %v", key) + continue + } + + if actualNode.Name != expectedNode.Name { + t.Errorf("indexByName()[%v].Name = %v, want %v", key, actualNode.Name, expectedNode.Name) + } + } + }) + } +} + +func Test_scheduleTaskNodes(t *testing.T) { + ctl := gomock.NewController(t) + defer ctl.Finish() + + // Mock time for consistent testing + mockTimeNow := time.Now() + o := timeNow + timeNow = func() time.Time { + return mockTimeNow + } + defer func() { + timeNow = o + }() + + tests := []struct { + name string + taskNodes []*taskNode + freePods []*corev1.Pod + batchSbx *sandboxv1alpha1.BatchSandbox + expectedTaskNodes []*taskNode + expectedRemainingFreePods int + expectedSetCalls map[string]*api.Task // IP -> Expected Task + }{ + { + name: "assign free pods to unassigned task nodes", + taskNodes: []*taskNode{ + { + ObjectMeta: metav1.ObjectMeta{Name: "task-1"}, + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"echo", "hello"}, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "task-2"}, + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"echo", "world"}, + }, + }, + }, + }, + freePods: []*corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{Name: "pod-1"}, + Status: corev1.PodStatus{PodIP: "1.1.1.1"}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "pod-2"}, + Status: corev1.PodStatus{PodIP: "1.1.1.2"}, + }, + }, + batchSbx: &sandboxv1alpha1.BatchSandbox{ + ObjectMeta: v1.ObjectMeta{Name: "test-batch"}, + }, + expectedTaskNodes: []*taskNode{ + { + ObjectMeta: metav1.ObjectMeta{Name: "task-1"}, + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"echo", "hello"}, + }, + }, + IP: "1.1.1.1", + PodName: "pod-1", + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "task-2"}, + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"echo", "world"}, + }, + }, + IP: "1.1.1.2", + PodName: "pod-2", + }, + }, + expectedRemainingFreePods: 0, + expectedSetCalls: map[string]*api.Task{ + "1.1.1.1": { + Name: "task-1", + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"echo", "hello"}, + }, + }, + }, + "1.1.1.2": { + Name: "task-2", + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"echo", "world"}, + }, + }, + }, + }, + }, + { + name: "no free pods available", + taskNodes: []*taskNode{ + { + ObjectMeta: metav1.ObjectMeta{Name: "task-1"}, + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"echo", "hello"}, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "task-2"}, + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"echo", "world"}, + }, + }, + }, + }, + freePods: []*corev1.Pod{}, + batchSbx: &sandboxv1alpha1.BatchSandbox{ + ObjectMeta: v1.ObjectMeta{Name: "test-batch"}, + }, + expectedTaskNodes: []*taskNode{ + { + ObjectMeta: metav1.ObjectMeta{Name: "task-1"}, + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"echo", "hello"}, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "task-2"}, + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"echo", "world"}, + }, + }, + }, + }, + expectedRemainingFreePods: 0, + expectedSetCalls: map[string]*api.Task{}, + }, + { + name: "some task nodes already assigned", + taskNodes: []*taskNode{ + { + ObjectMeta: metav1.ObjectMeta{Name: "task-1"}, + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"echo", "hello"}, + }, + }, + IP: "1.1.1.1", + PodName: "pod-1", + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "task-2"}, + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"echo", "world"}, + }, + }, + }, + }, + freePods: []*corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{Name: "pod-2"}, + Status: corev1.PodStatus{PodIP: "1.1.1.2"}, + }, + }, + batchSbx: &sandboxv1alpha1.BatchSandbox{ + ObjectMeta: v1.ObjectMeta{Name: "test-batch"}, + }, + expectedTaskNodes: []*taskNode{ + { + ObjectMeta: metav1.ObjectMeta{Name: "task-1"}, + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"echo", "hello"}, + }, + }, + IP: "1.1.1.1", + PodName: "pod-1", + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "task-2"}, + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"echo", "world"}, + }, + }, + IP: "1.1.1.2", + PodName: "pod-2", + }, + }, + expectedRemainingFreePods: 0, + expectedSetCalls: map[string]*api.Task{ + "1.1.1.1": { + Name: "task-1", + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"echo", "hello"}, + }, + }, + }, + "1.1.1.2": { + Name: "task-2", + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"echo", "world"}, + }, + }, + }, + }, + }, + { + name: "more free pods than unassigned tasks", + taskNodes: []*taskNode{ + { + ObjectMeta: metav1.ObjectMeta{Name: "task-1"}, + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"echo", "hello"}, + }, + }, + IP: "1.1.1.1", + PodName: "pod-1", + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "task-2"}, + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"echo", "world"}, + }, + }, + }, + }, + freePods: []*corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{Name: "pod-2"}, + Status: corev1.PodStatus{PodIP: "1.1.1.2"}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "pod-3"}, + Status: corev1.PodStatus{PodIP: "1.1.1.3"}, + }, + }, + batchSbx: &sandboxv1alpha1.BatchSandbox{ + ObjectMeta: v1.ObjectMeta{Name: "test-batch"}, + }, + expectedTaskNodes: []*taskNode{ + { + ObjectMeta: metav1.ObjectMeta{Name: "task-1"}, + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"echo", "hello"}, + }, + }, + IP: "1.1.1.1", + PodName: "pod-1", + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "task-2"}, + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"echo", "world"}, + }, + }, + IP: "1.1.1.2", + PodName: "pod-2", + }, + }, + expectedRemainingFreePods: 1, + expectedSetCalls: map[string]*api.Task{ + "1.1.1.1": { + Name: "task-1", + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"echo", "hello"}, + }, + }, + }, + "1.1.1.2": { + Name: "task-2", + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"echo", "world"}, + }, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create mock task clients for each pod IP and task node + mockClients := make(map[string]*MocktaskClient) + + // Create task client creator function that returns mock clients + taskClientCreator := func(ip string) taskClient { + if mockClient, ok := mockClients[ip]; ok { + return mockClient + } + mockClient := NewMocktaskClient(ctl) + mockClients[ip] = mockClient + return mockClient + } + + // Set expectations for Set calls + for ip, expectedTask := range tt.expectedSetCalls { + mockClient := mockClients[ip] + if mockClient == nil { + mockClient = NewMocktaskClient(ctl) + mockClients[ip] = mockClient + } + mockClient.EXPECT().Set(gomock.Any(), expectedTask).Return(expectedTask, nil).Times(1) + } + + // Create scheduler + sch := &defaultTaskScheduler{ + taskNodes: tt.taskNodes, + freePods: tt.freePods, + maxConcurrency: defaultSchConcurrency, + taskClientCreator: taskClientCreator, + } + + // Call scheduleTaskNodes + err := sch.scheduleTaskNodes() + + // Verify no error + if err != nil { + t.Errorf("scheduleTaskNodes() error = %v, want nil", err) + } + + // Verify results + for i, expectedNode := range tt.expectedTaskNodes { + actualNode := tt.taskNodes[i] + + if actualNode.Name != expectedNode.Name { + t.Errorf("taskNode[%d].Name = %v, want %v", i, actualNode.Name, expectedNode.Name) + } + + if actualNode.IP != expectedNode.IP { + t.Errorf("taskNode[%d].IP = %v, want %v", i, actualNode.IP, expectedNode.IP) + } + + if actualNode.PodName != expectedNode.PodName { + t.Errorf("taskNode[%d].PodName = %v, want %v", i, actualNode.PodName, expectedNode.PodName) + } + } + + // Verify remaining free pods + if len(sch.freePods) != tt.expectedRemainingFreePods { + t.Errorf("scheduleTaskNodes() remaining freePods length = %v, want %v", len(sch.freePods), tt.expectedRemainingFreePods) + } + }) + } +} + +func Test_parseTaskState(t *testing.T) { + mockTimeNow := time.Now() + + tests := []struct { + name string + taskStatus *sandboxv1alpha1.TaskStatus + expected TaskState + }{ + { + name: "running task", + taskStatus: &sandboxv1alpha1.TaskStatus{ + State: sandboxv1alpha1.TaskState{ + Running: &sandboxv1alpha1.TaskStateRunning{ + StartedAt: metav1.NewTime(mockTimeNow), + }, + }, + }, + expected: RunningTaskState, + }, + { + name: "succeed task", + taskStatus: &sandboxv1alpha1.TaskStatus{ + State: sandboxv1alpha1.TaskState{ + Terminated: &sandboxv1alpha1.TaskStateTerminated{ + ExitCode: 0, + FinishedAt: metav1.NewTime(mockTimeNow), + }, + }, + }, + expected: SucceedTaskState, + }, + { + name: "failed task", + taskStatus: &sandboxv1alpha1.TaskStatus{ + State: sandboxv1alpha1.TaskState{ + Terminated: &sandboxv1alpha1.TaskStateTerminated{ + ExitCode: 1, + FinishedAt: metav1.NewTime(mockTimeNow), + }, + }, + }, + expected: FailedTaskState, + }, + { + name: "unknown task state", + taskStatus: &sandboxv1alpha1.TaskStatus{ + State: sandboxv1alpha1.TaskState{}, + }, + expected: UnknownTaskState, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := parseTaskState(tt.taskStatus) + if result != tt.expected { + t.Errorf("parseTaskState() = %v, want %v", result, tt.expected) + } + }) + } +} + +func Test_initTaskNodes(t *testing.T) { + type args struct { + tasks []*sandboxv1alpha1.Task + } + tests := []struct { + name string + args args + want []*taskNode + wantErr bool + }{ + { + name: "init success", + args: args{ + tasks: []*sandboxv1alpha1.Task{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "test-task-0", + Namespace: "default", + Labels: map[string]string{"app": "test"}, + Annotations: map[string]string{"annotation": "value"}, + }, + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"tail", "-f", "/dev/null"}, + }, + }, + }, + }, + }, + want: []*taskNode{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "test-task-0", + Namespace: "default", + Labels: map[string]string{"app": "test"}, + Annotations: map[string]string{"annotation": "value"}, + }, + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"tail", "-f", "/dev/null"}, + }, + }, + }, + }, + }, + { + name: "init multiple tasks", + args: args{ + tasks: []*sandboxv1alpha1.Task{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "test-task-0", + Namespace: "default", + }, + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"echo", "hello"}, + }, + }, + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "test-task-1", + Namespace: "default", + }, + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"echo", "world"}, + }, + }, + }, + }, + }, + want: []*taskNode{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "test-task-0", + Namespace: "default", + }, + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"echo", "hello"}, + }, + }, + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "test-task-1", + Namespace: "default", + }, + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"echo", "world"}, + }, + }, + }, + }, + }, + { + name: "init empty tasks", + args: args{ + tasks: []*sandboxv1alpha1.Task{}, + }, + want: []*taskNode{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := initTaskNodes(tt.args.tasks) + if (err != nil) != tt.wantErr { + t.Errorf("initTaskNodes() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("initTaskNodes() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/kubernetes/internal/scheduler/interface.go b/kubernetes/internal/scheduler/interface.go new file mode 100644 index 00000000..d5e67fc5 --- /dev/null +++ b/kubernetes/internal/scheduler/interface.go @@ -0,0 +1,32 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scheduler + +import ( + corev1 "k8s.io/api/core/v1" + + sandboxv1alpha1 "github.com/alibaba/OpenSandbox/sandbox-k8s/api/v1alpha1" +) + +type TaskScheduler interface { + Schedule() error + UpdatePods(pod []*corev1.Pod) + ListTask() []Task + StopTask() []Task +} + +func NewTaskScheduler(name string, tasks []*sandboxv1alpha1.Task, pods []*corev1.Pod, resPolicyWhenTaskCompleted sandboxv1alpha1.TaskResourcePolicy) (TaskScheduler, error) { + return newTaskScheduler(name, tasks, pods, resPolicyWhenTaskCompleted) +} diff --git a/kubernetes/internal/scheduler/mock/interface.go b/kubernetes/internal/scheduler/mock/interface.go new file mode 100644 index 00000000..7d421734 --- /dev/null +++ b/kubernetes/internal/scheduler/mock/interface.go @@ -0,0 +1,91 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: internal/task/scheduler/interface.go + +// Package mock_scheduler is a generated GoMock package. +package mock_scheduler + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + v1 "k8s.io/api/core/v1" + + scheduler "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/scheduler" +) + +// MockTaskScheduler is a mock of TaskScheduler interface. +type MockTaskScheduler struct { + ctrl *gomock.Controller + recorder *MockTaskSchedulerMockRecorder +} + +// MockTaskSchedulerMockRecorder is the mock recorder for MockTaskScheduler. +type MockTaskSchedulerMockRecorder struct { + mock *MockTaskScheduler +} + +// NewMockTaskScheduler creates a new mock instance. +func NewMockTaskScheduler(ctrl *gomock.Controller) *MockTaskScheduler { + mock := &MockTaskScheduler{ctrl: ctrl} + mock.recorder = &MockTaskSchedulerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockTaskScheduler) EXPECT() *MockTaskSchedulerMockRecorder { + return m.recorder +} + +// ListTask mocks base method. +func (m *MockTaskScheduler) ListTask() []scheduler.Task { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListTask") + ret0, _ := ret[0].([]scheduler.Task) + return ret0 +} + +// ListTask indicates an expected call of ListTask. +func (mr *MockTaskSchedulerMockRecorder) ListTask() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListTask", reflect.TypeOf((*MockTaskScheduler)(nil).ListTask)) +} + +// Schedule mocks base method. +func (m *MockTaskScheduler) Schedule() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Schedule") + ret0, _ := ret[0].(error) + return ret0 +} + +// Schedule indicates an expected call of Schedule. +func (mr *MockTaskSchedulerMockRecorder) Schedule() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Schedule", reflect.TypeOf((*MockTaskScheduler)(nil).Schedule)) +} + +// StopTask mocks base method. +func (m *MockTaskScheduler) StopTask() []scheduler.Task { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StopTask") + ret0, _ := ret[0].([]scheduler.Task) + return ret0 +} + +// StopTask indicates an expected call of StopTask. +func (mr *MockTaskSchedulerMockRecorder) StopTask() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StopTask", reflect.TypeOf((*MockTaskScheduler)(nil).StopTask)) +} + +// UpdatePods mocks base method. +func (m *MockTaskScheduler) UpdatePods(pod []*v1.Pod) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "UpdatePods", pod) +} + +// UpdatePods indicates an expected call of UpdatePods. +func (mr *MockTaskSchedulerMockRecorder) UpdatePods(pod interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdatePods", reflect.TypeOf((*MockTaskScheduler)(nil).UpdatePods), pod) +} diff --git a/kubernetes/internal/scheduler/mock/types.go b/kubernetes/internal/scheduler/mock/types.go new file mode 100644 index 00000000..03940884 --- /dev/null +++ b/kubernetes/internal/scheduler/mock/types.go @@ -0,0 +1,92 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: internal/task/scheduler/types.go + +// Package mock_scheduler is a generated GoMock package. +package mock_scheduler + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + + scheduler "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/scheduler" +) + +// MockTask is a mock of Task interface. +type MockTask struct { + ctrl *gomock.Controller + recorder *MockTaskMockRecorder +} + +// MockTaskMockRecorder is the mock recorder for MockTask. +type MockTaskMockRecorder struct { + mock *MockTask +} + +// NewMockTask creates a new mock instance. +func NewMockTask(ctrl *gomock.Controller) *MockTask { + mock := &MockTask{ctrl: ctrl} + mock.recorder = &MockTaskMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockTask) EXPECT() *MockTaskMockRecorder { + return m.recorder +} + +// GetName mocks base method. +func (m *MockTask) GetName() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetName") + ret0, _ := ret[0].(string) + return ret0 +} + +// GetName indicates an expected call of GetName. +func (mr *MockTaskMockRecorder) GetName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetName", reflect.TypeOf((*MockTask)(nil).GetName)) +} + +// GetPodName mocks base method. +func (m *MockTask) GetPodName() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPodName") + ret0, _ := ret[0].(string) + return ret0 +} + +// GetPodName indicates an expected call of GetPodName. +func (mr *MockTaskMockRecorder) GetPodName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPodName", reflect.TypeOf((*MockTask)(nil).GetPodName)) +} + +// GetState mocks base method. +func (m *MockTask) GetState() scheduler.TaskState { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetState") + ret0, _ := ret[0].(scheduler.TaskState) + return ret0 +} + +// GetState indicates an expected call of GetState. +func (mr *MockTaskMockRecorder) GetState() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetState", reflect.TypeOf((*MockTask)(nil).GetState)) +} + +// IsResourceReleased mocks base method. +func (m *MockTask) IsResourceReleased() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsResourceReleased") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsResourceReleased indicates an expected call of IsResourceReleased. +func (mr *MockTaskMockRecorder) IsResourceReleased() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsResourceReleased", reflect.TypeOf((*MockTask)(nil).IsResourceReleased)) +} diff --git a/kubernetes/internal/scheduler/recovery.go b/kubernetes/internal/scheduler/recovery.go new file mode 100644 index 00000000..1a5a4d0f --- /dev/null +++ b/kubernetes/internal/scheduler/recovery.go @@ -0,0 +1,80 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scheduler + +import ( + "context" + + v1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" + + api "github.com/alibaba/OpenSandbox/sandbox-k8s/pkg/task-executor" +) + +// recover reconstructs the task scheduler state from existing pods and their endpoints +// This function is used to restore the scheduler state after a restart +func (sch *defaultTaskScheduler) recover() error { + var err error + sch.once.Do(func() { + sch.recoverTaskNodesStatus() + klog.Infof("task scheduler recovered, BatchSandbox %s, task_nodes=%d, all_pods=%d", sch.name, len(sch.taskNodes), len(sch.allPods)) + }) + return err +} + +func (sch *defaultTaskScheduler) recoverTaskNodesStatus() error { + ips := make([]string, 0, len(sch.allPods)/2) + pods := make([]*v1.Pod, 0, len(sch.allPods)/2) + for i := range sch.allPods { + pod := sch.allPods[i] + if pod.Status.PodIP == "" { + continue + } + ips = append(ips, pod.Status.PodIP) + pods = append(pods, pod) + } + if len(ips) == 0 { + return nil + } + // TODO: When the agent starts stopping a task, if a recovery occurs at this moment, + // the recovery may complete after the agent has already finished stopping the task and returned an empty task list. + // This could cause the scheduler to be unable to determine whether the task was never executed or has already completed. + // It might lead to duplicate execution, but it ensures at-least-once delivery semantics. + tasks := sch.taskStatusCollector.Collect(context.Background(), ips) + for i := range ips { + ip := ips[i] + pod := pods[i] + task := tasks[ip] + if task == nil || pod == nil { + continue + } + if tNode := sch.taskNodeByNameIndex[task.Name]; tNode != nil { + recoverOneTaskNode(tNode, task, pod.Status.PodIP, pod.Name) + } else { + } + // TODO do we need to stop tasks not belong us? e.g users ScaleIn []*sandboxv1alpha1.Task + } + return nil +} + +func recoverOneTaskNode(tNode *taskNode, currentTask *api.Task, ip string, podName string) { + tNode.Status = currentTask + tNode.transTaskState(parseTaskState(¤tTask.Status)) + tNode.IP = ip + tNode.PodName = podName + if currentTask.DeletionTimestamp != nil { + tNode.transSchState(stateReleasing) + } +} diff --git a/kubernetes/internal/scheduler/recovery_test.go b/kubernetes/internal/scheduler/recovery_test.go new file mode 100644 index 00000000..b6af7e9c --- /dev/null +++ b/kubernetes/internal/scheduler/recovery_test.go @@ -0,0 +1,297 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scheduler + +import ( + "reflect" + "sync" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/golang/mock/gomock" + + sandboxv1alpha1 "github.com/alibaba/OpenSandbox/sandbox-k8s/api/v1alpha1" + api "github.com/alibaba/OpenSandbox/sandbox-k8s/pkg/task-executor" +) + +func Test_recoverOneTaskNode(t *testing.T) { + mockTimeNow := time.Now() + o := timeNow + timeNow = func() time.Time { + return mockTimeNow + } + defer func() { + timeNow = o + }() + testNow := metav1.Time{Time: mockTimeNow} + testTask := &api.Task{ + Name: "test", + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"sleep"}, + }, + }, + Status: sandboxv1alpha1.TaskStatus{ + State: sandboxv1alpha1.TaskState{ + Running: &sandboxv1alpha1.TaskStateRunning{ + StartedAt: testNow, + }, + }, + }, + } + testReleasingTask := &api.Task{ + Name: "test", + DeletionTimestamp: &testNow, + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"sleep"}, + }, + }, + Status: sandboxv1alpha1.TaskStatus{ + State: sandboxv1alpha1.TaskState{ + Running: &sandboxv1alpha1.TaskStateRunning{ + StartedAt: testNow, + }, + }, + }, + } + type args struct { + tNode *taskNode + currentTask *api.Task + ip string + podName string + } + tests := []struct { + name string + args args + expectTaskNode *taskNode + }{ + { + name: "running task", + args: args{ + tNode: &taskNode{}, + currentTask: testTask, + ip: "1.2.3.4", + podName: "foo-bar", + }, + expectTaskNode: &taskNode{ + Status: testTask, + IP: "1.2.3.4", + PodName: "foo-bar", + tState: RunningTaskState, + tStateLastTransTime: &mockTimeNow, + }, + }, + { + name: "releasing task", + args: args{ + tNode: &taskNode{}, + currentTask: testReleasingTask, + ip: "1.2.3.4", + podName: "foo-bar", + }, + expectTaskNode: &taskNode{ + Status: testReleasingTask, + IP: "1.2.3.4", + PodName: "foo-bar", + sState: stateReleasing, + sStateLastTransTime: &mockTimeNow, + tState: RunningTaskState, + tStateLastTransTime: &mockTimeNow, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + recoverOneTaskNode(tt.args.tNode, tt.args.currentTask, tt.args.ip, tt.args.podName) + if tt.expectTaskNode != nil { + if !reflect.DeepEqual(tt.expectTaskNode, tt.args.tNode) { + t.Errorf("recoverOneTaskNode, want %+v, got %+v", tt.expectTaskNode, tt.args.tNode) + } + } + }) + } +} + +func Test_defaultTaskScheduler_recoverTaskNodesStatus(t *testing.T) { + mockTimeNow := time.Now() + o := timeNow + timeNow = func() time.Time { + return mockTimeNow + } + defer func() { + timeNow = o + }() + ctl := gomock.NewController(t) + defer ctl.Finish() + testNow := metav1.Now() + testTaskNode := &taskNode{ + ObjectMeta: v1.ObjectMeta{ + Name: "bsbx-0", + }, + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"hello"}, + }, + }, + } + testTask := &api.Task{ + Name: testTaskNode.Name, + Spec: testTaskNode.Spec, + Status: sandboxv1alpha1.TaskStatus{ + State: sandboxv1alpha1.TaskState{ + Running: &sandboxv1alpha1.TaskStateRunning{ + StartedAt: testNow, + }, + }, + }, + } + recoveredTestTaskNode := &taskNode{ + ObjectMeta: v1.ObjectMeta{ + Name: "bsbx-0", + }, + Spec: sandboxv1alpha1.TaskSpec{ + Container: &sandboxv1alpha1.ContainerTask{ + Command: []string{"hello"}, + }, + }, + Status: testTask, + PodName: "test-0", + IP: "1.2.3.4", + tState: RunningTaskState, + tStateLastTransTime: &mockTimeNow, + } + + type fields struct { + freePods []*corev1.Pod + allPods []*corev1.Pod + taskNodes []*taskNode + taskNodeByNameIndex map[string]*taskNode + maxConcurrency int + once sync.Once + taskStatusCollector taskStatusCollector + } + tests := []struct { + name string + fields fields + wantErr bool + expectTaskNodes []*taskNode + }{ + { + name: "recover nothing, pod pending", + fields: fields{ + allPods: []*corev1.Pod{{ + ObjectMeta: v1.ObjectMeta{Name: "test-0"}, + }}, + taskNodes: []*taskNode{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "bsbx-0", + }, + }, + }, + }, + expectTaskNodes: []*taskNode{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "bsbx-0", + }, + }, + }, + }, + { + name: "recover nothing, client return nil task via endpoint", + fields: fields{ + allPods: []*corev1.Pod{{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-0", + }, + Status: corev1.PodStatus{ + PodIP: "1.2.3.4", + }, + }}, + taskNodes: []*taskNode{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "bsbx-0", + }, + }, + }, + taskStatusCollector: func() taskStatusCollector { + mock := NewMocktaskStatusCollector(ctl) + mock.EXPECT().Collect(gomock.Any(), []string{"1.2.3.4"}).Return(map[string]*api.Task{"1.2.3.4": nil}).Times(1) + return mock + }(), + }, + expectTaskNodes: []*taskNode{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "bsbx-0", + }, + }, + }, + }, + { + name: "recover successfully, client return running task via endpoint", + fields: fields{ + allPods: []*corev1.Pod{{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-0", + }, + Status: corev1.PodStatus{ + PodIP: "1.2.3.4", + }, + }}, + taskNodes: []*taskNode{testTaskNode}, + taskNodeByNameIndex: map[string]*taskNode{ + "bsbx-0": testTaskNode, + }, + taskStatusCollector: func() taskStatusCollector { + mock := NewMocktaskStatusCollector(ctl) + mock.EXPECT().Collect(gomock.Any(), []string{"1.2.3.4"}).Return(map[string]*api.Task{"1.2.3.4": testTask}).Times(1) + return mock + }(), + }, + expectTaskNodes: []*taskNode{ + recoveredTestTaskNode, + }, + }, + } + for i := range tests { + tt := &tests[i] + t.Run(tt.name, func(t *testing.T) { + sch := &defaultTaskScheduler{ + freePods: tt.fields.freePods, + allPods: tt.fields.allPods, + taskNodes: tt.fields.taskNodes, + taskNodeByNameIndex: tt.fields.taskNodeByNameIndex, + maxConcurrency: tt.fields.maxConcurrency, + taskStatusCollector: tt.fields.taskStatusCollector, + } + if err := sch.recoverTaskNodesStatus(); (err != nil) != tt.wantErr { + t.Errorf("defaultTaskScheduler.recoverTaskNodesStatus() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.expectTaskNodes != nil { + if !reflect.DeepEqual(tt.expectTaskNodes, sch.taskNodes) { + t.Errorf("recoverTaskNodesStatus, want %+v, got %+v", tt.expectTaskNodes, sch.taskNodes) + } + } + }) + } +} diff --git a/kubernetes/internal/scheduler/status_collector.go b/kubernetes/internal/scheduler/status_collector.go new file mode 100644 index 00000000..e7673289 --- /dev/null +++ b/kubernetes/internal/scheduler/status_collector.go @@ -0,0 +1,73 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scheduler + +import ( + "context" + "sync" + + "k8s.io/klog/v2" + + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/utils" + api "github.com/alibaba/OpenSandbox/sandbox-k8s/pkg/task-executor" +) + +type taskClientCreator func(ip string) taskClient + +func newTaskStatusCollector(creator taskClientCreator) taskStatusCollector { + return &defaultTaskStatusCollector{creator: creator} +} + +// TODO error +type taskStatusCollector interface { + Collect(ctx context.Context, ipList []string) map[string]*api.Task /*ip<->task*/ +} + +// TODO maybe cache +type defaultTaskStatusCollector struct { + creator taskClientCreator +} + +func (s *defaultTaskStatusCollector) Collect(ctx context.Context, ipList []string) map[string]*api.Task { + semaphore := make(chan struct{}, len(ipList)) + var wg sync.WaitGroup + var mu sync.Mutex + ret := make(map[string]*api.Task, len(ipList)) + for idx := range ipList { + ip := ipList[idx] + semaphore <- struct{}{} + wg.Add(1) + go func(ip string) { + defer func() { + <-semaphore + wg.Done() + }() + ctx, cancel := context.WithTimeout(ctx, defaultTimeout) + defer cancel() + client := s.creator(ip) + task, err := client.Get(ctx) + if err != nil { + klog.Errorf("failed to GetTask for IP %s, err %v", ip, err) + } else if task != nil { + mu.Lock() + ret[ip] = task + mu.Unlock() + } + }(ip) + } + wg.Wait() + klog.Infof("Collect task status %s", utils.DumpJSON(ret)) + return ret +} diff --git a/kubernetes/internal/scheduler/status_collector_mock.go b/kubernetes/internal/scheduler/status_collector_mock.go new file mode 100644 index 00000000..d8aa1e2a --- /dev/null +++ b/kubernetes/internal/scheduler/status_collector_mock.go @@ -0,0 +1,51 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: internal/task/scheduler/status_collector.go + +// Package scheduler is a generated GoMock package. +package scheduler + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + + api "github.com/alibaba/OpenSandbox/sandbox-k8s/pkg/task-executor" +) + +// MocktaskStatusCollector is a mock of taskStatusCollector interface. +type MocktaskStatusCollector struct { + ctrl *gomock.Controller + recorder *MocktaskStatusCollectorMockRecorder +} + +// MocktaskStatusCollectorMockRecorder is the mock recorder for MocktaskStatusCollector. +type MocktaskStatusCollectorMockRecorder struct { + mock *MocktaskStatusCollector +} + +// NewMocktaskStatusCollector creates a new mock instance. +func NewMocktaskStatusCollector(ctrl *gomock.Controller) *MocktaskStatusCollector { + mock := &MocktaskStatusCollector{ctrl: ctrl} + mock.recorder = &MocktaskStatusCollectorMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MocktaskStatusCollector) EXPECT() *MocktaskStatusCollectorMockRecorder { + return m.recorder +} + +// Collect mocks base method. +func (m *MocktaskStatusCollector) Collect(ctx context.Context, ipList []string) map[string]*api.Task { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Collect", ctx, ipList) + ret0, _ := ret[0].(map[string]*api.Task) + return ret0 +} + +// Collect indicates an expected call of Collect. +func (mr *MocktaskStatusCollectorMockRecorder) Collect(ctx, ipList interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Collect", reflect.TypeOf((*MocktaskStatusCollector)(nil).Collect), ctx, ipList) +} diff --git a/kubernetes/internal/scheduler/types.go b/kubernetes/internal/scheduler/types.go new file mode 100644 index 00000000..6e7ef952 --- /dev/null +++ b/kubernetes/internal/scheduler/types.go @@ -0,0 +1,33 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scheduler + +type Task interface { + GetName() string + GetState() TaskState + GetPodName() string + // IsResourceReleased task resource is released + // TODO func name is strange + IsResourceReleased() bool +} + +type TaskState string + +const ( + RunningTaskState TaskState = "RUNNING" + FailedTaskState TaskState = "FAILED" + SucceedTaskState TaskState = "SUCCEED" + UnknownTaskState TaskState = "UNKNOWN" +) diff --git a/kubernetes/internal/task-executor/config/config.go b/kubernetes/internal/task-executor/config/config.go new file mode 100644 index 00000000..9e8f8eee --- /dev/null +++ b/kubernetes/internal/task-executor/config/config.go @@ -0,0 +1,78 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "flag" + "os" + "time" +) + +type Config struct { + DataDir string + ListenAddr string + CRISocket string + ReadTimeout time.Duration + WriteTimeout time.Duration + ReconcileInterval time.Duration + EnableSidecarMode bool + EnableContainerMode bool + MainContainerName string +} + +func NewConfig() *Config { + return &Config{ + DataDir: "/var/lib/sandbox/tasks", + ListenAddr: "0.0.0.0:5758", + CRISocket: "/var/run/containerd/containerd.sock", + ReadTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + ReconcileInterval: 500 * time.Millisecond, + EnableContainerMode: false, + EnableSidecarMode: false, + MainContainerName: "main", + } +} + +func (c *Config) LoadFromEnv() { + if v := os.Getenv("DATA_DIR"); v != "" { + c.DataDir = v + } + if v := os.Getenv("LISTEN_ADDR"); v != "" { + c.ListenAddr = v + } + if v := os.Getenv("CRI_SOCKET"); v != "" { + c.CRISocket = v + } + if v := os.Getenv("ENABLE_CONTAINER_MODE"); v == "true" { + c.EnableContainerMode = true + } + if v := os.Getenv("ENABLE_SIDECAR_MODE"); v == "true" { + c.EnableSidecarMode = true + } + if v := os.Getenv("MAIN_CONTAINER_NAME"); v != "" { + c.MainContainerName = v + } +} + +func (c *Config) LoadFromFlags() { + flag.StringVar(&c.DataDir, "data-dir", c.DataDir, "data storage directory") + flag.StringVar(&c.ListenAddr, "listen-addr", c.ListenAddr, "service listen address") + flag.StringVar(&c.CRISocket, "cri-socket", c.CRISocket, "CRI socket path for container runner mode") + flag.BoolVar(&c.EnableContainerMode, "enable-container-mode", c.EnableContainerMode, "enable container runner mode") + flag.BoolVar(&c.EnableSidecarMode, "enable-sidecar-mode", c.EnableSidecarMode, "enable sidecar runner mode") + flag.StringVar(&c.MainContainerName, "main-container-name", c.MainContainerName, "main container name") + flag.Parse() +} diff --git a/kubernetes/internal/task-executor/manager/interface.go b/kubernetes/internal/task-executor/manager/interface.go new file mode 100644 index 00000000..ee899dfd --- /dev/null +++ b/kubernetes/internal/task-executor/manager/interface.go @@ -0,0 +1,40 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package manager + +import ( + "context" + + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/types" +) + +// TaskManager defines the contract for managing tasks in memory. +type TaskManager interface { + Create(ctx context.Context, task *types.Task) (*types.Task, error) + // Sync synchronizes the current task list with the desired state. + // It deletes tasks not in the desired list and creates new ones. + // Returns the current task list after synchronization. + Sync(ctx context.Context, desired []*types.Task) ([]*types.Task, error) + + Get(ctx context.Context, id string) (*types.Task, error) + + List(ctx context.Context) ([]*types.Task, error) + + Delete(ctx context.Context, id string) error + + Start(ctx context.Context) + + Stop() +} diff --git a/kubernetes/internal/task-executor/manager/task_manager.go b/kubernetes/internal/task-executor/manager/task_manager.go new file mode 100644 index 00000000..2f39376b --- /dev/null +++ b/kubernetes/internal/task-executor/manager/task_manager.go @@ -0,0 +1,502 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package manager + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "k8s.io/klog/v2" + + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/config" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/runtime" + store "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/storage" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/types" +) + +const ( + // Maximum number of concurrent tasks (enforcing single task limitation) + maxConcurrentTasks = 1 +) + +type taskManager struct { + mu sync.RWMutex + tasks map[string]*types.Task // name -> task + // TODO we need design queue for pending tasks + activeTasks int // Count of active tasks (not deleted AND not terminated) + store store.TaskStore + executor runtime.Executor + config *config.Config + + // Reconcile loop control + stopCh chan struct{} + doneCh chan struct{} +} + +// NewTaskManager creates a new task manager instance. +func NewTaskManager(cfg *config.Config, taskStore store.TaskStore, exec runtime.Executor) (TaskManager, error) { + if cfg == nil { + return nil, fmt.Errorf("config cannot be nil") + } + if taskStore == nil { + return nil, fmt.Errorf("task store cannot be nil") + } + if exec == nil { + return nil, fmt.Errorf("executor cannot be nil") + } + + return &taskManager{ + tasks: make(map[string]*types.Task), + store: taskStore, + executor: exec, + config: cfg, + stopCh: make(chan struct{}), + doneCh: make(chan struct{}), + }, nil +} + +// isTaskActive checks if the task is counting towards the concurrency limit. +// A task is active if it is NOT marked for deletion AND NOT in a terminated state. +func (m *taskManager) isTaskActive(task *types.Task) bool { + if task == nil { + return false + } + if task.DeletionTimestamp != nil { + return false + } + state := task.Status.State + return state == types.TaskStatePending || state == types.TaskStateRunning +} + +// Create creates a new task and starts execution. +func (m *taskManager) Create(ctx context.Context, task *types.Task) (*types.Task, error) { + if task == nil { + return nil, fmt.Errorf("task cannot be nil") + } + if task.Name == "" { + return nil, fmt.Errorf("task name cannot be empty") + } + + m.mu.Lock() + defer m.mu.Unlock() + + // Check if task already exists + if _, exists := m.tasks[task.Name]; exists { + return nil, fmt.Errorf("task %s already exists", task.Name) + } + + // Enforce single task limitation using the cached counter + if m.activeTasks >= maxConcurrentTasks { + return nil, fmt.Errorf("maximum concurrent tasks (%d) reached, cannot create new task", maxConcurrentTasks) + } + + // Persist task to store + if err := m.store.Create(ctx, task); err != nil { + return nil, fmt.Errorf("failed to persist task: %w", err) + } + + // Start task execution + if err := m.executor.Start(ctx, task); err != nil { + // Rollback - delete from store + if delErr := m.store.Delete(ctx, task.Name); delErr != nil { + klog.ErrorS(delErr, "failed to rollback task creation", "name", task.Name) + } + return nil, fmt.Errorf("failed to start task: %w", err) + } + + // Inspect immediately to populate status (Running/Waiting) so API response is not empty + if status, err := m.executor.Inspect(ctx, task); err == nil { + task.Status = *status + // Persist the PID and initial status + if err := m.store.Update(ctx, task); err != nil { + klog.ErrorS(err, "failed to persist initial task status", "name", task.Name) + } + } else { + klog.ErrorS(err, "failed to inspect task after start", "name", task.Name) + } + + // Safety fallback: Ensure task has a state + if task.Status.State == "" { + task.Status.State = types.TaskStatePending + task.Status.Reason = "Initialized" + } + + // Add to memory + m.tasks[task.Name] = task + if m.isTaskActive(task) { + m.activeTasks++ + } + + klog.InfoS("task created successfully", "name", task.Name) + return task, nil +} + +// Sync synchronizes the current task list with the desired state. +// It deletes tasks not in the desired list and creates new ones. +// Returns the current task list and any errors encountered during sync. +func (m *taskManager) Sync(ctx context.Context, desired []*types.Task) ([]*types.Task, error) { + if desired == nil { + return nil, fmt.Errorf("desired task list cannot be nil") + } + + m.mu.Lock() + defer m.mu.Unlock() + + // Build desired task map + desiredMap := make(map[string]*types.Task) + for _, task := range desired { + if task != nil && task.Name != "" { + desiredMap[task.Name] = task + } + } + + // Collect errors during sync + var syncErrors []error + + // Delete tasks not in desired list + for name, task := range m.tasks { + if _, ok := desiredMap[name]; !ok { + if err := m.softDeleteLocked(ctx, task); err != nil { + klog.ErrorS(err, "failed to delete task during sync", "name", name) + syncErrors = append(syncErrors, fmt.Errorf("failed to delete task %s: %w", name, err)) + } + } + } + + // Create new tasks + for name, task := range desiredMap { + if _, exists := m.tasks[name]; !exists { + if err := m.createTaskLocked(ctx, task); err != nil { + klog.ErrorS(err, "failed to create task during sync", "name", name) + syncErrors = append(syncErrors, fmt.Errorf("failed to create task %s: %w", name, err)) + } + } + } + + // Return current task list with aggregated errors + if len(syncErrors) > 0 { + return m.listTasksLocked(), errors.Join(syncErrors...) + } + return m.listTasksLocked(), nil +} + +// Get retrieves a task by name. +func (m *taskManager) Get(ctx context.Context, name string) (*types.Task, error) { + if name == "" { + return nil, fmt.Errorf("task name cannot be empty") + } + + m.mu.RLock() + defer m.mu.RUnlock() + + task, exists := m.tasks[name] + if !exists { + return nil, fmt.Errorf("task %s not found", name) + } + + return task, nil +} + +// List returns all tasks. +func (m *taskManager) List(ctx context.Context) ([]*types.Task, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + return m.listTasksLocked(), nil +} + +// Delete removes a task by marking it for deletion (soft delete). +// The reconcile loop will handle the actual stopping and removal. +func (m *taskManager) Delete(ctx context.Context, name string) error { + if name == "" { + return fmt.Errorf("task name cannot be empty") + } + + m.mu.Lock() + defer m.mu.Unlock() + + task, exists := m.tasks[name] + if !exists { + return nil + } + + return m.softDeleteLocked(ctx, task) +} + +// softDeleteLocked marks a task for deletion without acquiring the lock. +func (m *taskManager) softDeleteLocked(ctx context.Context, task *types.Task) error { + if task.DeletionTimestamp != nil { + return nil // Already marked + } + + // If the task was active, decrement the active count + if m.isTaskActive(task) { + m.activeTasks-- + } + + now := time.Now() + task.DeletionTimestamp = &now + + if err := m.store.Update(ctx, task); err != nil { + return fmt.Errorf("failed to mark task for deletion: %w", err) + } + + klog.InfoS("task marked for deletion", "name", task.Name) + return nil +} + +// Start initializes the manager, loads tasks from store, and starts the reconcile loop. +func (m *taskManager) Start(ctx context.Context) { + klog.InfoS("starting task manager") + + // Recover tasks from store + if err := m.recoverTasks(ctx); err != nil { + klog.ErrorS(err, "failed to recover tasks from store") + } + + // Start reconcile loop + go m.reconcileLoop(ctx) + + klog.InfoS("task manager started") +} + +// Stop stops the reconcile loop and cleans up resources. +func (m *taskManager) Stop() { + klog.InfoS("stopping task manager") + close(m.stopCh) + <-m.doneCh + klog.InfoS("task manager stopped") +} + +// createTaskLocked creates a task without acquiring the lock (must be called with lock held). +func (m *taskManager) createTaskLocked(ctx context.Context, task *types.Task) error { + if task == nil || task.Name == "" { + return fmt.Errorf("invalid task") + } + + // Check if already exists + if _, exists := m.tasks[task.Name]; exists { + return fmt.Errorf("task %s already exists", task.Name) + } + + // Enforce single task limitation using the cached counter + if m.activeTasks >= maxConcurrentTasks { + return fmt.Errorf("maximum concurrent tasks (%d) reached, cannot create new task", maxConcurrentTasks) + } + + // Persist to store + if err := m.store.Create(ctx, task); err != nil { + return fmt.Errorf("failed to persist task: %w", err) + } + + // Start execution + if err := m.executor.Start(ctx, task); err != nil { + // Rollback + m.store.Delete(ctx, task.Name) + return fmt.Errorf("failed to start task: %w", err) + } + + // Inspect immediately to populate status (Running/Waiting) so API response is not empty + if status, err := m.executor.Inspect(ctx, task); err == nil { + task.Status = *status + // Persist the PID and initial status + if err := m.store.Update(ctx, task); err != nil { + klog.ErrorS(err, "failed to persist initial task status", "name", task.Name) + } + } else { + klog.ErrorS(err, "failed to inspect task after start", "name", task.Name) + } + + // Add to memory + m.tasks[task.Name] = task + if m.isTaskActive(task) { + m.activeTasks++ + } + return nil +} + +// deleteTaskLocked deletes a task without acquiring the lock (must be called with lock held). +func (m *taskManager) deleteTaskLocked(ctx context.Context, name string) error { + task, exists := m.tasks[name] + if !exists { + // Already deleted, no error + klog.InfoS("task not found, skipping delete", "name", name) + return nil + } + + // Stop task execution + if err := m.executor.Stop(ctx, task); err != nil { + klog.ErrorS(err, "failed to stop task", "name", name) + // Continue with deletion even if stop fails + } + + // Delete from store + if err := m.store.Delete(ctx, name); err != nil { + return fmt.Errorf("failed to delete task from store: %w", err) + } + + // Remove from memory + delete(m.tasks, name) + + klog.InfoS("task deleted successfully", "name", name) + return nil +} + +// listTasksLocked returns all tasks without acquiring the lock (must be called with lock held). +func (m *taskManager) listTasksLocked() []*types.Task { + tasks := make([]*types.Task, 0, len(m.tasks)) + for _, task := range m.tasks { + if task != nil { + tasks = append(tasks, task) + } + } + return tasks +} + +// recoverTasks loads tasks from store and recovers their state. +func (m *taskManager) recoverTasks(ctx context.Context) error { + klog.InfoS("recovering tasks from store") + + tasks, err := m.store.List(ctx) + if err != nil { + return fmt.Errorf("failed to list tasks from store: %w", err) + } + + m.mu.Lock() + defer m.mu.Unlock() + + for _, task := range tasks { + if task == nil { + continue + } + + // Inspect task to get current status + status, err := m.executor.Inspect(ctx, task) + if err != nil { + klog.ErrorS(err, "failed to inspect task during recovery", "name", task.Name) + continue + } + + // Update task status + task.Status = *status + + // Add to memory + m.tasks[task.Name] = task + + // Update active count + if m.isTaskActive(task) { + m.activeTasks++ + } + + klog.InfoS("recovered task", "name", task.Name, "state", task.Status.State, "deleting", task.DeletionTimestamp != nil) + } + + klog.InfoS("task recovery completed", "count", len(m.tasks)) + return nil +} + +// reconcileLoop periodically synchronizes task states. +func (m *taskManager) reconcileLoop(ctx context.Context) { + ticker := time.NewTicker(m.config.ReconcileInterval) + defer ticker.Stop() + defer close(m.doneCh) + + for { + select { + case <-ticker.C: + m.reconcileTasks(ctx) + case <-m.stopCh: + klog.InfoS("reconcile loop stopped") + return + case <-ctx.Done(): + klog.InfoS("reconcile loop context cancelled") + return + } + } +} + +// reconcileTasks updates the status of all tasks and handles deletion. +func (m *taskManager) reconcileTasks(ctx context.Context) { + m.mu.RLock() + tasks := make([]*types.Task, 0, len(m.tasks)) + for _, task := range m.tasks { + if task != nil { + tasks = append(tasks, task) + } + } + m.mu.RUnlock() + + // Update each task's status + for _, task := range tasks { + status, err := m.executor.Inspect(ctx, task) + if err != nil { + klog.ErrorS(err, "failed to inspect task", "name", task.Name) + continue + } + + // Acquire lock to safely update status and active count + m.mu.Lock() + wasActive := m.isTaskActive(task) + + // Update status + task.Status = *status + + isActive := m.isTaskActive(task) + + // If task transitioned from Active -> Inactive (Terminated), decrement active count + if wasActive && !isActive { + m.activeTasks-- + } + m.mu.Unlock() + + // Handle Deletion + if task.DeletionTimestamp != nil { + if task.Status.State == types.TaskStateSucceeded || task.Status.State == types.TaskStateFailed { + // Task is fully terminated, finalize deletion (remove from store/memory) + klog.InfoS("task terminated, finalizing deletion", "name", task.Name) + m.mu.Lock() + if err := m.deleteTaskLocked(ctx, task.Name); err != nil { + klog.ErrorS(err, "failed to finalize task deletion", "name", task.Name) + } + m.mu.Unlock() + continue + } else { + // Task is still running, trigger Stop + klog.InfoS("stopping task marked for deletion", "name", task.Name) + if err := m.executor.Stop(ctx, task); err != nil { + klog.ErrorS(err, "failed to stop task", "name", task.Name) + } + } + } + + // Update task status in memory only. + // We do not need to persist to store here because Persistent fields (Spec, PID, etc.) do not change during the reconcile loop. + // The Status struct IS persisted, but we choose not to persist every few seconds if only runtime state changes. + // However, since we made Status a first-class citizen and it's small, we COULD persist it. + // But for performance, we stick to the decision: only persist on significant changes (Create/Delete). + // Note: If we want to persist ExitCode/FinishedAt, we might need to Update store when state changes to Terminated. + // Let's add that optimization: if state changed to Terminated, persist it. + if wasActive && !isActive { + if err := m.store.Update(ctx, task); err != nil { + klog.ErrorS(err, "failed to update task status in store", "name", task.Name) + } + } + } +} + +// createTaskLocked creates a task without acquiring the lock (must be called with lock held). diff --git a/kubernetes/internal/task-executor/manager/task_manager_test.go b/kubernetes/internal/task-executor/manager/task_manager_test.go new file mode 100644 index 00000000..5df92762 --- /dev/null +++ b/kubernetes/internal/task-executor/manager/task_manager_test.go @@ -0,0 +1,510 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package manager + +import ( + "context" + "testing" + "time" + + "github.com/alibaba/OpenSandbox/sandbox-k8s/api/v1alpha1" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/config" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/runtime" + store "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/storage" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/types" +) + +func setupTestManager(t *testing.T) (TaskManager, *config.Config) { + cfg := &config.Config{ + DataDir: t.TempDir(), + EnableSidecarMode: false, + ReconcileInterval: 100 * time.Millisecond, + } + + taskStore, err := store.NewFileStore(cfg.DataDir) + if err != nil { + t.Fatalf("failed to create store: %v", err) + } + + exec, err := runtime.NewProcessExecutor(cfg) + if err != nil { + t.Fatalf("failed to create executor: %v", err) + } + + mgr, err := NewTaskManager(cfg, taskStore, exec) + if err != nil { + t.Fatalf("failed to create manager: %v", err) + } + + return mgr, cfg +} + +func cleanupTask(t *testing.T, mgr TaskManager, name string) { + ctx := context.Background() + mgr.Delete(ctx, name) + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + _, err := mgr.Get(ctx, name) + if err != nil { + return + } + time.Sleep(100 * time.Millisecond) + } + t.Logf("Task %s not deleted within timeout during cleanup", name) +} + +func TestNewTaskManager(t *testing.T) { + cfg := &config.Config{ + DataDir: t.TempDir(), + } + taskStore, _ := store.NewFileStore(cfg.DataDir) + exec, _ := runtime.NewProcessExecutor(cfg) + + tests := []struct { + name string + cfg *config.Config + store store.TaskStore + executor runtime.Executor + wantErr bool + }{ + { + name: "nil config", + cfg: nil, + store: taskStore, + executor: exec, + wantErr: true, + }, + { + name: "nil store", + cfg: cfg, + store: nil, + executor: exec, + wantErr: true, + }, + { + name: "nil executor", + cfg: cfg, + store: taskStore, + executor: nil, + wantErr: true, + }, + { + name: "valid parameters", + cfg: cfg, + store: taskStore, + executor: exec, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mgr, err := NewTaskManager(tt.cfg, tt.store, tt.executor) + if (err != nil) != tt.wantErr { + t.Errorf("NewTaskManager() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !tt.wantErr && mgr == nil { + t.Error("NewTaskManager() returned nil manager") + } + }) + } +} + +func TestTaskManager_Create(t *testing.T) { + mgr, _ := setupTestManager(t) + ctx := context.Background() + + tests := []struct { + name string + task *types.Task + wantErr bool + }{ + { + name: "nil task", + task: nil, + wantErr: true, + }, + { + name: "empty task name", + task: &types.Task{ + Name: "", + Spec: v1alpha1.TaskSpec{ + Process: &v1alpha1.ProcessTask{ + Command: []string{"echo", "test"}, + }, + }, + }, + wantErr: true, + }, + { + name: "valid task", + task: &types.Task{ + Name: "test-task", + Spec: v1alpha1.TaskSpec{ + Process: &v1alpha1.ProcessTask{ + Command: []string{"sh", "-c", "echo hello && exit 0"}, + }, + }, + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + created, err := mgr.Create(ctx, tt.task) + if (err != nil) != tt.wantErr { + t.Errorf("Create() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if !tt.wantErr { + if created == nil { + t.Error("Create() returned nil task") + } + if created != nil && created.Name != tt.task.Name { + t.Errorf("Create() task name = %v, want %v", created.Name, tt.task.Name) + } + + // Wait for task to complete naturally + time.Sleep(200 * time.Millisecond) + // Then clean up + if tt.task != nil { + mgr.Delete(ctx, tt.task.Name) + } + } + }) + } +} + +func TestTaskManager_CreateDuplicate(t *testing.T) { + mgr, _ := setupTestManager(t) + mgr.Start(context.Background()) + defer mgr.Stop() + + ctx := context.Background() + + task := &types.Task{ + Name: "duplicate-task", + Spec: v1alpha1.TaskSpec{ + Process: &v1alpha1.ProcessTask{ + Command: []string{"echo", "test"}, + }, + }, + } + + // First create should succeed + _, err := mgr.Create(ctx, task) + if err != nil { + t.Fatalf("First Create() failed: %v", err) + } + defer cleanupTask(t, mgr, task.Name) + + // Second create should fail + _, err = mgr.Create(ctx, task) + if err == nil { + t.Error("Create() should fail for duplicate task") + } +} + +func TestTaskManager_CreateMaxConcurrentTasks(t *testing.T) { + mgr, _ := setupTestManager(t) + mgr.Start(context.Background()) + defer mgr.Stop() + + ctx := context.Background() + + task1 := &types.Task{ + Name: "task-1", + Spec: v1alpha1.TaskSpec{ + Process: &v1alpha1.ProcessTask{ + Command: []string{"sleep", "10"}, + }, + }, + } + + // Create first task + _, err := mgr.Create(ctx, task1) + if err != nil { + t.Fatalf("First Create() failed: %v", err) + } + defer cleanupTask(t, mgr, task1.Name) + + // Try to create second task - should fail due to max concurrent limit + task2 := &types.Task{ + Name: "task-2", + Spec: v1alpha1.TaskSpec{ + Process: &v1alpha1.ProcessTask{ + Command: []string{"echo", "test"}, + }, + }, + } + + _, err = mgr.Create(ctx, task2) + if err == nil { + t.Error("Create() should fail when max concurrent tasks reached") + cleanupTask(t, mgr, task2.Name) + } +} + +func TestTaskManager_Get(t *testing.T) { + mgr, _ := setupTestManager(t) + mgr.Start(context.Background()) + defer mgr.Stop() + + ctx := context.Background() + + task := &types.Task{ + Name: "get-task", + Spec: v1alpha1.TaskSpec{ + Process: &v1alpha1.ProcessTask{ + Command: []string{"echo", "get"}, + }, + }, + } + + // Create task + _, err := mgr.Create(ctx, task) + if err != nil { + t.Fatalf("Create() failed: %v", err) + } + defer cleanupTask(t, mgr, task.Name) + + // Get task + got, err := mgr.Get(ctx, task.Name) + if err != nil { + t.Fatalf("Get() failed: %v", err) + } + + if got.Name != task.Name { + t.Errorf("Get() name = %v, want %v", got.Name, task.Name) + } +} + +func TestTaskManager_GetNotFound(t *testing.T) { + mgr, _ := setupTestManager(t) + ctx := context.Background() + + _, err := mgr.Get(ctx, "non-existent") + if err == nil { + t.Error("Get() should fail for non-existent task") + } +} + +func TestTaskManager_GetEmptyName(t *testing.T) { + mgr, _ := setupTestManager(t) + ctx := context.Background() + + _, err := mgr.Get(ctx, "") + if err == nil { + t.Error("Get() should fail for empty name") + } +} + +func TestTaskManager_List(t *testing.T) { + mgr, _ := setupTestManager(t) + ctx := context.Background() + + // Initially empty + tasks, err := mgr.List(ctx) + if err != nil { + t.Fatalf("List() failed: %v", err) + } + if len(tasks) != 0 { + t.Errorf("List() initial count = %d, want 0", len(tasks)) + } + + // Create a task + task := &types.Task{ + Name: "list-task", + Spec: v1alpha1.TaskSpec{ + Process: &v1alpha1.ProcessTask{ + Command: []string{"echo", "list"}, + }, + }, + } + + _, err = mgr.Create(ctx, task) + if err != nil { + t.Fatalf("Create() failed: %v", err) + } + defer mgr.Delete(ctx, task.Name) + + // List should return 1 task + tasks, err = mgr.List(ctx) + if err != nil { + t.Fatalf("List() failed: %v", err) + } + if len(tasks) != 1 { + t.Errorf("List() count = %d, want 1", len(tasks)) + } + if tasks[0].Name != task.Name { + t.Errorf("List() task name = %v, want %v", tasks[0].Name, task.Name) + } +} + +func TestTaskManager_Delete(t *testing.T) { + mgr, _ := setupTestManager(t) + // Start the manager to enable the reconcile loop + mgr.Start(context.Background()) + defer mgr.Stop() + + ctx := context.Background() + + task := &types.Task{ + Name: "delete-task", + Spec: v1alpha1.TaskSpec{ + Process: &v1alpha1.ProcessTask{ + Command: []string{"echo", "delete"}, + }, + }, + } + + // Create task + _, err := mgr.Create(ctx, task) + if err != nil { + t.Fatalf("Create() failed: %v", err) + } + + // Delete task (soft delete) + err = mgr.Delete(ctx, task.Name) + if err != nil { + t.Errorf("Delete() failed: %v", err) + } + + // Verify task is marked for deletion but still exists + got, err := mgr.Get(ctx, task.Name) + if err != nil { + t.Fatalf("Get() should succeed after Delete() (soft delete): %v", err) + } + if got.DeletionTimestamp == nil { + t.Error("DeletionTimestamp should be set after Delete()") + } + + // Wait for task to be finalized + timeout := 5 * time.Second + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + _, err := mgr.Get(ctx, task.Name) + if err != nil { + // Task is gone, success + return + } + time.Sleep(100 * time.Millisecond) + } + t.Error("Task was not finalized (deleted) within timeout") +} + +func TestTaskManager_DeleteNonExistent(t *testing.T) { + mgr, _ := setupTestManager(t) + ctx := context.Background() + + // Delete non-existent task should not error + err := mgr.Delete(ctx, "non-existent") + if err != nil { + t.Errorf("Delete() should not fail for non-existent task: %v", err) + } +} + +func TestTaskManager_Sync(t *testing.T) { + mgr, _ := setupTestManager(t) + // Start the manager to enable the reconcile loop + mgr.Start(context.Background()) + defer mgr.Stop() + + ctx := context.Background() + + // Create initial task + task1 := &types.Task{ + Name: "sync-task-1", + Spec: v1alpha1.TaskSpec{ + Process: &v1alpha1.ProcessTask{ + Command: []string{"echo", "1"}, + }, + }, + } + + _, err := mgr.Create(ctx, task1) + if err != nil { + t.Fatalf("Create() failed: %v", err) + } + + // Sync with new desired state (task1 removed, task2 added) + task2 := &types.Task{ + Name: "sync-task-2", + Spec: v1alpha1.TaskSpec{ + Process: &v1alpha1.ProcessTask{ + Command: []string{"echo", "2"}, + }, + }, + } + + // Sync triggers soft delete for task1 and creation of task2 + current, err := mgr.Sync(ctx, []*types.Task{task2}) + if err != nil { + t.Fatalf("Sync() failed: %v", err) + } + defer mgr.Delete(ctx, task2.Name) + + // Verify task1 is marked for deletion in the returned list + var task1Found bool + for _, t1 := range current { + if t1.Name == task1.Name { + task1Found = true + if t1.DeletionTimestamp == nil { + t.Error("task1 should be marked for deletion after Sync()") + } + } + } + if !task1Found { + // It's possible it was deleted super fast, but unlikely + t.Log("task1 not found in Sync result (maybe already deleted?)") + } + + // Verify task2 is created + var task2Found bool + for _, t2 := range current { + if t2.Name == task2.Name { + task2Found = true + } + } + if !task2Found { + t.Error("task2 should be present after Sync()") + } + + // Wait for task1 to be finalized + timeout := 5 * time.Second + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + _, err := mgr.Get(ctx, task1.Name) + if err != nil { + // Task is gone, success + return + } + time.Sleep(100 * time.Millisecond) + } + t.Error("task1 should be deleted after Sync()") +} + +func TestTaskManager_SyncNil(t *testing.T) { + mgr, _ := setupTestManager(t) + ctx := context.Background() + + _, err := mgr.Sync(ctx, nil) + if err == nil { + t.Error("Sync() should fail for nil desired list") + } +} diff --git a/kubernetes/internal/task-executor/runtime/composite.go b/kubernetes/internal/task-executor/runtime/composite.go new file mode 100644 index 00000000..c279c63c --- /dev/null +++ b/kubernetes/internal/task-executor/runtime/composite.go @@ -0,0 +1,97 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package runtime + +import ( + "context" + "fmt" + + "k8s.io/klog/v2" + + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/config" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/types" +) + +func NewExecutor(cfg *config.Config) (Executor, error) { + if cfg == nil { + return nil, fmt.Errorf("config cannot be nil") + } + + // 1. Initialize ProcessExecutor (Always available for Host/Sidecar modes) + procExec, err := NewProcessExecutor(cfg) + if err != nil { + return nil, fmt.Errorf("failed to create process executor: %w", err) + } + klog.InfoS("process executor initialized.", "enableSidecar", cfg.EnableSidecarMode, "mainContainer", cfg.MainContainerName) + + // 2. Initialize ContainerExecutor (Optional) + var containerExec Executor + if cfg.EnableContainerMode { + klog.InfoS("container executor initialized", "criSocket", cfg.CRISocket) + containerExec, err = newContainerExecutor(cfg) + if err != nil { + return nil, fmt.Errorf("failed to create container executor: %w", err) + } + } + // 3. Return Composite + return &compositeExecutor{ + processExec: procExec, + containerExec: containerExec, + }, nil +} + +// compositeExecutor dispatches tasks to the appropriate underlying executor +type compositeExecutor struct { + processExec Executor + containerExec Executor +} + +func (e *compositeExecutor) getDelegate(task *types.Task) (Executor, error) { + if task == nil { + return nil, fmt.Errorf("task cannot be nil") + } + executor := e.processExec + if task.Spec.Container != nil { + executor = e.containerExec + } + if executor == nil { + return nil, fmt.Errorf("no executor available for task: %s", task.Name) + } + return executor, nil +} + +func (e *compositeExecutor) Start(ctx context.Context, task *types.Task) error { + delegate, err := e.getDelegate(task) + if err != nil { + return err + } + return delegate.Start(ctx, task) +} + +func (e *compositeExecutor) Inspect(ctx context.Context, task *types.Task) (*types.Status, error) { + delegate, err := e.getDelegate(task) + if err != nil { + return nil, err + } + return delegate.Inspect(ctx, task) +} + +func (e *compositeExecutor) Stop(ctx context.Context, task *types.Task) error { + delegate, err := e.getDelegate(task) + if err != nil { + return err + } + return delegate.Stop(ctx, task) +} diff --git a/kubernetes/internal/task-executor/runtime/container.go b/kubernetes/internal/task-executor/runtime/container.go new file mode 100644 index 00000000..63fc0ba0 --- /dev/null +++ b/kubernetes/internal/task-executor/runtime/container.go @@ -0,0 +1,55 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package runtime + +import ( + "context" + "errors" + "fmt" + + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/config" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/types" +) + +type containerExecutor struct { + config *config.Config +} + +// newContainerExecutor creates a new container-based task executor. +// This is a placeholder implementation - container mode is not yet supported. +func newContainerExecutor(cfg *config.Config) (Executor, error) { + if cfg == nil { + return nil, fmt.Errorf("config cannot be nil") + } + + return &containerExecutor{ + config: cfg, + }, nil +} + +// Start is not implemented for container mode yet. +func (e *containerExecutor) Start(ctx context.Context, task *types.Task) error { + return errors.New("container mode is not implemented yet - use process mode instead") +} + +// Inspect is not implemented for container mode yet. +func (e *containerExecutor) Inspect(ctx context.Context, task *types.Task) (*types.Status, error) { + return nil, errors.New("container mode is not implemented yet - use process mode instead") +} + +// Stop is not implemented for container mode yet. +func (e *containerExecutor) Stop(ctx context.Context, task *types.Task) error { + return errors.New("container mode is not implemented yet - use process mode instead") +} diff --git a/kubernetes/internal/task-executor/runtime/interface.go b/kubernetes/internal/task-executor/runtime/interface.go new file mode 100644 index 00000000..4609ccef --- /dev/null +++ b/kubernetes/internal/task-executor/runtime/interface.go @@ -0,0 +1,30 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package runtime + +import ( + "context" + + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/types" +) + +// Executor defines the contract for running tasks across different modes. +type Executor interface { + Start(ctx context.Context, task *types.Task) error + // Inspect retrieves the current runtime state. + Inspect(ctx context.Context, task *types.Task) (*types.Status, error) + + Stop(ctx context.Context, task *types.Task) error +} diff --git a/kubernetes/internal/task-executor/runtime/process.go b/kubernetes/internal/task-executor/runtime/process.go new file mode 100644 index 00000000..2995e98b --- /dev/null +++ b/kubernetes/internal/task-executor/runtime/process.go @@ -0,0 +1,456 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package runtime + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" + + "k8s.io/klog/v2" + + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/config" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/types" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/utils" +) + +const ( + ExitFile = "exit" + PidFile = "pid" + StdoutFile = "stdout.log" + StderrFile = "stderr.log" +) + +// processExecutor handles both Host and Sidecar modes as they share the same +// shim-based process execution model. +type processExecutor struct { + config *config.Config + rootDir string +} + +func NewProcessExecutor(config *config.Config) (Executor, error) { + return &processExecutor{rootDir: config.DataDir, config: config}, nil +} + +func (e *processExecutor) Start(ctx context.Context, task *types.Task) error { + if task == nil { + return fmt.Errorf("task cannot be nil") + } + taskDir, err := utils.SafeJoin(e.rootDir, task.Name) + if err != nil { + return fmt.Errorf("invalid task name: %w", err) + } + pidPath := filepath.Join(taskDir, PidFile) + exitPath := filepath.Join(taskDir, ExitFile) + + // 1. Construct the user command securely + var cmdList []string + if task.Spec.Process != nil { + cmdList = append(task.Spec.Process.Command, task.Spec.Process.Args...) + } else { + return fmt.Errorf("process spec is required for process executor but task.Spec.Process is nil (task name: %s)", task.Name) + } + + if len(cmdList) == 0 { + return fmt.Errorf("no command specified in process spec (task name: %s)", task.Name) + } + + // Use shell escaping to prevent command injection + safeCmdStr := shellEscape(cmdList) + shimScript := e.buildShimScript(exitPath, safeCmdStr) + + // 2. Prepare the execution command based on mode + var cmd *exec.Cmd + + if e.config.EnableSidecarMode { + // Sidecar Logic: Find target PID and use nsenter + targetPID, err := e.findPidByEnvVar("SANDBOX_MAIN_CONTAINER", e.config.MainContainerName) + if err != nil { + return fmt.Errorf("failed to resolve target PID: %w", err) + } + + nsenterArgs := []string{ + "-t", strconv.Itoa(targetPID), + "--mount", "--uts", "--ipc", "--net", "--pid", + "--", + "/bin/sh", "-c", shimScript, + } + cmd = exec.Command("nsenter", nsenterArgs...) + klog.InfoS("Starting sidecar task", "id", task.Name, "targetPID", targetPID) + + } else { + // Host Logic: Direct execution + // Use exec.Command instead of CommandContext to ensure the process survives + // after the HTTP request context is cancelled. + cmd = exec.Command("/bin/sh", "-c", shimScript) + klog.InfoS("Starting host task", "name", task.Name, "cmd", safeCmdStr, "exitPath", exitPath) + } + + // Set process group ID to isolate from parent process lifecycle + // This applies to both Host and Sidecar (nsenter) processes + cmd.SysProcAttr = &syscall.SysProcAttr{ + Setpgid: true, // Create new process group + Pgid: 0, // Use PID as PGID + } + + // 3. Execute common logic (logs, shim start) + return e.executeCommand(task, cmd, pidPath) +} + +// executeCommand handles log setup and process starting +func (e *processExecutor) executeCommand(task *types.Task, cmd *exec.Cmd, pidPath string) error { + if task == nil || cmd == nil { + return fmt.Errorf("task and cmd cannot be nil") + } + + taskDir, err := utils.SafeJoin(e.rootDir, task.Name) + if err != nil { + return fmt.Errorf("invalid task name: %w", err) + } + + stdoutPath := filepath.Join(taskDir, StdoutFile) + stderrPath := filepath.Join(taskDir, StderrFile) + + stdoutFile, err := os.OpenFile(stdoutPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + if err != nil { + return fmt.Errorf("failed to open stdout: %w", err) + } + + stderrFile, err := os.OpenFile(stderrPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + if err != nil { + stdoutFile.Close() + return fmt.Errorf("failed to open stderr: %w", err) + } + + cmd.Stdout = stdoutFile + cmd.Stderr = stderrFile + + // Apply environment variables from ProcessTask spec + if task.Spec.Process != nil { + // Start with current environment + cmd.Env = os.Environ() + // Add task-specific environment variables + for _, env := range task.Spec.Process.Env { + if env.Name != "" { + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", env.Name, env.Value)) + } + } + + // Apply working directory + if task.Spec.Process.WorkingDir != "" { + cmd.Dir = task.Spec.Process.WorkingDir + klog.InfoS("Set working directory", "name", task.Name, "workingDir", task.Spec.Process.WorkingDir) + } + } + + if err := cmd.Start(); err != nil { + klog.ErrorS(err, "failed to start command", "name", task.Name) + stdoutFile.Close() + stderrFile.Close() + return fmt.Errorf("failed to start cmd: %w", err) + } + + // Write PID to file immediately (Host-side PID) + // This fixes the issue where sidecar tasks would write the container-internal PID + pid := cmd.Process.Pid + if err := os.WriteFile(pidPath, []byte(strconv.Itoa(pid)), 0644); err != nil { + klog.ErrorS(err, "failed to write pid file", "name", task.Name) + // Try to kill the process since we failed to track it + _ = cmd.Process.Kill() + stdoutFile.Close() + stderrFile.Close() + return fmt.Errorf("failed to write pid file: %w", err) + } + + klog.InfoS("Task command started successfully", "name", task.Name, "pid", pid) + + // Close file descriptors in parent; child process has inherited them + stdoutFile.Close() + stderrFile.Close() + + // Wait for process in background + go func() { + if err := cmd.Wait(); err != nil { + klog.ErrorS(err, "task process exited with error", "name", task.Name) + } else { + klog.InfoS("task process exited successfully", "name", task.Name) + } + }() + return nil +} + +func (e *processExecutor) buildShimScript(exitPath, cmdStr string) string { + // The shim script acts as a mini-init process. + // 1. It runs the user command in the background. + // 2. It traps SIGTERM and forwards it to the child process. + // 3. It waits for the child to exit and captures the exit code. + // This ensures graceful shutdown propagation in sidecar/host modes. + script := fmt.Sprintf(` +cleanup() { + if [ -n "$CHILD_PID" ]; then + kill -TERM "$CHILD_PID" 2>/dev/null + fi +} +trap cleanup TERM + +%s & +CHILD_PID=$! +wait "$CHILD_PID" +EXIT_CODE=$? + +printf "%%d" $EXIT_CODE > %s +exit $EXIT_CODE +`, cmdStr, shellEscapePath(exitPath)) + klog.InfoS("Generated shim script", "exitPath", exitPath, "script", script) + return script +} + +func (e *processExecutor) Inspect(ctx context.Context, task *types.Task) (*types.Status, error) { + taskDir, err := utils.SafeJoin(e.rootDir, task.Name) + if err != nil { + return nil, fmt.Errorf("invalid task name: %w", err) + } + exitPath := filepath.Join(taskDir, ExitFile) + pidPath := filepath.Join(taskDir, PidFile) + + status := &types.Status{ + State: types.TaskStateUnknown, + } + var pid int + + // 1. Check Exit File (Completed) + if exitData, err := os.ReadFile(exitPath); err == nil { + fileInfo, _ := os.Stat(exitPath) + exitCode, _ := strconv.Atoi(string(exitData)) + + status.ExitCode = exitCode + finishedAt := fileInfo.ModTime() + status.FinishedAt = &finishedAt + + if exitCode == 0 { + status.State = types.TaskStateSucceeded + status.Reason = "Succeeded" + } else { + status.State = types.TaskStateFailed + status.Reason = "Failed" + } + + // Try to read start time from PID file + if pidFileInfo, err := os.Stat(pidPath); err == nil { + startedAt := pidFileInfo.ModTime() + status.StartedAt = &startedAt + } + + return status, nil + } + + // 2. Check PID File (Running) + if pidData, err := os.ReadFile(pidPath); err == nil { + pid, _ = strconv.Atoi(strings.TrimSpace(string(pidData))) + fileInfo, _ := os.Stat(pidPath) + startedAt := fileInfo.ModTime() + status.StartedAt = &startedAt + + if isProcessRunning(pid) { + status.State = types.TaskStateRunning + } else { + // Process crashed + status.State = types.TaskStateFailed + status.ExitCode = 137 // Assume kill/crash + status.Reason = "ProcessCrashed" + status.Message = "Process exited without writing exit code" + // Use ModTime as FinishedAt for crash approximation + status.FinishedAt = &startedAt + } + return status, nil + } + + // 3. Pending + status.State = types.TaskStatePending + status.Reason = "Pending" + return status, nil +} + +func (e *processExecutor) Stop(ctx context.Context, task *types.Task) error { + // Read from pid file (Root PID: nsenter or sh) + taskDir, err := utils.SafeJoin(e.rootDir, task.Name) + if err != nil { + return fmt.Errorf("invalid task name: %w", err) + } + pidPath := filepath.Join(taskDir, PidFile) + pidData, err := os.ReadFile(pidPath) + if err != nil { + return nil // pid file does not exist, process might not be started + } + var pid int + pid, err = strconv.Atoi(strings.TrimSpace(string(pidData))) + if err != nil || pid == 0 { + return nil + } + klog.InfoS("Read PID from pid file", "name", task.Name, "pid", pid) + + // Target the process group (negative PID) + pgid := -pid + + // Determine target PID to signal + targetPID := 0 + if e.config.EnableSidecarMode { + // In Sidecar mode, pid is nsenter. We need to signal its child (Shim). + // We use /proc//task//children which is O(1) compared to scanning /proc. + children, err := getChildrenPIDs(pid) + if err == nil && len(children) > 0 { + targetPID = children[0] // Assume first child is Shim + klog.InfoS("Sidecar mode: targeted Shim process via /proc/children", "nsenterPID", pid, "shimPID", targetPID) + } else { + klog.Warning("Sidecar mode: failed to find child process via /proc/children, falling back to PGID", "pid", pid, "err", err) + } + } else { + // In Host mode, pid is the Shim itself. + targetPID = pid + } + + // 1. Send SIGTERM + // If we found a specific target (Shim), signal it. It will trap and forward to child. + // If not (or if signal fails), fallback to signaling the group. + killedShim := false + if targetPID > 0 { + if err := syscall.Kill(targetPID, syscall.SIGTERM); err == nil { + killedShim = true + } else if err != syscall.ESRCH { + klog.ErrorS(err, "Failed to send SIGTERM to target process", "targetPID", targetPID) + } + } + + if !killedShim { + // Fallback: kill the group. + // Note: In Sidecar mode, this might kill nsenter before Shim exits, risking zombies. + _ = syscall.Kill(pgid, syscall.SIGTERM) + } + + // 2. Wait for process to exit (Graceful shutdown) + // Poll every 500ms for up to 10 seconds + timeout := 10 * time.Second + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if !isProcessRunning(pid) { + return nil + } + time.Sleep(500 * time.Millisecond) + } + + // 3. Force Kill (SIGKILL) + klog.InfoS("Process did not exit after timeout, sending SIGKILL", "pgid", pgid) + if targetPID > 0 { + _ = syscall.Kill(targetPID, syscall.SIGKILL) + } + _ = syscall.Kill(pgid, syscall.SIGKILL) + + return nil +} + +// getChildrenPIDs reads /proc//task//children to find direct children. +// This requires kernel 3.5+ and CONFIG_PROC_CHILDREN. +func getChildrenPIDs(pid int) ([]int, error) { + path := fmt.Sprintf("/proc/%d/task/%d/children", pid, pid) + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + var pids []int + for _, field := range strings.Fields(string(data)) { + if id, err := strconv.Atoi(field); err == nil { + pids = append(pids, id) + } + } + return pids, nil +} + +// Helpers +func isProcessRunning(pid int) bool { + process, err := os.FindProcess(pid) + if err != nil { + return false + } + return process.Signal(syscall.Signal(0)) == nil +} + +// shellEscape quotes arguments for safe shell execution +func shellEscape(args []string) string { + quoted := make([]string, len(args)) + for i, s := range args { + quoted[i] = shellEscapePath(s) + } + return strings.Join(quoted, " ") +} + +// shellEscapePath escapes a single string for safe shell execution. +// It wraps the string in single quotes and escapes any embedded single quotes. +// e.g., foo'bar -> 'foo'\”bar' +func shellEscapePath(s string) string { + return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'" +} + +// findPidByEnvVar finds a process by checking for a specific environment variable. +// It looks for processes with SANDBOX_MAIN_CONTAINER= in their environment. +func (e *processExecutor) findPidByEnvVar(envName, expectedValue string) (int, error) { + procDir, err := os.Open("/proc") + if err != nil { + return 0, fmt.Errorf("failed to open /proc: %w", err) + } + defer procDir.Close() + + entries, err := procDir.Readdirnames(-1) + if err != nil { + return 0, fmt.Errorf("failed to read /proc entries: %w", err) + } + + selfPID := os.Getpid() + targetEnv := fmt.Sprintf("%s=%s", envName, expectedValue) + + for _, entry := range entries { + pid, err := strconv.Atoi(entry) + if err != nil { + continue + } + if pid == selfPID { + continue + } + + // Read process environment + envPath := filepath.Join("/proc", entry, "environ") + envData, err := os.ReadFile(envPath) + if err != nil { + continue + } + + // Environment variables are null-separated + envVars := strings.Split(string(envData), "\x00") + for _, env := range envVars { + if env == targetEnv { + klog.InfoS("Found main container by environment variable", "pid", pid, "env", targetEnv) + return pid, nil + } + } + } + + return 0, fmt.Errorf("no process found with environment variable %s=%s", envName, expectedValue) +} diff --git a/kubernetes/internal/task-executor/runtime/process_test.go b/kubernetes/internal/task-executor/runtime/process_test.go new file mode 100644 index 00000000..e2a25649 --- /dev/null +++ b/kubernetes/internal/task-executor/runtime/process_test.go @@ -0,0 +1,247 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package runtime + +import ( + "context" + "os" + "os/exec" + "testing" + "time" + + "github.com/alibaba/OpenSandbox/sandbox-k8s/api/v1alpha1" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/config" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/types" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/utils" + "github.com/stretchr/testify/assert" +) + +func setupTestExecutor(t *testing.T) (Executor, string) { + dataDir := t.TempDir() + cfg := &config.Config{ + DataDir: dataDir, + EnableSidecarMode: false, + } + executor, err := NewProcessExecutor(cfg) + if err != nil { + t.Fatalf("Failed to create executor: %v", err) + } + return executor, dataDir +} + +func TestProcessExecutor_Lifecycle(t *testing.T) { + // Skip if not running on Linux/Unix-like systems where sh is available + if _, err := exec.LookPath("sh"); err != nil { + t.Skip("sh not found, skipping process executor test") + } + + executor, _ := setupTestExecutor(t) + pExecutor := executor.(*processExecutor) + ctx := context.Background() + + // 1. Create a task that runs for a while + task := &types.Task{ + Name: "long-running", + Spec: v1alpha1.TaskSpec{ + Process: &v1alpha1.ProcessTask{ + Command: []string{"/bin/sh", "-c", "sleep 10"}, + }, + }, + } + + // Create task directory manually (normally handled by store) + + taskDir, err := utils.SafeJoin(pExecutor.rootDir, task.Name) + assert.Nil(t, err) + os.MkdirAll(taskDir, 0755) + + // 2. Start + if err := executor.Start(ctx, task); err != nil { + t.Fatalf("Start failed: %v", err) + } + + // 3. Inspect (Running) + status, err := executor.Inspect(ctx, task) + if err != nil { + t.Fatalf("Inspect failed: %v", err) + } + if status.State != types.TaskStateRunning { + t.Errorf("Task should be running, got: %s", status.State) + } + + // 4. Stop + if err := executor.Stop(ctx, task); err != nil { + t.Fatalf("Stop failed: %v", err) + } + + // 5. Inspect (Terminated) + // Wait a bit for file to be written + time.Sleep(100 * time.Millisecond) + status, err = executor.Inspect(ctx, task) + if err != nil { + t.Fatalf("Inspect failed: %v", err) + } + // sleep command killed by signal results in non-zero exit code, so it's Failed + if status.State != types.TaskStateFailed { + t.Errorf("Task should be failed (terminated), got: %s", status.State) + } +} + +func TestProcessExecutor_ShortLived(t *testing.T) { + if _, err := exec.LookPath("sh"); err != nil { + t.Skip("sh not found") + } + + executor, _ := setupTestExecutor(t) + pExecutor := executor.(*processExecutor) + ctx := context.Background() + + task := &types.Task{ + Name: "short-lived", + Spec: v1alpha1.TaskSpec{ + Process: &v1alpha1.ProcessTask{ + Command: []string{"echo", "done"}, + }, + }, + } + taskDir, err := utils.SafeJoin(pExecutor.rootDir, task.Name) + assert.Nil(t, err) + os.MkdirAll(taskDir, 0755) + + if err := executor.Start(ctx, task); err != nil { + t.Fatalf("Start failed: %v", err) + } + + // Wait for process to finish + time.Sleep(200 * time.Millisecond) + + status, err := executor.Inspect(ctx, task) + if err != nil { + t.Fatalf("Inspect failed: %v", err) + } + if status.State != types.TaskStateSucceeded { + t.Errorf("Task should be succeeded, got: %s", status.State) + } + if status.ExitCode != 0 { + t.Errorf("Exit code should be 0, got %d", status.ExitCode) + } +} + +func TestProcessExecutor_Failure(t *testing.T) { + if _, err := exec.LookPath("sh"); err != nil { + t.Skip("sh not found") + } + + executor, _ := setupTestExecutor(t) + pExecutor := executor.(*processExecutor) + ctx := context.Background() + + task := &types.Task{ + Name: "failing-task", + Spec: v1alpha1.TaskSpec{ + Process: &v1alpha1.ProcessTask{ + Command: []string{"/bin/sh", "-c", "exit 1"}, + }, + }, + } + taskDir, err := utils.SafeJoin(pExecutor.rootDir, task.Name) + assert.Nil(t, err) + os.MkdirAll(taskDir, 0755) + + if err := executor.Start(ctx, task); err != nil { + t.Fatalf("Start failed: %v", err) + } + + time.Sleep(200 * time.Millisecond) + + status, err := executor.Inspect(ctx, task) + if err != nil { + t.Fatalf("Inspect failed: %v", err) + } + if status.State != types.TaskStateFailed { + t.Errorf("Task should be failed") + } else if status.ExitCode != 1 { + t.Errorf("Exit code should be 1, got %d", status.ExitCode) + } +} + +func TestProcessExecutor_InvalidArgs(t *testing.T) { + exec, _ := setupTestExecutor(t) + ctx := context.Background() + + // Nil task + if err := exec.Start(ctx, nil); err == nil { + t.Error("Start should fail with nil task") + } + + // Missing process spec + task := &types.Task{ + Name: "invalid", + Spec: v1alpha1.TaskSpec{}, + } + if err := exec.Start(ctx, task); err == nil { + t.Error("Start should fail with missing process spec") + } +} + +func TestShellEscape(t *testing.T) { + tests := []struct { + input []string + expected string + }{ + {[]string{"echo", "hello"}, "'echo' 'hello'"}, + {[]string{"echo", "hello world"}, "'echo' 'hello world'"}, + {[]string{"foo'bar"}, "'foo'\\''bar'"}, + } + + for _, tt := range tests { + got := shellEscape(tt.input) + if got != tt.expected { + t.Errorf("shellEscape(%v) = %q, want %q", tt.input, got, tt.expected) + } + } +} + +func TestNewExecutor(t *testing.T) { + // 1. Container mode + Host Mode + cfg := &config.Config{ + EnableContainerMode: true, + } + e, err := NewExecutor(cfg) + if err != nil { + t.Fatalf("NewExecutor(container) failed: %v", err) + } + if _, ok := e.(*compositeExecutor); !ok { + t.Error("NewExecutor should return CompositeExecutor") + } + + // 2. Process mode only + cfg = &config.Config{ + EnableContainerMode: false, + DataDir: t.TempDir(), + } + e, err = NewExecutor(cfg) + if err != nil { + t.Fatalf("NewExecutor(process) failed: %v", err) + } + if _, ok := e.(*compositeExecutor); !ok { + t.Error("NewExecutor should return CompositeExecutor") + } + + // 3. Nil config + if _, err := NewExecutor(nil); err == nil { + t.Error("NewExecutor should fail with nil config") + } +} diff --git a/kubernetes/internal/task-executor/server/handler.go b/kubernetes/internal/task-executor/server/handler.go new file mode 100644 index 00000000..92fe206e --- /dev/null +++ b/kubernetes/internal/task-executor/server/handler.go @@ -0,0 +1,315 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "encoding/json" + "fmt" + "net/http" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" + + "github.com/alibaba/OpenSandbox/sandbox-k8s/api/v1alpha1" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/config" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/manager" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/types" + api "github.com/alibaba/OpenSandbox/sandbox-k8s/pkg/task-executor" +) + +// ErrorResponse represents a standard error response. +type ErrorResponse struct { + Code string `json:"code"` + Message string `json:"message"` +} + +type Handler struct { + manager manager.TaskManager + config *config.Config +} + +func NewHandler(mgr manager.TaskManager, cfg *config.Config) *Handler { + if mgr == nil { + klog.Warning("TaskManager is nil, handler may not work properly") + } + if cfg == nil { + klog.Warning("Config is nil, handler may not work properly") + } + return &Handler{ + manager: mgr, + config: cfg, + } +} + +func (h *Handler) CreateTask(w http.ResponseWriter, r *http.Request) { + if h.manager == nil { + writeError(w, http.StatusInternalServerError, "task manager not initialized") + return + } + + // Parse request body + var apiTask api.Task + if err := json.NewDecoder(r.Body).Decode(&apiTask); err != nil { + writeError(w, http.StatusBadRequest, fmt.Sprintf("invalid request body: %v", err)) + return + } + + // Validate task + if apiTask.Name == "" { + writeError(w, http.StatusBadRequest, "task name is required") + return + } + + // Convert to internal model + task := h.convertAPIToInternalTask(&apiTask) + if task == nil { + writeError(w, http.StatusBadRequest, "failed to convert task") + return + } + + // Create task + created, err := h.manager.Create(r.Context(), task) + if err != nil { + klog.ErrorS(err, "failed to create task", "name", apiTask.Name) + writeError(w, http.StatusInternalServerError, fmt.Sprintf("failed to create task: %v", err)) + return + } + + // Convert back to API model + response := convertInternalToAPITask(created) + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + json.NewEncoder(w).Encode(response) + + klog.InfoS("task created via API", "name", apiTask.Name) +} + +func (h *Handler) SyncTasks(w http.ResponseWriter, r *http.Request) { + if h.manager == nil { + writeError(w, http.StatusInternalServerError, "task manager not initialized") + return + } + + // Parse request body - array of tasks + var apiTasks []api.Task + if err := json.NewDecoder(r.Body).Decode(&apiTasks); err != nil { + writeError(w, http.StatusBadRequest, fmt.Sprintf("invalid request body: %v", err)) + return + } + + // Convert to internal model + desired := make([]*types.Task, 0, len(apiTasks)) + for i := range apiTasks { + if apiTasks[i].Name == "" { + continue // Skip invalid tasks + } + task := h.convertAPIToInternalTask(&apiTasks[i]) + if task != nil { + desired = append(desired, task) + } + } + + // Sync tasks + current, err := h.manager.Sync(r.Context(), desired) + if err != nil { + klog.ErrorS(err, "failed to sync tasks") + writeError(w, http.StatusInternalServerError, fmt.Sprintf("failed to sync tasks: %v", err)) + return + } + + // Convert back to API model + response := make([]api.Task, 0, len(current)) + for _, task := range current { + if task != nil { + response = append(response, *convertInternalToAPITask(task)) + } + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + + klog.InfoS("tasks synced via API", "count", len(response)) +} + +func (h *Handler) GetTask(w http.ResponseWriter, r *http.Request) { + if h.manager == nil { + writeError(w, http.StatusInternalServerError, "task manager not initialized") + return + } + + // Extract task ID from path + taskID := r.PathValue("id") + if taskID == "" { + writeError(w, http.StatusBadRequest, "task id is required") + return + } + + // Get task + task, err := h.manager.Get(r.Context(), taskID) + if err != nil { + klog.ErrorS(err, "failed to get task", "id", taskID) + writeError(w, http.StatusNotFound, fmt.Sprintf("task not found: %v", err)) + return + } + + // Convert to API model + response := convertInternalToAPITask(task) + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func (h *Handler) ListTasks(w http.ResponseWriter, r *http.Request) { + if h.manager == nil { + writeError(w, http.StatusInternalServerError, "task manager not initialized") + return + } + + // List all tasks + tasks, err := h.manager.List(r.Context()) + if err != nil { + klog.ErrorS(err, "failed to list tasks") + writeError(w, http.StatusInternalServerError, fmt.Sprintf("failed to list tasks: %v", err)) + return + } + + // Convert to API model + response := make([]api.Task, 0, len(tasks)) + for _, task := range tasks { + if task != nil { + response = append(response, *convertInternalToAPITask(task)) + } + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +// Health returns the health status of the task executor +func (h *Handler) Health(w http.ResponseWriter, r *http.Request) { + response := map[string]string{ + "status": "healthy", + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) +} + +func (h *Handler) DeleteTask(w http.ResponseWriter, r *http.Request) { + if h.manager == nil { + writeError(w, http.StatusInternalServerError, "task manager not initialized") + return + } + + // Extract task ID from path + taskID := r.PathValue("id") + if taskID == "" { + writeError(w, http.StatusBadRequest, "task id is required") + return + } + + // Delete task + err := h.manager.Delete(r.Context(), taskID) + if err != nil { + klog.ErrorS(err, "failed to delete task", "id", taskID) + writeError(w, http.StatusInternalServerError, fmt.Sprintf("failed to delete task: %v", err)) + return + } + + w.WriteHeader(http.StatusNoContent) + klog.InfoS("task deleted via API", "id", taskID) +} + +// writeError writes an error response in JSON format. +func writeError(w http.ResponseWriter, code int, message string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + json.NewEncoder(w).Encode(ErrorResponse{ + Code: http.StatusText(code), + Message: message, + }) +} + +// convertAPIToInternalTask converts api.Task to types.Task. +func (h *Handler) convertAPIToInternalTask(apiTask *api.Task) *types.Task { + if apiTask == nil { + return nil + } + task := &types.Task{ + Name: apiTask.Name, + Spec: apiTask.Spec, + } + // Initialize default status + task.Status = types.Status{ + State: types.TaskStatePending, + } + + return task +} + +// convertInternalToAPITask converts types.Task to api.Task. +func convertInternalToAPITask(task *types.Task) *api.Task { + if task == nil { + return nil + } + + apiTask := &api.Task{ + Name: task.Name, + Spec: task.Spec, + } + + // Map internal Status to v1alpha1.TaskStatus + apiStatus := v1alpha1.TaskStatus{ + State: v1alpha1.TaskState{}, + } + + switch task.Status.State { + case types.TaskStatePending: + apiStatus.State.Waiting = &v1alpha1.TaskStateWaiting{ + Reason: task.Status.Reason, + } + case types.TaskStateRunning: + if task.Status.StartedAt != nil { + t := metav1.NewTime(*task.Status.StartedAt) + apiStatus.State.Running = &v1alpha1.TaskStateRunning{ + StartedAt: t, + } + } else { + apiStatus.State.Running = &v1alpha1.TaskStateRunning{} + } + case types.TaskStateSucceeded, types.TaskStateFailed: + term := &v1alpha1.TaskStateTerminated{ + ExitCode: int32(task.Status.ExitCode), + Reason: task.Status.Reason, + Message: task.Status.Message, + } + if task.Status.StartedAt != nil { + t := metav1.NewTime(*task.Status.StartedAt) + term.StartedAt = t + } + if task.Status.FinishedAt != nil { + t := metav1.NewTime(*task.Status.FinishedAt) + term.FinishedAt = t + } + apiStatus.State.Terminated = term + default: + apiStatus.State.Waiting = &v1alpha1.TaskStateWaiting{ + Reason: "Unknown", + } + } + + apiTask.Status = apiStatus + return apiTask +} diff --git a/kubernetes/internal/task-executor/server/handler_test.go b/kubernetes/internal/task-executor/server/handler_test.go new file mode 100644 index 00000000..c66f85e7 --- /dev/null +++ b/kubernetes/internal/task-executor/server/handler_test.go @@ -0,0 +1,246 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/alibaba/OpenSandbox/sandbox-k8s/api/v1alpha1" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/config" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/types" + api "github.com/alibaba/OpenSandbox/sandbox-k8s/pkg/task-executor" +) + +// MockTaskManager implements manager.TaskManager for testing +type MockTaskManager struct { + tasks map[string]*types.Task + err error +} + +func NewMockTaskManager() *MockTaskManager { + return &MockTaskManager{ + tasks: make(map[string]*types.Task), + } +} + +func (m *MockTaskManager) Create(ctx context.Context, task *types.Task) (*types.Task, error) { + if m.err != nil { + return nil, m.err + } + m.tasks[task.Name] = task + return task, nil +} + +func (m *MockTaskManager) Sync(ctx context.Context, desired []*types.Task) ([]*types.Task, error) { + if m.err != nil { + return nil, m.err + } + m.tasks = make(map[string]*types.Task) + var result []*types.Task + for _, t := range desired { + m.tasks[t.Name] = t + result = append(result, t) + } + return result, nil +} + +func (m *MockTaskManager) Get(ctx context.Context, id string) (*types.Task, error) { + if m.err != nil { + return nil, m.err + } + if t, ok := m.tasks[id]; ok { + return t, nil + } + return nil, fmt.Errorf("not found") +} + +func (m *MockTaskManager) List(ctx context.Context) ([]*types.Task, error) { + if m.err != nil { + return nil, m.err + } + var list []*types.Task + for _, t := range m.tasks { + list = append(list, t) + } + return list, nil +} + +func (m *MockTaskManager) Delete(ctx context.Context, id string) error { + if m.err != nil { + return m.err + } + delete(m.tasks, id) + return nil +} + +func (m *MockTaskManager) Start(ctx context.Context) {} +func (m *MockTaskManager) Stop() {} + +func TestHandler_Health(t *testing.T) { + cfg := &config.Config{} + h := NewHandler(NewMockTaskManager(), cfg) + req := httptest.NewRequest("GET", "/health", nil) + w := httptest.NewRecorder() + + h.Health(w, req) + + if w.Code != http.StatusOK { + t.Errorf("Health returned status %d", w.Code) + } +} + +func TestHandler_CreateTask(t *testing.T) { + mgr := NewMockTaskManager() + cfg := &config.Config{} + h := NewHandler(mgr, cfg) + + task := api.Task{ + Name: "test-task", + Spec: v1alpha1.TaskSpec{ + Process: &v1alpha1.ProcessTask{ + Command: []string{"echo"}, + }, + }, + } + body, _ := json.Marshal(task) + + req := httptest.NewRequest("POST", "/tasks", bytes.NewReader(body)) + w := httptest.NewRecorder() + + h.CreateTask(w, req) + + if w.Code != http.StatusCreated { + t.Errorf("CreateTask returned status %d", w.Code) + } + + if _, ok := mgr.tasks["test-task"]; !ok { + t.Error("Task was not created in manager") + } +} + +func TestHandler_GetTask(t *testing.T) { + mgr := NewMockTaskManager() + mgr.tasks["test-task"] = &types.Task{Name: "test-task"} + cfg := &config.Config{} + h := NewHandler(mgr, cfg) + + router := NewRouter(h) + req := httptest.NewRequest("GET", "/tasks/test-task", nil) + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Errorf("GetTask returned status %d", w.Code) + } + + var resp api.Task + json.NewDecoder(w.Body).Decode(&resp) + if resp.Name != "test-task" { + t.Errorf("GetTask returned name %s", resp.Name) + } +} + +func TestHandler_DeleteTask(t *testing.T) { + mgr := NewMockTaskManager() + mgr.tasks["test-task"] = &types.Task{Name: "test-task"} + cfg := &config.Config{} + h := NewHandler(mgr, cfg) + router := NewRouter(h) + + req := httptest.NewRequest("DELETE", "/tasks/test-task", nil) + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + if w.Code != http.StatusNoContent { + t.Errorf("DeleteTask returned status %d", w.Code) + } + + if _, ok := mgr.tasks["test-task"]; ok { + t.Error("Task was not deleted from manager") + } +} + +func TestHandler_ListTasks(t *testing.T) { + mgr := NewMockTaskManager() + mgr.tasks["task-1"] = &types.Task{Name: "task-1"} + mgr.tasks["task-2"] = &types.Task{Name: "task-2"} + cfg := &config.Config{} + h := NewHandler(mgr, cfg) + + req := httptest.NewRequest("GET", "/getTasks", nil) + w := httptest.NewRecorder() + + h.ListTasks(w, req) + + if w.Code != http.StatusOK { + t.Errorf("ListTasks returned status %d", w.Code) + } + + var resp []api.Task + json.NewDecoder(w.Body).Decode(&resp) + if len(resp) != 2 { + t.Errorf("ListTasks returned %d tasks, want 2", len(resp)) + } +} + +func TestHandler_SyncTasks(t *testing.T) { + mgr := NewMockTaskManager() + cfg := &config.Config{} + h := NewHandler(mgr, cfg) + + tasks := []api.Task{ + {Name: "task-1", Spec: v1alpha1.TaskSpec{Process: &v1alpha1.ProcessTask{}}}, + } + body, _ := json.Marshal(tasks) + + req := httptest.NewRequest("POST", "/setTasks", bytes.NewReader(body)) + w := httptest.NewRecorder() + + h.SyncTasks(w, req) + + if w.Code != http.StatusOK { + t.Errorf("SyncTasks returned status %d", w.Code) + } + + if _, ok := mgr.tasks["task-1"]; !ok { + t.Error("Task was not synced to manager") + } +} + +func TestHandler_Errors(t *testing.T) { + mgr := NewMockTaskManager() + mgr.err = errors.New("mock error") + cfg := &config.Config{} + h := NewHandler(mgr, cfg) + + // Create fail + task := api.Task{Name: "fail"} + body, _ := json.Marshal(task) + req := httptest.NewRequest("POST", "/tasks", bytes.NewReader(body)) + w := httptest.NewRecorder() + h.CreateTask(w, req) + if w.Code != http.StatusInternalServerError { + t.Errorf("CreateTask should fail with 500, got %d", w.Code) + } +} diff --git a/kubernetes/internal/task-executor/server/router.go b/kubernetes/internal/task-executor/server/router.go new file mode 100644 index 00000000..52d3bd00 --- /dev/null +++ b/kubernetes/internal/task-executor/server/router.go @@ -0,0 +1,32 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "net/http" +) + +func NewRouter(h *Handler) http.Handler { + mux := http.NewServeMux() + + mux.HandleFunc("POST /setTasks", h.SyncTasks) + mux.HandleFunc("GET /getTasks", h.ListTasks) + mux.HandleFunc("POST /tasks", h.CreateTask) + mux.HandleFunc("GET /tasks/{id}", h.GetTask) + mux.HandleFunc("DELETE /tasks/{id}", h.DeleteTask) + mux.HandleFunc("GET /health", h.Health) + + return mux +} diff --git a/kubernetes/internal/task-executor/storage/file_store.go b/kubernetes/internal/task-executor/storage/file_store.go new file mode 100644 index 00000000..936830b2 --- /dev/null +++ b/kubernetes/internal/task-executor/storage/file_store.go @@ -0,0 +1,285 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package store + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sync" + + "k8s.io/klog/v2" + + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/types" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/utils" +) + +type fileStore struct { + dataDir string + locks sync.Map // key: taskName, value: *sync.RWMutex +} + +// NewFileStore creates a new file-based task store. +func NewFileStore(dataDir string) (TaskStore, error) { + if dataDir == "" { + return nil, fmt.Errorf("dataDir cannot be empty") + } + + if err := os.MkdirAll(dataDir, 0755); err != nil { + return nil, fmt.Errorf("failed to create data directory %s: %w", dataDir, err) + } + + testFile := filepath.Join(dataDir, ".test") + if err := os.WriteFile(testFile, []byte("test"), 0644); err != nil { + return nil, fmt.Errorf("data directory %s is not writable: %w", dataDir, err) + } + os.Remove(testFile) + + klog.InfoS("initialized file store", "dataDir", dataDir) + + return &fileStore{ + dataDir: dataDir, + }, nil +} + +// getTaskLock retrieves or creates a lock for a specific task. +func (s *fileStore) getTaskLock(name string) *sync.RWMutex { + val, _ := s.locks.LoadOrStore(name, &sync.RWMutex{}) + return val.(*sync.RWMutex) +} + +// Create persists a new task to disk. +func (s *fileStore) Create(ctx context.Context, task *types.Task) error { + if task == nil { + return fmt.Errorf("task cannot be nil") + } + if task.Name == "" { + return fmt.Errorf("task name cannot be empty") + } + + mu := s.getTaskLock(task.Name) + mu.Lock() + defer mu.Unlock() + + taskDir, err := utils.SafeJoin(s.dataDir, task.Name) + if err != nil { + return fmt.Errorf("invalid task name: %w", err) + } + + if _, err := os.Stat(taskDir); err == nil { + return fmt.Errorf("task %s already exists", task.Name) + } + + if err := os.MkdirAll(taskDir, 0755); err != nil { + return fmt.Errorf("failed to create task directory: %w", err) + } + + if err := s.writeTaskFile(taskDir, task); err != nil { + os.RemoveAll(taskDir) + return err + } + + klog.InfoS("created task", "name", task.Name, "dir", taskDir) + return nil +} + +// Update updates an existing task's runtime information. +func (s *fileStore) Update(ctx context.Context, task *types.Task) error { + if task == nil { + return fmt.Errorf("task cannot be nil") + } + if task.Name == "" { + return fmt.Errorf("task name cannot be empty") + } + + mu := s.getTaskLock(task.Name) + mu.Lock() + defer mu.Unlock() + + taskDir, err := utils.SafeJoin(s.dataDir, task.Name) + if err != nil { + return fmt.Errorf("invalid task name: %w", err) + } + + // Check if task exists + if _, err := os.Stat(taskDir); os.IsNotExist(err) { + return fmt.Errorf("task %s does not exist", task.Name) + } + + // Write task data + if err := s.writeTaskFile(taskDir, task); err != nil { + return err + } + + klog.InfoS("updated task", "name", task.Name) + return nil +} + +// Get retrieves a task by name. +func (s *fileStore) Get(ctx context.Context, name string) (*types.Task, error) { + if name == "" { + return nil, fmt.Errorf("task name cannot be empty") + } + + mu := s.getTaskLock(name) + mu.RLock() + defer mu.RUnlock() + + taskDir, err := utils.SafeJoin(s.dataDir, name) + if err != nil { + return nil, fmt.Errorf("invalid task name: %w", err) + } + + // Check if task exists + if _, err := os.Stat(taskDir); os.IsNotExist(err) { + return nil, fmt.Errorf("task %s not found", name) + } + + return s.readTaskFile(taskDir, name) +} + +// List returns all tasks in the store. +func (s *fileStore) List(ctx context.Context) ([]*types.Task, error) { + // Read all task directories + // Note: We don't have a global lock, so the list of tasks might change during iteration. + // This is acceptable for a file-based store. + entries, err := os.ReadDir(s.dataDir) + if err != nil { + return nil, fmt.Errorf("failed to read data directory: %w", err) + } + + tasks := make([]*types.Task, 0, len(entries)) + for _, entry := range entries { + if !entry.IsDir() { + continue + } + + taskName := entry.Name() + taskDir, err := utils.SafeJoin(s.dataDir, taskName) + if err != nil { + klog.ErrorS(err, "invalid task directory, skipping", "name", taskName) + continue + } + + // Acquire read lock for this specific task + mu := s.getTaskLock(taskName) + mu.RLock() + task, err := s.readTaskFile(taskDir, taskName) + mu.RUnlock() + + if err != nil { + klog.ErrorS(err, "failed to read task, skipping", "name", taskName) + continue + } + + tasks = append(tasks, task) + } + + return tasks, nil +} + +// Delete removes a task from the store. +func (s *fileStore) Delete(ctx context.Context, name string) error { + if name == "" { + return fmt.Errorf("task name cannot be empty") + } + + mu := s.getTaskLock(name) + mu.Lock() + defer mu.Unlock() + + taskDir, err := utils.SafeJoin(s.dataDir, name) + if err != nil { + return fmt.Errorf("invalid task name: %w", err) + } + + // Check if task exists + if _, err := os.Stat(taskDir); os.IsNotExist(err) { + klog.InfoS("task already deleted", "name", name) + return nil + } + + // Remove task directory + if err := os.RemoveAll(taskDir); err != nil { + return fmt.Errorf("failed to delete task %s: %w", name, err) + } + + klog.InfoS("deleted task", "name", name) + return nil +} + +// getTaskFilePath returns the file path for a task's JSON file. +func (s *fileStore) getTaskFilePath(taskDir string) string { + return filepath.Join(taskDir, "task.json") +} + +// writeTaskFile writes task data to disk atomically using temp file + rename. +func (s *fileStore) writeTaskFile(taskDir string, task *types.Task) error { + // Marshal to JSON + data, err := json.MarshalIndent(task, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal task: %w", err) + } + + taskFile := s.getTaskFilePath(taskDir) + tmpFile := taskFile + ".tmp" + + // Write to temporary file + if err := os.WriteFile(tmpFile, data, 0644); err != nil { + return fmt.Errorf("failed to write temp file: %w", err) + } + + // Sync to ensure data is written to disk + f, err := os.Open(tmpFile) + if err != nil { + os.Remove(tmpFile) + return fmt.Errorf("failed to open temp file for sync: %w", err) + } + if err := f.Sync(); err != nil { + f.Close() + os.Remove(tmpFile) + return fmt.Errorf("failed to sync temp file: %w", err) + } + f.Close() + + // Atomically rename temp file to final file + if err := os.Rename(tmpFile, taskFile); err != nil { + os.Remove(tmpFile) + return fmt.Errorf("failed to rename temp file: %w", err) + } + + return nil +} + +// readTaskFile reads task data from disk. +func (s *fileStore) readTaskFile(taskDir, taskName string) (*types.Task, error) { + taskFile := s.getTaskFilePath(taskDir) + + // Read file + data, err := os.ReadFile(taskFile) + if err != nil { + return nil, fmt.Errorf("failed to read task file: %w", err) + } + + // Unmarshal JSON + var task types.Task + if err := json.Unmarshal(data, &task); err != nil { + return nil, fmt.Errorf("failed to unmarshal task file: %w", err) + } + + return &task, nil +} diff --git a/kubernetes/internal/task-executor/storage/file_store_test.go b/kubernetes/internal/task-executor/storage/file_store_test.go new file mode 100644 index 00000000..5692eee0 --- /dev/null +++ b/kubernetes/internal/task-executor/storage/file_store_test.go @@ -0,0 +1,226 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package store + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/alibaba/OpenSandbox/sandbox-k8s/api/v1alpha1" + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/types" +) + +func TestNewFileStore(t *testing.T) { + // Test case 1: Valid directory + tmpDir := t.TempDir() + store, err := NewFileStore(tmpDir) + if err != nil { + t.Fatalf("NewFileStore failed: %v", err) + } + if store == nil { + t.Fatal("NewFileStore returned nil store") + } + + // Test case 2: Empty directory + _, err = NewFileStore("") + if err == nil { + t.Fatal("NewFileStore should fail with empty dir") + } +} + +func TestFileStore_CRUD(t *testing.T) { + tmpDir := t.TempDir() + store, err := NewFileStore(tmpDir) + if err != nil { + t.Fatalf("Failed to create store: %v", err) + } + + ctx := context.Background() + task := &types.Task{ + Name: "test-task", + Spec: v1alpha1.TaskSpec{ + Process: &v1alpha1.ProcessTask{ + Command: []string{"echo", "hello"}, + }, + }, + } + + // 1. Create + if err := store.Create(ctx, task); err != nil { + t.Fatalf("Create failed: %v", err) + } + + // Verify file exists + taskDir := filepath.Join(tmpDir, task.Name) + if _, err := os.Stat(taskDir); os.IsNotExist(err) { + t.Error("Task directory was not created") + } + + // 2. Get + got, err := store.Get(ctx, task.Name) + if err != nil { + t.Fatalf("Get failed: %v", err) + } + if got.Name != task.Name { + t.Errorf("Get returned wrong name: got %s, want %s", got.Name, task.Name) + } + + // 3. Update + now := time.Now() + got.DeletionTimestamp = &now + + if err := store.Update(ctx, got); err != nil { + t.Fatalf("Update failed: %v", err) + } + + updated, err := store.Get(ctx, task.Name) + if err != nil { + t.Fatalf("Get after update failed: %v", err) + } + if updated.DeletionTimestamp == nil { + t.Error("Update failed to persist DeletionTimestamp") + } + + // 4. List + tasks, err := store.List(ctx) + if err != nil { + t.Fatalf("List failed: %v", err) + } + if len(tasks) != 1 { + t.Errorf("List returned %d tasks, want 1", len(tasks)) + } + if tasks[0].Name != task.Name { + t.Errorf("List returned wrong task: %s", tasks[0].Name) + } + + // 5. Delete + if err := store.Delete(ctx, task.Name); err != nil { + t.Fatalf("Delete failed: %v", err) + } + + // Verify deletion + if _, err := store.Get(ctx, task.Name); err == nil { + t.Error("Get should fail after delete") + } + + tasks, err = store.List(ctx) + if err != nil { + t.Fatalf("List failed: %v", err) + } + if len(tasks) != 0 { + t.Errorf("List returned %d tasks after delete, want 0", len(tasks)) + } + + // Verify directory gone + if _, err := os.Stat(taskDir); !os.IsNotExist(err) { + t.Error("Task directory still exists after delete") + } +} + +func TestFileStore_EdgeCases(t *testing.T) { + tmpDir := t.TempDir() + store, _ := NewFileStore(tmpDir) + ctx := context.Background() + + // Create with nil task + if err := store.Create(ctx, nil); err == nil { + t.Error("Create should fail with nil task") + } + + // Create with empty name + if err := store.Create(ctx, &types.Task{}); err == nil { + t.Error("Create should fail with empty name") + } + + // Create duplicate + task := &types.Task{Name: "dup"} + store.Create(ctx, task) + if err := store.Create(ctx, task); err == nil { + t.Error("Create should fail for duplicate task") + } + + // Update non-existent + if err := store.Update(ctx, &types.Task{Name: "missing"}); err == nil { + t.Error("Update should fail for non-existent task") + } + + // Get non-existent + if _, err := store.Get(ctx, "missing"); err == nil { + t.Error("Get should fail for non-existent task") + } + + // Delete non-existent + if err := store.Delete(ctx, "missing"); err != nil { + t.Errorf("Delete should not fail for non-existent task, got %v", err) + } +} + +func TestFileStore_CorruptedData(t *testing.T) { + tmpDir := t.TempDir() + store, _ := NewFileStore(tmpDir) + ctx := context.Background() + + // Manually create a corrupted task file + taskDir := filepath.Join(tmpDir, "corrupted") + os.MkdirAll(taskDir, 0755) + os.WriteFile(filepath.Join(taskDir, "task.json"), []byte("{invalid-json"), 0644) + + // List should skip corrupted task + tasks, err := store.List(ctx) + if err != nil { + t.Fatalf("List failed: %v", err) + } + if len(tasks) != 0 { + t.Errorf("List should skip corrupted task, got %d", len(tasks)) + } + + // Get should fail for corrupted task + if _, err := store.Get(ctx, "corrupted"); err == nil { + t.Error("Get should fail for corrupted task") + } +} + +// TestConcurrency verifies thread safety +func TestFileStore_Concurrency(t *testing.T) { + tmpDir := t.TempDir() + store, _ := NewFileStore(tmpDir) + ctx := context.Background() + taskName := "concurrent-task" + + store.Create(ctx, &types.Task{Name: taskName}) + + done := make(chan bool) + for i := 0; i < 10; i++ { + go func(id int) { + store.Update(ctx, &types.Task{ + Name: taskName, + Spec: v1alpha1.TaskSpec{ + Process: &v1alpha1.ProcessTask{ + Args: []string{time.Now().String()}, + }, + }, + }) + store.Get(ctx, taskName) + done <- true + }(i) + } + + for i := 0; i < 10; i++ { + <-done + } +} diff --git a/kubernetes/internal/task-executor/storage/interface.go b/kubernetes/internal/task-executor/storage/interface.go new file mode 100644 index 00000000..0a9c511a --- /dev/null +++ b/kubernetes/internal/task-executor/storage/interface.go @@ -0,0 +1,34 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package store + +import ( + "context" + + "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/task-executor/types" +) + +// TaskStore defines the contract for persisting task state. +type TaskStore interface { + Create(ctx context.Context, task *types.Task) error + + Update(ctx context.Context, task *types.Task) error + + Get(ctx context.Context, name string) (*types.Task, error) + + List(ctx context.Context) ([]*types.Task, error) + + Delete(ctx context.Context, name string) error +} diff --git a/kubernetes/internal/task-executor/types/task.go b/kubernetes/internal/task-executor/types/task.go new file mode 100644 index 00000000..211dcdad --- /dev/null +++ b/kubernetes/internal/task-executor/types/task.go @@ -0,0 +1,52 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "time" + + "github.com/alibaba/OpenSandbox/sandbox-k8s/api/v1alpha1" +) + +// TaskState defines the simplified internal state of a task. +type TaskState string + +const ( + TaskStatePending TaskState = "Pending" + TaskStateRunning TaskState = "Running" + TaskStateSucceeded TaskState = "Succeeded" + TaskStateFailed TaskState = "Failed" + TaskStateUnknown TaskState = "Unknown" +) + +// Status represents the internal status of a task. +// This is decoupled from the Kubernetes API status. +type Status struct { + State TaskState `json:"state"` + Reason string `json:"reason,omitempty"` + Message string `json:"message,omitempty"` + ExitCode int `json:"exitCode,omitempty"` + StartedAt *time.Time `json:"startedAt,omitempty"` + FinishedAt *time.Time `json:"finishedAt,omitempty"` +} + +type Task struct { + Name string `json:"name"` + DeletionTimestamp *time.Time `json:"deletionTimestamp,omitempty"` + Spec v1alpha1.TaskSpec `json:"spec"` + + // Status is now a first-class citizen and persisted. + Status Status `json:"status"` +} diff --git a/kubernetes/internal/task-executor/utils/pathutil.go b/kubernetes/internal/task-executor/utils/pathutil.go new file mode 100644 index 00000000..bcdedf00 --- /dev/null +++ b/kubernetes/internal/task-executor/utils/pathutil.go @@ -0,0 +1,53 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "fmt" + "os" + "path/filepath" +) + +func SafeJoin(baseDir, userPath string) (string, error) { + joinedPath := filepath.Join(baseDir, userPath) + + absBaseDir, err := filepath.Abs(baseDir) + if err != nil { + return "", fmt.Errorf("failed to resolve base directory absolute path: %w", err) + } + absJoinedPath, err := filepath.Abs(joinedPath) + if err != nil { + return "", fmt.Errorf("failed to resolve joined path absolute path: %w", err) + } + + if !isSubPath(absBaseDir, absJoinedPath) { + return "", fmt.Errorf("path traversal detected") + } + + return absJoinedPath, nil +} + +func isSubPath(parent, child string) bool { + if len(parent) == 0 { + return false + } + + parentWithSep := parent + if !os.IsPathSeparator(parent[len(parent)-1]) { + parentWithSep = parent + string(filepath.Separator) + } + + return child == parent || (len(child) > len(parentWithSep) && child[:len(parentWithSep)] == parentWithSep) +} diff --git a/kubernetes/internal/task-executor/utils/pathutil_test.go b/kubernetes/internal/task-executor/utils/pathutil_test.go new file mode 100644 index 00000000..76117718 --- /dev/null +++ b/kubernetes/internal/task-executor/utils/pathutil_test.go @@ -0,0 +1,84 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "os" + "path/filepath" + "testing" +) + +func TestSafeJoin(t *testing.T) { + tempDir, err := os.MkdirTemp("", "safejoin-test") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + defer os.RemoveAll(tempDir) + + tests := []struct { + name string + baseDir string + userPath string + wantErr bool + }{ + { + name: "valid path", + baseDir: tempDir, + userPath: "foo", + wantErr: false, + }, + { + name: "valid nested path", + baseDir: tempDir, + userPath: "foo/bar", + wantErr: false, + }, + { + name: "path traversal attempt", + baseDir: tempDir, + userPath: "../foo", + wantErr: true, + }, + { + name: "path traversal to root (treated as relative)", + baseDir: tempDir, + userPath: "/etc/passwd", + wantErr: false, + }, + { + name: "complex traversal", + baseDir: tempDir, + userPath: "foo/../../bar", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := SafeJoin(tt.baseDir, tt.userPath) + if (err != nil) != tt.wantErr { + t.Errorf("SafeJoin() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !tt.wantErr { + expected := filepath.Join(tt.baseDir, tt.userPath) + absExpected, _ := filepath.Abs(expected) + if got != absExpected { + t.Errorf("SafeJoin() = %v, want %v", got, absExpected) + } + } + }) + } +} diff --git a/kubernetes/internal/utils/controller/util.go b/kubernetes/internal/utils/controller/util.go new file mode 100644 index 00000000..e1edc202 --- /dev/null +++ b/kubernetes/internal/utils/controller/util.go @@ -0,0 +1,25 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +// GetControllerKey return key of CloneSet. +func GetControllerKey(obj metav1.Object) string { + return types.NamespacedName{Namespace: obj.GetNamespace(), Name: obj.GetName()}.String() +} diff --git a/kubernetes/internal/utils/expectations/init.go b/kubernetes/internal/utils/expectations/init.go new file mode 100644 index 00000000..d7916dd7 --- /dev/null +++ b/kubernetes/internal/utils/expectations/init.go @@ -0,0 +1,26 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package expectations + +import ( + "flag" + "time" +) + +func init() { + flag.DurationVar(&ExpectationTimeout, "expectation-timeout", time.Minute*5, "The expectation timeout. Defaults 5min") +} + +var ExpectationTimeout time.Duration diff --git a/kubernetes/internal/utils/expectations/resource_version_expectation.go b/kubernetes/internal/utils/expectations/resource_version_expectation.go new file mode 100644 index 00000000..65ec0eb2 --- /dev/null +++ b/kubernetes/internal/utils/expectations/resource_version_expectation.go @@ -0,0 +1,119 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package expectations + +import ( + "strconv" + "sync" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +type ResourceVersionExpectation interface { + Expect(obj metav1.Object) + Observe(obj metav1.Object) + IsSatisfied(obj metav1.Object) (bool, time.Duration) + Delete(obj metav1.Object) +} + +func NewResourceVersionExpectation() ResourceVersionExpectation { + return &realResourceVersionExpectation{objectVersions: make(map[types.UID]*objectCacheVersions, 100)} +} + +type realResourceVersionExpectation struct { + sync.Mutex + objectVersions map[types.UID]*objectCacheVersions +} + +type objectCacheVersions struct { + version string + firstUnsatisfiedTimestamp time.Time +} + +func (r *realResourceVersionExpectation) Expect(obj metav1.Object) { + r.Lock() + defer r.Unlock() + + expectations := r.objectVersions[obj.GetUID()] + if expectations == nil { + r.objectVersions[obj.GetUID()] = &objectCacheVersions{} + } + if isResourceVersionNewer(r.objectVersions[obj.GetUID()].version, obj.GetResourceVersion()) { + r.objectVersions[obj.GetUID()].version = obj.GetResourceVersion() + } +} + +func (r *realResourceVersionExpectation) Observe(obj metav1.Object) { + r.Lock() + defer r.Unlock() + + expectations := r.objectVersions[obj.GetUID()] + if expectations == nil { + return + } + if isResourceVersionNewer(r.objectVersions[obj.GetUID()].version, obj.GetResourceVersion()) { + delete(r.objectVersions, obj.GetUID()) + } +} + +func (r *realResourceVersionExpectation) IsSatisfied(obj metav1.Object) (bool, time.Duration) { + r.Lock() + defer r.Unlock() + + expectations := r.objectVersions[obj.GetUID()] + if expectations == nil { + return true, 0 + } + + if isResourceVersionNewer(r.objectVersions[obj.GetUID()].version, obj.GetResourceVersion()) { + delete(r.objectVersions, obj.GetUID()) + } + _, existing := r.objectVersions[obj.GetUID()] + if existing { + if r.objectVersions[obj.GetUID()].firstUnsatisfiedTimestamp.IsZero() { + r.objectVersions[obj.GetUID()].firstUnsatisfiedTimestamp = time.Now() + } + + return false, time.Since(r.objectVersions[obj.GetUID()].firstUnsatisfiedTimestamp) + } + + return !existing, 0 +} + +func (r *realResourceVersionExpectation) Delete(obj metav1.Object) { + r.Lock() + defer r.Unlock() + delete(r.objectVersions, obj.GetUID()) +} + +func isResourceVersionNewer(old, new string) bool { + if len(old) == 0 { + return true + } + + oldCount, err := strconv.ParseUint(old, 10, 64) + if err != nil { + return true + } + + newCount, err := strconv.ParseUint(new, 10, 64) + if err != nil { + return false + } + + return newCount >= oldCount +} diff --git a/kubernetes/internal/utils/expectations/resource_version_expectation_test.go b/kubernetes/internal/utils/expectations/resource_version_expectation_test.go new file mode 100644 index 00000000..f815ea15 --- /dev/null +++ b/kubernetes/internal/utils/expectations/resource_version_expectation_test.go @@ -0,0 +1,66 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package expectations + +import ( + "testing" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestResourceVersionExpectation(t *testing.T) { + cases := []struct { + expect *v1.Pod + observe *v1.Pod + isSatisfied *v1.Pod + result bool + }{ + { + expect: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "2"}}, + observe: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "1"}}, + isSatisfied: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "1"}}, + result: false, + }, + { + expect: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "2"}}, + observe: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "2"}}, + isSatisfied: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "2"}}, + result: true, + }, + { + expect: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "2"}}, + observe: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "1"}}, + isSatisfied: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "2"}}, + result: true, + }, + { + expect: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "2"}}, + observe: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "2"}}, + isSatisfied: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ResourceVersion: "3"}}, + result: true, + }, + } + + for i, testCase := range cases { + c := NewResourceVersionExpectation() + c.Expect(testCase.expect) + c.Observe(testCase.observe) + got, _ := c.IsSatisfied(testCase.isSatisfied) + if got != testCase.result { + t.Fatalf("#%d expected %v, got %v", i, testCase.result, got) + } + } +} diff --git a/kubernetes/internal/utils/expectations/scale_expectations.go b/kubernetes/internal/utils/expectations/scale_expectations.go new file mode 100644 index 00000000..4ca02fc3 --- /dev/null +++ b/kubernetes/internal/utils/expectations/scale_expectations.go @@ -0,0 +1,147 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package expectations + +import ( + "sync" + "time" + + "k8s.io/apimachinery/pkg/util/sets" +) + +// ScaleAction is the action of scale, like create and delete. +type ScaleAction string + +const ( + // Create action + Create ScaleAction = "create" + // Delete action + Delete ScaleAction = "delete" +) + +// ScaleExpectations is an interface that allows users to set and wait on expectations of pods scale. +type ScaleExpectations interface { + ExpectScale(controllerKey string, action ScaleAction, name string) + ObserveScale(controllerKey string, action ScaleAction, name string) + SatisfiedExpectations(controllerKey string) (bool, time.Duration, map[ScaleAction][]string) + DeleteExpectations(controllerKey string) + GetExpectations(controllerKey string) map[ScaleAction]sets.String +} + +// NewScaleExpectations returns a common ScaleExpectations. +func NewScaleExpectations() ScaleExpectations { + return &realScaleExpectations{ + controllerCache: make(map[string]*realControllerScaleExpectations), + } +} + +type realScaleExpectations struct { + sync.Mutex + // key: parent key, workload namespace/name + controllerCache map[string]*realControllerScaleExpectations +} + +type realControllerScaleExpectations struct { + // item: name for this object + objsCache map[ScaleAction]sets.String + firstUnsatisfiedTimestamp time.Time +} + +func (r *realScaleExpectations) GetExpectations(controllerKey string) map[ScaleAction]sets.String { + r.Lock() + defer r.Unlock() + + expectations := r.controllerCache[controllerKey] + if expectations == nil { + return nil + } + + res := make(map[ScaleAction]sets.String, len(expectations.objsCache)) + for k, v := range expectations.objsCache { + res[k] = sets.NewString(v.List()...) + } + + return res +} + +func (r *realScaleExpectations) ExpectScale(controllerKey string, action ScaleAction, name string) { + r.Lock() + defer r.Unlock() + + expectations := r.controllerCache[controllerKey] + if expectations == nil { + expectations = &realControllerScaleExpectations{ + objsCache: make(map[ScaleAction]sets.String), + } + r.controllerCache[controllerKey] = expectations + } + + if s := expectations.objsCache[action]; s != nil { + s.Insert(name) + } else { + expectations.objsCache[action] = sets.NewString(name) + } +} + +func (r *realScaleExpectations) ObserveScale(controllerKey string, action ScaleAction, name string) { + r.Lock() + defer r.Unlock() + + expectations := r.controllerCache[controllerKey] + if expectations == nil { + return + } + + s := expectations.objsCache[action] + if s == nil { + return + } + s.Delete(name) + + for _, s := range expectations.objsCache { + if s.Len() > 0 { + return + } + } + delete(r.controllerCache, controllerKey) +} + +func (r *realScaleExpectations) SatisfiedExpectations(controllerKey string) (bool, time.Duration, map[ScaleAction][]string) { + r.Lock() + defer r.Unlock() + + expectations := r.controllerCache[controllerKey] + if expectations == nil { + return true, 0, nil + } + + for a, s := range expectations.objsCache { + if s.Len() > 0 { + if expectations.firstUnsatisfiedTimestamp.IsZero() { + expectations.firstUnsatisfiedTimestamp = time.Now() + } + return false, time.Since(expectations.firstUnsatisfiedTimestamp), map[ScaleAction][]string{a: s.List()} + } + } + + delete(r.controllerCache, controllerKey) + return true, 0, nil +} + +func (r *realScaleExpectations) DeleteExpectations(controllerKey string) { + r.Lock() + defer r.Unlock() + delete(r.controllerCache, controllerKey) +} diff --git a/kubernetes/internal/utils/expectations/scale_expectations_test.go b/kubernetes/internal/utils/expectations/scale_expectations_test.go new file mode 100644 index 00000000..250b2141 --- /dev/null +++ b/kubernetes/internal/utils/expectations/scale_expectations_test.go @@ -0,0 +1,50 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package expectations + +import ( + "testing" +) + +func TestScale(t *testing.T) { + e := NewScaleExpectations() + controllerKey01 := "default/cs01" + controllerKey02 := "default/cs02" + pod01 := "pod01" + pod02 := "pod02" + + e.ExpectScale(controllerKey01, Create, pod01) + e.ExpectScale(controllerKey01, Create, pod02) + e.ExpectScale(controllerKey01, Delete, pod01) + if ok, _, _ := e.SatisfiedExpectations(controllerKey01); ok { + t.Fatalf("expected not satisfied") + } + + e.ObserveScale(controllerKey01, Create, pod02) + e.ObserveScale(controllerKey01, Create, pod01) + if ok, _, _ := e.SatisfiedExpectations(controllerKey01); ok { + t.Fatalf("expected not satisfied") + } + + e.ObserveScale(controllerKey02, Delete, pod01) + if ok, _, _ := e.SatisfiedExpectations(controllerKey01); ok { + t.Fatalf("expected not satisfied") + } + + e.ObserveScale(controllerKey01, Delete, pod01) + if ok, _, _ := e.SatisfiedExpectations(controllerKey01); !ok { + t.Fatalf("expected satisfied") + } +} diff --git a/kubernetes/internal/utils/fieldindex/register.go b/kubernetes/internal/utils/fieldindex/register.go new file mode 100644 index 00000000..808d9b16 --- /dev/null +++ b/kubernetes/internal/utils/fieldindex/register.go @@ -0,0 +1,65 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fieldindex + +import ( + "context" + "sync" + + v1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + + sandboxv1alpha1 "github.com/alibaba/OpenSandbox/sandbox-k8s/api/v1alpha1" +) + +const ( + IndexNameForOwnerRefUID = "ownerRefUID" + IndexNameForPoolRef = "poolRef" +) + +var ( + registerOnce sync.Once +) + +var OwnerIndexFunc = func(obj client.Object) []string { + var owners []string + for _, ref := range obj.GetOwnerReferences() { + owners = append(owners, string(ref.UID)) + } + return owners +} + +var PoolRefIndexFunc = func(obj client.Object) []string { + batchSandbox, ok := obj.(*sandboxv1alpha1.BatchSandbox) + if ok { + return []string{batchSandbox.Spec.PoolRef} + } + return nil +} + +func RegisterFieldIndexes(c cache.Cache) error { + var err error + registerOnce.Do(func() { + // pod ownerReference + if err = c.IndexField(context.TODO(), &v1.Pod{}, IndexNameForOwnerRefUID, OwnerIndexFunc); err != nil { + return + } + if err = c.IndexField(context.TODO(), &sandboxv1alpha1.BatchSandbox{}, IndexNameForPoolRef, PoolRefIndexFunc); err != nil { + return + } + }) + return err +} diff --git a/kubernetes/internal/utils/finalizer.go b/kubernetes/internal/utils/finalizer.go new file mode 100644 index 00000000..9eaec724 --- /dev/null +++ b/kubernetes/internal/utils/finalizer.go @@ -0,0 +1,65 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "context" + "errors" + + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +type FinalizerOpType string + +const ( + AddFinalizerOpType FinalizerOpType = "Add" + RemoveFinalizerOpType FinalizerOpType = "Remove" +) + +func UpdateFinalizer(c client.Client, object client.Object, op FinalizerOpType, finalizer string) error { + switch op { + case AddFinalizerOpType, RemoveFinalizerOpType: + default: + return errors.New("UpdateFinalizer Func 'op' parameter must be 'Add' or 'Remove'") + } + + key := client.ObjectKeyFromObject(object) + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + fetchedObject := object.DeepCopyObject().(client.Object) + getErr := c.Get(context.TODO(), key, fetchedObject) + if getErr != nil { + return getErr + } + finalizers := fetchedObject.GetFinalizers() + switch op { + case AddFinalizerOpType: + if controllerutil.ContainsFinalizer(fetchedObject, finalizer) { + return nil + } + finalizers = append(finalizers, finalizer) + case RemoveFinalizerOpType: + finalizerSet := sets.NewString(finalizers...) + if !finalizerSet.Has(finalizer) { + return nil + } + finalizers = finalizerSet.Delete(finalizer).List() + } + fetchedObject.SetFinalizers(finalizers) + return c.Update(context.TODO(), fetchedObject) + }) +} diff --git a/kubernetes/internal/utils/helper.go b/kubernetes/internal/utils/helper.go new file mode 100644 index 00000000..b349f15f --- /dev/null +++ b/kubernetes/internal/utils/helper.go @@ -0,0 +1,31 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// GetAnnotation from metaObject annotations +func GetAnnotation(obj metav1.Object, key string) string { + if obj == nil { + return "" + } + annotations := obj.GetAnnotations() + if annotations == nil { + return "" + } + return annotations[key] +} diff --git a/kubernetes/internal/utils/json.go b/kubernetes/internal/utils/json.go new file mode 100644 index 00000000..546b0386 --- /dev/null +++ b/kubernetes/internal/utils/json.go @@ -0,0 +1,48 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "encoding/json" + "reflect" +) + +// DumpJSON returns the JSON encoding +func DumpJSON(o interface{}) string { + j, _ := json.Marshal(o) + return string(j) +} + +// IsJSONObjectEqual checks if two objects are equal after encoding json +func IsJSONObjectEqual(o1, o2 interface{}) bool { + if reflect.DeepEqual(o1, o2) { + return true + } + + oj1, _ := json.Marshal(o1) + oj2, _ := json.Marshal(o2) + os1 := string(oj1) + os2 := string(oj2) + if os1 == os2 { + return true + } + + om1 := make(map[string]interface{}) + om2 := make(map[string]interface{}) + _ = json.Unmarshal(oj1, &om1) + _ = json.Unmarshal(oj2, &om2) + + return reflect.DeepEqual(om1, om2) +} diff --git a/kubernetes/internal/utils/pod.go b/kubernetes/internal/utils/pod.go new file mode 100644 index 00000000..337b2451 --- /dev/null +++ b/kubernetes/internal/utils/pod.go @@ -0,0 +1,182 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "fmt" + "time" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + apimachineryvalidation "k8s.io/apimachinery/pkg/api/validation" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" +) + +// IsPodAvailable returns true if a pod is available; false otherwise. +// Precondition for an available pod is that it must be ready. On top +// of that, there are two cases when a pod can be considered available: +// 1. minReadySeconds == 0, or +// 2. LastTransitionTime (is set) + minReadySeconds < current time +func IsPodAvailable(pod *v1.Pod, minReadySeconds int32, now metav1.Time) bool { + if !IsPodReady(pod) { + return false + } + + c := GetPodReadyCondition(pod.Status) + minReadySecondsDuration := time.Duration(minReadySeconds) * time.Second + if minReadySeconds == 0 || (!c.LastTransitionTime.IsZero() && c.LastTransitionTime.Add(minReadySecondsDuration).Before(now.Time)) { + return true + } + return false +} + +// IsPodReady returns true if a pod is ready; false otherwise. +func IsPodReady(pod *v1.Pod) bool { + return IsPodReadyConditionTrue(pod.Status) +} + +// IsPodTerminal returns true if a pod is terminal, all containers are stopped and cannot ever regress. +func IsPodTerminal(pod *v1.Pod) bool { + return IsPodPhaseTerminal(pod.Status.Phase) +} + +// IsPodPhaseTerminal returns true if the pod's phase is terminal. +func IsPodPhaseTerminal(phase v1.PodPhase) bool { + return phase == v1.PodFailed || phase == v1.PodSucceeded +} + +// IsPodReadyConditionTrue returns true if a pod is ready; false otherwise. +func IsPodReadyConditionTrue(status v1.PodStatus) bool { + condition := GetPodReadyCondition(status) + return condition != nil && condition.Status == v1.ConditionTrue +} + +// IsContainersReadyConditionTrue returns true if a pod is ready; false otherwise. +func IsContainersReadyConditionTrue(status v1.PodStatus) bool { + condition := GetContainersReadyCondition(status) + return condition != nil && condition.Status == v1.ConditionTrue +} + +// GetPodReadyCondition extracts the pod ready condition from the given status and returns that. +// Returns nil if the condition is not present. +func GetPodReadyCondition(status v1.PodStatus) *v1.PodCondition { + _, condition := GetPodCondition(&status, v1.PodReady) + return condition +} + +// GetContainersReadyCondition extracts the containers ready condition from the given status and returns that. +// Returns nil if the condition is not present. +func GetContainersReadyCondition(status v1.PodStatus) *v1.PodCondition { + _, condition := GetPodCondition(&status, v1.ContainersReady) + return condition +} + +// GetPodCondition extracts the provided condition from the given status and returns that. +// Returns nil and -1 if the condition is not present, and the index of the located condition. +func GetPodCondition(status *v1.PodStatus, conditionType v1.PodConditionType) (int, *v1.PodCondition) { + if status == nil { + return -1, nil + } + return GetPodConditionFromList(status.Conditions, conditionType) +} + +// GetPodConditionFromList extracts the provided condition from the given list of condition and +// returns the index of the condition and the condition. Returns -1 and nil if the condition is not present. +func GetPodConditionFromList(conditions []v1.PodCondition, conditionType v1.PodConditionType) (int, *v1.PodCondition) { + if conditions == nil { + return -1, nil + } + for i := range conditions { + if conditions[i].Type == conditionType { + return i, &conditions[i] + } + } + return -1, nil +} + +func GetPodFromTemplate( + template *v1.PodTemplateSpec, + parentObject runtime.Object, + controllerRef *metav1.OwnerReference, +) (*v1.Pod, error) { + desiredLabels := getPodsLabelSet(template) + desiredFinalizers := getPodsFinalizers(template) + desiredAnnotations := getPodsAnnotationSet(template) + accessor, err := meta.Accessor(parentObject) + if err != nil { + return nil, fmt.Errorf("parentObject does not have ObjectMeta, %v", err) + } + prefix := getPodsPrefix(accessor.GetName()) + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: desiredLabels, + Annotations: desiredAnnotations, + GenerateName: prefix, + Finalizers: desiredFinalizers, + }, + } + if controllerRef != nil { + pod.OwnerReferences = append(pod.OwnerReferences, *controllerRef) + } + pod.Spec = *template.Spec.DeepCopy() + return pod, nil +} + +func getPodsLabelSet(template *v1.PodTemplateSpec) labels.Set { + desiredLabels := make(labels.Set) + for k, v := range template.Labels { + desiredLabels[k] = v + } + return desiredLabels +} + +func getPodsFinalizers(template *v1.PodTemplateSpec) []string { + desiredFinalizers := make([]string, len(template.Finalizers)) + copy(desiredFinalizers, template.Finalizers) + return desiredFinalizers +} + +func getPodsAnnotationSet(template *v1.PodTemplateSpec) labels.Set { + desiredAnnotations := make(labels.Set) + for k, v := range template.Annotations { + desiredAnnotations[k] = v + } + return desiredAnnotations +} + +func getPodsPrefix(controllerName string) string { + // use the dash (if the name isn't too long) to make the pod name a bit prettier + prefix := fmt.Sprintf("%s-", controllerName) + if len(apimachineryvalidation.NameIsDNSSubdomain(prefix, true)) != 0 { + prefix = controllerName + } + return prefix +} + +func IsAssigned(pod *v1.Pod) bool { + return pod != nil && (pod.Spec.NodeName != "" || pod.Status.PodIP != "") +} + +func PodNameSorter(a, b *v1.Pod) int { + if a.Name < b.Name { + return -1 + } else if a.Name > b.Name { + return 1 + } + return 0 +} diff --git a/kubernetes/internal/utils/requeueduration/duration.go b/kubernetes/internal/utils/requeueduration/duration.go new file mode 100644 index 00000000..2ca0e670 --- /dev/null +++ b/kubernetes/internal/utils/requeueduration/duration.go @@ -0,0 +1,95 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package requeueduration + +import ( + "fmt" + "sync" + "time" +) + +// DurationStore can store a duration map for multiple workloads +type DurationStore struct { + store sync.Map +} + +func (dm *DurationStore) Push(key string, newDuration time.Duration) { + value, _ := dm.store.LoadOrStore(key, &Duration{}) + requeueDuration, ok := value.(*Duration) + if !ok { + dm.store.Delete(key) + return + } + requeueDuration.Update(newDuration) +} + +func (dm *DurationStore) Pop(key string) time.Duration { + value, ok := dm.store.Load(key) + if !ok { + return 0 + } + defer dm.store.Delete(key) + requeueDuration, ok := value.(*Duration) + if !ok { + return 0 + } + return requeueDuration.Get() +} + +// Duration helps calculate the shortest non-zore duration to requeue +type Duration struct { + sync.Mutex + duration time.Duration + message string +} + +func (rd *Duration) Update(newDuration time.Duration) { + rd.Lock() + defer rd.Unlock() + if newDuration > 0 { + if rd.duration <= 0 || newDuration < rd.duration { + rd.duration = newDuration + } + } +} + +func (rd *Duration) UpdateWithMsg(newDuration time.Duration, format string, args ...interface{}) { + rd.Lock() + defer rd.Unlock() + if newDuration > 0 { + if rd.duration <= 0 || newDuration < rd.duration { + rd.duration = newDuration + rd.message = fmt.Sprintf(format, args...) + } + } +} + +func (rd *Duration) Merge(rd2 *Duration) { + rd2.Lock() + defer rd2.Unlock() + rd.UpdateWithMsg(rd2.duration, "%s", rd2.message) +} + +func (rd *Duration) Get() time.Duration { + rd.Lock() + defer rd.Unlock() + return rd.duration +} + +func (rd *Duration) GetWithMsg() (time.Duration, string) { + rd.Lock() + defer rd.Unlock() + return rd.duration, rd.message +} diff --git a/kubernetes/pkg/task-executor/client.go b/kubernetes/pkg/task-executor/client.go new file mode 100644 index 00000000..8a14eabf --- /dev/null +++ b/kubernetes/pkg/task-executor/client.go @@ -0,0 +1,145 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package task_executor + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "k8s.io/klog/v2" +) + +type Client struct { + baseURL string + httpClient *http.Client +} + +func NewClient(baseURL string) *Client { + if baseURL == "" { + klog.Warning("baseURL is empty, client may not work properly") + } + return &Client{ + baseURL: baseURL, + httpClient: &http.Client{ + Timeout: 30 * time.Second, + }, + } +} + +// Set creates or updates a task on the remote server. +// If task is nil, it sends a delete request. +func (c *Client) Set(ctx context.Context, task *Task) (*Task, error) { + if c == nil { + return nil, fmt.Errorf("client is nil") + } + + var req *http.Request + var err error + + if task == nil { + // Delete request - send nil to clear tasks + req, err = http.NewRequestWithContext(ctx, "POST", c.baseURL+"/setTasks", bytes.NewReader([]byte("[]"))) + } else { + // Create/Update request + data, err := json.Marshal([]Task{*task}) + if err != nil { + return nil, fmt.Errorf("failed to marshal task: %w", err) + } + req, err = http.NewRequestWithContext(ctx, "POST", c.baseURL+"/setTasks", bytes.NewReader(data)) + } + + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + + // Send request with retry + var resp *http.Response + resp, err = c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("network error after retries: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("server error: status=%d, body=%s", resp.StatusCode, string(body)) + } + + // Parse response - expect array of tasks + var tasks []Task + if err := json.NewDecoder(resp.Body).Decode(&tasks); err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + + if task != nil && len(tasks) > 0 { + // Find the task we just set + for i := range tasks { + if tasks[i].Name == task.Name { + return &tasks[i], nil + } + } + } + + if task == nil { + // Delete succeeded + return nil, nil + } + + return task, nil +} + +// Get retrieves the current task list from the remote server. +func (c *Client) Get(ctx context.Context) (*Task, error) { + if c == nil { + return nil, fmt.Errorf("client is nil") + } + + req, err := http.NewRequestWithContext(ctx, "GET", c.baseURL+"/getTasks", nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("network error: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("server error: status=%d, body=%s", resp.StatusCode, string(body)) + } + + // Parse response - expect array of tasks + var tasks []Task + if err := json.NewDecoder(resp.Body).Decode(&tasks); err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + + // Return the first task (single task mode) + if len(tasks) > 0 { + return &tasks[0], nil + } + + // No tasks + return nil, nil +} diff --git a/kubernetes/pkg/task-executor/types.go b/kubernetes/pkg/task-executor/types.go new file mode 100644 index 00000000..7f0be340 --- /dev/null +++ b/kubernetes/pkg/task-executor/types.go @@ -0,0 +1,35 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package task_executor + +import ( + "github.com/alibaba/OpenSandbox/sandbox-k8s/api/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// Task represents the internal local task resource (LocalTask) +// It follows the Kubernetes resource model with Metadata, Spec, and Status. +type Task struct { + Name string `json:"name"` + DeletionTimestamp *metav1.Time `json:"deletionTimestamp,omitempty"` + + // Spec defines the desired behavior of the task. + // We reuse the v1alpha1.TaskSpec to ensure consistency with the controller API. + Spec v1alpha1.TaskSpec `json:"spec"` + + // Status describes the current state of the task. + // We reuse the v1alpha1.TaskStatus to ensure consistency with the controller API. + Status v1alpha1.TaskStatus `json:"status"` +} diff --git a/kubernetes/test/e2e/e2e_suite_test.go b/kubernetes/test/e2e/e2e_suite_test.go new file mode 100644 index 00000000..7d8fd3f3 --- /dev/null +++ b/kubernetes/test/e2e/e2e_suite_test.go @@ -0,0 +1,84 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "fmt" + "os" + "os/exec" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/alibaba/OpenSandbox/sandbox-k8s/test/utils" +) + +var ( + // projectImage is the name of the image which will be build and loaded + // with the code source changes to be tested. + projectImage = "example.com/sandbox-k8s:v0.0.1" + + // taskExecutorImage is the name of the task-executor image + taskExecutorImage = "example.com/task-executor:v0.0.1" + + // sandboxImage is a lightweight image used for sandbox containers in tests + // Using task-executor image instead of ubuntu:latest to avoid download issues in certain network environments + sandboxImage = taskExecutorImage +) + +// TestE2E runs the end-to-end (e2e) test suite for the project. These tests execute in an isolated, +// temporary environment to validate project changes with the purposed to be used in CI jobs. +// The default setup requires Kind, builds/loads the Manager Docker image locally. +func TestE2E(t *testing.T) { + RegisterFailHandler(Fail) + _, _ = fmt.Fprintf(GinkgoWriter, "Starting sandbox-k8s integration test suite\n") + RunSpecs(t, "e2e suite") +} + +var _ = BeforeSuite(func() { + dockerBuildArgs := os.Getenv("DOCKER_BUILD_ARGS") + + By("building the manager(Operator) image") + makeArgs := []string{"docker-build", fmt.Sprintf("IMG=%s", projectImage)} + if dockerBuildArgs != "" { + makeArgs = append(makeArgs, fmt.Sprintf("DOCKER_BUILD_ARGS=%s", dockerBuildArgs)) + } + cmd := exec.Command("make", makeArgs...) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build the manager(Operator) image") + + By("building the task-executor image") + makeArgs = []string{"docker-build-task-executor", fmt.Sprintf("IMG=%s", taskExecutorImage)} + if dockerBuildArgs != "" { + makeArgs = append(makeArgs, fmt.Sprintf("DOCKER_BUILD_ARGS=%s", dockerBuildArgs)) + } + cmd = exec.Command("make", makeArgs...) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build the task-executor image") + + // If you want to change the e2e test vendor from Kind, ensure the image is + // built and available before running the tests. Also, remove the following block. + By("loading the manager(Operator) image on Kind") + err = utils.LoadImageToKindClusterWithName(projectImage) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the manager(Operator) image into Kind") + + By("loading the task-executor image on Kind") + err = utils.LoadImageToKindClusterWithName(taskExecutorImage) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the task-executor image into Kind") +}) + +var _ = AfterSuite(func() { +}) diff --git a/kubernetes/test/e2e/e2e_test.go b/kubernetes/test/e2e/e2e_test.go new file mode 100644 index 00000000..98ea07c3 --- /dev/null +++ b/kubernetes/test/e2e/e2e_test.go @@ -0,0 +1,1323 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "text/template" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/alibaba/OpenSandbox/sandbox-k8s/test/utils" +) + +// namespace where the project is deployed in +const namespace = "sandbox-k8s-system" + +var _ = Describe("Manager", Ordered, func() { + var controllerPodName string + + // Before running the tests, set up the environment by creating the namespace, + // enforce the restricted security policy to the namespace, installing CRDs, + // and deploying the controller. + BeforeAll(func() { + By("creating manager namespace") + cmd := exec.Command("kubectl", "create", "ns", namespace) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create namespace") + + By("labeling the namespace to enforce the restricted security policy") + cmd = exec.Command("kubectl", "label", "--overwrite", "ns", namespace, + "pod-security.kubernetes.io/enforce=restricted") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to label namespace with restricted policy") + + By("installing CRDs") + cmd = exec.Command("make", "install") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to install CRDs") + + By("deploying the controller-manager") + cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectImage)) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") + }) + + // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, + // and deleting the namespace. + AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + + By("undeploying the controller-manager") + cmd = exec.Command("make", "undeploy") + _, _ = utils.Run(cmd) + + By("uninstalling CRDs") + cmd = exec.Command("make", "uninstall") + _, _ = utils.Run(cmd) + + By("removing manager namespace") + cmd = exec.Command("kubectl", "delete", "ns", namespace) + _, _ = utils.Run(cmd) + }) + + // After each test, check for failures and collect logs, events, + // and pod descriptions for debugging. + AfterEach(func() { + specReport := CurrentSpecReport() + if specReport.Failed() { + By("Fetching controller manager pod logs") + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + controllerLogs, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Controller logs:\n %s", controllerLogs) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Controller logs: %s", err) + } + + By("Fetching Kubernetes events") + cmd = exec.Command("kubectl", "get", "events", "-n", namespace, "--sort-by=.lastTimestamp") + eventsOutput, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Kubernetes events:\n%s", eventsOutput) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Kubernetes events: %s", err) + } + + By("Fetching curl-metrics logs") + cmd = exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Metrics logs:\n %s", metricsOutput) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get curl-metrics logs: %s", err) + } + + By("Fetching controller manager pod description") + cmd = exec.Command("kubectl", "describe", "pod", controllerPodName, "-n", namespace) + podDescription, err := utils.Run(cmd) + if err == nil { + fmt.Println("Pod description:\n", podDescription) + } else { + fmt.Println("Failed to describe controller pod") + } + } + }) + + SetDefaultEventuallyTimeout(2 * time.Minute) + SetDefaultEventuallyPollingInterval(time.Second) + + Context("Manager", func() { + It("should run successfully", func() { + By("validating that the controller-manager pod is running as expected") + verifyControllerUp := func(g Gomega) { + // Get the name of the controller-manager pod + goTemplate := `{{ range .items }}` + + `{{ if not .metadata.deletionTimestamp }}` + + `{{ .metadata.name }}` + + `{{ "\n" }}{{ end }}{{ end }}` + cmd := exec.Command("kubectl", "get", + "pods", "-l", "control-plane=controller-manager", + "-o", "go-template="+goTemplate, + "-n", namespace, + ) + + podOutput, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") + podNames := utils.GetNonEmptyLines(podOutput) + g.Expect(podNames).To(HaveLen(1), "expected 1 controller pod running") + controllerPodName = podNames[0] + g.Expect(controllerPodName).To(ContainSubstring("controller-manager")) + + // Validate the pod's status + cmd = exec.Command("kubectl", "get", + "pods", controllerPodName, "-o", "jsonpath={.status.phase}", + "-n", namespace, + ) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Running"), "Incorrect controller-manager pod status") + } + Eventually(verifyControllerUp).Should(Succeed()) + }) + }) + + Context("Pool", func() { + BeforeAll(func() { + By("waiting for controller to be ready") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager", + "-n", namespace, "-o", "jsonpath={.items[0].status.phase}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Running")) + }, 2*time.Minute).Should(Succeed()) + }) + + It("should correctly create pods and maintain pool status", func() { + const poolName = "test-pool-basic" + const testNamespace = "default" + const poolMin = 2 + const poolMax = 5 + const bufferMin = 1 + const bufferMax = 3 + + By("creating a basic Pool") + poolYAML, err := renderTemplate("testdata/pool-basic.yaml", map[string]interface{}{ + "PoolName": poolName, + "SandboxImage": sandboxImage, + "Namespace": testNamespace, + "BufferMax": bufferMax, + "BufferMin": bufferMin, + "PoolMax": poolMax, + "PoolMin": poolMin, + }) + Expect(err).NotTo(HaveOccurred()) + + poolFile := filepath.Join("/tmp", "test-pool-basic.yaml") + err = os.WriteFile(poolFile, []byte(poolYAML), 0644) + Expect(err).NotTo(HaveOccurred()) + defer os.Remove(poolFile) + + cmd := exec.Command("kubectl", "apply", "-f", poolFile) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create Pool") + + By("verifying Pool creates pods and maintains correct status") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pool", poolName, "-n", testNamespace, + "-o", "jsonpath={.status}") + statusOutput, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + + g.Expect(statusOutput).To(ContainSubstring(`"total":`), "Pool status should have total field") + g.Expect(statusOutput).To(ContainSubstring(`"allocated":`), "Pool status should have allocated field") + g.Expect(statusOutput).To(ContainSubstring(`"available":`), "Pool status should have available field") + + cmd = exec.Command("kubectl", "get", "pool", poolName, "-n", testNamespace, + "-o", "jsonpath={.status.total}") + totalStr, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + total := 0 + if totalStr != "" { + fmt.Sscanf(totalStr, "%d", &total) + } + g.Expect(total).To(BeNumerically(">=", poolMin), "Pool total should be >= poolMin") + g.Expect(total).To(BeNumerically("<=", poolMax), "Pool total should be <= poolMax") + }, 2*time.Minute).Should(Succeed()) + + By("verifying pods are created") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "-n", testNamespace, + "-l", fmt.Sprintf("sandbox.opensandbox.io/pool-name=%s", poolName), + "-o", "jsonpath={.items[*].metadata.name}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).NotTo(BeEmpty(), "Pool should create pods") + }, 2*time.Minute).Should(Succeed()) + + By("cleaning up the Pool") + cmd = exec.Command("kubectl", "delete", "pool", poolName, "-n", testNamespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should correctly manage capacity when poolMin and poolMax change", func() { + const poolName = "test-pool-capacity" + const testNamespace = "default" + + By("creating a Pool with initial capacity") + poolYAML, err := renderTemplate("testdata/pool-basic.yaml", map[string]interface{}{ + "PoolName": poolName, + "SandboxImage": sandboxImage, + "Namespace": testNamespace, + "BufferMax": 3, + "BufferMin": 1, + "PoolMax": 5, + "PoolMin": 2, + }) + Expect(err).NotTo(HaveOccurred()) + + poolFile := filepath.Join("/tmp", "test-pool-capacity.yaml") + err = os.WriteFile(poolFile, []byte(poolYAML), 0644) + Expect(err).NotTo(HaveOccurred()) + defer os.Remove(poolFile) + + cmd := exec.Command("kubectl", "apply", "-f", poolFile) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("waiting for initial Pool to be ready") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pool", poolName, "-n", testNamespace, + "-o", "jsonpath={.status.total}") + totalStr, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + total := 0 + if totalStr != "" { + fmt.Sscanf(totalStr, "%d", &total) + } + g.Expect(total).To(BeNumerically(">=", 2)) + }, 2*time.Minute).Should(Succeed()) + + By("increasing poolMin to trigger scale up") + poolYAML, err = renderTemplate("testdata/pool-basic.yaml", map[string]interface{}{ + "PoolName": poolName, + "SandboxImage": sandboxImage, + "Namespace": testNamespace, + "BufferMax": 3, + "BufferMin": 1, + "PoolMax": 10, + "PoolMin": 5, + }) + Expect(err).NotTo(HaveOccurred()) + + err = os.WriteFile(poolFile, []byte(poolYAML), 0644) + Expect(err).NotTo(HaveOccurred()) + + cmd = exec.Command("kubectl", "apply", "-f", poolFile) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("verifying Pool scales up to meet new poolMin") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pool", poolName, "-n", testNamespace, + "-o", "jsonpath={.status.total}") + totalStr, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + total := 0 + if totalStr != "" { + fmt.Sscanf(totalStr, "%d", &total) + } + g.Expect(total).To(BeNumerically(">=", 5), "Pool should scale up to meet poolMin=5") + g.Expect(total).To(BeNumerically("<=", 10), "Pool should not exceed poolMax=10") + }, 2*time.Minute).Should(Succeed()) + + By("decreasing poolMax to below current total") + poolYAML, err = renderTemplate("testdata/pool-basic.yaml", map[string]interface{}{ + "PoolName": poolName, + "SandboxImage": sandboxImage, + "Namespace": testNamespace, + "BufferMax": 2, + "BufferMin": 1, + "PoolMax": 3, + "PoolMin": 2, + }) + Expect(err).NotTo(HaveOccurred()) + + err = os.WriteFile(poolFile, []byte(poolYAML), 0644) + Expect(err).NotTo(HaveOccurred()) + + cmd = exec.Command("kubectl", "apply", "-f", poolFile) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("verifying Pool respects new poolMax constraint") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pool", poolName, "-n", testNamespace, + "-o", "jsonpath={.status.total}") + totalStr, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + total := 0 + if totalStr != "" { + fmt.Sscanf(totalStr, "%d", &total) + } + g.Expect(total).To(BeNumerically("<=", 3), "Pool should scale down to meet poolMax=3") + }, 2*time.Minute).Should(Succeed()) + + By("cleaning up the Pool") + cmd = exec.Command("kubectl", "delete", "pool", poolName, "-n", testNamespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should upgrade pool template correctly", func() { + const poolName = "test-pool-upgrade" + const testNamespace = "default" + const batchSandboxName = "test-bs-for-upgrade" + + By("creating a Pool with initial template") + poolYAML, err := renderTemplate("testdata/pool-basic.yaml", map[string]interface{}{ + "PoolName": poolName, + "SandboxImage": sandboxImage, + "Namespace": testNamespace, + "BufferMax": 3, + "BufferMin": 2, + "PoolMax": 5, + "PoolMin": 2, + }) + Expect(err).NotTo(HaveOccurred()) + + poolFile := filepath.Join("/tmp", "test-pool-upgrade.yaml") + err = os.WriteFile(poolFile, []byte(poolYAML), 0644) + Expect(err).NotTo(HaveOccurred()) + defer os.Remove(poolFile) + + cmd := exec.Command("kubectl", "apply", "-f", poolFile) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("waiting for Pool to be ready") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pool", poolName, "-n", testNamespace, + "-o", "jsonpath={.status.total}") + totalStr, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(totalStr).NotTo(BeEmpty()) + }, 2*time.Minute).Should(Succeed()) + + By("allocating a pod from the pool via BatchSandbox") + batchSandboxYAML, err := renderTemplate("testdata/batchsandbox-pooled-no-expire.yaml", map[string]interface{}{ + "BatchSandboxName": batchSandboxName, + "Namespace": testNamespace, + "Replicas": 1, + "PoolName": poolName, + }) + Expect(err).NotTo(HaveOccurred()) + + bsFile := filepath.Join("/tmp", "test-bs-upgrade.yaml") + err = os.WriteFile(bsFile, []byte(batchSandboxYAML), 0644) + Expect(err).NotTo(HaveOccurred()) + defer os.Remove(bsFile) + + cmd = exec.Command("kubectl", "apply", "-f", bsFile) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("waiting for BatchSandbox to allocate pod") + var allocatedPodNames []string + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "batchsandbox", batchSandboxName, "-n", testNamespace, + "-o", "jsonpath={.status.allocated}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("1")) + + cmd = exec.Command("kubectl", "get", "batchsandbox", batchSandboxName, "-n", testNamespace, + "-o", "jsonpath={.metadata.annotations.sandbox\\.opensandbox\\.io/alloc-status}") + allocStatusJSON, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(allocStatusJSON).NotTo(BeEmpty(), "alloc-status annotation should exist") + + var allocStatus struct { + Pods []string `json:"pods"` + } + err = json.Unmarshal([]byte(allocStatusJSON), &allocStatus) + g.Expect(err).NotTo(HaveOccurred()) + + allocatedPodNames = allocStatus.Pods + g.Expect(len(allocatedPodNames)).To(Equal(1), "Should have 1 allocated pod") + }, 2*time.Minute).Should(Succeed()) + + By("getting all pool pods") + cmd = exec.Command("kubectl", "get", "pods", "-n", testNamespace, + "-l", fmt.Sprintf("sandbox.opensandbox.io/pool-name=%s", poolName), + "-o", "jsonpath={.items[*].metadata.name}") + allPoolPodsStr, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + allPoolPods := strings.Fields(allPoolPodsStr) + + By("calculating available pods (all pool pods - allocated pods)") + availablePodsBeforeUpgrade := []string{} + allocatedPodMap := make(map[string]bool) + for _, podName := range allocatedPodNames { + allocatedPodMap[podName] = true + } + for _, podName := range allPoolPods { + if !allocatedPodMap[podName] { + availablePodsBeforeUpgrade = append(availablePodsBeforeUpgrade, podName) + } + } + + By("updating Pool template with new environment variable") + updatedPoolYAML, err := renderTemplate("testdata/pool-with-env.yaml", map[string]interface{}{ + "PoolName": poolName, + "Namespace": testNamespace, + "SandboxImage": sandboxImage, + "BufferMax": 3, + "BufferMin": 2, + "PoolMax": 5, + "PoolMin": 2, + }) + Expect(err).NotTo(HaveOccurred()) + + err = os.WriteFile(poolFile, []byte(updatedPoolYAML), 0644) + Expect(err).NotTo(HaveOccurred()) + + cmd = exec.Command("kubectl", "apply", "-f", poolFile) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("verifying allocated pod is NOT upgraded") + Consistently(func(g Gomega) { + for _, allocatedPod := range allocatedPodNames { + cmd := exec.Command("kubectl", "get", "pod", allocatedPod, "-n", testNamespace, + "-o", "jsonpath={.metadata.name}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal(allocatedPod), "Allocated pod should not be recreated") + } + }, 30*time.Second, 3*time.Second).Should(Succeed()) + + By("verifying available pods are recreated with new template") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "-n", testNamespace, + "-l", fmt.Sprintf("sandbox.opensandbox.io/pool-name=%s", poolName), + "-o", "jsonpath={.items[*].metadata.name}") + allPodsAfterStr, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + allPodsAfter := strings.Fields(allPodsAfterStr) + + // Get currently allocated pods + cmd = exec.Command("kubectl", "get", "batchsandbox", batchSandboxName, "-n", testNamespace, + "-o", "jsonpath={.metadata.annotations.sandbox\\.opensandbox\\.io/alloc-status}") + allocStatusJSON, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + + var allocStatus struct { + Pods []string `json:"pods"` + } + err = json.Unmarshal([]byte(allocStatusJSON), &allocStatus) + g.Expect(err).NotTo(HaveOccurred()) + + currentAllocatedPods := make(map[string]bool) + for _, podName := range allocStatus.Pods { + currentAllocatedPods[podName] = true + } + + // Calculate available pods after upgrade + availablePodsAfterUpgrade := []string{} + for _, podName := range allPodsAfter { + if !currentAllocatedPods[podName] { + availablePodsAfterUpgrade = append(availablePodsAfterUpgrade, podName) + } + } + + // Check if at least one available pod was recreated + recreated := false + for _, oldPod := range availablePodsBeforeUpgrade { + found := false + for _, newPod := range availablePodsAfterUpgrade { + if oldPod == newPod { + found = true + break + } + } + if !found { + recreated = true + break + } + } + g.Expect(recreated).To(BeTrue(), "At least one available pod should be recreated") + }, 3*time.Minute).Should(Succeed()) + + By("verifying new pods have the upgraded environment variable") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "-n", testNamespace, + "-l", fmt.Sprintf("sandbox.opensandbox.io/pool-name=%s", poolName), + "-o", "json") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + + var podList struct { + Items []struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + Spec struct { + Containers []struct { + Name string `json:"name"` + Env []struct { + Name string `json:"name"` + Value string `json:"value"` + } `json:"env"` + } `json:"containers"` + } `json:"spec"` + } `json:"items"` + } + err = json.Unmarshal([]byte(output), &podList) + g.Expect(err).NotTo(HaveOccurred()) + + // Get currently allocated pods + cmd = exec.Command("kubectl", "get", "batchsandbox", batchSandboxName, "-n", testNamespace, + "-o", "jsonpath={.metadata.annotations.sandbox\\.opensandbox\\.io/alloc-status}") + allocStatusJSON, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + + var allocStatus struct { + Pods []string `json:"pods"` + } + err = json.Unmarshal([]byte(allocStatusJSON), &allocStatus) + g.Expect(err).NotTo(HaveOccurred()) + + allocatedPodMap := make(map[string]bool) + for _, podName := range allocStatus.Pods { + allocatedPodMap[podName] = true + } + + // Find at least one available pod with UPGRADED=true + foundUpgraded := false + for _, pod := range podList.Items { + if !allocatedPodMap[pod.Metadata.Name] { + // This is an available pod + for _, container := range pod.Spec.Containers { + if container.Name == "sandbox-container" { + for _, env := range container.Env { + if env.Name == "UPGRADED" && env.Value == "true" { + foundUpgraded = true + break + } + } + } + } + } + } + g.Expect(foundUpgraded).To(BeTrue(), "At least one available pod should have UPGRADED=true env var") + }, 2*time.Minute).Should(Succeed()) + + By("cleaning up BatchSandbox and Pool") + cmd = exec.Command("kubectl", "delete", "batchsandbox", batchSandboxName, "-n", testNamespace) + _, _ = utils.Run(cmd) + + cmd = exec.Command("kubectl", "delete", "pool", poolName, "-n", testNamespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + }) + }) + + Context("BatchSandbox", func() { + BeforeAll(func() { + By("waiting for controller to be ready") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager", + "-n", namespace, "-o", "jsonpath={.items[0].status.phase}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Running")) + }, 2*time.Minute).Should(Succeed()) + }) + + It("should work correctly in non-pooled mode", func() { + const batchSandboxName = "test-bs-non-pooled" + const testNamespace = "default" + const replicas = 2 + + By("creating a non-pooled BatchSandbox") + bsYAML, err := renderTemplate("testdata/batchsandbox-non-pooled.yaml", map[string]interface{}{ + "BatchSandboxName": batchSandboxName, + "SandboxImage": sandboxImage, + "Namespace": testNamespace, + "Replicas": replicas, + }) + Expect(err).NotTo(HaveOccurred()) + + bsFile := filepath.Join("/tmp", "test-bs-non-pooled.yaml") + err = os.WriteFile(bsFile, []byte(bsYAML), 0644) + Expect(err).NotTo(HaveOccurred()) + defer os.Remove(bsFile) + + cmd := exec.Command("kubectl", "apply", "-f", bsFile) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("verifying pods are created directly from template") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "-n", testNamespace, + "-o", "json") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + + var podList struct { + Items []struct { + Metadata struct { + Name string `json:"name"` + OwnerReferences []struct { + Kind string `json:"kind"` + Name string `json:"name"` + UID string `json:"uid"` + } `json:"ownerReferences"` + } `json:"metadata"` + } `json:"items"` + } + err = json.Unmarshal([]byte(output), &podList) + g.Expect(err).NotTo(HaveOccurred()) + + // Find pods owned by this BatchSandbox + ownedPods := []string{} + for _, pod := range podList.Items { + for _, owner := range pod.Metadata.OwnerReferences { + if owner.Kind == "BatchSandbox" && owner.Name == batchSandboxName { + ownedPods = append(ownedPods, pod.Metadata.Name) + break + } + } + } + g.Expect(len(ownedPods)).To(Equal(replicas), "Should create %d pods", replicas) + }, 2*time.Minute).Should(Succeed()) + + By("verifying BatchSandbox status is correctly updated") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "batchsandbox", batchSandboxName, "-n", testNamespace, + "-o", "jsonpath={.status}") + statusOutput, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(statusOutput).To(ContainSubstring(fmt.Sprintf(`"replicas":%d`, replicas))) + g.Expect(statusOutput).To(ContainSubstring(fmt.Sprintf(`"allocated":%d`, replicas))) + g.Expect(statusOutput).To(ContainSubstring(fmt.Sprintf(`"ready":%d`, replicas))) + }, 2*time.Minute).Should(Succeed()) + + By("verifying endpoint annotation is set") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "batchsandbox", batchSandboxName, "-n", testNamespace, + "-o", "jsonpath={.metadata.annotations.sandbox\\.opensandbox\\.io/endpoints}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).NotTo(BeEmpty()) + endpoints := strings.Split(output, ",") + g.Expect(len(endpoints)).To(Equal(replicas)) + }, 30*time.Second).Should(Succeed()) + + By("cleaning up BatchSandbox") + cmd = exec.Command("kubectl", "delete", "batchsandbox", batchSandboxName, "-n", testNamespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("verifying pods are deleted") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "-n", testNamespace, "-o", "json") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + + var podList struct { + Items []struct { + Metadata struct { + Name string `json:"name"` + DeletionTimestamp *string `json:"deletionTimestamp"` + OwnerReferences []struct { + Kind string `json:"kind"` + Name string `json:"name"` + } `json:"ownerReferences"` + } `json:"metadata"` + } `json:"items"` + } + err = json.Unmarshal([]byte(output), &podList) + g.Expect(err).NotTo(HaveOccurred()) + + // Check no pods are owned by this BatchSandbox or they have deletionTimestamp + for _, pod := range podList.Items { + for _, owner := range pod.Metadata.OwnerReferences { + if owner.Kind == "BatchSandbox" && owner.Name == batchSandboxName { + g.Expect(pod.Metadata.DeletionTimestamp).NotTo(BeNil(), + "Pod %s owned by BatchSandbox should have deletionTimestamp set", pod.Metadata.Name) + } + } + } + }, 2*time.Minute).Should(Succeed()) + }) + + It("should work correctly in pooled mode", func() { + const poolName = "test-pool-for-bs" + const batchSandboxName = "test-bs-pooled" + const testNamespace = "default" + const replicas = 2 + + By("creating a Pool") + poolYAML, err := renderTemplate("testdata/pool-basic.yaml", map[string]interface{}{ + "PoolName": poolName, + "SandboxImage": sandboxImage, + "Namespace": testNamespace, + "BufferMax": 3, + "BufferMin": 2, + "PoolMax": 5, + "PoolMin": 2, + }) + Expect(err).NotTo(HaveOccurred()) + + poolFile := filepath.Join("/tmp", "test-pool-for-bs.yaml") + err = os.WriteFile(poolFile, []byte(poolYAML), 0644) + Expect(err).NotTo(HaveOccurred()) + defer os.Remove(poolFile) + + cmd := exec.Command("kubectl", "apply", "-f", poolFile) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("waiting for Pool to be ready") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pool", poolName, "-n", testNamespace, + "-o", "jsonpath={.status.total}") + totalStr, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(totalStr).NotTo(BeEmpty()) + }, 2*time.Minute).Should(Succeed()) + + By("creating a pooled BatchSandbox") + bsYAML, err := renderTemplate("testdata/batchsandbox-pooled-no-expire.yaml", map[string]interface{}{ + "BatchSandboxName": batchSandboxName, + "SandboxImage": sandboxImage, + "Namespace": testNamespace, + "Replicas": replicas, + "PoolName": poolName, + }) + Expect(err).NotTo(HaveOccurred()) + + bsFile := filepath.Join("/tmp", "test-bs-pooled.yaml") + err = os.WriteFile(bsFile, []byte(bsYAML), 0644) + Expect(err).NotTo(HaveOccurred()) + defer os.Remove(bsFile) + + cmd = exec.Command("kubectl", "apply", "-f", bsFile) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("verifying BatchSandbox allocates pods from pool") + Eventually(func(g Gomega) { + // Verify alloc-status annotation contains pool pod names + cmd = exec.Command("kubectl", "get", "batchsandbox", batchSandboxName, "-n", testNamespace, + "-o", "jsonpath={.metadata.annotations.sandbox\\.opensandbox\\.io/alloc-status}") + allocStatusJSON, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(allocStatusJSON).NotTo(BeEmpty(), "alloc-status annotation should exist") + + var allocStatus struct { + Pods []string `json:"pods"` + } + err = json.Unmarshal([]byte(allocStatusJSON), &allocStatus) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(allocStatus.Pods)).To(Equal(replicas), "Should have %d pods in alloc-status", replicas) + + // Verify the pods in alloc-status are from the pool + for _, podName := range allocStatus.Pods { + cmd = exec.Command("kubectl", "get", "pod", podName, "-n", testNamespace, + "-o", "jsonpath={.metadata.labels.sandbox\\.opensandbox\\.io/pool-name}") + poolLabel, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(poolLabel).To(Equal(poolName), "Pod %s should be from pool %s", podName, poolName) + } + }, 2*time.Minute).Should(Succeed()) + + By("verifying BatchSandbox status is correctly updated") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "batchsandbox", batchSandboxName, "-n", testNamespace, + "-o", "jsonpath={.status}") + statusOutput, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(statusOutput).To(ContainSubstring(fmt.Sprintf(`"replicas":%d`, replicas))) + g.Expect(statusOutput).To(ContainSubstring(fmt.Sprintf(`"ready":%d`, replicas))) + }, 30*time.Second).Should(Succeed()) + + By("verifying endpoint annotation is set") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "batchsandbox", batchSandboxName, "-n", testNamespace, + "-o", "jsonpath={.metadata.annotations.sandbox\\.opensandbox\\.io/endpoints}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).NotTo(BeEmpty()) + endpoints := strings.Split(output, ",") + g.Expect(len(endpoints)).To(Equal(replicas)) + }, 30*time.Second).Should(Succeed()) + + By("recording Pool allocated count") + cmd = exec.Command("kubectl", "get", "pool", poolName, "-n", testNamespace, + "-o", "jsonpath={.status.allocated}") + allocatedBefore, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("cleaning up BatchSandbox") + cmd = exec.Command("kubectl", "delete", "batchsandbox", batchSandboxName, "-n", testNamespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("verifying pods are returned to pool") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pool", poolName, "-n", testNamespace, + "-o", "jsonpath={.status.allocated}") + allocatedAfter, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + + before := 0 + if allocatedBefore != "" { + fmt.Sscanf(allocatedBefore, "%d", &before) + } + after := 0 + if allocatedAfter != "" { + fmt.Sscanf(allocatedAfter, "%d", &after) + } + g.Expect(after).To(BeNumerically("<", before), "Allocated count should decrease") + }, 30*time.Second).Should(Succeed()) + + By("cleaning up Pool") + cmd = exec.Command("kubectl", "delete", "pool", poolName, "-n", testNamespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should expire and delete non-pooled BatchSandbox correctly", func() { + const batchSandboxName = "test-bs-expire-non-pooled" + const testNamespace = "default" + const replicas = 1 + + By("creating a non-pooled BatchSandbox with expireTime") + expireTime := time.Now().Add(45 * time.Second).UTC().Format(time.RFC3339) + + bsYAML, err := renderTemplate("testdata/batchsandbox-non-pooled-expire.yaml", map[string]interface{}{ + "BatchSandboxName": batchSandboxName, + "Namespace": testNamespace, + "Replicas": replicas, + "ExpireTime": expireTime, + "SandboxImage": sandboxImage, + }) + Expect(err).NotTo(HaveOccurred()) + + bsFile := filepath.Join("/tmp", "test-bs-expire-non-pooled.yaml") + err = os.WriteFile(bsFile, []byte(bsYAML), 0644) + Expect(err).NotTo(HaveOccurred()) + defer os.Remove(bsFile) + + cmd := exec.Command("kubectl", "apply", "-f", bsFile) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("verifying BatchSandbox is created") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "batchsandbox", batchSandboxName, "-n", testNamespace, + "-o", "jsonpath={.status.allocated}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal(fmt.Sprintf("%d", replicas))) + }, 2*time.Minute).Should(Succeed()) + + By("recording pod names") + cmd = exec.Command("kubectl", "get", "pods", "-n", testNamespace, "-o", "json") + output, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + var podList struct { + Items []struct { + Metadata struct { + Name string `json:"name"` + OwnerReferences []struct { + Kind string `json:"kind"` + Name string `json:"name"` + } `json:"ownerReferences"` + } `json:"metadata"` + } `json:"items"` + } + err = json.Unmarshal([]byte(output), &podList) + Expect(err).NotTo(HaveOccurred()) + + podNamesList := []string{} + for _, pod := range podList.Items { + for _, owner := range pod.Metadata.OwnerReferences { + if owner.Kind == "BatchSandbox" && owner.Name == batchSandboxName { + podNamesList = append(podNamesList, pod.Metadata.Name) + break + } + } + } + Expect(len(podNamesList)).To(BeNumerically(">", 0), "Should have pods owned by BatchSandbox") + + By("waiting for BatchSandbox to expire and be deleted") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "batchsandbox", batchSandboxName, "-n", testNamespace) + _, err := utils.Run(cmd) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("not found")) + }, 2*time.Minute).Should(Succeed()) + + By("verifying pods are deleted") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "-n", testNamespace, "-o", "json") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + + var currentPodList struct { + Items []struct { + Metadata struct { + Name string `json:"name"` + DeletionTimestamp *string `json:"deletionTimestamp"` + OwnerReferences []struct { + Kind string `json:"kind"` + Name string `json:"name"` + } `json:"ownerReferences"` + } `json:"metadata"` + } `json:"items"` + } + err = json.Unmarshal([]byte(output), ¤tPodList) + g.Expect(err).NotTo(HaveOccurred()) + + // Verify no pods are owned by the deleted BatchSandbox or they have deletionTimestamp + for _, pod := range currentPodList.Items { + for _, owner := range pod.Metadata.OwnerReferences { + if owner.Kind == "BatchSandbox" && owner.Name == batchSandboxName { + g.Expect(pod.Metadata.DeletionTimestamp).NotTo(BeNil(), + "Pod %s owned by BatchSandbox should have deletionTimestamp set", pod.Metadata.Name) + } + } + } + }, 30*time.Second).Should(Succeed()) + }) + + It("should expire and return pooled BatchSandbox pods to pool", func() { + const poolName = "test-pool-for-expire" + const batchSandboxName = "test-bs-expire-pooled" + const testNamespace = "default" + const replicas = 1 + + By("creating a Pool") + poolYAML, err := renderTemplate("testdata/pool-basic.yaml", map[string]interface{}{ + "PoolName": poolName, + "SandboxImage": sandboxImage, + "Namespace": testNamespace, + "BufferMax": 3, + "BufferMin": 2, + "PoolMax": 5, + "PoolMin": 2, + }) + Expect(err).NotTo(HaveOccurred()) + + poolFile := filepath.Join("/tmp", "test-pool-for-expire.yaml") + err = os.WriteFile(poolFile, []byte(poolYAML), 0644) + Expect(err).NotTo(HaveOccurred()) + defer os.Remove(poolFile) + + cmd := exec.Command("kubectl", "apply", "-f", poolFile) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("waiting for Pool to be ready") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pool", poolName, "-n", testNamespace, + "-o", "jsonpath={.status.total}") + totalStr, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(totalStr).NotTo(BeEmpty()) + }, 2*time.Minute).Should(Succeed()) + + By("recording Pool allocated count before BatchSandbox creation") + cmd = exec.Command("kubectl", "get", "pool", poolName, "-n", testNamespace, + "-o", "jsonpath={.status.allocated}") + allocatedBeforeBS, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("creating a pooled BatchSandbox with expireTime") + expireTime := time.Now().Add(45 * time.Second).UTC().Format(time.RFC3339) + bsYAML, err := renderTemplate("testdata/batchsandbox-pooled.yaml", map[string]interface{}{ + "BatchSandboxName": batchSandboxName, + "SandboxImage": sandboxImage, + "Namespace": testNamespace, + "Replicas": replicas, + "PoolName": poolName, + "ExpireTime": expireTime, + }) + Expect(err).NotTo(HaveOccurred()) + + bsFile := filepath.Join("/tmp", "test-bs-expire-pooled.yaml") + err = os.WriteFile(bsFile, []byte(bsYAML), 0644) + Expect(err).NotTo(HaveOccurred()) + defer os.Remove(bsFile) + + cmd = exec.Command("kubectl", "apply", "-f", bsFile) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("recording pod names from alloc-status") + var podNamesList []string + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "batchsandbox", batchSandboxName, "-n", testNamespace, + "-o", "jsonpath={.metadata.annotations.sandbox\\.opensandbox\\.io/alloc-status}") + allocStatusJSON, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(allocStatusJSON).NotTo(BeEmpty()) + + var allocStatus struct { + Pods []string `json:"pods"` + } + err = json.Unmarshal([]byte(allocStatusJSON), &allocStatus) + g.Expect(err).NotTo(HaveOccurred()) + podNamesList = allocStatus.Pods + g.Expect(len(podNamesList)).To(BeNumerically(">", 0), "Should have allocated pods") + }, 2*time.Minute).Should(Succeed()) + + allocatedAfterBS := "" + By("verifying Pool allocated count increased after BatchSandbox allocation") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pool", poolName, "-n", testNamespace, + "-o", "jsonpath={.status.allocated}") + _allocatedAfterBS, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + allocatedAfterBS = _allocatedAfterBS + + before := 0 + if allocatedBeforeBS != "" { + fmt.Sscanf(allocatedBeforeBS, "%d", &before) + } + + after := 0 + if _allocatedAfterBS != "" { + fmt.Sscanf(allocatedAfterBS, "%d", &after) + } + + g.Expect(after).To(BeNumerically(">", before), "Pool allocated count should increase after BatchSandbox allocation") + }, 30*time.Second).Should(Succeed()) + + By("waiting for BatchSandbox to expire and be deleted") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "batchsandbox", batchSandboxName, "-n", testNamespace) + _, err := utils.Run(cmd) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("not found")) + }, 2*time.Minute).Should(Succeed()) + + By("verifying pods still exist and are returned to pool") + Eventually(func(g Gomega) { + for _, podName := range podNamesList { + cmd := exec.Command("kubectl", "get", "pod", podName, "-n", testNamespace, + "-o", "jsonpath={.metadata.name}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal(podName), "Pod should still exist") + } + }, 30*time.Second).Should(Succeed()) + + By("verifying Pool allocated count decreased after BatchSandbox expiration") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pool", poolName, "-n", testNamespace, + "-o", "jsonpath={.status.allocated}") + allocatedAfterExpiration, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + + before := 0 + if allocatedAfterBS != "" { + fmt.Sscanf(allocatedAfterBS, "%d", &before) + } + after := 0 + if allocatedAfterExpiration != "" { + fmt.Sscanf(allocatedAfterExpiration, "%d", &after) + } + g.Expect(after).To(BeNumerically("<", before), "Allocated count should decrease") + }, 30*time.Second).Should(Succeed()) + + By("cleaning up Pool") + cmd = exec.Command("kubectl", "delete", "pool", poolName, "-n", testNamespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + }) + }) + + Context("Task", func() { + BeforeAll(func() { + By("waiting for controller to be ready") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager", + "-n", namespace, "-o", "jsonpath={.items[0].status.phase}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Running")) + }, 2*time.Minute).Should(Succeed()) + }) + + It("should successfully manage Pool with task scheduling", func() { + const poolName = "test-pool" + const batchSandboxName = "test-batchsandbox-with-task" + const testNamespace = "default" + const replicas = 2 + + By("creating a Pool with task-executor sidecar") + poolTemplateFile := filepath.Join("testdata", "pool-with-task-executor.yaml") + poolYAML, err := renderTemplate(poolTemplateFile, map[string]interface{}{ + "PoolName": poolName, + "Namespace": testNamespace, + "TaskExecutorImage": taskExecutorImage, + }) + Expect(err).NotTo(HaveOccurred()) + + poolFile := filepath.Join("/tmp", "test-pool.yaml") + err = os.WriteFile(poolFile, []byte(poolYAML), 0644) + Expect(err).NotTo(HaveOccurred()) + + cmd := exec.Command("kubectl", "apply", "-f", poolFile) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create Pool") + + By("waiting for Pool to be ready") + verifyPoolReady := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pool", poolName, "-n", testNamespace, + "-o", "jsonpath={.status.total}") + output, err := utils.Run(cmd) + By(fmt.Sprintf("waiting for Pool to be ready, output %s", output)) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).NotTo(BeEmpty(), "Pool status.total should not be empty") + } + Eventually(verifyPoolReady, 2*time.Minute).Should(Succeed()) + + By("creating a BatchSandbox with process-based tasks using the Pool") + batchSandboxTemplateFile := filepath.Join("testdata", "batchsandbox-with-process-task.yaml") + batchSandboxYAML, err := renderTemplate(batchSandboxTemplateFile, map[string]interface{}{ + "BatchSandboxName": batchSandboxName, + "Namespace": testNamespace, + "Replicas": replicas, + "PoolName": poolName, + "TaskExecutorImage": taskExecutorImage, + }) + Expect(err).NotTo(HaveOccurred()) + + batchSandboxFile := filepath.Join("/tmp", "test-batchsandbox.yaml") + err = os.WriteFile(batchSandboxFile, []byte(batchSandboxYAML), 0644) + Expect(err).NotTo(HaveOccurred()) + + cmd = exec.Command("kubectl", "apply", "-f", batchSandboxFile) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create BatchSandbox") + + By("verifying BatchSandbox successfully allocated endpoints") + verifyBatchSandboxAllocated := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "batchsandbox", batchSandboxName, "-n", testNamespace, + "-o", "jsonpath={.status.allocated}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal(fmt.Sprintf("%d", replicas)), "BatchSandbox should allocate %d replicas", replicas) + } + Eventually(verifyBatchSandboxAllocated, 2*time.Minute).Should(Succeed()) + + By("verifying BatchSandbox endpoints are available") + verifyEndpoints := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "batchsandbox", batchSandboxName, "-n", testNamespace, + "-o", "jsonpath={.metadata.annotations.sandbox\\.opensandbox\\.io/endpoints}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).NotTo(BeEmpty(), "BatchSandbox should have sandbox.opensandbox.io/endpoints annotation") + endpoints := strings.Split(output, ",") + g.Expect(len(endpoints)).To(Equal(replicas), "Should have %d endpoints", replicas) + } + Eventually(verifyEndpoints, 30*time.Second).Should(Succeed()) + + By("verifying BatchSandbox status is as expected") + verifyBatchSandboxStatus := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "batchsandbox", batchSandboxName, "-n", testNamespace, + "-o", "jsonpath={.status}") + statusOutput, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(statusOutput).To(ContainSubstring(fmt.Sprintf(`"replicas":%d`, replicas))) + g.Expect(statusOutput).To(ContainSubstring(fmt.Sprintf(`"allocated":%d`, replicas))) + g.Expect(statusOutput).To(ContainSubstring(fmt.Sprintf(`"ready":%d`, replicas))) + } + Eventually(verifyBatchSandboxStatus, 30*time.Second).Should(Succeed()) + + By("verifying all tasks are successfully scheduled and succeeded") + verifyTasksSucceeded := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "batchsandbox", batchSandboxName, "-n", testNamespace, + "-o", "jsonpath={.status.taskSucceed}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal(fmt.Sprintf("%d", replicas)), "All tasks should succeed") + + cmd = exec.Command("kubectl", "get", "batchsandbox", batchSandboxName, "-n", testNamespace, + "-o", "jsonpath={.status.taskFailed}") + output, err = utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("0"), "No tasks should fail") + } + Eventually(verifyTasksSucceeded, 2*time.Minute).Should(Succeed()) + + By("recording Pool status before deletion") + cmd = exec.Command("kubectl", "get", "pool", poolName, "-n", testNamespace, + "-o", "jsonpath={.status.allocated}") + poolAllocatedBefore, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("deleting the BatchSandbox") + cmd = exec.Command("kubectl", "delete", "batchsandbox", batchSandboxName, "-n", testNamespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to delete BatchSandbox") + + By("verifying all tasks are unloaded and BatchSandbox is deleted") + verifyBatchSandboxDeleted := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "batchsandbox", batchSandboxName, "-n", testNamespace) + _, err := utils.Run(cmd) + g.Expect(err).To(HaveOccurred(), "BatchSandbox should be deleted") + g.Expect(err.Error()).To(ContainSubstring("not found")) + } + Eventually(verifyBatchSandboxDeleted, 2*time.Minute).Should(Succeed()) + + By("verifying pods are returned to the Pool") + verifyPodsReturnedToPool := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pool", poolName, "-n", testNamespace, + "-o", "jsonpath={.status.allocated}") + poolAllocatedAfter, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + + beforeCount := 0 + if poolAllocatedBefore != "" { + fmt.Sscanf(poolAllocatedBefore, "%d", &beforeCount) + } + afterCount := 0 + if poolAllocatedAfter != "" { + fmt.Sscanf(poolAllocatedAfter, "%d", &afterCount) + } + g.Expect(afterCount).To(BeNumerically("<=", beforeCount), + "Pool allocated count should decrease or stay same after BatchSandbox deletion") + } + Eventually(verifyPodsReturnedToPool, 30*time.Second).Should(Succeed()) + + By("cleaning up the Pool") + cmd = exec.Command("kubectl", "delete", "pool", poolName, "-n", testNamespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to delete Pool") + + By("cleaning up temporary files") + os.Remove(poolFile) + os.Remove(batchSandboxFile) + }) + }) + +}) + +// renderTemplate renders a YAML template file with the given data. +func renderTemplate(templateFile string, data map[string]interface{}) (string, error) { + dir, err := utils.GetProjectDir() + if err != nil { + return "", err + } + + fullPath := filepath.Join(dir, "test", "e2e", templateFile) + tmplContent, err := os.ReadFile(fullPath) + if err != nil { + return "", fmt.Errorf("failed to read template file %s: %w", fullPath, err) + } + + tmpl, err := template.New("yaml").Parse(string(tmplContent)) + if err != nil { + return "", fmt.Errorf("failed to parse template: %w", err) + } + + var buf bytes.Buffer + err = tmpl.Execute(&buf, data) + if err != nil { + return "", fmt.Errorf("failed to execute template: %w", err) + } + + return buf.String(), nil +} diff --git a/kubernetes/test/e2e/testdata/batchsandbox-non-pooled-expire.yaml b/kubernetes/test/e2e/testdata/batchsandbox-non-pooled-expire.yaml new file mode 100644 index 00000000..e144497d --- /dev/null +++ b/kubernetes/test/e2e/testdata/batchsandbox-non-pooled-expire.yaml @@ -0,0 +1,14 @@ +apiVersion: sandbox.opensandbox.io/v1alpha1 +kind: BatchSandbox +metadata: + name: {{.BatchSandboxName}} + namespace: {{.Namespace}} +spec: + replicas: {{.Replicas}} + expireTime: "{{.ExpireTime}}" + template: + spec: + containers: + - name: sandbox-container + image: {{.SandboxImage}} + command: ["sleep", "3600"] diff --git a/kubernetes/test/e2e/testdata/batchsandbox-non-pooled.yaml b/kubernetes/test/e2e/testdata/batchsandbox-non-pooled.yaml new file mode 100644 index 00000000..fbd3d7df --- /dev/null +++ b/kubernetes/test/e2e/testdata/batchsandbox-non-pooled.yaml @@ -0,0 +1,13 @@ +apiVersion: sandbox.opensandbox.io/v1alpha1 +kind: BatchSandbox +metadata: + name: {{.BatchSandboxName}} + namespace: {{.Namespace}} +spec: + replicas: {{.Replicas}} + template: + spec: + containers: + - name: sandbox-container + image: {{.SandboxImage}} + command: ["sleep", "3600"] \ No newline at end of file diff --git a/kubernetes/test/e2e/testdata/batchsandbox-pooled-no-expire.yaml b/kubernetes/test/e2e/testdata/batchsandbox-pooled-no-expire.yaml new file mode 100644 index 00000000..eeb550ad --- /dev/null +++ b/kubernetes/test/e2e/testdata/batchsandbox-pooled-no-expire.yaml @@ -0,0 +1,8 @@ +apiVersion: sandbox.opensandbox.io/v1alpha1 +kind: BatchSandbox +metadata: + name: {{.BatchSandboxName}} + namespace: {{.Namespace}} +spec: + replicas: {{.Replicas}} + poolRef: {{.PoolName}} diff --git a/kubernetes/test/e2e/testdata/batchsandbox-pooled.yaml b/kubernetes/test/e2e/testdata/batchsandbox-pooled.yaml new file mode 100644 index 00000000..a434145c --- /dev/null +++ b/kubernetes/test/e2e/testdata/batchsandbox-pooled.yaml @@ -0,0 +1,9 @@ +apiVersion: sandbox.opensandbox.io/v1alpha1 +kind: BatchSandbox +metadata: + name: {{.BatchSandboxName}} + namespace: {{.Namespace}} +spec: + replicas: {{.Replicas}} + poolRef: {{.PoolName}} + expireTime: "{{.ExpireTime}}" \ No newline at end of file diff --git a/kubernetes/test/e2e/testdata/batchsandbox-with-process-task.yaml b/kubernetes/test/e2e/testdata/batchsandbox-with-process-task.yaml new file mode 100644 index 00000000..ce69e9f3 --- /dev/null +++ b/kubernetes/test/e2e/testdata/batchsandbox-with-process-task.yaml @@ -0,0 +1,13 @@ +apiVersion: sandbox.opensandbox.io/v1alpha1 +kind: BatchSandbox +metadata: + name: {{.BatchSandboxName}} + namespace: {{.Namespace}} +spec: + replicas: {{.Replicas}} + poolRef: {{.PoolName}} + taskTemplate: + spec: + process: + command: ["echo"] + args: ["Hello from task"] diff --git a/kubernetes/test/e2e/testdata/pool-basic.yaml b/kubernetes/test/e2e/testdata/pool-basic.yaml new file mode 100644 index 00000000..03f30a79 --- /dev/null +++ b/kubernetes/test/e2e/testdata/pool-basic.yaml @@ -0,0 +1,17 @@ +apiVersion: sandbox.opensandbox.io/v1alpha1 +kind: Pool +metadata: + name: {{.PoolName}} + namespace: {{.Namespace}} +spec: + template: + spec: + containers: + - name: sandbox-container + image: {{.SandboxImage}} + command: ["sleep", "3600"] + capacitySpec: + bufferMax: {{.BufferMax}} + bufferMin: {{.BufferMin}} + poolMax: {{.PoolMax}} + poolMin: {{.PoolMin}} \ No newline at end of file diff --git a/kubernetes/test/e2e/testdata/pool-with-env.yaml b/kubernetes/test/e2e/testdata/pool-with-env.yaml new file mode 100644 index 00000000..8ff8c78b --- /dev/null +++ b/kubernetes/test/e2e/testdata/pool-with-env.yaml @@ -0,0 +1,20 @@ +apiVersion: sandbox.opensandbox.io/v1alpha1 +kind: Pool +metadata: + name: {{.PoolName}} + namespace: {{.Namespace}} +spec: + template: + spec: + containers: + - name: sandbox-container + image: {{.SandboxImage}} + command: ["sleep", "3600"] + env: + - name: UPGRADED + value: "true" + capacitySpec: + bufferMax: {{.BufferMax}} + bufferMin: {{.BufferMin}} + poolMax: {{.PoolMax}} + poolMin: {{.PoolMin}} diff --git a/kubernetes/test/e2e/testdata/pool-with-task-executor.yaml b/kubernetes/test/e2e/testdata/pool-with-task-executor.yaml new file mode 100644 index 00000000..b3e95f9b --- /dev/null +++ b/kubernetes/test/e2e/testdata/pool-with-task-executor.yaml @@ -0,0 +1,16 @@ +apiVersion: sandbox.opensandbox.io/v1alpha1 +kind: Pool +metadata: + name: {{.PoolName}} + namespace: {{.Namespace}} +spec: + template: + spec: + containers: + - name: task-executor + image: {{.TaskExecutorImage}} + capacitySpec: + bufferMax: 0 + bufferMin: 0 + poolMax: 10 + poolMin: 2 diff --git a/kubernetes/test/e2e_task/suite_test.go b/kubernetes/test/e2e_task/suite_test.go new file mode 100644 index 00000000..450edf3b --- /dev/null +++ b/kubernetes/test/e2e_task/suite_test.go @@ -0,0 +1,27 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e_task + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestE2E(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Task Executor E2E Suite") +} diff --git a/kubernetes/test/e2e_task/task_e2e_test.go b/kubernetes/test/e2e_task/task_e2e_test.go new file mode 100644 index 00000000..f4d4f4c3 --- /dev/null +++ b/kubernetes/test/e2e_task/task_e2e_test.go @@ -0,0 +1,151 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e_task + +import ( + "context" + "fmt" + "os" + "os/exec" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/alibaba/OpenSandbox/sandbox-k8s/api/v1alpha1" + api "github.com/alibaba/OpenSandbox/sandbox-k8s/pkg/task-executor" +) + +const ( + ImageName = "task-executor-e2e" + TargetContainer = "task-e2e-target" + ExecutorContainer = "task-e2e-executor" + VolumeName = "task-e2e-vol" + HostPort = "38080" +) + +var _ = Describe("Task Executor E2E", Ordered, func() { + var client *api.Client + + BeforeAll(func() { + // Check docker + _, err := exec.LookPath("docker") + Expect(err).NotTo(HaveOccurred(), "Docker not found, skipping E2E test") + + By("Building image") + cmd := exec.Command("docker", "build", + "--build-arg", "PACKAGE=cmd/task-executor/main.go", + "-t", ImageName, "-f", "../../Dockerfile", "../../") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + Expect(cmd.Run()).To(Succeed()) + + By("Cleaning up previous runs") + exec.Command("docker", "rm", "-f", TargetContainer, ExecutorContainer).Run() + exec.Command("docker", "volume", "rm", VolumeName).Run() + + By("Creating shared volume") + Expect(exec.Command("docker", "volume", "create", VolumeName).Run()).To(Succeed()) + + By("Starting target container") + targetCmd := exec.Command("docker", "run", "-d", "--name", TargetContainer, + "-v", fmt.Sprintf("%s:/tmp/tasks", VolumeName), + "-e", "SANDBOX_MAIN_CONTAINER=main", + "golang:1.24", "sleep", "infinity") + targetCmd.Stdout = os.Stdout + targetCmd.Stderr = os.Stderr + Expect(targetCmd.Run()).To(Succeed()) + + By("Starting executor container in Sidecar Mode") + execCmd := exec.Command("docker", "run", "-d", "--name", ExecutorContainer, + "-v", fmt.Sprintf("%s:/tmp/tasks", VolumeName), + "--privileged", + "-u", "0", + "--pid=container:"+TargetContainer, + "-p", HostPort+":8080", + ImageName, + "-enable-sidecar-mode=true", + "-main-container-name=main", + "-data-dir=/tmp/tasks") + execCmd.Stdout = os.Stdout + execCmd.Stderr = os.Stderr + Expect(execCmd.Run()).To(Succeed()) + + By("Waiting for executor to be ready") + client = api.NewClient(fmt.Sprintf("http://127.0.0.1:%s", HostPort)) + Eventually(func() error { + _, err := client.Get(context.Background()) + return err + }, 10*time.Second, 500*time.Millisecond).Should(Succeed(), "Executor failed to become ready") + }) + + AfterAll(func() { + By("Cleaning up containers") + if CurrentSpecReport().Failed() { + By("Dumping logs") + out, _ := exec.Command("docker", "logs", ExecutorContainer).CombinedOutput() + fmt.Printf("Executor Logs:\n%s\n", string(out)) + } + exec.Command("docker", "rm", "-f", TargetContainer, ExecutorContainer).Run() + exec.Command("docker", "volume", "rm", VolumeName).Run() + }) + + Context("When creating a short-lived task", func() { + taskName := "e2e-test-1" + + It("should run and succeed", func() { + By("Creating task") + task := &api.Task{ + Name: taskName, + Spec: v1alpha1.TaskSpec{ + Process: &v1alpha1.ProcessTask{ + Command: []string{"sleep", "2"}, + }, + }, + } + _, err := client.Set(context.Background(), task) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for task to succeed") + Eventually(func(g Gomega) { + got, err := client.Get(context.Background()) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(got).NotTo(BeNil()) + g.Expect(got.Name).To(Equal(taskName)) + + // Verify state + if got.Status.State.Terminated != nil { + g.Expect(got.Status.State.Terminated.ExitCode).To(BeZero()) + g.Expect(got.Status.State.Terminated.Reason).To(Equal("Succeeded")) + } else { + // Fail if not terminated yet (so Eventually retries) + g.Expect(got.Status.State.Terminated).NotTo(BeNil(), "Task status: %v", got.Status.State) + } + }, 10*time.Second, 1*time.Second).Should(Succeed()) + }) + + It("should be deletable", func() { + By("Deleting task") + _, err := client.Set(context.Background(), nil) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying deletion") + Eventually(func() *api.Task { + got, _ := client.Get(context.Background()) + return got + }, 5*time.Second, 500*time.Millisecond).Should(BeNil()) + }) + }) +}) diff --git a/kubernetes/test/utils/utils.go b/kubernetes/test/utils/utils.go new file mode 100644 index 00000000..25669e35 --- /dev/null +++ b/kubernetes/test/utils/utils.go @@ -0,0 +1,252 @@ +// Copyright 2025 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "bufio" + "bytes" + "fmt" + "os" + "os/exec" + "strings" + + . "github.com/onsi/ginkgo/v2" // nolint:revive,staticcheck +) + +const ( + prometheusOperatorVersion = "v0.77.1" + prometheusOperatorURL = "https://github.com/prometheus-operator/prometheus-operator/" + + "releases/download/%s/bundle.yaml" + + certmanagerVersion = "v1.16.3" + certmanagerURLTmpl = "https://github.com/cert-manager/cert-manager/releases/download/%s/cert-manager.yaml" +) + +func warnError(err error) { + _, _ = fmt.Fprintf(GinkgoWriter, "warning: %v\n", err) +} + +// Run executes the provided command within this context +func Run(cmd *exec.Cmd) (string, error) { + dir, _ := GetProjectDir() + cmd.Dir = dir + + if err := os.Chdir(cmd.Dir); err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "chdir dir: %q\n", err) + } + + cmd.Env = append(os.Environ(), "GO111MODULE=on") + command := strings.Join(cmd.Args, " ") + _, _ = fmt.Fprintf(GinkgoWriter, "running: %q\n", command) + output, err := cmd.CombinedOutput() + if err != nil { + return string(output), fmt.Errorf("%q failed with error %q: %w", command, string(output), err) + } + + return string(output), nil +} + +// InstallPrometheusOperator installs the prometheus Operator to be used to export the enabled metrics. +func InstallPrometheusOperator() error { + url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion) + cmd := exec.Command("kubectl", "create", "-f", url) + _, err := Run(cmd) + return err +} + +// UninstallPrometheusOperator uninstalls the prometheus +func UninstallPrometheusOperator() { + url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion) + cmd := exec.Command("kubectl", "delete", "-f", url) + if _, err := Run(cmd); err != nil { + warnError(err) + } +} + +// IsPrometheusCRDsInstalled checks if any Prometheus CRDs are installed +// by verifying the existence of key CRDs related to Prometheus. +func IsPrometheusCRDsInstalled() bool { + // List of common Prometheus CRDs + prometheusCRDs := []string{ + "prometheuses.monitoring.coreos.com", + "prometheusrules.monitoring.coreos.com", + "prometheusagents.monitoring.coreos.com", + } + + cmd := exec.Command("kubectl", "get", "crds", "-o", "custom-columns=NAME:.metadata.name") + output, err := Run(cmd) + if err != nil { + return false + } + crdList := GetNonEmptyLines(output) + for _, crd := range prometheusCRDs { + for _, line := range crdList { + if strings.Contains(line, crd) { + return true + } + } + } + + return false +} + +// UninstallCertManager uninstalls the cert manager +func UninstallCertManager() { + url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) + cmd := exec.Command("kubectl", "delete", "-f", url) + if _, err := Run(cmd); err != nil { + warnError(err) + } +} + +// InstallCertManager installs the cert manager bundle. +func InstallCertManager() error { + url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) + cmd := exec.Command("kubectl", "apply", "-f", url) + if _, err := Run(cmd); err != nil { + return err + } + // Wait for cert-manager-webhook to be ready, which can take time if cert-manager + // was re-installed after uninstalling on a cluster. + cmd = exec.Command("kubectl", "wait", "deployment.apps/cert-manager-webhook", + "--for", "condition=Available", + "--namespace", "cert-manager", + "--timeout", "5m", + ) + + _, err := Run(cmd) + return err +} + +// IsCertManagerCRDsInstalled checks if any Cert Manager CRDs are installed +// by verifying the existence of key CRDs related to Cert Manager. +func IsCertManagerCRDsInstalled() bool { + // List of common Cert Manager CRDs + certManagerCRDs := []string{ + "certificates.cert-manager.io", + "issuers.cert-manager.io", + "clusterissuers.cert-manager.io", + "certificaterequests.cert-manager.io", + "orders.acme.cert-manager.io", + "challenges.acme.cert-manager.io", + } + + // Execute the kubectl command to get all CRDs + cmd := exec.Command("kubectl", "get", "crds") + output, err := Run(cmd) + if err != nil { + return false + } + + // Check if any of the Cert Manager CRDs are present + crdList := GetNonEmptyLines(output) + for _, crd := range certManagerCRDs { + for _, line := range crdList { + if strings.Contains(line, crd) { + return true + } + } + } + + return false +} + +// LoadImageToKindClusterWithName loads a local docker image to the kind cluster +func LoadImageToKindClusterWithName(name string) error { + cluster := "kind" + if v, ok := os.LookupEnv("KIND_CLUSTER"); ok { + cluster = v + } + kindOptions := []string{"load", "docker-image", name, "--name", cluster} + cmd := exec.Command("kind", kindOptions...) + _, err := Run(cmd) + return err +} + +// GetNonEmptyLines converts given command output string into individual objects +// according to line breakers, and ignores the empty elements in it. +func GetNonEmptyLines(output string) []string { + var res []string + elements := strings.Split(output, "\n") + for _, element := range elements { + if element != "" { + res = append(res, element) + } + } + + return res +} + +// GetProjectDir will return the directory where the project is +func GetProjectDir() (string, error) { + wd, err := os.Getwd() + if err != nil { + return wd, fmt.Errorf("failed to get current working directory: %w", err) + } + wd = strings.ReplaceAll(wd, "/test/e2e", "") + return wd, nil +} + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return fmt.Errorf("failed to read file %q: %w", filename, err) + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %q to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return fmt.Errorf("failed to write to output: %w", err) + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + if _, err = out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)); err != nil { + return fmt.Errorf("failed to write to output: %w", err) + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err = out.WriteString("\n"); err != nil { + return fmt.Errorf("failed to write to output: %w", err) + } + } + + if _, err = out.Write(content[idx+len(target):]); err != nil { + return fmt.Errorf("failed to write to output: %w", err) + } + + // false positive + // nolint:gosec + if err = os.WriteFile(filename, out.Bytes(), 0644); err != nil { + return fmt.Errorf("failed to write file %q: %w", filename, err) + } + + return nil +}