Skip to content

Commit 9bac6f9

Browse files
yhwangyhwang
authored
Controller Impl (#5)
Use the kubebuilder to generate the skeleton of the controller and implement the first version of EvalJob CRD Create the EvalJob CRD Implement the controller to handle the CR objects Implement a driver to wrap the Python program that runs the lm-eval job Update the manifests to deploy the CRD, controller, and related resources --------- Signed-off-by: Yihong Wang <[email protected]>
1 parent e8b31ca commit 9bac6f9

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

60 files changed

+3845
-0
lines changed

‎.dockerignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file
2+
# Ignore build and test binaries.
3+
bin/

‎.gitignore‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,31 @@
88
**/.pytest_cache
99
.coverage
1010
*.egg-info/
11+
12+
# Binaries for programs and plugins
13+
*.exe
14+
*.exe~
15+
*.dll
16+
*.so
17+
*.dylib
18+
bin/*
19+
Dockerfile.cross
20+
21+
# Test binary, built with `go test -c`
22+
*.test
23+
24+
# Output of the go coverage tool, specifically when used with LiteIDE
25+
*.out
26+
27+
# Go workspace file
28+
go.work
29+
30+
# Kubernetes Generated files - skip generated files, except for vendored files
31+
!vendor/**/zz_generated.*
32+
33+
# editor and IDE paraphernalia
34+
.idea
35+
.vscode
36+
*.swp
37+
*.swo
38+
*~

‎Makefile‎

Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,212 @@
1+
# Image URL to use all building/pushing image targets
2+
IMG ?= quay.io/yhwang/lm-eval-aas-controller:latest
3+
IMG_DRIVER ?= quay.io/yhwang/lm-eval-aas-driver:latest
4+
# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
5+
ENVTEST_K8S_VERSION = 1.25.0
6+
7+
DOCKER_FILE ?= docker/Dockerfile.controller
8+
9+
DOCKER_FILE_DRIVER ?= docker/Dockerfile.driver
10+
11+
# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
12+
ifeq (,$(shell go env GOBIN))
13+
GOBIN=$(shell go env GOPATH)/bin
14+
else
15+
GOBIN=$(shell go env GOBIN)
16+
endif
17+
18+
# CONTAINER_TOOL defines the container tool to be used for building images.
19+
# Be aware that the target commands are only tested with Docker which is
20+
# scaffolded by default. However, you might want to replace it to use other
21+
# tools. (i.e. podman)
22+
CONTAINER_TOOL ?= docker
23+
24+
# Setting SHELL to bash allows bash commands to be executed by recipes.
25+
# Options are set to exit when a recipe line exits non-zero or a piped command fails.
26+
SHELL = /usr/bin/env bash -o pipefail
27+
.SHELLFLAGS = -ec
28+
29+
.PHONY: all
30+
all: build
31+
32+
##@ General
33+
34+
# The help target prints out all targets with their descriptions organized
35+
# beneath their categories. The categories are represented by '##@' and the
36+
# target descriptions by '##'. The awk command is responsible for reading the
37+
# entire set of makefiles included in this invocation, looking for lines of the
38+
# file as xyz: ## something, and then pretty-format the target and help. Then,
39+
# if there's a line with ##@ something, that gets pretty-printed as a category.
40+
# More info on the usage of ANSI control characters for terminal formatting:
41+
# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters
42+
# More info on the awk command:
43+
# http://linuxcommand.org/lc3_adv_awk.php
44+
45+
.PHONY: help
46+
help: ## Display this help.
47+
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
48+
49+
##@ Development
50+
51+
.PHONY: manifests
52+
manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
53+
$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
54+
55+
.PHONY: generate
56+
generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
57+
$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
58+
59+
.PHONY: fmt
60+
fmt: ## Run go fmt against code.
61+
go fmt ./...
62+
63+
.PHONY: vet
64+
vet: ## Run go vet against code.
65+
go vet ./...
66+
67+
.PHONY: test
68+
test: manifests generate fmt vet envtest ## Run tests.
69+
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -i --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out
70+
71+
# Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors.
72+
.PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up.
73+
test-e2e:
74+
go test ./test/e2e/ -v -ginkgo.v
75+
76+
.PHONY: lint
77+
lint: golangci-lint ## Run golangci-lint linter
78+
$(GOLANGCI_LINT) run
79+
80+
.PHONY: lint-fix
81+
lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes
82+
$(GOLANGCI_LINT) run --fix
83+
84+
##@ Build
85+
86+
.PHONY: build
87+
build: manifests generate fmt vet ## Build manager binary.
88+
go build -o bin/manager cmd/controller/main.go
89+
go build -o bin/driver cmd/driver/main.go
90+
91+
.PHONY: run
92+
run: manifests generate fmt vet ## Run a controller from your host.
93+
go run ./cmd/controller/main.go
94+
95+
# If you wish to build the manager image targeting other platforms you can use the --platform flag.
96+
# (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it.
97+
# More info: https://docs.docker.com/develop/develop-images/build_enhancements/
98+
.PHONY: docker-build
99+
docker-build: ## Build docker image with the manager.
100+
$(CONTAINER_TOOL) build -t ${IMG} -f $(DOCKER_FILE) .
101+
102+
.PHONY: docker-build-driver
103+
docker-build-driver: ## Build docker image with the manager.
104+
$(CONTAINER_TOOL) build -t ${IMG_DRIVER} -f $(DOCKER_FILE_DRIVER) .
105+
106+
.PHONY: docker-push
107+
docker-push: ## Push docker image with the manager.
108+
$(CONTAINER_TOOL) push ${IMG}
109+
110+
.PHONY: docker-push-driver
111+
docker-push-driver: ## Push docker image with the manager.
112+
$(CONTAINER_TOOL) push ${IMG_DRIVER}
113+
114+
# PLATFORMS defines the target platforms for the manager image be built to provide support to multiple
115+
# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to:
116+
# - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/
117+
# - have enabled BuildKit. More info: https://docs.docker.com/develop/develop-images/build_enhancements/
118+
# - be able to push the image to your registry (i.e. if you do not set a valid value via IMG=<myregistry/image:<tag>> then the export will fail)
119+
# To adequately provide solutions that are compatible with multiple platforms, you should consider using this option.
120+
PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le
121+
.PHONY: docker-buildx
122+
docker-buildx: ## Build and push docker image for the manager for cross-platform support
123+
# copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile
124+
sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross
125+
- $(CONTAINER_TOOL) buildx create --name fms-lm-eval-service-builder
126+
$(CONTAINER_TOOL) buildx use fms-lm-eval-service-builder
127+
- $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross .
128+
- $(CONTAINER_TOOL) buildx rm fms-lm-eval-service-builder
129+
rm Dockerfile.cross
130+
131+
.PHONY: build-installer
132+
build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment.
133+
mkdir -p dist
134+
cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
135+
$(KUSTOMIZE) build config/default > dist/install.yaml
136+
137+
##@ Deployment
138+
139+
ifndef ignore-not-found
140+
ignore-not-found = false
141+
endif
142+
143+
.PHONY: install
144+
install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
145+
$(KUSTOMIZE) build config/crd | $(KUBECTL) apply -f -
146+
147+
.PHONY: uninstall
148+
uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
149+
$(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f -
150+
151+
.PHONY: deploy
152+
deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
153+
cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
154+
$(KUSTOMIZE) build config/default | $(KUBECTL) apply -f -
155+
156+
.PHONY: undeploy
157+
undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
158+
$(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f -
159+
160+
##@ Dependencies
161+
162+
## Location to install dependencies to
163+
LOCALBIN ?= $(shell pwd)/bin
164+
$(LOCALBIN):
165+
mkdir -p $(LOCALBIN)
166+
167+
## Tool Binaries
168+
KUBECTL ?= kubectl
169+
KUSTOMIZE ?= $(LOCALBIN)/kustomize-$(KUSTOMIZE_VERSION)
170+
CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen-$(CONTROLLER_TOOLS_VERSION)
171+
ENVTEST ?= $(LOCALBIN)/setup-envtest-$(ENVTEST_VERSION)
172+
GOLANGCI_LINT = $(LOCALBIN)/golangci-lint-$(GOLANGCI_LINT_VERSION)
173+
174+
## Tool Versions
175+
KUSTOMIZE_VERSION ?= v5.4.1
176+
CONTROLLER_TOOLS_VERSION ?= v0.15.0
177+
ENVTEST_VERSION ?= release-0.18
178+
GOLANGCI_LINT_VERSION ?= v1.57.2
179+
180+
.PHONY: kustomize
181+
kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary.
182+
$(KUSTOMIZE): $(LOCALBIN)
183+
$(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v5,$(KUSTOMIZE_VERSION))
184+
185+
.PHONY: controller-gen
186+
controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary.
187+
$(CONTROLLER_GEN): $(LOCALBIN)
188+
$(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen,$(CONTROLLER_TOOLS_VERSION))
189+
190+
.PHONY: envtest
191+
envtest: $(ENVTEST) ## Download setup-envtest locally if necessary.
192+
$(ENVTEST): $(LOCALBIN)
193+
$(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest,$(ENVTEST_VERSION))
194+
195+
.PHONY: golangci-lint
196+
golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary.
197+
$(GOLANGCI_LINT): $(LOCALBIN)
198+
$(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint,${GOLANGCI_LINT_VERSION})
199+
200+
# go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist
201+
# $1 - target path with name of binary (ideally with version)
202+
# $2 - package url which can be installed
203+
# $3 - specific version of package
204+
define go-install-tool
205+
@[ -f $(1) ] || { \
206+
set -e; \
207+
package=$(2)@$(3) ;\
208+
echo "Downloading $${package}" ;\
209+
GOBIN=$(LOCALBIN) go install $${package} ;\
210+
mv "$$(echo "$(1)" | sed "s/-$(3)$$//")" $(1) ;\
211+
}
212+
endef

‎PROJECT‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# Code generated by tool. DO NOT EDIT.
2+
# This file is used to track the info used to scaffold your project
3+
# and allow the plugins properly work.
4+
# More info: https://book.kubebuilder.io/reference/project-config.html
5+
domain: github.com
6+
layout:
7+
- go.kubebuilder.io/v4
8+
projectName: fms-lm-eval-service
9+
repo: github.com/foundation-model-stack/fms-lm-eval-service
10+
resources:
11+
- api:
12+
crdVersion: v1
13+
namespaced: true
14+
controller: true
15+
domain: github.com
16+
group: lm-eval-service
17+
kind: EvalJob
18+
path: github.com/foundation-model-stack/fms-lm-eval-service/api/v1beta1
19+
version: v1beta1
20+
webhooks:
21+
defaulting: true
22+
validation: true
23+
webhookVersion: v1
24+
version: "3"

0 commit comments

Comments
 (0)