Skip to content

Commit aa1e6f8

Browse files
ArvindthiruArvindthiru
authored
feat: add init container to install fleet CRDs (Azure#1141)
2 parents ced2c6c + 061372e commit aa1e6f8

File tree

16 files changed

+917
-27
lines changed

16 files changed

+917
-27
lines changed

‎Makefile‎

Lines changed: 45 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,12 @@ endif
66
HUB_AGENT_IMAGE_VERSION ?= $(TAG)
77
MEMBER_AGENT_IMAGE_VERSION ?= $(TAG)
88
REFRESH_TOKEN_IMAGE_VERSION ?= $(TAG)
9+
CRD_INSTALLER_IMAGE_VERSION ?= $(TAG)
910

1011
HUB_AGENT_IMAGE_NAME ?= hub-agent
1112
MEMBER_AGENT_IMAGE_NAME ?= member-agent
12-
REFRESH_TOKEN_IMAGE_NAME := refresh-token
13+
REFRESH_TOKEN_IMAGE_NAME ?= refresh-token
14+
CRD_INSTALLER_IMAGE_NAME ?= crd-installer
1315

1416
KUBECONFIG ?= $(HOME)/.kube/config
1517
HUB_SERVER_URL ?= https://172.19.0.2:6443
@@ -48,7 +50,7 @@ ENVTEST_K8S_VERSION = 1.30.0
4850
# ENVTEST_VER is the version of the ENVTEST binary
4951
ENVTEST_VER = v0.0.0-20240317073005-bd9ea79e8d18
5052
ENVTEST_BIN := setup-envtest
51-
ENVTEST := $(abspath $(TOOLS_BIN_DIR)/$(ENVTEST_BIN)-$(ENVTEST_VER))
53+
ENVTEST := $(abspath $(TOOLS_BIN_DIR)/$(ENVTEST_BIN)-$(ENVTEST_VER))
5254

5355
# Scripts
5456
GO_INSTALL := ./hack/go-install.sh
@@ -100,7 +102,7 @@ staticcheck: $(STATICCHECK)
100102
$(STATICCHECK) ./...
101103

102104
.PHONY: fmt
103-
fmt: $(GOIMPORTS) ## Run go fmt against code.
105+
fmt: $(GOIMPORTS) ## Run go fmt against code.
104106
go fmt ./...
105107
$(GOIMPORTS) -local go.goms.io/fleet -w $$(go list -f {{.Dir}} ./...)
106108

@@ -121,17 +123,18 @@ create-member-kind-cluster:
121123
kind create cluster --name $(MEMBER_KIND_CLUSTER_NAME) --image=$(KIND_IMAGE) --config=$(CLUSTER_CONFIG) --kubeconfig=$(KUBECONFIG)
122124

123125
load-hub-docker-image:
124-
kind load docker-image --name $(HUB_KIND_CLUSTER_NAME) $(REGISTRY)/$(HUB_AGENT_IMAGE_NAME):$(HUB_AGENT_IMAGE_VERSION)
126+
kind load docker-image --name $(HUB_KIND_CLUSTER_NAME) $(REGISTRY)/$(HUB_AGENT_IMAGE_NAME):$(HUB_AGENT_IMAGE_VERSION)
125127

126128
load-member-docker-image:
127-
kind load docker-image --name $(MEMBER_KIND_CLUSTER_NAME) $(REGISTRY)/$(REFRESH_TOKEN_IMAGE_NAME):$(REFRESH_TOKEN_IMAGE_VERSION) $(REGISTRY)/$(MEMBER_AGENT_IMAGE_NAME):$(MEMBER_AGENT_IMAGE_VERSION)
129+
kind load docker-image --name $(MEMBER_KIND_CLUSTER_NAME) $(REGISTRY)/$(REFRESH_TOKEN_IMAGE_NAME):$(REFRESH_TOKEN_IMAGE_VERSION)
130+
kind load docker-image --name $(MEMBER_KIND_CLUSTER_NAME) $(REGISTRY)/$(MEMBER_AGENT_IMAGE_NAME):$(MEMBER_AGENT_IMAGE_VERSION)
128131

129132
## --------------------------------------
130133
## test
131134
## --------------------------------------
132135

133136
.PHONY: test
134-
test: manifests generate fmt vet local-unit-test integration-test## Run tests.
137+
test: manifests generate fmt vet local-unit-test integration-test ## Run tests.
135138

136139
##
137140
## workaround to bypass the pkg/controllers/workv1alpha1 tests failure
@@ -148,26 +151,26 @@ integration-test: $(ENVTEST) ## Run tests.
148151
export CGO_ENABLED=1 && \
149152
export KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" && \
150153
ginkgo -v -p --race --cover --coverpkg=./pkg/scheduler/... ./test/scheduler && \
151-
ginkgo -v -p --race --cover --coverpkg=./... ./test/apis/... && \
154+
ginkgo -v -p --race --cover --coverpkg=./... ./test/apis/... ./test/crdinstaller && \
152155
go test ./test/integration/... -coverpkg=./... -race -coverprofile=it-coverage.xml -v
153156

154157
## local tests & e2e tests
155158

156159
install-hub-agent-helm:
157160
kind export kubeconfig --name $(HUB_KIND_CLUSTER_NAME)
158161
helm install hub-agent ./charts/hub-agent/ \
159-
--set image.pullPolicy=Never \
160-
--set image.repository=$(REGISTRY)/$(HUB_AGENT_IMAGE_NAME) \
161-
--set image.tag=$(HUB_AGENT_IMAGE_VERSION) \
162-
--set logVerbosity=5 \
163-
--set namespace=fleet-system \
164-
--set enableWebhook=true \
165-
--set webhookServiceName=fleetwebhook \
166-
--set webhookClientConnectionType=service \
167-
--set enableV1Alpha1APIs=true \
168-
--set enableV1Beta1APIs=false \
169-
--set enableClusterInventoryAPI=true \
170-
--set logFileMaxSize=1000000
162+
--set image.pullPolicy=Never \
163+
--set image.repository=$(REGISTRY)/$(HUB_AGENT_IMAGE_NAME) \
164+
--set image.tag=$(HUB_AGENT_IMAGE_VERSION) \
165+
--set logVerbosity=5 \
166+
--set namespace=fleet-system \
167+
--set enableWebhook=true \
168+
--set webhookServiceName=fleetwebhook \
169+
--set webhookClientConnectionType=service \
170+
--set enableV1Alpha1APIs=true \
171+
--set enableV1Beta1APIs=false \
172+
--set enableClusterInventoryAPI=true \
173+
--set logFileMaxSize=1000000
171174

172175
.PHONY: e2e-v1alpha1-hub-kubeconfig-secret
173176
e2e-v1alpha1-hub-kubeconfig-secret:
@@ -187,12 +190,13 @@ install-member-agent-helm: install-hub-agent-helm e2e-v1alpha1-hub-kubeconfig-se
187190
--set config.hubURL=$$HUB_SERVER_URL \
188191
--set image.repository=$(REGISTRY)/$(MEMBER_AGENT_IMAGE_NAME) \
189192
--set image.tag=$(MEMBER_AGENT_IMAGE_VERSION) \
190-
--set refreshtoken.repository=$(REGISTRY)/$(REFRESH_TOKEN_IMAGE_NAME) \
191-
--set refreshtoken.tag=$(REFRESH_TOKEN_IMAGE_VERSION) \
192-
--set image.pullPolicy=Never --set refreshtoken.pullPolicy=Never \
193-
--set config.memberClusterName="kind-$(MEMBER_KIND_CLUSTER_NAME)" \
194-
--set logVerbosity=5 \
195-
--set namespace=fleet-system
193+
--set refreshtoken.repository=$(REGISTRY)/$(REFRESH_TOKEN_IMAGE_NAME) \
194+
--set refreshtoken.tag=$(REFRESH_TOKEN_IMAGE_VERSION) \
195+
--set image.pullPolicy=Never \
196+
--set refreshtoken.pullPolicy=Never \
197+
--set config.memberClusterName="kind-$(MEMBER_KIND_CLUSTER_NAME)" \
198+
--set logVerbosity=5 \
199+
--set namespace=fleet-system
196200
# to make sure member-agent reads the token file.
197201
kubectl delete pod --all -n fleet-system
198202

@@ -206,7 +210,7 @@ run-e2e-v1alpha1: build-e2e-v1alpha1
206210
create-kind-cluster: create-hub-kind-cluster create-member-kind-cluster install-helm
207211

208212
.PHONY: install-helm
209-
install-helm: load-hub-docker-image load-member-docker-image install-member-agent-helm
213+
install-helm: load-hub-docker-image load-member-docker-image install-member-agent-helm
210214

211215
.PHONY: e2e-tests-v1alpha1
212216
e2e-tests-v1alpha1: create-kind-cluster run-e2e-v1alpha1
@@ -250,6 +254,7 @@ generate: $(CONTROLLER_GEN)
250254
build: generate fmt vet ## Build agent binaries.
251255
go build -o bin/hubagent cmd/hubagent/main.go
252256
go build -o bin/memberagent cmd/memberagent/main.go
257+
go build -o bin/crdinstaller cmd/crdinstaller/main.go
253258

254259
.PHONY: run-hubagent
255260
run-hubagent: manifests generate fmt vet ## Run a controllers from your host.
@@ -259,6 +264,10 @@ run-hubagent: manifests generate fmt vet ## Run a controllers from your host.
259264
run-memberagent: manifests generate fmt vet ## Run a controllers from your host.
260265
go run ./cmd/memberagent/main.go
261266

267+
.PHONY: run-crdinstaller
268+
run-crdinstaller: manifests generate fmt vet ## Run CRD installer from your host.
269+
go run ./cmd/crdinstaller/main.go --mode=$(MODE)
270+
262271
## --------------------------------------
263272
## Images
264273
## --------------------------------------
@@ -270,7 +279,7 @@ BUILDKIT_VERSION ?= v0.18.1
270279

271280
.PHONY: push
272281
push:
273-
$(MAKE) OUTPUT_TYPE="type=registry" docker-build-hub-agent docker-build-member-agent docker-build-refresh-token
282+
$(MAKE) OUTPUT_TYPE="type=registry" docker-build-hub-agent docker-build-member-agent docker-build-refresh-token docker-build-crd-installer
274283

275284
# By default, docker buildx create will pull image moby/buildkit:buildx-stable-1 and hit the too many requests error
276285
.PHONY: docker-buildx-builder
@@ -308,6 +317,15 @@ docker-build-refresh-token: docker-buildx-builder
308317
--pull \
309318
--tag $(REGISTRY)/$(REFRESH_TOKEN_IMAGE_NAME):$(REFRESH_TOKEN_IMAGE_VERSION) .
310319

320+
.PHONY: docker-build-crd-installer
321+
docker-build-crd-installer: docker-buildx-builder
322+
docker buildx build \
323+
--file docker/crd-installer.Dockerfile \
324+
--output=$(OUTPUT_TYPE) \
325+
--platform="linux/amd64" \
326+
--pull \
327+
--tag $(REGISTRY)/$(CRD_INSTALLER_IMAGE_NAME):$(CRD_INSTALLER_IMAGE_VERSION) .
328+
311329
## -----------------------------------
312330
## Cleanup
313331
## -----------------------------------

‎charts/hub-agent/templates/deployment.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,15 @@ spec:
1515
{{- include "hub-agent.selectorLabels" . | nindent 8 }}
1616
spec:
1717
serviceAccountName: {{ include "hub-agent.fullname" . }}-sa
18+
initContainers:
19+
{{- if and .Values.crdInstaller.enabled .Values.enableV1Beta1APIs }}
20+
- name: crd-installer
21+
image: "{{ .Values.crdInstaller.image.repository }}:{{ .Values.crdInstaller.image.tag }}"
22+
imagePullPolicy: {{ .Values.crdInstaller.image.pullPolicy }}
23+
args:
24+
- --mode=hub
25+
- --v={{ .Values.crdInstaller.logVerbosity }}
26+
{{- end }}
1827
containers:
1928
- name: {{ .Chart.Name }}
2029
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"

‎charts/hub-agent/values.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,15 @@ image:
1010
# Overrides the image tag whose default is the chart appVersion.
1111
tag: main
1212

13+
# CRD installer configuration.
14+
crdInstaller:
15+
enabled: false
16+
image:
17+
repository: ghcr.io/azure/fleet/crd-installer
18+
pullPolicy: Always
19+
tag: main
20+
logVerbosity: 2
21+
1322
logVerbosity: 5
1423

1524
enableWebhook: true

‎charts/member-agent/templates/deployment.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,15 @@ spec:
1616
spec:
1717
restartPolicy: Always
1818
serviceAccountName: {{ include "member-agent.fullname" . }}-sa
19+
initContainers:
20+
{{- if and .Values.crdInstaller.enabled .Values.enableV1Beta1APIs }}
21+
- name: crd-installer
22+
image: "{{ .Values.crdInstaller.image.repository }}:{{ .Values.crdInstaller.image.tag }}"
23+
imagePullPolicy: {{ .Values.crdInstaller.image.pullPolicy }}
24+
args:
25+
- --mode=member
26+
- --v={{ .Values.crdInstaller.logVerbosity }}
27+
{{- end }}
1928
containers:
2029
- name: {{ include "member-agent.fullname" . }}
2130
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"

‎charts/member-agent/values.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,15 @@ image:
55
pullPolicy: Always
66
tag: main
77

8+
# CRD installer configuration.
9+
crdInstaller:
10+
enabled: false
11+
image:
12+
repository: ghcr.io/azure/fleet/crd-installer
13+
pullPolicy: Always
14+
tag: main
15+
logVerbosity: 2
16+
817
logVerbosity: 5
918

1019
refreshtoken:

‎cmd/crdinstaller/main.go‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
/*
2+
Copyright (c) Microsoft Corporation.
3+
Licensed under the MIT license.
4+
*/
5+
6+
// Package main contains the CRD installer utility for KubeFleet.
7+
package main
8+
9+
import (
10+
"context"
11+
"flag"
12+
"fmt"
13+
14+
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
15+
"k8s.io/apimachinery/pkg/runtime"
16+
"k8s.io/klog/v2"
17+
ctrl "sigs.k8s.io/controller-runtime"
18+
"sigs.k8s.io/controller-runtime/pkg/client"
19+
"sigs.k8s.io/controller-runtime/pkg/log/zap"
20+
21+
"go.goms.io/fleet/cmd/crdinstaller/utils"
22+
)
23+
24+
var (
25+
mode = flag.String("mode", "", "Mode to run in: 'hub' or 'member' (required)")
26+
)
27+
28+
func main() {
29+
klog.InitFlags(nil)
30+
flag.Parse()
31+
32+
// Validate required flags.
33+
if *mode != "hub" && *mode != "member" {
34+
klog.Fatal("--mode flag must be either 'hub' or 'member'")
35+
}
36+
37+
klog.Infof("Starting CRD installer in %s mode", *mode)
38+
39+
// Print all flags for debugging.
40+
flag.VisitAll(func(f *flag.Flag) {
41+
klog.V(2).InfoS("flag:", "name", f.Name, "value", f.Value)
42+
})
43+
44+
// Set up controller-runtime logger.
45+
ctrl.SetLogger(zap.New(zap.UseDevMode(true)))
46+
47+
// Create context for API operations.
48+
ctx := ctrl.SetupSignalHandler()
49+
50+
// Get Kubernetes config using controller-runtime.
51+
config := ctrl.GetConfigOrDie()
52+
53+
// Create a scheme that knows about CRD types.
54+
scheme := runtime.NewScheme()
55+
if err := apiextensionsv1.AddToScheme(scheme); err != nil {
56+
klog.Fatalf("Failed to add apiextensions scheme: %v", err)
57+
}
58+
client, err := client.New(config, client.Options{
59+
Scheme: scheme,
60+
})
61+
62+
if err != nil {
63+
klog.Fatalf("Failed to create Kubernetes client: %v", err)
64+
}
65+
66+
// Install CRDs from the fixed location.
67+
const crdPath = "/workspace/config/crd/bases"
68+
if err := installCRDs(ctx, client, crdPath, *mode); err != nil {
69+
klog.Fatalf("Failed to install CRDs: %v", err)
70+
}
71+
72+
klog.Infof("Successfully installed %s CRDs", *mode)
73+
}
74+
75+
// installCRDs installs the CRDs from the specified directory based on the mode.
76+
func installCRDs(ctx context.Context, client client.Client, crdPath, mode string) error {
77+
// List of CRDs to install based on mode.
78+
crdsToInstall, err := utils.CollectCRDs(crdPath, mode, client.Scheme())
79+
if err != nil {
80+
return err
81+
}
82+
83+
if len(crdsToInstall) == 0 {
84+
return fmt.Errorf("no CRDs found for mode %s in directory %s", mode, crdPath)
85+
}
86+
87+
klog.Infof("Found %d CRDs to install for mode %s", len(crdsToInstall), mode)
88+
89+
// Install each CRD.
90+
for i := range crdsToInstall {
91+
if err := utils.InstallCRD(ctx, client, &crdsToInstall[i]); err != nil {
92+
return err
93+
}
94+
}
95+
96+
return nil
97+
}

0 commit comments

Comments
 (0)