Skip to content

Commit 7eb3030

Browse files
authored
split webhooks into multiple to avoid hitting max size (#5218)
1 parent cd58a03 commit 7eb3030

File tree

10 files changed

+196
-28
lines changed

10 files changed

+196
-28
lines changed

Taskfile.yml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -849,6 +849,7 @@ tasks:
849849
deps:
850850
- controller:generate-types
851851
- controller:generate-genruntime-deepcopy
852+
- scripts:build-python-venv
852853
sources:
853854
- "v2/api/**/*.go" # depends on all API types
854855
- "v2/pkg/genruntime/**/*.go" # Also depends on the genruntime types as they're embedded into the CRDs
@@ -858,13 +859,13 @@ tasks:
858859
- if [ -d "{{.CONTROLLER_OUTPUT}}/crd/generated/bases" ]; then find "{{.CONTROLLER_OUTPUT}}/crd/generated/bases" -type f -delete; fi
859860
- if [ -d "{{.CONTROLLER_OUTPUT}}/crd/generated/patches" ]; then find "{{.CONTROLLER_OUTPUT}}/crd/generated/patches" -type f -delete; fi
860861
- cd v2/api && controller-gen {{.OBJECT_OPTIONS}} paths=./...
861-
- cd v2/api && controller-gen {{.CRD_OPTIONS}} {{.WEBHOOK_OPTIONS}} {{.RBAC_OPTIONS}} paths=./...
862+
- cd v2/api && controller-gen {{.CRD_OPTIONS}} {{.RBAC_OPTIONS}} paths=./...
863+
- "{{.SCRIPTS_ROOT}}/venv/bin/python3 {{.SCRIPTS_ROOT}}/generate-webhooks-per-group.py --api-dir v2/api --output-dir {{.CONTROLLER_OUTPUT}}/webhook/generated"
862864
- cmd: cd v2/api && gofumpt -l -w . # format all generated code
863865
ignore_error: true # Just in case the code doesn't build
864866
vars:
865867
OBJECT_OPTIONS: object:headerFile={{.HEADER_FILE}}
866868
CRD_OPTIONS: crd:crdVersions=v1,allowDangerousTypes=true output:crd:artifacts:config={{.CONTROLLER_OUTPUT}}/crd/generated/bases
867-
WEBHOOK_OPTIONS: webhook output:webhook:artifacts:config={{.CONTROLLER_OUTPUT}}/webhook
868869
RBAC_OPTIONS: rbac:roleName=manager-role output:rbac:artifacts:config={{.CONTROLLER_OUTPUT}}/rbac
869870

870871
controller:check-crd-size:
Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
#!/usr/bin/env python3
2+
# Copyright (c) Microsoft Corporation.
3+
# Licensed under the MIT license.
4+
5+
"""
6+
Iterates through each API group folder and runs controller-gen to produce
7+
per-group webhook configurations (<group>-mwh.yaml / <group>-vwh.yaml),
8+
avoiding a single massive manifest that risks exceeding etcd size limits.
9+
10+
Usage: generate-webhooks-per-group.py --api-dir <API_DIR> --output-dir <OUTPUT_DIR>
11+
"""
12+
13+
import argparse
14+
import os
15+
import shutil
16+
import subprocess
17+
import sys
18+
import tempfile
19+
from concurrent.futures import ThreadPoolExecutor, as_completed
20+
from pathlib import Path
21+
22+
import yaml
23+
24+
KIND_SUFFIXES = {
25+
"MutatingWebhookConfiguration": "mwh",
26+
"ValidatingWebhookConfiguration": "vwh",
27+
}
28+
29+
30+
def has_webhook_markers(group_dir: Path) -> bool:
31+
"""Check if any Go file in the directory tree contains a kubebuilder:webhook marker."""
32+
for f in group_dir.rglob("*.go"):
33+
if "kubebuilder:webhook" in f.read_text(errors="replace"):
34+
return True
35+
return False
36+
37+
38+
def run_controller_gen(api_dir: Path, group: str, output_dir: Path) -> None:
39+
"""Run controller-gen webhook for a single group."""
40+
subprocess.run(
41+
[
42+
"controller-gen",
43+
"webhook",
44+
f"output:webhook:artifacts:config={output_dir}",
45+
f"paths=./{group}/...",
46+
],
47+
cwd=api_dir,
48+
check=True,
49+
)
50+
51+
52+
def split_manifests(manifests_path: Path, group: str, bases_dir: Path) -> list[str]:
53+
"""Split a multi-doc manifests.yaml into per-kind files with renamed metadata.name."""
54+
generated = []
55+
with open(manifests_path) as f:
56+
docs = list(yaml.safe_load_all(f))
57+
58+
for doc in docs:
59+
if doc is None:
60+
continue
61+
kind = doc.get("kind", "")
62+
suffix = KIND_SUFFIXES.get(kind)
63+
if suffix is None:
64+
print(f" WARNING: unexpected kind '{kind}' in {group}, skipping")
65+
continue
66+
67+
doc["metadata"]["name"] = f"{group}-{suffix}"
68+
out_file = bases_dir / f"{group}-{suffix}.yaml"
69+
with open(out_file, "w") as f:
70+
yaml.dump(doc, f, default_flow_style=False, sort_keys=False)
71+
72+
rel = f"bases/{out_file.name}"
73+
generated.append(rel)
74+
print(f" Generated: {out_file.name}")
75+
76+
return generated
77+
78+
79+
def write_kustomization(output_dir: Path, resources: list[str]) -> None:
80+
"""Write a kustomization.yaml listing all generated resources."""
81+
content = {
82+
"resources": resources,
83+
}
84+
kustomization_path = output_dir / "kustomization.yaml"
85+
with open(kustomization_path, "w") as f:
86+
f.write("# This file is auto-generated by generate-webhooks-per-group.py\n")
87+
f.write("# DO NOT EDIT manually\n")
88+
yaml.dump(content, f, default_flow_style=False, sort_keys=False)
89+
90+
91+
def main() -> None:
92+
parser = argparse.ArgumentParser(description=__doc__)
93+
parser.add_argument("--api-dir", required=True, help="Path to the api directory (e.g. v2/api)")
94+
parser.add_argument("--output-dir", required=True, help="Webhook output directory (e.g. v2/config/webhook/generated)")
95+
args = parser.parse_args()
96+
97+
api_dir = Path(args.api_dir).resolve()
98+
output_dir = Path(args.output_dir).resolve()
99+
bases_dir = output_dir / "bases"
100+
101+
# Clean and create output directories
102+
if bases_dir.exists():
103+
shutil.rmtree(bases_dir)
104+
bases_dir.mkdir(parents=True)
105+
106+
print("Generating per-group webhook configurations...")
107+
print(f" API directory: {api_dir}")
108+
print(f" Output directory: {output_dir}")
109+
110+
all_resources: list[str] = []
111+
112+
with tempfile.TemporaryDirectory() as tmp:
113+
tmp_path = Path(tmp)
114+
115+
# Phase 1: discover groups with webhook markers
116+
groups: list[str] = []
117+
group_tmp_dirs: dict[str, Path] = {}
118+
for group_dir in sorted(api_dir.iterdir()):
119+
if not group_dir.is_dir():
120+
continue
121+
if not has_webhook_markers(group_dir):
122+
continue
123+
124+
group = group_dir.name
125+
groups.append(group)
126+
group_tmp = tmp_path / group
127+
group_tmp.mkdir()
128+
group_tmp_dirs[group] = group_tmp
129+
130+
# Phase 2: run controller-gen concurrently for all groups
131+
print(f" Running controller-gen for {len(groups)} groups concurrently...")
132+
with ThreadPoolExecutor() as executor:
133+
futures = {
134+
executor.submit(run_controller_gen, api_dir, group, group_tmp_dirs[group]): group
135+
for group in groups
136+
}
137+
for future in as_completed(futures):
138+
group = futures[future]
139+
future.result() # raises on failure
140+
print(f" Completed: {group}")
141+
142+
# Phase 3: split and rename manifests
143+
for group in groups:
144+
manifests = tmp_path / group / "manifests.yaml"
145+
if not manifests.exists():
146+
print(f" WARNING: no manifests.yaml for {group}, skipping")
147+
continue
148+
all_resources.extend(split_manifests(manifests, group, bases_dir))
149+
150+
# Phase 4: write kustomization.yaml
151+
print(f"\nWriting kustomization.yaml with {len(all_resources)} resources...")
152+
write_kustomization(output_dir, all_resources)
153+
print(f"Done. Generated {len(all_resources)} webhook configuration files.")
154+
155+
156+
if __name__ == "__main__":
157+
main()

scripts/v2/package-helm-manifest.sh

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33
# package-helm-manifest.sh script is used to copy the generated files by kustomize and package the helm chart.
44
# The generated files are updated when a new resource is added. To eliminate the manual process of updating generated files, we use this script for automation.
55
# Generated files include below files:
6-
# - admissionregistration.k8s.io_v1_mutatingwebhookconfiguration_azureserviceoperator-mutating-webhook-configuration.yaml
7-
# - admissionregistration.k8s.io_v1_validatingwebhookconfiguration_azureserviceoperator-validating-webhook-configuration.yaml
6+
# - admissionregistration.k8s.io_v1_mutatingwebhookconfiguration_azureserviceoperator-<group>-mwh.yaml (one per API group)
7+
# - admissionregistration.k8s.io_v1_validatingwebhookconfiguration_azureserviceoperator-<group>-vwh.yaml (one per API group)
88
# - rbac.authorization.k8s.io_v1_clusterrole_azureserviceoperator-manager-role.yaml
99

1010
# Above files are always updated when a new resource is added
@@ -54,8 +54,8 @@ echo "Making sed replacements and copying generated yamls"
5454
for file in $(find "$TEMP_DIR" -type f)
5555
do
5656
# Append cluster or tenant guards to each file
57-
if [[ $file == *"mutating-webhook-configuration"* ]] ||
58-
[[ $file == *"validating-webhook-configuration"* ]] ||
57+
if [[ $file == *"_mutatingwebhookconfiguration_"* ]] ||
58+
[[ $file == *"_validatingwebhookconfiguration_"* ]] ||
5959
[[ $file == *"azureserviceoperator-manager-role.yaml" ]] ; then
6060
sed -i "1 s/^/$IF_CLUSTER\n/;$ a {{- end }}" "$file"
6161
sed -i 's/azureserviceoperator-system/{{ .Release.Namespace }}/g' "$file"

v2/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
config/crd/generated
22
config/webhook/manifests.yaml
3+
config/webhook/generated
34
config/rbac/role.yaml
45
test/perf/reports/graphs/
56
out/

v2/config/default/kustomization.yaml

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -35,14 +35,20 @@ patchesStrategicMerge:
3535
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in crd/kustomization.yaml
3636
- manager_webhook_patch.yaml
3737

38-
# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'.
39-
# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks.
40-
# 'CERTMANAGER' needs to be enabled to use ca injection
41-
- webhookcainjection_patch.yaml
42-
4338
# - manager_credentials_patch.yaml
4439

4540
patches:
41+
# [CERTMANAGER] Inject CA from cert-manager into all webhook configurations.
42+
- path: mutatingwebhook_cainjection_patch.yaml
43+
target:
44+
group: admissionregistration.k8s.io
45+
version: v1
46+
kind: MutatingWebhookConfiguration
47+
- path: validatingwebhook_cainjection_patch.yaml
48+
target:
49+
group: admissionregistration.k8s.io
50+
version: v1
51+
kind: ValidatingWebhookConfiguration
4652
- patch: |-
4753
- op: add
4854
path: /spec/template/spec/containers/0/args/-
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# This patch adds the cert-manager CA injection annotation to all MutatingWebhookConfigurations.
2+
# The variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) are substituted by kustomize.
3+
# Matching is done via the target selector in kustomization.yaml.
4+
apiVersion: admissionregistration.k8s.io/v1
5+
kind: MutatingWebhookConfiguration
6+
metadata:
7+
name: unused
8+
annotations:
9+
cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# This patch adds the cert-manager CA injection annotation to all ValidatingWebhookConfigurations.
2+
# The variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) are substituted by kustomize.
3+
# Matching is done via the target selector in kustomization.yaml.
4+
apiVersion: admissionregistration.k8s.io/v1
5+
kind: ValidatingWebhookConfiguration
6+
metadata:
7+
name: unused
8+
annotations:
9+
cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)

v2/config/default/webhookcainjection_patch.yaml

Lines changed: 0 additions & 15 deletions
This file was deleted.

v2/config/webhook/kustomization.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
namePrefix: azureserviceoperator-
22

33
resources:
4-
- manifests.yaml
4+
- generated
55
- service.yaml
66

77
configurations:

v2/internal/testcommon/kube_test_context_envtest.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ func createSharedEnvTest(
8080
}
8181

8282
crdPath := filepath.Join(root, "v2/out/envtest/crds")
83-
webhookPath := filepath.Join(root, "v2/config/webhook")
83+
webhookPath := filepath.Join(root, "v2/config/webhook/generated/bases")
8484

8585
environment := envtest.Environment{
8686
ErrorIfCRDPathMissing: true,

0 commit comments

Comments
 (0)