Skip to content

Commit a6c3d6b

Browse files
chrischdichrischdi
committed
add sidecar for kube-state-metrics and read crd config from cm + add option to add kustomizations via tilt
1 parent 21acce8 commit a6c3d6b

File tree

9 files changed

+187
-48
lines changed

9 files changed

+187
-48
lines changed

‎Tiltfile

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# -*- mode: Python -*-
1+
Tiltfile# -*- mode: Python -*-
22

33
envsubst_cmd = "./hack/tools/bin/envsubst"
44
clusterctl_cmd = "./bin/clusterctl"
@@ -482,6 +482,19 @@ def deploy_observability():
482482
objects = ["capi-visualizer:serviceaccount"],
483483
)
484484

485+
def deploy_kustomizations():
486+
for name in settings.get("deploy_kustomizations", []):
487+
yaml = read_file("./.tiltbuild/yaml/{}.kustomization.yaml".format(name))
488+
k8s_yaml(yaml)
489+
objs = decode_yaml_stream(yaml)
490+
print("objects")
491+
print(find_all_objects_names(objs))
492+
k8s_resource(
493+
new_name = name,
494+
objects = find_all_objects_names(objs),
495+
labels = ["kustomization"],
496+
)
497+
485498
def prepare_all():
486499
tools_arg = "--tools kustomize,envsubst,clusterctl "
487500
tilt_settings_file_arg = "--tilt-settings-file " + tilt_file
@@ -640,6 +653,8 @@ deploy_provider_crds()
640653

641654
deploy_observability()
642655

656+
deploy_kustomizations()
657+
643658
enable_providers()
644659

645660
cluster_templates()

‎hack/observability/kube-state-metrics/chart/cm-crd-sidecar.yaml

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: kube-state-metrics-crd-sidecar-script
5+
data:
6+
compile.sh: |
7+
#!/bin/sh
8+
9+
set -x
10+
11+
SOURCE_DIR="/tmp"
12+
TARGET_FILE="/etc/config/crd-config.yaml"
13+
# This script will use a temporary file to only overwrite the target file once.
14+
TARGET_FILE_TMP="${TARGET_FILE}.tmp"
15+
16+
# Create header
17+
cat << EOF > "${TARGET_FILE_TMP}"
18+
kind: CustomResourceStateMetrics
19+
spec:
20+
resources:
21+
EOF
22+
23+
# Append custom resource config of all files but remove headers
24+
for f in $(ls -1 ${SOURCE_DIR}/*.yaml); do
25+
cat $f | grep -v -E -e '^(-|kind: CustomResourceStateMetrics|spec:| +resources:)' \
26+
>> "${TARGET_FILE_TMP}"
27+
done
28+
29+
# Overwrite target file
30+
mv "${TARGET_FILE_TMP}" "${TARGET_FILE}"

‎hack/observability/kube-state-metrics/chart/kustomization.yaml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,12 @@ helmGlobals:
1111
# Otherwise "go mod tidy" picks up dependencies of go files contained in the Helm Chart.
1212
# "go mod tidy" ignores folders that begin with ".": https://pkg.go.dev/cmd/go#hdr-Package_lists_and_patterns.
1313
chartHome: .charts
14+
15+
resources:
16+
- rbac-crd-aggregation.yaml
17+
- cm-crd-sidecar.yaml
18+
19+
patches:
20+
- path: patch-crd-sidecar.yaml
21+
target:
22+
kind: Deployment

‎hack/observability/kube-state-metrics/chart/patch-crd-sidecar.yaml

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: kube-state-metrics
5+
spec:
6+
template:
7+
spec:
8+
containers:
9+
- env:
10+
- name: LABEL
11+
value: kube-state-metrics/custom-resource
12+
- name: FOLDER
13+
value: /tmp
14+
- name: RESOURCE
15+
value: configmap
16+
- name: NAMESPACE
17+
value: observability
18+
- name: SCRIPT
19+
value: /script/compile.sh
20+
image: kiwigrid/k8s-sidecar:latest
21+
name: crd-sidecar
22+
volumeMounts:
23+
- mountPath: /etc/config
24+
name: config-volume
25+
- mountPath: /script
26+
name: compile-script
27+
initContainers:
28+
- command:
29+
- /bin/sh
30+
- -c
31+
- |
32+
cat << EOF > "/etc/config/crd-config.yaml"
33+
kind: CustomResourceStateMetrics
34+
spec:
35+
resources: []
36+
EOF
37+
image: kiwigrid/k8s-sidecar:latest
38+
name: init-crd-config
39+
volumeMounts:
40+
- mountPath: /etc/config
41+
name: config-volume
42+
volumes:
43+
- configMap:
44+
defaultMode: 511
45+
name: kube-state-metrics-crd-sidecar-script
46+
name: compile-script

‎hack/observability/kube-state-metrics/chart/rbac-crd-aggregation.yaml

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
---
2+
# ClusterRole to aggregate other ClusterRoles for different Custom Resource Configurations
3+
apiVersion: rbac.authorization.k8s.io/v1
4+
kind: ClusterRole
5+
metadata:
6+
name: kube-state-metrics-aggregated-role
7+
aggregationRule:
8+
clusterRoleSelectors:
9+
- matchLabels:
10+
kube-state-metrics/aggregate-to-manager: "true"
11+
---
12+
# ClusterRoleBinding for the aggregation role
13+
apiVersion: rbac.authorization.k8s.io/v1
14+
kind: ClusterRoleBinding
15+
metadata:
16+
name: kube-state-metrics-custom-resource-rolebinding
17+
roleRef:
18+
apiGroup: rbac.authorization.k8s.io
19+
kind: ClusterRole
20+
name: kube-state-metrics-aggregated-role
21+
subjects:
22+
- kind: ServiceAccount
23+
name: kube-state-metrics
24+
namespace: observability

‎hack/observability/kube-state-metrics/chart/values.yaml

Lines changed: 2 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,8 @@ volumeMounts:
44
name: config-volume
55

66
volumes:
7-
- configMap:
8-
name: kube-state-metrics-crd-config
9-
name: config-volume
7+
- emptyDir: {}
8+
name: config-volume
109

1110
extraArgs:
1211
- "--custom-resource-state-config-file=/etc/config/crd-config.yaml"
@@ -21,33 +20,3 @@ rbac:
2120
- get
2221
- list
2322
- watch
24-
- apiGroups:
25-
- cluster.x-k8s.io
26-
resources:
27-
- clusterclasses
28-
- clusters
29-
- machinedeployments
30-
- machinepools
31-
- machinesets
32-
- machines
33-
- machinehealthchecks
34-
verbs:
35-
- get
36-
- list
37-
- watch
38-
- apiGroups:
39-
- controlplane.cluster.x-k8s.io
40-
resources:
41-
- kubeadmcontrolplanes
42-
verbs:
43-
- get
44-
- list
45-
- watch
46-
- apiGroups:
47-
- bootstrap.cluster.x-k8s.io
48-
resources:
49-
- kubeadmconfigs
50-
verbs:
51-
- get
52-
- list
53-
- watch

‎hack/observability/kube-state-metrics/crd-clusterrole.yaml

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
---
2+
apiVersion: rbac.authorization.k8s.io/v1
3+
kind: ClusterRole
4+
metadata:
5+
name: kube-state-metrics-custom-resource-capi
6+
labels:
7+
kube-state-metrics/aggregate-to-manager: "true"
8+
rules:
9+
- apiGroups:
10+
- cluster.x-k8s.io
11+
resources:
12+
- clusterclasses
13+
- clusters
14+
- machinedeployments
15+
- machinepools
16+
- machinesets
17+
- machines
18+
- machinehealthchecks
19+
verbs:
20+
- get
21+
- list
22+
- watch
23+
- apiGroups:
24+
- controlplane.cluster.x-k8s.io
25+
resources:
26+
- kubeadmcontrolplanes
27+
verbs:
28+
- get
29+
- list
30+
- watch
31+
- apiGroups:
32+
- bootstrap.cluster.x-k8s.io
33+
resources:
34+
- kubeadmconfigs
35+
verbs:
36+
- get
37+
- list
38+
- watch

‎hack/observability/kube-state-metrics/kustomization.yaml

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
resources:
22
- ../namespace.yaml
3-
# The kube-state-metrics helm chart will reference a configmap with name `kube-state-metrics-crd-config`.
4-
# The configMapGenerator below will create the configmap and append a hash suffix calculated from its
5-
# content to the name. Kustomize will append the suffix hash to all references in the helm chart, but
6-
# only when the helm chart content is referenced in "resources".
7-
# This would not work if the helm chart is configured in this file via the "helmCharts" option.
83
- ./chart
4+
- ./crd-clusterrole.yaml
95

106
namespace: observability
117

128
configMapGenerator:
13-
- name: kube-state-metrics-crd-config
9+
- name: kube-state-metrics-crd-config-capi
1410
files:
15-
- crd-config.yaml
11+
- capi.yaml=crd-config.yaml
12+
options:
13+
disableNameSuffixHash: true
14+
labels:
15+
kube-state-metrics/custom-resource: "true"

‎hack/tools/internal/tilt-prepare/main.go

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -104,13 +104,14 @@ var (
104104
// Types used to de-serialize the tilt-settings.yaml/json file from the Cluster API repository.
105105

106106
type tiltSettings struct {
107-
Debug map[string]tiltSettingsDebugConfig `json:"debug,omitempty"`
108-
ExtraArgs map[string]tiltSettingsExtraArgs `json:"extra_args,omitempty"`
109-
DeployCertManager *bool `json:"deploy_cert_manager,omitempty"`
110-
DeployObservability []string `json:"deploy_observability,omitempty"`
111-
EnableProviders []string `json:"enable_providers,omitempty"`
112-
AllowedContexts []string `json:"allowed_contexts,omitempty"`
113-
ProviderRepos []string `json:"provider_repos,omitempty"`
107+
Debug map[string]tiltSettingsDebugConfig `json:"debug,omitempty"`
108+
ExtraArgs map[string]tiltSettingsExtraArgs `json:"extra_args,omitempty"`
109+
DeployCertManager *bool `json:"deploy_cert_manager,omitempty"`
110+
DeployObservability []string `json:"deploy_observability,omitempty"`
111+
DeployKustomizations map[string]string `json:"deploy_kustomizations,omitempty"`
112+
EnableProviders []string `json:"enable_providers,omitempty"`
113+
AllowedContexts []string `json:"allowed_contexts,omitempty"`
114+
ProviderRepos []string `json:"provider_repos,omitempty"`
114115
}
115116

116117
type tiltSettingsDebugConfig struct {
@@ -308,6 +309,13 @@ func tiltResources(ctx context.Context, ts *tiltSettings) error {
308309
)
309310
}
310311

312+
for name, path := range ts.DeployKustomizations {
313+
name := fmt.Sprintf("%s.kustomization", name)
314+
tasks[name] = sequential(
315+
kustomizeTask(path, fmt.Sprintf("%s.yaml", name)),
316+
)
317+
}
318+
311319
// Add read configurations from provider repos
312320
for _, p := range ts.ProviderRepos {
313321
tiltProviderConfigs, err := loadTiltProvider(p)

0 commit comments

Comments
 (0)