Skip to content

Commit a3adcb7

Browse files
jimmidysonjimmidyson
authored
refactor: Update helm registry initialization (#961)
This commit updates the helm registry initialization by separating out the image used for copying the charts to the PVC and using the released mindthegap image directly for serving the bundles. This is a small security enhancement by using the minimal image that mindthegap already provides. This commit also copies a statically compiled version of `cp` to a scratch container in order to us a minimal container with no package manager or shell, again this is a minor security enhancement. This change means that the bundles are copied to a subdirectory on the PVC as globbing cannot be used without a shell present, whereas recursive copying works correctly. This is a breaking change but should not affect any users at this point (e.g. not yet included in any downstream releases). Finally, this commit updates the helm values to use a more structured approach. While this is a breaking change, the Helm chart is only used to generate the clusterctl provider components YAML and as such does not have any impact on existing users.
1 parent f0a51bb commit a3adcb7

File tree

17 files changed

+140
-107
lines changed

17 files changed

+140
-107
lines changed

‎.envrc‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ export DEVBOX_NO_ENVRC_UPDATE=1
77

88
eval "$(devbox generate direnv --print-envrc --env-file .dev-envrc)"
99

10-
dotenv_if_exists '.envrc.local'
11-
dotenv_if_exists '.envrc.e2e'
10+
source_env_if_exists '.envrc.local'
11+
source_env_if_exists '.envrc.e2e'
1212

1313
# check out https://www.jetpack.io/devbox/docs/ide_configuration/direnv/
1414
# for more details

‎.github/workflows/checks.yml‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -234,14 +234,19 @@ jobs:
234234
name: Build Docker images
235235
run: devbox run -- make release-snapshot
236236

237+
- if: steps.list-changed.outputs.changed == 'true'
238+
name: Export image tag
239+
id: export-image-tag
240+
run: echo test-image-tag="$(devbox run -- gojq -r .version dist/metadata.json)-$(devbox run -- go env GOARCH)" >> "$GITHUB_OUTPUT"
241+
237242
- if: steps.list-changed.outputs.changed == 'true'
238243
name: Sideload docker image
239244
run: |
240245
devbox run -- \
241246
kind load docker-image \
242247
--name "${KIND_CLUSTER_NAME}" \
243-
"ko.local/cluster-api-runtime-extensions-nutanix:$(devbox run -- gojq -r .version dist/metadata.json)-$(devbox run -- go env GOARCH)" \
244-
"ghcr.io/nutanix-cloud-native/caren-helm-reg:$(devbox run -- gojq -r .version dist/metadata.json)-$(devbox run -- go env GOARCH)"
248+
"ko.local/cluster-api-runtime-extensions-nutanix:${{ steps.export-image-tag.outputs.test-image-tag }}" \
249+
"ghcr.io/nutanix-cloud-native/caren-helm-reg:${{ steps.export-image-tag.outputs.test-image-tag }}"
245250
246251
- if: steps.list-changed.outputs.changed == 'true'
247252
name: Setup Cluster API and cert-manager
@@ -253,7 +258,7 @@ jobs:
253258
devbox run -- \
254259
ct install \
255260
--config charts/ct-config.yaml \
256-
--helm-extra-set-args "--set-string image.repository=ko.local/cluster-api-runtime-extensions-nutanix --set-string image.tag=$(devbox run -- gojq -r .version dist/metadata.json)-$(devbox run -- go env GOARCH) --set-string helmRepositoryImage.tag=$(devbox run -- gojq -r .version dist/metadata.json)-$(devbox run -- go env GOARCH)"
261+
--helm-extra-set-args "--set-string image.repository=ko.local/cluster-api-runtime-extensions-nutanix --set-string image.tag=${{ steps.export-image-tag.outputs.test-image-tag }} --set-string helmRepository.images.bundleInitializer.tag=${{ steps.export-image-tag.outputs.test-image-tag }}"
257262
env:
258263
KUBECONFIG: ${{ env.KIND_KUBECONFIG }}
259264

‎.goreleaser.yml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ before:
4444
$(helm template {{ .ProjectName }} ./charts/{{ .ProjectName }} \
4545
--namespace caren-system \
4646
--set-string image.tag=v{{ trimprefix .Version "v" }}{{ if .IsSnapshot }}-{{ .Runtime.Goarch }}{{ end }} \
47-
--set-string helmRepositoryImage.tag=v{{ trimprefix .Version "v" }}{{ if .IsSnapshot }}-{{ .Runtime.Goarch }} \
47+
--set-string helmRepository.images.bundleInitializer.tag=v{{ trimprefix .Version "v" }}{{ if .IsSnapshot }}-{{ .Runtime.Goarch }} \
4848
--set-string image.repository=ko.local/{{ .ProjectName }}{{ end }} \
4949
)
5050
EOF'
@@ -103,9 +103,9 @@ dockers:
103103
- image_templates:
104104
- 'ghcr.io/nutanix-cloud-native/caren-helm-reg:v{{ trimprefix .Version "v" }}-amd64'
105105
use: buildx
106-
dockerfile: ./hack/addons/mindthegap-helm-registry/Dockerfile
106+
dockerfile: ./hack/addons/helm-chart-bundler/Dockerfile
107107
extra_files:
108-
- hack/addons/mindthegap-helm-registry/repos.yaml
108+
- hack/addons/helm-chart-bundler/repos.yaml
109109
build_flag_templates:
110110
- "--platform=linux/amd64"
111111
- "--pull"
@@ -119,9 +119,9 @@ dockers:
119119
- image_templates:
120120
- 'ghcr.io/nutanix-cloud-native/caren-helm-reg:v{{ trimprefix .Version "v" }}-arm64'
121121
use: buildx
122-
dockerfile: ./hack/addons/mindthegap-helm-registry/Dockerfile
122+
dockerfile: ./hack/addons/helm-chart-bundler/Dockerfile
123123
extra_files:
124-
- hack/addons/mindthegap-helm-registry/repos.yaml
124+
- hack/addons/helm-chart-bundler/repos.yaml
125125
build_flag_templates:
126126
- "--platform=linux/arm64"
127127
- "--pull"

‎.pre-commit-config.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ repos:
5555
name: addons-repo-yaml
5656
entry: make template-helm-repository
5757
language: system
58-
files: "hack/addons/mindthegap-helm-registry/repos.yaml"
58+
files: "hack/addons/helm-chart-bundler/repos.yaml"
5959
pass_filenames: false
6060
- id: check-devbox-lock
6161
name: check-devbox-lock

‎charts/cluster-api-runtime-extensions-nutanix/README.md‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -32,12 +32,16 @@ A Helm chart for cluster-api-runtime-extensions-nutanix
3232
| deployment.replicas | int | `1` | |
3333
| env | object | `{}` | |
3434
| helmAddonsConfigMap | string | `"default-helm-addons-config"` | |
35-
| helmRepositoryImage.pullPolicy | string | `"IfNotPresent"` | |
36-
| helmRepositoryImage.repository | string | `"ghcr.io/nutanix-cloud-native/caren-helm-reg"` | |
37-
| helmRepositoryImage.tag | string | `""` | |
38-
| helmRepositorySecurityContext.fsGroup | int | `65534` | |
39-
| helmRepositorySecurityContext.runAsGroup | int | `65534` | |
40-
| helmRepositorySecurityContext.runAsUser | int | `65534` | |
35+
| helmRepository.enabled | bool | `true` | |
36+
| helmRepository.images.bundleInitializer.pullPolicy | string | `"IfNotPresent"` | |
37+
| helmRepository.images.bundleInitializer.repository | string | `"ghcr.io/nutanix-cloud-native/caren-helm-reg"` | |
38+
| helmRepository.images.bundleInitializer.tag | string | `""` | |
39+
| helmRepository.images.mindthegap.pullPolicy | string | `"IfNotPresent"` | |
40+
| helmRepository.images.mindthegap.repository | string | `"ghcr.io/mesosphere/mindthegap"` | |
41+
| helmRepository.images.mindthegap.tag | string | `"v1.16.0"` | |
42+
| helmRepository.securityContext.fsGroup | int | `65532` | |
43+
| helmRepository.securityContext.runAsGroup | int | `65532` | |
44+
| helmRepository.securityContext.runAsUser | int | `65532` | |
4145
| hooks.ccm.aws.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | |
4246
| hooks.ccm.aws.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-aws-ccm-helm-values-template"` | |
4347
| hooks.ccm.aws.k8sMinorVersionToCCMVersion."1.27" | string | `"v1.27.9"` | |
@@ -99,7 +103,6 @@ A Helm chart for cluster-api-runtime-extensions-nutanix
99103
| resources.requests.cpu | string | `"100m"` | |
100104
| resources.requests.memory | string | `"128Mi"` | |
101105
| securityContext.runAsUser | int | `65532` | |
102-
| selfHostedRegistry | bool | `true` | |
103106
| service.annotations | object | `{}` | |
104107
| service.port | int | `443` | |
105108
| service.type | string | `"ClusterIP"` | |

‎charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -10,48 +10,48 @@ data:
1010
aws-ccm: |
1111
ChartName: aws-cloud-controller-manager
1212
ChartVersion: 0.0.8
13-
RepositoryURL: {{ if .Values.selfHostedRegistry }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://kubernetes.github.io/cloud-provider-aws{{ end }}
13+
RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://kubernetes.github.io/cloud-provider-aws{{ end }}'
1414
aws-ebs-csi: |
1515
ChartName: aws-ebs-csi-driver
1616
ChartVersion: 2.35.1
17-
RepositoryURL: {{ if .Values.selfHostedRegistry }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://kubernetes-sigs.github.io/aws-ebs-csi-driver{{ end }}
17+
RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://kubernetes-sigs.github.io/aws-ebs-csi-driver{{ end }}'
1818
cilium: |
1919
ChartName: cilium
2020
ChartVersion: 1.16.2
21-
RepositoryURL: {{ if .Values.selfHostedRegistry }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://helm.cilium.io/{{ end }}
21+
RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://helm.cilium.io/{{ end }}'
2222
cluster-autoscaler: |
2323
ChartName: cluster-autoscaler
2424
ChartVersion: 9.40.0
25-
RepositoryURL: {{ if .Values.selfHostedRegistry }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://kubernetes.github.io/autoscaler{{ end }}
25+
RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://kubernetes.github.io/autoscaler{{ end }}'
2626
local-path-provisioner-csi: |
2727
ChartName: local-path-provisioner
2828
ChartVersion: 0.0.29
29-
RepositoryURL: {{ if .Values.selfHostedRegistry }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://charts.containeroo.ch{{ end }}
29+
RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://charts.containeroo.ch{{ end }}'
3030
metallb: |
3131
ChartName: metallb
3232
ChartVersion: 0.14.8
33-
RepositoryURL: {{ if .Values.selfHostedRegistry }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://metallb.github.io/metallb{{ end }}
33+
RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://metallb.github.io/metallb{{ end }}'
3434
nfd: |
3535
ChartName: node-feature-discovery
3636
ChartVersion: 0.16.4
37-
RepositoryURL: {{ if .Values.selfHostedRegistry }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://kubernetes-sigs.github.io/node-feature-discovery/charts{{ end }}
37+
RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://kubernetes-sigs.github.io/node-feature-discovery/charts{{ end }}'
3838
nutanix-ccm: |
3939
ChartName: nutanix-cloud-provider
4040
ChartVersion: 0.4.1
41-
RepositoryURL: {{ if .Values.selfHostedRegistry }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://nutanix.github.io/helm/{{ end }}
41+
RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://nutanix.github.io/helm/{{ end }}'
4242
nutanix-storage-csi: |
4343
ChartName: nutanix-csi-storage
4444
ChartVersion: 3.1.0
45-
RepositoryURL: {{ if .Values.selfHostedRegistry }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://nutanix.github.io/helm-releases/{{ end }}
45+
RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://nutanix.github.io/helm-releases/{{ end }}'
4646
snapshot-controller: |
4747
ChartName: snapshot-controller
4848
ChartVersion: 3.0.6
49-
RepositoryURL: {{ if .Values.selfHostedRegistry }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://piraeus.io/helm-charts/{{ end }}
49+
RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://piraeus.io/helm-charts/{{ end }}'
5050
tigera-operator: |
5151
ChartName: tigera-operator
5252
ChartVersion: v3.28.2
53-
RepositoryURL: {{ if .Values.selfHostedRegistry }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://docs.tigera.io/calico/charts{{ end }}
53+
RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://docs.tigera.io/calico/charts{{ end }}'
5454
kind: ConfigMap
5555
metadata:
5656
creationTimestamp: null
57-
name: {{ .Values.helmAddonsConfigMap }}
57+
name: '{{ .Values.helmAddonsConfigMap }}'

‎charts/cluster-api-runtime-extensions-nutanix/templates/helm-repository.yaml‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
# This file contains the manifests to run a helmRepository deployment which contains helm charts for our addons.
44
# The pod is built via goreleaser with configuration from hack/addons.
55
#
6-
{{ if .Values.selfHostedRegistry }}
6+
{{ if .Values.helmRepository.enabled }}
77
apiVersion: cert-manager.io/v1
88
kind: Issuer
99
metadata:
@@ -62,8 +62,8 @@ spec:
6262
spec:
6363
initContainers:
6464
- name: copy-charts
65-
image: "{{ .Values.helmRepositoryImage.repository }}:{{ default $.Chart.AppVersion .Values.helmRepositoryImage.tag }}"
66-
command: ["/bin/sh", "-c", "cp /charts/*.tar /helm-charts"]
65+
image: "{{ .Values.helmRepository.images.bundleInitializer.repository }}:{{ default $.Chart.AppVersion .Values.helmRepository.images.bundleInitializer.tag }}"
66+
command: ["/bin/cp", "-r", "/charts/", "/helm-charts/bundles/"]
6767
imagePullPolicy: "{{ .Values.image.pullPolicy }}"
6868
volumeMounts:
6969
- name: charts-volume
@@ -74,13 +74,12 @@ spec:
7474
- name: serve
7575
protocol: TCP
7676
containerPort: 5000
77-
image: "{{ .Values.helmRepositoryImage.repository }}:{{ default $.Chart.AppVersion .Values.helmRepositoryImage.tag }}"
77+
image: "{{ .Values.helmRepository.images.mindthegap.repository }}:{{ .Values.helmRepository.images.mindthegap.tag }}"
7878
imagePullPolicy: "{{ .Values.image.pullPolicy }}"
79-
command: ["/usr/bin/mindthegap"]
8079
args:
8180
- serve
8281
- bundle
83-
- --bundle=/helm-charts/helm-charts-*.tar
82+
- --bundle=/helm-charts/bundles/helm-charts-*.tar
8483
- --listen-port=5000
8584
- --listen-address=0.0.0.0
8685
- --tls-private-key-file=/certs/tls.key
@@ -101,7 +100,7 @@ spec:
101100
periodSeconds: 1
102101
priorityClassName: {{ .Values.priorityClassName }}
103102
securityContext:
104-
{{ with .Values.helmRepositorySecurityContext }}
103+
{{ with .Values.helmRepository.securityContext }}
105104
{{- toYaml . | nindent 8}}
106105
{{- end }}
107106
volumes:

‎charts/cluster-api-runtime-extensions-nutanix/values.schema.json‎

Lines changed: 48 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -38,30 +38,57 @@
3838
"helmAddonsConfigMap": {
3939
"type": "string"
4040
},
41-
"helmRepositoryImage": {
41+
"helmRepository": {
4242
"properties": {
43-
"pullPolicy": {
44-
"type": "string"
45-
},
46-
"repository": {
47-
"type": "string"
48-
},
49-
"tag": {
50-
"type": "string"
51-
}
52-
},
53-
"type": "object"
54-
},
55-
"helmRepositorySecurityContext": {
56-
"properties": {
57-
"fsGroup": {
58-
"type": "integer"
43+
"enabled": {
44+
"type": "boolean"
5945
},
60-
"runAsGroup": {
61-
"type": "integer"
46+
"images": {
47+
"properties": {
48+
"bundleInitializer": {
49+
"properties": {
50+
"pullPolicy": {
51+
"type": "string"
52+
},
53+
"repository": {
54+
"type": "string"
55+
},
56+
"tag": {
57+
"type": "string"
58+
}
59+
},
60+
"type": "object"
61+
},
62+
"mindthegap": {
63+
"properties": {
64+
"pullPolicy": {
65+
"type": "string"
66+
},
67+
"repository": {
68+
"type": "string"
69+
},
70+
"tag": {
71+
"type": "string"
72+
}
73+
},
74+
"type": "object"
75+
}
76+
},
77+
"type": "object"
6278
},
63-
"runAsUser": {
64-
"type": "integer"
79+
"securityContext": {
80+
"properties": {
81+
"fsGroup": {
82+
"type": "integer"
83+
},
84+
"runAsGroup": {
85+
"type": "integer"
86+
},
87+
"runAsUser": {
88+
"type": "integer"
89+
}
90+
},
91+
"type": "object"
6592
}
6693
},
6794
"type": "object"
@@ -575,9 +602,6 @@
575602
},
576603
"type": "object"
577604
},
578-
"selfHostedRegistry": {
579-
"type": "boolean"
580-
},
581605
"service": {
582606
"properties": {
583607
"annotations": {

‎charts/cluster-api-runtime-extensions-nutanix/values.yaml‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -110,8 +110,6 @@ hooks:
110110

111111
helmAddonsConfigMap: default-helm-addons-config
112112

113-
selfHostedRegistry: true
114-
115113
deployDefaultClusterClasses: true
116114

117115
# The ClusterClass and the Templates it references must be in the same namespace
@@ -132,10 +130,21 @@ image:
132130
tag: ""
133131
pullPolicy: IfNotPresent
134132

135-
helmRepositoryImage:
136-
repository: ghcr.io/nutanix-cloud-native/caren-helm-reg
137-
tag: ""
138-
pullPolicy: IfNotPresent
133+
helmRepository:
134+
enabled: true
135+
images:
136+
bundleInitializer:
137+
repository: ghcr.io/nutanix-cloud-native/caren-helm-reg
138+
tag: ""
139+
pullPolicy: IfNotPresent
140+
mindthegap:
141+
repository: ghcr.io/mesosphere/mindthegap
142+
tag: "v1.16.0"
143+
pullPolicy: IfNotPresent
144+
securityContext:
145+
runAsUser: 65532
146+
runAsGroup: 65532
147+
fsGroup: 65532
139148

140149
# -- Optional secrets used for pulling the container image
141150
imagePullSecrets: []
@@ -160,12 +169,6 @@ resources:
160169
securityContext:
161170
runAsUser: 65532
162171

163-
# The helm-repository containers are based on an Alpine image with a different nonroot user
164-
helmRepositorySecurityContext:
165-
runAsUser: 65534
166-
runAsGroup: 65534
167-
fsGroup: 65534
168-
169172
service:
170173
annotations: {}
171174
type: ClusterIP

‎hack/addons/add-warning-helm-configmap.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ readonly SCRIPT_DIR
88
# shellcheck source=hack/common.sh
99
source "${SCRIPT_DIR}/../common.sh"
1010
ASSETS_DIR="$(mktemp -d -p "${TMPDIR:-/tmp}")"
11+
trap 'rm -rf "${ASSETS_DIR}"' EXIT
12+
1113
mv "${GIT_REPO_ROOT}/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml" "${ASSETS_DIR}/helm-config.yaml"
1214
# add warning not to edit file directly
1315
cat <<EOF >"${GIT_REPO_ROOT}/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml"
@@ -19,5 +21,3 @@ $(cat "${GIT_REPO_ROOT}/hack/license-header.yaml.txt")
1921
#=================================================================
2022
$(cat "${ASSETS_DIR}/helm-config.yaml")
2123
EOF
22-
23-
sed -i s/placeholder/"{{ .Values.helmAddonsConfigMap }}"/g "${GIT_REPO_ROOT}/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml"

0 commit comments

Comments
 (0)