From f0d8cf044e3a94e63fbe75fb3d242d2e0e259dd6 Mon Sep 17 00:00:00 2001 From: Dustin Kaiser Date: Thu, 19 Sep 2024 16:49:57 +0200 Subject: [PATCH 1/6] wip --- charts/Makefile | 33 +++++++++++++++++++++------------ charts/README.md | 8 ++++++++ 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/charts/Makefile b/charts/Makefile index 8bf61fee..f33c5391 100644 --- a/charts/Makefile +++ b/charts/Makefile @@ -7,33 +7,32 @@ CONFIG_DIR := $(shell dirname $(REPO_CONFIG_LOCATION)) CHART_DIRS := $(wildcard $(REPO_BASE_DIR)/charts/*/) .PHONY: .check-helmfile-installed -.check-helmfile-installed: +.check-helmfile-installed: ## Checks if helmfile is installed @if ! command -v helmfile >/dev/null 2>&1; then \ echo "'helmfile' is not installed. Install it to continue ...";\ fi -helmfile.yaml: simcore-charts/helmfile.yaml +helmfile.yaml: simcore-charts/helmfile.yaml ## Copies the helmfile.yaml to the charts directory cp $(CONFIG_DIR)/$@ $(REPO_BASE_DIR)/charts/helmfile.yaml -simcore-charts/helmfile.yaml: +simcore-charts/helmfile.yaml: ## Copies the simcore helmfile to the charts directory cp $(CONFIG_DIR)/helmfile.simcore.yaml $(REPO_BASE_DIR)/charts/$@ .PHONY: helmfile-lint -helmfile-lint: .check-helmfile-installed helmfile.yaml +helmfile-lint: .check-helmfile-installed helmfile.yaml ## Lints the helmfile set -a; source $(REPO_CONFIG_LOCATION); set +a; \ helmfile lint .PHONY: .helmfile-local-post-install -.helmfile-local-post-install: +.helmfile-local-post-install: ## Post install steps for local helmfile deployment @$(MAKE) -s configure-local-hosts @echo ""; @echo "Cluster has been deployed locally: http://$(MACHINE_FQDN)"; @echo " For secure connections self-signed certificates are used."; - @echo " Install their root-ca certificate in your system for smooth experience."; - @echo " For insecure connections make sure to disable automatic https redirects in your browser."; + @echo " .PHONY: helmfile-apply -helmfile-apply: .check-helmfile-installed helmfile.yaml +helmfile-apply: .check-helmfile-installed helmfile.yaml ## Applies the helmfile configuration set -a; source $(REPO_CONFIG_LOCATION); set +a; \ helmfile -f $(REPO_BASE_DIR)/charts/helmfile.yaml apply @@ -41,17 +40,27 @@ helmfile-apply: .check-helmfile-installed helmfile.yaml $(MAKE) -s .helmfile-local-post-install; \ fi +.PHONY: helmfile-sync +helmfile-sync: .check-helmfile-installed helmfile.yaml ## Syncs the helmfile configuration + set -a; source $(REPO_CONFIG_LOCATION); set +a; \ + helmfile -f $(REPO_BASE_DIR)/charts/helmfile.yaml sync + + @if [ "$(MACHINE_FQDN)" = "osparc.local" ]; then \ + $(MAKE) -s .helmfile-local-post-install; \ + fi + + .PHONY: configure-local-hosts -configure-local-hosts: - @echo "Addings $(MACHINE_FQDN) hosts to /etc/hosts ..." +configure-local-hosts: ## Adds local hosts entries for the machine + @echo "Adding $(MACHINE_FQDN) hosts to /etc/hosts ..." @grep -q '127.0.0.1 k8s.monitoring.$(MACHINE_FQDN)' /etc/hosts || echo '127.0.0.1 k8s.monitoring.$(MACHINE_FQDN)' | sudo tee -a /etc/hosts .PHONY: helmfile-diff -helmfile-diff: .check-helmfile-installed helmfile.yaml +helmfile-diff: .check-helmfile-installed helmfile.yaml ## Shows the differences that would be applied by helmfile @set -a; source $(REPO_CONFIG_LOCATION); set +a; \ helmfile -f $(REPO_BASE_DIR)/charts/helmfile.yaml diff .PHONY: helmfile-delete -helmfile-delete: .check-helmfile-installed helmfile.yaml +helmfile-delete: .check-helmfile-installed helmfile.yaml ## Deletes the helmfile configuration @set -a; source $(REPO_CONFIG_LOCATION); set +a; \ helmfile -f $(REPO_BASE_DIR)/charts/helmfile.yaml delete diff --git a/charts/README.md b/charts/README.md index 13767e80..45d83630 100644 --- a/charts/README.md +++ b/charts/README.md @@ -23,6 +23,14 @@ source: https://kind.sigs.k8s.io/docs/user/quick-start Follow the instructions here: https://helm.sh/docs/intro/install/ +Install the helm-diff plugin: `helm plugin install https://github.com/databus23/helm-diff` + +`via https://doc.traefik.io/traefik/user-guides/crd-acme/#ingressroute-definition` +Install traefik-v3 CRDs: `kubectl apply -f https://raw.githubusercontent.com/traefik/traefik/v3.1/docs/content/reference/dynamic-configuration/kubernetes-crd-definition-v1.yml` + +`via https://doc.traefik.io/traefik/user-guides/crd-acme/#ingressroute-definition` +Install traefik-v3 RBAC: `kubectl apply -f https://raw.githubusercontent.com/traefik/traefik/v3.1/docs/content/reference/dynamic-configuration/kubernetes-crd-rbac.yml` + #### helmfile If you have a different OS / architecture, pick a different link from [release artifacts](https://github.com/helmfile/helmfile/releases) From 293f63c8c7971afeb0de64af16d01153bf76eed4 Mon Sep 17 00:00:00 2001 From: Dustin Kaiser Date: Thu, 24 Oct 2024 10:13:28 +0200 Subject: [PATCH 2/6] Add csi-s3 and have portainer use it --- .gitignore | 3 +++ charts/csi-s3/values.yaml.gotmpl | 7 +++++++ charts/portainer/values.yaml.gotmpl | 6 ++++++ 3 files changed, 16 insertions(+) create mode 100644 charts/csi-s3/values.yaml.gotmpl diff --git a/.gitignore b/.gitignore index 0c825bcd..24edb7f6 100644 --- a/.gitignore +++ b/.gitignore @@ -149,3 +149,6 @@ docker-compose.simcore.yml repo.config .temp .temp/** + +# By convention: `.secret` files are gitignored +**/*.secret diff --git a/charts/csi-s3/values.yaml.gotmpl b/charts/csi-s3/values.yaml.gotmpl new file mode 100644 index 00000000..7e6ff4c9 --- /dev/null +++ b/charts/csi-s3/values.yaml.gotmpl @@ -0,0 +1,7 @@ +secret: + accessKey: {{ requiredEnv "S3_ACCESS_KEY" }} + secretKey: {{ requiredEnv "S3_SECRET_KEY" }} + region: {{ requiredEnv "S3_REGION" }} + endpoint: {{ requiredEnv "S3_ENDPOINT" }} +storageClass: + singleBucket: {{ requiredEnv "S3_K8S_CSI_BUCKET_NAME" }} diff --git a/charts/portainer/values.yaml.gotmpl b/charts/portainer/values.yaml.gotmpl index e89f2457..edc56479 100644 --- a/charts/portainer/values.yaml.gotmpl +++ b/charts/portainer/values.yaml.gotmpl @@ -18,6 +18,12 @@ serviceAccount: # The name of the service account to use. # If not set and create is true, a name is generated using the fullname template name: portainer-sa-clusteradmin +persistence: + enabled: true + size: "10Gi" + annotations: {} + storageClass: "csi-s3" + existingClaim: podAnnotations: {} podLabels: {} From f7f72ec27e13232dababef209c92e2a3a1b983d4 Mon Sep 17 00:00:00 2001 From: Dustin Kaiser Date: Fri, 25 Oct 2024 08:37:18 +0200 Subject: [PATCH 3/6] Change request @hrytsuk 1GB max portainer volume size --- charts/portainer/values.yaml.gotmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/portainer/values.yaml.gotmpl b/charts/portainer/values.yaml.gotmpl index edc56479..1f5f5c44 100644 --- a/charts/portainer/values.yaml.gotmpl +++ b/charts/portainer/values.yaml.gotmpl @@ -20,7 +20,7 @@ serviceAccount: name: portainer-sa-clusteradmin persistence: enabled: true - size: "10Gi" + size: "1Gi" annotations: {} storageClass: "csi-s3" existingClaim: From c9c70d642e632e920a25cc76395cc56f1be8f134 Mon Sep 17 00:00:00 2001 From: Dustin Kaiser Date: Tue, 3 Dec 2024 16:08:00 +0100 Subject: [PATCH 4/6] Arch Linux Certificates Customization --- certificates/Makefile | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/certificates/Makefile b/certificates/Makefile index a9326900..9af700df 100644 --- a/certificates/Makefile +++ b/certificates/Makefile @@ -66,16 +66,10 @@ install-root-certificate: rootca.crt ## installs a certificate in the host syste echo "Is the DOCKER service ready? press when ready" && read -n 1; \ fi;\ echo "======================================";,\ - $(if $(IS_OSX), \ - sudo security add-trusted-cert -d -k /Library/Keychains/System.keychain $<; \ - echo "Please restart the DOCKER service now..." && read -n 1; \ - echo "Is the DOCKER service ready? press when ready" && read -n 1; \ - , \ - sudo cp $< /usr/local/share/ca-certificates/osparc.crt; \ - sudo update-ca-certificates -f; \ - echo "# restarting docker daemon"; \ + sudo cp $< /etc/ca-certificates/trust-source/anchors/osparc.crt; \ + sudo trust extract-compat && \ + echo "# restarting docker daemon" && \ sudo systemctl restart docker \ - ) \ ) From 94b996ac6e7030a2756ea7a0d047696f04964c29 Mon Sep 17 00:00:00 2001 From: Dustin Kaiser Date: Fri, 28 Mar 2025 09:52:19 +0100 Subject: [PATCH 5/6] Fixes https://github.com/ITISFoundation/osparc-simcore/issues/7363 --- .../contentpacks/osparc-custom-content-pack-v2.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/services/graylog/data/contentpacks/osparc-custom-content-pack-v2.json b/services/graylog/data/contentpacks/osparc-custom-content-pack-v2.json index f50cec48..2bd87c57 100644 --- a/services/graylog/data/contentpacks/osparc-custom-content-pack-v2.json +++ b/services/graylog/data/contentpacks/osparc-custom-content-pack-v2.json @@ -1,9 +1,9 @@ { "v": "1", - "id": "dfaeea11-bde6-4203-9cfe-6ca2a23ca22e", - "rev": 42, - "name": "osparc-custom-content-pack-v2", - "summary": "osparc-custom-content-pack-v2", + "id": "daaeea11-bde6-4203-9cfe-6ca2a23ca22e", + "rev": 1, + "name": "osparc-custom-content-pack-v3", + "summary": "osparc-custom-content-pack-v3", "description": "", "vendor": "Osparc team", "url": "", @@ -623,7 +623,7 @@ "configuration": { "grok_pattern": { "@type": "string", - "@value": "log_level=%{WORD:log_level} \\| log_timestamp=%{TIMESTAMP_ISO8601:log_timestamp} \\| log_source=%{DATA:log_source} \\| (log_uid=%{WORD:log_uid} \\| )?log_msg=%{GREEDYDATA:log_msg}" + "@value": "log_level=%{WORD:log_level} \\| log_timestamp=%{TIMESTAMP_ISO8601:log_timestamp} \\| log_source=%{DATA:log_source} \\| (log_uid=%{WORD:log_uid} \\| (log_oec=%{WORD:log_oec} \\| )?log_msg=%{GREEDYDATA:log_msg}" }, "named_captures_only": { "@type": "boolean", From 018b70838adac40aca0322cbb244a8326bfd6f4c Mon Sep 17 00:00:00 2001 From: Dustin Kaiser Date: Mon, 31 Mar 2025 09:24:52 +0200 Subject: [PATCH 6/6] fixes --- certificates/Makefile | 12 +++++++++--- .../contentpacks/osparc-custom-content-pack-v2.json | 8 ++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/certificates/Makefile b/certificates/Makefile index 9af700df..a9326900 100644 --- a/certificates/Makefile +++ b/certificates/Makefile @@ -66,10 +66,16 @@ install-root-certificate: rootca.crt ## installs a certificate in the host syste echo "Is the DOCKER service ready? press when ready" && read -n 1; \ fi;\ echo "======================================";,\ - sudo cp $< /etc/ca-certificates/trust-source/anchors/osparc.crt; \ - sudo trust extract-compat && \ - echo "# restarting docker daemon" && \ + $(if $(IS_OSX), \ + sudo security add-trusted-cert -d -k /Library/Keychains/System.keychain $<; \ + echo "Please restart the DOCKER service now..." && read -n 1; \ + echo "Is the DOCKER service ready? press when ready" && read -n 1; \ + , \ + sudo cp $< /usr/local/share/ca-certificates/osparc.crt; \ + sudo update-ca-certificates -f; \ + echo "# restarting docker daemon"; \ sudo systemctl restart docker \ + ) \ ) diff --git a/services/graylog/data/contentpacks/osparc-custom-content-pack-v2.json b/services/graylog/data/contentpacks/osparc-custom-content-pack-v2.json index 2bd87c57..3b5627f2 100644 --- a/services/graylog/data/contentpacks/osparc-custom-content-pack-v2.json +++ b/services/graylog/data/contentpacks/osparc-custom-content-pack-v2.json @@ -1,9 +1,9 @@ { "v": "1", - "id": "daaeea11-bde6-4203-9cfe-6ca2a23ca22e", - "rev": 1, - "name": "osparc-custom-content-pack-v3", - "summary": "osparc-custom-content-pack-v3", + "id": "dfaeea11-bde6-4203-9cfe-6ca2a23ca22e", + "rev": 42, + "name": "osparc-custom-content-pack-v2", + "summary": "osparc-custom-content-pack-v2", "description": "", "vendor": "Osparc team", "url": "",