Fix: Varia, Cleanup (#911)

* wip

* Add csi-s3 and have portainer use it

* Change request @Hrytsuk 1GB max portainer volume size

* Arch Linux Certificates Customization

* Add up-local makefile target for simcore stack

* Remove unused docker config file in monitoring stack

* Remove unused env-var - grafana

* Use up-local makefile target for local simcore in deploy_everything_locally.sh

* revert arch changes

* Refactor j2 calling convention, always pass .env file path explicitly

---------

Co-authored-by: Dustin Kaiser <[email protected]>

Author: mrnicegyu11

scripts/common.Makefile

@@ -253,13 +253,13 @@ venv: $(REPO_BASE_DIR)/.venv/bin/activate ## Creates a python virtual environmen
 ifeq ($(shell test -f j2cli_customization.py && echo -n yes),yes)
 
 define jinja
-	$(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) .env -o $(2) --customize j2cli_customization.py
+	$(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) $(2) -o $(3) --customize j2cli_customization.py
 endef
 
 else
 
 define jinja
-	$(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) .env -o $(2)
+	$(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) $(2) -o $(3)
 endef
 
 endif

scripts/deployments/deploy_everything_locally.bash

@@ -254,5 +254,8 @@ if [ "$start_opsstack" -eq 0 ]; then
 fi
 if [ "$start_simcore" -eq 0 ]; then
     log_info "starting simcore..."
-    "${repo_basedir}"/scripts/deployments/start_simcore_locally.bash
+    service_dir="${repo_basedir}"/services/simcore
+    pushd "${service_dir}"
+    call_make "." up-"$stack_target"
+    popd
 fi

services/maintenance-page/Makefile

@@ -33,7 +33,7 @@ up-master: up
 
 .PHONY: docker-compose.yml
 docker-compose.yml: .venv .env
-	@$(call jinja, docker-compose.yml.j2, docker-compose.yml.unlinted) && \
+	@$(call jinja, docker-compose.yml.j2, .env, docker-compose.yml.unlinted) && \
 	$(_yq) docker-compose.yml.unlinted > docker-compose.yml; \
 	rm docker-compose.yml.unlinted >/dev/null 2>&1;
 

services/monitoring/Makefile

@@ -74,7 +74,7 @@ ${TEMP_COMPOSE}-local: docker-compose.yml docker-compose.letsencrypt.dns.yml con
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.letsencrypt.dns.yml > $@
 
 docker-compose.yml: docker-compose.yml.j2 .env .venv pgsql_query_exporter_config.yaml
-	$(call jinja,$<,$@)
+	$(call jinja,$<,.env,$@)
 
 .PHONY: update.grafana.pwd
 update.grafana.pwd: .env ## Change grafana pwd
@@ -101,7 +101,7 @@ grafana-import: grafana/assets .venv ## Imports AND OVERWRITES the remote grafan
 
 .PHONY: config.grafana.dashboards
 config.grafana.dashboards: grafana/templates-provisioning/dashboards/simcore/Metrics-dashboard.json.j2 .venv #Configure dashboards for aws or dalco clusters
-	$(call jinja, $<, grafana/provisioning/dashboards/simcore/Metrics-dashboard.json)
+	$(call jinja, $<, .env, grafana/provisioning/dashboards/simcore/Metrics-dashboard.json)
 
 .PHONY: config.monitoring
 config.monitoring: grafana/template-config.monitoring ${REPO_CONFIG_LOCATION}
@@ -133,7 +133,7 @@ config.prometheus.ceph.simcore: ${REPO_CONFIG_LOCATION} .venv
 	@set -o allexport; \
 	source $<; \
 	set +o allexport; \
-	$(call jinja, prometheus/prometheus-ceph.yml.j2, prometheus/prometheus-ceph.yml); \
+	$(call jinja, prometheus/prometheus-ceph.yml.j2, .env, prometheus/prometheus-ceph.yml); \
 	cat prometheus/prometheus-base.yml | $(_yq) '. *+ load("prometheus/prometheus-simcore.yml")' | cat | \
 	$(_yq) '. *+ load("prometheus/prometheus-ceph.yml")' > prometheus/prometheus.yml; \
 	envsubst < prometheus/prometheus.yml > prometheus/prometheus.temp.yml; \
@@ -148,10 +148,10 @@ config.prometheus: ${REPO_CONFIG_LOCATION} .venv
 	mv prometheus/prometheus.temp.yml prometheus/prometheus.yml
 
 pgsql_query_exporter_config.yaml: pgsql_query_exporter_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env .venv
-	$(call jinja, $<, $@);
+	$(call jinja, $<, .env, $@);
 
 smokeping_prober_config.yaml: smokeping_prober_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env .venv
-	$(call jinja, $<, $@);
+	$(call jinja, $<, .env, $@);
 
 .PHONY: grafana/assets
 grafana/assets: ${REPO_CONFIG_LOCATION}

services/monitoring/docker-compose.yml.j2

@@ -25,8 +25,6 @@ configs:
     file: ./prometheus/prometheus-federation.yml
   prometheus_rules:
     file: ./prometheus/prometheus.rules.yml
-  grafana_image_renderer_config:
-    file: ./grafana-image-renderer/config.json
   pgsql_query_exporter_config:
     file: ./pgsql_query_exporter_config.yaml
   smokeping_prober_config:
@@ -249,7 +247,7 @@ services:
           cpus: "0.1"
 
   grafana:
-    image: grafana/grafana-oss:11.1.3
+    image: grafana/grafana-oss:11.2.4
     volumes:
       - grafana_data:/var/lib/grafana
     env_file:

services/monitoring/grafana-image-renderer/README.md

@@ -3,8 +3,6 @@ GF_SECURITY_ADMIN_USER=${SERVICES_USER}
 GF_USERS_ALLOW_SIGN_UP=false
 GF_SERVER_ROOT_URL=https://${MONITORING_DOMAIN}/grafana
 GF_INSTALL_PLUGINS=grafana-piechart-panel, grafana-worldmap-panel, grafana-polystat-panel
-# NOTE: grafana image rendering plugin: https://grafana.com/blog/2020/05/07/grafana-7.0-preview-new-image-renderer-plugin-to-replace-phantomjs/
-GF_RENDERING_SERVER_URL=http://grafana-image-renderer:8081/render
 GF_RENDERING_CALLBACK_URL=http://grafana:3000/
 GF_LOG_FILTERS=rendering:debug
 GF_SERVER_DOMAIN=${MONITORING_DOMAIN}

services/monitoring/grafana-image-renderer/config.json

@@ -52,6 +52,6 @@ ${TEMP_COMPOSE}-letsencrypt-dns: docker-compose.yml docker-compose.letsencrypt.d
 
 config.json: config.json.j2 .env .venv
 	# generate $@
-	@$(call jinja, $<, $@)
+	@$(call jinja, $<, .env, $@)
 	# validate and format $@
 	@cat $@ | jq > /dev/null

services/monitoring/grafana/template-config.monitoring

@@ -52,7 +52,7 @@ up-public: up-dalco
 # Helpers -------------------------------------------------
 
 docker-compose.yml:  docker-compose.yml.j2 .venv .env
-	@$(call jinja, $<, $@)
+	@$(call jinja, $<, .env, $@)
 
 .PHONY: ${TEMP_COMPOSE}-local
 ${TEMP_COMPOSE}-local: docker-compose.yml docker-compose.self-signed.yml .env