Merge remote-tracking branch 'upstream/main'

Author: mrnicegyu11

.github/PULL_REQUEST_TEMPLATE.md

@@ -18,6 +18,8 @@
 - [ ] Service has placement constraints or is global
 - [ ] Service is restartable
 - [ ] Service restart is zero-downtime
+- [ ] Service is monitored (via prometheus and grafana)
 - [ ] Service is not bound to one specific node (e.g. via files or volumes)
 - [ ] Relevant OPS E2E Test are added
-- [ ] Service's Public URL is included in maintenance mode -->
+- [ ] Service's Public URL is included in maintenance mode
+- [ ] Service's Public URL is included in testing mode -->

.pre-commit-config.yaml

@@ -79,6 +79,10 @@ repos:
     hooks:
       - id: shellcheck
         name: Shell scripts conform to shellcheck
+  - repo: https://github.com/antonbabenko/pre-commit-terraform
+    rev: v1.89.1 # Get the latest from: https://github.com/antonbabenko/pre-commit-terraform/releases
+    hooks:
+    -   id: terraform_fmt
   - repo: local
     hooks:
       - id: run-pylint

scripts/common.Makefile

@@ -16,7 +16,7 @@ IS_WIN  := $(strip $(if $(or $(IS_LINUX),$(IS_OSX),$(IS_WSL)),,$(OS)))
 $(if $(IS_WSL2),,$(if $(IS_WSL),$(error WSL1 is not supported in all recipes. Use WSL2 instead. Follow instructions in README.md),))
 
 # Check that a valid location to a config file is set.
-REPO_BASE_DIR := $(shell git rev-parse --show-toplevel)
+REPO_BASE_DIR := $(abspath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))..)
 export REPO_CONFIG_LOCATION := $(shell cat $(REPO_BASE_DIR)/.config.location)
 $(if $(REPO_CONFIG_LOCATION),,$(error The location of the repo.config file given in .config.location is invalid. Aborting))
 $(if $(shell cat $(REPO_CONFIG_LOCATION)),,$(error The location of the repo.config file given in .config.location is invalid. Aborting))

services/graylog/scripts/configure.py

@@ -49,7 +49,10 @@
 )
 def wait_graylog_is_online():
     _r = requests.get(
-        GRAYLOG_BASE_DOMAIN + "/api/system", auth=REQUESTS_AUTH, verify=False
+        GRAYLOG_BASE_DOMAIN + "/api/system",
+        auth=REQUESTS_AUTH,
+        verify=False,
+        timeout=10,
     )
 
     if _r.status_code == 401:

services/monitoring/Makefile

@@ -4,7 +4,7 @@
 # STACK_NAME defaults to name of the current directory. Should not to be changed if you follow GitOps operating procedures.
 STACK_NAME = $(notdir $(CURDIR))
 TEMP_COMPOSE=.stack.${STACK_NAME}.yaml
-REPO_BASE_DIR := $(shell git rev-parse --show-toplevel)
+REPO_BASE_DIR := $(abspath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))../..)
 
 # TARGETS --------------------------------------------------
 include ${REPO_BASE_DIR}/scripts/common.Makefile
@@ -76,28 +76,12 @@ ${TEMP_COMPOSE}-local: docker-compose.yml docker-compose.letsencrypt.dns.yml con
 docker-compose.yml: docker-compose.yml.j2 .env .venv pgsql_query_exporter_config.yaml
 	$(call jinja,$<,.env,$@)
 
-.PHONY: update.grafana.pwd
-update.grafana.pwd: .env ## Change grafana pwd
-	@set -o allexport; \
-	source $(REPO_CONFIG_LOCATION); \
-	set +o allexport; \
-	grafanacontainerid=$$(docker ps | grep grafana | awk '{print $$1;}');\
-	docker exec -ti $$grafanacontainerid grafana-cli admin reset-admin-password $$TRAEFIK_PASSWORD
-
-
-.PHONY: grafana-export
-grafana-export: .venv## Export the remote grafana dashboards and datasources TO YOUR LOCAL MACHINE
-	@cd grafana/scripts;\
-	source ${REPO_BASE_DIR}/.venv/bin/activate;\
-	pip install -r requirements.txt > /dev/null 2>&1;\
-	python3 export.py;
-
 .PHONY: grafana-import
-grafana-import: grafana/assets .venv ## Imports AND OVERWRITES the remote grafana dashboards and datasources FROM YOUR LOCAL MACHINE
-	@cd grafana/scripts;\
-	source ${REPO_BASE_DIR}/.venv/bin/activate;\
-	pip install -r requirements.txt > /dev/null 2>&1;\
-	python3 import.py
+grafana-import: grafana/assets ## Imports the remote grafana dashboards and datasources FROM YOUR LOCAL MACHINE
+	@pushd ${REPO_BASE_DIR}/services/monitoring/grafana && \
+	$(MAKE) terraform-plan && \
+	$(MAKE) terraform-apply; \
+	popd > /dev/null
 
 .PHONY: config.grafana.dashboards
 config.grafana.dashboards: grafana/templates-provisioning/dashboards/simcore/Metrics-dashboard.json.j2 .venv #Configure dashboards for aws or dalco clusters
@@ -155,15 +139,4 @@ smokeping_prober_config.yaml: smokeping_prober_config.yaml.j2 ${REPO_CONFIG_LOCA
 
 .PHONY: grafana/assets
 grafana/assets: ${REPO_CONFIG_LOCATION}
-	@if [ ! -d "$(shell dirname ${REPO_CONFIG_LOCATION})/assets/grafana" ]; then \
-		echo "Error: folder does not exist $$GRAFANA_ASSETS_DIR" >&2; \
-		exit 1; \
-	fi; \
-	if [ ! -d "$(shell dirname ${REPO_CONFIG_LOCATION})/../shared/assets/grafana" ]; then \
-		echo "Error: folder does not exist $$GRAFANA_COMMON_ASSETS_DIR" >&2; \
-		exit 1; \
-	fi; \
-	rm -rf $(REPO_BASE_DIR)/services/monitoring/grafana/assets || true; \
-	mkdir -p $(REPO_BASE_DIR)/services/monitoring/grafana/assets; \
-	cp -r $(shell dirname ${REPO_CONFIG_LOCATION})/assets/grafana/* $(REPO_BASE_DIR)/services/monitoring/grafana/assets; \
-	cp -r $(shell dirname ${REPO_CONFIG_LOCATION})/../shared/assets/grafana $(REPO_BASE_DIR)/services/monitoring/grafana/assets/shared; \
+	@$(MAKE_C) grafana assets

services/monitoring/docker-compose.yml.j2

@@ -377,9 +377,12 @@ services:
     networks:
       - monitored
     environment:
-      REDIS_ADDR: redis://{{REDIS_HOST}}:{{REDIS_PORT}}
-      REDIS_EXPORTER_CHECK_KEYS: db0=user_id*client_session_id*alive,db0=user_id*client_session_id*resources,db1=project_lock*,db3=*
+      REDIS_ADDR: ${REDIS_ADDRESS}
+{%- if REDIS_USER %}
+      REDIS_USER: ${REDIS_USER}
+{%- endif %}
       REDIS_PASSWORD: ${REDIS_PASSWORD}
+      REDIS_EXPORTER_CHECK_KEYS: db0=user_id*client_session_id*alive,db0=user_id*client_session_id*resources,db1=project_lock*,db3=*
     deploy:
       labels:
       - prometheus-job=redis-exporter

services/monitoring/grafana/Makefile

@@ -0,0 +1,79 @@
+.DEFAULT_GOAL := help
+REPO_BASE_DIR := $(abspath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))../../..)
+include ${REPO_BASE_DIR}/scripts/common.Makefile
+
+
+
+# Internal VARIABLES ------------------------------------------------
+TF_STATE_FILE := terraform/.terraform/terraform.tfstate
+.PHONY: terraform/main.tf
+terraform/main.tf: terraform/main.tf.j2 .venv $(REPO_CONFIG_LOCATION)
+	# generate $@
+	@$(call jinja, $<, $(REPO_CONFIG_LOCATION), $@)
+
+terraform-init: $(TF_STATE_FILE)  ## init terraform
+
+$(TF_STATE_FILE): $(REPO_CONFIG_LOCATION) terraform/main.tf
+	# terraform init
+	@set -a; source $<; set +a; \
+	if [ "$${GRAFANA_TERRAFORM_STATE_BACKEND_TYPE}" = "local" ]; then \
+	  terraform -chdir=./terraform init; \
+	else \
+	  terraform -chdir=./terraform init -backend-config="access_key=$${TF_GRAFANA_STATE_BACKEND_AWS_ACCESS_KEY_ID}" -backend-config="secret_key=$${TF_GRAFANA_STATE_BACKEND_AWS_SECRET_ACCESS_KEY}"; \
+	fi
+
+terraform/plan.cache:
+	@echo "$@ file not found. Run 'make terraform-plan' to generate it."
+	@exit 1
+
+.PHONY: terraform-plan
+terraform-plan: $(REPO_CONFIG_LOCATION) $(TF_STATE_FILE) ensure-grafana-online ## terraform plan
+	# terraform plan
+	@set -a; source $<; set +a; \
+	terraform -chdir=./terraform plan -out=plan.cache
+
+.PHONY: terraform-apply
+terraform-apply: $(REPO_CONFIG_LOCATION) terraform/plan.cache $(TF_STATE_FILE) ensure-grafana-online ## terraform apply
+	# terraform apply
+	@set -a; source $<; set +a; \
+	terraform -chdir=./terraform apply plan.cache
+
+.PHONY: ensure-grafana-online
+ensure-grafana-online:
+	@set -o allexport; \
+	source $(REPO_CONFIG_LOCATION); \
+	set +o allexport; \
+	url=$${TF_VAR_GRAFANA_URL}; \
+	echo "Waiting for grafana at $$url to become reachable..."; \
+	attempts=0; \
+	max_attempts=10; \
+	while [ $$attempts -lt $$max_attempts ]; do \
+		status_code=$$(curl -k -o /dev/null -s -w "%{http_code}" --max-time 10 $$url); \
+		if [ "$$status_code" -ge 200 ] && [ "$$status_code" -lt 400 ]; then \
+			echo "Grafana is online"; \
+			break; \
+		else \
+			echo "Grafana still unreachable, waiting 5s for grafana to become reachable... (Attempt $$((attempts+1)))"; \
+			sleep 5; \
+			attempts=$$((attempts + 1)); \
+		fi; \
+	done; \
+	if [ $$attempts -eq $$max_attempts ]; then \
+		echo "Max attempts reached, Grafana is still unreachable."; \
+		exit 1; \
+	fi;
+
+.PHONY: assets
+assets: ${REPO_CONFIG_LOCATION}
+	@if [ ! -d "$(shell dirname ${REPO_CONFIG_LOCATION})/assets/grafana" ]; then \
+		echo "Error: folder does not exist $$GRAFANA_ASSETS_DIR" >&2; \
+		exit 1; \
+	fi; \
+	if [ ! -d "$(shell dirname ${REPO_CONFIG_LOCATION})/../shared/assets/grafana" ]; then \
+		echo "Error: folder does not exist $$GRAFANA_COMMON_ASSETS_DIR" >&2; \
+		exit 1; \
+	fi; \
+	rm -rf $(REPO_BASE_DIR)/services/monitoring/grafana/assets || true; \
+	mkdir -p $(REPO_BASE_DIR)/services/monitoring/grafana/assets; \
+	cp -r $(shell dirname ${REPO_CONFIG_LOCATION})/assets/grafana/* $(REPO_BASE_DIR)/services/monitoring/grafana/assets; \
+	cp -r $(shell dirname ${REPO_CONFIG_LOCATION})/../shared/assets/grafana $(REPO_BASE_DIR)/services/monitoring/grafana/assets/shared; \