diff --git a/scripts/.gitignore b/scripts/.gitignore new file mode 100644 index 00000000..79cef29c --- /dev/null +++ b/scripts/.gitignore @@ -0,0 +1 @@ +wait4x diff --git a/scripts/common.Makefile b/scripts/common.Makefile index 86d008b8..5f3f4517 100644 --- a/scripts/common.Makefile +++ b/scripts/common.Makefile @@ -342,8 +342,13 @@ show-venv: venv ## show venv info @echo venv: $(VENV_DIR) .PHONY: install -install: requirements.txt venv ## install dependencies from ./requirements.txt - @VIRTUAL_ENV=$(VENV_DIR) $(UV) pip install --requirement $< +install: guard-optional-REQUIREMENTS_FILE venv ## install requirements.txt dependencies + @if [ -z "$(REQUIREMENTS_FILE)" ]; then \ + REQUIREMENTS_FILE=./requirements.txt; \ + else \ + REQUIREMENTS_FILE=$(REQUIREMENTS_FILE); \ + fi; \ + VIRTUAL_ENV=$(VENV_DIR) $(UV) pip install --requirement $$REQUIREMENTS_FILE # https://github.com/kolypto/j2cli?tab=readme-ov-file#customization ifeq ($(shell test -f j2cli_customization.py && echo -n yes),yes) @@ -362,3 +367,21 @@ define jinja endef endif + +# +# wait-fot-it functionality +# + +WAIT_FOR_IT := $(REPO_BASE_DIR)/scripts/wait4x + +alias: $(WAIT_FOR_IT) + +# https://github.com/wait4x/wait4x +$(WAIT_FOR_IT): ## installs wait4x utility for WAIT_FOR_IT functionality + # installing wait4x + @mkdir --parents /tmp/wait4x + @cd /tmp/wait4x && curl --silent --location --remote-name https://github.com/wait4x/wait4x/releases/download/v3.5.0/wait4x-linux-amd64.tar.gz + @tar -xf /tmp/wait4x/wait4x-linux-amd64.tar.gz -C /tmp/wait4x + @mv /tmp/wait4x/wait4x $@ + @rm -rf /tmp/wait4x + @$@ version diff --git a/services/monitoring/grafana/Makefile b/services/monitoring/grafana/Makefile index 0c088ed1..d6db6db3 100644 --- a/services/monitoring/grafana/Makefile +++ b/services/monitoring/grafana/Makefile @@ -39,29 +39,11 @@ terraform-apply: $(REPO_CONFIG_LOCATION) terraform/plan.cache $(TF_STATE_FILE) e terraform -chdir=./terraform apply plan.cache .PHONY: ensure-grafana-online -ensure-grafana-online: +ensure-grafana-online: $(WAIT_FOR_IT) @set -o allexport; \ source $(REPO_CONFIG_LOCATION); \ set +o allexport; \ - url=$${TF_VAR_GRAFANA_URL}; \ - echo "Waiting for grafana at $$url to become reachable..."; \ - attempts=0; \ - max_attempts=10; \ - while [ $$attempts -lt $$max_attempts ]; do \ - status_code=$$(curl -k -o /dev/null -s -w "%{http_code}" --max-time 10 $$url); \ - if [ "$$status_code" -ge 200 ] && [ "$$status_code" -lt 400 ]; then \ - echo "Grafana is online"; \ - break; \ - else \ - echo "Grafana still unreachable, waiting 5s for grafana to become reachable... (Attempt $$((attempts+1)))"; \ - sleep 5; \ - attempts=$$((attempts + 1)); \ - fi; \ - done; \ - if [ $$attempts -eq $$max_attempts ]; then \ - echo "Max attempts reached, Grafana is still unreachable."; \ - exit 1; \ - fi; + $(WAIT_FOR_IT) http $$TF_VAR_GRAFANA_URL --timeout=120s --interval=5s --expect-status-code 200 .PHONY: assets assets: ${REPO_CONFIG_LOCATION} diff --git a/services/portainer/Makefile b/services/portainer/Makefile index 9337e5d3..33520b87 100644 --- a/services/portainer/Makefile +++ b/services/portainer/Makefile @@ -15,26 +15,22 @@ include ${REPO_BASE_DIR}/scripts/common.Makefile .PHONY: up ## Deploys portainer stack up: .init .env secrets ${TEMP_COMPOSE} @docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE} ${STACK_NAME} + @$(MAKE) --noprint configure-portainer-registry .PHONY: up-local ## Deploys portainer stack for local deployment up-local: .init .env secrets ${TEMP_COMPOSE} ${TEMP_COMPOSE}-local @docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-local ${STACK_NAME} - -.PHONY: up-letsencrypt-http ## Deploys portainer stack using let's encrypt http challenge -up-letsencrypt-http: .init .env secrets ${TEMP_COMPOSE}-letsencrypt-http - @docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-letsencrypt-http ${STACK_NAME} - -.PHONY: up-letsencrypt-dns ## Deploys portainer stack using let's encrypt dns challenge -up-letsencrypt-dns: .init .env secrets ${TEMP_COMPOSE}-letsencrypt-dns - @docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-letsencrypt-dns ${STACK_NAME} + @$(MAKE) --noprint configure-portainer-registry .PHONY: up-dalco ## Deploys portainer stack for Dalco Cluster up-dalco: .init .env secrets ${TEMP_COMPOSE}-dalco @docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-dalco ${STACK_NAME} + @$(MAKE) --noprint configure-portainer-registry .PHONY: up-aws ## Deploys portainer stack for AWS up-aws: .init .env secrets ${TEMP_COMPOSE}-aws @docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-aws ${STACK_NAME} + @$(MAKE) --noprint configure-portainer-registry .PHONY: up-public ## Deploys portainer stack for public access Cluster up-public: up-dalco @@ -42,23 +38,15 @@ up-public: up-dalco .PHONY: up-master ## Deploys portainer stack for master Cluster up-master: .init .env secrets ${TEMP_COMPOSE}-master @docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-master ${STACK_NAME} + @$(MAKE) --noprint configure-portainer-registry -.PHONY: configure-registry -configure-registry: ## Add if necessary dockerhub registry configuration to portainer. - @set -o allexport; \ - source $(REPO_CONFIG_LOCATION); \ - set +o allexport; \ - while [ "$$(curl -s -o /dev/null -I -w "%{http_code}" --max-time 10 -H "Accept: application/json" -H "Content-Type: application/json" -X GET https://"$$MONITORING_DOMAIN"/portainer/#/auth)" != 200 ]; do\ - echo "waiting for portainer to run...";\ - sleep 5s;\ - done;\ - echo "Updating docker-hub config";\ - authentificationToken=$$(curl -o /dev/null -X POST "https://"$$MONITORING_DOMAIN"/portainer/api/auth" -H "Content-Type: application/json" -d "{ \"Username\": \"$${PORTAINER_ADMIN_LOGIN}\", \"Password\": \"$${PORTAINER_ADMIN_PWD}\"}"); \ - authentificationToken=$$(echo "$$authentificationToken" | jq --raw-output '.jwt'); \ - update_hub=$$(curl -o /dev/null -X PUT "https://"$$MONITORING_DOMAIN"/portainer/api/dockerhub" -H "accept: application/json" -H \ - "Authorization: Bearer $${authentificationToken}" -H "Content-Type: application/json" \ - -d "{ \"Authentication\": true, \"Username\": \"$$DOCKER_HUB_LOGIN\", \"Password\": \"$$DOCKER_HUB_PASSWORD\"}"); \ +.PHONY: configure-portainer-registry +configure-portainer-registry: venv $(VENV_BIN)/python $(WAIT_FOR_IT) ## Add registry to Portainer + @$(MAKE) --no-print install REQUIREMENTS_FILE=./scripts/requirements.txt + @set -o allexport; source $(REPO_CONFIG_LOCATION); set +o allexport; \ + $(WAIT_FOR_IT) http $$PORTAINER_URL --timeout=120s --interval=5s --expect-status-code 200 && \ + $(VENV_BIN)/python ./scripts/configure_portainer_registry.py # Helpers ------------------------------------------------- @@ -67,14 +55,6 @@ configure-registry: ## Add if necessary dockerhub registry configuration to por ${TEMP_COMPOSE}: docker-compose.yml .env @${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< > $@ -.PHONY: ${TEMP_COMPOSE}-letsencrypt-http -${TEMP_COMPOSE}-letsencrypt-http: docker-compose.yml docker-compose.letsencrypt.http.yml .env - @${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.letsencrypt.http.yml > $@ - -.PHONY: ${TEMP_COMPOSE}-letsencrypt-dns -${TEMP_COMPOSE}-letsencrypt-dns: docker-compose.yml docker-compose.letsencrypt.dns.yml .env - @${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.letsencrypt.dns.yml > $@ - .PHONY: ${TEMP_COMPOSE}-dalco ${TEMP_COMPOSE}-dalco: docker-compose.yml docker-compose.dalco.yml .env @${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.dalco.yml > $@ diff --git a/services/portainer/docker-compose.letsencrypt.dns.yml b/services/portainer/docker-compose.letsencrypt.dns.yml deleted file mode 100644 index 88fbcdb6..00000000 --- a/services/portainer/docker-compose.letsencrypt.dns.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: "3.7" -services: - portainer: - deploy: - labels: - - traefik.http.routers.portainer.tls.certresolver=myresolver diff --git a/services/portainer/docker-compose.letsencrypt.http.yml b/services/portainer/docker-compose.letsencrypt.http.yml deleted file mode 100644 index 74cf115f..00000000 --- a/services/portainer/docker-compose.letsencrypt.http.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: "3.7" -services: - portainer: - deploy: - labels: - - traefik.http.routers.portainer.tls.certresolver=lehttpchallenge diff --git a/services/portainer/scripts/configure_portainer_registry.py b/services/portainer/scripts/configure_portainer_registry.py new file mode 100644 index 00000000..017076e8 --- /dev/null +++ b/services/portainer/scripts/configure_portainer_registry.py @@ -0,0 +1,138 @@ +import logging +import os +from enum import Enum +from typing import TypedDict + +import requests +from tenacity import retry + +logger = logging.getLogger(__name__) + + +# https://app.swaggerhub.com/apis/portainer/portainer-ce/2.27.6#/portainer.Registry +class RegistryType(Enum): + DOCKER_HUB = 6 + + +class Registry(TypedDict): + Id: int + Name: str + URL: str + Authentication: bool + Username: str + Type: RegistryType + + +@retry +def get_portainer_api_auth_token( + portainer_api_url: str, portainer_username: str, portainer_password: str +) -> str: + # https://app.swaggerhub.com/apis/portainer/portainer-ce/2.27.6#/auth/AuthenticateUser + response = requests.post( + f"{portainer_api_url}/auth", + # https://app.swaggerhub.com/apis/portainer/portainer-ce/2.27.6#/auth.authenticatePayload + json={"Username": portainer_username, "Password": portainer_password}, + ) + + try: + response.raise_for_status() + except requests.HTTPError as e: + logger.error("Failed to authenticate with Portainer API: %s", e.response.text) + raise + + return response.json()["jwt"] + + +@retry +def get_registries(portainer_api_url: str, auth_token: str) -> list[Registry]: + # https://app.swaggerhub.com/apis/portainer/portainer-ce/2.27.6#/registries/RegistryList + response = requests.get( + f"{portainer_api_url}/registries", + headers={"Authorization": f"Bearer {auth_token}"}, + ) + + try: + response.raise_for_status() + except requests.HTTPError as e: + logger.error("Failed to fetch registries: %s", e.response.text) + raise + + return response.json() + + +@retry +def create_authenticated_dockerhub_registry( + portainer_api_url: str, + auth_token: str, + dockerhub_username: str, + dockerhub_password: str, + registry_name: str = "IT'IS Foundation", +) -> None: + # https://app.swaggerhub.com/apis/portainer/portainer-ce/2.27.6#/registries/RegistryCreate + response = requests.post( + f"{portainer_api_url}/registries", + headers={"Authorization": f"Bearer {auth_token}"}, + # https://app.swaggerhub.com/apis/portainer/portainer-ce/2.27.6#/registries.registryCreatePayload + json={ + "name": registry_name, + "url": "docker.io", + "authentication": True, + "username": dockerhub_username, + "password": dockerhub_password, + "type": RegistryType.DOCKER_HUB.value, + }, + ) + + try: + response.raise_for_status() + except requests.HTTPError as e: + logger.error( + "Failed to create authenticated Docker Hub registry: %s", e.response.text + ) + raise + + return response.json() + + +def main(): + logger.info("Configuring Portainer registries...") + + portainer_username = os.environ["SERVICES_USER"] + portainer_password = os.environ["SERVICES_PASSWORD"] + portainer_api_url = os.environ["PORTAINER_URL"] + "/api" + + dockerhub_username = os.environ["DOCKER_HUB_LOGIN"] + dockerhub_password = os.environ["DOCKER_HUB_PASSWORD"] + + portainer_jwt_token = get_portainer_api_auth_token( + portainer_api_url, portainer_username, portainer_password + ) + + registries = get_registries(portainer_api_url, portainer_jwt_token) + + if not any( + r["Type"] == RegistryType.DOCKER_HUB.value and r["Authentication"] is True + for r in registries + ): + logging.info("Creating authenticated Docker Hub registry in Portainer...") + create_authenticated_dockerhub_registry( + portainer_api_url, + portainer_jwt_token, + dockerhub_username, + dockerhub_password, + ) + else: + logging.info("Portainer already has an authenticated Docker Hub registry.") + + logging.info("Portainer registries configuration completed.") + + +def configure_logging(): + logging.basicConfig( + level=logging.INFO, + ) + + +if __name__ == "__main__": + configure_logging() + main() diff --git a/services/portainer/scripts/requirements.txt b/services/portainer/scripts/requirements.txt new file mode 100644 index 00000000..e58f026e --- /dev/null +++ b/services/portainer/scripts/requirements.txt @@ -0,0 +1,2 @@ +tenacity==9.1.2 +requests==2.32.4