Skip to content

Commit 8b54592

Browse files
mrnicegyu11YuryHrytsuk
mrnicegyu11
and
YuryHrytsuk
authored
Refactor: ops-traefik configuration (2nd attempt) (ITISFoundation#1118)
* wip * Add csi-s3 and have portainer use it * Change request @Hrytsuk 1GB max portainer volume size * Arch Linux Certificates Customization * Fix pgsql exporter failure * [Kubernetes] Introduce on-prem persistent Storage (Longhorn) 🎉 (ITISFoundation#979) * Introduce longhorn chart * Further longhorn configuration * Longhorn: further settings configuration * Fix longhorn configuration bugs Extra: introduce longhorn pv vales for portainer * Add comment for deletion longhorn * Further longhorn configuration * Add README.md for Longhorn wit FAQ * Update Longhorn readme * Update readme * Futher LH configuration * Update LH's Readme * Update Longhorn Readme * Improve LH's Readme * LH: Reduce reserved default disk space to 5% Since we use a dedicated disk for LH, we can go ahead with 5% * Use values to set Longhorn storage class * Update LH's Readme * LH Readme: add requirements reference * PR Review: bring back portainer s3 pv * LH: decrease portinaer volume size * Experimental: Try to add tracing to simcore-traefik on master * Fixes ITISFoundation/osparc-simcore#7363 * Refactor ops-traefik: use env-var based config * fixes * fixes * Re-add REDIS_EXTERNAL_PORT env var * wip * wip * initial working version on osparc.local * Update traefik to 3.4.4 * osparc.local fixes * Traefik v3 syntax fixes * revert faulty commit --------- Co-authored-by: Dustin Kaiser <[email protected]> Co-authored-by: YH <[email protected]>
1 parent 12ccb60 commit 8b54592

15 files changed

+109
-328
lines changed

‎scripts/common.Makefile‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,13 +35,13 @@ endif
3535
export DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL:=$(shell set -o allexport; \
3636
source $(REPO_CONFIG_LOCATION); \
3737
if [ -z "$${DEPLOYMENT_FQDNS}" ]; then \
38-
DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL="(Host(\`$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (Host(\`invitations.$$MACHINE_FQDN\`))|| (HostRegexp(\`services.$$MACHINE_FQDN\`,\`{subhost:[a-zA-Z0-9-]+}.services.$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (HostRegexp(\`services.testing.$$MACHINE_FQDN\`,\`{subhost:[a-zA-Z0-9-]+}.services.testing.$$MACHINE_FQDN\`) && PathPrefix(\`/\`))"; \
38+
DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL="(Host(\`$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (Host(\`invitations.$$MACHINE_FQDN\`))|| (HostRegexp(\`services.$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (HostRegexp(\`services.testing.$$MACHINE_FQDN\`) && PathPrefix(\`/\`))"; \
3939
else \
4040
IFS=', ' read -r -a hosts <<< "$${DEPLOYMENT_FQDNS}"; \
41-
DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL="(Host(\`$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (Host(\`invitations.$$MACHINE_FQDN\`))|| (HostRegexp(\`services.$$MACHINE_FQDN\`,\`{subhost:[a-zA-Z0-9-]+}.services.$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (HostRegexp(\`services.testing.$$MACHINE_FQDN\`,\`{subhost:[a-zA-Z0-9-]+}.services.testing.$$MACHINE_FQDN\`) && PathPrefix(\`/\`))"; \
41+
DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL="(Host(\`$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (Host(\`invitations.$$MACHINE_FQDN\`))|| (HostRegexp(\`services.$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (HostRegexp(\`services.testing.$$MACHINE_FQDN\`) && PathPrefix(\`/\`))"; \
4242
for element in "$${hosts[@]}"; \
4343
do \
44-
DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL="$$DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL || (Host(\`$$element\`) && PathPrefix(\`/\`)) || (Host(\`invitations.$$element\`)) || (HostRegexp(\`services.$$element\`,\`{subhost:[a-zA-Z0-9-]+}.services.$$element\`) && PathPrefix(\`/\`)) || (HostRegexp(\`services.testing.$$element\`,\`{subhost:[a-zA-Z0-9-]+}.services.testing.$$element\`) && PathPrefix(\`/\`))";\
44+
DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL="$$DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL || (Host(\`$$element\`) && PathPrefix(\`/\`)) || (Host(\`invitations.$$element\`)) || (HostRegexp(\`services.$$element\`) && PathPrefix(\`/\`)) || (HostRegexp(\`services.testing.$$element\`) && PathPrefix(\`/\`))";\
4545
done; \
4646
DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL="$$DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL"; \
4747
fi; \
@@ -68,13 +68,13 @@ export DEPLOYMENT_FQDNS_CAPTURE_INVITATIONS:=$(shell set -o allexport; \
6868
export DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_MAINTENANCE_PAGE:=$(shell set -o allexport; \
6969
source $(REPO_CONFIG_LOCATION); \
7070
if [ -z "$${DEPLOYMENT_FQDNS}" ]; then \
71-
DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_MAINTENANCE_PAGE="(Host(\`$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (HostRegexp(\`services.$$MACHINE_FQDN\`,\`{subhost:[a-zA-Z0-9-]+}.services.$$MACHINE_FQDN\`) && PathPrefix(\`/\`))"; \
71+
DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_MAINTENANCE_PAGE="(Host(\`$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (HostRegexp(\`services.$$MACHINE_FQDN\`) && PathPrefix(\`/\`))"; \
7272
else \
7373
IFS=', ' read -r -a hosts <<< "$${DEPLOYMENT_FQDNS}"; \
74-
DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_MAINTENANCE_PAGE="(Host(\`$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (HostRegexp(\`services.$$MACHINE_FQDN\`,\`{subhost:[a-zA-Z0-9-]+}.services.$$MACHINE_FQDN\`) && PathPrefix(\`/\`))"; \
74+
DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_MAINTENANCE_PAGE="(Host(\`$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (HostRegexp(\`services.$$MACHINE_FQDN\`) && PathPrefix(\`/\`))"; \
7575
for element in "$${hosts[@]}"; \
7676
do \
77-
DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_MAINTENANCE_PAGE="$$DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_MAINTENANCE_PAGE || (Host(\`$$element\`) && PathPrefix(\`/\`)) || (HostRegexp(\`services.$$element\`,\`{subhost:[a-zA-Z0-9-]+}.services.$$element\`) && PathPrefix(\`/\`))";\
77+
DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_MAINTENANCE_PAGE="$$DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_MAINTENANCE_PAGE || (Host(\`$$element\`) && PathPrefix(\`/\`)) || (HostRegexp(\`services.$$element\`) && PathPrefix(\`/\`))";\
7878
done; \
7979
DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_MAINTENANCE_PAGE="$$DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_MAINTENANCE_PAGE"; \
8080
fi; \

‎services/graylog/docker-compose.local.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
version: "3.7"
21
services:
32
mongodb:
43
deploy:

‎services/graylog/docker-compose.yml.j2‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
version: "3.7"
21
services:
32
# MongoDB: https://hub.docker.com/_/mongo/
43
mongodb:

‎services/maintenance-page/docker-compose.yml.j2‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ services:
2828
- traefik.enable=true
2929
- traefik.swarm.network=${PUBLIC_NETWORK}
3030
- traefik.http.routers.{{"maintenance_" + j2item.replace('@','').replace(' ','').replace('.','').replace('-','').replace('\'','') + "_html"}}.priority={{MAINTENANCE_PAGES_TRAEFIK_PRIORITY}}
31-
- traefik.http.routers.{{"maintenance_" + j2item.replace('@','').replace(' ','').replace('.','').replace('-','').replace('\'','') + "_html"}}.rule=Host(`{{VENDOR_MANUAL_SUBDOMAIN_PREFIX}}.{{j2item}}`) || (Host(`{{j2item}}`) && PathPrefix(`/`)) || (HostRegexp(`services.{{j2item}}`,`{subhost:[a-zA-Z0-9-]+}.services.{{j2item}}`) && PathPrefix(`/`))
31+
- traefik.http.routers.{{"maintenance_" + j2item.replace('@','').replace(' ','').replace('.','').replace('-','').replace('\'','') + "_html"}}.rule=Host(`{{VENDOR_MANUAL_SUBDOMAIN_PREFIX}}.{{j2item}}`) || (Host(`{{j2item}}`) && PathPrefix(`/`)) || (HostRegexp(`services.{{j2item}}`) && PathPrefix(`/`))
3232
- traefik.http.routers.{{"maintenance_" + j2item.replace('@','').replace(' ','').replace('.','').replace('-','').replace('\'','') + "_html"}}.tls=true
3333
- traefik.http.services.{{"maintenance_" + j2item.replace('@','').replace(' ','').replace('.','').replace('-','').replace('\'','') + "_html"}}.loadbalancer.server.port=80
3434
- traefik.http.routers.{{"maintenance_" + j2item.replace('@','').replace(' ','').replace('.','').replace('-','').replace('\'','') + "_html"}}.entrypoints=https

‎services/portainer/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ up: .init .env secrets ${TEMP_COMPOSE}
2020
.PHONY: up-local ## Deploys portainer stack for local deployment
2121
up-local: .init .env secrets ${TEMP_COMPOSE} ${TEMP_COMPOSE}-local
2222
@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-local ${STACK_NAME}
23-
@$(MAKE) --no-print-directory configure-portainer-registry
23+
@$(MAKE) configure-portainer-registry
2424

2525
.PHONY: up-dalco ## Deploys portainer stack for Dalco Cluster
2626
up-dalco: .init .env secrets ${TEMP_COMPOSE}-dalco

‎services/portainer/scripts/configure_portainer_registry.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ def get_portainer_api_auth_token(
3232
f"{portainer_api_url}/auth",
3333
# https://app.swaggerhub.com/apis/portainer/portainer-ce/2.27.6#/auth.authenticatePayload
3434
json={"Username": portainer_username, "Password": portainer_password},
35+
verify=False,
3536
)
3637

3738
try:
@@ -49,6 +50,7 @@ def get_registries(portainer_api_url: str, auth_token: str) -> list[Registry]:
4950
response = requests.get(
5051
f"{portainer_api_url}/registries",
5152
headers={"Authorization": f"Bearer {auth_token}"},
53+
verify=False,
5254
)
5355

5456
try:
@@ -81,6 +83,7 @@ def create_authenticated_dockerhub_registry(
8183
"password": dockerhub_password,
8284
"type": RegistryType.DOCKER_HUB.value,
8385
},
86+
verify=False,
8487
)
8588

8689
try:

‎services/portainer/template.env‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
PORTAINER_ADMIN_PWD=${PORTAINER_PASSWORD}
22
PORTAINER_ADMIN_LOGIN=${PORTAINER_USER}
33
MONITORING_DOMAIN=${MONITORING_DOMAIN}
4-
4+
PORTAINER_URL=${PORTAINER_URL}
55
PUBLIC_NETWORK=${PUBLIC_NETWORK}

‎services/traefik/Makefile‎

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -49,38 +49,32 @@ ${TEMP_COMPOSE}-local: docker-compose.yml docker-compose.local.yml .env traefik_
4949
${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.local.yml > $@
5050

5151
.PHONY: ${TEMP_COMPOSE}-aws
52-
${TEMP_COMPOSE}-aws: docker-compose.yml docker-compose.aws.yml .env
52+
${TEMP_COMPOSE}-aws: docker-compose.yml .env
5353
@set -o allexport; \
5454
source .env; \
5555
set +o allexport; \
56-
${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.aws.yml > $@
56+
${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< > $@
5757

5858
.PHONY: ${TEMP_COMPOSE}-dalco
59-
${TEMP_COMPOSE}-dalco: docker-compose.yml docker-compose.dalco.yml .env
59+
${TEMP_COMPOSE}-dalco: docker-compose.yml .env
6060
@set -o allexport; \
6161
source .env; \
6262
set +o allexport; \
63-
${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.dalco.yml > $@
63+
${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< > $@
6464

6565
.PHONY: ${TEMP_COMPOSE}-public
66-
${TEMP_COMPOSE}-public: docker-compose.yml docker-compose.public.yml .env
66+
${TEMP_COMPOSE}-public: docker-compose.yml .env
6767
@set -o allexport; \
6868
source .env; \
6969
set +o allexport; \
70-
${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.public.yml > $@
70+
${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< > $@
7171

7272
.PHONY: ${TEMP_COMPOSE}-master
73-
${TEMP_COMPOSE}-master: docker-compose.yml docker-compose.master.yml .env
73+
${TEMP_COMPOSE}-master: docker-compose.yml .env
7474
@set -o allexport; \
7575
source .env; \
7676
set +o allexport; \
77-
${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.master.yml > $@
78-
79-
.PHONY: docker-compose.letsencrypt.dns.yml
80-
docker-compose.letsencrypt.dns.yml: venv .env
81-
@$(call jinja, docker-compose.letsencrypt.dns.yml.j2, .env, docker-compose.letsencrypt.dns.yml.unlinted) && \
82-
$(_yq) docker-compose.letsencrypt.dns.yml.unlinted > docker-compose.letsencrypt.dns.yml; \
83-
rm docker-compose.letsencrypt.dns.yml.unlinted >/dev/null 2>&1;
77+
${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< > $@
8478

8579
.PHONY: docker-compose.yml
8680
docker-compose.yml: traefik_dynamic_config.yml venv .env

‎services/traefik/docker-compose.aws.yml‎

Lines changed: 0 additions & 54 deletions
This file was deleted.

‎services/traefik/docker-compose.dalco.yml‎

Lines changed: 0 additions & 44 deletions
This file was deleted.

0 commit comments

Comments
 (0)