compose.yaml

# This document uses the open "Docker Compose V2" file specification.
# More info: https://www.compose-spec.io and https://github.com/compose-spec/compose-spec/blob/master/spec.md
# Please note this is different from the "Compose file version 2" specification. Which is for Docker Compose V1.
# Hence no `version` is specified in this file.

services:
  # nginx proxy container, also runs certbot
  webserver:
    platform: linux/amd64
    image: ${DOCKER_IMAGE_WEBSERVER:-${DOCKER_REGISTRY:-ghcr.io/internetstandards}/webserver:${RELEASE}}
    restart: unless-stopped
    logging:
      driver: $LOGGING_DRIVER
      options:
        tag: '{{.Name}}'
    mem_limit: $DEFAULT_MEMORY_LIMIT
    # disable swap by settings swap to the memory limit
    memswap_limit: $DEFAULT_MEMORY_LIMIT
    networks:
      internal:
      public-internet: {}

    ports:
      - $WEBSERVER_PORT
      - $WEBSERVER_PORT_TLS/tcp
      - $WEBSERVER_PORT_TLS/udp
      - $WEBSERVER_PORT_IPV6/tcp
      - $WEBSERVER_PORT_IPV6_TLS/tcp
      - $WEBSERVER_PORT_IPV6_TLS/udp

    environment:
      - INTERNETNL_DOMAINNAME
      - IPV6_TEST_ADDR
      - MONITORING_AUTH_RAW
      - AUTH_ALL_URLS
      - ALLOW_LIST
      # required for authentication.sh to restrict access when debug is on
      - DEBUG
      - ENABLE_BATCH
      - LETSENCRYPT_STAGING
      - LETSENCRYPT_EMAIL
      - CERTBOT_SERVER
      - CERTBOT_EAB_KID
      - CERTBOT_EAB_HMAC_KEY
      - REDIRECT_DOMAINS
      - NGINX_PROXY_CACHE
      - INTERNETNL_BRANDING
      - LANGUAGES

    # webserver does not depend on any of the other services directly. So it can
    # be started and kept running independently from the other services to
    # provide stale cache or a maintenance page.
    depends_on: {}

    volumes:
      # persist certbot configuration between restarts
      - certbot-config:/etc/letsencrypt
      - htpasswd-files:/etc/nginx/htpasswd/external
      - nginx-logs-exporter:/var/log/nginx/prometheus-nginxlog-exporter/

    healthcheck:
      test: ["CMD", "service", "nginx", "status"]
      interval: $HEALTHCHECK_INTERVAL
      start_interval: $HEALTHCHECK_START_INTERVAL
      start_period: 1m
      retries: 10

  # django container
  app:
    platform: linux/amd64
    image: ${DOCKER_IMAGE_APP:-${DOCKER_REGISTRY:-ghcr.io/internetstandards}/internet.nl:${RELEASE}}
    restart: unless-stopped
    logging:
      driver: $LOGGING_DRIVER
      options:
        tag: '{{.Name}}'
    mem_limit: $HIGH_MEMORY_LIMIT
    # disable swap by settings swap to the memory limit
    memswap_limit: $HIGH_MEMORY_LIMIT
    networks:
      internal:
      public-internet: {}
    entrypoint: uwsgi
    command: >
      --module=internetnl.wsgi:application
      --http=0.0.0.0:8080
      --master
      --processes=4
      --stats 127.0.0.1:1717 --stats-http
    depends_on:
      db-migrate:
        # wait for DB migration to be completed
        condition: service_completed_successfully
      redis:
        condition: service_healthy
      rabbitmq:
        condition: service_healthy
      postgres:
        condition: service_healthy
      unbound:
        condition: service_healthy
        required: false
      resolver-validating:
        # restart this service when the resolver is restarted and possibly has a new IP, so the old IP is no longer used
        restart: true
        condition: service_healthy
    # set hostname for Sentry
    hostname: app
    environment:
      - INTERNET_NL_CHECK_SUPPORT_IPV6
      - INTERNET_NL_CHECK_SUPPORT_DNSSEC
      - INTERNET_NL_CHECK_SUPPORT_MAIL
      - INTERNET_NL_CHECK_SUPPORT_TLS
      - INTERNET_NL_CHECK_SUPPORT_APPSECPRIV
      - INTERNET_NL_CHECK_SUPPORT_RPKI
      - PUBLIC_SUFFIX_LIST_URL
      - ENABLE_BATCH
      - ENABLE_HOF
      - RABBIT_HOST=rabbitmq:15672
      - SECRET_KEY
      - GENERATE_SECRET_KEY
      - DB_HOST=postgres
      - DB_NAME=internetnl_db1
      - DB_USER=internetnl
      - DB_PASSWORD=password
      - CELERY_BROKER_URL=amqp://guest:guest@rabbitmq:5672//
      - CELERY_RESULT_BACKEND=redis://redis:6379/0
      - CACHE_LOCATION=redis://redis:6379/0
      - ROUTINATOR_URL
      - DJANGO_IS_PROXIED=True
      - STATSD_HOST=statsd
      - ALLOWED_HOSTS
      - DEBUG
      - DEBUG_LOG
      - DEBUG_LOG_UNBOUND
      - INTEGRATION_TESTS
      - INTERNETNL_LOG_LEVEL
      - INTERNETNL_CACHE_TTL
      - CONN_TEST_DOMAIN
      - SMTP_EHLO_DOMAIN
      - IPV6_TEST_ADDR
      - SENTRY_DSN
      - SENTRY_ENVIRONMENT
      - SENTRY_SERVER_NAME
      - MATOMO_URL
      - MATOMO_SITEID
      - MATOMO_SUBDOMAIN_TRACKING
      - MANUAL_HOF_PAGES
      - CLIENT_RATE_LIMIT
      - INTERNETNL_BRANDING
      - INTERNETNL_DOMAINNAME
      - USER_AGENT_URL
      - LANGUAGES
      - PAGE_CACHE_TIME_SECONDS
    healthcheck:
      test: ["CMD", "nc", "-z", "127.0.0.1", "8080"]
      interval: $HEALTHCHECK_INTERVAL
      start_interval: $HEALTHCHECK_START_INTERVAL
      start_period: 1m
      retries: 10

    volumes:
      - batch_results:/app/batch_results
      - manual-hof:/app/manual-hall-of-fame/

  # django DB migrations, runs to completion and exits with 0
  db-migrate:
    platform: linux/amd64
    image: ${DOCKER_IMAGE_APP:-${DOCKER_REGISTRY:-ghcr.io/internetstandards}/internet.nl:${RELEASE}}
    networks:
      - internal
    command: migrate
    # this container runs to completion and exits with 0
    restart: on-failure
    logging:
      driver: $LOGGING_DRIVER
      options:
        tag: '{{.Name}}'
    depends_on:
      postgres:
        condition: service_healthy
    environment:
      - RABBIT_HOST=rabbitmq:15672
      - SECRET_KEY
      - GENERATE_SECRET_KEY
      - DB_HOST=postgres
      - DB_NAME=internetnl_db1
      - DB_USER=internetnl
      - DB_PASSWORD=password
      # disable redis cache as it is not used in db migrations
      - CACHE_LOCATION=
      # disable batch checks
      - ENABLE_BATCH=False
      - ENABLE_HOF=False
      - DEBUG
      - DEBUG_LOG
      - DEBUG_LOG_UNBOUND
      - INTERNETNL_LOG_LEVEL
      - INTERNETNL_CACHE_TTL
      - WORKER_CONCURRENCY

  worker: &worker
    platform: linux/amd64
    image: ${DOCKER_IMAGE_APP:-${DOCKER_REGISTRY:-ghcr.io/internetstandards}/internet.nl:${RELEASE}}
    deploy:
      replicas: $WORKER_REPLICAS
    restart: unless-stopped
    mem_limit: $WORKER_MEMORY_LIMIT
    # disable swap by setting swap to the memory limit
    memswap_limit: $WORKER_MEMORY_LIMIT
    logging:
      driver: $LOGGING_DRIVER
      options:
        tag: '{{.Name}}'
    networks:
      - internal
      - public-internet

    # run as root because permission need to be set on batch_results directory, drop privileges in entrypoint.sh
    user: root
    entrypoint: /entrypoint-worker.sh
    command: celery --app internetnl worker --without-gossip --pool=eventlet --time-limit=300 --concurrency=$WORKER_CONCURRENCY
      --queues default,celery,db_worker,ipv6_worker,mail_worker,web_worker,resolv_worker,dnssec_worker,rpki_worker,batch_main,batch_callback,batch_scheduler
    # time after which a SIGKILL is sent to celery after a SIGTERM (warm shutdown), default 10s
    # insufficient short grace period causes issues on batch when tasks are killed during the hourly worker restart
    stop_grace_period: 10m

    depends_on:
      db-migrate:
        # wait for DB migration to be completed
        condition: service_completed_successfully
      redis:
        condition: service_healthy
      rabbitmq:
        condition: service_healthy
      postgres:
        condition: service_healthy
      routinator:
        # TODO: convert to service_healthy when healthcheck is added to routinator
        condition: service_started
        # routinator can sometimes be disabled in profiles
        required: false
      unbound:
        condition: service_healthy
        required: false
      resolver-validating:
        # restart this service when the resolver is restarted and possibly has a new IP, so the old IP is no longer used
        restart: true
        condition: service_healthy
    # set hostname for Sentry
    hostname: worker
    environment:
      - INTERNET_NL_CHECK_SUPPORT_IPV6
      - INTERNET_NL_CHECK_SUPPORT_DNSSEC
      - INTERNET_NL_CHECK_SUPPORT_MAIL
      - INTERNET_NL_CHECK_SUPPORT_TLS
      - INTERNET_NL_CHECK_SUPPORT_APPSECPRIV
      - INTERNET_NL_CHECK_SUPPORT_RPKI
      - PUBLIC_SUFFIX_LIST_URL
      - ENABLE_BATCH
      - ENABLE_HOF
      - RABBIT_HOST=rabbitmq:15672
      - SECRET_KEY
      - GENERATE_SECRET_KEY
      - DB_HOST=postgres
      - DB_NAME=internetnl_db1
      - DB_USER=internetnl
      - DB_PASSWORD=password
      - CELERY_BROKER_URL=amqp://guest:guest@rabbitmq:5672//
      - CELERY_RESULT_BACKEND=redis://redis:6379/0
      - CACHE_LOCATION=redis://redis:6379/0
      - DJANGO_SETTINGS_MODULE=internetnl.settings
      - STATSD_HOST=statsd
      - DEBUG
      - DEBUG_LOG
      - DEBUG_LOG_UNBOUND
      - CELERY_LOG_LEVEL
      - INTEGRATION_TESTS
      - INTERNETNL_LOG_LEVEL
      - INTERNETNL_CACHE_TTL
      - ROUTINATOR_URL
      - CONN_TEST_DOMAIN
      - SMTP_EHLO_DOMAIN
      - INTERNETNL_BRANDING
      - INTERNETNL_DOMAINNAME
      - USER_AGENT_URL
      - SENTRY_DSN
      - SENTRY_ENVIRONMENT
      - SENTRY_SERVER_NAME
      # even though eventlet is used and childs are not really threads or processes, this settings reduces
      # issues with high memory usage, probably because some objects are freed?
      - CELERYD_MAX_TASKS_PER_CHILD=100

    volumes:
      - batch_results:/app/batch_results

    healthcheck:
      test: ["CMD", "/bin/sh", "-c", "celery --app=internetnl inspect ping --destination=celery@$(hostname)"]
      interval: $HEALTHCHECK_INTERVAL
      start_interval: $HEALTHCHECK_START_INTERVAL
      start_period: 1m
      retries: 10

  # worker for queue with potential memory leak
  worker-nassl:
    # copy all attributes from 'worker' service and overwrite specific settings below
    <<: *worker
    mem_limit: $WORKER_NASSL_MEMORY_LIMIT
    # disable swap by settings swap to the memory limit
    memswap_limit: $WORKER_NASSL_MEMORY_LIMIT

    command: celery --app internetnl worker --without-gossip --pool=eventlet --time-limit=300 --concurrency=$WORKER_CONCURRENCY
      --queues nassl_worker,batch_nassl
    # set hostname for Sentry
    hostname: worker-nassl
    # required because current version of nassl cannot be compile for ARM (eg: Apple Silicon)
    platform: linux/amd64

  # worker for slow and long running tasks that could require a lot of memory (eg: hof update)
  worker-slow:
    # copy all attributes from 'worker' service and overwrite specific settings below
    <<: *worker
    mem_limit: $WORKER_SLOW_MEMORY_LIMIT
    # disable swap by settings swap to the memory limit
    memswap_limit: $WORKER_SLOW_MEMORY_LIMIT
    deploy:
      replicas: $WORKER_SLOW_REPLICAS

    command: celery --app internetnl worker --without-gossip --pool=eventlet --time-limit=600 --concurrency=$WORKER_SLOW_CONCURRENCY
      --queues slow_db_worker,batch_slow

  # celery task queue
  beat:
    platform: linux/amd64
    image: ${DOCKER_IMAGE_APP:-${DOCKER_REGISTRY:-ghcr.io/internetstandards}/internet.nl:${RELEASE}}
    restart: unless-stopped
    logging:
      driver: $LOGGING_DRIVER
      options:
        tag: '{{.Name}}'
    mem_limit: $DEFAULT_MEMORY_LIMIT
    # disable swap by settings swap to the memory limit
    memswap_limit: $DEFAULT_MEMORY_LIMIT
    networks:
      - internal
      - public-internet
    entrypoint: celery
    command: --app internetnl beat
    depends_on:
      db-migrate:
        # wait for DB migration to be completed
        condition: service_completed_successfully
      redis:
        condition: service_healthy
      rabbitmq:
        condition: service_healthy
      postgres:
        condition: service_healthy
      routinator:
        # TODO: convert to service_healthy when healthcheck is added to routinator
        condition: service_started
        # routinator can sometimes be disabled in profiles
        required: false
    environment:
      - INTERNET_NL_CHECK_SUPPORT_IPV6
      - INTERNET_NL_CHECK_SUPPORT_DNSSEC
      - INTERNET_NL_CHECK_SUPPORT_MAIL
      - INTERNET_NL_CHECK_SUPPORT_TLS
      - INTERNET_NL_CHECK_SUPPORT_APPSECPRIV
      - INTERNET_NL_CHECK_SUPPORT_RPKI
      - ENABLE_BATCH
      - ENABLE_HOF
      - RABBIT_HOST=rabbitmq:15672
      - SECRET_KEY
      - GENERATE_SECRET_KEY
      - DB_HOST=postgres
      - DB_NAME=internetnl_db1
      - DB_USER=internetnl
      - DB_PASSWORD=password
      - CELERY_BROKER_URL=amqp://guest:guest@rabbitmq:5672//
      - CELERY_RESULT_BACKEND=redis://redis:6379/0
      - CACHE_LOCATION=redis://redis:6379/0
      - DJANGO_SETTINGS_MODULE=internetnl.settings
      - STATSD_HOST=statsd
      - DEBUG
      - DEBUG_LOG
      - DEBUG_LOG_UNBOUND
      - INTEGRATION_TESTS
      - INTERNETNL_LOG_LEVEL
      - INTERNETNL_CACHE_TTL
      - ROUTINATOR_URL
      - CONN_TEST_DOMAIN
      - SMTP_EHLO_DOMAIN
      - BATCH_SCHEDULER_INTERVAL
      - SENTRY_DSN
      - SENTRY_ENVIRONMENT
      - SENTRY_SERVER_NAME
    healthcheck:
      test: ["CMD", "pgrep", "-f", "celery"]
      interval: $HEALTHCHECK_INTERVAL
      start_interval: $HEALTHCHECK_START_INTERVAL
      start_period: 1m
      retries: 10

  # redis caches state, also used for:
  # - MAC address lookup
  # - Django page cache
  # - client DNS resolver IPs in connectiontest
  redis:
    image: ${DOCKER_IMAGE_REDIS}
    restart: unless-stopped
    logging:
      driver: $LOGGING_DRIVER
      options:
        tag: '{{.Name}}'
    mem_limit: $DEFAULT_MEMORY_LIMIT
    # disable swap by settings swap to the memory limit
    memswap_limit: $DEFAULT_MEMORY_LIMIT
    networks:
      internal:
    healthcheck:
      test: ["CMD", "redis-cli","ping"]
      interval: $HEALTHCHECK_INTERVAL
      start_interval: $HEALTHCHECK_START_INTERVAL
      start_period: 1m
      retries: 10
    volumes:
      - redis:/data

  # rabbitmq message-broker
  rabbitmq:
    image: ${DOCKER_IMAGE_RABBITMQ}
    configs:
      - source: rabbitmq_config
        target: /etc/rabbitmq/rabbitmq.conf
        # permissions should be set correctly, temporary workaround until fix is released:
        # https://github.com/docker/compose/pull/12666
        mode: 0444
    restart: unless-stopped
    logging:
      driver: $LOGGING_DRIVER
      options:
        tag: '{{.Name}}'
    mem_limit: $DEFAULT_MEMORY_LIMIT
    # disable swap by settings swap to the memory limit
    memswap_limit: $DEFAULT_MEMORY_LIMIT
    user: rabbitmq
    networks:
      internal:
    ports:
      # expose admin GUI to localhost
      - $RABBITMQ_GUI
    healthcheck:
      test: ["CMD", "rabbitmq-diagnostics", "--quiet", "check_running"]
      interval: $HEALTHCHECK_INTERVAL
      start_interval: $HEALTHCHECK_START_INTERVAL
      # rabbitmq can initially take a long time to start
      start_period: 5m
      retries: 10
    volumes:
      - rabbitmq:/var/lib/rabbitmq

  # database
  postgres:
    image: ${DOCKER_IMAGE_POSTGRES}
    restart: unless-stopped
    logging:
      driver: $LOGGING_DRIVER
      options:
        tag: '{{.Name}}'
    mem_limit: $HIGH_MEMORY_LIMIT
    # disable swap by settings swap to the memory limit
    memswap_limit: $HIGH_MEMORY_LIMIT
    networks:
      internal:
    environment:
      - POSTGRES_USER
      - POSTGRES_PASSWORD
      - POSTGRES_DB
    healthcheck:
      test: pg_isready -U $POSTGRES_USER -d $POSTGRES_DB
      interval: $HEALTHCHECK_INTERVAL
      start_interval: $HEALTHCHECK_START_INTERVAL
      start_period: 1m
      retries: 10
    volumes:
      - postgres:/var/lib/postgresql/data

  # for RPKI
  routinator:
    image: ${DOCKER_IMAGE_ROUTINATOR}
    restart: unless-stopped
    logging:
      driver: $LOGGING_DRIVER
      options:
        tag: '{{.Name}}'
    mem_limit: $HIGH_MEMORY_LIMIT
    # disable swap by settings swap to the memory limit
    memswap_limit: $HIGH_MEMORY_LIMIT
    networks:
      internal:
      public-internet: {}
    volumes:
      - routinator:/home/routinator/.rpki-cache/

    healthcheck:
      # verify routinator webserver is responsive
      test: wget -q -O/dev/null http://127.0.0.1:9556/
      # TODO: should verify the API endpoint status, but because it takes a long time to start
      # needs refactoring like splitting up the api and sync parts
      # test: wget -q -O/dev/null http://127.0.0.1:9556/api/v1/status
      interval: $HEALTHCHECK_INTERVAL
      start_interval: $HEALTHCHECK_START_INTERVAL
      # routinator has to rsync a lot of data initally which can take a long time
      start_period: 30m
      retries: 10

    profiles:
      - routinator

  # unbound DNS server used for connection test
  unbound:
    platform: linux/amd64
    image: ${DOCKER_IMAGE_UNBOUND:-${DOCKER_REGISTRY:-ghcr.io/internetstandards}/unbound:${RELEASE}}
    depends_on:
      redis:
        condition: service_healthy

    restart: unless-stopped
    logging:
      driver: $LOGGING_DRIVER
      options:
        tag: '{{.Name}}'
    mem_limit: $DEFAULT_MEMORY_LIMIT
    # disable swap by settings swap to the memory limit
    memswap_limit: $DEFAULT_MEMORY_LIMIT
    networks:
      internal:
      public-internet: {}
    ports:
      - $UNBOUND_PORT_TCP
      - $UNBOUND_PORT_UDP
      - $UNBOUND_PORT_IPV6_TCP
      - $UNBOUND_PORT_IPV6_UDP

    environment:
      - DEBUG_LOG_UNBOUND
      - IPV4_IP_PUBLIC
      - IPV6_IP_PUBLIC
      - CONN_TEST_DOMAIN

    volumes:
      # stores DNSSEC key information and signed zone files
      - unbound-zones:/opt/unbound/etc/unbound/zones/

    healthcheck:
      test: ["CMD", "unbound-control", "status"]
      interval: $HEALTHCHECK_INTERVAL
      start_interval: $HEALTHCHECK_START_INTERVAL
      start_period: 1m
      retries: 10

  # unbound resolver used for ldns-dane that require DNSSEC validation
  resolver-validating:
    platform: linux/amd64
    image: ${DOCKER_IMAGE_UNBOUND:-${DOCKER_REGISTRY:-ghcr.io/internetstandards}/unbound:${RELEASE}}

    entrypoint: /entrypoint-resolver.sh
    command: ["resolver-validating.conf"]

    restart: unless-stopped
    logging:
      driver: $LOGGING_DRIVER
      options:
        tag: '{{.Name}}'
    mem_limit: $DEFAULT_MEMORY_LIMIT
    # disable swap by settings swap to the memory limit
    memswap_limit: $DEFAULT_MEMORY_LIMIT
    networks:
      internal:
      public-internet: {}

    environment:
      - DEBUG_LOG_UNBOUND
      - INTERNETNL_CACHE_TTL

    healthcheck:
      test: ["CMD", "unbound-control", "-c", "/opt/unbound/etc/unbound/resolver-validating.conf", "status"]
      interval: $HEALTHCHECK_INTERVAL
      start_interval: $HEALTHCHECK_START_INTERVAL
      start_period: 1m
      retries: 10

  # cron with periodic tasks
  cron:
    platform: linux/amd64
    image: ${DOCKER_IMAGE_UTIL:-${DOCKER_REGISTRY:-ghcr.io/internetstandards}/util:${RELEASE}}
    command: crond -f -d7
    environment:
      - HOSTERS_HOF_URL
      - DB_HOST=postgres
      - DB_NAME=internetnl_db1
      - DB_USER=internetnl
      - DB_PASSWORD=password
      - CRON_DAILY_POSTGRESQL_BACKUP
      - CRON_WEEKLY_POSTGRESQL_BACKUP
      - CRON_DAILY_DELETE_BATCH_RESULTS
      - CRON_15MIN_RUN_TESTS
      - CRON_DAILY_TRUNCATE_EXPORTER_LOGS
      - INTERNETNL_DOMAINNAME
      - INTERNETNL_CACHE_TTL
      - TEST_DOMAINS_SITE
      - TEST_DOMAINS_MAIL

    restart: unless-stopped
    logging:
      driver: $LOGGING_DRIVER
      options:
        tag: '{{.Name}}'
    mem_limit: $DEFAULT_MEMORY_LIMIT
    # disable swap by settings swap to the memory limit
    memswap_limit: $DEFAULT_MEMORY_LIMIT
    networks:
      internal: {}
      public-internet: {}


    volumes:
      - manual-hof:/app/manual-hall-of-fame/
      - postgres-backups:/var/lib/postgresql/backups
      - nginx-logs-exporter:/var/log/nginx/prometheus-nginxlog-exporter/
      - prometheus-textfile-directory:/prometheus-textfile-directory
      - batch_results:/app/batch_results

    healthcheck:
      test: ["CMD", "pgrep", "crond"]
      interval: $HEALTHCHECK_INTERVAL
      start_interval: $HEALTHCHECK_START_INTERVAL
      start_period: 1m
      retries: 10

  # cron daemon with access to Docker socket but no networking
  cron-docker:
    platform: linux/amd64
    image: ${DOCKER_IMAGE_UTIL:-${DOCKER_REGISTRY:-ghcr.io/internetstandards}/util:${RELEASE}}
    command: crond -f -d7 -c /etc/crontabs-docker
    environment:
      - AUTO_UPDATE_TO
      - WORKER_SLOW_REPLICAS
      - DOCKER_REGISTRY
      - CRON_DAILY_DATABASE_CLEANUP
      - CRON_WORKER_RESTART
      - INTERNETNL_INSTALL_BASE
    restart: unless-stopped
    logging:
      driver: $LOGGING_DRIVER
      options:
        tag: '{{.Name}}'
    mem_limit: $DEFAULT_MEMORY_LIMIT
    # disable swap by settings swap to the memory limit
    memswap_limit: $DEFAULT_MEMORY_LIMIT
    network_mode: "none"
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - $INTERNETNL_INSTALL_BASE:/opt/Internet.nl

  grafana:
    platform: linux/amd64
    image: ${DOCKER_IMAGE_GRAFANA:-${DOCKER_REGISTRY:-ghcr.io/internetstandards}/grafana:${RELEASE}}

    environment:
      - GF_AUTH_ANONYMOUS_ENABLED=true
      - GF_AUTH_ANONYMOUS_ORG_NAME=Main Org.
      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
      - GF_AUTH_BASIC_ENABLED=false
      - GF_AUTH_DISABLE_LOGIN_FORM=true
      - GF_AUTH_DISABLE_SIGNOUT_MENU=true
      - GF_USERS_ALLOW_SIGN_UP=false
      - GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/etc/dashboards/home.json
      - GF_SERVER_ROOT_URL=http://localhost:8080/grafana/
      - GF_SERVER_SERVE_FROM_SUB_PATH=true
      # disable grafana from trying to reach out and filling logging with failed DNS queries
      - GF_ANALYTICS_CHECK_FOR_UPDATES=false
      - GF_ANALYTICS_REPORTING_ENABLED=false
      - GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES=false
      # disables live functionality which is not used, because it uses websockets which don't play well with authentication
      - GF_LIVE_MAX_CONNECTIONS=0
      - INTERNETNL_DOMAINNAME

    volumes:
      - grafana-data:/var/lib/grafana

    restart: unless-stopped
    logging:
      driver: $LOGGING_DRIVER
      options:
        tag: '{{.Name}}'
    mem_limit: $DEFAULT_MEMORY_LIMIT
    # disable swap by settings swap to the memory limit
    memswap_limit: $DEFAULT_MEMORY_LIMIT
    networks:
      internal:

    profiles:
      - monitoring

  prometheus:
    image: ${DOCKER_IMAGE_PROMETHEUS}

    command:
      - --config.file=/prometheus.yaml
      - --web.external-url=/prometheus/
      - --storage.tsdb.retention.time=$PROMETHEUS_RETENTION_TIME
      - --storage.tsdb.retention.size=$PROMETHEUS_RETENTION_SIZE
      - --web.enable-admin-api

    configs:
      - source: prometheus_config
        target: /prometheus.yaml
        # permissions should be set correctly, temporary workaround until fix is released:
        # https://github.com/docker/compose/pull/12666
        mode: 0444
      - source: prometheus_rules_config
        target: /prometheus-rules.yaml
        # permissions should be set correctly, temporary workaround until fix is released:
        # https://github.com/docker/compose/pull/12666
        mode: 0444

    restart: unless-stopped
    logging:
      driver: $LOGGING_DRIVER
      options:
        tag: '{{.Name}}'
    mem_limit: $DEFAULT_MEMORY_LIMIT
    # disable swap by settings swap to the memory limit
    memswap_limit: $DEFAULT_MEMORY_LIMIT
    networks:
      internal:

    volumes:
      - prometheus-data:/prometheus

    profiles:
      - monitoring

  # requires monitoring profile
  alertmanager:
    image: ${DOCKER_IMAGE_PROMETHEUS_ALERTMANAGER}

    command:
      - --config.file=/alertmanager.yaml
      - --web.external-url=https://$INTERNETNL_DOMAINNAME/alertmanager/
      - --cluster.listen-address=

    configs:
      - source: alertmanager_config
        target: /alertmanager.yaml
        # permissions should be set correctly, temporary workaround until fix is released:
        # https://github.com/docker/compose/pull/12666
        mode: 0444

    restart: unless-stopped
    logging:
      driver: $LOGGING_DRIVER
      options:
        tag: '{{.Name}}'
    networks:
      internal:
      public-internet: {}

    profiles:
      - alertmanager

  postgresql-exporter:
    image: ${DOCKER_IMAGE_POSTGRESQL_EXPORTER}

    environment:
      - DATA_SOURCE_NAME=postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres:5432/$POSTGRES_DB?sslmode=disable

    restart: unless-stopped
    logging:
      driver: $LOGGING_DRIVER
      options:
        tag: '{{.Name}}'
    mem_limit: $DEFAULT_MEMORY_LIMIT
    # disable swap by settings swap to the memory limit
    memswap_limit: $DEFAULT_MEMORY_LIMIT
    networks:
      - internal

    profiles:
      - monitoring

  redis-exporter:
    image: ${DOCKER_IMAGE_REDIS_EXPORTER}

    environment:
      - REDIS_ADDR=redis://redis:6379

    restart: unless-stopped
    logging:
      driver: $LOGGING_DRIVER
      options:
        tag: '{{.Name}}'
    mem_limit: $DEFAULT_MEMORY_LIMIT
    # disable swap by settings swap to the memory limit
    memswap_limit: $DEFAULT_MEMORY_LIMIT
    networks:
      - internal

    profiles:
      - monitoring

  statsd-exporter:
    image: ${DOCKER_IMAGE_STATSD_EXPORTER}

    command:
      - --statsd.listen-udp=:8125
      - --statsd.listen-tcp=:8125

    restart: unless-stopped
    logging:
      driver: $LOGGING_DRIVER
      options:
        tag: '{{.Name}}'
    mem_limit: $DEFAULT_MEMORY_LIMIT
    # disable swap by settings swap to the memory limit
    memswap_limit: $DEFAULT_MEMORY_LIMIT
    networks:
      internal:
        aliases:
          - statsd

    profiles:
      - monitoring

  celery-exporter:
    platform: linux/amd64
    image: ${DOCKER_IMAGE_CELERY_EXPORTER}
    command:
      - --broker-url=amqp://guest:guest@rabbitmq:5672/
      - --accept-content=json,pickle
      # retry connecting to broker on failure
      - --retry-interval=10
    restart: unless-stopped
    logging:
      driver: $LOGGING_DRIVER
      options:
        tag: '{{.Name}}'
    mem_limit: $DEFAULT_MEMORY_LIMIT
    # disable swap by settings swap to the memory limit
    memswap_limit: $DEFAULT_MEMORY_LIMIT
    networks:
      - internal
    depends_on:
      rabbitmq:
        condition: service_healthy

    profiles:
      - monitoring

  node-exporter:
    # https://github.com/prometheus/node_exporter#docker
    image: ${DOCKER_IMAGE_NODE_EXPORTER}
    command:
      - --path.rootfs=/host
      - --collector.systemd
      # disable metrics about the exporter itself
      - --web.disable-exporter-metrics
      # ignore docker container interfaces
      - --collector.netdev.device-exclude=veth
      # ignore docker container interfaces
      - --collector.netclass.ignored-devices=veth
      - --collector.textfile
      - --collector.textfile.directory=/prometheus-textfile-directory
    restart: unless-stopped
    logging:
      driver: $LOGGING_DRIVER
      options:
        tag: '{{.Name}}'
    mem_limit: $DEFAULT_MEMORY_LIMIT
    # disable swap by settings swap to the memory limit
    memswap_limit: $DEFAULT_MEMORY_LIMIT
    pid: host
    networks:
      - internal
    volumes:
      - /:/host:ro
      - /var/run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket
      - prometheus-textfile-directory:/prometheus-textfile-directory

    profiles:
      - monitoring

  docker_stats_exporter:
    # https://github.com/jan4843/docker_stats_exporter
    platform: linux/amd64
    image: ${DOCKER_IMAGE_DOCKER_STATSD_EXPORTER}
    environment:
      LABEL_state: '{{.Container.State}}'
      LABEL_compose_project: '{{index .Container.Labels "com.docker.compose.project"}}'
    restart: unless-stopped
    logging:
      driver: $LOGGING_DRIVER
      options:
        tag: '{{.Name}}'
    mem_limit: $DEFAULT_MEMORY_LIMIT
    # disable swap by settings swap to the memory limit
    memswap_limit: $DEFAULT_MEMORY_LIMIT
    networks:
      - internal
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock

    profiles:
      - monitoring

  nginx_logs_exporter:
    platform: linux/amd64
    image: ${DOCKER_IMAGE_NGINX_LOGS_EXPORTER}
    command:
      - -config-file=/config.hcl
    configs:
      - source: nginx_logs_exporter_config
        target: /config.hcl
        # permissions should be set correctly, temporary workaround until fix is released:
        # https://github.com/docker/compose/pull/12666
        mode: 0444

    restart: unless-stopped
    logging:
      driver: $LOGGING_DRIVER
      options:
        tag: '{{.Name}}'
    mem_limit: $DEFAULT_MEMORY_LIMIT
    # disable swap by settings swap to the memory limit
    memswap_limit: $DEFAULT_MEMORY_LIMIT
    networks:
      - internal
    volumes:
      - nginx-logs-exporter:/var/log/nginx/prometheus-nginxlog-exporter/

    profiles:
      - monitoring

volumes:
  postgres: {}
  postgres-backups: {}
  redis: {}
  rabbitmq: {}
  routinator: {}
  batch_results: {}
  certbot-config: {}
  htpasswd-files: {}
  unbound-zones: {}
  # permanent storage for Prometheus metrics
  prometheus-data: {}
  # prometheus metric textfile shared volume
  prometheus-textfile-directory: {}
  # permanent storage for Grafana custom dashboards
  grafana-data: {}
  # shares hosters HoF file between cron and app
  manual-hof: {}
  # shares nginx log files with log exporter
  nginx-logs-exporter: {}

configs:
  nginx_logs_exporter_config:
    content: |
      namespace "nginx" {
        format = "$$remote_addr $$host $$remote_user [$$time_local] $$proxy_host \"$$request\" $$status $$body_bytes_sent \"$$http_referer\" \"$$http_user_agent\" \"$$http_x_forwarded_for\""

        source = {
          files = ["/var/log/nginx/prometheus-nginxlog-exporter/access.log"]
        }

        relabel "proxy_host" { from = "proxy_host" }
        relabel "remote_user" { from = "remote_user" }
      }
  rabbitmq_config:
    content: |
      # RabbitMQ requires messages to be acked in 30 minutes or else it will mark  a client as stale. This conflicts
      # with Celery tasks that have a countdown or eta set beyond 30 minutes as they are not acked before this time.
      # Setting this timeout to 1 year here to work around this issue. Related error and info:
      # amqp.exceptions.PreconditionFailed: (0, 0): (406) PRECONDITION_FAILED - consumer ack timed out on channel 1
      # https://github.com/celery/celery/issues/6760
      consumer_timeout = 31622400000
  prometheus_config:
    content: |
      global:
        scrape_interval: 10s
        scrape_timeout: 5s
      rule_files:
        - /prometheus-rules.yaml
      alerting:
        alertmanagers:
          - path_prefix: /alertmanager
            static_configs:
              - targets:
                  - alertmanager:9093
      scrape_configs:
        - &scrape_config
          scheme: http
          metrics_path: /metrics
          metric_relabel_configs:
            # drop exporter internal metrics
            - source_labels: [ "__name__" ]
              regex: '(go|python|process)_.+'
              action: drop
          job_name: postgresql_exporter
          static_configs: [{targets: ["postgresql-exporter:9187"]}]
        - <<: *scrape_config
          job_name: redis_exporter
          static_configs: [{targets: ["redis-exporter:9121"]}]
        - <<: *scrape_config
          job_name: rabbitmq_exporter
          static_configs: [{targets: ["rabbitmq:15692"]}]
        - <<: *scrape_config
          job_name: statsd_exporter
          static_configs: [{targets: ["statsd-exporter:9102"]}]
        - <<: *scrape_config
          job_name: celery_exporter
          static_configs: [{targets: ["celery-exporter:9808"]}]
        - <<: *scrape_config
          job_name: node_exporter
          static_configs: [{targets: ["node-exporter:9100"]}]
        - <<: *scrape_config
          job_name: docker_stats_exporter
          static_configs: [{targets: ["docker_stats_exporter:9338"]}]
        - <<: *scrape_config
          job_name: nginx_logs_exporter
          static_configs: [{targets: ["nginx_logs_exporter:4040"]}]
  prometheus_rules_config:
    content: |
      groups:
      - name: End to end monitoring
        rules:
        - alert: HighTestRuntimeSite
          # when site probes for 2 or more of the test domains take longer than 30 seconds something is wrong
          expr: count(tests_test_runtime_seconds{test="site"} >= 30) >= 2
          annotations:
            host: $INTERNETNL_DOMAINNAME
            summary: Two or more tests for web take longer to complete than expected
            dashboard: 'https://$INTERNETNL_DOMAINNAME/grafana/d/af7d1d82-c0f9-4d8d-bc03-542c4c4c75c0/periodic-tests'
        - alert: StaleTestResults
          # when the metrics for the periodic tests have not been updated for an hour
          expr: time() - node_textfile_mtime_seconds > 3600
          annotations:
            host: $INTERNETNL_DOMAINNAME
            summary: Two or more tests for web take longer to complete than expected
            dashboard: 'https://$INTERNETNL_DOMAINNAME/grafana/d/af7d1d82-c0f9-4d8d-bc03-542c4c4c75c0/periodic-tests'
        - alert: HighTestRuntimeMail
          # when mail probes for 2 or more of the oltest domains take longer than 70 seconds something is wrong
          expr: count(tests_test_runtime_seconds{test="site"} >= 70) >= 2
          annotations:
            host: $INTERNETNL_DOMAINNAME
            summary: Two or more tests for mail take longer to complete than expected
            dashboard: 'https://$INTERNETNL_DOMAINNAME/grafana/d/af7d1d82-c0f9-4d8d-bc03-542c4c4c75c0/periodic-tests'
        - alert: OOMKills
          expr: increase(node_vmstat_oom_kill[5m]) > 1
          annotations:
            host: $INTERNETNL_DOMAINNAME
            summary: One or more containers/processes where OOM killed
            dashboard: 'https://$INTERNETNL_DOMAINNAME/grafana/d/bdd9dac3-b85a-4420-8158-bd92e06da08d/batch?viewPanel=5'

  alertmanager_config:
    content: |
      global:
        smtp_from: $ALERTMANAGER_MAIL_FROM
        smtp_smarthost: $ALERTMANAGER_SMTP_HOST:$ALERTMANAGER_SMTP_PORT
        smtp_require_tls: true
        smtp_auth_username: $ALERTMANAGER_SMTP_USER
        smtp_auth_password: $ALERTMANAGER_SMTP_PASSWORD
      route:
        receiver: alerts
        routes:
          - receiver: alerts
      receivers:
        - name: alerts
          email_configs:
            - to: $ALERTMANAGER_MAIL_TO
              headers:
                subject: $ALERTMANAGER_SUBJECT

networks:
  # disable default network
  default:
    driver: none
  internal:
    internal: true
    driver: bridge
    # no IPv6 required
    enable_ipv6: false
    ipam:
      driver: default
      config:
      - subnet: $IPV4_SUBNET_INTERNAL

  # allows connected services to access public internet
  public-internet:
    # required to enable IPv6 on Docker Desktop runtime
    enable_ipv6: true
    driver: bridge
    driver_opts:
      # required to enable IPv6 on Colima Docker runtime
      com.docker.network.enable_ipv6: "true"
      # network for internal communication between services
      com.docker.network.bridge.enable_icc: "true"
    ipam:
      driver: default
      config:
      - subnet: $IPV6_SUBNET_PUBLIC
        gateway: $IPV6_GATEWAY_PUBLIC
      - subnet: $IPV4_SUBNET_PUBLIC