docker-compose.yml

#####
# Docker Compose file to start a full ESGF node
#
# Required environment variables:
#   ESGF_CONFIG : Directory containing configuration
#   ESGF_HOSTNAME: Host name for the deployment
#   ESGF_DATA: Directory containing ESGF data files
###

version: '3.4'


# Use a YAML anchor to define common properties
#   The "x-" prefix means Compose will treat this element as an extension and not throw a syntax error
x-esgf-properties: &esgf_properties
  ESGF_HOSTNAME:
  ESGF_PEER_GROUP: esgf-test
  ESGF_ATTRIBUTE_SERVICE_ENDPOINT: "https://$ESGF_HOSTNAME/esgf-idp/saml/soap/secure/attributeService.htm"
  ESGF_REGISTRATION_SERVICE_ENDPOINT: "https://$ESGF_HOSTNAME/esgf-idp/secure/registrationService.htm"
  ESGF_OPENID_ENDPOINT: "https://$ESGF_HOSTNAME/esgf-idp/idp/openidServer.htm"
  ESGF_DATABASE_HOST: esgf-postgres-security
  ESGF_DATABASE_PORT: "5432"
  ESGF_DATABASE_USER: dbsuper
  ESGF_DATABASE_NAME: esgcet
  ESGF_SEARCH_SERVICE_ENDPOINT: "https://$ESGF_HOSTNAME/esg-search/search"
  ESGF_IDP_PEER: $ESGF_HOSTNAME
  ESGF_INDEX_PEER: $ESGF_HOSTNAME
  ESGF_PUBLISH_SERVICE_ENDPOINT: "https://$ESGF_HOSTNAME/esg-search/remote/secure/client-cert/hessian/publishingService"
  ESGF_SOLR_QUERY_URL: "http://esgf-solr-slave:8983/solr"
  ESGF_SOLR_PUBLISH_URL: "http://esgf-solr-master:8983/solr"
  ESGF_AUTHORIZATION_SERVICE_ENDPOINT: "https://$ESGF_HOSTNAME/esg-orp/saml/soap/secure/authorizationService.htm"
  ESGF_SLCS_ENDPOINT: "https://$ESGF_HOSTNAME/esgf-slcs"


services:
  # The publisher is included here so it can be used with docker-compose run
  # The command is modified to a simple quit with 0 exit status
  esgf-publisher:
    image: "${ESGF_HUB}/${ESGF_PREFIX}publisher:${ESGF_VERSION}"
    environment:
      ESGF_DATABASE_HOST: esgf-postgres-esgcet
      ESGF_DATABASE_PORT: "5432"
      ESGF_DATABASE_USER: esgcet
      ESGF_DATABASE_PASSWORD_FILE: /esg/secrets/esgcet-database-password
      ESGF_DATABASE_NAME: esgcet
      ESGF_HESSIAN_METADATA_URL: "https://${ESGF_HOSTNAME}/esgcet/remote/hessian/guest/remoteMetadataService"
      ESGF_HESSIAN_URL: "https://${ESGF_HOSTNAME}/esg-search/remote/secure/client-cert/hessian/publishingService"
      ESGF_TDS_CATALOG_URL: "https://${ESGF_HOSTNAME}/thredds/catalog/esgcet"
      ESGF_TDS_REINIT_URL: "https://${ESGF_HOSTNAME}/thredds/admin/debug?Catalogs/recheck"
      ESGF_TDS_REINIT_ERROR_URL: "https://${ESGF_HOSTNAME}/thredds/admin/content/logs/catalogInit.log"
      ESGF_TDS_USERNAME: rootAdmin
      ESGF_TDS_PASSWORD_FILE: /esg/secrets/rootadmin-password
      ESGF_SLCS_CERTIFICATE_URL: "https://${ESGF_HOSTNAME}/esgf-slcs/onlineca/certificate/"
    volumes:
      - "$ESGF_DATA:/esg/data"
      - "tds-content:/esg/content/thredds/esgcet"
      # Mount only the secrets we actually need
      - "$ESGF_CONFIG/secrets/esgcet-database-password:/esg/secrets/esgcet-database-password:ro"
      - "$ESGF_CONFIG/secrets/rootadmin-password:/esg/secrets/rootadmin-password:ro"
      # Make sure the trusted certificate bundle is available
      - "$ESGF_CONFIG/certificates/esg-trust-bundle.pem:/esg/certificates/esg-trust-bundle.pem:ro"
      # Allow the publisher configuration to be overridden
      - "$ESGF_CONFIG/publisher:/esg/config/esgcet/.overrides:ro"
    # Use a command that exits immediately with success, but don't change the entrypoint
    command: ["true"]
    depends_on:
      - esgf-postgres-esgcet
      - esgf-index-node
      - esgf-idp-node
      - esgf-orp
      - esgf-tds

  esgf-proxy:
    image: "${ESGF_HUB}/${ESGF_PREFIX}proxy:${ESGF_VERSION}"
    restart: always
    ports:
      - "80:80"
      - "443:443"
    volumes:
      - "$ESGF_CONFIG/certificates/hostcert:/etc/nginx/ssl:ro"
      # Make sure the trusted certificate bundle is available
      - "$ESGF_CONFIG/certificates/esg-trust-bundle.pem:/esg/certificates/esg-trust-bundle.pem:ro"
    environment:
      ESGF_HOSTNAME:
      ESGF_SOLR_UPSTREAM: http://esgf-solr-slave:8983
      ESGF_INDEX_NODE_UPSTREAM: http://esgf-index-node:8080
      ESGF_TDS_UPSTREAM: http://esgf-tds:8080
      ESGF_COG_UPSTREAM: http://esgf-cog:8000
      ESGF_ORP_UPSTREAM: http://esgf-orp:8080
      ESGF_IDP_UPSTREAM: http://esgf-idp-node:8080
      ESGF_SLCS_UPSTREAM: http://esgf-slcs:8000
      ESGF_AUTH_UPSTREAM: http://esgf-auth:8000
    depends_on:
      - esgf-solr-slave
      - esgf-index-node
      - esgf-tds
      - esgf-orp
      - esgf-idp-node
      - esgf-slcs
      - esgf-auth
      - esgf-cog

  esgf-solr-master:
    image: "${ESGF_HUB}/${ESGF_PREFIX}solr:${ESGF_VERSION}"
    restart: always
    environment:
      ESGF_SOLR_ROLE: master
    volumes:
      - "solr-master-home:/esg/solr-home"

  esgf-solr-slave:
    image: "${ESGF_HUB}/${ESGF_PREFIX}solr:${ESGF_VERSION}"
    restart: always
    environment:
      ESGF_SOLR_MASTER_URL: http://esgf-solr-master:8983/solr
      # Use a shorter replication interval for the local slave than the default
      ESGF_SOLR_REPLICATION_INTERVAL: "00:00:60"
    volumes:
      - "solr-slave-home:/esg/solr-home"
    depends_on:
      - esgf-solr-master

  esgf-postgres-esgcet:
    image: "${ESGF_HUB}/${ESGF_PREFIX}postgres:${ESGF_VERSION}"
    restart: always
    environment:
      POSTGRESQL_DATABASE: esgcet
      POSTGRESQL_USER: esgcet
      POSTGRESQL_PASSWORD_FILE: /esg/secrets/esgcet-database-password
    volumes:
      - "$ESGF_CONFIG/secrets/esgcet-database-password:/esg/secrets/esgcet-database-password:ro"
      - "postgres-esgcet-data:/var/lib/pgsql/data"

  esgf-postgres-security:
    image: "${ESGF_HUB}/${ESGF_PREFIX}postgres-security:${ESGF_VERSION}"
    restart: always
    environment:
      POSTGRESQL_DATABASE: esgcet
      POSTGRESQL_USER: dbsuper
      POSTGRESQL_PASSWORD_FILE: /esg/secrets/security-database-password
      ESGF_ROOTADMIN_EMAIL: "CoG@${ESGF_HOSTNAME}"
      ESGF_ROOTADMIN_USERNAME: rootAdmin
      ESGF_ROOTADMIN_OPENID: https://${ESGF_HOSTNAME}/esgf-idp/openid/rootAdmin
      ESGF_ROOTADMIN_PASSWORD_FILE: /esg/secrets/rootadmin-password
    volumes:
      # Mount only the secrets we actually need
      - "$ESGF_CONFIG/secrets/security-database-password:/esg/secrets/security-database-password:ro"
      - "$ESGF_CONFIG/secrets/rootadmin-password:/esg/secrets/rootadmin-password:ro"
      - "postgres-security-data:/var/lib/pgsql/data"

  esgf-orp:
    image: "${ESGF_HUB}/${ESGF_PREFIX}orp:${ESGF_VERSION}"
    restart: always
    environment:
      <<: *esgf_properties
      ESGF_KEYSTORE_ALIAS: $ESGF_HOSTNAME
      ESGF_SAML_CERT_FILE: /esg/certificates/hostcert/hostcert.crt
      ESGF_SAML_KEY_FILE: /esg/certificates/hostcert/hostcert.key
    volumes:
      # Mount the host certificate and key for SAML signing
      - "$ESGF_CONFIG/certificates/hostcert:/esg/certificates/hostcert:ro"
      # Make sure the trusted certificate bundle is available
      - "$ESGF_CONFIG/certificates/esg-trust-bundle.pem:/esg/certificates/esg-trust-bundle.pem:ro"
      # Allow the main ESGF configuration to be overridden
      - "$ESGF_CONFIG/config:/esg/config/.overrides:ro"

  esgf-index-node:
    image: "${ESGF_HUB}/${ESGF_PREFIX}index-node:${ESGF_VERSION}"
    restart: always
    environment:
      <<: *esgf_properties
      ESGF_SOLR_SHARDS: "esgf-solr-slave:8983/solr"
    volumes:
      # Make sure the trusted certificate bundle is available
      - "$ESGF_CONFIG/certificates/esg-trust-bundle.pem:/esg/certificates/esg-trust-bundle.pem:ro"
      # Allow the main ESGF configuration to be overridden
      - "$ESGF_CONFIG/config:/esg/config/.overrides:ro"
    depends_on:
      - esgf-solr-master
      - esgf-solr-slave
      - esgf-orp

  esgf-idp-node:
    image: "${ESGF_HUB}/${ESGF_PREFIX}idp-node:${ESGF_VERSION}"
    restart: always
    environment:
      <<: *esgf_properties
      ESGF_DATABASE_PASSWORD_FILE: /esg/config/.esg_pg_pass
    volumes:
      - "$ESGF_CONFIG/secrets/security-database-password:/esg/config/.esg_pg_pass:ro"
      - "$ESGF_CONFIG/secrets/rootadmin-password:/esg/config/.esgf_pass:ro"
      # Make sure the trusted certificate bundle is available
      - "$ESGF_CONFIG/certificates/esg-trust-bundle.pem:/esg/certificates/esg-trust-bundle.pem:ro"
      # Allow the main ESGF configuration to be overridden
      - "$ESGF_CONFIG/config:/esg/config/.overrides:ro"
    depends_on:
      - esgf-postgres-security

  esgf-tds:
    image: "${ESGF_HUB}/${ESGF_PREFIX}tds:${ESGF_VERSION}"
    environment:
      <<: *esgf_properties
      ESGF_ORP_HOME_ENDPOINT: "https://$ESGF_HOSTNAME/esg-orp/home.htm"
      ESGF_AUTH_HOME_ENDPOINT: "https://$ESGF_HOSTNAME/esgf-auth/home/"
      ESGF_REGISTRATION_RELAY_ENDPOINT: "https://$ESGF_HOSTNAME/esg-orp/registration-request.htm"
      ESGF_COOKIE_SECRET_KEY_FILE: /esg/secrets/shared-cookie-secret-key
      ESGF_TDS_ADMIN_USERNAME: rootAdmin
      ESGF_TDS_ADMIN_PASSWORD_FILE: /esg/config/.esgf_pass
    volumes:
      - "tds-content:/esg/content/thredds/esgcet:ro"
      - "$ESGF_DATA:/esg/data:ro"
      # Mount only the secrets we actually need
      - "$ESGF_CONFIG/secrets/shared-cookie-secret-key:/esg/secrets/shared-cookie-secret-key:ro"
      - "$ESGF_CONFIG/secrets/rootadmin-password:/esg/config/.esgf_pass:ro"
      # Make sure the trusted certificate bundle is available
      - "$ESGF_CONFIG/certificates/esg-trust-bundle.pem:/esg/certificates/esg-trust-bundle.pem:ro"
      # Allow the main ESGF configuration to be overridden
      - "$ESGF_CONFIG/config:/esg/config/.overrides:ro"
      # Allow the THREDDS config XML and root catalog to be overridden
      # Note that this is separate to the esgcet catalog being generated by the publisher
      - "$ESGF_CONFIG/thredds:/esg/content/thredds/.overrides:ro"

  esgf-postgres-cog:
    image: "${ESGF_HUB}/${ESGF_PREFIX}postgres:${ESGF_VERSION}"
    restart: always
    environment:
      POSTGRESQL_DATABASE: cogdb
      POSTGRESQL_USER: dbsuper
      POSTGRESQL_PASSWORD_FILE: /esg/secrets/cog-database-password
    volumes:
      - "$ESGF_CONFIG/secrets/cog-database-password:/esg/secrets/cog-database-password:ro"
      - "postgres-cog-data:/var/lib/pgsql/data"

  esgf-cog:
    image: "${ESGF_HUB}/${ESGF_PREFIX}cog:${ESGF_VERSION}"
    restart: always
    environment:
      <<: *esgf_properties
      ESGF_COG_SITE_NAME: "Local CoG"
      ESGF_COG_HOME_PROJECT: TestProject
      ESGF_COG_TIME_ZONE: "Europe/London"
      ESGF_COG_SECRET_KEY_FILE: /esg/secrets/cog-secret-key
      ESGF_COG_DATABASE_HOST: esgf-postgres-cog
      ESGF_COG_DATABASE_PORT: "5432"
      ESGF_COG_DATABASE_NAME: cogdb
      ESGF_COG_DATABASE_USER: dbsuper
      ESGF_COG_DATABASE_PASSWORD_FILE: /esg/secrets/cog-database-password
    volumes:
      - "$ESGF_CONFIG/secrets/security-database-password:/esg/config/.esg_pg_pass:ro"
      - "$ESGF_CONFIG/secrets/cog-database-password:/esg/secrets/cog-database-password:ro"
      - "$ESGF_CONFIG/secrets/rootadmin-password:/esg/config/.esgf_pass:ro"
      - "$ESGF_CONFIG/secrets/cog-secret-key:/esg/secrets/cog-secret-key:ro"
      # Make sure the trusted certificate bundle is available
      - "$ESGF_CONFIG/certificates/esg-trust-bundle.pem:/esg/certificates/esg-trust-bundle.pem:ro"
      # Allow the main ESGF configuration to be overridden
      - "$ESGF_CONFIG/config:/esg/config/.overrides:ro"
    depends_on:
      - esgf-postgres-security
      - esgf-postgres-cog

  esgf-postgres-auth:
    image: "${ESGF_HUB}/${ESGF_PREFIX}postgres:${ESGF_VERSION}"
    restart: always
    environment:
      POSTGRESQL_DATABASE: auth
      POSTGRESQL_USER: authuser
      POSTGRESQL_PASSWORD_FILE: /esg/secrets/auth-database-password
    volumes:
      - "$ESGF_CONFIG/secrets/auth-database-password:/esg/secrets/auth-database-password:ro"
      - "postgres-auth-data:/var/lib/pgsql/data"

  esgf-auth:
    image: "${ESGF_HUB}/${ESGF_PREFIX}auth:${ESGF_VERSION}"
    restart: always
    environment:
      <<: *esgf_properties
      SCRIPT_NAME: /esgf-auth
      ESGF_AUTH_SECRET_KEY_FILE: /esg/secrets/auth-secret-key
      ESGF_COOKIE_SECRET_KEY_FILE: /esg/secrets/shared-cookie-secret-key
      # Database settings
      DJANGO_DATABASE_DEFAULT_ENGINE: django.db.backends.postgresql
      DJANGO_DATABASE_DEFAULT_NAME: auth
      DJANGO_DATABASE_DEFAULT_HOST: esgf-postgres-auth
      DJANGO_DATABASE_DEFAULT_PORT: "5432"
      DJANGO_DATABASE_DEFAULT_USER: authuser
      DJANGO_DATABASE_DEFAULT_PASSWORD_FILE: /esg/secrets/auth-database-password
    volumes:
      - "$ESGF_CONFIG/secrets/auth-secret-key:/esg/secrets/auth-secret-key:ro"
      - "$ESGF_CONFIG/secrets/shared-cookie-secret-key:/esg/secrets/shared-cookie-secret-key:ro"
      - "$ESGF_CONFIG/secrets/auth-database-password:/esg/secrets/auth-database-password:ro"
      # Make sure the trusted certificate bundle is available
      - "$ESGF_CONFIG/certificates/esg-trust-bundle.pem:/esg/certificates/esg-trust-bundle.pem:ro"
      # Allow the main ESGF configuration to be overridden
      - "$ESGF_CONFIG/config:/esg/config/.overrides:ro"
      # Allow the ESGF auth configuration to be overridden
      # In particular, this allows an esgf_oauth2.json to be dropped in with OAuth2 credentials
      - "$ESGF_CONFIG/auth:/esg/auth/.overrides:ro"
    depends_on:
      - esgf-postgres-auth

  esgf-postgres-slcs:
    image: "${ESGF_HUB}/${ESGF_PREFIX}postgres:${ESGF_VERSION}"
    restart: always
    environment:
      POSTGRESQL_DATABASE: slcs
      POSTGRESQL_USER: slcsuser
      POSTGRESQL_PASSWORD_FILE: /esg/secrets/slcs-database-password
    volumes:
      - "$ESGF_CONFIG/secrets/slcs-database-password:/esg/secrets/slcs-database-password:ro"
      - "postgres-slcs-data:/var/lib/pgsql/data"

  esgf-slcs:
    image: "${ESGF_HUB}/${ESGF_PREFIX}slcs:${ESGF_VERSION}"
    restart: always
    environment:
      # Generic Django settings
      SCRIPT_NAME: /esgf-slcs
      DJANGO_ALLOWED_HOSTS: $ESGF_HOSTNAME
      DJANGO_SECRET_KEY_FILE: /esg/secrets/slcs-secret-key
      # SLCS database
      DJANGO_DATABASE_DEFAULT_ENGINE: django.db.backends.postgresql
      DJANGO_DATABASE_DEFAULT_NAME: slcs
      DJANGO_DATABASE_DEFAULT_HOST: esgf-postgres-slcs
      DJANGO_DATABASE_DEFAULT_PORT: "5432"
      DJANGO_DATABASE_DEFAULT_USER: slcsuser
      DJANGO_DATABASE_DEFAULT_PASSWORD_FILE: /esg/secrets/slcs-database-password
      # ESGF User database
      DJANGO_DATABASE_USERDB_ENGINE: django.db.backends.postgresql
      DJANGO_DATABASE_USERDB_NAME: esgcet
      DJANGO_DATABASE_USERDB_HOST: esgf-postgres-security
      DJANGO_DATABASE_USERDB_PORT: "5432"
      DJANGO_DATABASE_USERDB_USER: dbsuper
      DJANGO_DATABASE_USERDB_PASSWORD_FILE: /esg/secrets/security-database-password
      # Make the rootAdmin account a superuser
      DJANGO_CREATE_SUPERUSER: "1"
      DJANGO_SUPERUSER_USERNAME: "rootAdmin"
      DJANGO_SUPERUSER_EMAIL: "CoG@${ESGF_HOSTNAME}"
      # Other SLCS-specific settings
      ESGF_SLCS_URL: "https://${ESGF_HOSTNAME}/esgf-slcs"
      ESGF_SLCS_BASIC_AUTH_REALM: $ESGF_HOSTNAME
      ONLINECA_SUBJECT_NAME_TEMPLATE: "/DC=esgf/CN={user}"
      ONLINECA_CERT_PATH: /esg/certificates/slcsca/ca.crt
      ONLINECA_KEY_PATH: /esg/certificates/slcsca/ca.key
      ONLINECA_TRUSTROOTS_DIR: /esg/certificates/slcsca/trustroots
    volumes:
      - "$ESGF_CONFIG/secrets/slcs-secret-key:/esg/secrets/slcs-secret-key:ro"
      - "$ESGF_CONFIG/secrets/slcs-database-password:/esg/secrets/slcs-database-password:ro"
      - "$ESGF_CONFIG/secrets/security-database-password:/esg/secrets/security-database-password:ro"
      - "$ESGF_CONFIG/certificates/slcsca/ca.crt:/esg/certificates/slcsca/ca.crt:ro"
      - "$ESGF_CONFIG/certificates/slcsca/ca.key:/esg/certificates/slcsca/ca.key:ro"
      # Just mount an empty volume as the trustroots for now
      - "slcs-trustroots:/esg/certificates/slcsca/trustroots/:ro"
      # Make sure the trusted certificate bundle is available
      - "$ESGF_CONFIG/certificates/esg-trust-bundle.pem:/esg/certificates/esg-trust-bundle.pem:ro"
    depends_on:
      - esgf-postgres-security
      - esgf-postgres-slcs


# Use Docker volumes to store persistent data
volumes:
  solr-master-home:
  solr-slave-home:
  postgres-auth-data:
  postgres-cog-data:
  postgres-esgcet-data:
  postgres-security-data:
  postgres-slcs-data:
  tds-content:
  slcs-trustroots: