Run docker-compose environment with network_mode=host with external database. #35193

husseljoo · 2024-02-12T22:05:08Z

husseljoo
Feb 12, 2024

So I want to run all container services in the docker-compose.yaml in network_mode=host.
And I want to connect to an external database.
I followed this guide (https://docs.airbyte.com/operator-guides/configuring-airbyte-db) for the external database connection part.

The database starts successfully, and the system is ready to accept connections.

docker run --name airbyte-postgres \
  --rm \
  --network host \
  -e POSTGRES_PASSWORD=postgres \
  -e POSTGRES_USER=postgres \
  -e POSTGRES_DB=postgres \
  -e PGPORT=3000 \
  --health-cmd="pg_isready -U postgres -d postgres" \
  --health-interval=2s \
  --health-timeout=10s \
  --health-retries=3 \
  postgres:13

The database is up normally and the system is ready to accept connections

Here is the compose file I modified:

version: "3.8"
#https://github.com/compose-spec/compose-spec/blob/master/spec.md#using-extensions-as-fragments
x-logging: &default-logging
  options:
    max-size: "100m"
    max-file: "5"
  driver: json-file
services:
  # hook in case we need to add init behavior
  # every root service (no depends_on) should depend on init
  init:
    image: airbyte/init:${VERSION}
    network_mode: host
    logging: *default-logging
    container_name: init
    command: /bin/sh -c "./scripts/create_mount_directories.sh /local_parent ${HACK_LOCAL_ROOT_PARENT} ${LOCAL_ROOT}"
    environment:
      - LOCAL_ROOT=${LOCAL_ROOT}
      - HACK_LOCAL_ROOT_PARENT=${HACK_LOCAL_ROOT_PARENT}
    volumes:
      - ${HACK_LOCAL_ROOT_PARENT}:/local_parent
  bootloader:
    image: airbyte/bootloader:${VERSION}
    network_mode: host
    logging: *default-logging
    container_name: airbyte-bootloader
    environment:
      - AIRBYTE_VERSION=${VERSION}
      - CONNECTOR_REGISTRY_BASE_URL=${CONNECTOR_REGISTRY_BASE_URL:-}
      - CONNECTOR_REGISTRY_SEED_PROVIDER=${CONNECTOR_REGISTRY_SEED_PROVIDER:-}
      - DATABASE_PASSWORD=${DATABASE_PASSWORD}
      - DATABASE_URL=${DATABASE_URL}
      - DATABASE_USER=${DATABASE_USER}
      - LOCAL_CONNECTOR_CATALOG_PATH=${LOCAL_CONNECTOR_CATALOG_PATH}
      - LOG_LEVEL=${LOG_LEVEL}
    depends_on:
      init:
        condition: service_completed_successfully
  # db:
  #   image: airbyte/db:${VERSION}
  #   network_mode: host
  #   logging: *default-logging
  #   container_name: airbyte-db
  #   restart: unless-stopped
  #   environment:
  #     - CONFIG_DATABASE_PASSWORD=${CONFIG_DATABASE_PASSWORD:-}
  #     - CONFIG_DATABASE_URL=${CONFIG_DATABASE_URL:-}
  #     - CONFIG_DATABASE_USER=${CONFIG_DATABASE_USER:-}
  #     - DATABASE_PASSWORD=${DATABASE_PASSWORD}
  #     - DATABASE_URL=${DATABASE_URL}
  #     - DATABASE_USER=${DATABASE_USER}
  #     - POSTGRES_PASSWORD=${DATABASE_PASSWORD}
  #     - POSTGRES_USER=${DATABASE_USER}
  #   volumes:
  #     - db:/var/lib/postgresql/data
  worker:
    image: airbyte/worker:${VERSION}
    network_mode: host
    logging: *default-logging
    container_name: airbyte-worker
    restart: unless-stopped
    environment:
      - ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS=${ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS}
      - ACTIVITY_MAX_ATTEMPT=${ACTIVITY_MAX_ATTEMPT}
      - ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS=${ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS}
      - AIRBYTE_ROLE=${AIRBYTE_ROLE:-}
      - AIRBYTE_VERSION=${VERSION}
      - APPLY_FIELD_SELECTION=${APPLY_FIELD_SELECTION}
      - AUTO_DETECT_SCHEMA=${AUTO_DETECT_SCHEMA}
      - AUTO_DISABLE_FAILING_CONNECTIONS=${AUTO_DISABLE_FAILING_CONNECTIONS}
      - CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=${CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION:-}
      - CONFIG_DATABASE_PASSWORD=${CONFIG_DATABASE_PASSWORD:-}
      - CONFIG_DATABASE_URL=${CONFIG_DATABASE_URL:-}
      - CONFIG_DATABASE_USER=${CONFIG_DATABASE_USER:-}
      - CONFIG_ROOT=${CONFIG_ROOT}
      - CONNECTOR_REGISTRY_BASE_URL=${CONNECTOR_REGISTRY_BASE_URL:-}
      - DATABASE_PASSWORD=${DATABASE_PASSWORD}
      - DATABASE_URL=${DATABASE_URL}
      - DATABASE_USER=${DATABASE_USER}
      - DEPLOYMENT_MODE=${DEPLOYMENT_MODE}
      - DD_AGENT_HOST=${DD_AGENT_HOST}
      - DD_DOGSTATSD_PORT=${DD_DOGSTATSD_PORT}
      - FEATURE_FLAG_CLIENT=${FEATURE_FLAG_CLIENT}
      - FIELD_SELECTION_WORKSPACES=${FIELD_SELECTION_WORKSPACES}
      - INTERNAL_API_HOST=${INTERNAL_API_HOST}
      - JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=${JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION:-}
      - JOB_ERROR_REPORTING_SENTRY_DSN=${JOB_ERROR_REPORTING_SENTRY_DSN}
      - JOB_ERROR_REPORTING_STRATEGY=${JOB_ERROR_REPORTING_STRATEGY}
      - JOB_MAIN_CONTAINER_CPU_LIMIT=${JOB_MAIN_CONTAINER_CPU_LIMIT}
      - JOB_MAIN_CONTAINER_CPU_REQUEST=${JOB_MAIN_CONTAINER_CPU_REQUEST}
      - JOB_MAIN_CONTAINER_MEMORY_LIMIT=${JOB_MAIN_CONTAINER_MEMORY_LIMIT}
      - JOB_MAIN_CONTAINER_MEMORY_REQUEST=${JOB_MAIN_CONTAINER_MEMORY_REQUEST}
      - LAUNCHDARKLY_KEY=${LAUNCHDARKLY_KEY}
      - LOCAL_DOCKER_MOUNT=${LOCAL_DOCKER_MOUNT}
      - LOCAL_ROOT=${LOCAL_ROOT}
      - LOG_CONNECTOR_MESSAGES=${LOG_CONNECTOR_MESSAGES}
      - LOG_LEVEL=${LOG_LEVEL}
      - MAX_CHECK_WORKERS=${MAX_CHECK_WORKERS}
      - MAX_DISCOVER_WORKERS=${MAX_DISCOVER_WORKERS}
      - MAX_NOTIFY_WORKERS=${MAX_NOTIFY_WORKERS}
      - MAX_SPEC_WORKERS=${MAX_SPEC_WORKERS}
      - MAX_SYNC_WORKERS=${MAX_SYNC_WORKERS}
      - METRIC_CLIENT=${METRIC_CLIENT}
      - MICROMETER_METRICS_ENABLED=${MICROMETER_METRICS_ENABLED}
      - MICROMETER_METRICS_STATSD_FLAVOR=${MICROMETER_METRICS_STATSD_FLAVOR}
      - MICRONAUT_ENVIRONMENTS=${WORKERS_MICRONAUT_ENVIRONMENTS}
      - NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT=${NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT}
      - NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST=${NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST}
      - NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT=${NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT}
      - NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST=${NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST}
      - OTEL_COLLECTOR_ENDPOINT=${OTEL_COLLECTOR_ENDPOINT}
      - PUBLISH_METRICS=${PUBLISH_METRICS}
      - SECRET_PERSISTENCE=${SECRET_PERSISTENCE}
      - SEGMENT_WRITE_KEY=${SEGMENT_WRITE_KEY}
      - SHOULD_RUN_NOTIFY_WORKFLOWS=${SHOULD_RUN_NOTIFY_WORKFLOWS}
      - STATSD_HOST=${STATSD_HOST}
      - STATSD_PORT=${STATSD_PORT}
      - SYNC_JOB_INIT_RETRY_TIMEOUT_MINUTES=${SYNC_JOB_INIT_RETRY_TIMEOUT_MINUTES}
      - SYNC_JOB_MAX_ATTEMPTS=${SYNC_JOB_MAX_ATTEMPTS}
      - SYNC_JOB_MAX_TIMEOUT_DAYS=${SYNC_JOB_MAX_TIMEOUT_DAYS}
      - TEMPORAL_HOST=${TEMPORAL_HOST}
      - TRACKING_STRATEGY=${TRACKING_STRATEGY}
      - WEBAPP_URL=${WEBAPP_URL}
      - WORKER_LOGS_STORAGE_TYPE=${WORKER_LOGS_STORAGE_TYPE}
      - WORKER_STATE_STORAGE_TYPE=${WORKER_STATE_STORAGE_TYPE}
      - WORKFLOW_FAILURE_RESTART_DELAY_SECONDS=${WORKFLOW_FAILURE_RESTART_DELAY_SECONDS}
      - WORKLOAD_API_HOST=${WORKLOAD_API_URL}
      - WORKSPACE_DOCKER_MOUNT=${WORKSPACE_DOCKER_MOUNT}
      - WORKSPACE_ROOT=${WORKSPACE_ROOT}
    configs:
      - flags
    volumes:
      - ${DOCKER_SOCKET:-/var/run/docker.sock}:/var/run/docker.sock
      - workspace:${WORKSPACE_ROOT}
      - ${LOCAL_ROOT}:${LOCAL_ROOT}
    depends_on:
      bootloader:
        condition: service_completed_successfully
  server:
    image: airbyte/server:${VERSION}
    network_mode: host
    logging: *default-logging
    container_name: airbyte-server
    restart: unless-stopped
    environment:
      - AIRBYTE_ROLE=${AIRBYTE_ROLE:-}
      - AIRBYTE_VERSION=${VERSION}
      - AUTO_DETECT_SCHEMA=${AUTO_DETECT_SCHEMA}
      - CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=${CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION:-}
      - CONFIG_DATABASE_PASSWORD=${CONFIG_DATABASE_PASSWORD:-}
      - CONFIG_DATABASE_URL=${CONFIG_DATABASE_URL:-}
      - CONFIG_DATABASE_USER=${CONFIG_DATABASE_USER:-}
      - CONFIG_ROOT=${CONFIG_ROOT}
      - CONNECTOR_REGISTRY_BASE_URL=${CONNECTOR_REGISTRY_BASE_URL:-}
      - DATABASE_PASSWORD=${DATABASE_PASSWORD}
      - DATABASE_URL=${DATABASE_URL}
      - DATABASE_USER=${DATABASE_USER}
      - DD_AGENT_HOST=${DD_AGENT_HOST}
      - DD_DOGSTATSD_PORT=${DD_DOGSTATSD_PORT}
      - FEATURE_FLAG_CLIENT=${FEATURE_FLAG_CLIENT}
      - GITHUB_STORE_BRANCH=${GITHUB_STORE_BRANCH}
      - JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=${JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION:-}
      - JOB_ERROR_REPORTING_SENTRY_DSN=${JOB_ERROR_REPORTING_SENTRY_DSN}
      - JOB_ERROR_REPORTING_STRATEGY=${JOB_ERROR_REPORTING_STRATEGY}
      - JOB_MAIN_CONTAINER_CPU_LIMIT=${JOB_MAIN_CONTAINER_CPU_LIMIT}
      - JOB_MAIN_CONTAINER_CPU_REQUEST=${JOB_MAIN_CONTAINER_CPU_REQUEST}
      - JOB_MAIN_CONTAINER_MEMORY_LIMIT=${JOB_MAIN_CONTAINER_MEMORY_LIMIT}
      - JOB_MAIN_CONTAINER_MEMORY_REQUEST=${JOB_MAIN_CONTAINER_MEMORY_REQUEST}
      - LAUNCHDARKLY_KEY=${LAUNCHDARKLY_KEY}
      - LOG_LEVEL=${LOG_LEVEL}
      - METRIC_CLIENT=${METRIC_CLIENT}
      - MAX_NOTIFY_WORKERS=5
      - MICROMETER_METRICS_ENABLED=${MICROMETER_METRICS_ENABLED}
      - MICROMETER_METRICS_STATSD_FLAVOR=${MICROMETER_METRICS_STATSD_FLAVOR}
      - MICRONAUT_ENVIRONMENTS=${WORKERS_MICRONAUT_ENVIRONMENTS}
      - NEW_SCHEDULER=${NEW_SCHEDULER}
      - PUBLISH_METRICS=${PUBLISH_METRICS}
      - SECRET_PERSISTENCE=${SECRET_PERSISTENCE}
      - SEGMENT_WRITE_KEY=${SEGMENT_WRITE_KEY}
      - SHOULD_RUN_NOTIFY_WORKFLOWS=${SHOULD_RUN_NOTIFY_WORKFLOWS}
      - STATSD_HOST=${STATSD_HOST}
      - STATSD_PORT=${STATSD_PORT}
      - TEMPORAL_HOST=${TEMPORAL_HOST}
      - TRACKING_STRATEGY=${TRACKING_STRATEGY}
      - WEBAPP_URL=${WEBAPP_URL}
      - WORKER_ENVIRONMENT=${WORKER_ENVIRONMENT}
      - WORKSPACE_ROOT=${WORKSPACE_ROOT}
      - CONNECTOR_BUILDER_SERVER_API_HOST=${CONNECTOR_BUILDER_SERVER_API_HOST}
    configs:
      - flags
    volumes:
      - workspace:${WORKSPACE_ROOT}
      - data:${CONFIG_ROOT}
      - ${LOCAL_ROOT}:${LOCAL_ROOT}
      - ./configs:/app/configs:ro
    depends_on:
      bootloader:
        condition: service_completed_successfully
  webapp:
    image: airbyte/webapp:${VERSION}
    network_mode: host
    logging: *default-logging
    container_name: airbyte-webapp
    restart: unless-stopped
    environment:
      - INTERNAL_API_HOST=${INTERNAL_API_HOST}
      - CONNECTOR_BUILDER_API_HOST=${CONNECTOR_BUILDER_API_HOST}
      - KEYCLOAK_INTERNAL_HOST=localhost # placeholder to ensure the webapp's nginx config is valid
    depends_on:
      bootloader:
        condition: service_completed_successfully
  airbyte-temporal:
    image: airbyte/temporal:${VERSION}
    network_mode: host
    logging: *default-logging
    container_name: airbyte-temporal
    restart: unless-stopped
    environment:
      - DB=postgresql
      - DB_PORT=${DATABASE_PORT}
      - DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development.yaml
      - LOG_LEVEL=${LOG_LEVEL}
      - POSTGRES_PWD=${DATABASE_PASSWORD}
      - POSTGRES_SEEDS=${DATABASE_HOST}
      - POSTGRES_USER=${DATABASE_USER}
    volumes:
      - ./temporal/dynamicconfig:/etc/temporal/config/dynamicconfig
  airbyte-cron:
    image: airbyte/cron:${VERSION}
    network_mode: host
    logging: *default-logging
    container_name: airbyte-cron
    restart: unless-stopped
    environment:
      - AIRBYTE_VERSION=${VERSION}
      - CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=${CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION}
      - CONNECTOR_REGISTRY_BASE_URL=${CONNECTOR_REGISTRY_BASE_URL:-}
      - CONNECTOR_REGISTRY_SEED_PROVIDER=${CONNECTOR_REGISTRY_SEED_PROVIDER:-}
      - DATABASE_PASSWORD=${DATABASE_PASSWORD}
      - DATABASE_URL=${DATABASE_URL}
      - DATABASE_USER=${DATABASE_USER}
      - DD_AGENT_HOST=${DD_AGENT_HOST}
      - DD_DOGSTATSD_PORT=${DD_DOGSTATSD_PORT}
      - DEPLOYMENT_MODE=${DEPLOYMENT_MODE}
      - LOG_LEVEL=${LOG_LEVEL}
      - METRIC_CLIENT=${METRIC_CLIENT}
      - MICROMETER_METRICS_ENABLED=${MICROMETER_METRICS_ENABLED}
      - MICROMETER_METRICS_STATSD_FLAVOR=${MICROMETER_METRICS_STATSD_FLAVOR}
      - MICRONAUT_ENVIRONMENTS=${CRON_MICRONAUT_ENVIRONMENTS}
      - PUBLISH_METRICS=${PUBLISH_METRICS}
      - SEGMENT_WRITE_KEY=${SEGMENT_WRITE_KEY}
      - STATSD_HOST=${STATSD_HOST}
      - STATSD_PORT=${STATSD_PORT}
      - TEMPORAL_HISTORY_RETENTION_IN_DAYS=${TEMPORAL_HISTORY_RETENTION_IN_DAYS}
      - TRACKING_STRATEGY=${TRACKING_STRATEGY}
      - UPDATE_DEFINITIONS_CRON_ENABLED=${UPDATE_DEFINITIONS_CRON_ENABLED}
      - WORKSPACE_ROOT=${WORKSPACE_ROOT}
      # - TEMPORAL_HOST=${TEMPORAL_HOST}
    configs:
      - flags
    volumes:
      - workspace:${WORKSPACE_ROOT}
    depends_on:
      bootloader:
        condition: service_completed_successfully
  airbyte-api-server:
    image: airbyte/airbyte-api-server:${VERSION}
    network_mode: host
    logging: *default-logging
    container_name: airbyte-api-server
    restart: unless-stopped
    environment:
      - AIRBYTE_API_HOST=${AIRBYTE_API_HOST}
      - AIRBYTE_VERSION=${VERSION}
      - DEPLOYMENT_MODE=${DEPLOYMENT_MODE}
      - INTERNAL_API_HOST=${INTERNAL_API_URL} # Non-ideal, but the existing URL can't have https:// added because it's used for nginx conf
      - LOG_LEVEL=${LOG_LEVEL}
      - MICROMETER_METRICS_ENABLED=${MICROMETER_METRICS_ENABLED}
      - MICROMETER_METRICS_STATSD_FLAVOR=${MICROMETER_METRICS_STATSD_FLAVOR}
      - SEGMENT_WRITE_KEY=${SEGMENT_WRITE_KEY}
      - STATSD_HOST=${STATSD_HOST}
      - STATSD_PORT=${STATSD_PORT}
      - TRACKING_STRATEGY=${TRACKING_STRATEGY}
    depends_on:
      bootloader:
        condition: service_completed_successfully
  airbyte-connector-builder-server:
    image: airbyte/connector-builder-server:${VERSION}
    network_mode: host
    logging: *default-logging
    container_name: airbyte-connector-builder-server
    restart: unless-stopped
    environment:
      - AIRBYTE_VERSION=${VERSION}
      - CDK_VERSION=${CDK_VERSION}
      - DEPLOYMENT_MODE=${DEPLOYMENT_MODE}
      - METRIC_CLIENT=${METRIC_CLIENT}
      - MICROMETER_METRICS_ENABLED=${MICROMETER_METRICS_ENABLED}
      - MICROMETER_METRICS_STATSD_FLAVOR=${MICROMETER_METRICS_STATSD_FLAVOR}
      - PYTHON_VERSION=${PYTHON_VERSION}
      - SEGMENT_WRITE_KEY=${SEGMENT_WRITE_KEY}
      - STATSD_HOST=${STATSD_HOST}
      - STATSD_PORT=${STATSD_PORT}
      - TRACKING_STRATEGY=${TRACKING_STRATEGY}
    depends_on:
      bootloader:
        condition: service_completed_successfully
  airbyte-proxy:
    image: airbyte/proxy:${VERSION}
    network_mode: host
    container_name: airbyte-proxy
    restart: unless-stopped
    environment:
      - BASIC_AUTH_USERNAME=${BASIC_AUTH_USERNAME}
      - BASIC_AUTH_PASSWORD=${BASIC_AUTH_PASSWORD}
      - BASIC_AUTH_PROXY_TIMEOUT=${BASIC_AUTH_PROXY_TIMEOUT}
    depends_on:
      - webapp
      - server
      - airbyte-api-server
volumes:
  workspace:
    name: ${WORKSPACE_DOCKER_MOUNT}
  # the data volume is only needed for backward compatibility; when users upgrade
  # from an old Airbyte version that relies on file-based configs, the server needs
  # to read this volume to copy their configs to the database
  data:
    name: ${DATA_DOCKER_MOUNT}
  db:
    name: ${DB_DOCKER_MOUNT}
configs:
  flags:
    file: ./flags.yml

Here are the modified env files:

# This file only contains Docker relevant variables.
#
# Variables with defaults have been omitted to avoid duplication of defaults.
# The only exception to the non-default rule are env vars related to scaling.
#
# See https://github.com/airbytehq/airbyte/blob/master/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java
# for the latest environment variables.
#
# # Contributors - please organise this env file according to the above linked file.


### SHARED ###
VERSION=0.50.48

# When using the airbyte-db via default docker image
CONFIG_ROOT=/data
DATA_DOCKER_MOUNT=airbyte_data
DB_DOCKER_MOUNT=airbyte_db

# Workspace storage for running jobs (logs, etc)
WORKSPACE_ROOT=/tmp/workspace
WORKSPACE_DOCKER_MOUNT=airbyte_workspace

# Local mount to access local files from filesystem
# todo (cgardens) - when we are mount raw directories instead of named volumes, *_DOCKER_MOUNT must
# be the same as *_ROOT.
# Issue: https://github.com/airbytehq/airbyte/issues/578
LOCAL_ROOT=/tmp/airbyte_local
LOCAL_DOCKER_MOUNT=/tmp/airbyte_local
# todo (cgardens) - hack to handle behavior change in docker compose. *_PARENT directories MUST
# already exist on the host filesystem and MUST be parents of *_ROOT.
# Issue: https://github.com/airbytehq/airbyte/issues/577
HACK_LOCAL_ROOT_PARENT=/tmp

# Storage type
WORKER_LOGS_STORAGE_TYPE=local
WORKER_STATE_STORAGE_TYPE=local

# Proxy Configuration
# Set to empty values, e.g. "" to disable basic auth
BASIC_AUTH_USERNAME=airbyte
BASIC_AUTH_PASSWORD=password
BASIC_AUTH_PROXY_TIMEOUT=900

### DATABASE ###
# Airbyte Internal Job Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db
DATABASE_USER=postgres
DATABASE_PASSWORD=postgres
DATABASE_HOST=postgres
DATABASE_PORT=3000
DATABASE_DB=postgres
# translate manually DATABASE_URL=jdbc:postgresql://${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB} (do not include the username or password here)
DATABASE_URL=jdbc:postgresql://localhost:3000/postgres
JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=0.40.26.001

# Airbyte Internal Config Database, defaults to Job Database if empty. Explicitly left empty to mute docker compose warnings.
# CONFIG_DATABASE_USER=
# CONFIG_DATABASE_PASSWORD=
# CONFIG_DATABASE_URL=
# CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=0.40.23.002

### AIRBYTE SERVICES ###
TEMPORAL_HOST=localhost:7233
INTERNAL_API_HOST=localhost:8001
INTERNAL_API_URL=http://localhost:8001
CONNECTOR_BUILDER_API_HOST=localhost:80
WEBAPP_URL=http://localhost:8000/
WORKLOAD_API_HOST=localhost:8007
WORKLOAD_API_URL=http://localhost:8007
# Although not present as an env var, required for webapp configuration.
CONNECTOR_BUILDER_API_URL=/connector-builder-api
AIRBYTE_API_HOST=localhost:8006
CONNECTOR_BUILDER_SERVER_API_HOST=http://localhost:80
 
# ### AIRBYTE SERVICES ###
# TEMPORAL_HOST=localhost:7233
# INTERNAL_API_HOST=localhost:8023
# INTERNAL_API_URL=http://localhost:8023
# CONNECTOR_BUILDER_API_HOST=localhost:8022
# WEBAPP_URL=http://localhost:8020/
# WORKLOAD_API_HOST=localhost:8007
# WORKLOAD_API_URL=http://localhost:8007
# # Although not present as an env var, required for webapp configuration.
# CONNECTOR_BUILDER_API_URL=/connector-builder-api
# AIRBYTE_API_HOST=localhost:8006
# CONNECTOR_BUILDER_SERVER_API_HOST=http://localhost:80


### JOBS ###
# Relevant to scaling.
SYNC_JOB_MAX_ATTEMPTS=3
SYNC_JOB_MAX_TIMEOUT_DAYS=3
SYNC_JOB_INIT_RETRY_TIMEOUT_MINUTES=5
JOB_MAIN_CONTAINER_CPU_REQUEST=
JOB_MAIN_CONTAINER_CPU_LIMIT=
JOB_MAIN_CONTAINER_MEMORY_REQUEST=
JOB_MAIN_CONTAINER_MEMORY_LIMIT=

NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT=
NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST=
NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT=
NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST=

### LOGGING/MONITORING/TRACKING ###
TRACKING_STRATEGY=segment
SEGMENT_WRITE_KEY=7UDdp5K55CyiGgsauOr2pNNujGvmhaeu
JOB_ERROR_REPORTING_STRATEGY=logging
# Although not present as an env var, expected by Log4J configuration.
LOG_LEVEL=INFO


### APPLICATIONS ###
# Worker #
WORKERS_MICRONAUT_ENVIRONMENTS=control-plane
# Cron #
CRON_MICRONAUT_ENVIRONMENTS=control-plane
# Relevant to scaling.
MAX_SYNC_WORKERS=5
MAX_SPEC_WORKERS=5
MAX_CHECK_WORKERS=5
MAX_DISCOVER_WORKERS=5
MAX_NOTIFY_WORKERS=5
SHOULD_RUN_NOTIFY_WORKFLOWS=true
# Temporal Activity configuration
ACTIVITY_MAX_ATTEMPT=
ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS=
ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS=
WORKFLOW_FAILURE_RESTART_DELAY_SECONDS=

### FEATURE FLAGS ###
AUTO_DISABLE_FAILING_CONNECTIONS=false

FEATURE_FLAG_CLIENT=config

### MONITORING FLAGS ###
PUBLISH_METRICS=false
# Accepted values are datadog and otel (open telemetry)
METRIC_CLIENT=
# Useful only when metric client is set to be otel. Must start with http:// or https://.
OTEL_COLLECTOR_ENDPOINT="http://localhost:4317"
# Useful only when metric client is set to be datadog.
DD_AGENT_HOST=
DD_DOGSTATSD_PORT=
# Useful only when enabling Micrometer metrics
MICROMETER_METRICS_ENABLED=false
MICROMETER_METRICS_STATSD_FLAVOR=
STATSD_HOST=
STATSD_PORT=

AUTO_DETECT_SCHEMA=true

SECRET_PERSISTENCE=TESTING_CONFIG_DB_TABLE

# To test local catalog changes, set the below variable to the path of your local catalog.
# https://docs.airbyte.com/connector-development/testing-connectors/testing-a-local-catalog-in-development
LOCAL_CONNECTOR_CATALOG_PATH=

I face the following errors:

airbyte-temporal  | nc: bad address 'postgres'
airbyte-temporal  | Waiting for PostgreSQL to startup.

And subsqeuently:

- airbyte-server
Caused by: io.grpc.netty.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: localhost/127.0.0.1:7233
 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused
     at io.grpc.netty.shaded.io.netty.channel.unix.Errors.newConnectException0(Errors.java:155) ~[grpc-netty-shaded-1.55.1.jar:1.55.1]
     at io.grpc.netty.shaded.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:128) ~[grpc-netty-shaded-1.55.1.jar:1.55.1]
     at io.grpc.netty.shaded.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[grpc-netty-shaded-1.55.1.jar:1.55.1]
     at io.grpc.netty.shaded.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:710) ~[grpc-netty-shaded-1.55.1.jar:1.55.1]
     at io.grpc.netty.shaded.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:687) ~[grpc-netty-shaded-1.55.1.jar:1.55.1]
     at io.grpc.netty.shaded.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[grpc-netty-shaded-1.55.1.jar:1.55.1]
     at io.grpc.netty.shaded.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:489) ~[grpc-netty-shaded-1.55.1.jar:1.55.1]
     at io.grpc.netty.shaded.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[grpc-netty-shaded-1.55.1.jar:1.55.1]
     at io.grpc.netty.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[grpc-netty-shaded-1.55.1.jar:1.55.1]
     at io.grpc.netty.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[grpc-netty-shaded-1.55.1.jar:1.55.1]
     at io.grpc.netty.shaded.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[grpc-netty-shaded-1.55.1.jar:1.55.1]
     at java.base/java.lang.Thread.run(Thread.java:1583) ~[?:?]
 2024-02-12 21:30:14 WARN i.a.c.t.TemporalUtils(getTemporalClientWhenConnected):159 - Waiting for namespace default to be initialized in temporal...

airbyte-worker
- Caused by: io.grpc.netty.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: localhost/127.0.0.1:7233
Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused
    at io.grpc.netty.shaded.io.netty.channel.unix.Errors.newConnectException0(Errors.java:155) ~[grpc-netty-shaded-1.55.1.jar:1.55.1]
    at io.grpc.netty.shaded.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:128) ~[grpc-netty-shaded-1.55.1.jar:1.55.1]
    at io.grpc.netty.shaded.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[grpc-netty-shaded-1.55.1.jar:1.55.1]
    at io.grpc.netty.shaded.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:710) ~[grpc-netty-shaded-1.55.1.jar:1.55.1]
    at io.grpc.netty.shaded.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:687) ~[grpc-netty-shaded-1.55.1.jar:1.55.1]
    at io.grpc.netty.shaded.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[grpc-netty-shaded-1.55.1.jar:1.55.1]
    at io.grpc.netty.shaded.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:489) ~[grpc-netty-shaded-1.55.1.jar:1.55.1]
    at io.grpc.netty.shaded.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[grpc-netty-shaded-1.55.1.jar:1.55.1]
    at io.grpc.netty.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[grpc-netty-shaded-1.55.1.jar:1.55.1]
    at io.grpc.netty.shaded.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[grpc-netty-shaded-1.55.1.jar:1.55.1]
    at io.grpc.netty.shaded.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[grpc-netty-shaded-1.55.1.jar:1.55.1]
    at java.base/java.lang.Thread.run(Thread.java:1583) ~[?:?]

Moreover, the airbyte-cron service gives this error:

airbyte-cron                      | Feb 12, 2024 9:30:14 PM io.grpc.internal.ManagedChannelImpl$NameResolverListener handleErrorInSyncContext
airbyte-cron                      | WARNING: [Channel<33>: (airbyte-temporal:7233)] Failed to resolve name. status=Status{code=UNAVAILABLE, description=Unable to resolve host airbyte-temporal, cause=java.lang.RuntimeException: java.net.UnknownHostException: airbyte-temporal
airbyte-cron                      |     at io.grpc.internal.DnsNameResolver.resolveAddresses(DnsNameResolver.java:223)
airbyte-cron                      |     at io.grpc.internal.DnsNameResolver.doResolve(DnsNameResolver.java:282)
airbyte-cron                      |     at io.grpc.grpclb.GrpclbNameResolver.doResolve(GrpclbNameResolver.java:63)
airbyte-cron                      |     at io.grpc.internal.DnsNameResolver$Resolve.run(DnsNameResolver.java:318)
airbyte-cron                      |     at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
airbyte-cron                      |     at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
airbyte-cron                      |     at java.base/java.lang.Thread.run(Thread.java:1583)
airbyte-cron                      | Caused by: java.net.UnknownHostException: airbyte-temporal
airbyte-cron                      |     at java.base/java.net.InetAddress$CachedLookup.get(InetAddress.java:988)
airbyte-cron                      |     at java.base/java.net.InetAddress$NameServiceAddresses.get(InetAddress.java:1191)
airbyte-cron                      |     at java.base/java.net.InetAddress.getAllByName0(InetAddress.java:1818)
airbyte-cron                      |     at java.base/java.net.InetAddress.getAllByName(InetAddress.java:1688)
airbyte-cron                      |     at io.grpc.internal.DnsNameResolver$JdkAddressResolver.resolveAddress(DnsNameResolver.java:632)
airbyte-cron                      |     at io.grpc.internal.DnsNameResolver.resolveAddresses(DnsNameResolver.java:219)
airbyte-cron                      |     ... 6 more

I'm not sure why that is the case, as I relplaced every service name with "localhost".
I fear this may be hardcoded in the Dockerfile.
Lastly is there a way to get the source Dockerfiles for each container image.
Are there any specific configurations I need to adjust in the Airbyte docker-compose.yaml file for the host network mode?

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

Run docker-compose environment with network_mode=host with external database. #35193

Uh oh!

{{title}}

Uh oh!

Uh oh!

{{editor}}'s edit

{{editor}}'s edit

Uh oh!

Replies: 0 comments

Select a reply

Uh oh!

Run docker-compose environment with network_mode=host with external database. #35193

Uh oh!

Uh oh!

husseljoo Feb 12, 2024

Replies: 0 comments

husseljoo
Feb 12, 2024