Skip to content

Commit f7e4ddf

Browse files
aaronsteersaaronsteers
committed
refactor, use build args
1 parent 7a87b62 commit f7e4ddf

File tree

2 files changed

+167
-75
lines changed

2 files changed

+167
-75
lines changed

‎airbyte_cdk/utils/docker.py‎

Lines changed: 71 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -2,56 +2,31 @@
22

33
from __future__ import annotations
44

5+
import json
56
import logging
7+
import os
68
import subprocess
79
import sys
810
from pathlib import Path
911

1012
import click
1113

1214
from airbyte_cdk.models.connector_metadata import MetadataFile
15+
from airbyte_cdk.utils.docker_image_templates import (
16+
DOCKERIGNORE_TEMPLATE,
17+
PYTHON_CONNECTOR_DOCKERFILE_TEMPLATE,
18+
)
1319

1420
logger = logging.getLogger(__name__)
1521

16-
# This template accepts the following variables:
17-
# - base_image: The base image to use for the build
18-
# - extra_build_steps: Additional build steps to include in the Dockerfile
19-
# - connector_snake_name: The snake_case name of the connector
20-
# - connector_kebab_name: The kebab-case name of the connector
21-
DOCKERFILE_TEMPLATE = """
22-
FROM {base_image} AS builder
23-
24-
WORKDIR /airbyte/integration_code
25-
26-
COPY . ./
27-
COPY {connector_snake_name} ./{connector_snake_name}
28-
{extra_build_steps}
29-
30-
# TODO: Pre-install uv on the base image to speed up the build.
31-
# (uv is still faster even with the extra step.)
32-
RUN pip install --no-cache-dir uv
33-
34-
RUN python -m uv pip install --no-cache-dir .
35-
36-
FROM {base_image}
37-
38-
WORKDIR /airbyte/integration_code
39-
40-
COPY --from=builder /usr/local /usr/local
41-
42-
COPY . .
43-
44-
ENV AIRBYTE_ENTRYPOINT="{connector_kebab_name}"
45-
ENTRYPOINT ["{connector_kebab_name}"]
46-
"""
47-
4822

4923
def _build_image(
5024
context_dir: Path,
5125
dockerfile: Path,
5226
metadata: MetadataFile,
5327
tag: str,
5428
arch: str,
29+
build_args: dict[str, str | None] | None = None,
5530
) -> str:
5631
"""Build a Docker image for the specified architecture.
5732
@@ -72,10 +47,22 @@ def _build_image(
7247
tag,
7348
str(context_dir),
7449
]
50+
if build_args:
51+
for key, value in build_args.items():
52+
if value is not None:
53+
docker_args.append(f"--build-arg={key}={value}")
54+
else:
55+
docker_args.append(f"--build-arg={key}")
56+
7557
print(f"Building image: {tag} ({arch})")
76-
run_docker_command(
77-
docker_args,
78-
)
58+
try:
59+
run_docker_command(
60+
docker_args,
61+
)
62+
except subprocess.CalledProcessError as e:
63+
print(f"ERROR: Failed to build image using Docker args: {docker_args}")
64+
exit(1)
65+
raise
7966
return tag
8067

8168

@@ -119,15 +106,10 @@ def build_connector_image(
119106
dockerfile_path = connector_directory / "build" / "docker" / "Dockerfile"
120107
dockerignore_path = connector_directory / "build" / "docker" / "Dockerfile.dockerignore"
121108

122-
extra_build_steps: str = ""
109+
extra_build_script: str = ""
123110
build_customization_path = connector_directory / "build_customization.py"
124111
if build_customization_path.exists():
125-
extra_build_steps = "\n".join(
126-
[
127-
"COPY build_customization.py ./",
128-
"RUN python3 build_customization.py",
129-
]
130-
)
112+
extra_build_script = str(build_customization_path)
131113

132114
dockerfile_path.parent.mkdir(parents=True, exist_ok=True)
133115
if not metadata.data.connectorBuildOptions:
@@ -138,34 +120,15 @@ def build_connector_image(
138120

139121
base_image = metadata.data.connectorBuildOptions.baseImage
140122

141-
dockerfile_path.write_text(
142-
DOCKERFILE_TEMPLATE.format(
143-
base_image=base_image,
144-
connector_snake_name=connector_snake_name,
145-
connector_kebab_name=connector_kebab_name,
146-
extra_build_steps=extra_build_steps,
147-
)
148-
)
149-
dockerignore_path.write_text(
150-
"\n".join(
151-
[
152-
"# This file is auto-generated. Do not edit.",
153-
"build/",
154-
".venv/",
155-
"secrets/",
156-
"!setup.py",
157-
"!pyproject.toml",
158-
"!poetry.lock",
159-
"!poetry.toml",
160-
"!components.py",
161-
"!requirements.txt",
162-
"!README.md",
163-
"!metadata.yaml",
164-
"!build_customization.py",
165-
# f"!{connector_snake_name}/",
166-
]
167-
)
168-
)
123+
dockerfile_path.write_text(PYTHON_CONNECTOR_DOCKERFILE_TEMPLATE)
124+
dockerignore_path.write_text(DOCKERIGNORE_TEMPLATE)
125+
126+
build_args: dict[str, str | None] = {
127+
"BASE_IMAGE": base_image,
128+
"CONNECTOR_SNAKE_NAME": connector_snake_name,
129+
"CONNECTOR_KEBAB_NAME": connector_kebab_name,
130+
"EXTRA_BUILD_SCRIPT": extra_build_script,
131+
}
169132

170133
base_tag = f"{metadata.data.dockerRepository}:{tag}"
171134
arch_images: list[str] = []
@@ -182,6 +145,7 @@ def build_connector_image(
182145
metadata=metadata,
183146
tag=docker_tag,
184147
arch=arch,
148+
build_args=build_args,
185149
)
186150
)
187151

@@ -190,7 +154,7 @@ def build_connector_image(
190154
new_tags=[base_tag],
191155
)
192156
if not no_verify:
193-
if verify_image(base_tag):
157+
if verify_connector_image(base_tag):
194158
click.echo(f"Build completed successfully: {base_tag}")
195159
sys.exit(0)
196160
else:
@@ -201,19 +165,35 @@ def build_connector_image(
201165
sys.exit(0)
202166

203167

204-
def run_docker_command(cmd: list[str]) -> None:
168+
def run_docker_command(
169+
cmd: list[str],
170+
*,
171+
check: bool = True,
172+
capture_output: bool = False,
173+
) -> subprocess.CompletedProcess:
205174
"""Run a Docker command as a subprocess.
206175
176+
Args:
177+
cmd: The command to run as a list of strings.
178+
check: If True, raises an exception if the command fails. If False, the caller is
179+
responsible for checking the return code.
180+
capture_output: If True, captures stdout and stderr and returns to the caller.
181+
If False, the output is printed to the console.
182+
207183
Raises:
208184
subprocess.CalledProcessError: If the command fails and check is True.
209185
"""
210-
logger.debug(f"Running command: {' '.join(cmd)}")
186+
print(f"Running command: {' '.join(cmd)}")
211187

212188
process = subprocess.run(
213189
cmd,
214190
text=True,
215191
check=True,
192+
# If capture_output=True, stderr and stdout are captured and returned to caller:
193+
capture_output=capture_output,
194+
env={**os.environ, "DOCKER_BUILDKIT": "1"},
216195
)
196+
return process
217197

218198

219199
def verify_docker_installation() -> bool:
@@ -225,7 +205,9 @@ def verify_docker_installation() -> bool:
225205
return False
226206

227207

228-
def verify_image(image_name: str) -> bool:
208+
def verify_connector_image(
209+
image_name: str,
210+
) -> bool:
229211
"""Verify the built image by running the spec command.
230212
231213
Args:
@@ -239,7 +221,21 @@ def verify_image(image_name: str) -> bool:
239221
cmd = ["docker", "run", "--rm", image_name, "spec"]
240222

241223
try:
242-
run_docker_command(cmd)
224+
result = run_docker_command(
225+
cmd,
226+
check=True,
227+
capture_output=True,
228+
)
229+
# check that the output is valid JSON
230+
if result.stdout:
231+
try:
232+
json.loads(result.stdout)
233+
except json.JSONDecodeError:
234+
logger.error("Invalid JSON output from spec command.")
235+
return False
236+
else:
237+
logger.error("No output from spec command.")
238+
return False
243239
except subprocess.CalledProcessError as e:
244240
logger.error(f"Image verification failed: {e.stderr}")
245241
return False

‎airbyte_cdk/utils/docker_image_templates.py‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
2+
"""A collection of Dockerfile templates for building Airbyte connectors.
3+
4+
The templates are designed to be used with the Airbyte CDK and can be customized
5+
for different connectors and architectures.
6+
7+
These templates are used to generate connector images.
8+
"""
9+
10+
DOCKERIGNORE_TEMPLATE: str = "\n".join(
11+
[
12+
"# This file is auto-generated. Do not edit.",
13+
# "*,"
14+
"build/",
15+
".venv/",
16+
"secrets/",
17+
"!setup.py",
18+
"!pyproject.toml",
19+
"!poetry.lock",
20+
"!poetry.toml",
21+
"!components.py",
22+
"!requirements.txt",
23+
"!README.md",
24+
"!metadata.yaml",
25+
"!build_customization.py",
26+
"!source_*",
27+
"!destination_*",
28+
]
29+
)
30+
31+
PYTHON_CONNECTOR_DOCKERFILE_TEMPLATE = """
32+
# syntax=docker/dockerfile:1
33+
# check=skip=all
34+
ARG BASE_IMAGE
35+
36+
FROM ${BASE_IMAGE} AS builder
37+
ARG BASE_IMAGE
38+
ARG CONNECTOR_SNAKE_NAME
39+
ARG CONNECTOR_KEBAB_NAME
40+
ARG EXTRA_PREREQS_SCRIPT=""
41+
42+
WORKDIR /airbyte/integration_code
43+
44+
COPY . ./
45+
COPY ${CONNECTOR_SNAKE_NAME} ./${CONNECTOR_SNAKE_NAME}
46+
47+
# Conditionally copy and execute the extra build script if provided
48+
RUN if [ -n "${EXTRA_PREREQS_SCRIPT}" ]; then \
49+
cp ${EXTRA_PREREQS_SCRIPT} ./extra_prereqs_script && \
50+
./extra_prereqs_script; \
51+
fi
52+
53+
# TODO: Pre-install uv on the base image to speed up the build.
54+
# (uv is still faster even with the extra step.)
55+
RUN pip install --no-cache-dir uv
56+
RUN python -m uv pip install --no-cache-dir .
57+
58+
FROM ${BASE_IMAGE}
59+
ARG CONNECTOR_SNAKE_NAME
60+
ARG CONNECTOR_KEBAB_NAME
61+
ARG BASE_IMAGE
62+
63+
WORKDIR /airbyte/integration_code
64+
65+
COPY --from=builder /usr/local /usr/local
66+
COPY --chmod=755 <<EOT /entrypoint.sh
67+
#!/usr/bin/env bash
68+
set -e
69+
70+
${CONNECTOR_KEBAB_NAME} "\$\@"
71+
EOT
72+
73+
ENV AIRBYTE_ENTRYPOINT="/entrypoint.sh"
74+
ENTRYPOINT ["/entrypoint.sh"]
75+
"""
76+
77+
MANIFEST_ONLY_DOCKERFILE_TEMPLATE = """
78+
ARG BASE_IMAGE
79+
ARG CONNECTOR_SNAKE_NAME
80+
ARG CONNECTOR_KEBAB_NAME
81+
82+
FROM ${BASE_IMAGE} AS builder
83+
84+
WORKDIR /airbyte/integration_code
85+
86+
COPY . ./
87+
COPY --chmod=755 <<EOT /entrypoint.sh
88+
#!/usr/bin/env bash
89+
set -e
90+
91+
${CONNECTOR_KEBAB_NAME} "$@"
92+
EOT
93+
94+
ENV AIRBYTE_ENTRYPOINT="/entrypoint.sh"
95+
ENTRYPOINT ["/entrypoint.sh"]
96+
"""

0 commit comments

Comments
 (0)