add support for building images for java connectors

Author: aaronsteers

airbyte_cdk/utils/docker.py

@@ -16,6 +16,7 @@
 from airbyte_cdk.models.connector_metadata import ConnectorLanguage, MetadataFile
 from airbyte_cdk.utils.docker_image_templates import (
     DOCKERIGNORE_TEMPLATE,
+    JAVA_CONNECTOR_DOCKERFILE_TEMPLATE,
     MANIFEST_ONLY_DOCKERFILE_TEMPLATE,
     PYTHON_CONNECTOR_DOCKERFILE_TEMPLATE,
 )
@@ -62,6 +63,10 @@ def _build_image(
 
     Raises: ConnectorImageBuildError if the build fails.
     """
+    if metadata.data.language == ConnectorLanguage.JAVA:
+        # For Java connectors, the context directory is the repo root.
+        context_dir = context_dir.parent.parent.parent
+
     docker_args: list[str] = [
         "docker",
         "build",
@@ -248,10 +253,7 @@ def get_dockerfile_template(
         return MANIFEST_ONLY_DOCKERFILE_TEMPLATE
 
     if metadata.data.language == ConnectorLanguage.JAVA:
-        raise ValueError(
-            f"Java and Kotlin connectors are not yet supported. "
-            "Please use airbyte-ci or gradle to build your image."
-        )
+        return JAVA_CONNECTOR_DOCKERFILE_TEMPLATE
 
     raise ValueError(
         f"Unsupported connector language: {metadata.data.language}. "
@@ -322,10 +324,21 @@ def verify_connector_image(
         )
         # check that the output is valid JSON
         if result.stdout:
-            try:
-                json.loads(result.stdout)
-            except json.JSONDecodeError:
-                logger.error("Invalid JSON output from spec command.")
+            found_spec_output = False
+            for line in result.stdout.split("\n"):
+                if line.strip():
+                    try:
+                        # Check if the line is a valid JSON object
+                        msg = json.loads(line)
+                        if isinstance(msg, dict) and "type" in msg and msg["type"] == "SPEC":
+                            found_spec_output = True
+
+                    except json.JSONDecodeError as e:
+                        logger.error(f"Invalid JSON output from spec command: {e}: {line}")
+                        return False
+
+            if not found_spec_output:
+                logger.error("No valid JSON output found for spec command.")
                 return False
         else:
             logger.error("No output from spec command.")

airbyte_cdk/utils/docker_image_templates.py

@@ -99,3 +99,77 @@
 ENV AIRBYTE_ENTRYPOINT="python ./main.py"
 ENTRYPOINT ["python", "./main.py"]
 """
+
+#########################
+# JAVA CONNECTOR IMAGE ##
+#########################
+
+JAVA_CONNECTOR_DOCKERFILE_TEMPLATE = """
+# Java connector Dockerfile for Airbyte
+# This Dockerfile replicates the dagger-based build process for Java connectors
+
+# Build arguments
+ARG JDK_VERSION=21-al2023
+
+# Base image - using Amazon Corretto (Amazon's distribution of OpenJDK)
+FROM amazoncorretto:${JDK_VERSION}
+ARG CONNECTOR_KEBAB_NAME
+
+# Install required packages and set up the non-root user
+RUN set -o xtrace && \
+    yum install -y shadow-utils tar openssl findutils && \
+    yum update -y --security && \
+    yum clean all && \
+    rm -rf /var/cache/yum && \
+    echo "Creating airbyte user and group..." && \
+    groupadd --gid 1000 airbyte && \
+    useradd --uid 1000 --gid airbyte --shell /bin/bash --create-home airbyte && \
+    echo "Creating directories..." && \
+    mkdir /secrets && \
+    mkdir /config && \
+    mkdir --mode 755 /airbyte && \
+    mkdir --mode 755 /custom_cache && \
+    echo "Setting permissions..." && \
+    chown -R airbyte:airbyte /airbyte && \
+    chown -R airbyte:airbyte /custom_cache && \
+    chown -R airbyte:airbyte /secrets && \
+    chown -R airbyte:airbyte /config && \
+    chown -R airbyte:airbyte /usr/share/pki/ca-trust-source && \
+    chown -R airbyte:airbyte /etc/pki/ca-trust && \
+    chown -R airbyte:airbyte /tmp
+
+# Download required scripts
+WORKDIR /airbyte
+ADD https://raw.githubusercontent.com/airbytehq/airbyte/6d8a3a2bc4f4ca79f10164447a90fdce5c9ad6f9/airbyte-integrations/bases/base/base.sh /airbyte/base.sh
+ADD https://raw.githubusercontent.com/airbytehq/airbyte/6d8a3a2bc4f4ca79f10164447a90fdce5c9ad6f9/airbyte-integrations/bases/base-java/javabase.sh /airbyte/javabase.sh
+ADD https://dtdg.co/latest-java-tracer /airbyte/dd-java-agent.jar
+
+# Set permissions for downloaded files
+RUN chmod +x /airbyte/base.sh /airbyte/javabase.sh && \
+    chown airbyte:airbyte /airbyte/base.sh /airbyte/javabase.sh /airbyte/dd-java-agent.jar
+
+# Set environment variables
+ENV AIRBYTE_SPEC_CMD="/airbyte/javabase.sh --spec"
+ENV AIRBYTE_CHECK_CMD="/airbyte/javabase.sh --check"
+ENV AIRBYTE_DISCOVER_CMD="/airbyte/javabase.sh --discover"
+ENV AIRBYTE_READ_CMD="/airbyte/javabase.sh --read"
+ENV AIRBYTE_WRITE_CMD="/airbyte/javabase.sh --write"
+ENV AIRBYTE_ENTRYPOINT="/airbyte/base.sh"
+ENV APPLICATION="${CONNECTOR_KEBAB_NAME}"
+
+# Add the connector TAR file and extract it
+COPY airbyte-integrations/connectors/${CONNECTOR_KEBAB_NAME}/build/distributions/${CONNECTOR_KEBAB_NAME}.tar /tmp/${CONNECTOR_KEBAB_NAME}.tar
+RUN tar xf /tmp/${CONNECTOR_KEBAB_NAME}.tar --strip-components=1 -C /airbyte && \
+    rm -rf /tmp/${CONNECTOR_KEBAB_NAME}.tar && \
+    chown -R airbyte:airbyte /airbyte
+
+# Set the non-root user
+USER airbyte
+
+# Set entrypoint
+ENTRYPOINT ["/airbyte/base.sh"]
+
+# Add labels
+LABEL io.airbyte.version="0.1.0"
+LABEL io.airbyte.name="airbyte/${CONNECTOR_KEBAB_NAME}"
+"""