From a83bfea6497ddc75ca5708d2884c6141e0b2623a Mon Sep 17 00:00:00 2001 From: ChristoGrab Date: Wed, 9 Apr 2025 17:12:37 -0700 Subject: [PATCH 1/5] update dockerfile to use sdm cli --- Dockerfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index d2bce875d..2916d6f9b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ COPY dist/*.whl ./dist/ RUN poetry config virtualenvs.create false \ && poetry install --only main --no-interaction --no-ansi || true -# Build and install the package +# Install source-declarative-manifest CLI and its dependencies RUN pip install dist/*.whl # Recreate the original structure @@ -33,7 +33,6 @@ RUN rm -rf dist/ pyproject.toml poetry.lock README.md # Set ownership of /airbyte to the non-root airbyte user and group (1000:1000) RUN chown -R 1000:1000 /airbyte -# Set the entrypoint -ENV AIRBYTE_ENTRYPOINT="python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] +# Set the entrypoint to use the source-declarative-manifest CLI +ENTRYPOINT ["source-declarative-manifest"] USER airbyte From 824c63f22432fe95a9b9eca96f7d490ba8b63a9f Mon Sep 17 00:00:00 2001 From: ChristoGrab Date: Thu, 17 Apr 2025 11:20:43 -0700 Subject: [PATCH 2/5] add readme first draft --- airbyte_cdk/cli/README.md | 65 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 airbyte_cdk/cli/README.md diff --git a/airbyte_cdk/cli/README.md b/airbyte_cdk/cli/README.md new file mode 100644 index 000000000..410ff8226 --- /dev/null +++ b/airbyte_cdk/cli/README.md @@ -0,0 +1,65 @@ +# Source Declarative Manifest CLI Usage Guide + +This guide explains how to install and use the Source Declarative Manifest (SDM) CLI tool for Airbyte connector development. + +## Installation + +### Standard Installation + +```bash +pipx install airbyte-cdk +``` + +### Alternative Installation (If encountering distutils errors) + +If you encounter errors related to missing distutils module, try using this alternative approach: + +```bash +python -m pip install --user pipx +python -m pipx install airbyte-cdk +``` + +## Using the CLI + +The SDM CLI follows standard Airbyte connector command patterns: + +```bash +source-declarative-manifest [command] --config /path/to/config.json +``` + +Where [command] can be: + +spec - Show connector specification +check - Verify connection to the source +discover - List available streams +read - Read data from streams + +:::caution +When developing locally (outside a Docker container), the CLI operates in "remote manifest mode" and expects your manifest to be included in your configuration file. +::: + +### Steps for Local Testing + +1. Convert your manifest from YAML to JSON + +Your manifest is defined in YAML, but must be converted to JSON for the config file. You can use an [online tool](https://onlineyamltools.com/convert-yaml-to-json) to do so. + +Create a config file that includes both your config parameters AND the manifest. Add your entire manifest as a JSON object under the `__injected_declarative_manifest` key + +Example: + +```json +{ + "api_key": "very_secret_key", + "start_time": "04:20", + "__injected_declarative_manifest": { + // Insert the JSON version of your manifest here + } +} +``` + +2. Run the command with your config file + +```bash +source-declarative-manifest check --config /relative/path/to/your/config.json +``` From 9e1f74266524c51ead039211f2e80a5a77921f44 Mon Sep 17 00:00:00 2001 From: ChristoGrab Date: Thu, 17 Apr 2025 12:50:05 -0700 Subject: [PATCH 3/5] chore: update readme --- airbyte_cdk/cli/README.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/airbyte_cdk/cli/README.md b/airbyte_cdk/cli/README.md index 410ff8226..9e3becdd9 100644 --- a/airbyte_cdk/cli/README.md +++ b/airbyte_cdk/cli/README.md @@ -10,12 +10,9 @@ This guide explains how to install and use the Source Declarative Manifest (SDM) pipx install airbyte-cdk ``` -### Alternative Installation (If encountering distutils errors) - -If you encounter errors related to missing distutils module, try using this alternative approach: +If you encounter an error related to a missing `distutils` module, very that you are running Python version `<=3.11` and try running: ```bash -python -m pip install --user pipx python -m pipx install airbyte-cdk ``` @@ -58,8 +55,9 @@ Example: } ``` -2. Run the command with your config file +2. Run the command against your config file ```bash -source-declarative-manifest check --config /relative/path/to/your/config.json +source-declarative-manifest check --config /relative/path/to/config.json +source-declarative-manifest read --config /relative/path/to/config.json --catalog /relative/path/to/catalog.json ``` From 7b75141a49862dfb8d5f2f9392ffdcf7d036e7ed Mon Sep 17 00:00:00 2001 From: Christo Grabowski <108154848+ChristoGrab@users.noreply.github.com> Date: Thu, 17 Apr 2025 15:52:08 -0400 Subject: [PATCH 4/5] Update airbyte_cdk/cli/README.md --- airbyte_cdk/cli/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte_cdk/cli/README.md b/airbyte_cdk/cli/README.md index 9e3becdd9..0943ebf8e 100644 --- a/airbyte_cdk/cli/README.md +++ b/airbyte_cdk/cli/README.md @@ -10,7 +10,7 @@ This guide explains how to install and use the Source Declarative Manifest (SDM) pipx install airbyte-cdk ``` -If you encounter an error related to a missing `distutils` module, very that you are running Python version `<=3.11` and try running: +If you encounter an error related to a missing `distutils` module, verify that you are running Python version `<=3.11` and try running: ```bash python -m pipx install airbyte-cdk From 79f28c0550ad6a9c9745d41fdcea5e37cac94a5d Mon Sep 17 00:00:00 2001 From: ChristoGrab Date: Thu, 17 Apr 2025 13:04:03 -0700 Subject: [PATCH 5/5] chore: add AIRBYTE_ENTRYPOINT --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 2916d6f9b..1a3e19bee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,5 +34,6 @@ RUN rm -rf dist/ pyproject.toml poetry.lock README.md RUN chown -R 1000:1000 /airbyte # Set the entrypoint to use the source-declarative-manifest CLI +ENV AIRBYTE_ENTRYPOINT="source-declarative-manifest" ENTRYPOINT ["source-declarative-manifest"] USER airbyte