diff --git a/debug_manifest/.gitignore b/debug_manifest/.gitignore new file mode 100644 index 000000000..7fbd5d4ce --- /dev/null +++ b/debug_manifest/.gitignore @@ -0,0 +1,5 @@ +# Ignore all resource files +resources/* + +# But do not ignore .gitkeep files, to keep an empty `resources` directory +!resources/.gitkeep diff --git a/debug_manifest/README.md b/debug_manifest/README.md new file mode 100644 index 000000000..fc747333f --- /dev/null +++ b/debug_manifest/README.md @@ -0,0 +1,64 @@ +# Configuring the Debugger + +## For VSCode + +### Set Up the debugger configuration (one-time setup step) +To configure the debugger in VSCode to run the `debug_manifest`, follow these steps: + +1. Clone or Open the existing `airbyte-python-cdk` project in VSCode. +2. Click on the Run and Debug icon in the Activity Bar on the side of the window. +3. Click on the `create a launch.json file` link to create a new configuration file. +4. Select `Python` from the list of environments. +5. Replace the contents of the generated `launch.json` file with the following configuration: + +```json +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Debug Manifest", + "type": "debugpy", + "request": "launch", + "console": "integratedTerminal", + "cwd": "${workspaceFolder}/debug_manifest", + "python": "/bin/python", + "module": "debug_manifest", + "args": [ + // SPECIFY THE COMMAND: [spec, check, discover, read] + "read", + // SPECIFY THE CONFIG + "--config", + // PATH TO THE CONFIG FILE + "resources/config.json", + // SPECIFY THE CATALOG + "--catalog", + // PATH TO THE CATALOG FILE + "resources/catalog.json", + // SPECIFY THE STATE (optional) + // "--state", + // PATH TO THE STATE FILE + // "resources/state.json", + // ADDITIONAL FLAGS, like `--debug` (optional) + "--debug" + ], + } + ] +} +``` + +6. Save the `launch.json` file. +7. Install `CDK dependencies` by running `poetry install --all-extras` +8. Replace the `"python": "/bin/python"` with the correct interpreter `PATH` pointing to the `CDK env` installed from Step `7` (use `which python` to have the complete python path), to wire the CDK env to the debugger. Alternatively you can switch the default interpreter you use in your IDE. +### Set up the necessary resources to use within the manifest-only connector +* These resources are ignored by `git`, in the `.gitignore`, thus should not be committed +1. Put the `config.json` inside the `/airbyte_cdk/debug_manifest/resources` (this will hold the `source input configuration`). +2. Put the `catalog.json` inside the `/airbyte_cdk/debug_manifest/resources` (this will hold the `configured catalog` for the target source). +3. Put the `manifest.yaml` inside the `/airbyte_cdk/debug_manifest/resources` +4. (Optional) Put the `state.json` inside the `/airbyte_cdk/debug_manifest/resources` + +## Debugging Steps +1. Set any necessary breakpoints in your code, or `CDK` components code. +2. Press `F5` / `Shift + CMD + D` / click the green play button in the `Run and Debug` view to start debugging. +3. Iterate over the `2` and `3`, to debug your `manifest-only` source. + +Basically, you're now able to run the `manifest-only` sources like the regular python source, with `spec`, `check`, `discover` and `read` commands, as well as having the `--debug` alongside. diff --git a/debug_manifest/debug_manifest.py b/debug_manifest/debug_manifest.py new file mode 100644 index 000000000..c520d0b0c --- /dev/null +++ b/debug_manifest/debug_manifest.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import sys +from typing import Any, Mapping + +from airbyte_cdk.entrypoint import launch +from airbyte_cdk.sources.declarative.yaml_declarative_source import ( + YamlDeclarativeSource, +) + +configuration: Mapping[str, Any] = { + "path_to_yaml": "resources/manifest.yaml", +} + + +def debug_manifest(source: YamlDeclarativeSource, args: list[str]) -> None: + """ + Run the debug manifest with the given source and arguments. + """ + launch(source, args) + + +if __name__ == "__main__": + debug_manifest( + YamlDeclarativeSource(**configuration), + sys.argv[1:], + ) diff --git a/debug_manifest/resources/.gitkeep b/debug_manifest/resources/.gitkeep new file mode 100644 index 000000000..e69de29bb