fix: add explicit docker volume mapping (resolves permission denied errors); fix: remove 'cdk:low-code' logic with false-positives for manifest-only connector detection; feat: add option to auto-print full log file text, enabled by default in CI environments

.github/workflows/python_pytest.yml

@@ -75,6 +75,13 @@ jobs:
         name: fasttest-coverage
         path: htmlcov/
 
+    - name: Upload logs to GitHub Artifacts
+      if: failure()
+      uses: actions/upload-artifact@v4
+      with:
+        name: pytest-fast-test-logs
+        path: /tmp/airbyte/logs/
+
   pytest-no-creds:
     name: Pytest (No Creds)
     runs-on: ubuntu-latest
@@ -122,6 +129,13 @@ jobs:
         name: nocreds-test-coverage
         path: htmlcov/
 
+    - name: Upload logs to GitHub Artifacts
+      if: failure()
+      uses: actions/upload-artifact@v4
+      with:
+        name: pytest-no-creds-test-logs
+        path: /tmp/airbyte/logs/
+
   pytest:
     name: Pytest (All, Python ${{ matrix.python-version }}, ${{ matrix.os }})
     # Don't run on forks. Run on pushes to main, and on PRs that are not from forks.
@@ -189,6 +203,13 @@ jobs:
         name: py${{ matrix.python-version }}-${{ matrix.os }}-test-coverage
         path: htmlcov/
 
+    - name: Upload logs to GitHub Artifacts
+      if: failure()
+      uses: actions/upload-artifact@v4
+      with:
+        name: py${{ matrix.python-version }}-${{ matrix.os }}-test-logs
+        path: /tmp/airbyte/logs/
+
   dependency-analysis:
     name: Dependency Analysis with Deptry
     runs-on: ubuntu-latest

airbyte/_executors/util.py

@@ -42,8 +42,8 @@ def _try_get_source_manifest(
 
     Raises:
         - `PyAirbyteInputError`: If `source_name` is `None`.
-        - `HTTPError`: If fetching the URL was unsuccessful.
-        - `YAMLError`: If parsing the YAML failed.
+        - `AirbyteConnectorInstallationError`: If the registry file cannot be downloaded or if the
+          manifest YAML cannot be parsed.
     """
     if source_name is None:
         raise exc.PyAirbyteInputError(
@@ -62,7 +62,16 @@ def _try_get_source_manifest(
         url=manifest_url,
         headers={"User-Agent": f"PyAirbyte/{get_version()}"},
     )
-    response.raise_for_status()  # Raise HTTPError exception if the download failed
+    try:
+        response.raise_for_status()  # Raise HTTPError exception if the download failed
+    except requests.exceptions.HTTPError as ex:
+        raise exc.AirbyteConnectorInstallationError(
+            message="Failed to download the connector manifest.",
+            context={
+                "manifest_url": manifest_url,
+            },
+        ) from ex
+
     try:
         return cast("dict", yaml.safe_load(response.text))
     except yaml.YAMLError as ex:

airbyte/constants.py

@@ -109,3 +109,18 @@ def _str_to_bool(value: str) -> bool:
 This setting helps you make informed choices about data privacy and operation in restricted and
 air-gapped environments.
 """
+
+AIRBYTE_PRINT_FULL_ERROR_LOGS: bool = _str_to_bool(
+    os.getenv(
+        key="AIRBYTE_PRINT_FULL_ERROR_LOGS",
+        default=os.getenv("CI", "false"),
+    )
+)
+"""Whether to print full error logs when an error occurs.
+This setting helps in debugging by providing detailed logs when errors occur. This is especially
+helpful in ephemeral environments like CI/CD pipelines where log files may not be persisted after
+the pipeline run.
+
+If not set, the default value is `False` for non-CI environments.
+If running in a CI environment ("CI" env var is set), then the default value is `True`.
+"""

airbyte/exceptions.py

@@ -44,6 +44,8 @@
 from textwrap import indent
 from typing import TYPE_CHECKING, Any
 
+from airbyte.constants import AIRBYTE_PRINT_FULL_ERROR_LOGS
+
 
 if TYPE_CHECKING:
     from airbyte._util.api_duck_types import AirbyteApiResponseDuckType
@@ -68,6 +70,7 @@ class PyAirbyteError(Exception):
     help_url: str | None = None
     log_text: str | list[str] | None = None
     log_file: Path | None = None
+    print_full_log: bool = AIRBYTE_PRINT_FULL_ERROR_LOGS
     context: dict[str, Any] | None = None
     message: str | None = None
     original_exception: Exception | None = None
@@ -93,6 +96,7 @@ def __str__(self) -> str:
             "log_text",
             "context",
             "log_file",
+            "print_full_log",
             "original_exception",
         ]
         display_properties = {
@@ -119,9 +123,6 @@ def __str__(self) -> str:
         if context_str:
             exception_str += "\n    " + context_str
 
-        if self.log_file:
-            exception_str += f"\n    Log file: {self.log_file.absolute()!s}"
-
         if self.log_text:
             if isinstance(self.log_text, list):
                 self.log_text = "\n".join(self.log_text)
@@ -131,6 +132,16 @@ def __str__(self) -> str:
         if self.original_exception:
             exception_str += VERTICAL_SEPARATOR + f"\nCaused by: {self.original_exception!s}"
 
+        if self.log_file:
+            if self.print_full_log:
+                exception_str += (
+                    f"\n    Full log file text from {self.log_file.absolute()!s}:"
+                    + VERTICAL_SEPARATOR
+                    + self.log_file.read_text()
+                    + VERTICAL_SEPARATOR
+                )
+            else:
+                exception_str += f"\n    Log file: {self.log_file.absolute()!s}"
         return exception_str
 
     def __repr__(self) -> str:

airbyte/sources/registry.py

@@ -30,16 +30,13 @@
 _REGISTRY_ENV_VAR = "AIRBYTE_LOCAL_REGISTRY"
 _REGISTRY_URL = "https://connectors.airbyte.com/files/registries/v0/oss_registry.json"
 
-_LOWCODE_CDK_TAG = "cdk:low-code"
-
 _PYTHON_LANGUAGE = "python"
 _MANIFEST_ONLY_LANGUAGE = "manifest-only"
 
 _PYTHON_LANGUAGE_TAG = f"language:{_PYTHON_LANGUAGE}"
 _MANIFEST_ONLY_TAG = f"language:{_MANIFEST_ONLY_LANGUAGE}"
 
 _LOWCODE_CONNECTORS_NEEDING_PYTHON: list[str] = [
-    "source-adjust",
     "source-alpha-vantage",
     "source-amplitude",
     "source-apify-dataset",
@@ -50,13 +47,14 @@
     "source-braintree",
     "source-braze",
     "source-chargebee",
-    "source-close-com",
     "source-commercetools",
+    "source-eventbrite",
     "source-facebook-pages",
     "source-fastbill",
     "source-freshdesk",
     "source-gitlab",
     "source-gnews",
+    "source-gong",
     "source-greenhouse",
     "source-instagram",
     "source-instatus",
@@ -74,7 +72,6 @@
     "source-okta",
     "source-orb",
     "source-outreach",
-    "source-partnerstack",
     "source-paypal-transaction",
     "source-pinterest",
     "source-pipedrive",
@@ -83,17 +80,17 @@
     "source-prestashop",
     "source-public-apis",
     "source-qualaroo",
-    "source-quickbooks",
     "source-railz",
     "source-recharge",
     "source-recurly",
     "source-retently",
     "source-rss",
     "source-salesloft",
+    "source-service-now",
     "source-slack",
     "source-surveymonkey",
-    "source-tiktok-marketing",
     "source-the-guardian-api",
+    "source-tiktok-marketing",
     "source-trello",
     "source-typeform",
     "source-us-census",
@@ -106,21 +103,15 @@
     "source-zenloop",
     "source-zoom",
 ]
-_LOWCODE_CONNECTORS_FAILING_VALIDATION = [
-    "source-amazon-ads",
-]
+_LOWCODE_CONNECTORS_FAILING_VALIDATION: list[str] = []
 # Connectors that return 404 or some other misc exception.
-_LOWCODE_CONNECTORS_UNEXPECTED_ERRORS: list[str] = []
-# (CDK) FileNotFoundError: Unable to find spec.yaml or spec.json in the package.
-_LOWCODE_CDK_FILE_NOT_FOUND_ERRORS: list[str] = [
-    "source-apple-search-ads",
+_LOWCODE_CONNECTORS_UNEXPECTED_ERRORS: list[str] = [
+    "source-adjust",
+    "source-amazon-ads",
     "source-marketo",
-    "source-n8n",
-    "source-onesignal",
-    "source-postmarkapp",
-    "source-sentry",
-    "source-unleash",
 ]
+# (CDK) FileNotFoundError: Unable to find spec.yaml or spec.json in the package.
+_LOWCODE_CDK_FILE_NOT_FOUND_ERRORS: list[str] = []
 _LOWCODE_CONNECTORS_EXCLUDED: list[str] = [
     *_LOWCODE_CONNECTORS_FAILING_VALIDATION,
     *_LOWCODE_CONNECTORS_UNEXPECTED_ERRORS,
@@ -219,9 +210,6 @@ def _registry_entry_to_connector_metadata(entry: dict) -> ConnectorMetadata:
             InstallType.PYTHON if language == Language.PYTHON and pypi_enabled else None,
             InstallType.JAVA if language == Language.JAVA else None,
             InstallType.YAML if language == Language.MANIFEST_ONLY else None,
-            # TODO: Remove 'cdk:low-code' check once all connectors are migrated to manifest-only.
-            # https://github.com/airbytehq/PyAirbyte/issues/348
-            InstallType.YAML if _LOWCODE_CDK_TAG in tags else None,
         ]
         if x
     }