Feat: Clean up public API and improve docs (#109)

Author: aaronsteers

airbyte/__init__.py

@@ -2,27 +2,34 @@
 
 .. include:: ../README.md
 
+## API Reference
+
 """
 from __future__ import annotations
 
-from airbyte import caches, datasets, registry, secrets
-from airbyte._factories.connector_factories import get_source
+from airbyte import caches, datasets, documents, exceptions, results, secrets, sources
 from airbyte.caches.bigquery import BigQueryCache
 from airbyte.caches.duckdb import DuckDBCache
-from airbyte.caches.factories import get_default_cache, new_local_cache
+from airbyte.caches.util import get_default_cache, new_local_cache
 from airbyte.datasets import CachedDataset
-from airbyte.registry import get_available_connectors
 from airbyte.results import ReadResult
 from airbyte.secrets import SecretSource, get_secret
-from airbyte.source import Source
+from airbyte.sources import registry
+from airbyte.sources.base import Source
+from airbyte.sources.registry import get_available_connectors
+from airbyte.sources.util import get_source
 
 
 __all__ = [
     # Modules
     "caches",
     "datasets",
+    "documents",
+    "exceptions",
     "registry",
+    "results",
     "secrets",
+    "sources",
     # Factories
     "get_available_connectors",
     "get_default_cache",

airbyte/_executor.py

@@ -13,7 +13,7 @@
 from rich import print
 
 from airbyte import exceptions as exc
-from airbyte.registry import ConnectorMetadata
+from airbyte.sources.registry import ConnectorMetadata
 
 
 if TYPE_CHECKING:

airbyte/_factories/__init__.py

@@ -49,7 +49,7 @@
 
 if TYPE_CHECKING:
     from airbyte.caches.base import CacheBase
-    from airbyte.source import Source
+    from airbyte.sources.base import Source
 
 
 HASH_SEED = "PyAirbyte:"

airbyte/_util/telemetry.py

@@ -2,12 +2,13 @@
 """Base module for all caches."""
 from __future__ import annotations
 
+from airbyte.caches import bigquery, duckdb, motherduck, postgres, snowflake, util
 from airbyte.caches.base import CacheBase
 from airbyte.caches.duckdb import DuckDBCache
-from airbyte.caches.factories import get_default_cache, new_local_cache
 from airbyte.caches.motherduck import MotherDuckCache
 from airbyte.caches.postgres import PostgresCache
 from airbyte.caches.snowflake import SnowflakeCache
+from airbyte.caches.util import get_default_cache, new_local_cache
 
 
 # We export these classes for easy access: `airbyte.caches...`
@@ -21,4 +22,12 @@
     "MotherDuckCache",
     "PostgresCache",
     "SnowflakeCache",
+    # Submodules,
+    "util",
+    "base",
+    "bigquery",
+    "duckdb",
+    "motherduck",
+    "postgres",
+    "snowflake",
 ]

airbyte/caches/__init__.py

@@ -1,4 +1,6 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+"""Utility functions for working with caches."""
+
 from __future__ import annotations
 
 from pathlib import Path

airbyte/caches/factories.py airbyte/caches/util.pyairbyte/caches/factories.py renamed to airbyte/caches/util.py

@@ -1,5 +1,13 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
-"""Methods for converting Airbyte records into documents."""
+"""This module contains the `Documents` class for converting Airbyte records into documents.
+
+Generally you will not create `Documents` objects directly. Instead, you can use one of the
+following methods to generate documents from records:
+
+- `Source.get_documents()`: Get an iterable of documents from a source.
+- `Dataset.to_documents()`: Get an iterable of documents from a dataset.
+"""
+
 from __future__ import annotations
 
 from typing import TYPE_CHECKING, Any, Optional
@@ -46,3 +54,8 @@ def page_content(self) -> str:
         with the LangChain project's `Document` class.
         """
         return self.content
+
+
+__all__ = [
+    "Document",
+]

airbyte/documents.py

@@ -6,10 +6,12 @@
 property prints rather than sentence-like string concatenation.
 
 E.g. Instead of this:
-> Subprocess failed with exit code '1'
+
+> `Subprocess failed with exit code '1'`
 
 We do this:
-> Subprocess failed. (exit_code=1)
+
+> `Subprocess failed. (exit_code=1)`
 
 The benefit of this approach is that we can easily support structured logging, and we can
 easily add new properties to exceptions without having to update all the places where they

airbyte/exceptions.py

@@ -126,3 +126,9 @@ def get_secret(
         secret_name=secret_name,
         sources=[str(s) for s in sources],
     )
+
+
+__all__ = [
+    "get_secret",
+    "SecretSource",
+]

airbyte/secrets.py

@@ -0,0 +1,25 @@
+"""Sources connectors module for PyAirbyte."""
+from __future__ import annotations
+
+from airbyte.sources import base, util
+from airbyte.sources.registry import (
+    ConnectorMetadata,
+    get_available_connectors,
+    get_connector_metadata,
+)
+from airbyte.sources.util import get_source
+
+
+__all__ = [
+    # Submodules
+    "base",
+    "util",
+    # Factories
+    "get_source",
+    # Helper Functions
+    "get_available_connectors",
+    "get_connector_metadata",
+    # Classes
+    "Source",
+    "ConnectorMetadata",
+]