Merge pull request #216 from posit-dev/feat-data-path-strings

feat: data path strings

Author: rich-iannone

docs/_quarto.yml

@@ -211,6 +211,7 @@ quartodoc:
         - name: assistant
         - name: load_dataset
         - name: get_data_path
+        - name: connect_to_table
     - title: Utility Functions
       desc: >
         The *Utility Functions* group contains functions that are useful accessing metadata about

pointblank/__init__.py

@@ -30,6 +30,7 @@
 from pointblank.validate import (
     Validate,
     config,
+    connect_to_table,
     get_action_metadata,
     get_column_count,
     get_data_path,
@@ -63,6 +64,7 @@
     "load_dataset",
     "get_data_path",
     "config",
+    "connect_to_table",
     "preview",
     "missing_vals_tbl",
     "get_action_metadata",

pointblank/validate.py

@@ -104,6 +104,7 @@
     "Validate",
     "load_dataset",
     "config",
+    "connect_to_table",
     "preview",
     "missing_vals_tbl",
     "get_action_metadata",
@@ -2110,6 +2111,236 @@ def get_val_info(self) -> dict[str, any]:
         return self.val_info
 
 
+def connect_to_table(connection_string: str) -> Any:
+    """
+    Connect to a database table using a connection string.
+
+    This utility function tests whether a connection string leads to a valid table and returns
+    the table object if successful. It provides helpful error messages when no table is specified
+    or when backend dependencies are missing.
+
+    Parameters
+    ----------
+    connection_string
+        A database connection string with a required table specification using the `::table_name`
+        suffix. Supported formats are outlined in the *Supported Connection String Formats* section.
+
+    Returns
+    -------
+    Any
+        An Ibis table object for the specified database table.
+
+    Supported Connection String Formats
+    -----------------------------------
+    The `connection_string` parameter must include a valid connection string with a table name
+    specified using the `::` syntax. For example:
+
+    - DuckDB: `"duckdb:///path/to/database.ddb::table_name"`
+    - SQLite: `"sqlite:///path/to/database.db::table_name"`
+    - PostgreSQL: `"postgresql://user:password@localhost:5432/database::table_name"`
+    - MySQL: `"mysql://user:password@localhost:3306/database::table_name"`
+    - BigQuery: `"bigquery://project/dataset::table_name"`
+    - Snowflake: `"snowflake://user:password@account/database/schema::table_name"`
+
+    If the connection string does not include a table name, the function will attempt to connect to
+    the database and list available tables, providing guidance on how to specify a table.
+
+    Examples
+    --------
+    Connect to a DuckDB table:
+
+    ```{python}
+    import pointblank as pb
+
+    # Get path to a DuckDB database file from package data
+    duckdb_path = pb.get_data_path("game_revenue", "duckdb")
+
+    # Connect to the `game_revenue` table in the DuckDB database
+    game_revenue = pb.connect_to_table(f"duckdb:///{duckdb_path}::game_revenue")
+
+    # Use with the `preview()` function
+    pb.preview(game_revenue)
+    ```
+
+    Here are some backend-specific connection examples:
+
+    ```python
+    # PostgreSQL
+    pg_table = pb.connect_to_table(
+        "postgresql://user:password@localhost:5432/warehouse::customer_data"
+    )
+
+    # SQLite
+    sqlite_table = pb.connect_to_table("sqlite:///local_data.db::products")
+
+    # BigQuery
+    bq_table = pb.connect_to_table("bigquery://my-project/analytics::daily_metrics")
+    ```
+
+    This function requires the Ibis library with appropriate backend drivers:
+
+    ```bash
+    # You can install a set of common backends:
+    pip install 'ibis-framework[duckdb,postgres,mysql,sqlite]'
+
+    # ...or specific backends as needed:
+    pip install 'ibis-framework[duckdb]'    # for DuckDB
+    pip install 'ibis-framework[postgres]'  # for PostgreSQL
+    ```
+    """
+    # Check if Ibis is available
+    if not _is_lib_present(lib_name="ibis"):
+        raise ImportError(
+            "The Ibis library is not installed but is required for database connection strings.\n"
+            "Install it with: pip install 'ibis-framework[duckdb]' (or other backend as needed)"
+        )
+
+    import ibis
+
+    # Check if connection string includes table specification
+    if "::" not in connection_string:
+        # Try to connect to get available tables for helpful error message
+        try:
+            # Extract the base connection string (without table name)
+            base_connection = connection_string
+
+            # Connect to the database
+            conn = ibis.connect(base_connection)
+
+            # Get list of available tables
+            try:
+                available_tables = conn.list_tables()
+            except Exception:
+                available_tables = []
+
+            conn.disconnect()
+
+            # Create helpful error message
+            if available_tables:
+                table_list = "\n".join(f"  - {table}" for table in available_tables)
+                error_msg = (
+                    f"No table specified in connection string: {connection_string}\n\n"
+                    f"Available tables in the database:\n{table_list}\n\n"
+                    f"To access a specific table, use the format:\n"
+                    f"  {connection_string}::TABLE_NAME\n\n"
+                    f"Examples:\n"
+                )
+                # Add examples with first few table names
+                for table in available_tables[:3]:
+                    error_msg += f"  {connection_string}::{table}\n"
+            else:
+                error_msg = (
+                    f"No table specified in connection string: {connection_string}\n\n"
+                    f"No tables found in the database or unable to list tables.\n\n"
+                    f"To access a specific table, use the format:\n"
+                    f"  {connection_string}::TABLE_NAME"
+                )
+
+            raise ValueError(error_msg)
+
+        except Exception as e:
+            if isinstance(e, ValueError):
+                raise  # Re-raise our custom ValueError
+
+            # Check for backend-specific errors and provide installation guidance
+            error_str = str(e).lower()
+            backend_install_map = {
+                "duckdb": "pip install 'ibis-framework[duckdb]'",
+                "postgresql": "pip install 'ibis-framework[postgres]'",
+                "postgres": "pip install 'ibis-framework[postgres]'",
+                "mysql": "pip install 'ibis-framework[mysql]'",
+                "sqlite": "pip install 'ibis-framework[sqlite]'",
+                "bigquery": "pip install 'ibis-framework[bigquery]'",
+                "snowflake": "pip install 'ibis-framework[snowflake]'",
+            }
+
+            # Check if this is a missing backend dependency
+            for backend, install_cmd in backend_install_map.items():
+                if backend in error_str and ("not found" in error_str or "no module" in error_str):
+                    raise ConnectionError(
+                        f"Missing {backend.upper()} backend for Ibis. Install it with:\n"
+                        f"  {install_cmd}\n\n"
+                        f"Original error: {e}\n\n"
+                        f"Supported connection string formats:\n"
+                        f"- DuckDB: 'duckdb:///path/to/file.ddb::table_name'\n"
+                        f"- SQLite: 'sqlite:///path/to/file.db::table_name'\n"
+                        f"- PostgreSQL: 'postgresql://user:pass@host:port/db::table_name'\n"
+                        f"- MySQL: 'mysql://user:pass@host:port/db::table_name'\n"
+                        f"- BigQuery: 'bigquery://project/dataset::table_name'\n"
+                        f"- Snowflake: 'snowflake://user:pass@account/db/schema::table_name'\n"
+                        f"\nNote: Use '::table_name' to specify the table within the database."
+                    ) from e
+
+            # Generic connection error
+            raise ConnectionError(
+                f"Failed to connect to database using connection string: {connection_string}\n"
+                f"Error: {e}\n\n"
+                f"No table specified. Use the format: {connection_string}::TABLE_NAME"
+            ) from e
+
+    # Split connection string and table name
+    try:
+        base_connection, table_name = connection_string.rsplit("::", 1)
+    except ValueError:
+        raise ValueError(f"Invalid connection string format: {connection_string}")
+
+    # Connect to database and get table
+    try:
+        conn = ibis.connect(base_connection)
+        table = conn.table(table_name)
+        return table
+
+    except Exception as e:
+        # Check for backend-specific errors and provide installation guidance
+        error_str = str(e).lower()
+        backend_install_map = {
+            "duckdb": "pip install 'ibis-framework[duckdb]'",
+            "postgresql": "pip install 'ibis-framework[postgres]'",
+            "postgres": "pip install 'ibis-framework[postgres]'",
+            "mysql": "pip install 'ibis-framework[mysql]'",
+            "sqlite": "pip install 'ibis-framework[sqlite]'",
+            "bigquery": "pip install 'ibis-framework[bigquery]'",
+            "snowflake": "pip install 'ibis-framework[snowflake]'",
+        }
+
+        # Check if this is a missing backend dependency
+        for backend, install_cmd in backend_install_map.items():
+            if backend in error_str and ("not found" in error_str or "no module" in error_str):
+                raise ConnectionError(
+                    f"Missing {backend.upper()} backend for Ibis. Install it with:\n"
+                    f"  {install_cmd}\n\n"
+                    f"Original error: {e}"
+                ) from e
+
+        # Check if table doesn't exist
+        if "table" in error_str and ("not found" in error_str or "does not exist" in error_str):
+            # Try to get available tables for helpful message
+            try:
+                available_tables = conn.list_tables()
+                if available_tables:
+                    table_list = "\n".join(f"  - {table}" for table in available_tables)
+                    raise ValueError(
+                        f"Table '{table_name}' not found in database.\n\n"
+                        f"Available tables:\n{table_list}\n\n"
+                        f"Check the table name and try again with:\n"
+                        f"  {base_connection}::CORRECT_TABLE_NAME"
+                    ) from e
+                else:
+                    raise ValueError(
+                        f"Table '{table_name}' not found and no tables available in database."
+                    ) from e
+            except Exception:
+                raise ValueError(
+                    f"Table '{table_name}' not found in database. "
+                    f"Check the table name and connection string."
+                ) from e
+
+        # Generic connection error
+        raise ConnectionError(
+            f"Failed to connect to table '{table_name}' using: {base_connection}\nError: {e}"
+        ) from e
+
+
 @dataclass
 class Validate:
     """
@@ -2143,11 +2374,13 @@ class Validate:
     ----------
     data
         The table to validate, which could be a DataFrame object, an Ibis table object, a CSV
-        file path, or a Parquet file path. When providing a CSV or Parquet file path (as a string
-        or `pathlib.Path` object), the file will be automatically loaded using an available
-        DataFrame library (Polars or Pandas). Parquet input also supports glob patterns,
-        directories containing .parquet files, and Spark-style partitioned datasets. Read the
-        *Supported Input Table Types* section for details on the supported table types.
+        file path, a Parquet file path, or a database connection string. When providing a CSV or
+        Parquet file path (as a string or `pathlib.Path` object), the file will be automatically
+        loaded using an available DataFrame library (Polars or Pandas). Parquet input also supports
+        glob patterns, directories containing .parquet files, and Spark-style partitioned datasets.
+        Connection strings enable direct database access via Ibis with optional table specification
+        using the `::table_name` suffix. Read the *Supported Input Table Types* section for details
+        on the supported table types.
     tbl_name
         An optional name to assign to the input table object. If no value is provided, a name will
         be generated based on whatever information is available. This table name will be displayed
@@ -2220,6 +2453,7 @@ class Validate:
     - CSV files (string path or `pathlib.Path` object with `.csv` extension)
     - Parquet files (string path, `pathlib.Path` object, glob pattern, directory with `.parquet`
     extension, or partitioned dataset)
+    - Database connection strings (RFC 3986 URI format with optional table specification)
 
     The table types marked with an asterisk need to be prepared as Ibis tables (with type of
     `ibis.expr.types.relations.Table`). Furthermore, the use of `Validate` with such tables requires
@@ -2230,6 +2464,18 @@ class Validate:
     provided. The file will be automatically detected and loaded using the best available DataFrame
     library. The loading preference is Polars first, then Pandas as a fallback.
 
+    Connection strings follow database URL formats and must also specify a table using the
+    `::table_name` suffix. Examples include:
+
+    - `"duckdb:///path/to/database.ddb::table_name"`
+    - `"sqlite:///path/to/database.db::table_name"`
+    - `"postgresql://user:password@localhost:5432/database::table_name"`
+    - `"mysql://user:password@localhost:3306/database::table_name"`
+    - `"bigquery://project/dataset::table_name"`
+    - `"snowflake://user:password@account/database/schema::table_name"`
+
+    When using connection strings, the Ibis library with the appropriate backend driver is required.
+
     Thresholds
     ----------
     The `thresholds=` parameter is used to set the failure-condition levels for all validation
@@ -2612,6 +2858,33 @@ def send_report():
 
     Both Polars and Pandas handle partitioned datasets natively, so this works seamlessly with
     either DataFrame library. The loading preference is Polars first, then Pandas as a fallback.
+
+    ### Working with Database Connection Strings
+
+    The `Validate` class supports database connection strings for direct validation of database
+    tables. Connection strings must specify a table using the `::table_name` suffix:
+
+    ```{python}
+    # Get path to a DuckDB database file from package data
+    duckdb_path = pb.get_data_path("game_revenue", "duckdb")
+
+    validation_9 = (
+        pb.Validate(
+            data=f"duckdb:///{duckdb_path}::game_revenue",
+            label="DuckDB Game Revenue Validation"
+        )
+        .col_exists(["player_id", "session_id", "item_revenue"])
+        .col_vals_gt(columns="item_revenue", value=0)
+        .interrogate()
+    )
+
+    validation_9
+    ```
+
+    For comprehensive documentation on supported connection string formats, error handling, and
+    installation requirements, see the [`connect_to_table()`](`pointblank.connect_to_table`)
+    function. This function handles all the connection logic and provides helpful error messages
+    when table specifications are missing or backend dependencies are not installed.
     """
 
     data: FrameT | Any
@@ -2625,6 +2898,9 @@ def send_report():
     locale: str | None = None
 
     def __post_init__(self):
+        # Handle connection string input for the data parameter
+        self.data = self._process_connection_string_input(self.data)
+
         # Handle CSV file input for the data parameter
         self.data = self._process_csv_input(self.data)
 
@@ -2675,6 +2951,33 @@ def __post_init__(self):
 
         self.validation_info = []
 
+    def _process_connection_string_input(self, data: FrameT | Any) -> FrameT | Any:
+        """
+        Process data parameter to handle database connection strings.
+
+        Uses the `connect_to_table()` utility function to handle RFC 3986 URI-formatted
+        connection strings with table specifications. Returns the original data if it's
+        not a connection string.
+
+        For more details on supported connection string formats, see the documentation
+        for `connect_to_table()`.
+        """
+        # Check if data is a string that looks like a connection string
+        if not isinstance(data, str):
+            return data
+
+        # Basic connection string patterns
+        connection_patterns = [
+            "://",  # General URL-like pattern
+        ]
+
+        # Check if it looks like a connection string
+        if not any(pattern in data for pattern in connection_patterns):
+            return data
+
+        # Use the utility function to connect to the table
+        return connect_to_table(data)
+
     def _process_csv_input(self, data: FrameT | Any) -> FrameT | Any:
         """
         Process data parameter to handle CSV file inputs.