Merge branch 'main' into fix-plotly-utf8

Author: ankatiyar

kedro-airflow/pyproject.toml

@@ -98,5 +98,5 @@ select = [
 ]
 ignore = ["E501"]  # Black takes care of line-too-long
 
-[tool.ruff.per-file-ignores]
+[tool.ruff.lint.per-file-ignores]
 "{tests,features}/*" = ["T201", "PLR2004", "PLR0915", "PLW1510"]

kedro-datasets/RELEASE.md

@@ -1,11 +1,26 @@
 # Upcoming Release
+
 ## Major features and improvements
 
 - Group datasets documentation according to the dependencies to clean up the nav bar.
 
+- Added the following new **experimental** datasets:
+
+| Type                                | Description                                              | Location                                |
+| ----------------------------------- | -------------------------------------------------------- | --------------------------------------- |
+| `langchain.LangChainPromptDataset`  | Kedro dataset for loading LangChain prompts              | `kedro_datasets_experimental.langchain` |
+
 ## Bug fixes and other changes
 - Add HTMLPreview type.
 
+## Major features and improvements
+
+- Added the following new experimental datasets:
+
+| Type                           | Description                                                   | Location                             |
+|--------------------------------|---------------------------------------------------------------|--------------------------------------|
+| `pypdf.PDFDataset`             | A dataset to read PDF files and extract text using pypdf      | `kedro_datasets_experimental.pypdf`  |
+
 # Release 8.1.0
 ## Major features and improvements
 
@@ -15,14 +30,16 @@
 | ------------------------------ | ------------------------------------------------------------- | ------------------------------------ |
 | `polars.PolarsDatabaseDataset` | A dataset to load and save data to a SQL backend using Polars | `kedro_datasets_experimental.polars` |
 
+- Added `mode` save argument to `ibis.TableDataset`, supporting "append", "overwrite", "error"/"errorifexists", and "ignore" save modes. The deprecated `overwrite` save argument is mapped to `mode` for backward compatibility and will be removed in a future release. Specifying both `mode` and `overwrite` results in an error.
+
 ## Bug fixes and other changes
 
 - Added primary key constraint to BaseTable.
 - Added save/load with `use_pyarrow=True` save_args for LazyPolarsDataset partitioned parquet files.
 - Updated the json schema for Kedro 1.0.0.
 
-## Breaking Changes
 ## Community contributions
+
 - [Minura Punchihewa](https://github.com/MinuraPunchihewa)
 - [gitgud5000](https://github.com/gitgud5000)
 
@@ -56,7 +73,6 @@ Many thanks to the following Kedroids for contributing PRs to this release:
 - [Seohyun Park](https://github.com/soyamimi)
 - [Daniel Russell-Brain](https://github.com/killerfridge)
 
-
 # Release 7.0.0
 
 ## Major features and improvements

kedro-datasets/docs/api/kedro_datasets_experimental/index.md

@@ -11,7 +11,9 @@ Name | Description
 [langchain.ChatCohereDataset](langchain.ChatCohereDataset.md) | ``ChatCohereDataset`` loads a ChatCohere `langchain` model.
 [langchain.ChatOpenAIDataset](langchain.ChatOpenAIDataset.md) | OpenAI dataset used to access credentials at runtime.
 [langchain.OpenAIEmbeddingsDataset](langchain.OpenAIEmbeddingsDataset.md) | ``OpenAIEmbeddingsDataset`` loads a OpenAIEmbeddings `langchain` model.
+[langchain.LangChainPromptDataset](langchain.LangChainPromptDataset.md) | ``LangChainPromptDataset`` loads a `langchain` prompt template.
 [netcdf.NetCDFDataset](netcdf.NetCDFDataset.md) | ``NetCDFDataset`` loads/saves data from/to a NetCDF file using an underlying filesystem (e.g.: local, S3, GCS). It uses xarray to handle the NetCDF file.
+[pypdf.PDFDataset](pypdf.PDFDataset.md) | ``PDFDataset`` loads data from PDF files using pypdf to extract text from pages. Read-only dataset.
 [polars.PolarsDatabaseDataset](polars.PolarsDatabaseDataset.md) | ``PolarsDatabaseDataset`` implementation to access databases as Polars DataFrames. It supports reading from a SQL query and writing to a database table.
 [prophet.ProphetModelDataset](prophet.ProphetModelDataset.md) | ``ProphetModelDataset`` loads/saves Facebook Prophet models to a JSON file using an underlying filesystem (e.g., local, S3, GCS). It uses Prophet's built-in serialisation to handle the JSON file.
 [pytorch.PyTorchDataset](pytorch.PyTorchDataset.md) | ``PyTorchDataset`` loads and saves PyTorch models' `state_dict` using PyTorch's recommended zipfile serialization protocol. To avoid security issues with Pickle.

kedro-datasets/docs/api/kedro_datasets_experimental/langchain.LangChainPromptDataset.md

@@ -0,0 +1,4 @@
+::: kedro_datasets_experimental.langchain.LangChainPromptDataset
+    options:
+        members: true
+        show_source: true

kedro-datasets/docs/api/kedro_datasets_experimental/pypdf.PDFDataset.md

@@ -0,0 +1,4 @@
+::: kedro_datasets_experimental.pypdf.PDFDataset
+    options:
+        members: true
+        show_source: true

kedro-datasets/kedro_datasets/ibis/table_dataset.py

@@ -1,18 +1,37 @@
 """Provide data loading and saving functionality for Ibis's backends."""
 from __future__ import annotations
 
+import sys
 from copy import deepcopy
+from enum import auto
 from typing import TYPE_CHECKING, Any, ClassVar
+from warnings import warn
+
+if sys.version_info >= (3, 11):
+    from enum import StrEnum  # pragma: no cover
+else:
+    from backports.strenum import StrEnum  # pragma: no cover
 
 import ibis.expr.types as ir
-from kedro.io import AbstractDataset
+from kedro.io import AbstractDataset, DatasetError
 
+from kedro_datasets import KedroDeprecationWarning
 from kedro_datasets._utils import ConnectionMixin
 
 if TYPE_CHECKING:
     from ibis import BaseBackend
 
 
+class SaveMode(StrEnum):
+    """`SaveMode` is used to specify the expected behavior of saving a table."""
+
+    APPEND = auto()
+    OVERWRITE = auto()
+    ERROR = auto()
+    ERRORIFEXISTS = auto()
+    IGNORE = auto()
+
+
 class TableDataset(ConnectionMixin, AbstractDataset[ir.Table, ir.Table]):
     """`TableDataset` loads/saves data from/to Ibis table expressions.
 
@@ -28,14 +47,18 @@ class TableDataset(ConnectionMixin, AbstractDataset[ir.Table, ir.Table]):
             database: company.db
           save_args:
             materialized: table
+            mode: append
 
         motorbikes:
           type: ibis.TableDataset
           table_name: motorbikes
           connection:
             backend: duckdb
             database: company.db
-        ```
+          save_args:
+            materialized: view
+            mode: overwrite
+    ```
 
         Using the [Python API](https://docs.kedro.org/en/stable/catalog-data/advanced_data_catalog_usage/):
 
@@ -62,7 +85,7 @@ class TableDataset(ConnectionMixin, AbstractDataset[ir.Table, ir.Table]):
     DEFAULT_LOAD_ARGS: ClassVar[dict[str, Any]] = {}
     DEFAULT_SAVE_ARGS: ClassVar[dict[str, Any]] = {
         "materialized": "view",
-        "overwrite": True,
+        "mode": "overwrite",
     }
 
     _CONNECTION_GROUP: ClassVar[str] = "ibis"
@@ -109,7 +132,12 @@ def __init__(  # noqa: PLR0913
                 `create_{materialized}` method. By default, ``ir.Table``
                 objects are materialized as views. To save a table using
                 a different materialization strategy, supply a value for
-                `materialized` in `save_args`.
+                `materialized` in `save_args`. The `mode` parameter controls
+                the behavior when saving data:
+                - _"overwrite"_: Overwrite existing data in the table.
+                - _"append"_: Append contents of the new data to the existing table (does not overwrite).
+                - _"error"_ or _"errorifexists"_: Throw an exception if the table already exists.
+                - _"ignore"_: Silently ignore the operation if the table already exists.
             metadata: Any arbitrary metadata. This is ignored by Kedro,
                 but may be consumed by users or external plugins.
         """
@@ -134,6 +162,28 @@ def __init__(  # noqa: PLR0913
 
         self._materialized = self._save_args.pop("materialized")
 
+        # Handle mode/overwrite conflict.
+        if save_args and "mode" in save_args and "overwrite" in self._save_args:
+            raise ValueError("Cannot specify both 'mode' and deprecated 'overwrite'.")
+
+        # Map legacy overwrite if present.
+        if "overwrite" in self._save_args:
+            warn(
+                "'overwrite' is deprecated and will be removed in a future release. "
+                "Please use 'mode' instead.",
+                KedroDeprecationWarning,
+                stacklevel=2,
+            )
+            legacy = self._save_args.pop("overwrite")
+            # Remove any lingering 'mode' key from defaults to avoid
+            # leaking into writer kwargs.
+            del self._save_args["mode"]
+            mode = "overwrite" if legacy else "error"
+        else:
+            mode = self._save_args.pop("mode")
+
+        self._mode = SaveMode(mode)
+
     def _connect(self) -> BaseBackend:
         import ibis  # noqa: PLC0415
 
@@ -151,7 +201,21 @@ def load(self) -> ir.Table:
 
     def save(self, data: ir.Table) -> None:
         writer = getattr(self.connection, f"create_{self._materialized}")
-        writer(self._table_name, data, **self._save_args)
+        if self._mode == "append":
+            if not self._exists():
+                writer(self._table_name, data, overwrite=False, **self._save_args)
+            elif hasattr(self.connection, "insert"):
+                self.connection.insert(self._table_name, data, **self._save_args)
+            else:
+                raise DatasetError(
+                    f"The {self.connection.name} backend for Ibis does not support inserts."
+                )
+        elif self._mode == "overwrite":
+            writer(self._table_name, data, overwrite=True, **self._save_args)
+        elif self._mode in {"error", "errorifexists"}:
+            writer(self._table_name, data, overwrite=False, **self._save_args)
+        elif self._mode == "ignore" and not self._exists():
+            writer(self._table_name, data, overwrite=False, **self._save_args)
 
     def _describe(self) -> dict[str, Any]:
         load_args = deepcopy(self._load_args)
@@ -165,6 +229,7 @@ def _describe(self) -> dict[str, Any]:
             "load_args": load_args,
             "save_args": save_args,
             "materialized": self._materialized,
+            "mode": self._mode,
         }
 
     def _exists(self) -> bool:

kedro-datasets/kedro_datasets_experimental/langchain/__init__.py

@@ -7,19 +7,23 @@
     from ._anthropic import ChatAnthropicDataset
     from ._cohere import ChatCohereDataset
     from ._openai import ChatOpenAIDataset, OpenAIEmbeddingsDataset
+    from .langchain_prompt_dataset import LangChainPromptDataset
+
 except (ImportError, RuntimeError):
     # For documentation builds that might fail due to dependency issues
     # https://github.com/pylint-dev/pylint/issues/4300#issuecomment-1043601901
     ChatAnthropicDataset: Any
     ChatOpenAIDataset: Any
     OpenAIEmbeddingsDataset: Any
     ChatCohereDataset: Any
+    LangChainPromptDataset: Any
 
 __getattr__, __dir__, __all__ = lazy.attach(
     __name__,
     submod_attrs={
         "_openai": ["ChatOpenAIDataset", "OpenAIEmbeddingsDataset"],
         "_anthropic": ["ChatAnthropicDataset"],
         "_cohere": ["ChatCohereDataset"],
+        "langchain_prompt_dataset": ["LangChainPromptDataset"],
     },
 )