Merge branch 'main' into feature/iceberg-merge-into-full-functionality

Author: kukushking

.github/workflows/pr-linter.yml

@@ -10,10 +10,11 @@ on:
 
 jobs:
   lint:
+    name: Validate PR title
     runs-on: ubuntu-latest
     permissions:
-      statuses: write
+      pull-requests: read
     steps:
-      - uses: aslafy-z/conventional-pr-title-action@v3
+      - uses: amannn/action-semantic-pull-request@v6
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

awswrangler/_distributed.py

@@ -177,7 +177,7 @@ def get(cls) -> MemoryFormatEnum:
             return cls._enum if cls._enum else cls.get_installed()
 
     @classmethod
-    def set(cls, name: EngineLiteral) -> None:
+    def set(cls, name: MemoryFormatLiteral) -> None:
         """Set the memory format."""
         with cls._lock:
             cls._enum = MemoryFormatEnum[name.upper()]

awswrangler/athena/_write_iceberg.py

@@ -707,6 +707,7 @@ def to_iceberg(  # noqa: PLR0913
                 keep_files=False,
                 data_source=data_source,
                 workgroup=workgroup,
+                s3_output=s3_output,
                 encryption=encryption,
                 kms_key=kms_key,
                 boto3_session=boto3_session,
@@ -722,6 +723,7 @@ def to_iceberg(  # noqa: PLR0913
                 wg_config=wg_config,
                 database=database,
                 data_source=data_source,
+                s3_output=s3_output,
                 encryption=encryption,
                 kms_key=kms_key,
                 boto3_session=boto3_session,

awswrangler/oracle.py

@@ -607,7 +607,7 @@ def detect_oracle_decimal_datatype(cursor: Any) -> dict[str, pa.DataType]:
     if isinstance(cursor, oracledb.Cursor):
         # Oracle stores DECIMAL as the NUMBER type
 
-        for name, db_type, display_size, internal_size, precision, scale, null_ok in cursor.description:
+        for name, db_type, display_size, internal_size, precision, scale, null_ok in cursor.description:  # type: ignore
             _logger.debug((name, db_type, display_size, internal_size, precision, scale, null_ok))
 
             if db_type == oracledb.DB_TYPE_NUMBER and scale is not None and scale > 0:

awswrangler/s3/_read_deltalake.py

@@ -25,6 +25,7 @@ def _set_default_storage_options_kwargs(
 ) -> dict[str, Any]:
     defaults = {key.upper(): value for key, value in _utils.boto3_to_primitives(boto3_session=boto3_session).items()}
     defaults["AWS_REGION"] = defaults.pop("REGION_NAME")
+    defaults["AWS_SESSION_TOKEN"] = "" if defaults["AWS_SESSION_TOKEN"] is None else defaults["AWS_SESSION_TOKEN"]
     s3_additional_kwargs = s3_additional_kwargs or {}
     return {
         **defaults,

awswrangler/s3/_write_deltalake.py

@@ -29,7 +29,7 @@ def _set_default_storage_options_kwargs(
 ) -> dict[str, Any]:
     defaults = {key.upper(): value for key, value in _utils.boto3_to_primitives(boto3_session=boto3_session).items()}
     defaults["AWS_REGION"] = defaults.pop("REGION_NAME")
-
+    defaults["AWS_SESSION_TOKEN"] = "" if defaults["AWS_SESSION_TOKEN"] is None else defaults["AWS_SESSION_TOKEN"]
     s3_additional_kwargs = s3_additional_kwargs or {}
 
     s3_lock_arguments = {}

test_infra/pyproject.toml

@@ -2,7 +2,7 @@
 name = "awswrangler-test-infrastructure"
 version = "3.13.0"
 description = "CDK test infrastructure for AWS SDK for pandas"
-authors = ["Amazon Web Services"]
+authors = [{ name = "Amazon Web Services" }]
 license = {text = "Apache-2.0"}
 requires-python = ">=3.9, <4.0"
 

tox.ini

@@ -3,6 +3,7 @@ envlist = py{39,310,311,312,313}
 isolated_build = True
 
 [testenv]
+runner = uv-venv-lock-runner
 passenv =
        AWS_PROFILE
        AWS_DEFAULT_REGION
@@ -15,6 +16,7 @@ allowlist_externals =
        pytest
        uv
 commands_pre =
+       uv pip install pyarrow==20.0.0
        uv sync --frozen --verbose --extra deltalake --extra gremlin --extra mysql --extra opencypher --extra opensearch --extra oracle --extra postgres --extra redshift --extra sparql --extra sqlserver --extra geopandas
 commands =
        uv run pytest -n {posargs} -s -v --timeout=600 --reruns=1 --reruns-delay=30 \
@@ -38,6 +40,7 @@ allowlist_externals =
        pytest
        uv
 commands_pre =
+       uv pip install pyarrow==20.0.0
        uv sync --frozen --verbose --all-extras
 commands =
        uv run pytest -n {posargs} -s -v --timeout=600 --reruns=1 --reruns-delay=30 \