Merge remote-tracking branch 'upstream/main' into gb/merge-upstream-a… (#30)

* Merge remote-tracking branch 'upstream/main' into gb/merge-upstream-arrow-57.1

* Fix merge mistakes

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* format

* Disable python bindings again

* Fix clippy errors

* Enable tests

* Fix merge mistake

* .

* .

* .

* Clippy fix

* fix: Reserved sort order ID cannot contain any fields (apache#1978)

## Which issue does this PR close?

- Closes apache#1963.

## What changes are included in this PR?

This change validates that table metadata with reserved sort order ID
(0) cannot contain fields associated with it. If this is found, we error
out instead of silently parsing arbitrary field values.

## Are these changes tested?

Added the unit test described in the issue and verified that the check
is now enforced.

* feat(datafusion): Add LIMIT pushdown support (apache#2006)

Implement LIMIT pushdown to optimize queries with LIMIT clauses by
stopping data processing once the limit is reached. This reduces
unnecessary I/O and computation for queries that only need a subset of
rows.

Changes:
- Add limit field to IcebergTableScan to track row limit
- Apply limit at stream level by filtering/slicing record batches
- Update IcebergTableProvider and IcebergStaticTableProvider to pass
limit parameter to scan
- Add comprehensive tests for limit pushdown functionality

## Which issue does this PR close?


- Closes #.

## What changes are included in this PR?


## Are these changes tested?

Co-authored-by: Claude Sonnet 4.5 <noreply@anthropic.com>

* Redo comment

* Add case-sensitive attribute to incremental scan for consistency

---------

Co-authored-by: Aditya Subrahmanyan <aditya.subrahmanyan@gmail.com>
Co-authored-by: Liang-Chi Hsieh <viirya@gmail.com>
Co-authored-by: Claude Sonnet 4.5 <noreply@anthropic.com>

Author: 4 people

.asf.yaml

@@ -43,7 +43,9 @@ github:
         dismiss_stale_reviews: false
 
       required_linear_history: true
-  del_branch_on_merge: true
+  pull_requests:
+    # auto-delete head branches after being merged
+    del_branch_on_merge: true
   features:
     wiki: false
     issues: true

.cargo/audit.toml

@@ -25,4 +25,15 @@ ignore = [
   #
   # Introduced by hive_metastore, tracked at https://github.com/cloudwego/pilota/issues/293
   "RUSTSEC-2024-0388",
+  # `paste` is unmaintained; consider using an alternative
+  #
+  # Introduced by hive_metastore, tracked at https://github.com/cloudwego/pilota/issues/293
+  "RUSTSEC-2024-0436",
+  # `rustls-pemfile` is unmaintained
+  #
+  # Introduced by object_store, see https://github.com/apache/arrow-rs-object-store/issues/564
+  "RUSTSEC-2025-0134",
+
+  # Tracked here: https://github.com/paupino/rust-decimal/issues/766
+  "RUSTSEC-2026-0001",
 ]

.github/workflows/audit.yml

@@ -40,12 +40,11 @@ jobs:
   security_audit:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
         with:
           rust-version: stable
       - uses: rustsec/audit-check@v2.0.0
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          ignore: RUSTSEC-2024-0436

.github/workflows/bindings_python_ci.yml

@@ -41,10 +41,21 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  # check-rust:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v6
+  #     - name: Check format
+  #       working-directory: "bindings/python"
+  #       run: cargo fmt --all -- --check
+  #     - name: Check clippy
+  #       working-directory: "bindings/python"
+  #       run: cargo clippy --all-targets --all-features -- -D warnings
+
   check-python:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - uses: astral-sh/setup-uv@v7
         with:
           version: "0.9.3"
@@ -60,3 +71,41 @@ jobs:
         working-directory: "bindings/python"
         run: |
           uvx ruff check .
+
+  # test:
+  #   runs-on: ${{ matrix.os }}
+  #   strategy:
+  #     matrix:
+  #       os:
+  #         - ubuntu-latest
+  #         - macos-latest
+  #         - windows-latest
+  #   steps:
+  #     - uses: actions/checkout@v6
+  #     - uses: actions/setup-python@v6
+  #       with:
+  #         python-version: 3.12
+  #     - uses: PyO3/maturin-action@v1
+  #       with:
+  #         working-directory: "bindings/python"
+  #         command: build
+  #         args: --out dist --sdist
+  #     - uses: astral-sh/setup-uv@v7
+  #       with:
+  #         version: "0.9.3"
+  #         enable-cache: true
+  #     - name: Sync dependencies
+  #       working-directory: "bindings/python"
+  #       shell: bash
+  #       run: |
+  #         make install
+  #     - name: Install built wheel
+  #       working-directory: "bindings/python"
+  #       shell: bash
+  #       run: |
+  #         uv pip install --reinstall dist/pyiceberg_core-*.whl
+  #     - name: Run tests
+  #       working-directory: "bindings/python"
+  #       shell: bash
+  #       run: |
+  #         make test

.github/workflows/ci.yml

@@ -52,7 +52,7 @@ jobs:
           - ubuntu-latest
           - macos-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
@@ -101,7 +101,7 @@ jobs:
           root-reserve-mb: 10240
           temp-reserve-mb: 10240
 
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
@@ -126,7 +126,7 @@ jobs:
           - macos-latest
           - windows-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
@@ -143,7 +143,6 @@ jobs:
       matrix:
         test-suite:
           - { name: "default", args: "--all-targets --all-features --workspace" }
-          - { name: "smol", args: "--all-targets --no-default-features --features smol --features storage-all --workspace" }
           - { name: "doc", args: "--doc --all-features --workspace" }
     name: Unit Tests (${{ matrix.test-suite.name }})
     steps:
@@ -158,7 +157,7 @@ jobs:
           root-reserve-mb: 10240
           temp-reserve-mb: 10240
 
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
@@ -180,7 +179,7 @@ jobs:
     name: Verify MSRV
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: Install protoc
         uses: arduino/setup-protoc@v3
         with:

.github/workflows/ci_typos.yml

@@ -40,6 +40,6 @@ jobs:
     env:
       FORCE_COLOR: 1
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: Check typos
-        uses: crate-ci/typos@v1.39.2
+        uses: crate-ci/typos@v1.41.0

.github/workflows/publish.yml

@@ -42,10 +42,11 @@ jobs:
           - "crates/catalog/glue"
           - "crates/catalog/hms"
           - "crates/catalog/rest"
+          - "crates/catalog/s3tables"
           - "crates/catalog/sql"
           - "crates/integrations/datafusion"
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder

.github/workflows/release_python.yml

@@ -85,7 +85,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: [validate-release-tag]
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Install toml-cli
         if: ${{ needs.validate-release-tag.outputs.is-rc == 'true' }}
@@ -107,7 +107,7 @@ jobs:
           command: sdist
           args: -o dist
       - name: Upload sdist
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: wheels-sdist
           path: bindings/python/dist
@@ -128,7 +128,7 @@ jobs:
             }
           - { os: ubuntu-latest, target: "armv7l" }
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Install toml-cli
         if: ${{ needs.validate-release-tag.outputs.is-rc == 'true' }}
@@ -159,7 +159,7 @@ jobs:
           command: build
           args: --release -o dist
       - name: Upload wheels
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: wheels-${{ matrix.os }}-${{ matrix.target }}
           path: bindings/python/dist
@@ -178,7 +178,7 @@ jobs:
 
     steps:
       - name: Download all the dists
-        uses: actions/download-artifact@v6
+        uses: actions/download-artifact@v7
         with:
           pattern: wheels-*
           merge-multiple: true

.github/workflows/release_python_nightly.yml

@@ -43,7 +43,7 @@ jobs:
     if: github.repository == 'apache/iceberg-rust' # Only run for apache repo
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - uses: ./.github/actions/overwrite-package-version # Overwrite package version with timestamp
         with:
@@ -56,7 +56,7 @@ jobs:
           args: -o dist
 
       - name: Upload sdist
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: wheels-sdist
           path: bindings/python/dist
@@ -78,7 +78,7 @@ jobs:
             }
           - { os: ubuntu-latest, target: "armv7l" }
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - uses: ./.github/actions/overwrite-package-version # Overwrite package version with timestamp
         with:
@@ -102,7 +102,7 @@ jobs:
           args: --release -o dist
 
       - name: Upload wheels
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: wheels-${{ matrix.os }}-${{ matrix.target }}
           path: bindings/python/dist
@@ -120,17 +120,44 @@ jobs:
 
     steps:
       - name: Download all the dists
-        uses: actions/download-artifact@v6
+        uses: actions/download-artifact@v7
         with:
           pattern: wheels-*
           merge-multiple: true
           path: bindings/python/dist
       - name: List downloaded artifacts
         run: ls -R bindings/python/dist
       - name: Publish to TestPyPI
+        id: publish-testpypi
+        continue-on-error: true
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           repository-url: https://test.pypi.org/legacy/
           skip-existing: true
           packages-dir: bindings/python/dist
           verbose: true
+      - name: Display error message on publish failure
+        if: steps.publish-testpypi.outcome == 'failure'
+        run: |
+          echo "::error::Failed to publish to TestPyPI"
+          echo ""
+          echo "⚠️  TestPyPI Publish Failed"
+          echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+          echo ""
+          echo "This may be due to TestPyPI storage limits."
+          echo "See: https://docs.pypi.org/project-management/storage-limits"
+          echo ""
+          echo "To resolve this issue, use the pypi-cleanup utility to clean up old TestPyPI artifacts:"
+          echo "https://pypi.org/project/pypi-cleanup/"
+          echo ""
+          echo "  uvx pypi-cleanup --package pyiceberg-core --host https://test.pypi.org/ \\"
+          echo "    --verbose -d 10 --do-it --username <username>"
+          echo ""
+          echo "Requirements:"
+          echo "  • Must be a maintainer for pyiceberg-core on TestPyPI"
+          echo "    (https://test.pypi.org/project/pyiceberg-core)"
+          echo "  • Requires TestPyPI password and 2FA"
+          echo "  • ⚠️  ONLY do this for TestPyPI, NOT for production PyPI!"
+          echo ""
+          echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+          exit 1

.github/workflows/stale.yml

@@ -31,7 +31,7 @@ jobs:
     if: github.repository_owner == 'apache'
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/stale@v10.1.0
+      - uses: actions/stale@v10.1.1
         with:
           stale-issue-label: 'stale'
           exempt-issue-labels: 'not-stale'