Merge branch 'main' into feature/implement-project-node-for-insert-into-datafusion

Author: fvaleye

.github/ISSUE_TEMPLATE/iceberg_bug_report.yml

@@ -28,7 +28,8 @@ body:
       description: What Apache Iceberg Rust version are you using?
       multiple: false
       options:
-        - 0.5.1 (latest version)
+        - 0.6.0 (latest version)
+        - 0.5.1
         - 0.4.0
         - 0.3.0
         - 0.2.0

.github/actions/overwrite-package-version/action.yml

@@ -38,14 +38,24 @@ runs:
       env:
         TIMESTAMP: ${{ inputs.timestamp }}
       run: |
-        CURRENT_VERSION=$(python -c "import toml; print(toml.load('bindings/python/pyproject.toml')['project']['version'])")
-        NEW_VERSION="${CURRENT_VERSION}.dev${TIMESTAMP}"
-        NEW_VERSION=$NEW_VERSION python -c "
+        # Read the current version from the Rust crate's Cargo.toml
+        CURRENT_VERSION=$(python -c "import toml; print(toml.load('bindings/python/Cargo.toml')['package']['version'])")
+        export NEW_VERSION="${CURRENT_VERSION}.dev${TIMESTAMP}"
+        echo "Current version: ${CURRENT_VERSION}"
+        echo "New dev version: ${NEW_VERSION}"
+
+        # Update bindings/python/pyproject.toml:
+        # - Set project.version = NEW_VERSION
+        # - Ensure 'version' is not listed in project.dynamic
+        python -c "
         import toml
         import os
         config = toml.load('bindings/python/pyproject.toml')
-        config['project']['version'] = os.environ['NEW_VERSION']
+        new_version = os.environ['NEW_VERSION']
+        config['project']['version'] = new_version
+        if 'dynamic' in config['project']:
+          config['project']['dynamic'] = [v for v in config['project']['dynamic'] if v != 'version']
         with open('bindings/python/pyproject.toml', 'w') as f:
             toml.dump(config, f)
-        print(f'Updated version to: {config[\"project\"][\"version\"]}')
+        print(f'Updated version to: {new_version}')
         "

.github/workflows/bindings_python_ci.yml

@@ -22,8 +22,26 @@ on:
     branches:
       - main
   pull_request:
-    branches:
-      - main
+    paths:
+      - '**'                                       # Include all files and directories in the repository by default.
+      - '!.github/workflows/**'                    # Exclude all workflow files
+      - '.github/workflows/bindings_python_ci.yml' # except the current file.
+      - '!.github/actions/**'                      # Exclude custom actions
+      - 'bindings/python/**'                       # Include source codes and its dependencies
+      - '!crates/**'
+      - 'crates/iceberg/**'
+      - 'crates/integrations/datafusion/**'
+      - '!.github/ISSUE_TEMPLATE/**'               # Exclude files and directories that don't impact tests or code like templates, metadata, and documentation.
+      - '!scripts/**'
+      - '!website/**'
+      - '!.asf.yml'
+      - '!.gitattributes'
+      - '!.gitignore'
+      - '!CONTRIBUTING.md'
+      - '!CHANGELOG.md'
+      - '!LICENSE'
+      - '!NOTICE'
+      - '!README.md'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}

.github/workflows/ci.yml

@@ -22,8 +22,22 @@ on:
     branches:
       - main
   pull_request:
-    branches:
-      - main
+    paths:
+      - '**'                              # Include all files and directories in the repository by default.
+      - '!.github/workflows/**'           # Exclude all workflow files
+      - '.github/workflows/ci.yml'        # except the current file.
+      - '!.github/actions/**'             # Exclude custom actions
+      - '!.github/ISSUE_TEMPLATE/**'      # Exclude files and directories that don't impact tests or code like templates, metadata, and documentation.
+      - '!scripts/**'
+      - '!website/**'
+      - '!.asf.yml'
+      - '!.gitattributes'
+      - '!.gitignore'
+      - '!CONTRIBUTING.md'
+      - '!CHANGELOG.md'
+      - '!LICENSE'
+      - '!NOTICE'
+      - '!README.md'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
@@ -155,19 +169,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v5
-      - name: Setup Nightly Rust toolchain
-        uses: ./.github/actions/setup-builder
       - name: Install protoc
         uses: arduino/setup-protoc@v3
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Generate minimal versions lockfile
-        run: |
-          cargo generate-lockfile -Z direct-minimal-versions
       - name: Setup MSRV Rust toolchain
         uses: ./.github/actions/setup-builder
         with:
           rust-version: ${{ env.rust_msrv }}
+      - name: Setup Nightly Rust toolchain
+        uses: ./.github/actions/setup-builder
       - name: Check MSRV
-        run: |
-          cargo +${{ env.rust_msrv }} check --locked --workspace
+        run: make check-msrv

.github/workflows/website.yml

@@ -22,8 +22,9 @@ on:
     branches:
       - main
   pull_request:
-    branches:
-      - main
+    paths:
+      - ".github/workflows/website.yml"
+      - "website/**"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}

CHANGELOG.md

@@ -24,6 +24,108 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](https://semver.org/).
 
+## [v0.7.0] - 2025-09-17
+
+### Breaking Changes
+
+* **API Changes:**
+  * feat(writer): Make LocationGenerator partition-aware by @CTTY in https://github.com/apache/iceberg-rust/pull/1625
+  * feat(catalog): Implement catalog loader for in memory by @lliangyu-lin in https://github.com/apache/iceberg-rust/pull/1623
+
+### All Changes
+
+* chore: expose field id mapping from schema by @dentiny in https://github.com/apache/iceberg-rust/pull/1498
+* Read ManifestList V1 with V2 projection. by @rambleraptor in https://github.com/apache/iceberg-rust/pull/1482
+* refine: refine the interface of SnapshotProducer by @ZENOTME in https://github.com/apache/iceberg-rust/pull/1490
+* feat(transaction): Add retry logic to transaction by @CTTY in https://github.com/apache/iceberg-rust/pull/1484
+* fix: Fix mock dependency by @dentiny in https://github.com/apache/iceberg-rust/pull/1507
+* Bump version to 0.6.0 (Round 1) by @Xuanwo in https://github.com/apache/iceberg-rust/pull/1506
+* feat(catalog): Add register_table to Catalog trait by @CTTY in https://github.com/apache/iceberg-rust/pull/1509
+* chore: Bump python version for release by @Xuanwo in https://github.com/apache/iceberg-rust/pull/1522
+* chore: Bump opendal to 0.54 by @Xuanwo in https://github.com/apache/iceberg-rust/pull/1524
+* Preparing 0.6.0 release (update dep list) by @Xuanwo in https://github.com/apache/iceberg-rust/pull/1525
+* chore: expose default schema id by @dentiny in https://github.com/apache/iceberg-rust/pull/1526
+* Add support for custom credential loader for S3 FileIO by @phillipleblanc in https://github.com/apache/iceberg-rust/pull/1528
+* docs: fix release process docs by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/1538
+* fix: Use CatalogCommitConflicts for TableRequirement by @CTTY in https://github.com/apache/iceberg-rust/pull/1532
+* fix: retain metadata_location from StaticTable::into_table by @colinmarc in https://github.com/apache/iceberg-rust/pull/1536
+* infra: use `toml-cli` to manually set version in github action by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/1537
+* refactor: Add `read_from()` and `write_to()` to `TableMetadata` by @CTTY in https://github.com/apache/iceberg-rust/pull/1523
+* Add `uv` instructions to `bindings/python/README.md` by @Fokko in https://github.com/apache/iceberg-rust/pull/1514
+* feat: append action allows snapshot properties by @dentiny in https://github.com/apache/iceberg-rust/pull/1534
+* feat(catalog): Implement `update_table` for `MemoryCatalog` by @CTTY in https://github.com/apache/iceberg-rust/pull/1549
+* fix ci. by @liurenjie1024 in https://github.com/apache/iceberg-rust/pull/1557
+* fix(writer): Fix current_written_size for ParquetWriter by @CTTY in https://github.com/apache/iceberg-rust/pull/1556
+* chore(deps): Bump tokio from 1.46.1 to 1.47.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1552
+* fix(parquet_writer): correctly update upper bound by @Erigara in https://github.com/apache/iceberg-rust/pull/1520
+* feat(core): Implement `RollingFileWriter` to help split data into multiple files by @CTTY in https://github.com/apache/iceberg-rust/pull/1547
+* Refactor: use metadatalocaiton:new_with_table_location in catalog create_table by @yingjianwu98 in https://github.com/apache/iceberg-rust/pull/1558
+* Updates Podman Instructions, Docker-Compose, & Spark Script by @Kurtiscwright in https://github.com/apache/iceberg-rust/pull/1553
+* Rest: Implement register table by @gabeiglio in https://github.com/apache/iceberg-rust/pull/1521
+* feat(iceberg-datafusion): make IcebergTableScan public by @colinmarc in https://github.com/apache/iceberg-rust/pull/1570
+* chore: upgrade roaring to 0.11 by @dentiny in https://github.com/apache/iceberg-rust/pull/1577
+* refactor: remove TrackWriter by @mapleFU in https://github.com/apache/iceberg-rust/pull/1575
+* feat(spec): Expose DataFile serde functions by @CTTY in https://github.com/apache/iceberg-rust/pull/1563
+* Finish Adding Default Implementations for V1→V2 by @Kurtiscwright in https://github.com/apache/iceberg-rust/pull/1579
+* feat: add bulk insertion to deletion vector by @dentiny in https://github.com/apache/iceberg-rust/pull/1578
+* Try infer file io scheme from metadata location ahead of warehouse path by @daiyy in https://github.com/apache/iceberg-rust/pull/1590
+* feat(datafusion): Add IcebergCommitExec to commit the written data files by @CTTY in https://github.com/apache/iceberg-rust/pull/1588
+* chore(deps): Bump actions/download-artifact from 4 to 5 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1593
+* feat(catalog): Add catalog loader and builder implementation for rest catalog by @lliangyu-lin in https://github.com/apache/iceberg-rust/pull/1580
+* chore(deps): Update typo to 1.35.3 by @Marvel-Gu in https://github.com/apache/iceberg-rust/pull/1597
+* minor: Update cargo lock for rest catalog loader by @CTTY in https://github.com/apache/iceberg-rust/pull/1596
+* feat(datafusion): Implement IcebergWriteExec for DataFusion write support by @CTTY in https://github.com/apache/iceberg-rust/pull/1585
+* feat(arrow): Allow ArrowArrayAccessor to use field.name to match fields by @CTTY in https://github.com/apache/iceberg-rust/pull/1561
+* fix(tests): resolve flaky catalog tests by waiting for S3 bucket creation by @fvaleye in https://github.com/apache/iceberg-rust/pull/1599
+* chore(deps): Bump actions/checkout from 4 to 5 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1613
+* chore(deps): Bump crate-ci/typos from 1.35.3 to 1.35.4 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1614
+* chore(deps): Bump uuid from 1.17.0 to 1.18.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1615
+* Refactor: remove unnecessary async for ArrowReader::read by @ZENOTME in https://github.com/apache/iceberg-rust/pull/1608
+* feat(catalog): implement catalog loader for glue by @lliangyu-lin in https://github.com/apache/iceberg-rust/pull/1603
+* Include license and notice files in published crates by @ankane in https://github.com/apache/iceberg-rust/pull/1601
+* docs: tidying; update version refs; describe various catalogues by @atcol in https://github.com/apache/iceberg-rust/pull/1616
+* feat(catalog): implement catalog loader for s3tables by @fvaleye in https://github.com/apache/iceberg-rust/pull/1598
+* feat(catalog): Implement catalog loader for hms by @lliangyu-lin in https://github.com/apache/iceberg-rust/pull/1612
+* feat(catalog): Implement update_table for GlueCatalog by @CTTY in https://github.com/apache/iceberg-rust/pull/1584
+* fix(schema): check if new schema field names conflict with existing partition field names by @fvaleye in https://github.com/apache/iceberg-rust/pull/1619
+* chore(deps): Bump crate-ci/typos from 1.35.4 to 1.35.5 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1629
+* chore(deps): Bump tempfile from 3.20.0 to 3.21.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1628
+* chore(deps): Bump aws-sdk-glue from 1.116.0 to 1.117.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1627
+* feat(catalog): Implement register_table for glue catalog by @Yiyang-C in https://github.com/apache/iceberg-rust/pull/1568
+* feat(catalog): Implement catalog loader for in memory by @lliangyu-lin in https://github.com/apache/iceberg-rust/pull/1623
+* feat(datafusion): Implement `insert_into` for `IcebergTableProvider` by @CTTY in https://github.com/apache/iceberg-rust/pull/1600
+* fix: await get catalog in datafusion integration test and update version in python binding by @lliangyu-lin in https://github.com/apache/iceberg-rust/pull/1635
+* chore: Replace async-std with smol by @manuzhang in https://github.com/apache/iceberg-rust/pull/1640
+* chore(deps): Bump tracing-subscriber from 0.3.19 to 0.3.20 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1637
+* chore(deps): Bump aws-sdk-s3tables from 1.35.0 to 1.36.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1642
+* chore(deps): Bump crate-ci/typos from 1.35.5 to 1.35.7 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1641
+* chore(deps): Bump aws-sdk-glue from 1.117.0 to 1.118.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1643
+* Allow V2 reader to read v1 manifests by @emkornfield in https://github.com/apache/iceberg-rust/pull/1634
+* feat(writer): Make LocationGenerator partition-aware by @CTTY in https://github.com/apache/iceberg-rust/pull/1625
+* feat(sqllogictest): Add sqllogictest schedule definition and parsing by @lliangyu-lin in https://github.com/apache/iceberg-rust/pull/1630
+* fix: Add support for unsigned Arrow datatypes in schema conversion by @gkpanda4 in https://github.com/apache/iceberg-rust/pull/1617
+* Partition Writer Support Part 1: add partition splitter by @ZENOTME in https://github.com/apache/iceberg-rust/pull/1040
+* test: pyiceberg-core, pin datafusion and unpin pyarrow by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/1648
+* infra: no need to manually install crates in CI by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/1654
+* Add LICENSE and NOTICE files for pyiceberg-core source distribution and wheel by @sumanth-manchala in https://github.com/apache/iceberg-rust/pull/1653
+* docs: add instructions to check license header in "How to verify a release" by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/1662
+* chore(deps): Bump actions/setup-python from 5 to 6 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1658
+* upgrade MSRV from `1.85` to `1.87` by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/1655
+* chore(deps): Bump crate-ci/typos from 1.35.7 to 1.36.2 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1659
+* ci: only use `-Z direct-minimal-versions` for MSRV check by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/1656
+* Bump Avro to 0.20 by @Fokko in https://github.com/apache/iceberg-rust/pull/1644
+* Scan Delete Support Part 6: Equality Delete Parsing by @sdd in https://github.com/apache/iceberg-rust/pull/1017
+* Remove rustversion pin by @Fokko in https://github.com/apache/iceberg-rust/pull/1664
+* refactor(test): remove MockLocationGenerator and its usages (#1645) by @pushcoder in https://github.com/apache/iceberg-rust/pull/1663
+* feat: bump datafusion to 48 by @colinmarc in https://github.com/apache/iceberg-rust/pull/1501
+* chore: Add staleness bot for issues by @emkornfield in https://github.com/apache/iceberg-rust/pull/1665
+* chore(deps): Bump actions/stale from 9.1.0 to 10.0.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1671
+* chore(deps): Bump tempfile from 3.21.0 to 3.22.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1672
+* docs: Update download links to 0.6.0 by @manuzhang in https://github.com/apache/iceberg-rust/pull/1677
+* docs: Fix Podman and Orbstack pages not found by @manuzhang in https://github.com/apache/iceberg-rust/pull/1678
+* Update bug report template with the latest version, 0.6.0 by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/1679
+* Expose Avro reader to PyIceberg by @Fokko in https://github.com/apache/iceberg-rust/pull/1328
+
 ## [v0.6.0] - 2025-07-14
 
 ### Breaking Changes

CONTRIBUTING.md

@@ -103,8 +103,8 @@ cargo 1.69.0 (6e9a83356 2023-04-12)
 
 Currently, iceberg-rust uses Docker to set up environment for integration tests. Native Docker has some limitations, please check (https://github.com/apache/iceberg-rust/pull/748). Please use Orbstack or Podman.
 
-For MacOS users, you can install [OrbStack as a docker alternative](docs/contributing/orbstack.md).
-For Podman users, refer to [Using Podman instead of Docker](docs/contributing/podman.md)
+For MacOS users, you can install [OrbStack as a docker alternative](reference/orbstack.md).
+For Podman users, refer to [Using Podman instead of Docker](reference/podman.md)
 
 ## Build