Merge branch 'main' into main

Author: Fokko

.asf.yaml

@@ -21,7 +21,7 @@
 # https://cwiki.apache.org/confluence/display/INFRA/Git+-+.asf.yaml+features
 
 github:
-  description: "Apache PyIceberg"
+  description: "PyIceberg"
   homepage: https://py.iceberg.apache.org/
   labels:
     - iceberg

.github/ISSUE_TEMPLATE/iceberg_bug_report.yml

@@ -28,7 +28,9 @@ body:
       description: What Apache Iceberg version are you using?
       multiple: false
       options:
-        - "0.8.1 (latest release)"
+        - "0.9.1 (latest release)"
+        - "0.9.0"
+        - "0.8.1"
         - "0.8.0"
         - "0.7.1"
         - "0.7.0"

.github/dependabot.yml

@@ -22,9 +22,9 @@ updates:
   - package-ecosystem: "pip"
     directory: "/"
     schedule:
-      interval: "daily"
+      interval: "weekly"
     open-pull-requests-limit: 50
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
-      interval: "daily"
+      interval: "weekly"

.github/pull_request_template.md

@@ -0,0 +1,14 @@
+<!--
+Thanks for opening a pull request!
+-->
+
+<!-- In the case this PR will resolve an issue, please replace ${GITHUB_ISSUE_ID} below with the actual Github issue id. -->
+<!-- Closes #${GITHUB_ISSUE_ID} -->
+
+# Rationale for this change
+
+## Are these changes tested?
+
+## Are there any user-facing changes?
+
+<!-- In the case of user-facing changes, please add the changelog label. -->

.github/workflows/check-md-link.yml

@@ -36,4 +36,4 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@master
-    - uses: gaurav-nelson/github-action-markdown-link-check@v1
+    - uses: tcort/github-action-markdown-link-check@v1

.github/workflows/license_check.yml

@@ -24,5 +24,5 @@ jobs:
   rat:
     runs-on: ubuntu-22.04
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
     - run: dev/check-license

.github/workflows/nightly-pypi-build.yml

@@ -31,7 +31,7 @@ jobs:
     outputs:
       VERSION: ${{ steps.set-version.outputs.VERSION }}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           fetch-depth: 1
 
@@ -71,7 +71,7 @@ jobs:
 
     steps:
     - name: Download all the artifacts
-      uses: actions/download-artifact@v4
+      uses: actions/download-artifact@v5
       with:
         merge-multiple: true
         path: dist/

.github/workflows/pypi-build-artifacts.yml

@@ -35,7 +35,7 @@ jobs:
         os: [ ubuntu-22.04, windows-2022, macos-13, macos-14 ]
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           fetch-depth: 1
 
@@ -62,7 +62,7 @@ jobs:
         if: startsWith(matrix.os, 'ubuntu')
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.22.0
+        uses: pypa/cibuildwheel@v3.1.4
         with:
           output-dir: wheelhouse
           config-file: "pyproject.toml"

.github/workflows/python-ci-docs.yml

@@ -35,7 +35,7 @@ jobs:
     runs-on: ubuntu-22.04
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: Install poetry
         run: make install-poetry
       - uses: actions/setup-python@v5

.github/workflows/python-ci.yml

@@ -50,17 +50,59 @@ jobs:
         python: ['3.9', '3.10', '3.11', '3.12']
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
     - name: Install poetry
       run: make install-poetry
     - uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python }}
-        cache: poetry
-        cache-dependency-path: ./poetry.lock
+    - name: Install system dependencies
+      run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos
     - name: Install
       run: make install-dependencies
-    - name: Linters
+    - name: Run linters
       run: make lint
-    - name: Tests
-      run: make test-coverage
+    - name: Run unit tests with coverage
+      run: COVERAGE=1 make test
+    - name: Generate coverage report (85%) # Coverage threshold should only increase over time — never decrease it!
+      run: COVERAGE_FAIL_UNDER=85 make coverage-report
+
+  integration-test:
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        python: ['3.9', '3.10', '3.11', '3.12']
+
+    steps:
+    - uses: actions/checkout@v5
+    - name: Install system dependencies
+      run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos
+    - name: Install
+      run: make install
+
+    - name: Run integration tests with coverage
+      run: COVERAGE=1 make test-integration
+    - name: Show debug logs
+      if: ${{ failure() }}
+      run: docker compose -f dev/docker-compose.yml logs
+
+    - name: Run s3 integration tests with coverage
+      run: COVERAGE=1 make test-s3
+    - name: Show debug logs
+      if: ${{ failure() }}
+      run: docker compose -f dev/docker-compose.yml logs
+
+    - name: Run adls integration tests with coverage
+      run: COVERAGE=1 make test-adls
+    - name: Show debug logs
+      if: ${{ failure() }}
+      run: docker compose -f dev/docker-compose-azurite.yml logs
+
+    - name: Run gcs integration tests with coverage
+      run: COVERAGE=1 make test-gcs
+    - name: Show debug logs
+      if: ${{ failure() }}
+      run: docker compose -f dev/docker-compose-gcs-server.yml logs
+
+    - name: Generate coverage report (75%) # Coverage threshold should only increase over time — never decrease it!
+      run: COVERAGE_FAIL_UNDER=75 make coverage-report