Merge branch 'main' into fix/json-serializable-and-expression

Author: Aniketsy

.github/workflows/pypi-build-artifacts.yml

@@ -42,7 +42,6 @@ jobs:
       - uses: actions/setup-python@v6
         with:
           python-version: |
-            3.9
             3.10
             3.11
             3.12
@@ -62,14 +61,14 @@ jobs:
         if: startsWith(matrix.os, 'ubuntu')
 
       - name: Build wheels
-        uses: pypa/[email protected].0
+        uses: pypa/[email protected].1
         with:
           output-dir: wheelhouse
           config-file: "pyproject.toml"
         env:
           # Ignore 32 bit architectures
           CIBW_ARCHS: "auto64"
-          CIBW_PROJECT_REQUIRES_PYTHON: ">=3.9,<3.13"
+          CIBW_PROJECT_REQUIRES_PYTHON: ">=3.10"
           CIBW_TEST_REQUIRES: "pytest==7.4.2 moto==5.0.1"
           CIBW_TEST_COMMAND: "pytest {project}/tests/avro/test_decoder.py"
           # Ignore tests for pypy since not all dependencies are compiled for it

.github/workflows/python-ci-docs.yml

@@ -45,3 +45,5 @@ jobs:
         run: make docs-install
       - name: Build docs
         run: make docs-build
+      - name: Run linters
+        run: make lint

.github/workflows/python-ci.yml

@@ -47,7 +47,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python: ['3.9', '3.10', '3.11', '3.12']
+        python: ['3.10', '3.11', '3.12']
 
     steps:
     - uses: actions/checkout@v5
@@ -71,10 +71,13 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python: ['3.9', '3.10', '3.11', '3.12']
+        python: ['3.10', '3.11', '3.12']
 
     steps:
     - uses: actions/checkout@v5
+    - uses: actions/setup-python@v6
+      with:
+        python-version: ${{ matrix.python }}
     - name: Install system dependencies
       run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos
     - name: Install

.github/workflows/stale.yml

@@ -31,7 +31,7 @@ jobs:
     if: github.repository_owner == 'apache'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v10.0.0
+      - uses: actions/stale@v10.1.0
         with:
           stale-issue-label: 'stale'
           exempt-issue-labels: 'not-stale'

.github/workflows/svn-build-artifacts.yml

@@ -42,7 +42,6 @@ jobs:
       - uses: actions/setup-python@v6
         with:
           python-version: |
-            3.9
             3.10
             3.11
             3.12
@@ -57,14 +56,14 @@ jobs:
         if: startsWith(matrix.os, 'ubuntu')
 
       - name: Build wheels
-        uses: pypa/[email protected].0
+        uses: pypa/[email protected].1
         with:
           output-dir: wheelhouse
           config-file: "pyproject.toml"
         env:
           # Ignore 32 bit architectures
           CIBW_ARCHS: "auto64"
-          CIBW_PROJECT_REQUIRES_PYTHON: ">=3.9,<3.13"
+          CIBW_PROJECT_REQUIRES_PYTHON: ">=3.10"
           CIBW_TEST_REQUIRES: "pytest==7.4.2 moto==5.0.1"
           CIBW_TEST_COMMAND: "pytest {project}/tests/avro/test_decoder.py"
           # Ignore tests for pypy since not all dependencies are compiled for it

.pre-commit-config.yaml

@@ -37,7 +37,12 @@ repos:
     hooks:
       - id: mypy
         args:
-          [--install-types, --non-interactive, --config=pyproject.toml]
+          [--config=pyproject.toml]
+        additional_dependencies:
+          - types-cachetools
+          - types-pytest-lazy-fixture
+          - types-pytz
+          - types-requests
   - repo: https://github.com/igorshubovych/markdownlint-cli
     rev: v0.45.0
     hooks:

Makefile

@@ -83,8 +83,8 @@ install: install-poetry install-dependencies ## Install Poetry and dependencies
 check-license: ## Check license headers
 	./dev/check-license
 
-lint: ## Run code linters via pre-commit
-	$(POETRY) run pre-commit run --all-files
+lint: ## Run code linters via prek (pre-commit hooks)
+	$(POETRY) run prek run -a
 
 # ===============
 # Testing Section
@@ -100,10 +100,8 @@ test-integration: test-integration-setup test-integration-exec test-integration-
 test-integration-setup: ## Start Docker services for integration tests
 	docker compose -f dev/docker-compose-integration.yml kill
 	docker compose -f dev/docker-compose-integration.yml rm -f
-	docker compose -f dev/docker-compose-integration.yml up -d
-	sleep 10
-	docker compose -f dev/docker-compose-integration.yml cp ./dev/provision.py spark-iceberg:/opt/spark/provision.py
-	docker compose -f dev/docker-compose-integration.yml exec -T spark-iceberg ipython ./provision.py
+	docker compose -f dev/docker-compose-integration.yml up -d --wait
+	$(POETRY) run python dev/provision.py
 
 test-integration-exec: ## Run integration tests (excluding provision)
 	$(TEST_RUNNER) pytest tests/ -m integration $(PYTEST_ARGS)

dev/.rat-excludes

@@ -4,3 +4,4 @@ build
 .git
 .gitignore
 poetry.lock
+mkdocs/*

dev/Dockerfile

@@ -17,9 +17,8 @@
 
 services:
   spark-iceberg:
-    image: python-integration
     container_name: pyiceberg-spark
-    build: .
+    build: spark/
     networks:
       iceberg_net:
     depends_on:
@@ -37,6 +36,12 @@ services:
       - rest:rest
       - hive:hive
       - minio:minio
+    healthcheck:
+      test: ["CMD", "sh", "-c", "netstat -an | grep 15002 | grep LISTEN"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 90s
   rest:
     image: apache/iceberg-rest-fixture
     container_name: pyiceberg-rest
@@ -87,7 +92,7 @@ services:
       "
   hive:
     build: hive/
-    container_name: hive
+    container_name: pyiceberg-hive
     hostname: hive
     networks:
       iceberg_net: