Merge branch 'main' of github.com:apache/iceberg-python into fd-add-ability-to-delete-full-data-files

Author: Fokko

Makefile

@@ -41,6 +41,7 @@ test-integration:
 	docker compose -f dev/docker-compose-integration.yml rm -f
 	docker compose -f dev/docker-compose-integration.yml up -d
 	sleep 10
+	docker compose -f dev/docker-compose-integration.yml cp ./dev/provision.py spark-iceberg:/opt/spark/provision.py
 	docker compose -f dev/docker-compose-integration.yml exec -T spark-iceberg ipython ./provision.py
 	poetry run pytest tests/ -v -m integration ${PYTEST_ARGS}
 
@@ -63,6 +64,8 @@ test-coverage:
 	docker compose -f dev/docker-compose-integration.yml up -d
 	sh ./dev/run-azurite.sh
 	sh ./dev/run-gcs-server.sh
+	sleep 10
+	docker compose -f dev/docker-compose-integration.yml cp ./dev/provision.py spark-iceberg:/opt/spark/provision.py
 	docker compose -f dev/docker-compose-integration.yml exec -T spark-iceberg ipython ./provision.py
 	poetry run coverage run --source=pyiceberg/ -m pytest tests/ ${PYTEST_ARGS}
 	poetry run coverage report -m --fail-under=90

dev/Dockerfile

@@ -36,10 +36,10 @@ ENV PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9.7-src.zip:$
 RUN mkdir -p ${HADOOP_HOME} && mkdir -p ${SPARK_HOME} && mkdir -p /home/iceberg/spark-events
 WORKDIR ${SPARK_HOME}
 
-ENV SPARK_VERSION=3.4.3
-ENV ICEBERG_SPARK_RUNTIME_VERSION=3.4_2.12
-ENV ICEBERG_VERSION=1.4.2
-ENV PYICEBERG_VERSION=0.5.1
+ENV SPARK_VERSION=3.5.0
+ENV ICEBERG_SPARK_RUNTIME_VERSION=3.5_2.12
+ENV ICEBERG_VERSION=1.5.0
+ENV PYICEBERG_VERSION=0.6.0
 
 RUN curl --retry 3 -s -C - https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz -o spark-${SPARK_VERSION}-bin-hadoop3.tgz \
  && tar xzf spark-${SPARK_VERSION}-bin-hadoop3.tgz --directory /opt/spark --strip-components 1 \

mkdocs/docs/contributing.md

@@ -71,7 +71,7 @@ pip3 install -e ".[s3fs,hive]"
 Install it directly for GitHub (not recommended), but sometimes handy:
 
 ```
-pip install "git+https://github.com/apache/iceberg-python.git#egg=pyiceberg[s3fs]"
+pip install "git+https://github.com/apache/iceberg-python.git#egg=pyiceberg[pyarrow]"
 ```
 
 ## Linting

mkdocs/docs/how-to-release.md

@@ -143,6 +143,10 @@ https://pypi.org/project/pyiceberg/$VERSION/
 
 And can be installed using: pip3 install pyiceberg==$VERSION
 
+Instructions for verifying a release can be found here:
+
+* https://py.iceberg.apache.org/verify-release/
+
 Please download, verify, and test.
 
 Please vote in the next 72 hours.

mkdocs/docs/verify-release.md

@@ -44,12 +44,21 @@ curl https://dist.apache.org/repos/dist/dev/iceberg/KEYS -o KEYS
 gpg --import KEYS
 ```
 
+Set an environment variable to the version to verify and path to use
+
+```sh
+export PYICEBERG_VERSION=<version> # e.g. 0.6.1rc3
+export PYICEBERG_VERIFICATION_DIR=/tmp/pyiceberg/${PYICEBERG_VERSION}
+```
+
 Next, verify the `.asc` file.
 
 ```sh
-svn checkout https://dist.apache.org/repos/dist/dev/iceberg/pyiceberg-0.5.0rc1/ /tmp/pyiceberg/
+svn checkout https://dist.apache.org/repos/dist/dev/iceberg/pyiceberg-${PYICEBERG_VERSION}/ ${PYICEBERG_VERIFICATION_DIR}
+
+cd ${PYICEBERG_VERIFICATION_DIR}
 
-for name in $(ls /tmp/pyiceberg/pyiceberg-*.whl /tmp/pyiceberg/pyiceberg-*.tar.gz)
+for name in $(ls pyiceberg-*.whl pyiceberg-*.tar.gz)
 do
     gpg --verify ${name}.asc ${name}
 done
@@ -58,8 +67,8 @@ done
 ## Verifying checksums
 
 ```sh
-cd  /tmp/pyiceberg/
-for name in $(ls /tmp/pyiceberg/pyiceberg-*.whl.sha512 /tmp/pyiceberg/pyiceberg-*.tar.gz.sha512)
+cd ${PYICEBERG_VERIFICATION_DIR}
+for name in $(ls pyiceberg-*.whl.sha512 pyiceberg-*.tar.gz.sha512)
 do
     shasum -a 512 --check ${name}
 done
@@ -68,8 +77,9 @@ done
 ## Verifying License Documentation
 
 ```sh
-tar xzf pyiceberg-0.5.0.tar.gz
-cd pyiceberg-0.5.0
+export PYICEBERG_RELEASE_VERSION=${PYICEBERG_VERSION/rc?/}  # remove rcX qualifier
+tar xzf pyiceberg-${PYICEBERG_RELEASE_VERSION}.tar.gz
+cd pyiceberg-${PYICEBERG_RELEASE_VERSION}
 ```
 
 Run RAT checks to validate license header:
@@ -102,18 +112,20 @@ And then run the tests:
 make test
 ```
 
-To run the full integration tests:
+To run the full test coverage:
 
 ```sh
-make test-s3
+make test-coverage
 ```
 
-This will include a Minio S3 container being spun up.
+This will spin up Docker containers to faciliate running test coverage.
 
 # Cast the vote
 
 Votes are cast by replying to the release candidate announcement email on the dev mailing list with either `+1`, `0`, or `-1`. For example :
 
-> \[ \] +1 Release this as PyIceberg 0.3.0 \[ \] +0 \[ \] -1 Do not release this because…
+> \[ \] +1 Release this as PyIceberg 0.3.0 <br>
+> \[ \] +0 <br>
+> \[ \] -1 Do not release this because… <br>
 
 In addition to your vote, it’s customary to specify if your vote is binding or non-binding. Only members of the Project Management Committee have formally binding votes. If you’re unsure, you can specify that your vote is non-binding. To read more about voting in the Apache framework, checkout the [Voting](https://www.apache.org/foundation/voting.html) information page on the Apache foundation’s website.

mkdocs/requirements.txt

@@ -23,6 +23,6 @@ mkdocstrings-python==1.8.0
 mkdocs-literate-nav==0.6.1
 mkdocs-autorefs==0.5.0
 mkdocs-gen-files==0.5.0
-mkdocs-material==9.5.10
+mkdocs-material==9.5.18
 mkdocs-material-extensions==1.3.1
 mkdocs-section-index==0.3.8