Kevinjqliu/refactor mkdocs

.asf.yaml

@@ -38,6 +38,7 @@ github:
         required_approving_review_count: 1
 
       required_linear_history: true
+  del_branch_on_merge: true
   features:
     wiki: true
     issues: true

mkdocs/requirements.txt → .codespellrc

@@ -14,15 +14,5 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
-mkdocs==1.6.1
-griffe==1.5.1
-jinja2==3.1.4
-mkdocstrings==0.27.0
-mkdocstrings-python==1.12.2
-mkdocs-literate-nav==0.6.1
-mkdocs-autorefs==1.2.0
-mkdocs-gen-files==0.5.0
-mkdocs-material==9.5.46
-mkdocs-material-extensions==1.3.1
-mkdocs-section-index==0.3.9
+[codespell]
+ignore-words-list = BoundIn,fo,MoR,NotIn,notIn,oT

.github/ISSUE_TEMPLATE/iceberg_bug_report.yml

@@ -28,7 +28,8 @@ body:
       description: What Apache Iceberg version are you using?
       multiple: false
       options:
-        - "0.8.0 (latest release)"
+        - "0.8.1 (latest release)"
+        - "0.8.0"
         - "0.7.1"
         - "0.7.0"
         - "0.6.1"

.github/workflows/check-md-link.yml

@@ -22,10 +22,14 @@ name: Check Markdown links
 on:
   push:
     paths:
-      - mkdocs/**
+      - '.github/workflows/check-md-link.yml'
+      - 'mkdocs/**'
     branches:
     - 'main'
   pull_request:
+    paths:
+      - '.github/workflows/check-md-link.yml'
+      - 'mkdocs/**'
 
 jobs:
   markdown-link-check:

.github/workflows/python-ci-docs.yml

@@ -36,12 +36,12 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+      - name: Install poetry
+        run: make install-poetry
       - uses: actions/setup-python@v5
         with:
           python-version: 3.12
       - name: Install
-        working-directory: ./mkdocs
-        run: pip install -r requirements.txt
-      - name: Build
-        working-directory: ./mkdocs
-        run: mkdocs build --strict
+        run: make docs-install
+      - name: Build docs
+        run: make docs-build

.github/workflows/python-ci.yml

@@ -24,6 +24,19 @@ on:
     branches:
     - 'main'
   pull_request:
+    paths:
+    - '**'                              # Include all files and directories in the repository by default.
+    - '!.github/workflows/**'           # Exclude all workflow files
+    - '.github/workflows/python-ci.yml' # except the current file.
+    - '!.github/ISSUE_TEMPLATE/**'      # Exclude files and directories that don't impact tests or code like templates, metadata, and documentation.
+    - '!.gitignore'
+    - '!.asf.yml'
+    - '!mkdocs/**'
+    - '!.gitattributes'
+    - '!README.md'
+    - '!CONTRIBUTING.md'
+    - '!LICENSE'
+    - '!NOTICE'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

.github/workflows/python-integration.yml

@@ -24,6 +24,19 @@ on:
     branches:
     - 'main'
   pull_request:
+    paths:
+    - '**'                                       # Include all files and directories in the repository by default.
+    - '!.github/workflows/**'                    # Exclude all workflow files
+    - '.github/workflows/python-integration.yml' # except the current file.
+    - '!.github/ISSUE_TEMPLATE/**'               # Exclude files and directories that don't impact tests or code like templates, metadata, and documentation.
+    - '!.gitignore'
+    - '!.asf.yml'
+    - '!mkdocs/**'
+    - '!.gitattributes'
+    - '!README.md'
+    - '!CONTRIBUTING.md'
+    - '!LICENSE'
+    - '!NOTICE'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

.github/workflows/python-release-docs.yml

@@ -31,15 +31,15 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+      - name: Install poetry
+        run: make install-poetry
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
-      - name: Install
-        working-directory: ./mkdocs
-        run: pip install -r requirements.txt
-      - name: Build
-        working-directory: ./mkdocs
-        run: mkdocs build --strict
+      - name: Install docs
+        run: make docs-install
+      - name: Build docs
+        run: make docs-build
       - name: Copy
         working-directory: ./mkdocs
         run: mv ./site /tmp/site

.pre-commit-config.yaml

@@ -69,6 +69,10 @@ repos:
           # --line-length is set to a high value to deal with very long lines
           - --line-length
           - '99999'
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.3.0
+    hooks:
+      - id: codespell
 ci:
     autofix_commit_msg: |
         [pre-commit.ci] auto fixes from pre-commit.com hooks

Makefile

@@ -22,12 +22,12 @@ help:  ## Display this help
 install-poetry:  ## Install poetry if the user has not done that yet.
 	 @if ! command -v poetry &> /dev/null; then \
          echo "Poetry could not be found. Installing..."; \
-         pip install --user poetry==1.8.4; \
+         pip install --user poetry==1.8.5; \
      else \
          echo "Poetry is already installed."; \
      fi
 
-install-dependencies: ## Install dependencies including dev and all extras
+install-dependencies: ## Install dependencies including dev, docs, and all extras
 	poetry install --all-extras
 
 install: | install-poetry install-dependencies
@@ -97,3 +97,12 @@ clean: ## Clean up the project Python working environment
 	@find . -name "*.pyd" -exec echo Deleting {} \; -delete
 	@find . -name "*.pyo" -exec echo Deleting {} \; -delete
 	@echo "Cleanup complete"
+
+docs-install:
+	poetry install --with docs
+
+docs-serve:
+	poetry run mkdocs serve -f mkdocs/mkdocs.yml
+
+docs-build:
+	poetry run mkdocs build -f mkdocs/mkdocs.yml --strict

dev/Dockerfile

@@ -36,21 +36,22 @@ ENV PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9.7-src.zip:$
 RUN mkdir -p ${HADOOP_HOME} && mkdir -p ${SPARK_HOME} && mkdir -p /home/iceberg/spark-events
 WORKDIR ${SPARK_HOME}
 
+# Remember to also update `tests/conftest`'s spark setting
 ENV SPARK_VERSION=3.5.3
 ENV ICEBERG_SPARK_RUNTIME_VERSION=3.5_2.12
 ENV ICEBERG_VERSION=1.6.0
-ENV PYICEBERG_VERSION=0.8.0
+ENV PYICEBERG_VERSION=0.8.1
 
-RUN curl --retry 3 -s -C - https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz -o spark-${SPARK_VERSION}-bin-hadoop3.tgz \
+RUN curl --retry 5 -s -C - https://dlcdn.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz -o spark-${SPARK_VERSION}-bin-hadoop3.tgz \
  && tar xzf spark-${SPARK_VERSION}-bin-hadoop3.tgz --directory /opt/spark --strip-components 1 \
  && rm -rf spark-${SPARK_VERSION}-bin-hadoop3.tgz
 
 # Download iceberg spark runtime
-RUN curl -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar -Lo iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar \
+RUN curl --retry 5 -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar -Lo iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar \
  && mv iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar /opt/spark/jars
 
 # Download AWS bundle
-RUN curl -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar -Lo /opt/spark/jars/iceberg-aws-bundle-${ICEBERG_VERSION}.jar
+RUN curl --retry 5 -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar -Lo /opt/spark/jars/iceberg-aws-bundle-${ICEBERG_VERSION}.jar
 
 COPY spark-defaults.conf /opt/spark/conf
 ENV PATH="/opt/spark/sbin:/opt/spark/bin:${PATH}"

dev/docker-compose-azurite.yml

@@ -14,7 +14,6 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-version: "3"
 
 services:
   azurite:

dev/docker-compose-gcs-server.yml

@@ -14,7 +14,6 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-version: "3"
 
 services:
   gcs-server:

dev/docker-compose-integration.yml

@@ -14,7 +14,6 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-version: "3"
 
 services:
   spark-iceberg:
@@ -41,7 +40,7 @@ services:
       - hive:hive
       - minio:minio
   rest:
-    image: tabulario/iceberg-rest
+    image: apache/iceberg-rest-fixture
     container_name: pyiceberg-rest
     networks:
       iceberg_net:

dev/docker-compose.yml

@@ -14,7 +14,6 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-version: "3"
 
 services:
   minio:

dev/provision.py

@@ -22,7 +22,17 @@
 from pyiceberg.schema import Schema
 from pyiceberg.types import FixedType, NestedField, UUIDType
 
-spark = SparkSession.builder.getOrCreate()
+# The configuration is important, otherwise we get many small
+# parquet files with a single row. When a positional delete
+# hits the Parquet file with one row, the parquet file gets
+# dropped instead of having a merge-on-read delete file.
+spark = (
+    SparkSession
+        .builder
+        .config("spark.sql.shuffle.partitions", "1")
+        .config("spark.default.parallelism", "1")
+        .getOrCreate()
+)
 
 catalogs = {
     'rest': load_catalog(
@@ -120,10 +130,6 @@
     """
     )
 
-    # Partitioning is not really needed, but there is a bug:
-    # https://github.com/apache/iceberg/pull/7685
-    spark.sql(f"ALTER TABLE {catalog_name}.default.test_positional_mor_deletes ADD PARTITION FIELD years(dt) AS dt_years")
-
     spark.sql(
         f"""
     INSERT INTO {catalog_name}.default.test_positional_mor_deletes
@@ -168,10 +174,6 @@
     """
     )
 
-    # Partitioning is not really needed, but there is a bug:
-    # https://github.com/apache/iceberg/pull/7685
-    spark.sql(f"ALTER TABLE {catalog_name}.default.test_positional_mor_double_deletes ADD PARTITION FIELD years(dt) AS dt_years")
-
     spark.sql(
         f"""
     INSERT INTO {catalog_name}.default.test_positional_mor_double_deletes

mkdocs/README.md

@@ -22,7 +22,6 @@ The pyiceberg docs are stored in `docs/`.
 ## Running docs locally
 
 ```sh
-pip3 install -r requirements.txt
-mkdocs serve
-open http://localhost:8000/
+make docs-install
+make docs-serve
 ```

mkdocs/docs/SUMMARY.md

@@ -29,6 +29,7 @@
 - Releases
     - [Verify a release](verify-release.md)
     - [How to release](how-to-release.md)
+    - [Release Notes](https://github.com/apache/iceberg-python/releases)
 - [Code Reference](reference/)
 
 <!-- markdown-link-check-enable-->