Merge branch 'main' of github.com:apache/iceberg-python into fd-add-ability-to-delete-full-data-files

Author: Fokko

.pre-commit-config.yaml

@@ -75,3 +75,14 @@ repos:
           # --line-length is set to a high value to deal with very long lines
           - --line-length
           - '99999'
+ci:
+    autofix_commit_msg: |
+        [pre-commit.ci] auto fixes from pre-commit.com hooks
+
+        for more information, see https://pre-commit.ci
+    autofix_prs: true
+    autoupdate_branch: ''
+    autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate'
+    autoupdate_schedule: weekly
+    skip: []
+    submodules: false

Makefile

@@ -16,7 +16,7 @@
 # under the License.
 
 install-poetry:
-	pip install poetry==1.7.1
+	pip install poetry==1.8.2
 
 install-dependencies:
 	poetry install -E pyarrow -E hive -E s3fs -E glue -E adlfs -E duckdb -E ray -E sql-postgres -E gcsfs -E sql-sqlite -E daft

mkdocs/requirements.txt

@@ -15,14 +15,14 @@
 # specific language governing permissions and limitations
 # under the License.
 
-mkdocs==1.5.3
-griffe==0.40.1
+mkdocs==1.6.0
+griffe==0.44.0
 jinja2==3.1.3
-mkdocstrings==0.24.0
-mkdocstrings-python==1.8.0
+mkdocstrings==0.25.0
+mkdocstrings-python==1.10.0
 mkdocs-literate-nav==0.6.1
-mkdocs-autorefs==0.5.0
+mkdocs-autorefs==1.0.1
 mkdocs-gen-files==0.5.0
-mkdocs-material==9.5.18
+mkdocs-material==9.5.20
 mkdocs-material-extensions==1.3.1
 mkdocs-section-index==0.3.8

poetry.lock

@@ -206,7 +206,7 @@ class PyArrowFile(InputFile, OutputFile):
         >>> # output_file.create().write(b'foobytes')
     """
 
-    _fs: FileSystem
+    _filesystem: FileSystem
     _path: str
     _buffer_size: int
 
@@ -333,7 +333,7 @@ def parse_location(location: str) -> Tuple[str, str, str]:
         if not uri.scheme:
             return "file", uri.netloc, os.path.abspath(location)
         elif uri.scheme == "hdfs":
-            return uri.scheme, uri.netloc, location
+            return uri.scheme, uri.netloc, uri.path
         else:
             return uri.scheme, uri.netloc, f"{uri.netloc}{uri.path}"
 

pyiceberg/io/pyarrow.py

@@ -59,9 +59,9 @@ def bytes_required(value: Union[int, Decimal]) -> int:
         int: the minimum number of bytes needed to serialize the value.
     """
     if isinstance(value, int):
-        return (value.bit_length() + 7) // 8
+        return (value.bit_length() + 8) // 8
     elif isinstance(value, Decimal):
-        return (decimal_to_unscaled(value).bit_length() + 7) // 8
+        return (decimal_to_unscaled(value).bit_length() + 8) // 8
 
     raise ValueError(f"Unsupported value: {value}")
 

pyiceberg/utils/decimal.py

@@ -58,7 +58,7 @@ fsspec = ">=2023.1.0,<2025.1.0"
 pyparsing = ">=3.1.0,<4.0.0"
 zstandard = ">=0.13.0,<1.0.0"
 tenacity = ">=8.2.3,<9.0.0"
-pyarrow = { version = ">=9.0.0,<16.0.0", optional = true }
+pyarrow = { version = ">=9.0.0,<17.0.0", optional = true }
 pandas = { version = ">=1.0.0,<3.0.0", optional = true }
 duckdb = { version = ">=0.5.0,<1.0.0", optional = true }
 ray = { version = ">=2.0.0,<2.10.0", optional = true }
@@ -84,7 +84,7 @@ requests-mock = "1.12.1"
 moto = { version = "^5.0.2", extras = ["server"] }
 typing-extensions = "4.11.0"
 pytest-mock = "3.14.0"
-pyspark = "3.5.0"
+pyspark = "3.5.1"
 cython = "3.0.8"
 deptry = ">=0.14,<0.17"
 docutils = "!=0.21"

pyproject.toml

@@ -1644,9 +1644,9 @@ def check_results(location: str, expected_schema: str, expected_netloc: str, exp
         assert netloc == expected_netloc
         assert uri == expected_uri
 
-    check_results("hdfs://127.0.0.1:9000/root/foo.txt", "hdfs", "127.0.0.1:9000", "hdfs://127.0.0.1:9000/root/foo.txt")
-    check_results("hdfs://127.0.0.1/root/foo.txt", "hdfs", "127.0.0.1", "hdfs://127.0.0.1/root/foo.txt")
-    check_results("hdfs://clusterA/root/foo.txt", "hdfs", "clusterA", "hdfs://clusterA/root/foo.txt")
+    check_results("hdfs://127.0.0.1:9000/root/foo.txt", "hdfs", "127.0.0.1:9000", "/root/foo.txt")
+    check_results("hdfs://127.0.0.1/root/foo.txt", "hdfs", "127.0.0.1", "/root/foo.txt")
+    check_results("hdfs://clusterA/root/foo.txt", "hdfs", "clusterA", "/root/foo.txt")
 
     check_results("/root/foo.txt", "file", "", "/root/foo.txt")
     check_results("/root/tmp/foo.txt", "file", "", "/root/tmp/foo.txt")

tests/io/test_pyarrow.py

@@ -14,9 +14,11 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+from decimal import Decimal
+
 import pytest
 
-from pyiceberg.utils.decimal import decimal_required_bytes
+from pyiceberg.utils.decimal import decimal_required_bytes, decimal_to_bytes
 
 
 def test_decimal_required_bytes() -> None:
@@ -38,3 +40,10 @@ def test_decimal_required_bytes() -> None:
     with pytest.raises(ValueError) as exc_info:
         decimal_required_bytes(precision=-1)
     assert "(0, 40]" in str(exc_info.value)
+
+
+def test_decimal_to_bytes() -> None:
+    # Check the boundary between 2 and 3 bytes.
+    # 2 bytes has a minimum of -32,768 and a maximum value of 32,767 (inclusive).
+    assert decimal_to_bytes(Decimal('32767.')) == b'\x7f\xff'
+    assert decimal_to_bytes(Decimal('32768.')) == b'\x00\x80\x00'