dependency updates

Author: robamu

README.md

@@ -29,12 +29,25 @@ The following features have not been implemented yet. PRs or notifications for d
 
 You can install this package from PyPI
 
-Install command assuming an active virtual environment:
+For example, using [`uv`](https://docs.astral.sh/uv/)
+
+Setting up virtual environment:
+
+```sh
+uv venv
+```
+
+Regular install:
 
 ```sh
-pip install cfdp-py
+uv pip install -e .
 ```
 
+Interactive install with testing support:
+
+```sh
+uv pip install -e ".[test]"
+```
 
 # Examples
 
@@ -47,14 +60,13 @@ If you want to run the tests, it is recommended to install `pytest` and `coverag
 first. You also have to install the package with the optional `test` feature:
 
 ```sh
-pip install coverage pytest
-pip install cfdp-py[test]
+uv pip install -e ".[test]"
 ```
 
 Running tests regularly:
 
 ```sh
-pytest .
+pytest
 ```
 
 Running tests with coverage:
@@ -91,11 +103,11 @@ make doctest
 Linting:
 
 ```sh
-ruff check .
+ruff check
 ```
 
 Formatting:
 
 ```sh
-ruff format .
+ruff format
 ```

docs/conf.py

@@ -64,7 +64,7 @@
 intersphinx_mapping = {
     "python": ("https://docs.python.org/3", None),
     "spacepackets": ("https://spacepackets.readthedocs.io/en/latest/", None),
-    "crcmod": ("https://crcmod.sourceforge.net/", None),
+    "fastcrc": ("https://fastcrc.readthedocs.io/en/latest/", None),
 }
 
 # -- Options for HTML output -------------------------------------------------

examples/cfdp-libre-cube-crosstest/requirements.txt

@@ -1 +1,3 @@
 cfdp @ git+https://gitlab.com/librecube/lib/python-cfdp.git
+# Install local cfdp-py interactively
+-e ../..

justfile

@@ -1,5 +1,11 @@
 # Run with `just all`
-all: fmt check test
+all: fmt check coverage
+
+setup:
+  uv venv
+
+install:
+  uv pip install -e ".[test]"
 
 fmt:
   ruff format
@@ -9,3 +15,7 @@ check:
 
 test:
   pytest
+
+coverage:
+  coverage run -m pytest
+  coverage report

pyproject.toml

@@ -27,14 +27,16 @@ classifiers = [
   "Topic :: Scientific/Engineering"
 ]
 dependencies = [
-  "spacepackets>=0.26.0, <=0.28",
-  "crcmod~=1.7",
+  "spacepackets>=0.30,<0.32",
+  "fastcrc~=0.3",
   "deprecation~=2.1",
 ]
 
 [project.optional-dependencies]
 test = [
-    "pyfakefs~=5.2"
+    "pyfakefs~=5.2",
+    "pytest",
+    "coverage"
 ]
 lint = ["ruff"]
 

src/cfdppy/filestore.py

@@ -7,10 +7,11 @@
 import os
 import platform
 import shutil
+import struct
 import subprocess
 from typing import TYPE_CHECKING, BinaryIO
 
-from crcmod.predefined import PredefinedCrc
+import fastcrc
 from spacepackets.cfdp.defs import NULL_CHECKSUM_U32, ChecksumType
 from spacepackets.cfdp.tlv import FilestoreResponseStatusCode
 
@@ -351,9 +352,12 @@ def checksum_type_to_crcmod_str(self, checksum_type: ChecksumType) -> str | None
             return "crc32c"
         raise ChecksumNotImplemented(checksum_type)
 
+    """
+
     def _generate_crc_calculator(self, checksum_type: ChecksumType) -> PredefinedCrc:
         self._verify_checksum(checksum_type)
         return PredefinedCrc(self.checksum_type_to_crcmod_str(checksum_type))
+    """
 
     def calculate_checksum(
         self,
@@ -370,16 +374,26 @@ def calculate_checksum(
             return calc_modular_checksum(file_path)
         if segment_len == 0:
             raise ValueError("segment length can not be 0")
-        crc_obj = self._generate_crc_calculator(checksum_type)
+        func = None
+        if checksum_type == ChecksumType.CRC_32:
+            func = fastcrc.crc32.iso_hdlc
+        elif checksum_type == ChecksumType.CRC_32C:
+            func = fastcrc.crc32.iscsi
+        else:
+            raise ChecksumNotImplemented(checksum_type)
+        current = func(b"")
         current_offset = 0
         # Calculate the file CRC
         with open(file_path, "rb") as file:
             while current_offset < size_to_verify:
                 read_len = min(segment_len, size_to_verify - current_offset)
                 if read_len > 0:
-                    crc_obj.update(self.read_from_opened_file(file, current_offset, read_len))
+                    current = func(
+                        self.read_from_opened_file(file, current_offset, read_len), current
+                    )
                 current_offset += read_len
-            return crc_obj.digest()
+            assert current is not None
+            return struct.pack("!I", current)
 
 
 HostFilestore = NativeFilestore

src/cfdppy/restricted_filestore.py

@@ -133,12 +133,14 @@ def calculate_checksum(
 
     def verify_checksum(
         self,
-        checksum: bytes,
+        checksum: int | bytes,
         checksum_type: ChecksumType,
         file_path: Path,
         size_to_verify: int,
         segment_len: int = 4096,
     ) -> bool:
+        if isinstance(checksum, int):
+            checksum = checksum.to_bytes(4, "big")
         """Verify checksum of file."""
         return super().verify_checksum(
             checksum=checksum,

tests/test_dest_handler.py

@@ -224,7 +224,7 @@ def _insert_file_segment(
         )
         return fsm_res
 
-    def _generic_insert_eof_pdu(self, file_size: int, checksum: bytes) -> FsmResult:
+    def _generic_insert_eof_pdu(self, file_size: int, checksum: int | bytes) -> FsmResult:
         eof_pdu = EofPdu(file_size=file_size, file_checksum=checksum, pdu_conf=self.src_pdu_conf)
         fsm_res = self.dest_handler.state_machine(eof_pdu)
         if self.expected_mode == TransmissionMode.UNACKNOWLEDGED:

tests/test_dest_handler_acked.py

@@ -1,6 +1,7 @@
 import struct
 import time
 
+import fastcrc
 from spacepackets.cfdp import (
     NULL_CHECKSUM_U32,
     ConditionCode,
@@ -18,7 +19,6 @@
     FinishedPdu,
     TransactionStatus,
 )
-from spacepackets.crc import mkPredefinedCrcFun
 
 from cfdppy.defs import CfdpState
 from cfdppy.handler.dest import FsmResult, TransactionStep
@@ -45,12 +45,12 @@ def test_acked_small_file_transfer(self):
         file_content = b"Hello World!"
         with open(self.src_file_path, "wb") as of:
             of.write(file_content)
-        crc32_func = mkPredefinedCrcFun("crc32")
-        crc32 = struct.pack("!I", crc32_func(file_content))
+        crc32 = fastcrc.crc32.iso_hdlc(file_content)
+        crc32_bytes = struct.pack("!I", crc32)
         # Basic acknowledged empty file transfer.
         self._generic_regular_transfer_init(len(file_content))
         self._insert_file_segment(file_content, 0)
-        fsm_res = self._generic_insert_eof_pdu(len(file_content), crc32)
+        fsm_res = self._generic_insert_eof_pdu(len(file_content), crc32_bytes)
         self._generic_eof_recv_indication_check(fsm_res)
         self._generic_verify_eof_ack_packet(fsm_res, TransactionStep.WAITING_FOR_FINISHED_ACK)
         finished_pdu = self._generic_no_error_finished_pdu_check_acked(fsm_res)
@@ -99,11 +99,11 @@ def test_deferred_missing_file_segment_handling(self):
         file_content = b"Hello World!"
         with open(self.src_file_path, "wb") as of:
             of.write(file_content)
-        crc32_func = mkPredefinedCrcFun("crc32")
-        crc32 = struct.pack("!I", crc32_func(file_content))
+        crc32 = fastcrc.crc32.iso_hdlc(file_content)
+        crc32_bytes = struct.pack("!I", crc32)
         self._generic_regular_transfer_init(len(file_content))
         self._insert_file_segment(file_content[0:5], 0)
-        fsm_res = self._generic_insert_eof_pdu(len(file_content), crc32)
+        fsm_res = self._generic_insert_eof_pdu(len(file_content), crc32_bytes)
         self._generic_eof_recv_indication_check(fsm_res)
         self._generic_verify_eof_ack_packet(fsm_res, TransactionStep.WAITING_FOR_MISSING_DATA)
         self._state_checker(fsm_res, 1, CfdpState.BUSY, TransactionStep.WAITING_FOR_MISSING_DATA)
@@ -126,8 +126,8 @@ def test_immediate_missing_file_seg_handling_0(self):
         file_content = b"Hello World!"
         with open(self.src_file_path, "wb") as of:
             of.write(file_content)
-        crc32_func = mkPredefinedCrcFun("crc32")
-        crc32 = struct.pack("!I", crc32_func(file_content))
+        crc32 = fastcrc.crc32.iso_hdlc(file_content)
+        crc32_bytes = struct.pack("!I", crc32)
         self._generic_regular_transfer_init(len(file_content))
         # A middle segment is missing now, with the expected lost segment tuple to be (3, 6). The
         # lost segment is immediately supplied.
@@ -137,7 +137,7 @@ def test_immediate_missing_file_seg_handling_0(self):
         # Insert the missing file content.
         self._insert_file_segment(file_content[3:6], 3, expected_progress=12)
         # All lost segments were delivered, regular transfer finish.
-        fsm_res = self._generic_insert_eof_pdu(len(file_content), crc32)
+        fsm_res = self._generic_insert_eof_pdu(len(file_content), crc32_bytes)
         self._generic_eof_recv_indication_check(fsm_res)
         self._generic_verify_eof_ack_packet(fsm_res, TransactionStep.WAITING_FOR_FINISHED_ACK)
         finished_pdu = self._generic_no_error_finished_pdu_check_acked(fsm_res)
@@ -148,8 +148,8 @@ def test_immediate_missing_file_seg_handling_1(self):
         file_content = b"Hello World!"
         with open(self.src_file_path, "wb") as of:
             of.write(file_content)
-        crc32_func = mkPredefinedCrcFun("crc32")
-        crc32 = struct.pack("!I", crc32_func(file_content))
+        crc32 = fastcrc.crc32.iso_hdlc(file_content)
+        crc32_bytes = struct.pack("!I", crc32)
         self._generic_regular_transfer_init(len(file_content))
         # Simulate the second segment being lost, with more than one segment following after that.
         self._insert_file_segment(file_content[0:2], 0)
@@ -160,7 +160,7 @@ def test_immediate_missing_file_seg_handling_1(self):
         # Now insert the missing segment.
         self._insert_file_segment(file_content[2:4], 2, expected_progress=12)
         # All lost segments were delivered, regular transfer finish.
-        fsm_res = self._generic_insert_eof_pdu(len(file_content), crc32)
+        fsm_res = self._generic_insert_eof_pdu(len(file_content), crc32_bytes)
         self._generic_eof_recv_indication_check(fsm_res)
         self._generic_verify_eof_ack_packet(fsm_res, TransactionStep.WAITING_FOR_FINISHED_ACK)
         finished_pdu = self._generic_no_error_finished_pdu_check_acked(fsm_res)
@@ -171,8 +171,8 @@ def test_immediate_multi_missing_segment_handling(self):
         file_content = b"Hello World!"
         with open(self.src_file_path, "wb") as of:
             of.write(file_content)
-        crc32_func = mkPredefinedCrcFun("crc32")
-        crc32 = struct.pack("!I", crc32_func(file_content))
+        crc32 = fastcrc.crc32.iso_hdlc(file_content)
+        crc32_bytes = struct.pack("!I", crc32)
         self._generic_regular_transfer_init(len(file_content))
         self._insert_file_segment(file_content[0:2], 0)
         self._insert_file_segment(file_content[4:6], 4, 1)
@@ -187,7 +187,7 @@ def test_immediate_multi_missing_segment_handling(self):
         self._insert_file_segment(file_content[2:4], 2, expected_progress=12)
         self._insert_file_segment(file_content[6:8], 6, expected_progress=12)
         # All lost segments were delivered, regular transfer finish.
-        fsm_res = self._generic_insert_eof_pdu(len(file_content), crc32)
+        fsm_res = self._generic_insert_eof_pdu(len(file_content), crc32_bytes)
         self._generic_eof_recv_indication_check(fsm_res)
         self._generic_verify_eof_ack_packet(fsm_res, TransactionStep.WAITING_FOR_FINISHED_ACK)
         finished_pdu = self._generic_no_error_finished_pdu_check_acked(fsm_res)
@@ -198,8 +198,8 @@ def test_immediate_missing_segment_also_rerequested_after_eof(self):
         file_content = b"Hello World!"
         with open(self.src_file_path, "wb") as of:
             of.write(file_content)
-        crc32_func = mkPredefinedCrcFun("crc32")
-        crc32 = struct.pack("!I", crc32_func(file_content))
+        crc32 = fastcrc.crc32.iso_hdlc(file_content)
+        crc32_bytes = struct.pack("!I", crc32)
         self._generic_regular_transfer_init(len(file_content))
         # Missing middle segment
         self._insert_file_segment(file_content[0:2], 0)
@@ -208,7 +208,7 @@ def test_immediate_missing_segment_also_rerequested_after_eof(self):
         self._generic_verify_missing_segment_requested(0, len(file_content), [(2, 6)])
 
         # All lost segments were delivered, regular transfer finish.
-        fsm_res = self._generic_insert_eof_pdu(len(file_content), crc32)
+        fsm_res = self._generic_insert_eof_pdu(len(file_content), crc32_bytes)
         self._generic_eof_recv_indication_check(fsm_res)
         self._generic_verify_eof_ack_packet(fsm_res, TransactionStep.WAITING_FOR_MISSING_DATA)
         self._state_checker(fsm_res, 1, CfdpState.BUSY, TransactionStep.WAITING_FOR_MISSING_DATA)
@@ -230,8 +230,8 @@ def test_multi_segment_missing_deferred_handling(self):
         file_content = b"Hello World!"
         with open(self.src_file_path, "wb") as of:
             of.write(file_content)
-        crc32_func = mkPredefinedCrcFun("crc32")
-        crc32 = struct.pack("!I", crc32_func(file_content))
+        crc32 = fastcrc.crc32.iso_hdlc(file_content)
+        crc32_bytes = struct.pack("!I", crc32)
         self._generic_regular_transfer_init(len(file_content))
         self._insert_file_segment(file_content[0:2], 0)
         self._insert_file_segment(file_content[4:6], 4, 1)
@@ -242,7 +242,7 @@ def test_multi_segment_missing_deferred_handling(self):
         self._insert_file_segment(file_content[8:], 8, 1)
         self._generic_verify_missing_segment_requested(0, len(file_content), [(6, 8)])
 
-        fsm_res = self._generic_insert_eof_pdu(len(file_content), crc32)
+        fsm_res = self._generic_insert_eof_pdu(len(file_content), crc32_bytes)
         self._generic_eof_recv_indication_check(fsm_res)
         self._generic_verify_eof_ack_packet(fsm_res, TransactionStep.WAITING_FOR_MISSING_DATA)
         self._state_checker(fsm_res, 1, CfdpState.BUSY, TransactionStep.WAITING_FOR_MISSING_DATA)
@@ -274,8 +274,8 @@ def test_missing_metadata_pdu(self):
         file_content = b"Hello World!"
         with open(self.src_file_path, "wb") as of:
             of.write(file_content)
-        crc32_func = mkPredefinedCrcFun("crc32")
-        crc32 = struct.pack("!I", crc32_func(file_content))
+        crc32 = fastcrc.crc32.iso_hdlc(file_content)
+        crc32_bytes = struct.pack("!I", crc32)
         self._insert_file_segment(
             file_content[0:2],
             0,
@@ -308,7 +308,7 @@ def test_missing_metadata_pdu(self):
             2,
         )
         # All lost segments were delivered, regular transfer finish.
-        fsm_res = self._generic_insert_eof_pdu(len(file_content), crc32)
+        fsm_res = self._generic_insert_eof_pdu(len(file_content), crc32_bytes)
         self._generic_eof_recv_indication_check(fsm_res)
         self._generic_verify_eof_ack_packet(fsm_res, TransactionStep.WAITING_FOR_FINISHED_ACK)
         finished_pdu = self._generic_no_error_finished_pdu_check_acked(fsm_res)
@@ -337,8 +337,8 @@ def test_metadata_eof_only_missing_metadata(self):
     def _generic_deferred_lost_segment_handling_with_timeout(self, file_content: bytes):
         with open(self.src_file_path, "wb") as of:
             of.write(file_content)
-        crc32_func = mkPredefinedCrcFun("crc32")
-        crc32 = struct.pack("!I", crc32_func(file_content))
+        crc32 = fastcrc.crc32.iso_hdlc(file_content)
+        crc32_bytes = struct.pack("!I", crc32)
         self._generic_regular_transfer_init(len(file_content))
         self._insert_file_segment(file_content[0:2], 0)
         self._insert_file_segment(file_content[4:6], 4, 1)
@@ -350,7 +350,7 @@ def _generic_deferred_lost_segment_handling_with_timeout(self, file_content: byt
         self._generic_verify_missing_segment_requested(0, len(file_content), [(6, 8)])
 
         # This should trigger deferred EOF handling.
-        fsm_res = self._generic_insert_eof_pdu(len(file_content), crc32)
+        fsm_res = self._generic_insert_eof_pdu(len(file_content), crc32_bytes)
         self._generic_eof_recv_indication_check(fsm_res)
         self._generic_verify_eof_ack_packet(fsm_res, TransactionStep.WAITING_FOR_MISSING_DATA)
         self._state_checker(fsm_res, 1, CfdpState.BUSY, TransactionStep.WAITING_FOR_MISSING_DATA)