[CDF-27407] Skip library checksum validation for downloads (#2774)

## Description

After careful consideration, decided to remove checksum validation. 

## Bump

- [x] Patch
- [ ] Skip

## Changelog

### Changed

- External module libraries are configured with a URL only; checksum
fields and post-download SHA256 verification are removed.

Author: ronpal

.github/workflows/build.yml

@@ -125,7 +125,7 @@ jobs:
       - name: Build the package
         run: uv build
       - name: Initialize project
-        run: uv run cdf modules init demo_project --clean --all --library-url "https://github.com/cognitedata/toolkit-data/releases/download/latest/packages.zip" --library-checksum "sha256:b70c207a78d6c94e0977a733571420b250ca641ac8802d8310f1ffe05bf7b2eb"
+        run: uv run cdf modules init demo_project --clean --all --library-url "https://github.com/cognitedata/toolkit-data/releases/download/latest/packages.zip"
       - name: "Pre-processing for demo environment"
         run: uv run python ./demo/preproc.py --modules all
       - name: "Build the templates"

.github/workflows/demo.yml

@@ -30,7 +30,7 @@ jobs:
       - name: Build the package
         run: uv build
       - name: Initialize project
-        run: uv run cdf modules init demo_project --clean --all --library-url "https://github.com/cognitedata/toolkit-data/releases/download/latest/packages.zip" --library-checksum "sha256:b70c207a78d6c94e0977a733571420b250ca641ac8802d8310f1ffe05bf7b2eb"
+        run: uv run cdf modules init demo_project --clean --all --library-url "https://github.com/cognitedata/toolkit-data/releases/download/latest/packages.zip"
       - name: "Pre-processing for demo environment"
         run: uv run python ./demo/preproc.py
       - name: "Build the templates"

LIBRARIES.md

@@ -13,7 +13,6 @@ external-libraries = true
 
 [library.package_1]
 url = "https://raw.githubusercontent.com/cognitedata/toolkit-data/librarian/builtins.zip"
-checksum = "sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"
 
 ```
 
@@ -26,9 +25,6 @@ The library must be available over https. Authentication is not currently suppor
 To publish a library, create a repository that contains one or more downloadable zip files.
 The zip file must have the structure and content described below.
 
-The checksum is mandatory. It is used to verify the integrity of the downloaded zip file.
-It must be a SHA-256 checksum of the zip file.
-
 ### <package_1>.zip
 
 Zip file content must be structured like this:

cdf.toml

@@ -35,5 +35,4 @@ version = "0.0.0"
 
 
 [library.toolkit-data]
-url = "https://github.com/cognitedata/toolkit-data/releases/download/latest/packages.zip" 
-checksum = "sha256:b70c207a78d6c94e0977a733571420b250ca641ac8802d8310f1ffe05bf7b2eb"
+url = "https://github.com/cognitedata/toolkit-data/releases/download/latest/packages.zip"

cognite_toolkit/_cdf_tk/apps/_modules_app.py

@@ -73,7 +73,7 @@ def init(
             typer.Option(
                 "--library-checksum",
                 "-c",
-                help="Checksum of the library to add to the project.",
+                help="Library zip checksum (optional; accepted for compatibility, not verified).",
             ),
         ] = None,
         verbose: Annotated[
@@ -92,12 +92,6 @@ def init(
             # This is only used for logging purposes in the command.
             client = EnvironmentVariables.create_from_environment().get_client()
 
-        if library_url and not library_checksum:
-            raise typer.BadParameter(
-                "--library-checksum must be provided when --library-url is specified.",
-                param_hint="--library-checksum",
-            )
-
         with ModulesCommand(client=client) as cmd:
             cmd.run(
                 lambda: cmd.init(

cognite_toolkit/_cdf_tk/cdf_toml.py

@@ -80,7 +80,6 @@ def load(cls, raw: dict[str, Any]) -> Self:
 @dataclass
 class Library:
     url: str
-    checksum: str
 
     def __post_init__(self) -> None:
         self._validate()
@@ -102,10 +101,7 @@ def load(cls, raw: dict[str, Any]) -> Self:
         if "url" not in raw:
             raise ValueError("Library configuration must contain 'url' field.")
 
-        if "checksum" not in raw:
-            raise ValueError("Library configuration must contain 'checksum' field.")
-
-        return cls(**raw)
+        return cls(url=raw["url"])
 
 
 @dataclass

cognite_toolkit/_cdf_tk/commands/modules.py

@@ -5,7 +5,6 @@
 import tempfile
 import zipfile
 from collections import Counter
-from hashlib import sha256
 from pathlib import Path
 from types import TracebackType
 from typing import Any, Literal
@@ -58,7 +57,6 @@
 from cognite_toolkit._cdf_tk.exceptions import ToolkitError, ToolkitRequiredValueError, ToolkitValueError
 from cognite_toolkit._cdf_tk.hints import verify_module_directory
 from cognite_toolkit._cdf_tk.tk_warnings import MediumSeverityWarning
-from cognite_toolkit._cdf_tk.tk_warnings.other import HighSeverityWarning
 from cognite_toolkit._cdf_tk.utils import humanize_collection, read_yaml_file
 from cognite_toolkit._cdf_tk.utils.file import safe_read, safe_rmtree, safe_write, yaml_safe_dump
 from cognite_toolkit._cdf_tk.utils.modules import module_directory_from_path
@@ -290,6 +288,8 @@ def init(
         library_url: str | None = None,
         library_checksum: str | None = None,
     ) -> None:
+        _ = library_checksum
+
         if not organization_dir:
             organization_dir = ModulesCommand._prompt_organization_dir()
 
@@ -298,11 +298,7 @@ def init(
         # Determine which library to use (if any)
         library: Library | None = None
         if library_url:
-            if not library_checksum:
-                raise ToolkitRequiredValueError(
-                    "The '--library-checksum' is required when '--library-url' is provided."
-                )
-            library = Library(url=library_url, checksum=library_checksum)
+            library = Library(url=library_url)
         elif not (organization_dir / CDFToml.file_name).exists():
             # Load default library from resources when cdf.toml doesn't exist
             default_cdf_toml = CDFToml.load(cwd=RESOURCES_PATH, use_singleton=False)
@@ -887,7 +883,6 @@ def _get_available_packages(self, user_library: Library | None = None) -> tuple[
                     )
                     file_path = self._temp_download_dir / filename
                     self._download(library.url, file_path)
-                    self._validate_checksum(library.checksum, file_path)
                     self._unpack(file_path)
                     packages = Packages().load(file_path.parent)
                     if packages.warnings:
@@ -996,40 +991,6 @@ def _download(self, url: str, file_path: Path) -> None:
         except requests.exceptions.RequestException as e:
             raise ToolkitError(f"Error downloading file from {url}: {e}") from e
 
-    def _validate_checksum(self, checksum: str, file_path: Path) -> None:
-        """
-        Compares the checksum of the downloaded file with the expected checksum.
-        """
-
-        if checksum.lower().startswith("sha256:"):
-            checksum = checksum[7:]
-        else:
-            raise ToolkitValueError(f"Unsupported checksum format: {checksum}. Expected 'sha256:' prefix")
-
-        chunk_size: int = 8192
-        sha256_hash = sha256()
-        try:
-            with open(file_path, "rb") as f:
-                # Read the file in chunks to handle large files efficiently
-                for chunk in iter(lambda: f.read(chunk_size), b""):
-                    sha256_hash.update(chunk)
-            calculated = sha256_hash.hexdigest()
-        except OSError as e:
-            raise ToolkitError(f"Failed to calculate checksum for {file_path}: {e}") from e
-        except Exception as e:
-            raise ToolkitError(f"Unexpected error during checksum calculation for {file_path}: {e}") from e
-
-        if calculated != checksum:
-            self.warn(
-                HighSeverityWarning(
-                    f"The provided checksum sha256:{checksum} does not match downloaded file hash sha256:{calculated}.\n"
-                    "Please verify the checksum with the source and update cdf.toml if needed.\n"
-                    "This may indicate that the package content has changed."
-                )
-            )
-        else:
-            print("[green]✓ Checksum verified[/green]")
-
     def _unpack(self, file_path: Path) -> None:
         """
         Unzips the downloaded file to the specified output path.

cognite_toolkit/_resources/cdf.toml

@@ -15,5 +15,4 @@ functions = false
 
 
 [library.toolkit-data]
-url = "https://github.com/cognitedata/toolkit-data/releases/download/latest/packages.zip" 
-checksum = "sha256:b70c207a78d6c94e0977a733571420b250ca641ac8802d8310f1ffe05bf7b2eb"
+url = "https://github.com/cognitedata/toolkit-data/releases/download/latest/packages.zip"

tests/test_unit/test_cdf_tk/test_commands/test_about.py

@@ -51,7 +51,6 @@ class TestAboutCommand:
                 version = "0.0.0"
                 [library.custom]
                 url = "https://example.com/packages.zip"
-                checksum = "abc123"
                 """,
                 [],
             ),
@@ -68,7 +67,6 @@ class TestAboutCommand:
                 graphql = true
                 [library.my-lib]
                 url = "https://example.com/lib.zip"
-                checksum = "xyz789"
                 """,
                 [],
             ),
@@ -89,10 +87,8 @@ class TestAboutCommand:
                 version = "0.0.0"
                 [library.lib1]
                 url = "https://example.com/lib1.zip"
-                checksum = "aaa"
                 [library.lib2]
                 url = "https://example.com/lib2.zip"
-                checksum = "bbb"
                 """,
                 [],
             ),

tests/test_unit/test_cdf_tk/test_commands/test_modules.py

@@ -1,8 +1,6 @@
 from __future__ import annotations
 
-import hashlib
 import json
-import re
 import zipfile
 from pathlib import Path
 from unittest.mock import MagicMock
@@ -17,7 +15,6 @@
 from cognite_toolkit._cdf_tk.constants import MODULES
 from cognite_toolkit._cdf_tk.data_classes import ModuleLocation, Package, Packages
 from cognite_toolkit._cdf_tk.exceptions import ToolkitError
-from cognite_toolkit._cdf_tk.tk_warnings.other import HighSeverityWarning
 from tests.data import COMPLETE_ORG, EXTERNAL_PACKAGE
 from tests.test_unit.utils import MockQuestionary
 
@@ -307,31 +304,6 @@ def test_unpack_errors_os_error_during_write(self, tmp_path: Path, monkeypatch:
         assert isinstance(excinfo.value.__cause__, OSError)
         assert "No space left on device" in str(excinfo.value.__cause__)
 
-    def test_checksum_format(self, tmp_path: Path) -> None:
-        invalid_checksum = "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"
-        with pytest.raises(ToolkitError) as excinfo:
-            ModulesCommand(module_source_dir=COMPLETE_ORG_MODULES)._validate_checksum(
-                invalid_checksum, Path(tmp_path / "test_file.zip")
-            )
-
-        assert "Unsupported checksum format" in str(excinfo.value)
-
-    def test_checksum_success(self, tmp_path: Path, monkeypatch: MonkeyPatch) -> None:
-        file_path = tmp_path / "test_file.zip"
-        dummy_file_content = b"PK\x05\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
-        file_path.write_bytes(dummy_file_content)
-
-        expected_checksum = f"sha256:{hashlib.sha256(dummy_file_content).hexdigest()}"
-
-        cmd = ModulesCommand(print_warning=True, skip_tracking=True, module_source_dir=COMPLETE_ORG / MODULES)
-        try:
-            cmd._validate_checksum(
-                checksum=expected_checksum,
-                file_path=file_path,  # Pass the correct Path object
-            )
-        except ToolkitError as e:
-            pytest.fail(f"'_validate_checksum' raised an unexpected ToolkitError: {e}")
-
     def test_download_deletes_existing_file(self, tmp_path: Path, monkeypatch: MonkeyPatch) -> None:
         """Test that _download method deletes existing zip files before downloading."""
         # Create a stale zip file that should be deleted
@@ -404,25 +376,6 @@ def test_iterate_modules_finds_modules_in_temp_download_dir(self, tmp_path: Path
 
         shutil.rmtree(mock_module_dir)
 
-    def test_checksum_mismatch_prints_warning(self, tmp_path: Path, capsys) -> None:
-        file_path = tmp_path / "mismatch.zip"
-        # Write some bytes so we get a deterministic SHA256
-        file_bytes = b"dummy-bytes-for-checksum-test"
-        file_path.write_bytes(file_bytes)
-
-        # Intentionally use a different checksum than the file's actual hash
-        wrong_checksum = "sha256:" + hashlib.sha256(b"some-other-content").hexdigest()
-
-        cmd = ModulesCommand(print_warning=True, skip_tracking=True, module_source_dir=COMPLETE_ORG / MODULES)
-        cmd._validate_checksum(wrong_checksum, file_path)
-
-        assert len(cmd.warning_list) == 1
-        warning = cmd.warning_list[0]
-        assert isinstance(warning, HighSeverityWarning)
-        # Expect: two SHA256 hex hashes in the message, one for provided and one for calculated
-        pattern = r"^The provided checksum sha256:[0-9a-f]{64} does not match downloaded file hash sha256:[0-9a-f]{64}"
-        assert re.search(pattern, warning.message_raw)
-
     def test_list_json_output_is_parseable(self, tmp_path: Path, monkeypatch: MonkeyPatch, capsys) -> None:
         location = MagicMock()
         location.path = Path("modules/my_module")