Merge branch 'fix/test_nvs_gen_check_read_only_part_support' into 'master'

fix: test_nvs_gen_check.py support for read-only NVS partitions

Closes IDF-12238

See merge request espressif/esp-idf!36829

Author: adokitkat

components/nvs_flash/nvs_partition_tool/nvs_check.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 from typing import Dict
 from typing import List
@@ -21,10 +21,10 @@
 blob_chunks: List[NVS_Entry] = []
 
 
-def check_partition_size(nvs_partition: NVS_Partition, nvs_log: NVS_Logger) -> bool:
+def check_partition_size(nvs_partition: NVS_Partition, nvs_log: NVS_Logger, read_only: bool=False) -> bool:
     """ Checks if the partition is large enough and has enough pages
     """
-    if len(nvs_partition.raw_data) / 0x1000 < 3:
+    if len(nvs_partition.raw_data) // 0x1000 < 3 and not read_only:
         nvs_log.info(
             nvs_log.yellow(
                 'NVS Partition size must be at least 0x3000 (4kiB * 3 pages == 12kiB)!'
@@ -38,7 +38,7 @@ def check_partition_size(nvs_partition: NVS_Partition, nvs_log: NVS_Logger) -> b
             )
         )
         return False
-    if len(nvs_partition.pages) < 3:
+    if len(nvs_partition.pages) < 3 and not read_only:
         nvs_log.info(
             nvs_log.yellow(
                 'NVS Partition must contain 3 pages (sectors) at least to function properly!'

components/nvs_flash/nvs_partition_tool/test_nvs_gen_check.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
-# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
+from importlib.metadata import version
 from io import BufferedRandom
 from io import BytesIO
 from pathlib import Path
@@ -21,6 +22,18 @@
 from nvs_parser import nvs_const
 from nvs_parser import NVS_Entry
 from nvs_parser import NVS_Partition
+from packaging.version import Version
+
+
+NVS_PART_GEN_VERSION_SKIP = '0.1.8'
+
+
+# Temporary workaround for pytest skipping tests based on the version of the esp-idf-nvs-partition-gen package
+@pytest.fixture(scope='session', autouse=True)
+def before() -> None:
+    ver = version('esp-idf-nvs-partition-gen')
+    if Version(ver) < Version(NVS_PART_GEN_VERSION_SKIP):
+        pytest.skip('pass')
 
 
 class SilentLogger(NVS_Logger):
@@ -101,7 +114,7 @@ def create_entry_data_bytearray(namespace_index: int, entry_type: int, span: int
 
 @pytest.fixture
 def generate_nvs() -> Callable:
-    def _execute_nvs_setup(nvs_setup_func: Callable, size: int = 0x4000, output: Optional[Path] = None) -> NVS_Partition:
+    def _execute_nvs_setup(nvs_setup_func: Callable, output: Optional[Path] = None) -> NVS_Partition:
         nvs_file: Optional[Union[BytesIO, BufferedRandom]] = None
         if output is None:
             nvs_file = BytesIO()
@@ -110,15 +123,7 @@ def _execute_nvs_setup(nvs_setup_func: Callable, size: int = 0x4000, output: Opt
                 nvs_file = open(output, 'wb+')
             except OSError as e:
                 raise RuntimeError(f'Cannot open file {output}, error: {e}')
-        size_fixed = nvs_partition_gen.check_size(str(size))
-        nvs_obj = nvs_partition_gen.nvs_open(
-            result_obj=nvs_file,
-            input_size=size_fixed,
-            version=nvs_partition_gen.Page.VERSION2,
-            is_encrypt=False,
-            key=None
-        )
-        nvs_setup_func(nvs_obj)
+        nvs_obj = nvs_setup_func(nvs_file)
         nvs_partition_gen.nvs_close(nvs_obj)
         nvs_file.seek(0)
         nvs_parsed = NVS_Partition('test', bytearray(nvs_file.read()))
@@ -128,16 +133,36 @@ def _execute_nvs_setup(nvs_setup_func: Callable, size: int = 0x4000, output: Opt
 
 
 # Setup functions
-def setup_ok_primitive(nvs_obj: NVS) -> None:
+def setup_ok_primitive(nvs_file: Optional[Union[BytesIO, BufferedRandom]]) -> NVS:
+    size_fixed, read_only = nvs_partition_gen.check_size(str(0x4000))
+    nvs_obj = nvs_partition_gen.nvs_open(
+        result_obj=nvs_file,
+        input_size=size_fixed,
+        version=nvs_partition_gen.Page.VERSION2,
+        is_encrypt=False,
+        key=None,
+        read_only=read_only
+    )
+
     nvs_partition_gen.write_entry(nvs_obj, 'storage', 'namespace', '', '')
     nvs_partition_gen.write_entry(nvs_obj, 'int32_test', 'data', 'i32', str(42))
     nvs_partition_gen.write_entry(nvs_obj, 'uint32_test', 'data', 'u32', str(42))
     nvs_partition_gen.write_entry(nvs_obj, 'int8_test', 'data', 'i8', str(100))
 
+    return nvs_obj
+
+
+def setup_ok_variable_len(nvs_file: Optional[Union[BytesIO, BufferedRandom]]) -> NVS:
+    size_fixed, read_only = nvs_partition_gen.check_size(str(0x5000))
+    nvs_obj = nvs_partition_gen.nvs_open(
+        result_obj=nvs_file,
+        input_size=size_fixed,
+        version=nvs_partition_gen.Page.VERSION2,
+        is_encrypt=False,
+        key=None,
+        read_only=read_only
+    )
 
-def setup_ok_variable_len(nvs_obj: NVS) -> None:
-    size_fixed = nvs_partition_gen.check_size(str('0x5000'))
-    nvs_obj.size = size_fixed
     nvs_partition_gen.write_entry(nvs_obj, 'storage', 'namespace', '', '')
     nvs_partition_gen.write_entry(nvs_obj, 'short_string_key', 'data', 'string', 'Hello world!')
     nvs_partition_gen.write_entry(nvs_obj, 'blob_key', 'file', 'binary',
@@ -147,10 +172,20 @@ def setup_ok_variable_len(nvs_obj: NVS) -> None:
     nvs_partition_gen.write_entry(nvs_obj, 'multi_blob_key', 'file', 'binary',
                                   '../nvs_partition_generator/testdata/sample_multipage_blob.bin')
 
+    return nvs_obj
+
+
+def setup_ok_mixed(nvs_file: Optional[Union[BytesIO, BufferedRandom]]) -> NVS:
+    size_fixed, read_only = nvs_partition_gen.check_size(str(0x6000))
+    nvs_obj = nvs_partition_gen.nvs_open(
+        result_obj=nvs_file,
+        input_size=size_fixed,
+        version=nvs_partition_gen.Page.VERSION2,
+        is_encrypt=False,
+        key=None,
+        read_only=read_only
+    )
 
-def setup_ok_mixed(nvs_obj: NVS) -> None:
-    size_fixed = nvs_partition_gen.check_size(str('0x6000'))
-    nvs_obj.size = size_fixed
     prim_types = ['i8', 'u8', 'i16', 'u16', 'i32', 'u32']
 
     nvs_partition_gen.write_entry(nvs_obj, 'storage', 'namespace', '', '')
@@ -173,10 +208,20 @@ def setup_ok_mixed(nvs_obj: NVS) -> None:
     nvs_partition_gen.write_entry(nvs_obj, 'blob_key', 'file', 'binary',
                                   '../nvs_partition_generator/testdata/sample_multipage_blob.bin')
 
+    return nvs_obj
+
+
+def setup_bad_mixed_same_key_different_page(nvs_file: Optional[Union[BytesIO, BufferedRandom]]) -> NVS:
+    size_fixed, read_only = nvs_partition_gen.check_size(str(0x6000))
+    nvs_obj = nvs_partition_gen.nvs_open(
+        result_obj=nvs_file,
+        input_size=size_fixed,
+        version=nvs_partition_gen.Page.VERSION2,
+        is_encrypt=False,
+        key=None,
+        read_only=read_only
+    )
 
-def setup_bad_mixed_same_key_different_page(nvs_obj: NVS) -> None:
-    size_fixed = nvs_partition_gen.check_size(str('0x6000'))
-    nvs_obj.size = size_fixed
     prim_types = ['i8', 'u8', 'i16', 'u16', 'i32', 'u32']
 
     nvs_partition_gen.write_entry(nvs_obj, 'storage', 'namespace', '', '')
@@ -215,8 +260,20 @@ def setup_bad_mixed_same_key_different_page(nvs_obj: NVS) -> None:
     # the current used namespace index
     nvs_partition_gen.write_entry(nvs_obj, 'storage', 'namespace', '', '')
 
+    return nvs_obj
+
+
+def setup_bad_same_key_primitive(nvs_file: Optional[Union[BytesIO, BufferedRandom]]) -> NVS:
+    size_fixed, read_only = nvs_partition_gen.check_size(str(0x4000))
+    nvs_obj = nvs_partition_gen.nvs_open(
+        result_obj=nvs_file,
+        input_size=size_fixed,
+        version=nvs_partition_gen.Page.VERSION2,
+        is_encrypt=False,
+        key=None,
+        read_only=read_only
+    )
 
-def setup_bad_same_key_primitive(nvs_obj: NVS) -> None:
     nvs_partition_gen.write_entry(nvs_obj, 'storage', 'namespace', '', '')
     nvs_partition_gen.write_entry(nvs_obj, 'unique_key', 'data', 'i16', str(1234))
     nvs_partition_gen.write_entry(nvs_obj, 'same_key', 'data', 'i32', str(42))
@@ -225,17 +282,39 @@ def setup_bad_same_key_primitive(nvs_obj: NVS) -> None:
     nvs_partition_gen.write_entry(nvs_obj, 'another_same_key', 'data', 'u16', str(321))
     nvs_partition_gen.write_entry(nvs_obj, 'another_same_key', 'data', 'u16', str(456))
 
+    return nvs_obj
+
+
+def setup_bad_same_key_variable_len(nvs_file: Optional[Union[BytesIO, BufferedRandom]]) -> NVS:
+    size_fixed, read_only = nvs_partition_gen.check_size(str(0x4000))
+    nvs_obj = nvs_partition_gen.nvs_open(
+        result_obj=nvs_file,
+        input_size=size_fixed,
+        version=nvs_partition_gen.Page.VERSION2,
+        is_encrypt=False,
+        key=None,
+        read_only=read_only
+    )
 
-def setup_bad_same_key_variable_len(nvs_obj: NVS) -> None:
     nvs_partition_gen.write_entry(nvs_obj, 'storage', 'namespace', '', '')
     nvs_partition_gen.write_entry(nvs_obj, 'same_string_key', 'data', 'string', 'Hello')
     nvs_partition_gen.write_entry(nvs_obj, 'same_string_key', 'data', 'string', 'world!')
     nvs_partition_gen.write_entry(nvs_obj, 'unique_string_key', 'data', 'string', 'I am unique!')
 
+    return nvs_obj
+
+
+def setup_bad_same_key_blob_index(nvs_file: Optional[Union[BytesIO, BufferedRandom]]) -> NVS:
+    size_fixed, read_only = nvs_partition_gen.check_size(str(0x6000))
+    nvs_obj = nvs_partition_gen.nvs_open(
+        result_obj=nvs_file,
+        input_size=size_fixed,
+        version=nvs_partition_gen.Page.VERSION2,
+        is_encrypt=False,
+        key=None,
+        read_only=read_only
+    )
 
-def setup_bad_same_key_blob_index(nvs_obj: NVS) -> None:
-    size_fixed = nvs_partition_gen.check_size(str('0x6000'))
-    nvs_obj.size = size_fixed
     nvs_partition_gen.write_entry(nvs_obj, 'storage', 'namespace', '', '')
     nvs_partition_gen.write_entry(nvs_obj, 'blob_key', 'file', 'binary',
                                   '../nvs_partition_generator/testdata/sample_multipage_blob.bin')
@@ -244,6 +323,28 @@ def setup_bad_same_key_blob_index(nvs_obj: NVS) -> None:
     nvs_partition_gen.write_entry(nvs_obj, 'blob_key', 'file', 'binary',
                                   '../nvs_partition_generator/testdata/sample_multipage_blob.bin')  # Duplicate key
 
+    return nvs_obj
+
+
+def setup_read_only(nvs_file: Optional[Union[BytesIO, BufferedRandom]]) -> NVS:
+    size_fixed, read_only = nvs_partition_gen.check_size(str(0x1000))
+    nvs_obj = nvs_partition_gen.nvs_open(
+        result_obj=nvs_file,
+        input_size=size_fixed,
+        version=nvs_partition_gen.Page.VERSION2,
+        is_encrypt=False,
+        key=None,
+        read_only=read_only
+    )
+
+    nvs_partition_gen.write_entry(nvs_obj, 'storage', 'namespace', '', '')
+    nvs_partition_gen.write_entry(nvs_obj, 'int32_test', 'data', 'i32', str(42))
+    nvs_partition_gen.write_entry(nvs_obj, 'uint32_test', 'data', 'u32', str(42))
+    nvs_partition_gen.write_entry(nvs_obj, 'int8_test', 'data', 'i8', str(100))
+    nvs_partition_gen.write_entry(nvs_obj, 'short_string_key', 'data', 'string', 'Hello world!')
+
+    return nvs_obj
+
 
 # Helper functions
 def prepare_duplicate_list(nvs: NVS_Partition) -> Dict[str, List[NVS_Entry]]:
@@ -331,3 +432,12 @@ def test_check_duplicates_bad_same_key_blob_index(generate_nvs: Callable, setup_
     # however there are 2 duplicates of each blob_index and blob_data
     assert len(list(duplicates.values())[0]) == 6  # 6 entries with the blob_key (2x blob_index, 4x blob_data)
     nvs_check.integrity_check(nvs, logger)
+
+
+@pytest.mark.parametrize('setup_func', [setup_read_only])
+def test_check_read_only_partition(generate_nvs: Callable, setup_func: Callable) -> None:
+    nvs = generate_nvs(setup_func)
+    assert nvs.raw_data is not None
+    assert len(nvs.raw_data) == 0x1000
+    assert nvs_check.check_partition_size(nvs, logger, read_only=True)
+    assert not nvs_check.check_empty_page_present(nvs, logger)