feat(esp_partition): Adds new esp_partition APIs

Author: KonstantinKondrashov

components/app_update/test_apps/test_app_update/main/test_switch_ota.c

@@ -101,24 +101,6 @@ static void copy_app_partition_with_offset(esp_ota_handle_t update_handle, const
     ESP_LOGI(TAG, "finish the copy process");
 }
 
-#if defined(CONFIG_BOOTLOADER_FACTORY_RESET) || defined(CONFIG_BOOTLOADER_APP_TEST)
-/* @brief Copies partition from source partition to destination partition.
- *
- * Partitions can be of any types and subtypes.
- * @param[in] dst_partition - Destination partition
- * @param[in] src_partition - Source partition
- */
-static void copy_partition(const esp_partition_t *dst_partition, const esp_partition_t *src_partition)
-{
-    const void *partition_bin = NULL;
-    esp_partition_mmap_handle_t data_map;
-    TEST_ESP_OK(esp_partition_mmap(src_partition, 0, src_partition->size, ESP_PARTITION_MMAP_DATA, &partition_bin, &data_map));
-    TEST_ESP_OK(esp_partition_erase_range(dst_partition, 0, dst_partition->size));
-    TEST_ESP_OK(esp_partition_write(dst_partition, 0, (const void *)partition_bin, dst_partition->size));
-    esp_partition_munmap(data_map);
-}
-#endif
-
 /* @brief Get the next partition of OTA for the update.
  *
  * @return The next partition of OTA(OTA0-15).
@@ -530,7 +512,7 @@ static void test_flow5(void)
             ESP_LOGI(TAG, "Factory");
             TEST_ASSERT_EQUAL(ESP_PARTITION_SUBTYPE_APP_FACTORY, cur_app->subtype);
             set_output_pin(CONFIG_BOOTLOADER_NUM_PIN_APP_TEST);
-            copy_partition(esp_partition_find_first(ESP_PARTITION_TYPE_APP, ESP_PARTITION_SUBTYPE_APP_TEST, NULL), cur_app);
+            esp_partition_copy(esp_partition_find_first(ESP_PARTITION_TYPE_APP, ESP_PARTITION_SUBTYPE_APP_TEST, NULL), 0, cur_app, 0, cur_app->size);
             esp_restart();
             break;
         case 3:

components/esp_partition/host_test/partition_api_test/main/partition_api_test.c

@@ -722,6 +722,48 @@ TEST(partition_api, test_partition_power_off_emulation)
     free(test_data_ptr);
 }
 
+TEST(partition_api, test_partition_copy)
+{
+    const esp_partition_t *factory_part = esp_partition_find_first(ESP_PARTITION_TYPE_APP, ESP_PARTITION_SUBTYPE_APP_FACTORY, NULL);
+    TEST_ASSERT_NOT_NULL(factory_part);
+
+    const esp_partition_t *ota0_part = esp_partition_find_first(ESP_PARTITION_TYPE_APP, ESP_PARTITION_SUBTYPE_APP_OTA_0, NULL);
+    TEST_ASSERT_NOT_NULL(ota0_part);
+
+    TEST_ESP_OK(esp_partition_copy(ota0_part, 0, factory_part, 0, factory_part->size));
+    TEST_ESP_OK(esp_partition_copy(ota0_part, 0, factory_part, 0, SIZE_MAX));
+
+    TEST_ESP_ERR(ESP_ERR_INVALID_SIZE, esp_partition_copy(ota0_part, 0x1000000, factory_part, 0, SIZE_MAX));
+    TEST_ESP_ERR(ESP_ERR_INVALID_SIZE, esp_partition_copy(ota0_part, 0, factory_part, 0x1000000, SIZE_MAX));
+
+    TEST_ESP_ERR(ESP_ERR_INVALID_SIZE, esp_partition_copy(ota0_part, 0, factory_part, 0, SIZE_MAX - 1));
+
+    TEST_ESP_ERR(ESP_ERR_INVALID_SIZE, esp_partition_copy(ota0_part, UINT32_MAX - 1, factory_part, 0, 0x10000));
+    TEST_ESP_ERR(ESP_ERR_INVALID_SIZE, esp_partition_copy(ota0_part, 0, factory_part, UINT32_MAX - 1, 0x10000));
+
+    TEST_ESP_ERR(ESP_ERR_INVALID_SIZE, esp_partition_copy(ota0_part, UINT32_MAX - 1, factory_part, 0, SIZE_MAX));
+    TEST_ESP_ERR(ESP_ERR_INVALID_SIZE, esp_partition_copy(ota0_part, 0, factory_part, UINT32_MAX - 1, SIZE_MAX));
+}
+
+TEST(partition_api, test_partition_register_external)
+{
+    esp_err_t error;
+    const esp_partition_t *ota1_part = esp_partition_find_first(ESP_PARTITION_TYPE_APP, ESP_PARTITION_SUBTYPE_APP_OTA_1, NULL);
+    TEST_ASSERT_NULL(ota1_part);
+    const esp_partition_t *storage_part = esp_partition_find_first(ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_DATA_UNDEFINED, NULL);
+    error = esp_partition_register_external(NULL,
+        storage_part->address + storage_part->size, // place this new partition after the storage (the last part in the table)
+        1 * 1024 * 1024,
+        "ota_1",
+        ESP_PARTITION_TYPE_APP,
+        ESP_PARTITION_SUBTYPE_APP_OTA_1,
+        &ota1_part);
+    TEST_ESP_OK(error);
+    ota1_part = esp_partition_find_first(ESP_PARTITION_TYPE_APP, ESP_PARTITION_SUBTYPE_APP_OTA_1, NULL);
+    TEST_ASSERT_NOT_NULL(ota1_part);
+    TEST_ESP_OK(esp_partition_deregister_external(ota1_part));
+}
+
 TEST_GROUP_RUNNER(partition_api)
 {
     RUN_TEST_CASE(partition_api, test_partition_find_basic);
@@ -741,6 +783,8 @@ TEST_GROUP_RUNNER(partition_api)
     RUN_TEST_CASE(partition_api, test_partition_mmap_size_too_small);
     RUN_TEST_CASE(partition_api, test_partition_stats);
     RUN_TEST_CASE(partition_api, test_partition_power_off_emulation);
+    RUN_TEST_CASE(partition_api, test_partition_copy);
+    RUN_TEST_CASE(partition_api, test_partition_register_external);
 }
 
 static void run_all_tests(void)

components/esp_partition/host_test/partition_api_test/partition_table.csv

@@ -3,4 +3,5 @@
 nvs,        data, nvs,      0x9000,  0x6000,
 phy_init,   data, phy,      0xf000,  0x1000,
 factory,    app,  factory,  0x10000, 1M,
+ota_0,      app,  ota_0,    0x120000, 1M,
 storage,    data, ,             , 0x40000,

components/esp_partition/include/esp_partition.h

@@ -29,8 +29,8 @@ typedef struct esp_flash_t esp_flash_t;
  * @brief Enumeration which specifies memory space requested in an mmap call
  */
 typedef enum {
-    ESP_PARTITION_MMAP_DATA,    /**< map to data memory (Vaddr0), allows byte-aligned access, 4 MB total */
-    ESP_PARTITION_MMAP_INST,    /**< map to instruction memory (Vaddr1-3), allows only 4-byte-aligned access, 11 MB total */
+    ESP_PARTITION_MMAP_DATA,    /**< map to data memory (Vaddr0), allows byte-aligned access, (4 MB total - only for esp32) */
+    ESP_PARTITION_MMAP_INST,    /**< map to instruction memory (Vaddr1-3), allows only 4-byte-aligned access, (11 MB total - only for esp32) */
 } esp_partition_mmap_memory_t;
 
 /**
@@ -50,6 +50,8 @@ typedef uint32_t esp_partition_mmap_handle_t;
 typedef enum {
     ESP_PARTITION_TYPE_APP = 0x00,       //!< Application partition type
     ESP_PARTITION_TYPE_DATA = 0x01,      //!< Data partition type
+    ESP_PARTITION_TYPE_BOOTLOADER = 0x02, //!< Bootloader partition type
+    ESP_PARTITION_TYPE_PARTITION_TABLE = 0x03, //!< Partition table type
 
     ESP_PARTITION_TYPE_ANY = 0xff,       //!< Used to search for partitions with any type
 } esp_partition_type_t;
@@ -429,7 +431,7 @@ bool esp_partition_check_identity(const esp_partition_t* partition_1, const esp_
  * This API allows designating certain areas of external flash chips (identified by the esp_flash_t structure)
  * as partitions. This allows using them with components which access SPI flash through the esp_partition API.
  *
- * @param flash_chip  Pointer to the structure identifying the flash chip
+ * @param flash_chip  Pointer to the structure identifying the flash chip. If NULL then the internal flash chip is used (esp_flash_default_chip).
  * @param offset  Address in bytes, where the partition starts
  * @param size  Size of the partition in bytes
  * @param label  Partition name
@@ -472,6 +474,35 @@ void esp_partition_unload_all(void);
  */
 uint32_t esp_partition_get_main_flash_sector_size(void);
 
+/**
+ * @brief Copy data from a source partition at a specific offset to a destination partition at a specific offset.
+ *
+ * The destination offset must be aligned to the flash sector size (SPI_FLASH_SEC_SIZE = 0x1000).
+ * If "size" is SIZE_MAX, the entire destination partition (from dest_offset onward) will be erased,
+ * and the function will copy all of the source partition starting from src_offset into the destination.
+ * The function ensures that the destination partition is erased on sector boundaries (erase size is aligned up SPI_FLASH_SEC_SIZE).
+ *
+ * This function does the following:
+ * - erases the destination partition from dest_offset to the specified size (or the whole partition if "size" == SIZE_MAX),
+ * - maps data from the source partition in chunks,
+ * - writes the source data into the destination partition in corresponding chunks.
+ *
+ * @param dest_part   Pointer to a destination partition.
+ * @param dest_offset Offset in the destination partition where the data should be written (must be aligned to SPI_FLASH_SEC_SIZE = 0x1000).
+ * @param src_part    Pointer to a source partition (must be located on internal flash).
+ * @param src_offset  Offset in the source partition where the data should be read from.
+ * @param size        Number of bytes to copy from the source partition to the destination partition. If "size" is SIZE_MAX,
+ *                    the function copies from src_offset to the end of the source partition and erases
+ *                    the entire destination partition (from dest_offset onward).
+ *
+ * @return ESP_OK, if the source partition was copied successfully to the destination partition;
+ *         ESP_ERR_INVALID_ARG, if src_part or dest_part are incorrect, or if dest_offset is not sector aligned;
+ *         ESP_ERR_INVALID_SIZE, if the copy would go out of bounds of the source or destination partition;
+ *         ESP_ERR_NOT_ALLOWED, if the destination partition is read-only;
+ *         or one of the error codes from the lower-level flash driver.
+ */
+esp_err_t esp_partition_copy(const esp_partition_t* dest_part, uint32_t dest_offset, const esp_partition_t* src_part, uint32_t src_offset, size_t size);
+
 #ifdef __cplusplus
 }
 #endif

components/esp_partition/partition.c

@@ -9,6 +9,7 @@
 #include <string.h>
 #include <stdio.h>
 #include <sys/lock.h>
+#include <sys/param.h>
 
 /* interim to enable test_wl_host and test_fatfs_on_host compilation (both use IDF_TARGET_ESP32)
  * should go back to #include "sys/queue.h" once the tests are switched to CMake
@@ -24,11 +25,11 @@
 #include "esp_flash_partitions.h"
 #include "esp_attr.h"
 #include "esp_partition.h"
-#if !CONFIG_IDF_TARGET_LINUX
 #include "esp_flash.h"
+#if !CONFIG_IDF_TARGET_LINUX
 #include "esp_flash_encrypt.h"
-#include "spi_flash_mmap.h"
 #endif
+#include "spi_flash_mmap.h"
 #include "esp_log.h"
 #include "esp_rom_md5.h"
 #include "bootloader_util.h"
@@ -49,6 +50,8 @@
 #define INVARIANTS
 #endif
 
+#define ALIGN_UP(num, align) (((num) + ((align) - 1)) & ~((align) - 1))
+
 typedef struct partition_list_item_ {
     esp_partition_t info;
     bool user_registered;
@@ -68,6 +71,31 @@ static _lock_t s_partition_list_lock;
 
 static const char *TAG = "partition";
 
+static bool is_partition_encrypted(bool encryption_config, esp_partition_type_t type, esp_partition_subtype_t subtype)
+{
+#if CONFIG_IDF_TARGET_LINUX
+    (void) type;
+    (void) subtype;
+    (void) encryption_config;
+    return false;
+#else
+    bool ret_encrypted = encryption_config;
+    if (!esp_flash_encryption_enabled()) {
+        /* If flash encryption is not turned on, no partitions should be treated as encrypted */
+        ret_encrypted = false;
+    } else if (type == ESP_PARTITION_TYPE_APP
+                || (type == ESP_PARTITION_TYPE_BOOTLOADER)
+                || (type == ESP_PARTITION_TYPE_PARTITION_TABLE)
+                || (type == ESP_PARTITION_TYPE_DATA && subtype == ESP_PARTITION_SUBTYPE_DATA_OTA)
+                || (type == ESP_PARTITION_TYPE_DATA && subtype == ESP_PARTITION_SUBTYPE_DATA_NVS_KEYS)) {
+        /* If encryption is turned on, all app partitions and OTA data
+            are always encrypted */
+        ret_encrypted = true;
+    }
+    return ret_encrypted;
+#endif
+}
+
 // Create linked list of partition_list_item_t structures.
 // This function is called only once, with s_partition_list_lock taken.
 static esp_err_t load_partitions(void)
@@ -151,25 +179,10 @@ static esp_err_t load_partitions(void)
 #endif
         item->info.type = entry.type;
         item->info.subtype = entry.subtype;
-        item->info.encrypted = entry.flags & PART_FLAG_ENCRYPTED;
+        item->info.encrypted = is_partition_encrypted(entry.flags & PART_FLAG_ENCRYPTED, entry.type, entry.subtype);
         item->info.readonly = entry.flags & PART_FLAG_READONLY;
         item->user_registered = false;
 
-#if CONFIG_IDF_TARGET_LINUX
-        item->info.encrypted = false;
-#else
-        if (!esp_flash_encryption_enabled()) {
-            /* If flash encryption is not turned on, no partitions should be treated as encrypted */
-            item->info.encrypted = false;
-        } else if (entry.type == ESP_PARTITION_TYPE_APP
-                   || (entry.type == ESP_PARTITION_TYPE_DATA && entry.subtype == ESP_PARTITION_SUBTYPE_DATA_OTA)
-                   || (entry.type == ESP_PARTITION_TYPE_DATA && entry.subtype == ESP_PARTITION_SUBTYPE_DATA_NVS_KEYS)) {
-            /* If encryption is turned on, all app partitions and OTA data
-               are always encrypted */
-            item->info.encrypted = true;
-        }
-#endif
-
 #if CONFIG_NVS_COMPATIBLE_PRE_V4_3_ENCRYPTION_FLAG
         if (entry.type == ESP_PARTITION_TYPE_DATA &&
                 entry.subtype == ESP_PARTITION_SUBTYPE_DATA_NVS &&
@@ -392,10 +405,10 @@ esp_err_t esp_partition_register_external(esp_flash_t *flash_chip, size_t offset
         *out_partition = NULL;
     }
 
-#if CONFIG_IDF_TARGET_LINUX
-    return ESP_ERR_NOT_SUPPORTED;
-
-#else
+#if !CONFIG_IDF_TARGET_LINUX
+    if (flash_chip == NULL) {
+        flash_chip = esp_flash_default_chip;
+    }
     if (offset + size > flash_chip->size) {
         return ESP_ERR_INVALID_SIZE;
     }
@@ -415,7 +428,14 @@ esp_err_t esp_partition_register_external(esp_flash_t *flash_chip, size_t offset
     item->info.size = size;
     item->info.type = type;
     item->info.subtype = subtype;
+#if CONFIG_IDF_TARGET_LINUX
+    item->info.erase_size = ESP_PARTITION_EMULATED_SECTOR_SIZE;
     item->info.encrypted = false;
+#else
+    item->info.erase_size = SPI_FLASH_SEC_SIZE;
+    item->info.encrypted = (flash_chip == esp_flash_default_chip) ? is_partition_encrypted(false, type, subtype) : false;
+#endif // CONFIG_IDF_TARGET_LINUX
+    item->info.readonly = false;
     item->user_registered = true;
     strlcpy(item->info.label, label, sizeof(item->info.label));
 
@@ -466,3 +486,75 @@ esp_err_t esp_partition_deregister_external(const esp_partition_t *partition)
     _lock_release(&s_partition_list_lock);
     return result;
 }
+
+esp_err_t esp_partition_copy(const esp_partition_t* dest_part, uint32_t dest_offset, const esp_partition_t* src_part, uint32_t src_offset, size_t size)
+{
+    if (src_part == NULL || dest_part == NULL || src_part == dest_part) {
+        return ESP_ERR_INVALID_ARG;
+    }
+
+    if (src_offset > src_part->size || dest_offset > dest_part->size) {
+        return ESP_ERR_INVALID_SIZE;
+    }
+
+    // Check if the source partition is on external flash and return error
+#if !CONFIG_IDF_TARGET_LINUX
+    if (src_part->flash_chip != esp_flash_default_chip) {
+        ESP_LOGE(TAG, "Source partition is on external flash. Operation not supported.");
+        return ESP_ERR_NOT_SUPPORTED;
+    }
+#endif
+
+    size_t dest_erase_size = size;
+    if (size == SIZE_MAX) {
+        size = src_part->size - src_offset;
+        dest_erase_size = dest_part->size - dest_offset; // Erase the whole destination partition
+    }
+
+    uint32_t src_end_offset;
+    uint32_t dest_end_offset;
+    if ((__builtin_add_overflow(src_offset, size, &src_end_offset) || (src_end_offset > src_part->size))
+        || (__builtin_add_overflow(dest_offset, size, &dest_end_offset) || (dest_end_offset > dest_part->size))) { // with overflow checks
+        return ESP_ERR_INVALID_SIZE;
+    }
+
+    esp_err_t error = esp_partition_erase_range(dest_part, dest_offset, ALIGN_UP(dest_erase_size, SPI_FLASH_SEC_SIZE));
+    if (error) {
+        ESP_LOGE(TAG, "Erasing destination partition range failed (err=0x%x)", error);
+        return error;
+    }
+
+    uint32_t src_current_offset = src_offset;
+    uint32_t dest_current_offset = dest_offset;
+    size_t remaining_size = size;
+    /* Read the portion that fits in the free MMU pages */
+    uint32_t mmu_free_pages_count = spi_flash_mmap_get_free_pages(SPI_FLASH_MMAP_DATA);
+    int attempts_for_mmap = 0;
+    while (remaining_size > 0) {
+        uint32_t chunk_size = MIN(remaining_size, mmu_free_pages_count * SPI_FLASH_MMU_PAGE_SIZE);
+        esp_partition_mmap_handle_t src_part_map;
+        const void *src_data = NULL;
+        error = esp_partition_mmap(src_part, src_current_offset, chunk_size, ESP_PARTITION_MMAP_DATA, &src_data, &src_part_map);
+        if (error == ESP_OK) {
+            attempts_for_mmap = 0;
+            error = esp_partition_write(dest_part, dest_current_offset, src_data, chunk_size);
+            if (error != ESP_OK) {
+                ESP_LOGE(TAG, "Writing to destination partition failed (err=0x%x)", error);
+                esp_partition_munmap(src_part_map);
+                break;
+            }
+            esp_partition_munmap(src_part_map);
+        } else {
+            mmu_free_pages_count = spi_flash_mmap_get_free_pages(SPI_FLASH_MMAP_DATA);
+            chunk_size = 0;
+            if (++attempts_for_mmap >= 3) {
+                ESP_LOGE(TAG, "Failed to mmap source partition after a few attempts, mmu_free_pages = %" PRIu32 " (err=0x%x)", mmu_free_pages_count, error);
+                break;
+            }
+        }
+        src_current_offset += chunk_size;
+        dest_current_offset += chunk_size;
+        remaining_size -= chunk_size;
+    }
+    return error;
+}

components/spi_flash/CMakeLists.txt

@@ -2,6 +2,7 @@ idf_build_get_property(target IDF_TARGET)
 if(${target} STREQUAL "linux")
     idf_component_register(SRCS "linux/spi_flash_linux.c"
                           "linux/cache_utils.c"
+                          "linux/flash_mmap.c"
                            INCLUDE_DIRS include
                            PRIV_INCLUDE_DIRS include/spi_flash)
     return()

components/spi_flash/linux/flash_mmap.c

@@ -0,0 +1,12 @@
+/*
+ * SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#include "spi_flash_mmap.h"
+
+uint32_t spi_flash_mmap_get_free_pages(spi_flash_mmap_memory_t memory)
+{
+    (void) memory;
+    return 10;
+}