fatfs/add menuconfig options for different encodings

This commit adds character encoding configurations in for the fatfs component.
The FF_LFN_UNICODE definition in ffconf.h can now be changed to accept UTF-8 or
UTF-16 encoded filernames. Test cases using UTF-8 encoded file paths and names in
FatFs have also been added.

Closes #1183

Author: Dazza0

components/fatfs/Kconfig

@@ -107,6 +107,23 @@ config FATFS_MAX_LFN
    help
       Maximum long filename length. Can be reduced to save RAM.
 
+choice FATFS_API_ENCODING
+    prompt "API character encoding"
+    depends on !FATFS_LFN_NONE
+    default FATFS_API_ENCODING_ANSI_OEM
+    help
+        Choose encoding for character and string arguments/returns when using 
+        FATFS APIs. The encoding of arguments will usually depend on text 
+        editor settings.
+
+config FATFS_API_ENCODING_ANSI_OEM
+    bool "API uses ANSI/OEM encoding"
+config FATFS_API_ENCODING_UTF_16
+    bool "API uses UTF-16 encoding"
+config FATFS_API_ENCODING_UTF_8
+    bool "API uses UTF-8 encoding"
+endchoice
+
 config FATFS_FS_LOCK
    int "Number of simultaneously open files protected by lock function"
    default 0

components/fatfs/src/ffconf.h

@@ -129,7 +129,13 @@
 /  ff_memfree() in ffsystem.c, need to be added to the project. */
 
 
-#define FF_LFN_UNICODE	0
+#ifdef CONFIG_FATFS_API_ENCODING_UTF_8
+#define FF_LFN_UNICODE      2
+#elif defined(CONFIG_FATFS_API_ENCODING_UTF_16)
+#define FF_LFN_UNICODE      1
+#else /* CONFIG_FATFS_API_ENCODING_ANSI_OEM */
+#define FF_LFN_UNICODE      0
+#endif
 /* This option switches the character encoding on the API when LFN is enabled.
 /
 /   0: ANSI/OEM in current CP (TCHAR = char)
@@ -148,7 +154,7 @@
 /  on character encoding. When LFN is not enabled, these options have no effect. */
 
 
-#define FF_STRF_ENCODE	3
+#define FF_STRF_ENCODE      3
 /* When FF_LFN_UNICODE >= 1 with LFN enabled, string I/O functions, f_gets(),
 /  f_putc(), f_puts and f_printf() convert the character encoding in it.
 /  This option selects assumption of character encoding ON THE FILE to be

components/fatfs/test/test_fatfs_common.c

@@ -29,6 +29,7 @@
 #include "test_fatfs_common.h"
 
 const char* fatfs_test_hello_str = "Hello, World!\n";
+const char* fatfs_test_hello_str_utf = "世界，你好！\n";
 
 void test_fatfs_create_file_with_text(const char* name, const char* text)
 {
@@ -84,6 +85,17 @@ void test_fatfs_read_file(const char* filename)
     TEST_ASSERT_EQUAL(0, fclose(f));
 }
 
+void test_fatfs_read_file_utf_8(const char* filename)
+{
+    FILE* f = fopen(filename, "r");
+    TEST_ASSERT_NOT_NULL(f);
+    char buf[64] = { 0 };   //Doubled buffer size to allow for longer UTF-8 strings
+    int cb = fread(buf, 1, sizeof(buf), f);
+    TEST_ASSERT_EQUAL(strlen(fatfs_test_hello_str_utf), cb);
+    TEST_ASSERT_EQUAL(0, strcmp(fatfs_test_hello_str_utf, buf));
+    TEST_ASSERT_EQUAL(0, fclose(f));
+}
+
 void test_fatfs_open_max_files(const char* filename_prefix, size_t files_count)
 {
     FILE** files = calloc(files_count, sizeof(FILE*));
@@ -337,6 +349,85 @@ void test_fatfs_opendir_readdir_rewinddir(const char* dir_prefix)
     TEST_ASSERT_EQUAL(0, closedir(dir));
 }
 
+void test_fatfs_opendir_readdir_rewinddir_utf_8(const char* dir_prefix)
+{
+    char name_dir_inner_file[64];
+    char name_dir_inner[64];
+    char name_dir_file3[64];
+    char name_dir_file2[64];
+    char name_dir_file1[64];
+
+    snprintf(name_dir_inner_file, sizeof(name_dir_inner_file), "%s/内部目录/内部文件.txt", dir_prefix);
+    snprintf(name_dir_inner, sizeof(name_dir_inner), "%s/内部目录", dir_prefix);
+    snprintf(name_dir_file3, sizeof(name_dir_file3), "%s/文件三.bin", dir_prefix);
+    snprintf(name_dir_file2, sizeof(name_dir_file2), "%s/文件二.txt", dir_prefix);
+    snprintf(name_dir_file1, sizeof(name_dir_file1), "%s/文件一.txt", dir_prefix);
+
+    unlink(name_dir_inner_file);
+    rmdir(name_dir_inner);
+    unlink(name_dir_file1);
+    unlink(name_dir_file2);
+    unlink(name_dir_file3);
+    rmdir(dir_prefix);
+
+    TEST_ASSERT_EQUAL(0, mkdir(dir_prefix, 0755));
+    test_fatfs_create_file_with_text(name_dir_file1, "一号\n");
+    test_fatfs_create_file_with_text(name_dir_file2, "二号\n");
+    test_fatfs_create_file_with_text(name_dir_file3, "\0一\0二\0三");
+    TEST_ASSERT_EQUAL(0, mkdir(name_dir_inner, 0755));
+    test_fatfs_create_file_with_text(name_dir_inner_file, "三号\n");
+
+    DIR* dir = opendir(dir_prefix);
+    TEST_ASSERT_NOT_NULL(dir);
+    int count = 0;
+    const char* names[4];
+    while(count < 4) {
+        struct dirent* de = readdir(dir);
+        if (!de) {
+            break;
+        }
+        printf("found '%s'\n", de->d_name);
+        if (strcasecmp(de->d_name, "文件一.txt") == 0) {
+            TEST_ASSERT_TRUE(de->d_type == DT_REG);
+            names[count] = "文件一.txt";
+            ++count;
+        } else if (strcasecmp(de->d_name, "文件二.txt") == 0) {
+            TEST_ASSERT_TRUE(de->d_type == DT_REG);
+            names[count] = "文件二.txt";
+            ++count;
+        } else if (strcasecmp(de->d_name, "内部目录") == 0) {
+            TEST_ASSERT_TRUE(de->d_type == DT_DIR);
+            names[count] = "内部目录";
+            ++count;
+        } else if (strcasecmp(de->d_name, "文件三.bin") == 0) {
+            TEST_ASSERT_TRUE(de->d_type == DT_REG);
+            names[count] = "文件三.bin";
+            ++count;
+        } else {
+            TEST_FAIL_MESSAGE("unexpected directory entry");
+        }
+    }
+    TEST_ASSERT_EQUAL(count, 4);
+
+    rewinddir(dir);
+    struct dirent* de = readdir(dir);
+    TEST_ASSERT_NOT_NULL(de);
+    TEST_ASSERT_EQUAL(0, strcasecmp(de->d_name, names[0]));
+    seekdir(dir, 3);
+    de = readdir(dir);
+    TEST_ASSERT_NOT_NULL(de);
+    TEST_ASSERT_EQUAL(0, strcasecmp(de->d_name, names[3]));
+    seekdir(dir, 1);
+    de = readdir(dir);
+    TEST_ASSERT_NOT_NULL(de);
+    TEST_ASSERT_EQUAL(0, strcasecmp(de->d_name, names[1]));
+    seekdir(dir, 2);
+    de = readdir(dir);
+    TEST_ASSERT_NOT_NULL(de);
+    TEST_ASSERT_EQUAL(0, strcasecmp(de->d_name, names[2]));
+
+    TEST_ASSERT_EQUAL(0, closedir(dir));
+}
 
 typedef struct {
     const char* filename;
@@ -452,7 +543,6 @@ void test_fatfs_concurrent(const char* filename_prefix)
     vSemaphoreDelete(args4.done);
 }
 
-
 void test_fatfs_rw_speed(const char* filename, void* buf, size_t buf_size, size_t file_size, bool write)
 {
     const size_t buf_count = file_size / buf_size;
@@ -483,4 +573,3 @@ void test_fatfs_rw_speed(const char* filename, void* buf, size_t buf_size, size_
             (write)?"Wrote":"Read", file_size, buf_size, t_s * 1e3,
                     file_size / (1024.0f * 1024.0f * t_s));
 }
-

components/fatfs/test/test_fatfs_common.h

@@ -32,13 +32,16 @@
 
 
 const char* fatfs_test_hello_str;
+const char* fatfs_test_hello_str_utf;
 
 void test_fatfs_create_file_with_text(const char* name, const char* text);
 
 void test_fatfs_overwrite_append(const char* filename);
 
 void test_fatfs_read_file(const char* filename);
 
+void test_fatfs_read_file_utf_8(const char* filename);
+
 void test_fatfs_open_max_files(const char* filename_prefix, size_t files_count);
 
 void test_fatfs_lseek(const char* filename);
@@ -57,4 +60,6 @@ void test_fatfs_can_opendir(const char* path);
 
 void test_fatfs_opendir_readdir_rewinddir(const char* dir_prefix);
 
+void test_fatfs_opendir_readdir_rewinddir_utf_8(const char* dir_prefix);
+
 void test_fatfs_rw_speed(const char* filename, void* buf, size_t buf_size, size_t file_size, bool write);

components/fatfs/test/test_fatfs_sdmmc.c

@@ -50,6 +50,7 @@ static void test_teardown(void)
 }
 
 static const char* test_filename = "/sdcard/hello.txt";
+static const char* test_filename_utf_8 = "/sdcard/测试文件.txt";
 
 TEST_CASE("Mount fails cleanly without card inserted", "[fatfs][ignore]")
 {
@@ -239,3 +240,25 @@ TEST_CASE("(SD) mount two FAT partitions, SDMMC and WL, at the same time", "[fat
     fclose(f);
     TEST_ESP_OK(esp_vfs_fat_spiflash_unmount("/spiflash", wl_handle));
 }
+
+/*
+ * In FatFs menuconfig, set CONFIG_FATFS_API_ENCODING to UTF-8 and set the
+ * Codepage to CP936 (Simplified Chinese) in order to run the following tests.
+ * Ensure that the text editor is UTF-8 compatible when compiling these tests.
+ */
+#if defined(CONFIG_FATFS_API_ENCODING_UTF_8) && (CONFIG_FATFS_CODEPAGE == 936)
+TEST_CASE("(SD) can read file using UTF-8 encoded strings", "[fatfs][ignore]")
+{
+    test_setup();
+    test_fatfs_create_file_with_text(test_filename_utf_8, fatfs_test_hello_str_utf);
+    test_fatfs_read_file_utf_8(test_filename_utf_8);
+    test_teardown();
+}
+
+TEST_CASE("(SD) opendir, readdir, rewinddir, seekdir work as expected using UTF-8 encoded strings", "[fatfs][ignore]")
+{
+    test_setup();
+    test_fatfs_opendir_readdir_rewinddir_utf_8("/sdcard/目录");
+    test_teardown();
+}
+#endif

components/fatfs/test/test_fatfs_spiflash.c

@@ -96,7 +96,6 @@ TEST_CASE("(WL) can lseek", "[fatfs][wear_levelling]")
     test_teardown();
 }
 
-
 TEST_CASE("(WL) stat returns correct values", "[fatfs][wear_levelling]")
 {
     test_setup();
@@ -175,3 +174,25 @@ TEST_CASE("(WL) write/read speed test", "[fatfs][wear_levelling]")
     free(buf);
     test_teardown();
 }
+
+/*
+ * In FatFs menuconfig, set CONFIG_FATFS_API_ENCODING to UTF-8 and set the
+ * Codepage to CP936 (Simplified Chinese) in order to run the following tests.
+ * Ensure that the text editor is UTF-8 compatible when compiling these tests.
+ */
+#if defined(CONFIG_FATFS_API_ENCODING_UTF_8) && (CONFIG_FATFS_CODEPAGE == 936)
+TEST_CASE("(WL) can read file with UTF-8 encoded strings", "[fatfs][wear_levelling]")
+{
+    test_setup();
+    test_fatfs_create_file_with_text("/spiflash/测试文件.txt", fatfs_test_hello_str_utf);
+    test_fatfs_read_file_utf_8("/spiflash/测试文件.txt");
+    test_teardown();
+}
+
+TEST_CASE("(WL) opendir, readdir, rewinddir, seekdir work as expected using UTF-8 encoded strings", "[fatfs][wear_levelling]")
+{
+    test_setup();
+    test_fatfs_opendir_readdir_rewinddir_utf_8("/spiflash/目录");
+    test_teardown();
+}
+#endif