Revise approach to Unicode filenames on Windows (#5794)

Treat filenames as UTF-8 initially and fall back to ANSI functions
if conversion to UTF-16 fails

Adds new HDF5_PREFER_WINDOWS_CODE_PAGE environment variable to
prefer interpreting filenames according to the active Windows code
page rather than assuming UTF-8 encoding

Author: jhendersonHDF

doxygen/aliases

@@ -254,6 +254,7 @@ ALIASES += cpp_c_api_note="\anchor cpp_c_api_note \attention \Bold{C++ Developer
 ALIASES += callback_note="\attention \Bold{Leaving callback functions:}\n The callback function must return normally, even in the case of error.  Returning with H5_ITER_ERROR, instead of leaving by means of exceptions, exit() function, etc... will allow the HDF5 library to manage its resources and maintain a consistent state. See \ref cpp_c_api_note \"C++ Developers using HDF5 C-API functions\" warning for detail."
 ALIASES += par_compr_note="\attention If you are planning to use compression with parallel HDF5, ensure that calls to H5Dwrite() occur in collective mode. In other words, all MPI ranks (in the relevant communicator) call H5Dwrite() and pass a dataset transfer property list with the MPI-IO collective option property set to #H5FD_MPIO_COLLECTIVE_IO.\n Note that data transformations are currently \Bold{not} supported when writing to datasets in parallel and with compression enabled."
 ALIASES += sa_metadata_ops="\sa \li H5Pget_all_coll_metadata_ops() \li H5Pget_coll_metadata_write() \li H5Pset_all_coll_metadata_ops() \li H5Pset_coll_metadata_write() \li \ref maybe_metadata_reads"
+ALIASES += unicode_filename_note="\note On Windows, HDF5 assumes that a file name string is UTF-8 encoded and will attempt to convert it to UTF-16 before calling wide-character Windows API functions. If a file name string cannot be converted to UTF-16, the equivalent non-wide-character Windows API functions will be used, causing a file name string to be interpreted according to the active Windows code page. If an application desires that the active Windows code page be preferred, the environment variable #HDF5_PREFER_WINDOWS_CODE_PAGE can be set to the value '1' or 'TRUE' (case-insensitive)."
 
 ################################################################################
 # Specifications

release_docs/RELEASE.txt

@@ -780,6 +780,30 @@ Bug Fixes since HDF5-2.0.0 release
 ===================================
     Library
     -------
+    - Revised handling of Unicode filenames on Windows
+
+      In the HDF5 1.14.4 release, a change was made to address some issues
+      with the library's handling of code pages and file paths on Windows.
+      This change introduced other issues with the handling of UTF-8 file
+      names that caused breakage for software using the 1.14.4 and 1.14.5
+      releases of HDF5. That change was reverted for the 1.14.6 release
+      and the behavior has been slightly modified for this release.
+
+      On Windows, the library once again assumes that filename strings will
+      be UTF-8 encoded strings and will attempt to convert them to UTF-16
+      before passing them to Windows API functions. However, if the library
+      fails to convert a filename string to UTF-16, it will now fallback to
+      the equivalent Windows "ANSI" API functions which will interpret the
+      string according to the active Windows code page.
+
+      Support for a new environment variable, HDF5_PREFER_WINDOWS_CODE_PAGE,
+      was added in order to instruct HDF5 to prefer interpreting filenames
+      according to the active Windows code page rather than assuming UTF-8
+      encoding. If this environment variable is set to "1" or "TRUE"
+      (case-insensitive), the active code page will be preferred. If it is
+      unset or set to "0" or "FALSE" (case-insensitive), UTF-8 will be
+      preferred.
+    
     - Fixed an issue with caching in the ROS3 VFD
 
       The ROS3 VFD uses a very simple caching mechanism that caches the

src/H5Fpublic.h

@@ -269,9 +269,15 @@ extern "C" {
  *          container_name can be opened with the file access property list
  *          \p fapl_id.
  *
+ * \parblock
  * \note The H5Fis_accessible() function enables files to be checked with a
  *       given file access property list, unlike H5Fis_hdf5(), which only uses
  *       the default file driver when opening a file.
+ * \endparblock
+ *
+ * \parblock
+ * \unicode_filename_note
+ * \endparblock
  *
  * \since 1.12.0
  *
@@ -320,13 +326,21 @@ H5_DLL htri_t H5Fis_accessible(const char *container_name, hid_t fapl_id);
  * \par Example
  * \snippet H5F_examples.c minimal
  *
+ * \parblock
  * \note  #H5F_ACC_TRUNC and #H5F_ACC_EXCL are mutually exclusive; use
  *        exactly one.
+ * \endparblock
  *
+ * \parblock
  * \note An additional flag, #H5F_ACC_DEBUG, prints debug information. This
  *       flag can be combined with one of the above values using the bit-wise
  *       OR operator (\c |), but it is used only by HDF5 library developers;
  *       \Emph{it is neither tested nor supported for use in applications}.
+ * \endparblock
+ *
+ * \parblock
+ * \unicode_filename_note
+ * \endparblock
  *
  * \attention \Bold{Special case — File creation in the case of an already-open file:}
  *            If a file being created is already opened, by either a previous
@@ -420,8 +434,14 @@ H5_DLL hid_t H5Fcreate_async(const char *filename, unsigned flags, hid_t fcpl_id
  * \par Example
  * \snippet H5F_examples.c open
  *
+ * \parblock
  * \note  #H5F_ACC_RDWR and #H5F_ACC_RDONLY are mutually exclusive; use
  *        exactly one.
+ * \endparblock
+ *
+ * \parblock
+ * \unicode_filename_note
+ * \endparblock
  *
  * \attention \Bold{Special cases — Multiple opens:} A file can often be opened
  *            with a new H5Fopen() call without closing an already-open
@@ -666,6 +686,10 @@ H5_DLL herr_t H5Fclose_async(hid_t file_id, hid_t es_id);
  *          is an HDF5 file via H5Fis_accessible(). This is done to ensure that
  *          H5Fdelete() cannot be used as an arbitrary file deletion call.
  *
+ * \parblock
+ * \unicode_filename_note
+ * \endparblock
+ *
  * \since 1.12.0
  *
  */
@@ -1958,6 +1982,10 @@ H5_DLL herr_t H5Fset_latest_format(hid_t file_id, hbool_t latest_format);
  *
  * \details H5Fis_hdf5() determines whether a file is in the HDF5 format.
  *
+ * \parblock
+ * \unicode_filename_note
+ * \endparblock
+ *
  * \since 1.0.0
  * \deprecated 1.12.0 Deprecated in favor of the function H5Fis_accessible()
  *

src/H5private.h

@@ -900,12 +900,13 @@ H5_DLL int HDvasprintf(char **bufp, const char *fmt, va_list _ap);
 #if defined(H5_HAVE_WINDOW_PATH)
 
 /* directory delimiter for Windows: slash and backslash are acceptable on Windows */
-#define H5_DIR_SLASH_SEPC        '/'
-#define H5_DIR_SEPC              '\\'
-#define H5_DIR_SEPS              "\\"
-#define H5_CHECK_DELIMITER(SS)   ((SS == H5_DIR_SEPC) || (SS == H5_DIR_SLASH_SEPC))
-#define H5_CHECK_ABSOLUTE(NAME)  ((isalpha(NAME[0])) && (NAME[1] == ':') && (H5_CHECK_DELIMITER(NAME[2])))
-#define H5_CHECK_ABS_DRIVE(NAME) ((isalpha(NAME[0])) && (NAME[1] == ':'))
+#define H5_DIR_SLASH_SEPC      '/'
+#define H5_DIR_SEPC            '\\'
+#define H5_DIR_SEPS            "\\"
+#define H5_CHECK_DELIMITER(SS) ((SS == H5_DIR_SEPC) || (SS == H5_DIR_SLASH_SEPC))
+#define H5_CHECK_ABSOLUTE(NAME)                                                                              \
+    ((isalpha((unsigned char)NAME[0])) && (NAME[1] == ':') && (H5_CHECK_DELIMITER(NAME[2])))
+#define H5_CHECK_ABS_DRIVE(NAME) ((isalpha((unsigned char)NAME[0])) && (NAME[1] == ':'))
 #define H5_CHECK_ABS_PATH(NAME)  (H5_CHECK_DELIMITER(NAME[0]))
 
 #define H5_GET_LAST_DELIMITER(NAME, ptr)                                                                     \

src/H5public.h

@@ -222,8 +222,8 @@
  * opening a file. Valid values for this environment variable are
  * as follows:
  *
- *  "TRUE" or "1"  - Request that file locks should be used
- *  "FALSE" or "0" - Request that file locks should NOT be used
+ *  "TRUE" or "1"  - Request that file locks should be used <br />
+ *  "FALSE" or "0" - Request that file locks should NOT be used <br />
  *  "BEST_EFFORT"  - Request that file locks should be used and
  *                     that any locking errors caused by file
  *                     locking being disabled on the system
@@ -238,6 +238,18 @@
  * \since 1.14.0
  */
 #define HDF5_NOCLEANUP "HDF5_NOCLEANUP"
+/**
+ * Macro for environment variable used to instruct HDF5 to prefer
+ * Windows code pages over UTF-8 for functions that accept 'char *'
+ * parameters. Valid values for this environment variable are as
+ * follows (case-insensitive):
+ *
+ * "TRUE" or "1"  - Request that Windows code pages be preferred <br />
+ * "FALSE" or "0" - Request that UTF-8 be preferred <br />
+ *
+ * \since 2.0.0
+ */
+#define HDF5_PREFER_WINDOWS_CODE_PAGE "HDF5_PREFER_WINDOWS_CODE_PAGE"
 
 /**
  * Status return values.  Failed integer functions in HDF5 result almost

src/H5system.c

@@ -447,43 +447,57 @@ Wflock(int fd, int operation)
  *
  * Purpose:      Gets a UTF-16 string from an UTF-8 (or ASCII) string.
  *
- * Return:       Success:    A pointer to a UTF-16 string
- *                           This must be freed by the caller using H5MM_xfree()
- *               Failure:    NULL
+ *               On success, a pointer to the new UTF-16 string is returned
+ *               in `wstring`. This must be freed by the caller using
+ *               H5MM_xfree().
+ *
+ * Return:       Non-negative on success/Negative on failure
+ *
+ *               On failure, the result of GetLastError() is returned
+ *               through the `win_error` parameter, if non-NULL.
  *
  *-------------------------------------------------------------------------
  */
-wchar_t *
-H5_get_utf16_str(const char *s)
+herr_t
+H5_get_utf16_str(const char *s, wchar_t **wstring, uint32_t *win_error)
 {
     int      nwchars = -1;   /* Length of the UTF-16 buffer */
     wchar_t *ret_s   = NULL; /* UTF-16 version of the string */
 
     /* Get the number of UTF-16 characters needed */
-    if (0 == (nwchars = MultiByteToWideChar(CP_UTF8, 0, s, -1, NULL, 0)))
+    if (0 == (nwchars = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, s, -1, NULL, 0)))
         goto error;
 
     /* Allocate a buffer for the UTF-16 string */
-    if (NULL == (ret_s = (wchar_t *)H5MM_calloc(sizeof(wchar_t) * (size_t)nwchars)))
+    if (NULL == (ret_s = H5MM_calloc(sizeof(wchar_t) * (size_t)nwchars)))
         goto error;
 
     /* Convert the input UTF-8 string to UTF-16 */
-    if (0 == MultiByteToWideChar(CP_UTF8, 0, s, -1, ret_s, nwchars))
+    if (0 == MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, s, -1, ret_s, nwchars))
         goto error;
 
-    return ret_s;
+    *wstring = ret_s;
+
+    return SUCCEED;
 
 error:
+    /* Store error value first before doing anything else */
+    if (win_error)
+        *win_error = (uint32_t)GetLastError();
+
     if (ret_s)
         H5MM_xfree((void *)ret_s);
-    return NULL;
+
+    *wstring = NULL;
+
+    return FAIL;
 } /* end H5_get_utf16_str() */
 
 /*-------------------------------------------------------------------------
  * Function:     Wopen
  *
  * Purpose:      Equivalent of open(2) for use on Windows. Necessary to
- *               handle code pages and Unicode on that platform.
+ *               handle Unicode and code pages on that platform.
  *
  * Return:       Success:    A POSIX file descriptor
  *               Failure:    -1
@@ -492,9 +506,13 @@ H5_get_utf16_str(const char *s)
 int
 Wopen(const char *path, int oflag, ...)
 {
-    int      fd    = -1;   /* POSIX file descriptor to be returned */
-    wchar_t *wpath = NULL; /* UTF-16 version of the path */
-    int      pmode = 0;    /* mode (optionally set via variable args) */
+    uint32_t win_error        = 0;     /* Windows error code for failures */
+    wchar_t *wpath            = NULL;  /* UTF-16 version of the path */
+    herr_t   h5_ret           = FAIL;  /* HDF5 return code */
+    char    *env              = NULL;  /* Environment variable string */
+    bool     prefer_code_page = false; /* Whether to prefer using the Windows code page */
+    int      fd               = -1;    /* POSIX file descriptor to be returned */
+    int      pmode            = 0;     /* mode (optionally set via variable args) */
 
     /* _O_BINARY must be set in Windows to avoid CR-LF <-> LF EOL
      * transformations when performing I/O. Note that this will
@@ -511,30 +529,36 @@ Wopen(const char *path, int oflag, ...)
         va_end(vl);
     }
 
-    /* First try opening the file with the normal POSIX open() call.
-     * This will handle ASCII without additional processing as well as
-     * systems where code pages are being used instead of true Unicode.
+    /*
+     * Check HDF5_PREFER_WINDOWS_CODE_PAGE environment variable to
+     * determine how to handle the pathname.
      */
-    if ((fd = open(path, oflag, pmode)) >= 0) {
-        /* If this succeeds, we're done */
-        goto done;
+    env = getenv(HDF5_PREFER_WINDOWS_CODE_PAGE);
+    if (env && (*env != '\0')) {
+        if (0 == HDstrcasecmp(env, "true") || 0 == strcmp(env, "1"))
+            prefer_code_page = true;
     }
 
-    if (errno == ENOENT) {
-        /* Not found, reset errno and try with UTF-16 */
-        errno = 0;
-    }
-    else {
-        /* Some other error (like permissions), so just exit */
-        goto done;
-    }
-
-    /* Convert the input UTF-8 path to UTF-16 */
-    if (NULL == (wpath = H5_get_utf16_str(path)))
-        goto done;
+    /*
+     * Unless requested to prefer Windows code pages, try to convert
+     * the pathname from UTF-8 to UTF-16. If this fails, fallback to
+     * the normal POSIX open() call.
+     */
+    if (!prefer_code_page) {
+        h5_ret = H5_get_utf16_str(path, &wpath, &win_error);
+        if (h5_ret >= 0) {
+            /* Open the file using a UTF-16 path */
+            fd = _wopen(wpath, oflag, pmode);
+        }
+        else {
+            if (ERROR_NO_UNICODE_TRANSLATION != win_error)
+                goto done;
 
-    /* Open the file using a UTF-16 path */
-    fd = _wopen(wpath, oflag, pmode);
+            fd = open(path, oflag, pmode);
+        }
+    }
+    else
+        fd = open(path, oflag, pmode);
 
 done:
     H5MM_xfree(wpath);
@@ -546,7 +570,7 @@ Wopen(const char *path, int oflag, ...)
  * Function:     Wremove
  *
  * Purpose:      Equivalent of remove(3) for use on Windows. Necessary to
- *               handle code pages and Unicode on that platform.
+ *               handle Unicode and code pages on that platform.
  *
  * Return:       Success:    0
  *               Failure:    -1
@@ -555,33 +579,43 @@ Wopen(const char *path, int oflag, ...)
 int
 Wremove(const char *path)
 {
-    wchar_t *wpath = NULL; /* UTF-16 version of the path */
-    int      ret   = -1;
+    uint32_t win_error        = 0;     /* Windows error code for failures */
+    wchar_t *wpath            = NULL;  /* UTF-16 version of the path */
+    herr_t   h5_ret           = FAIL;  /* HDF5 return code */
+    char    *env              = NULL;  /* Environment variable string */
+    bool     prefer_code_page = false; /* Whether to prefer using the Windows code page */
+    int      ret              = -1;
 
-    /* First try removing the file with the normal POSIX remove() call.
-     * This will handle ASCII without additional processing as well as
-     * systems where code pages are being used instead of true Unicode.
+    /*
+     * Check HDF5_PREFER_WINDOWS_CODE_PAGE environment variable to
+     * determine how to handle the pathname.
      */
-    if ((ret = remove(path)) >= 0) {
-        /* If this succeeds, we're done */
-        goto done;
-    }
-
-    if (errno == ENOENT) {
-        /* Not found, reset errno and try with UTF-16 */
-        errno = 0;
-    }
-    else {
-        /* Some other error (like permissions), so just exit */
-        goto done;
+    env = getenv(HDF5_PREFER_WINDOWS_CODE_PAGE);
+    if (env && (*env != '\0')) {
+        if (0 == HDstrcasecmp(env, "true") || 0 == strcmp(env, "1"))
+            prefer_code_page = true;
     }
 
-    /* Convert the input UTF-8 path to UTF-16 */
-    if (NULL == (wpath = H5_get_utf16_str(path)))
-        goto done;
+    /*
+     * Unless requested to prefer Windows code pages, try to convert
+     * the pathname from UTF-8 to UTF-16. If this fails, fallback to
+     * the normal POSIX remove() call.
+     */
+    if (!prefer_code_page) {
+        h5_ret = H5_get_utf16_str(path, &wpath, &win_error);
+        if (h5_ret >= 0) {
+            /* Remove the file using a UTF-16 path */
+            ret = _wremove(wpath);
+        }
+        else {
+            if (ERROR_NO_UNICODE_TRANSLATION != win_error)
+                goto done;
 
-    /* Remove the file using a UTF-16 path */
-    ret = _wremove(wpath);
+            ret = remove(path);
+        }
+    }
+    else
+        ret = remove(path);
 
 done:
     H5MM_xfree(wpath);