Draft: Support Win32 long path

.gitattributes

@@ -5,6 +5,8 @@
 *.c text eol=lf
 *.h text eol=lf
 
+# These are explicitly windows files and should use crlf
+*.bat text eol=crlf
+
 # Denote files that should not be modified.
 *.odt binary
-

build/cmake/CMakeLists.txt

@@ -75,6 +75,22 @@ CMAKE_DEPENDENT_OPTION(BUILD_SHARED_LIBS "Build shared libraries" ON "NOT XXHASH
 CMAKE_HOST_SYSTEM_INFORMATION(RESULT PLATFORM QUERY OS_PLATFORM)
 message(STATUS "Architecture: ${PLATFORM}")
 
+# Detect target is Windows 10+/Windows Server 2016+ or not
+option(XXHASH_WIN_TARGET_WIN10 "Treat this build as targeting Windows 10 or later (MinGW too)" OFF)
+if (XXHASH_WIN_TARGET_WIN10)
+  add_compile_definitions(_WIN32_WINNT=0x0A00)
+endif()
+
+set(XXHASH_HAS_WIN10_TARGET FALSE)
+if (WIN32)
+  # MSVC and Windows >= 10
+  if (MSVC AND CMAKE_SYSTEM_VERSION VERSION_GREATER_EQUAL "10.0")
+    set(XXHASH_HAS_WIN10_TARGET TRUE)
+  elseif (XXHASH_WIN_TARGET_WIN10)
+    set(XXHASH_HAS_WIN10_TARGET TRUE)
+  endif()
+endif()
+
 # libxxhash
 if((DEFINED DISPATCH) AND (DEFINED PLATFORM))
   # Only support DISPATCH option on x86_64.
@@ -122,6 +138,10 @@ if(XXHASH_BUILD_XXHSUM)
   add_executable(${PROJECT_NAME}::xxhsum ALIAS xxhsum)
 
   target_link_libraries(xxhsum PRIVATE xxhash)
+  if (XXHASH_HAS_WIN10_TARGET)
+    target_compile_definitions(xxhsum PRIVATE XXHSUM_WIN32_LONGPATH=1)
+    target_link_libraries(xxhsum PRIVATE Pathcch.lib)
+  endif()
   target_include_directories(xxhsum PRIVATE "${XXHASH_DIR}")
 endif(XXHASH_BUILD_XXHSUM)
 

cli/xsum_os_specific.c

@@ -158,6 +158,14 @@ int main(int argc, const char* argv[])
 #else
 #  include <windows.h>
 #  include <wchar.h>
+#  if defined(XXHSUM_WIN32_LONGPATH) && (XXHSUM_WIN32_LONGPATH)
+#    include <pathcch.h> /* PathCchCanonicalizeEx, PathCchCombineEx */
+     /* Older Windows SDK (< WIN10 1703 (RS2)) doesn't contain the following macro */
+#    if defined(PATHCCH_DO_NOT_NORMALIZE_SEGMENTS) && \
+        defined(PATHCCH_ENSURE_IS_EXTENDED_LENGTH_PATH)
+#      define XXHSUM_WIN32_USE_PATHCCH 1
+#    endif
+#  endif
 
 /*****************************************************************************
  *                       Unicode conversion tools
@@ -201,6 +209,76 @@ static char* XSUM_narrowString(const wchar_t *str, int *lenOut)
     }
 }
 
+/*
+ * Converts a UTF-8 path to absolute extended-length path with "\\?\" prefix in UTF-16.
+ * Acts like strdup. The string must be freed afterwards.
+ * This version allows keeping the output length.
+ *
+ * Note: The \\?\ prefix (prefix with question) designates a file-system-only path.
+ * Unlike the \\.\ prefix (prefix with dot), it does not provide access to DOS device names (e.g. COM1, NUL, CON, etc).
+ */
+static wchar_t* XSUM_widenStringAsExtendedLengthPath(const char* path)
+{
+#if defined(XXHSUM_WIN32_USE_PATHCCH) && (XXHSUM_WIN32_USE_PATHCCH)
+    wchar_t* const wide_path = XSUM_widenString(path, NULL);  /* path in wchar_t */
+    size_t const path_len = strlen(path);
+    int const starts_with_extended_prefix = path_len >= 4 && path[0] == '\\' && path[1] == '\\' && path[2] == '?' && path[3] == '\\';
+
+    /* If path starts with "\\?\" */
+    if(starts_with_extended_prefix) {
+        /* just return wchar_t version of it. */
+        return wide_path;
+    } else {
+        wchar_t* result = NULL;
+
+        size_t const size_in_wchars  = 32768; /* 32767 wchar_t + NUL */
+        ULONG  const canonical_flags = PATHCCH_DO_NOT_NORMALIZE_SEGMENTS;
+        ULONG  const combine_flags   = canonical_flags | PATHCCH_ENSURE_IS_EXTENDED_LENGTH_PATH;
+
+        /* exl_path : buffer for extended length path */
+        wchar_t* const exl_path = (wchar_t*) malloc(size_in_wchars * sizeof(wchar_t));
+        if(exl_path != NULL) {
+            int const starts_with_unc_absolute = path_len >= 2 && path[0] == '\\' && path[1] == '\\';
+            int const starts_with_dos_absolute = path_len >= 3 && isalpha(path[0]) && path[1] == ':' && (path[2] == '\\' || path[2] == '/');
+
+            /* If path starts with "\\" or "[A-Za-z]:\" */
+            if(starts_with_unc_absolute || starts_with_dos_absolute) {
+                if(exl_path != NULL) {
+                    HRESULT const hr = PathCchCanonicalizeEx(exl_path, size_in_wchars, wide_path, canonical_flags);
+                    if(SUCCEEDED(hr)) {
+                        result = exl_path;
+                    }
+                }
+            } else {
+                /* path is relative path */
+                wchar_t* const cwd = (wchar_t*) malloc(size_in_wchars * sizeof(wchar_t));
+                if(cwd != NULL) {
+                    DWORD const n = GetCurrentDirectoryW((DWORD) size_in_wchars, cwd);
+                    if(n != 0 && n < size_in_wchars) {
+                        if(exl_path != NULL) {
+                            HRESULT const hr = PathCchCombineEx(exl_path, size_in_wchars, cwd, wide_path, combine_flags);
+                            if(SUCCEEDED(hr)) {
+                                result = exl_path;
+                            }
+                        }
+                    }
+                    free(cwd);
+                }
+            }
+
+            /* Tricky part: if result doesn't use exl_path, free exl_path */
+            if(result != exl_path) {
+                free(exl_path);
+            }
+        }
+        free(wide_path);
+        return result;
+    }
+#else
+    return XSUM_widenString(path, NULL);  /* path in wchar_t */
+#endif
+}
+
 
 
 /*****************************************************************************
@@ -211,12 +289,16 @@ static char* XSUM_narrowString(const wchar_t *str, int *lenOut)
  *
  * fopen will only accept ANSI filenames, which means that we can't open Unicode filenames.
  *
- * In order to open a Unicode filename, we need to convert filenames to UTF-16 and use _wfopen.
+ * In order to open a Unicode filename and long path, we need to convert filenames to UTF-16,
+ * absolute path, UNC and use _wfopen.
+ *
+ * Note: The \\?\ prefix designates a file-system-only path.
+ * Unlike the \\.\ prefix, it does not provide access to DOS device names (e.g. COM1, NUL, CON, etc).
  */
 XSUM_API FILE* XSUM_fopen(const char* filename, const char* mode)
 {
     FILE* f = NULL;
-    wchar_t* const wide_filename = XSUM_widenString(filename, NULL);
+    wchar_t* const wide_filename = XSUM_widenStringAsExtendedLengthPath(filename);
     if (wide_filename != NULL) {
         wchar_t* const wide_mode = XSUM_widenString(mode, NULL);
         if (wide_mode != NULL) {
@@ -234,7 +316,7 @@ XSUM_API FILE* XSUM_fopen(const char* filename, const char* mode)
 static int XSUM_stat(const char* infilename, XSUM_stat_t* statbuf)
 {
     int r = -1;
-    wchar_t* const wide_filename = XSUM_widenString(infilename, NULL);
+    wchar_t* const wide_filename = XSUM_widenStringAsExtendedLengthPath(infilename);
     if (wide_filename != NULL) {
         r = _wstat64(wide_filename, statbuf);
         free(wide_filename);

tests/windows/00-test-all.bat

@@ -0,0 +1,29 @@
+@echo off
+setlocal enabledelayedexpansion
+set /a errorno=1
+: _E = set the line number of the first !__! - 1
+set /a _E=8-1
+set "__=set /a _E+=1"
+
+!__! && for /f "delims=. tokens=1,2" %%E in ("%~n0%~x0") do set "TEST_NAME=%%E"
+!__! && for /F %%E in ('forfiles /m "%~nx0" /c "cmd /c echo 0x1b"') do set "_ESC=%%E"
+!__! && set "ORG_DIR=!CD!"
+!__! && :
+!__! && : cd to the directory which contains this batch file
+!__! && :
+!__! && cd /d "%~dp0"                   || goto :ERROR
+!__! && :
+!__! && : Invoke all test scripts
+!__! && :
+!__! && call .\build-with-cmake.bat     || goto :ERROR
+!__! && call .\test-long-path.bat       || goto :ERROR
+!__! && call .\test-trailing-period.bat || goto :ERROR
+!__! && call .\test-trailing-space.bat  || goto :ERROR
+
+echo Status =!_ESC![92m OK !_ESC![0m (%TEST_NAME%) && set /a errorno=0 && goto :END
+
+:ERROR
+echo !_ESC![2K Error = !_E! && echo Status =!_ESC![91m NG !_ESC![0m (%TEST_NAME%)
+
+:END
+cd /d "!ORG_DIR!" && exit /B !errorno!

tests/windows/README.md

@@ -0,0 +1,24 @@
+Windows test scripts
+====================
+
+This directory contains test scripts for Windows.
+
+
+Prerequisites
+-------------
+
+- Visual C++
+- git
+- cmake
+
+
+How to use
+----------
+
+```bat
+cmd.exe
+cd /d "%PUBLIC%"
+git clone https://github.com/Cyan4973/xxHash
+cd xxHash
+.\tests\windows\00-test-all.bat
+```

tests/windows/build-with-cmake.bat

@@ -0,0 +1,37 @@
+@echo off
+setlocal enabledelayedexpansion
+set /a errorno=1
+: _E = set the line number of the first !__! - 1
+set /a _E=8-1
+set "__=set /a _E+=1"
+
+!__! && for /f "delims=. tokens=1,2" %%E in ("%~n0%~x0") do set "TEST_NAME=%%E"
+!__! && for /F %%E in ('forfiles /m "%~nx0" /c "cmd /c echo 0x1b"') do set "_ESC=%%E"
+!__! && set "ORG_DIR=!CD!"
+!__! && :
+!__! && : cd to the directory which contains this batch file
+!__! && :
+!__! && cd /d "%~dp0"                                             || goto :ERROR
+!__! && :
+!__! && : XXHASH_DIR = root level directory of xxhash repository
+!__! && :
+!__! && cd ..\..                                                  || goto :ERROR
+!__! && set "XXHASH_DIR=!CD!"
+!__! && :
+!__! && : Build xxhsum with cmake
+!__! && :
+!__! && rmdir /S /Q   my_build 2>nul
+!__! && mkdir         my_build                                    || goto :ERROR
+!__! && cmake -B      my_build -S build/cmake                     || goto :ERROR
+!__! && cmake --build my_build --config Release                   || goto :ERROR
+!__! && set "XXHSUM_EXE=!XXHASH_DIR!\my_build\Release\xxhsum.exe"
+!__! && echo "!XXHSUM_EXE!" --version
+!__! &&      "!XXHSUM_EXE!" --version                             || goto :ERROR
+
+echo Status =!_ESC![92m OK !_ESC![0m (%TEST_NAME%) && set /a errorno=0 && goto :END
+
+:ERROR
+echo !_ESC![2K Error = !_E! && echo Status =!_ESC![91m NG !_ESC![0m (%TEST_NAME%)
+
+:END
+cd /d "!ORG_DIR!" && exit /B !errorno!

tests/windows/test-long-path.bat

@@ -0,0 +1,56 @@
+@echo off
+setlocal enabledelayedexpansion
+set /a errorno=1
+: _E = set the line number of the first !__! - 1
+set /a _E=8-1
+set "__=set /a _E+=1"
+
+!__! && for /f "delims=. tokens=1,2" %%E in ("%~n0%~x0") do set "TEST_NAME=%%E"
+!__! && for /F %%E in ('forfiles /m "%~nx0" /c "cmd /c echo 0x1b"') do set "_ESC=%%E"
+!__! && set "ORG_DIR=!CD!"
+!__! && :
+!__! && : cd to the directory which contains this batch file
+!__! && :
+!__! && cd /d "%~dp0"                                             || goto :ERROR
+!__! && :
+!__! && : XXHASH_DIR = root level directory of xxhash repository
+!__! && :
+!__! && cd ..\..                                                  || goto :ERROR
+!__! && set "XXHASH_DIR=!CD!"
+!__! && :
+!__! && : cd to %TMP%\xxHash_RANDOM_NAME
+!__! && :
+!__! && cd /d "!TMP!"                                             || goto :ERROR
+!__! && set "TMPNAME=xxHash_%TIME::=-%%RANDOM%"
+!__! && mkdir "!TMPNAME!"                                         || goto :ERROR
+!__! && cd    "!TMPNAME!"                                         || goto :ERROR
+!__! && :
+!__! && : Create long path > 300 chars
+!__! && :
+!__! && set "LONG_PATH=0---------1---------2---------3---------4---------5---------6---------7---------8---------9---------\a---------b---------c---------d---------e---------f---------g---------h---------i---------j---------\k---------l---------m---------n---------o---------p---------q---------r---------s---------t---------"
+!__! && rmdir /S /Q "!LONG_PATH!" 2>nul
+!__! && mkdir "!LONG_PATH!"                                       || goto :ERROR
+!__! && :
+!__! && : Copy the LICENSE file under !LONG_PATH!
+!__! && :
+!__! && copy "!XXHASH_DIR!\LICENSE" "!LONG_PATH!" >nul            || goto :ERROR
+!__! && :
+!__! && : Test xxhsum for !LONG_PATH!\LICENSE
+!__! && :
+!__! && set "XXHSUM_EXE=!XXHASH_DIR!\my_build\Release\xxhsum.exe"
+!__! && "!XXHSUM_EXE!" --version                                  || goto :ERROR
+!__! && "!XXHSUM_EXE!"     "!LONG_PATH!\LICENSE"                  || goto :ERROR
+!__! && "!XXHSUM_EXE!" -H0 "!LONG_PATH!\LICENSE" > test.xxh0      || goto :ERROR
+!__! && "!XXHSUM_EXE!" -H1 "!LONG_PATH!\LICENSE" > test.xxh1      || goto :ERROR
+!__! && "!XXHSUM_EXE!" -H2 "!LONG_PATH!\LICENSE" > test.xxh2      || goto :ERROR
+!__! && "!XXHSUM_EXE!" -H3 "!LONG_PATH!\LICENSE" > test.xxh3      || goto :ERROR
+!__! && type *.xxh*                                               || goto :ERROR
+!__! && "!XXHSUM_EXE!" -c test.xxh0 test.xxh1 test.xxh2 test.xxh3 || goto :ERROR
+
+echo Status =!_ESC![92m OK !_ESC![0m (%TEST_NAME%) && set /a errorno=0 && goto :END
+
+:ERROR
+echo !_ESC![2K Error = !_E! && echo Status =!_ESC![91m NG !_ESC![0m (%TEST_NAME%)
+
+:END
+cd /d "!ORG_DIR!" && exit /B !errorno!

tests/windows/test-trailing-period.bat

@@ -0,0 +1,52 @@
+@echo off
+setlocal enabledelayedexpansion
+set /a errorno=1
+: _E = set the line number of the first !__! - 1
+set /a _E=8-1
+set "__=set /a _E+=1"
+
+!__! && for /f "delims=. tokens=1,2" %%E in ("%~n0%~x0") do set "TEST_NAME=%%E"
+!__! && for /F %%E in ('forfiles /m "%~nx0" /c "cmd /c echo 0x1b"') do set "_ESC=%%E"
+!__! && set "ORG_DIR=!CD!"
+!__! && :
+!__! && : cd to the directory which contains this batch file
+!__! && :
+!__! && cd /d "%~dp0"                                             || goto :ERROR
+!__! && :
+!__! && : XXHASH_DIR = root level directory of xxhash repository
+!__! && :
+!__! && cd ..\..                                                  || goto :ERROR
+!__! && set "XXHASH_DIR=!CD!"
+!__! && :
+!__! && : cd to %TMP%\xxHash_RANDOM_NAME
+!__! && :
+!__! && cd /d "!TMP!"                                             || goto :ERROR
+!__! && set "TMPNAME=xxHash_%TIME::=-%%RANDOM%"
+!__! && mkdir "!TMPNAME!"                                         || goto :ERROR
+!__! && cd    "!TMPNAME!"                                         || goto :ERROR
+!__! && :
+!__! && : Delete test-specimen.
+!__! && :
+!__! && : Copy the LICENSE file as "test-specimen."
+!__! && :
+!__! && type "!XXHASH_DIR!\LICENSE" > "\\?\!CD!\test-specimen."   || goto :ERROR
+!__! && :
+!__! && : Test xxhsum for "test-specimen."
+!__! && :
+!__! && set "XXHSUM_EXE=!XXHASH_DIR!\my_build\Release\xxhsum.exe"
+!__! && "!XXHSUM_EXE!" --version                                  || goto :ERROR
+!__! && "!XXHSUM_EXE!"     "test-specimen."                       || goto :ERROR
+!__! && "!XXHSUM_EXE!" -H0 "test-specimen." > test.xxh0           || goto :ERROR
+!__! && "!XXHSUM_EXE!" -H1 "test-specimen." > test.xxh1           || goto :ERROR
+!__! && "!XXHSUM_EXE!" -H2 "test-specimen." > test.xxh2           || goto :ERROR
+!__! && "!XXHSUM_EXE!" -H3 "test-specimen." > test.xxh3           || goto :ERROR
+!__! && type *.xxh*                                               || goto :ERROR
+!__! && "!XXHSUM_EXE!" -c test.xxh0 test.xxh1 test.xxh2 test.xxh3 || goto :ERROR
+
+echo Status =!_ESC![92m OK !_ESC![0m (%TEST_NAME%) && set /a errorno=0 && goto :END
+
+:ERROR
+echo !_ESC![2K Error = !_E! && echo Status =!_ESC![91m NG !_ESC![0m (%TEST_NAME%)
+
+:END
+cd /d "!ORG_DIR!" && exit /B !errorno!