includes_normalizes: Full fix for long path support.

Use GetFullPathNameW() instead of GetFullPathNameA() to ensure that
normalization works for all long paths on Windows. Adjust unit
tests accordingly.

Fixes #2442

Author: lygstate

src/includes_normalize-win32.cc

@@ -14,28 +14,60 @@
 
 #include "includes_normalize.h"
 
-#include "string_piece.h"
-#include "string_piece_util.h"
-#include "util.h"
+#include <string.h>
+
+#include <windows.h>
 
 #include <algorithm>
 #include <iterator>
 #include <sstream>
 
-#include <windows.h>
+#include "string_piece.h"
+#include "string_piece_util.h"
+#include "util.h"
 
 namespace {
 
+constexpr const std::size_t MAX_PATH_LONG = 32767;
+
 bool InternalGetFullPathName(const StringPiece& file_name, char* buffer,
                              size_t buffer_length, std::string* err) {
-  DWORD result_size = GetFullPathNameA(file_name.AsString().c_str(),
-                                       buffer_length, buffer, NULL);
-  if (result_size == 0) {
-    *err = "GetFullPathNameA(" + file_name.AsString() + "): " +
-        GetLastErrorString();
+  // IMPORTANT: Using GetFullPathNameA() with a long paths will fail with
+  // "The filename or extension is too long" even if long path supported is
+  // enabled. GetFullPathNameW() must be used for this function to work!
+  buffer[0] = '\0';
+  // Convert to wide filename first.
+  std::wstring wide_filename;
+  if (!ConvertUTF8ToWin32Unicode(file_name, &wide_filename, err))
     return false;
-  } else if (result_size > buffer_length) {
-    *err = "path too long";
+
+  // Call GetFullPathNameW()
+  DWORD wide_full_size = GetFullPathNameW(wide_filename.c_str(), 0, NULL, NULL);
+  if (wide_full_size == 0) {
+    *err = "GetFullPathNameW(" +
+           std::string(wide_filename.begin(), wide_filename.end()) +
+           "): " + GetLastErrorString();
+    return false;
+  }
+
+  // NOTE: wide_full_size includes the null-terminating character.
+  std::wstring wide_path;
+  wide_path.resize(static_cast<size_t>(wide_full_size - 1));
+  DWORD wide_full_size2 =
+      GetFullPathNameW(wide_filename.c_str(), wide_full_size,
+                       const_cast<wchar_t*>(wide_path.data()), NULL);
+  if (wide_full_size2 == 0) {
+    *err = "GetFullPathNameW(" + file_name.AsString() + "): " + GetLastErrorString();
+    return false;
+  }
+
+  // Convert wide_path to Unicode.
+  int utf8_size =
+      WideCharToMultiByte(CP_UTF8, 0, wide_path.c_str(), wide_path.size() + 1,
+                          buffer, buffer_length, NULL, NULL);
+  if (utf8_size <= 0 || utf8_size >= buffer_length) {
+    *err = "WideCharToMultiByte(" + std::string(wide_path.begin(), wide_path.end()) +
+           "): " + GetLastErrorString();
     return false;
   }
   return true;
@@ -74,8 +106,8 @@ bool SameDrive(StringPiece a, StringPiece b, std::string* err) {
     return true;
   }
 
-  char a_absolute[_MAX_PATH];
-  char b_absolute[_MAX_PATH];
+  char a_absolute[MAX_PATH_LONG];
+  char b_absolute[MAX_PATH_LONG];
   if (!InternalGetFullPathName(a, a_absolute, sizeof(a_absolute), err)) {
     return false;
   }
@@ -144,7 +176,7 @@ std::string IncludesNormalize::AbsPath(StringPiece s, std::string* err) {
     return result;
   }
 
-  char result[_MAX_PATH];
+  char result[MAX_PATH_LONG];
   if (!InternalGetFullPathName(s, result, sizeof(result), err)) {
     return "";
   }
@@ -183,9 +215,9 @@ std::string IncludesNormalize::Relativize(
 
 bool IncludesNormalize::Normalize(const std::string& input, std::string* result,
                                   std::string* err) const {
-  char copy[_MAX_PATH + 1];
+  char copy[MAX_PATH_LONG + 1];
   size_t len = input.size();
-  if (len > _MAX_PATH) {
+  if (len > MAX_PATH_LONG) {
     *err = "path too long";
     return false;
   }

src/includes_normalize_test.cc

@@ -98,18 +98,20 @@ TEST(IncludesNormalize, DifferentDrive) {
 }
 
 TEST(IncludesNormalize, LongInvalidPath) {
+  // A long an invalid path that is larger than _MAX_PATH will still be
+  // canonicalized without crashing.
   const char kLongInputString[] =
       "C:\\Program Files (x86)\\Microsoft Visual Studio "
       "12.0\\VC\\INCLUDEwarning #31001: The dll for reading and writing the "
       "pdb (for example, mspdb110.dll) could not be found on your path. This "
       "is usually a configuration error. Compilation will continue using /Z7 "
       "instead of /Zi, but expect a similar error when you link your program.";
-  // Too long, won't be canonicalized. Ensure doesn't crash.
+  ASSERT_GT(sizeof(kLongInputString) - 1u, _MAX_PATH);
   std::string result, err;
   IncludesNormalize normalizer(".");
-  EXPECT_FALSE(
-      normalizer.Normalize(kLongInputString, &result, &err));
-  EXPECT_EQ("path too long", err);
+  EXPECT_TRUE(normalizer.Normalize(kLongInputString, &result, &err));
+  EXPECT_FALSE(result.empty());
+  EXPECT_TRUE(err.empty()) << err;
 
 
   // Construct max size path having cwd prefix.
@@ -150,19 +152,39 @@ TEST(IncludesNormalize, ShortRelativeButTooLongAbsolutePath) {
   EXPECT_TRUE(normalizer.Normalize("a", &result, &err));
   EXPECT_EQ("", err);
 
+  constexpr const int MAX_PATH_LONG = 32767;
+
   // Construct max size path having cwd prefix.
   // kExactlyMaxPath = "aaaa\\aaaa...aaaa\0";
-  char kExactlyMaxPath[_MAX_PATH + 1];
-  for (int i = 0; i < _MAX_PATH; ++i) {
-    if (i < _MAX_PATH - 1 && i % 10 == 4)
+  char kExactlyMaxPath[MAX_PATH_LONG + 1];
+  for (int i = 0; i < MAX_PATH_LONG; ++i) {
+    if (i < MAX_PATH_LONG - 1 && i % 10 == 4)
       kExactlyMaxPath[i] = '\\';
     else
       kExactlyMaxPath[i] = 'a';
   }
-  kExactlyMaxPath[_MAX_PATH] = '\0';
-  EXPECT_EQ(strlen(kExactlyMaxPath), static_cast<size_t>(_MAX_PATH));
+  kExactlyMaxPath[MAX_PATH_LONG] = '\0';
+  EXPECT_EQ(strlen(kExactlyMaxPath), static_cast<size_t>(MAX_PATH_LONG));
 
-  // Make sure a path that's exactly _MAX_PATH long fails with a proper error.
+  // Make sure a path that's exactly MAX_PATH_LONG long fails with a proper error.
   EXPECT_FALSE(normalizer.Normalize(kExactlyMaxPath, &result, &err));
   EXPECT_TRUE(err.find("GetFullPathName") != std::string::npos);
+
+  err = "";
+  // Make sure a path that's exactly _MAX_PATH long fails does not fail.
+  // Construct max size path having cwd prefix.
+  // exactly_max_path = "aaaa\\aaaa...aaaa\0";
+  std::string exactly_max_path;
+  for (int i = 0; i < _MAX_PATH; ++i) {
+    if (i < _MAX_PATH - 1 && i % 10 == 4)
+      exactly_max_path.push_back('\\');
+    else
+      exactly_max_path.push_back('a');
+  }
+  EXPECT_EQ(exactly_max_path.size(), static_cast<size_t>(_MAX_PATH));
+  EXPECT_TRUE(normalizer.Normalize(exactly_max_path, &result, &err)) << err;
+
+  // Make sue a path of _MAX_PATH + 1 characters also works.
+  std::string more_than_max_path = exactly_max_path + "\\a";
+  EXPECT_TRUE(normalizer.Normalize(more_than_max_path, &result, &err)) << err;
 }

src/util.cc

@@ -561,7 +561,30 @@ void Win32Fatal(const char* function, const char* hint) {
     Fatal("%s: %s", function, GetLastErrorString().c_str());
   }
 }
-#endif
+
+bool ConvertUTF8ToWin32Unicode(const StringPiece& input, std::wstring* output,
+                               std::string* err) {
+  output->clear();
+  if (input.empty())
+    return true;
+
+  int int_size = static_cast<int>(input.size());
+  if (static_cast<size_t>(int_size) != input.size()) {
+    *err = "Input string length > INT_MAX";
+    return false;
+  }
+  int wide_size =
+      MultiByteToWideChar(CP_UTF8, 0, input.begin(), int_size, nullptr, 0);
+  if (wide_size <= 0) {
+    *err = "MultiByteToWideChar(" + input.AsString() + "): " + GetLastErrorString();
+    return false;
+  }
+  output->resize(static_cast<size_t>(wide_size));
+  MultiByteToWideChar(CP_UTF8, 0, input.begin(), int_size,
+                      const_cast<wchar_t*>(output->data()), wide_size);
+  return true;
+}
+#endif  // _WIN32
 
 bool islatinalpha(int c) {
   // isalpha() is locale-dependent.

src/util.h

@@ -26,6 +26,8 @@
 #include <string>
 #include <vector>
 
+#include "string_piece.h"
+
 #if !defined(__has_cpp_attribute)
 #  define __has_cpp_attribute(x)  0
 #endif
@@ -133,6 +135,12 @@ std::string GetLastErrorString();
 /// Calls Fatal() with a function name and GetLastErrorString.
 NORETURN void Win32Fatal(const char* function, const char* hint = NULL);
 
+/// Convert UTF-8 string to Win32 Unicode.
+/// On success, set |*output| then return true.
+/// On Failure, clear |*output|, set |*err| then return false.
+bool ConvertUTF8ToWin32Unicode(const StringPiece& input, std::wstring* output,
+                               std::string* err);
+
 /// Naive implementation of C++ 20 std::bit_cast(), used to fix Clang and GCC
 /// [-Wcast-function-type] warning on casting result of GetProcAddress().
 template <class To, class From>

src/util_test.cc

@@ -502,3 +502,15 @@ TEST(StripAnsiEscapeCodes, StripColors) {
   EXPECT_EQ("affixmgr.cxx:286:15: warning: using the result... [-Wparentheses]",
             stripped);
 }
+
+#ifdef _WIN32
+TEST(ConvertUTF8ToWin32Unicode, Test) {
+  std::string err;
+  std::wstring output;
+  EXPECT_TRUE(ConvertUTF8ToWin32Unicode(StringPiece("B\xC3\xA9"
+                                                    "b\xC3\xA9"),
+                                        &output, &err));
+  EXPECT_TRUE(err.empty()) << err;
+  EXPECT_EQ(output, std::wstring(L"B\u00E9b\u00E9"));
+}
+#endif  // _WIN32