Merge pull request #490 from dmbryson/swift-5.1-branch

Merge 'master' into 'swift-5.1-branch'

Author: dmbryson

CMakeLists.txt

@@ -12,6 +12,8 @@ else()
   set(cmake_3_2_USES_TERMINAL USES_TERMINAL)
 endif()
 
+option(BUILD_SHARED_LIBS "Build Shared Libraries" ON)
+
 # Configure the default set of bindings to build.
 if(NOT DEFINED LLBUILD_SUPPORT_BINDINGS)
   set(LLBUILD_SUPPORT_BINDINGS "")

Xcode/Configs/Version.xcconfig

@@ -10,4 +10,4 @@
 //
 //===----------------------------------------------------------------------===//
 
-SWIFT_VERSION = 4.2
+SWIFT_VERSION = 5.0

include/llbuild/Basic/PlatformUtility.h

@@ -22,6 +22,9 @@
 #include <cstdint>
 #include <cstdio>
 #include <string>
+#if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__))
+#include <unistd.h>
+#endif
 #if !defined(_WIN32)
 #include <sys/resource.h>
 #endif
@@ -87,6 +90,37 @@ template <> struct FileDescriptorTraits<int> {
     return read(hFile, destinationBuffer, maxCharCount);
   }
 };
+
+enum class OSStyle {
+  Windows,
+  POSIX,
+
+#if defined(_WIN32)
+  Default = Windows,
+#elif defined(_POSIX_VERSION)
+  Default = POSIX,
+#endif
+};
+
+template <OSStyle = OSStyle::Default>
+struct ModuleTraits;
+
+#if defined(_WIN32)
+template <>
+struct ModuleTraits<OSStyle::Windows> {
+  using Handle = HMODULE;
+};
+#endif
+
+template <>
+struct ModuleTraits<OSStyle::POSIX> {
+  using Handle = void *;
+};
+
+ModuleTraits<>::Handle OpenLibrary(const char *);
+void *GetSymbolByname(ModuleTraits<>::Handle, const char *);
+void CloseLibrary(ModuleTraits<>::Handle);
+
 }
 }
 }

include/llbuild/Basic/ShellUtility.h

@@ -15,27 +15,35 @@
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/raw_ostream.h"
-
-using namespace llvm;
+#include <vector>
 
 namespace llbuild {
 namespace basic {
 
+#if defined(_WIN32)
+/// Formats a command line using the Windows command line escaping rules a la a
+/// reverse CommandLineToArgVW
+///
+/// \param args The arugments to escape
+///
+std::string formatWindowsCommandString(std::vector<std::string> args);
+#endif
+
 /// Appends a shell escaped string to an output stream.
 /// For e.g. hello -> hello, hello$world -> 'hello$world', input A -> 'input A'
 ///
 /// \param os Reference of the output stream to append to.
 ///
 /// \param string The string to be escaped and appended.
-/// 
-void appendShellEscapedString(llvm::raw_ostream& os, StringRef string);
+///
+void appendShellEscapedString(llvm::raw_ostream& os, llvm::StringRef string);
 
 /// Creates and returns a shell escaped string of the input.
 ///
 /// \param string The string to be escaped.
 ///
 /// \returns escaped string.
-std::string shellEscaped(StringRef string);
+std::string shellEscaped(llvm::StringRef string);
 
 }
 }

include/llbuild/Core/BuildDB.h

@@ -42,12 +42,12 @@ class BuildDBDelegate {
   /// implementations may (and do) cache these values for future use.
   ///
   /// \param key [out] The key whose unique ID is being returned.
-  virtual KeyID getKeyID(const KeyType& key) = 0;
+  virtual const KeyID getKeyID(const KeyType& key) = 0;
 
   /// Get the key corresponding to a key ID.
   ///
   /// This method must be thread safe, and must not fail.
-  virtual KeyType getKeyForID(KeyID key) = 0;
+  virtual KeyType getKeyForID(const KeyID key) = 0;
 };
 
 

lib/Basic/CMakeLists.txt

@@ -1,4 +1,4 @@
-add_llbuild_library(llbuildBasic
+add_llbuild_library(llbuildBasic STATIC
   ExecutionQueue.cpp
   FileInfo.cpp
   FileSystem.cpp
@@ -12,6 +12,8 @@ add_llbuild_library(llbuildBasic
   ShellUtility.cpp
   )
 
+target_link_libraries(llbuildBasic PRIVATE llvmSupport)
+
 if(${CMAKE_SYSTEM_NAME} MATCHES ".*BSD" OR ${CMAKE_SYSTEM_NAME} MATCHES "Linux")
   target_link_libraries(llbuildBasic pthread)
 endif()

lib/Basic/PlatformUtility.cpp

@@ -22,6 +22,9 @@
 #else
 #include <fnmatch.h>
 #include <unistd.h>
+#if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__))
+#include <dlfcn.h>
+#endif
 #endif
 #include <stdio.h>
 
@@ -336,3 +339,37 @@ std::string sys::getPathSeparators() {
   return "/";
 #endif
 }
+
+sys::ModuleTraits<>::Handle sys::OpenLibrary(const char *path) {
+#if defined(_WIN32)
+  int cchLength =
+      MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, path, strlen(path),
+                          nullptr, 0);
+  std::u16string buffer(cchLength + 1, 0);
+  MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, path, strlen(path),
+                      const_cast<LPWSTR>(reinterpret_cast<LPCWSTR>(buffer.data())),
+                      buffer.size());
+
+  return LoadLibraryW(reinterpret_cast<LPCWSTR>(buffer.data()));
+#else
+  return dlopen(path, RTLD_LAZY);
+#endif
+}
+
+void *sys::GetSymbolByname(sys::ModuleTraits<>::Handle handle,
+                           const char *name) {
+#if defined(_WIN32)
+  return GetProcAddress(handle, name);
+#else
+  return dlsym(handle, name);
+#endif
+}
+
+void sys::CloseLibrary(sys::ModuleTraits<>::Handle handle) {
+#if defined(_WIN32)
+  FreeLibrary(handle);
+#else
+  dlclose(handle);
+#endif
+}
+

lib/Basic/ShellUtility.cpp

@@ -13,11 +13,82 @@
 #include "llbuild/Basic/ShellUtility.h"
 #include "llvm/ADT/SmallString.h"
 
+using namespace llvm;
 namespace llbuild {
 namespace basic {
 
-void appendShellEscapedString(llvm::raw_ostream& os, StringRef string) {
 
+#if defined(_WIN32)
+std::string formatWindowsCommandArg(StringRef string) {
+  // Windows escaping, adapted from Daniel Colascione's "Everyone quotes
+  // command line arguments the wrong way" - Microsoft Developer Blog
+  const std::string needsQuote = " \t\n\v\"";
+  if (string.find_first_of(needsQuote) == std::string::npos) {
+    return string;
+  }
+
+  // To escape the command line, we surround the argument with quotes. However
+  // the complication comes due to how the Windows command line parser treats
+  // backslashes (\) and quotes (")
+  //
+  // - \ is normally treated as a literal backslash
+  //     - e.g. foo\bar\baz => foo\bar\baz
+  // - However, the sequence \" is treated as a literal "
+  //     - e.g. foo\"bar => foo"bar
+  //
+  // But then what if we are given a path that ends with a \? Surrounding
+  // foo\bar\ with " would be "foo\bar\" which would be an unterminated string
+  // since it ends on a literal quote. To allow this case the parser treats:
+  //
+  // - \\" as \ followed by the " metachar
+  // - \\\" as \ followed by a literal "
+  // - In general:
+  //     - 2n \ followed by " => n \ followed by the " metachar
+  //     - 2n+1 \ followed by " => n \ followed by a literal "
+  std::string escaped = "\"";
+  for (auto i = std::begin(string); i != std::end(string); ++i) {
+    int numBackslashes = 0;
+    while (i != string.end() && *i == '\\') {
+      ++i;
+      ++numBackslashes;
+    }
+
+    if (i == string.end()) {
+      // String ends with a backslash e.g. foo\bar\, escape all the backslashes
+      // then add the metachar " below
+      escaped.append(numBackslashes * 2, '\\');
+      break;
+    } else if (*i == '"') {
+      // This is  a string of \ followed by a " e.g. foo\"bar. Escape the
+      // backslashes and the quote
+      escaped.append(numBackslashes * 2 + 1, '\\');
+      escaped.push_back(*i);
+    } else {
+      // These are just literal backslashes
+      escaped.append(numBackslashes, '\\');
+      escaped.push_back(*i);
+    }
+  }
+  escaped.push_back('"');
+
+  return escaped;
+}
+
+std::string formatWindowsCommandString(std::vector<std::string> args) {
+    std::string commandLine;
+    for (auto& arg : args)
+        commandLine += formatWindowsCommandArg(arg) + " ";
+    if (commandLine.size())
+      commandLine.pop_back();
+    return commandLine;
+}
+#endif
+
+void appendShellEscapedString(llvm::raw_ostream& os, StringRef string) {
+#if defined(_WIN32)
+  os << formatWindowsCommandArg(string);
+  return;
+#else
   static const std::string whitelist = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890-_/:@#%+=.,";
   auto pos = string.find_first_not_of(whitelist);
 
@@ -27,32 +98,27 @@ void appendShellEscapedString(llvm::raw_ostream& os, StringRef string) {
     return;
   }
 
-#if defined(_WIN32)
-  std::string escQuote = "\"";
-#else
   std::string escQuote = "'";
-#endif
   // We only need to escape the single quote, if it isn't present we can
   // escape using single quotes.
-  auto singleQuotePos = string.find_first_of(escQuote, pos);
+  auto singleQuotePos = string.find_first_of("'" , pos);
   if (singleQuotePos == std::string::npos) {
-    os << escQuote;
-    os << string;
-    os << escQuote;
+    os << "'" << string << "'";
     return;
   }
 
   // Otherwise iterate and escape all the single quotes.
-  os << escQuote;
+  os << "'";
   os << string.slice(0, singleQuotePos);
   for (auto idx = singleQuotePos; idx < string.size(); idx++) {
-    if (string[idx] == escQuote[0]) {
-      os << escQuote << "\\" << escQuote << escQuote;
+    if (string[idx] == '\'') {
+      os << "'\\''";
     } else {
       os << string[idx];
     }
   }
-  os << escQuote;
+  os << "'";
+#endif
 }
 
 std::string shellEscaped(StringRef string) {

lib/Basic/Subprocess.cpp

@@ -14,6 +14,7 @@
 
 #include "llbuild/Basic/CrossPlatformCompatibility.h"
 #include "llbuild/Basic/PlatformUtility.h"
+#include "llbuild/Basic/ShellUtility.h"
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
@@ -72,6 +73,7 @@ int pthread_fchdir_np(int fd)
 
 #ifndef HAVE_POSIX_SPAWN_CHDIR
 #if defined(__sun) || \
+  (defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && __MAC_OS_X_VERSION_MIN_REQUIRED >= 101500) || \
   __GLIBC_PREREQ(2, 29)
 #define HAVE_POSIX_SPAWN_CHDIR 1
 #else
@@ -85,6 +87,12 @@ static int posix_spawn_file_actions_addchdir(posix_spawn_file_actions_t * __rest
 #if HAVE_POSIX_SPAWN_CHDIR
   return ::posix_spawn_file_actions_addchdir_np(file_actions, path);
 #else
+#if defined(__APPLE__) && __MAC_OS_X_VERSION_MAX_ALLOWED >= 101500
+  if (__builtin_available(macOS 10.15, *)) {
+    return ::posix_spawn_file_actions_addchdir_np(file_actions, path);
+  }
+#endif
+
   // Any other POSIX platform returns ENOSYS (Function not implemented),
   // to simplify the fallback logic around the call site.
   return ENOSYS;
@@ -409,7 +417,7 @@ void llbuild::basic::spawnProcess(
   // Form the complete C string command line.
   std::vector<std::string> argsStorage(commandLine.begin(), commandLine.end());
 #if defined(_WIN32)
-  std::string args = llvm::sys::flattenWindowsCommandLine(commandLine);
+  std::string args = llbuild::basic::formatWindowsCommandString(argsStorage);
 
   // Convert the command line string to utf16
   llvm::SmallVector<llvm::UTF16, 20> u16Executable;

lib/BuildSystem/BuildSystem.cpp

@@ -62,6 +62,7 @@
 #include <unistd.h>
 #endif
 
+using namespace llvm;
 using namespace llbuild;
 using namespace llbuild::basic;
 using namespace llbuild::core;
@@ -1873,7 +1874,13 @@ llvm::Optional<BuildValue> BuildSystemImpl::build(BuildKey key) {
   // queue to have notified the engine of the last task completion, but still
   // have other work to perform (e.g., informing the client of command
   // completion).
-  executionQueue.reset();
+  //
+  // This must hold the lock to prevent data racing on the executionQueue
+  // pointer (as can happen with cancellation) - rdar://problem/50993380
+  {
+    std::lock_guard<std::mutex> guard(executionQueueMutex);
+    executionQueue.reset();
+  }
 
   // Clear out the shell handlers, as we do not want to hold on to them across
   // multiple builds.
@@ -3234,11 +3241,11 @@ class ArchiveShellCommand : public ExternalCommand {
     llvm::raw_svector_ostream stream(result);
     bool first = true;
     for (const auto& arg: getArgs()) {
-      stream << arg;
       if (!first) {
         stream << " ";
-        first = false;
       }
+      first = false;
+      stream << arg;
     }
   }