Merge pull request #4554 from swiftwasm/main

[pull] swiftwasm from main

Author: kateinoigakukun

benchmark/scripts/compare_perf_tests.py

@@ -229,8 +229,8 @@ class PerformanceTestResult(object):
     statistics for normal distribution (MEAN, SD):
         #,TEST,SAMPLES,MIN(μs),MAX(μs),MEAN(μs),SD(μs),MEDIAN(μs),MAX_RSS(B)
     And new quantiles format with variable number of columns:
-        #,TEST,SAMPLES,MIN(μs),MEDIAN(μs),MAX(μs)
-        #,TEST,SAMPLES,MIN(μs),Q1(μs),Q2(μs),Q3(μs),MAX(μs),MAX_RSS(B)
+        #,TEST,SAMPLES,QMIN(μs),MEDIAN(μs),MAX(μs)
+        #,TEST,SAMPLES,QMIN(μs),Q1(μs),Q2(μs),Q3(μs),MAX(μs),MAX_RSS(B)
     The number of columns between MIN and MAX depends on the test driver's
     `--quantile`parameter. In both cases, the last column, MAX_RSS is optional.
     """
@@ -244,9 +244,10 @@ def __init__(self, csv_row, quantiles=False, memory=False, delta=False, meta=Fal
         self.name = csv_row[1]  # Name of the performance test
         self.num_samples = int(csv_row[2])  # Number of measurements taken
 
+        mem_index = (-1 if memory else 0) + (-3 if meta else 0)
         if quantiles:  # Variable number of columns representing quantiles
-            mem_index = (-1 if memory else 0) + (-3 if meta else 0)
             runtimes = csv_row[3:mem_index] if memory or meta else csv_row[3:]
+            last_runtime_index = mem_index - 1
             if delta:
                 runtimes = [int(x) if x else 0 for x in runtimes]
                 runtimes = functools.reduce(
@@ -277,20 +278,21 @@ def __init__(self, csv_row, quantiles=False, memory=False, delta=False, meta=Fal
                 sams.mean,
                 sams.sd,
             )
-            self.max_rss = (  # Maximum Resident Set Size (B)
-                int(csv_row[mem_index]) if memory else None
-            )
         else:  # Legacy format with statistics for normal distribution.
             self.min = int(csv_row[3])  # Minimum runtime (μs)
             self.max = int(csv_row[4])  # Maximum runtime (μs)
             self.mean = float(csv_row[5])  # Mean (average) runtime (μs)
             self.sd = float(csv_row[6])  # Standard Deviation (μs)
             self.median = int(csv_row[7])  # Median runtime (μs)
-            self.max_rss = (  # Maximum Resident Set Size (B)
-                int(csv_row[8]) if len(csv_row) > 8 else None
-            )
+            last_runtime_index = 7
             self.samples = None
 
+        self.max_rss = (  # Maximum Resident Set Size (B)
+            int(csv_row[mem_index]) if (
+                memory and len(csv_row) > (last_runtime_index + 1)
+            ) else None
+        )
+
         # Optional measurement metadata. The number of:
         # memory pages used, involuntary context switches and voluntary yields
         self.mem_pages, self.involuntary_cs, self.yield_count = (
@@ -427,7 +429,7 @@ def _store_memory_stats(self, max_rss, mem_pages):
         self.mem_pages = int(mem_pages)
 
     def _configure_format(self, header):
-        self.quantiles = "MEAN" not in header
+        self.quantiles = "QMIN" in header
         self.memory = "MAX_RSS" in header
         self.meta = "PAGES" in header
         self.delta = "𝚫" in header
@@ -453,7 +455,7 @@ def _configure_format(self, header):
                 Yield(len(self.samples), int(since_last_yield))
             )
         ),
-        re.compile(r"( *#[, \t]+TEST[, \t]+SAMPLES[, \t]+MIN.*)"): _configure_format,
+        re.compile(r"( *#[, \t]+TEST[, \t]+SAMPLES[, \t].*)"): _configure_format,
         # Environmental statistics: memory usage and context switches
         re.compile(
             r"\s+MAX_RSS \d+ - \d+ = (\d+) \((\d+) pages\)"

benchmark/scripts/test_compare_perf_tests.py

@@ -205,7 +205,7 @@ def test_init(self):
         self.assertEqual(r.samples, None)
 
         log_line = "1,AngryPhonebook,1,12045,12045,12045,0,12045,10510336"
-        r = PerformanceTestResult(log_line.split(","))
+        r = PerformanceTestResult(log_line.split(","), memory=True)
         self.assertEqual(r.max_rss, 10510336)
 
     def test_init_quantiles(self):
@@ -379,7 +379,11 @@ def test_merge(self):
         )[
             1:
         ]
-        results = list(map(PerformanceTestResult, [line.split(",") for line in tests]))
+
+        def makeResult(csv_row):
+            return PerformanceTestResult(csv_row, memory=True)
+
+        results = list(map(makeResult, [line.split(",") for line in tests]))
         results[2].setup = 9
         results[3].setup = 7
 
@@ -489,11 +493,14 @@ class OldAndNewLog(unittest.TestCase):
 3,Array2D,20,335831,400221,346622,0,346622
 1,AngryPhonebook,20,10458,12714,11000,0,11000"""
 
+    def makeResult(csv_row):
+        return PerformanceTestResult(csv_row, memory=True)
+
     old_results = dict(
         [
             (r.name, r)
             for r in map(
-                PerformanceTestResult,
+                makeResult,
                 [line.split(",") for line in old_log_content.splitlines()],
             )
         ]
@@ -503,7 +510,7 @@ class OldAndNewLog(unittest.TestCase):
         [
             (r.name, r)
             for r in map(
-                PerformanceTestResult,
+                makeResult,
                 [line.split(",") for line in new_log_content.splitlines()],
             )
         ]
@@ -557,14 +564,14 @@ def test_parse_results_formatted_text(self):
     def test_parse_quantiles(self):
         """Gathers samples from reported quantiles. Handles optional memory."""
         r = LogParser.results_from_string(
-            """#,TEST,SAMPLES,MIN(μs),MEDIAN(μs),MAX(μs)
+            """#,TEST,SAMPLES,QMIN(μs),MEDIAN(μs),MAX(μs)
 1,Ackermann,3,54383,54512,54601"""
         )["Ackermann"]
         self.assertEqual(
             [s.runtime for s in r.samples.all_samples], [54383, 54512, 54601]
         )
         r = LogParser.results_from_string(
-            """#,TEST,SAMPLES,MIN(μs),MEDIAN(μs),MAX(μs),MAX_RSS(B)
+            """#,TEST,SAMPLES,QMIN(μs),MEDIAN(μs),MAX(μs),MAX_RSS(B)
 1,Ackermann,3,54529,54760,55807,266240"""
         )["Ackermann"]
         self.assertEqual(
@@ -574,21 +581,21 @@ def test_parse_quantiles(self):
 
     def test_parse_delta_quantiles(self):
         r = LogParser.results_from_string(  # 2-quantile aka. median
-            "#,TEST,SAMPLES,MIN(μs),𝚫MEDIAN,𝚫MAX\n0,B,1,101,,"
+            "#,TEST,SAMPLES,QMIN(μs),𝚫MEDIAN,𝚫MAX\n0,B,1,101,,"
         )["B"]
         self.assertEqual(
             (r.num_samples, r.min, r.median, r.max, r.samples.count),
             (1, 101, 101, 101, 1),
         )
         r = LogParser.results_from_string(
-            "#,TEST,SAMPLES,MIN(μs),𝚫MEDIAN,𝚫MAX\n0,B,2,101,,1"
+            "#,TEST,SAMPLES,QMIN(μs),𝚫MEDIAN,𝚫MAX\n0,B,2,101,,1"
         )["B"]
         self.assertEqual(
             (r.num_samples, r.min, r.median, r.max, r.samples.count),
             (2, 101, 101, 102, 2),
         )
         r = LogParser.results_from_string(  # 20-quantiles aka. ventiles
-            "#,TEST,SAMPLES,MIN(μs),𝚫V1,𝚫V2,𝚫V3,𝚫V4,𝚫V5,𝚫V6,𝚫V7,𝚫V8,"
+            "#,TEST,SAMPLES,QMIN(μs),𝚫V1,𝚫V2,𝚫V3,𝚫V4,𝚫V5,𝚫V6,𝚫V7,𝚫V8,"
             + "𝚫V9,𝚫VA,𝚫VB,𝚫VC,𝚫VD,𝚫VE,𝚫VF,𝚫VG,𝚫VH,𝚫VI,𝚫VJ,𝚫MAX\n"
             + "202,DropWhileArray,200,214,,,,,,,,,,,,1,,,,,,2,16,464"
         )["DropWhileArray"]
@@ -617,13 +624,13 @@ def test_parse_meta(self):
             (3, 9, 50, 15, 36864),
         )
         r = LogParser.results_from_string(
-            "#,TEST,SAMPLES,MIN(μs),MAX(μs),PAGES,ICS,YIELD\n" + "0,B,1,4,4,8,31,15"
+            "#,TEST,SAMPLES,QMIN(μs),MAX(μs),PAGES,ICS,YIELD\n" + "0,B,1,4,4,8,31,15"
         )["B"]
         self.assertEqual(
             (r.min, r.mem_pages, r.involuntary_cs, r.yield_count), (4, 8, 31, 15)
         )
         r = LogParser.results_from_string(
-            "#,TEST,SAMPLES,MIN(μs),MAX(μs),MAX_RSS(B),PAGES,ICS,YIELD\n"
+            "#,TEST,SAMPLES,QMIN(μs),MAX(μs),MAX_RSS(B),PAGES,ICS,YIELD\n"
             + "0,B,1,5,5,32768,8,28,15"
         )["B"]
         self.assertEqual(
@@ -831,7 +838,8 @@ def test_values(self):
         self.assertEqual(
             ReportFormatter.values(
                 PerformanceTestResult(
-                    "1,AngryPhonebook,1,12045,12045,12045,0,12045,10510336".split(",")
+                    "1,AngryPhonebook,1,12045,12045,12045,0,12045,10510336".split(","),
+                    memory=True
                 )
             ),
             ("AngryPhonebook", "12045", "12045", "12045", "10510336"),

benchmark/utils/DriverUtils.swift

@@ -634,7 +634,8 @@ final class TestRunner {
       let index: (Int) -> String =
         { q == 2 ? "" : q <= 20 ?  base20[$0] : String($0) }
       let tail = (1..<q).map { prefix + index($0) } + ["MAX"]
-      return [withUnit("MIN")] + tail.map(c.delta ? withDelta : withUnit)
+      // QMIN identifies the quantile format, distinct from formats using "MIN"
+      return [withUnit("QMIN")] + tail.map(c.delta ? withDelta : withUnit)
     }
     return (
       ["#", "TEST", "SAMPLES"] +

cmake/modules/SwiftCXXUtils.cmake

@@ -2,6 +2,5 @@
 set(SWIFT_LIBSTDCXX_PLATFORMS
     "LINUX"
     "FREEBSD"
-    "OPENBSD"
     "CYGWIN"
     "HAIKU")

include/swift/AST/Types.h

@@ -4460,6 +4460,32 @@ class SILFunctionType final
   SILType getDirectFormalResultsType(SILModule &M,
                                      TypeExpansionContext expansion);
 
+  unsigned getNumIndirectFormalYields() const {
+    return isCoroutine() ? NumAnyIndirectFormalResults : 0;
+  }
+  /// Does this function have any formally indirect yields?
+  bool hasIndirectFormalYields() const {
+    return getNumIndirectFormalYields() != 0;
+  }
+  unsigned getNumDirectFormalYields() const {
+    return isCoroutine() ? NumAnyResults - NumAnyIndirectFormalResults : 0;
+  }
+
+  struct IndirectFormalYieldFilter {
+    bool operator()(SILYieldInfo yield) const {
+      return !yield.isFormalIndirect();
+    }
+  };
+
+  using IndirectFormalYieldIter =
+      llvm::filter_iterator<const SILYieldInfo *, IndirectFormalYieldFilter>;
+  using IndirectFormalYieldRange = iterator_range<IndirectFormalYieldIter>;
+
+  /// A range of SILYieldInfo for all formally Indirect Yields.
+  IndirectFormalYieldRange getIndirectFormalYields() const {
+    return llvm::make_filter_range(getYields(), IndirectFormalYieldFilter());
+  }
+
   /// Get a single non-address SILType for all SIL results regardless of whether
   /// they are formally indirect. The actual SIL result type of an apply
   /// instruction that calls this function depends on the current SIL stage and

include/swift/IDE/Utils.h

@@ -89,6 +89,7 @@ bool initCompilerInvocation(
     FrontendOptions::ActionType Action, DiagnosticEngine &Diags,
     StringRef UnresolvedPrimaryFile,
     llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FileSystem,
+    const std::string &swiftExecutablePath,
     const std::string &runtimeResourcePath,
     const std::string &diagnosticDocumentationPath, time_t sessionTimestamp,
     std::string &Error);

include/swift/SIL/SILValue.h

@@ -255,6 +255,9 @@ struct ValueOwnershipKind {
   explicit ValueOwnershipKind(unsigned newValue) : value(innerty(newValue)) {}
   ValueOwnershipKind(const SILFunction &f, SILType type,
                      SILArgumentConvention convention);
+  ValueOwnershipKind(const SILFunction &f, SILType type,
+                     SILArgumentConvention convention,
+                     SILModuleConventions moduleConventions);
 
   /// Parse Value into a ValueOwnershipKind.
   ///

lib/AST/ASTContext.cpp

@@ -3949,7 +3949,10 @@ SILFunctionType::SILFunctionType(
   } else {
     assert(normalResults.empty());    
     NumAnyResults = yields.size();
-    NumAnyIndirectFormalResults = 0; // unused
+    NumAnyIndirectFormalResults = std::count_if(
+        yields.begin(), yields.end(), [](const SILYieldInfo &yieldInfo) {
+          return yieldInfo.isFormalIndirect();
+        });
     memcpy(getMutableYields().data(), yields.data(),
            yields.size() * sizeof(SILYieldInfo));
   }

lib/ClangImporter/ClangImporter.cpp

@@ -73,6 +73,7 @@
 #include "llvm/Support/Path.h"
 #include "llvm/Support/YAMLParser.h"
 #include "llvm/Support/YAMLTraits.h"
+#include "llvm/Support/VirtualFileSystem.h"
 #include <algorithm>
 #include <memory>
 
@@ -740,14 +741,6 @@ importer::getNormalInvocationArguments(
     } else {
       // FIXME: Emit a warning of some kind.
     }
-
-    if (EnableCXXInterop) {
-      if (auto path =
-              getLibStdCxxModuleMapPath(searchPathOpts, triple, buffer)) {
-        invocationArgStrs.push_back(
-            (Twine("-fmodule-map-file=") + *path).str());
-      }
-    }
   }
 
   if (searchPathOpts.getSDKPath().empty()) {
@@ -917,6 +910,92 @@ importer::addCommonInvocationArguments(
   }
 }
 
+/// On Linux, some platform libraries (glibc, libstdc++) are not modularized.
+/// We inject modulemaps for those libraries into their include directories
+/// to allow using them from Swift.
+static SmallVector<std::pair<std::string, std::string>>
+getClangInvocationFileMapping(ASTContext &ctx) {
+  using Path = SmallString<128>;
+
+  const llvm::Triple &triple = ctx.LangOpts.Target;
+  // We currently only need this when building for Linux.
+  if (!triple.isOSLinux())
+    return {};
+
+  SearchPathOptions &searchPathOpts = ctx.SearchPathOpts;
+
+  Path sdkPath(searchPathOpts.getSDKPath());
+  if (sdkPath.empty())
+    sdkPath = "/";
+
+  // Currently only a modulemap for libstdc++ is injected.
+  if (!ctx.LangOpts.EnableCXXInterop)
+    return {};
+
+  Path actualModuleMapPath;
+  Path buffer;
+  if (auto path = getLibStdCxxModuleMapPath(searchPathOpts, triple, buffer))
+    actualModuleMapPath = path.getValue();
+  else
+    return {};
+
+  // Only inject the module map if it actually exists. It may not, for example
+  // if `swiftc -target x86_64-unknown-linux-gnu -emit-ir` is invoked using
+  // a Swift compiler not built for Linux targets.
+  if (!llvm::sys::fs::exists(actualModuleMapPath))
+    // FIXME: emit a warning of some kind.
+    return {};
+
+  // TODO: remove the libstdcxx.h header and reference all libstdc++ headers
+  // directly from the modulemap.
+  Path actualHeaderPath = actualModuleMapPath;
+  llvm::sys::path::remove_filename(actualHeaderPath);
+  llvm::sys::path::append(actualHeaderPath, "libstdcxx.h");
+
+  Path cxxStdlibsRoot(sdkPath);
+  llvm::sys::path::append(cxxStdlibsRoot, "usr", "include", "c++");
+  if (!llvm::sys::fs::exists(cxxStdlibsRoot))
+    return {};
+
+  // Collect all installed versions of libstdc++. We currently have no way to
+  // know which libstdc++ version will be used for this Clang invocation.
+  // TODO: extract this information from the Clang driver.
+  SmallVector<Path, 1> cxxStdlibDirs;
+  std::error_code errorCode;
+  for (llvm::vfs::directory_iterator
+           iter = ctx.SourceMgr.getFileSystem()->dir_begin(cxxStdlibsRoot,
+                                                           errorCode),
+           endIter;
+       !errorCode && iter != endIter; iter = iter.increment(errorCode)) {
+    cxxStdlibDirs.push_back(Path(iter->path()));
+  }
+
+  SmallVector<std::pair<std::string, std::string>> result;
+  // Inject a modulemap into the VFS for each of the libstdc++ versions.
+  for (const Path &cxxStdlibDir : cxxStdlibDirs) {
+    // Only inject the module map if the module does not already exist at
+    // {sysroot}/usr/include/module.{map,modulemap}.
+    Path injectedModuleMapLegacyPath(cxxStdlibDir);
+    llvm::sys::path::append(injectedModuleMapLegacyPath, "module.map");
+    if (llvm::sys::fs::exists(injectedModuleMapLegacyPath))
+      continue;
+
+    Path injectedModuleMapPath = cxxStdlibDir;
+    llvm::sys::path::append(injectedModuleMapPath, "module.modulemap");
+    if (llvm::sys::fs::exists(injectedModuleMapPath))
+      continue;
+
+    Path injectedHeaderPath = cxxStdlibDir;
+    llvm::sys::path::append(injectedHeaderPath, "libstdcxx.h");
+
+    result.push_back(
+        {std::string(injectedModuleMapPath), std::string(actualModuleMapPath)});
+    result.push_back(
+        {std::string(injectedHeaderPath), std::string(actualHeaderPath)});
+  }
+  return result;
+}
+
 bool ClangImporter::canReadPCH(StringRef PCHFilename) {
   if (!llvm::sys::fs::exists(PCHFilename))
     return false;
@@ -1169,9 +1248,10 @@ ClangImporter::create(ASTContext &ctx,
     }
   }
 
+  auto fileMapping = getClangInvocationFileMapping(ctx);
   // Wrap Swift's FS to allow Clang to override the working directory
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS =
-      llvm::vfs::RedirectingFileSystem::create({}, true,
+      llvm::vfs::RedirectingFileSystem::create(fileMapping, true,
                                                *ctx.SourceMgr.getFileSystem());
 
   // Create a new Clang compiler invocation.