Added partial support for compiling C++20 modules and header-units without scanning.

clang/include/clang/Driver/Options.td

@@ -9556,3 +9556,7 @@ def wasm_opt : Flag<["--"], "wasm-opt">,
   Group<m_Group>,
   HelpText<"Enable the wasm-opt optimizer (default)">,
   MarshallingInfoNegativeFlag<LangOpts<"NoWasmOpt">>;
+
+def no_scan_ipc : Flag<["-", "/"], "noScanIPC">,
+                  Visibility<[ClangOption, CC1Option, CLOption]>,
+                  HelpText<"Enable No scan IPC approach">;

clang/include/clang/Frontend/CompilerInstance.h

@@ -944,6 +944,11 @@ class CompilerInstance : public ModuleLoader {
 
   bool lookupMissingImports(StringRef Name, SourceLocation TriggerLoc) override;
 
+  void makeModuleAndDependenciesVisible(serialization::ModuleFile *ModFile,
+                                        Module *Mod);
+  Module *loadIPCReceivedHeaderUnit(StringRef FileName,
+                                    SourceLocation ImportLoc) override;
+
   void addDependencyCollector(std::shared_ptr<DependencyCollector> Listener) {
     DependencyCollectors.push_back(std::move(Listener));
   }

clang/include/clang/IPC2978/.clang-format-ignore

@@ -0,0 +1 @@
+*

clang/include/clang/IPC2978/IPCManagerBS.hpp

@@ -0,0 +1,39 @@
+
+#ifndef IPC_MANAGER_BS_HPP
+#define IPC_MANAGER_BS_HPP
+
+#include "clang/IPC2978/Manager.hpp"
+#include "clang/IPC2978/Messages.hpp"
+
+namespace N2978
+{
+
+// IPC Manager BuildSystem
+class IPCManagerBS : Manager
+{
+    friend tl::expected<IPCManagerBS, std::string> makeIPCManagerBS(std::string BMIIfHeaderUnitObjOtherwisePath);
+    bool connectedToCompiler = false;
+
+#ifdef _WIN32
+    explicit IPCManagerBS(void *hPipe_);
+#else
+    explicit IPCManagerBS(int fdSocket_);
+#endif
+
+  public:
+    IPCManagerBS(const IPCManagerBS &) = default;
+    IPCManagerBS &operator=(const IPCManagerBS &) = default;
+    IPCManagerBS(IPCManagerBS &&) = default;
+    IPCManagerBS &operator=(IPCManagerBS &&) = default;
+    tl::expected<void, std::string> receiveMessage(char (&ctbBuffer)[320], CTB &messageType) const;
+    [[nodiscard]] tl::expected<void, std::string> sendMessage(const BTCModule &moduleFile) const;
+    [[nodiscard]] tl::expected<void, std::string> sendMessage(const BTCNonModule &nonModule) const;
+    [[nodiscard]] tl::expected<void, std::string> sendMessage(const BTCLastMessage &lastMessage) const;
+    static tl::expected<ProcessMappingOfBMIFile, std::string> createSharedMemoryBMIFile(const BMIFile &bmiFile);
+    static tl::expected<void, std::string> closeBMIFileMapping(const ProcessMappingOfBMIFile &processMappingOfBMIFile);
+    void closeConnection() const;
+};
+
+tl::expected<IPCManagerBS, std::string> makeIPCManagerBS(std::string BMIIfHeaderUnitObjOtherwisePath);
+} // namespace N2978
+#endif // IPC_MANAGER_BS_HPP

clang/include/clang/IPC2978/IPCManagerCompiler.hpp

@@ -0,0 +1,182 @@
+
+#ifndef IPC_MANAGER_COMPILER_HPP
+#define IPC_MANAGER_COMPILER_HPP
+
+#include "clang/IPC2978/Manager.hpp"
+#include "clang/IPC2978/expected.hpp"
+
+struct CompilerTest;
+namespace N2978
+{
+
+enum class FileType : uint8_t
+{
+    MODULE,
+    HEADER_UNIT,
+    HEADER_FILE
+};
+
+struct Response
+{
+    // if type == HEADER_FILE, then fileSize has no meaning
+    BMIFile file;
+    FileType type;
+    bool user;
+    Response(BMIFile file_, FileType type_, bool user_);
+};
+
+// IPC Manager Compiler
+class IPCManagerCompiler : Manager
+{
+    friend struct ::CompilerTest;
+    template <typename T> tl::expected<T, std::string> receiveMessage() const;
+    // This is not exposed. sendCTBLastMessage calls this.
+    [[nodiscard]] tl::expected<void, std::string> receiveBTCLastMessage() const;
+    [[nodiscard]] tl::expected<BTCModule, std::string> receiveBTCModule(const CTBModule &moduleName);
+    [[nodiscard]] tl::expected<BTCNonModule, std::string> receiveBTCNonModule(const CTBNonModule &nonModule);
+
+    std::unordered_map<std::string, Response> responses;
+
+  public:
+    CTBLastMessage lastMessage{};
+#ifdef _WIN32
+    explicit IPCManagerCompiler(void *hPipe_);
+#else
+    explicit IPCManagerCompiler(int fdSocket_);
+#endif
+
+    // For FileType::HEADER_FILE, it can return FileType::HEADER_UNIT, otherwise it will return the request
+    // response. Either it will return from the cache or it will fetch it from the build-system
+    [[nodiscard]] tl::expected<Response, std::string> findResponse(const std::string &logicalName, FileType type);
+    [[nodiscard]] tl::expected<void, std::string> sendCTBLastMessage(const CTBLastMessage &lastMessage) const;
+    [[nodiscard]] tl::expected<void, std::string> sendCTBLastMessage(const CTBLastMessage &lastMessage,
+                                                                     const std::string &bmiFile,
+                                                                     const std::string &filePath) const;
+    static tl::expected<ProcessMappingOfBMIFile, std::string> readSharedMemoryBMIFile(const BMIFile &file);
+    static tl::expected<void, std::string> closeBMIFileMapping(const ProcessMappingOfBMIFile &processMappingOfBMIFile);
+    void closeConnection() const;
+};
+
+template <typename T> tl::expected<T, std::string> IPCManagerCompiler::receiveMessage() const
+{
+    // Read from the pipe.
+    char buffer[BUFFERSIZE];
+    uint32_t bytesRead;
+    if (const auto &r = readInternal(buffer); !r)
+    {
+        return tl::unexpected(r.error());
+    }
+    else
+    {
+        bytesRead = *r;
+    }
+
+    uint32_t bytesProcessed = 0;
+
+    if constexpr (std::is_same_v<T, BTCModule>)
+    {
+        const auto &r = readProcessMappingOfBMIFileFromPipe(buffer, bytesRead, bytesProcessed);
+        if (!r)
+        {
+            return tl::unexpected(r.error());
+        }
+
+        const auto &r2 = readBoolFromPipe(buffer, bytesRead, bytesProcessed);
+        if (!r2)
+        {
+            return tl::unexpected(r2.error());
+        }
+
+        const auto &r3 = readVectorOfModuleDepFromPipe(buffer, bytesRead, bytesProcessed);
+        if (!r3)
+        {
+            return tl::unexpected(r3.error());
+        }
+
+        BTCModule moduleFile;
+        moduleFile.requested = *r;
+        moduleFile.user = *r2;
+        moduleFile.modDeps = *r3;
+
+        if (bytesRead == bytesProcessed)
+        {
+            return moduleFile;
+        }
+    }
+    else if constexpr (std::is_same_v<T, BTCNonModule>)
+    {
+        const auto &r = readBoolFromPipe(buffer, bytesRead, bytesProcessed);
+        if (!r)
+        {
+            return tl::unexpected(r.error());
+        }
+
+        const auto &r2 = readBoolFromPipe(buffer, bytesRead, bytesProcessed);
+        if (!r2)
+        {
+            return tl::unexpected(r2.error());
+        }
+
+        const auto &r3 = readStringFromPipe(buffer, bytesRead, bytesProcessed);
+        if (!r3)
+        {
+            return tl::unexpected(r3.error());
+        }
+
+        const auto &r4 = readUInt32FromPipe(buffer, bytesRead, bytesProcessed);
+        if (!r4)
+        {
+            return tl::unexpected(r4.error());
+        }
+
+        const auto &r5 = readVectorOfStringFromPipe(buffer, bytesRead, bytesProcessed);
+        if (!r5)
+        {
+            return tl::unexpected(r5.error());
+        }
+
+        const auto &r6 = readVectorOfHeaderFileFromPipe(buffer, bytesRead, bytesProcessed);
+        if (!r6)
+        {
+            return tl::unexpected(r6.error());
+        }
+
+        const auto &r7 = readVectorOfHuDepFromPipe(buffer, bytesRead, bytesProcessed);
+        if (!r7)
+        {
+            return tl::unexpected(r7.error());
+        }
+
+        BTCNonModule nonModule;
+        nonModule.isHeaderUnit = *r;
+        nonModule.user = *r2;
+        nonModule.filePath = *r3;
+        nonModule.fileSize = *r4;
+        nonModule.logicalNames = *r5;
+        nonModule.headerFiles = *r6;
+        nonModule.huDeps = *r7;
+
+        if (bytesRead == bytesProcessed)
+        {
+            return nonModule;
+        }
+    }
+    else
+    {
+        static_assert(false && "Unknown type\n");
+    }
+
+    if (bytesRead != bytesProcessed)
+    {
+        return tl::unexpected(getErrorString(bytesRead, bytesProcessed));
+    }
+    std::string str = __FILE__;
+    str += ':';
+    str += std::to_string(__LINE__);
+    return tl::unexpected(getErrorString("N2978 IPC API internal error" + str));
+}
+[[nodiscard]] tl::expected<IPCManagerCompiler, std::string> makeIPCManagerCompiler(
+    std::string BMIIfHeaderUnitObjOtherwisePath);
+inline IPCManagerCompiler *managerCompiler;
+} // namespace N2978
+#endif // IPC_MANAGER_COMPILER_HPP

clang/include/clang/IPC2978/Manager.hpp

@@ -0,0 +1,143 @@
+
+#ifndef MANAGER_HPP
+#define MANAGER_HPP
+
+#include "clang/IPC2978/Messages.hpp"
+#include "clang/IPC2978/expected.hpp"
+
+#include <string>
+#include <vector>
+
+#define BUFFERSIZE 4096
+
+#ifdef _WIN32
+// The following variable is used in CreateNamedFunction.
+#define PIPE_TIMEOUT 5000
+#endif
+
+namespace tl
+{
+template <typename T, typename U> class expected;
+}
+
+namespace N2978
+{
+
+enum class ErrorCategory : uint8_t
+{
+    NONE,
+
+    // error-category for API errors
+    READ_FILE_ZERO_BYTES_READ,
+    INCORRECT_BTC_LAST_MESSAGE,
+    UNKNOWN_CTB_TYPE,
+};
+
+std::string getErrorString();
+std::string getErrorString(uint32_t bytesRead_, uint32_t bytesProcessed_);
+std::string getErrorString(ErrorCategory errorCategory_);
+// to facilitate error propagation.
+inline std::string getErrorString(std::string err)
+{
+    return err;
+}
+
+struct ProcessMappingOfBMIFile
+{
+    std::string_view file;
+#ifdef _WIN32
+    void *mapping;
+    void *view;
+#else
+    void *mapping;
+    uint32_t mappingSize;
+#endif
+};
+
+class Manager
+{
+  public:
+#ifdef _WIN32
+    void *hPipe = nullptr;
+#else
+    int fdSocket = 0;
+#endif
+
+    tl::expected<uint32_t, std::string> readInternal(char (&buffer)[BUFFERSIZE]) const;
+    tl::expected<void, std::string> writeInternal(const std::vector<char> &buffer) const;
+
+    static std::vector<char> getBufferWithType(CTB type);
+    static void writeUInt32(std::vector<char> &buffer, uint32_t value);
+    static void writeString(std::vector<char> &buffer, const std::string &str);
+    static void writeProcessMappingOfBMIFile(std::vector<char> &buffer, const BMIFile &file);
+    static void writeModuleDep(std::vector<char> &buffer, const ModuleDep &dep);
+    static void writeHuDep(std::vector<char> &buffer, const HuDep &dep);
+    static void writeHeaderFile(std::vector<char> &buffer, const HeaderFile &dep);
+    static void writeVectorOfStrings(std::vector<char> &buffer, const std::vector<std::string> &strs);
+    static void writeVectorOfProcessMappingOfBMIFiles(std::vector<char> &buffer, const std::vector<BMIFile> &files);
+    static void writeVectorOfModuleDep(std::vector<char> &buffer, const std::vector<ModuleDep> &deps);
+    static void writeVectorOfHuDeps(std::vector<char> &buffer, const std::vector<HuDep> &deps);
+    static void writeVectorOfHeaderFiles(std::vector<char> &buffer, const std::vector<HeaderFile> &headerFiles);
+
+    tl::expected<bool, std::string> readBoolFromPipe(char (&buffer)[BUFFERSIZE], uint32_t &bytesRead,
+                                                     uint32_t &bytesProcessed) const;
+    tl::expected<uint32_t, std::string> readUInt32FromPipe(char (&buffer)[BUFFERSIZE], uint32_t &bytesRead,
+                                                           uint32_t &bytesProcessed) const;
+    tl::expected<std::string, std::string> readStringFromPipe(char (&buffer)[BUFFERSIZE], uint32_t &bytesRead,
+                                                              uint32_t &bytesProcessed) const;
+    tl::expected<BMIFile, std::string> readProcessMappingOfBMIFileFromPipe(char (&buffer)[BUFFERSIZE],
+                                                                           uint32_t &bytesRead,
+                                                                           uint32_t &bytesProcessed) const;
+    tl::expected<std::vector<std::string>, std::string> readVectorOfStringFromPipe(char (&buffer)[BUFFERSIZE],
+                                                                                   uint32_t &bytesRead,
+                                                                                   uint32_t &bytesProcessed) const;
+    tl::expected<ModuleDep, std::string> readModuleDepFromPipe(char (&buffer)[BUFFERSIZE], uint32_t &bytesRead,
+                                                               uint32_t &bytesProcessed) const;
+    tl::expected<std::vector<ModuleDep>, std::string> readVectorOfModuleDepFromPipe(char (&buffer)[BUFFERSIZE],
+                                                                                    uint32_t &bytesRead,
+                                                                                    uint32_t &bytesProcessed) const;
+    tl::expected<HuDep, std::string> readHuDepFromPipe(char (&buffer)[BUFFERSIZE], uint32_t &bytesRead,
+                                                       uint32_t &bytesProcessed) const;
+    tl::expected<std::vector<HuDep>, std::string> readVectorOfHuDepFromPipe(char (&buffer)[BUFFERSIZE],
+                                                                            uint32_t &bytesRead,
+                                                                            uint32_t &bytesProcessed) const;
+    tl::expected<HeaderFile, std::string> readHeaderFileFromPipe(char (&buffer)[BUFFERSIZE], uint32_t &bytesRead,
+                                                                 uint32_t &bytesProcessed) const;
+    tl::expected<std::vector<HeaderFile>, std::string> readVectorOfHeaderFileFromPipe(char (&buffer)[BUFFERSIZE],
+                                                                                      uint32_t &bytesRead,
+                                                                                      uint32_t &bytesProcessed) const;
+    tl::expected<void, std::string> readNumberOfBytes(char *output, uint32_t size, char (&buffer)[BUFFERSIZE],
+                                                      uint32_t &bytesRead, uint32_t &bytesProcessed) const;
+};
+
+template <typename T, typename... Args> constexpr T *construct_at(T *p, Args &&...args)
+{
+    return ::new (static_cast<void *>(p)) T(std::forward<Args>(args)...);
+}
+
+template <typename T> T &getInitializedObjectFromBuffer(char (&buffer)[320])
+{
+    T &t = reinterpret_cast<T &>(buffer);
+    construct_at(&t);
+    return t;
+}
+
+inline std::string to16charHexString(const uint64_t v)
+{
+    static auto lut = "0123456789abcdef";
+    std::string out;
+    out.resize(16);
+    for (int i = 0; i < 8; ++i)
+    {
+        // extract byte in big-endian order:
+        const auto byte = static_cast<uint8_t>(v >> ((7 - i) * 8));
+        // high nibble:
+        out[2 * i] = lut[byte >> 4];
+        // low nibble:
+        out[2 * i + 1] = lut[byte & 0xF];
+    }
+    return out;
+}
+
+} // namespace N2978
+#endif // MANAGER_HPP