[clang][cas] Fall back to textual include for missing inferred submodules

When an inferred submodule is missing, because the umbrella header does
not actually include it, we need to fall back to textual include. This
was not working when included via PCH, because we were not serializing
the inference flag(s) in the include-tree.

rdar://107281193
(cherry picked from commit ecf7946)

Author: benlangmuir

clang/include/clang/CAS/IncludeTree.h

@@ -398,9 +398,13 @@ class IncludeTree::Module : public IncludeTreeBase<Module> {
     bool IsExplicit : 1;
     bool IsExternC : 1;
     bool IsSystem : 1;
+    bool InferSubmodules : 1;
+    bool InferExplicitSubmodules : 1;
+    bool InferExportWildcard : 1;
     ModuleFlags()
         : IsFramework(false), IsExplicit(false), IsExternC(false),
-          IsSystem(false) {}
+          IsSystem(false), InferSubmodules(false),
+          InferExplicitSubmodules(false), InferExportWildcard(false) {}
   };
 
   ModuleFlags getFlags() const;
@@ -449,7 +453,7 @@ class IncludeTree::Module : public IncludeTreeBase<Module> {
     if (!IncludeTreeBase::isValid(Node))
       return false;
     IncludeTreeBase Base(Node);
-    return Base.getData().size() > 1;
+    return Base.getData().size() > 2;
   }
   static bool isValid(ObjectStore &DB, ObjectRef Ref) {
     auto Node = DB.getProxy(Ref);
@@ -461,8 +465,8 @@ class IncludeTree::Module : public IncludeTreeBase<Module> {
   }
 
 private:
-  char rawFlags() const { return getData()[0]; }
-  StringRef dataAfterFlags() const { return getData().drop_front(); }
+  uint16_t rawFlags() const;
+  StringRef dataAfterFlags() const { return getData().drop_front(2); }
   bool hasExports() const;
   bool hasLinkLibraries() const;
   std::optional<unsigned> getExportsIndex() const;

clang/lib/CAS/IncludeTree.cpp

@@ -316,20 +316,26 @@ bool IncludeTree::FileList::isValid(const ObjectProxy &Node) {
          Data.size() == sizeof(uint32_t) + NumFiles * sizeof(FileSizeTy);
 }
 
-static constexpr char ModuleFlagFramework = 1 << 0;
-static constexpr char ModuleFlagExplicit = 1 << 1;
-static constexpr char ModuleFlagExternC = 1 << 2;
-static constexpr char ModuleFlagSystem = 1 << 3;
-static constexpr char ModuleFlagHasExports = 1 << 4;
-static constexpr char ModuleFlagHasLinkLibraries = 1 << 5;
+static constexpr uint16_t ModuleFlagFramework = 1 << 0;
+static constexpr uint16_t ModuleFlagExplicit = 1 << 1;
+static constexpr uint16_t ModuleFlagExternC = 1 << 2;
+static constexpr uint16_t ModuleFlagSystem = 1 << 3;
+static constexpr uint16_t ModuleFlagInferSubmodules = 1 << 4;
+static constexpr uint16_t ModuleFlagInferExplicitSubmodules = 1 << 5;
+static constexpr uint16_t ModuleFlagInferInferExportWildcard = 1 << 6;
+static constexpr uint16_t ModuleFlagHasExports = 1 << 7;
+static constexpr uint16_t ModuleFlagHasLinkLibraries = 1 << 8;
 
 IncludeTree::Module::ModuleFlags IncludeTree::Module::getFlags() const {
-  char Raw = rawFlags();
+  uint16_t Raw = rawFlags();
   ModuleFlags Flags;
   Flags.IsFramework = Raw & ModuleFlagFramework;
   Flags.IsExplicit = Raw & ModuleFlagExplicit;
   Flags.IsExternC = Raw & ModuleFlagExternC;
   Flags.IsSystem = Raw & ModuleFlagSystem;
+  Flags.InferSubmodules = Raw & ModuleFlagInferSubmodules;
+  Flags.InferExplicitSubmodules = Raw & ModuleFlagInferExplicitSubmodules;
+  Flags.InferExportWildcard = Raw & ModuleFlagInferInferExportWildcard;
   return Flags;
 }
 
@@ -362,14 +368,14 @@ IncludeTree::Module::create(ObjectStore &DB, StringRef ModuleName,
                             std::optional<ObjectRef> ExportList,
                             std::optional<ObjectRef> LinkLibraries) {
   // Data:
-  // - 1 byte for Flags
+  // - 2 bytes for Flags
   // - ModuleName (String)
   // Refs:
   // - Submodules (IncludeTreeModule)
   // - (optional) ExportList
   // - (optional) LinkLibaryList
 
-  char RawFlags = 0;
+  uint16_t RawFlags = 0;
   if (Flags.IsFramework)
     RawFlags |= ModuleFlagFramework;
   if (Flags.IsExplicit)
@@ -378,13 +384,22 @@ IncludeTree::Module::create(ObjectStore &DB, StringRef ModuleName,
     RawFlags |= ModuleFlagExternC;
   if (Flags.IsSystem)
     RawFlags |= ModuleFlagSystem;
+  if (Flags.InferSubmodules)
+    RawFlags |= ModuleFlagInferSubmodules;
+  if (Flags.InferExplicitSubmodules)
+    RawFlags |= ModuleFlagInferExplicitSubmodules;
+  if (Flags.InferExportWildcard)
+    RawFlags |= ModuleFlagInferInferExportWildcard;
   if (ExportList)
     RawFlags |= ModuleFlagHasExports;
   if (LinkLibraries)
     RawFlags |= ModuleFlagHasLinkLibraries;
 
   SmallString<64> Buffer;
-  Buffer.push_back(RawFlags);
+  llvm::raw_svector_ostream BufOS(Buffer);
+  llvm::support::endian::Writer Writer(BufOS, llvm::support::little);
+  Writer.write(RawFlags);
+
   Buffer.append(ModuleName);
 
   SmallVector<ObjectRef> Refs(Submodules);
@@ -396,6 +411,11 @@ IncludeTree::Module::create(ObjectStore &DB, StringRef ModuleName,
   return IncludeTreeBase::create(DB, Refs, Buffer);
 }
 
+uint16_t IncludeTree::Module::rawFlags() const {
+  return llvm::support::endian::read<uint16_t, llvm::support::little>(
+      getData().data());
+}
+
 bool IncludeTree::Module::hasExports() const {
   return rawFlags() & ModuleFlagHasExports;
 }
@@ -670,6 +690,15 @@ llvm::Error IncludeTree::Module::print(llvm::raw_ostream &OS, unsigned Indent) {
   if (Flags.IsSystem)
     OS << " (system)";
   OS << '\n';
+  if (Flags.InferSubmodules) {
+    if (Flags.InferExplicitSubmodules)
+      OS << "  explicit module *";
+    else
+      OS << "  module *";
+    if (Flags.InferExportWildcard)
+      OS << " { export * }";
+    OS << '\n';
+  }
   auto ExportList = getExports();
   if (!ExportList)
     return ExportList.takeError();

clang/lib/Frontend/FrontendAction.cpp

@@ -559,6 +559,9 @@ static Expected<Module *> makeIncludeTreeModule(CompilerInstance &CI,
   M->Kind = Module::IncludeTreeModuleMap;
   M->IsExternC = Flags.IsExternC;
   M->IsSystem = Flags.IsSystem;
+  M->InferSubmodules = Flags.InferSubmodules;
+  M->InferExplicitSubmodules = Flags.InferExplicitSubmodules;
+  M->InferExportWildcard = Flags.InferExportWildcard;
 
   auto ExportList = Mod.getExports();
   if (!ExportList)

clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp

@@ -517,6 +517,9 @@ getIncludeTreeModule(cas::ObjectStore &DB, Module *M) {
   Flags.IsExplicit = M->IsExplicit;
   Flags.IsExternC = M->IsExternC;
   Flags.IsSystem = M->IsSystem;
+  Flags.InferSubmodules = M->InferSubmodules;
+  Flags.InferExplicitSubmodules = M->InferExplicitSubmodules;
+  Flags.InferExportWildcard = M->InferExportWildcard;
 
   bool GlobalWildcardExport = false;
   SmallVector<ITModule::ExportList::Export> Exports;

clang/test/ClangScanDeps/modules-include-tree-missing-submodule.c

@@ -0,0 +1,92 @@
+// Ensure we fallback to textual inclusion for headers in incomplete umbrellas.
+
+// REQUIRES: ondisk_cas
+
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json
+// RUN: sed "s|DIR|%/t|g" %t/cdb_pch.json.template > %t/cdb_pch.json
+
+// RUN: clang-scan-deps -compilation-database %t/cdb_pch.json \
+// RUN:   -cas-path %t/cas -module-files-dir %t/outputs \
+// RUN:   -format experimental-include-tree-full -mode preprocess-dependency-directives \
+// RUN:   > %t/deps_pch.json
+
+// RUN: %deps-to-rsp %t/deps_pch.json --module-name Foo > %t/Foo.rsp
+// RUN: %deps-to-rsp %t/deps_pch.json --tu-index 0 > %t/pch.rsp
+// RUN: %clang @%t/Foo.rsp
+// RUN: %clang @%t/pch.rsp
+
+// RUN: clang-scan-deps -compilation-database %t/cdb.json \
+// RUN:   -cas-path %t/cas -module-files-dir %t/outputs \
+// RUN:   -format experimental-include-tree-full -mode preprocess-dependency-directives \
+// RUN:   > %t/deps.json
+
+// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp
+
+// Extract include-tree casids
+// RUN: cat %t/Foo.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/Foo.casid
+// RUN: cat %t/tu.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu.casid
+
+// RUN: echo "MODULE Foo" > %t/result.txt
+// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/Foo.casid >> %t/result.txt
+// RUN: echo "TRANSLATION UNIT" >> %t/result.txt
+// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu.casid >> %t/result.txt
+// RUN: FileCheck %s -input-file %t/result.txt -DPREFIX=%/t
+
+// CHECK-LABEL: MODULE Foo
+// CHECK: <module-includes> llvmcas://
+// CHECK: 1:1 <built-in> llvmcas://
+// CHECK: 2:1 [[PREFIX]]/Foo.framework/Headers/Foo.h llvmcas://
+// CHECK:   Submodule: Foo
+// CHECK-NOT: Bar
+// CHECK: Module Map:
+// CHECK: Foo (framework)
+// CHECK-NOT: Bar
+// CHECK:   module *
+// CHECK-NOT: Bar
+
+// CHECK-LABEL: TRANSLATION UNIT
+// CHECK: (PCH) llvmcas://
+// CHECK: [[PREFIX]]/tu.c llvmcas://
+// CHECK: 1:1 <built-in> llvmcas://
+// CHECK: 2:1 [[PREFIX]]/Foo.framework/Headers/Bar.h llvmcas://
+
+// RUN: %clang @%t/tu.rsp
+
+//--- cdb_pch.json.template
+[{
+  "file": "DIR/prefix.h",
+  "directory": "DIR",
+  "command": "clang -x c-header DIR/prefix.h -o DIR/prefix.h.pch -F DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache -Rcompile-job-cache"
+}]
+
+//--- cdb.json.template
+[{
+  "file": "DIR/tu.c",
+  "directory": "DIR",
+  "command": "clang -fsyntax-only DIR/tu.c -include prefix.h -F DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache -Rcompile-job-cache"
+}]
+
+//--- Foo.framework/Modules/module.modulemap
+framework module Foo {
+  umbrella header "Foo.h"
+  module * { export * }
+}
+
+//--- Foo.framework/Headers/Foo.h
+// Do not import Bar.h
+void foo(void);
+
+//--- Foo.framework/Headers/Bar.h
+void bar(void);
+
+//--- prefix.h
+#include <Foo/Foo.h>
+
+//--- tu.c
+#include <Foo/Bar.h>
+// FIXME: -Wincomplete-umbrella warning
+void tu(void) {
+  bar();
+}