Package CMO fixes.

* Do not use [serialized_for_package] for witness method before Package CMO.
* Update v-table and witness-table and their entry serialization.
  -- Allow non-serialized but visible entries in a serialized table so they
    can be optimized by other SIL opt passes.
* Only serialize MethodInst if it has the right visibility.

Resolves rdar://129089105&129088935

Author: elsh

lib/SIL/IR/SILWitnessTable.cpp

@@ -184,9 +184,7 @@ SerializedKind_t SILWitnessTable::conformanceSerializedKind(
 
   auto *nominal = conformance->getDeclContext()->getSelfNominalTypeDecl();
   if (nominal->getEffectiveAccess() >= accessLevelToCheck)
-    return optInPackage &&
-           conformance->getDeclContext()->getParentModule()->isResilient() ?
-           IsSerializedForPackage : IsSerialized;
+    return IsSerialized;
 
   return IsNotSerialized;
 }

lib/SILOptimizer/IPO/CrossModuleOptimization.cpp

@@ -67,12 +67,14 @@ class CrossModuleOptimization {
   bool everything;
 
   typedef llvm::DenseMap<SILFunction *, bool> FunctionFlags;
+  FunctionFlags canSerializeFlags;
 
 public:
   CrossModuleOptimization(SILModule &M, bool conservative, bool everything)
     : M(M), conservative(conservative), everything(everything) { }
 
-  void serializeFunctionsInModule(ArrayRef<SILFunction *> functions);
+  void trySerializeFunctions(ArrayRef<SILFunction *> functions);
+  void serializeFunctionsInModule(SILPassManager *manager);
   void serializeTablesInModule();
 
 private:
@@ -251,12 +253,8 @@ static bool isReferenceSerializeCandidate(SILGlobalVariable *G,
 }
 
 /// Select functions in the module which should be serialized.
-void CrossModuleOptimization::serializeFunctionsInModule(
+void CrossModuleOptimization::trySerializeFunctions(
     ArrayRef<SILFunction *> functions) {
-  FunctionFlags canSerializeFlags;
-
-  // The passed functions are already ordered bottom-up so the most
-  // nested referenced function is checked first.
   for (SILFunction *F : functions) {
     if (isSerializeCandidate(F, M.getOptions()) || everything) {
       if (canSerializeFunction(F, canSerializeFlags, /*maxDepth*/ 64)) {
@@ -266,31 +264,81 @@ void CrossModuleOptimization::serializeFunctionsInModule(
   }
 }
 
+void CrossModuleOptimization::serializeFunctionsInModule(SILPassManager *manager) {
+  // Reorder SIL funtions in the module bottom up so we can serialize
+  // the most nested referenced functions first and avoid unnecessary
+  // recursive checks.
+  BasicCalleeAnalysis *BCA = manager->getAnalysis<BasicCalleeAnalysis>();
+  BottomUpFunctionOrder BottomUpOrder(M, BCA);
+  auto bottomUpFunctions = BottomUpOrder.getFunctions();
+  trySerializeFunctions(bottomUpFunctions);
+}
+
 void CrossModuleOptimization::serializeTablesInModule() {
   if (!M.getSwiftModule()->serializePackageEnabled())
     return;
 
   for (const auto &vt : M.getVTables()) {
-    if (vt->isNotSerialized() &&
+    if (vt->getSerializedKind() != getRightSerializedKind(M) &&
         vt->getClass()->getEffectiveAccess() >= AccessLevel::Package) {
-      vt->setSerializedKind(getRightSerializedKind(M));
+      // This checks if a vtable entry is not serialized and attempts to
+      // serialize (and its references) if they have the right visibility.
+      // This should not be necessary but is added to ensure all applicable
+      // symbols are serialized. Whether serialized or not is cached so
+      // this check shouldn't be expensive.
+      auto unserializedClassMethodRange = llvm::make_filter_range(vt->getEntries(), [&](auto &entry) {
+        return entry.getImplementation()->getSerializedKind() != getRightSerializedKind(M);
+      });
+      std::vector<SILFunction *> classMethodsToSerialize;
+      llvm::transform(unserializedClassMethodRange,
+                      std::back_inserter(classMethodsToSerialize),
+                      [&](auto entry) {
+        return entry.getImplementation();
+      });
+      trySerializeFunctions(classMethodsToSerialize);
+
+      bool containsInternal = llvm::any_of(classMethodsToSerialize, [&](auto &method) {
+        // If the entry is internal, vtable should not be serialized.
+        // However, if the entry is not serialized but has the right
+        // visibility, it can still be referenced, thus the vtable
+        // should serialized.
+        return !method->hasValidLinkageForFragileRef(getRightSerializedKind(M));
+      });
+      if (!containsInternal)
+        vt->setSerializedKind(getRightSerializedKind(M));
     }
   }
 
+  // Witness thunks are not serialized, so serialize them here.
   for (auto &wt : M.getWitnessTables()) {
-    if (wt.isNotSerialized() && 
+    if (wt.getSerializedKind() != getRightSerializedKind(M) &&
         hasPublicOrPackageVisibility(wt.getLinkage(), /*includePackage*/ true)) {
-      for (auto &entry : wt.getEntries()) {
-        // Witness thunks are not serialized, so serialize them here.
-        if (entry.getKind() == SILWitnessTable::Method &&
-            entry.getMethodWitness().Witness->isNotSerialized() &&
-            isSerializeCandidate(entry.getMethodWitness().Witness,
-                                 M.getOptions())) {
-          entry.getMethodWitness().Witness->setSerializedKind(getRightSerializedKind(M));
-        }
-      }
-      // Then serialize the witness table itself.
-      wt.setSerializedKind(getRightSerializedKind(M));
+      // This checks if a wtable entry is not serialized and attempts to
+      // serialize (and its references) if they have the right visibility.
+      // This should not be necessary but is added to ensure all applicable
+      // symbols are serialized. Whether serialized or not is cached so
+      // this check shouldn't be expensive.
+      auto unserializedWTMethodRange = llvm::make_filter_range(wt.getEntries(), [&](auto &entry) {
+        return entry.getKind() == SILWitnessTable::Method &&
+        entry.getMethodWitness().Witness->getSerializedKind() != getRightSerializedKind(M);
+      });
+      std::vector<SILFunction *> wtMethodsToSerialize;
+      llvm::transform(unserializedWTMethodRange,
+                      std::back_inserter(wtMethodsToSerialize),
+                      [&](auto entry) {
+        return entry.getMethodWitness().Witness;
+      });
+      trySerializeFunctions(wtMethodsToSerialize);
+
+      bool containsInternal = llvm::any_of(wtMethodsToSerialize, [&](auto &method) {
+        // If the entry is internal, wtable should not be serialized.
+        // However, if the entry is not serialized but has the right
+        // visibility, it can still be referenced, thus the vtable
+        // should serialized.
+        return !method->hasValidLinkageForFragileRef(getRightSerializedKind(M));
+      });
+      if (!containsInternal)
+        wt.setSerializedKind(getRightSerializedKind(M));
     }
   }
 }
@@ -441,7 +489,13 @@ bool CrossModuleOptimization::canSerializeInstruction(
     return canUse;
   }
   if (auto *MI = dyn_cast<MethodInst>(inst)) {
-    return !MI->getMember().isForeign;
+    // If a class_method or witness_method is internal, it can't
+    // be serialized.
+    auto member = MI->getMember();
+    auto methodAccessScope = member.getDecl()->getFormalAccessScope(nullptr,
+                                               /*treatUsableFromInlineAsPublic*/ true);
+    return methodAccessScope.isPublicOrPackage() &&
+          !member.isForeign;
   }
   if (auto *REAI = dyn_cast<RefElementAddrInst>(inst)) {
     // In conservative mode, we don't support class field accesses of non-public
@@ -855,14 +909,7 @@ class CrossModuleOptimizationPass: public SILModuleTransform {
     }
 
     CrossModuleOptimization CMO(M, conservative, everything);
-
-    // Reorder SIL funtions in the module bottom up so we can serialize
-    // the most nested referenced functions first and avoid unnecessary
-    // recursive checks.
-    BasicCalleeAnalysis *BCA = PM->getAnalysis<BasicCalleeAnalysis>();
-    BottomUpFunctionOrder BottomUpOrder(M, BCA);
-    auto BottomUpFunctions = BottomUpOrder.getFunctions();
-    CMO.serializeFunctionsInModule(BottomUpFunctions);
+    CMO.serializeFunctionsInModule(PM);
 
     // Serialize SIL v-tables and witness-tables if package-cmo is enabled.
     CMO.serializeTablesInModule();