Merge pull request #20166 from DougGregor/demangle-fixes-5.0

[5.0] Incorporate recent fixes for demangling to metadata

Author: DougGregor

docs/ABI/Mangling.rst

@@ -9,9 +9,15 @@ Mangling
 --------
 ::
 
-  mangled-name ::= '$s' global
+  mangled-name ::= '$s' global  // Swift stable mangling
+  mangled-name ::= '_T0' global // Swift 4.0
+  mangled-name ::= '$S' global  // Swift 4.2
 
-All Swift-mangled names begin with this prefix.
+All Swift-mangled names begin with a common prefix. Since Swift 4.0, the
+compiler has used variations of the mangling described in this document, though
+pre-stable versions may not exactly conform to this description. By using
+distinct prefixes, tools can attempt to accommodate bugs and version variations
+in pre-stable versions of Swift.
 
 The basic mangling scheme is a list of 'operators' where the operators are
 structured in a post-fix order. For example the mangling may start with an
@@ -20,8 +26,6 @@ identifier has to be interpreted::
 
   4Test3FooC   // The trailing 'C' says that 'Foo' is a class in module 'Test'
 
-
-
 Operators are either identifiers or a sequence of one or more characters,
 like ``C`` for class.
 All operators share the same name-space. Important operators are a single
@@ -42,6 +46,48 @@ mangled name will start with the module name (after the ``_S``).
 In the following, productions which are only _part_ of an operator, are
 named with uppercase letters.
 
+Symbolic references
+~~~~~~~~~~~~~~~~~~~
+
+The Swift compiler emits mangled names into binary images to encode
+references to types for runtime instantiation and reflection. In a binary,
+these mangled names may embed pointers to runtime data
+structures in order to more efficiently represent locally-defined types.
+We call these pointers **symbolic references**.
+These references will be introduced by a control character in the range
+`\x01` ... `\x1F`, which indicates the kind of symbolic reference, followed by
+some number of arbitrary bytes *which may include null bytes*. Code that
+processes mangled names out of Swift binaries needs to be aware of symbolic
+references in order to properly terminate strings; a null terminator may be
+part of a symbolic reference.
+
+::
+
+  symbolic-reference ::= [\x01-\x17] .{4} // Relative symbolic reference
+   #if sizeof(void*) == 8
+     symbolic-reference ::= [\x18-\x1F] .{8} // Absolute symbolic reference
+   #elif sizeof(void*) == 4
+     symbolic-reference ::= [\x18-\x1F] .{4} // Absolute symbolic reference
+   #endif
+
+Symbolic references are only valid in compiler-emitted metadata structures
+and must only appear in read-only parts of a binary image. APIs and tools
+that interpret Swift mangled names from potentially uncontrolled inputs must
+refuse to interpret symbolic references.
+
+The following symbolic reference kinds are currently implemented:
+
+::
+
+   {any-generic-type, protocol} ::= '\x01' .{4}              // Reference points directly to context descriptor
+   {any-generic-type, protocol} ::= '\x02' .{4}              // Reference points indirectly to context descriptor
+   // The grammatical role of the symbolic reference is determined by the
+   // kind of context descriptor referenced
+
+   protocol-conformance-ref ::= '\x03' .{4}  // Reference points directly to protocol conformance descriptor (NOT IMPLEMENTED)
+   protocol-conformance-ref ::= '\x04' .{4}  // Reference points indirectly to protocol conformance descriptor (NOT IMPLEMENTED)
+
+
 Globals
 ~~~~~~~
 
@@ -84,7 +130,7 @@ Globals
   global ::= protocol-conformance 'WP'   // protocol witness table
   global ::= protocol-conformance 'Wa'   // protocol witness table accessor
 
-  global ::= protocol-conformance 'WG'   // generic protocol witness table
+  global ::= protocol-conformance 'WG'   // generic protocol witness table (HISTORICAL)
   global ::= protocol-conformance 'Wp'   // protocol witness table pattern
   global ::= protocol-conformance 'Wr'   // resilient witness table (HISTORICAL)
   global ::= protocol-conformance 'WI'   // generic protocol witness table instantiation function
@@ -549,18 +595,36 @@ Generics
 
 ::
 
-  protocol-conformance ::= type protocol module generic-signature?
+  protocol-conformance-context ::= protocol module generic-signature?
+
+  protocol-conformance ::= type protocol-conformance-context
 
 ``<protocol-conformance>`` refers to a type's conformance to a protocol. The
 named module is the one containing the extension or type declaration that
 declared the conformance.
 
 ::
 
+  protocol-conformance ::= type protocol
+
+If ``type`` is a generic parameter or associated type of one, then no module
+is mangled, because the conformance must be resolved from the generic
+environment.
+
   protocol-conformance ::= context identifier protocol identifier generic-signature?  // Property behavior conformance
 
 Property behaviors are implemented using private protocol conformances.
 
+::
+
+  concrete-protocol-conformance ::= type protocol-conformance-ref
+  protocol-conformance-ref ::= protocol module?
+
+A compact representation used to represent mangled protocol conformance witness
+arguments at runtime. The ``module`` is only specified for conformances that
+are "retroactive", meaning that the context in which the conformance is defined
+is in neither the protocol or type module.
+
 ::
 
   generic-signature ::= requirement* 'l'     // one generic parameter

include/swift/ABI/Metadata.h

@@ -1989,7 +1989,7 @@ struct TargetResilientWitnessTable final
 using ResilientWitnessTable = TargetResilientWitnessTable<InProcess>;
 
 /// \brief The control structure of a generic or resilient protocol
-/// conformance.
+/// conformance, which is embedded in the protocol conformance descriptor.
 ///
 /// Witness tables need to be instantiated at runtime in these cases:
 /// - For a generic conforming type, associated type requirements might be
@@ -2245,14 +2245,16 @@ struct TargetProtocolConformanceDescriptor final
              RelativeContextPointer<Runtime>,
              TargetGenericRequirementDescriptor<Runtime>,
              TargetResilientWitnessesHeader<Runtime>,
-             TargetResilientWitness<Runtime>> {
+             TargetResilientWitness<Runtime>,
+             TargetGenericWitnessTable<Runtime>> {
 
   using TrailingObjects = swift::ABI::TrailingObjects<
                              TargetProtocolConformanceDescriptor<Runtime>,
                              RelativeContextPointer<Runtime>,
                              TargetGenericRequirementDescriptor<Runtime>,
                              TargetResilientWitnessesHeader<Runtime>,
-                             TargetResilientWitness<Runtime>>;
+                             TargetResilientWitness<Runtime>,
+                             TargetGenericWitnessTable<Runtime>>;
   friend TrailingObjects;
 
   template<typename T>
@@ -2268,6 +2270,7 @@ struct TargetProtocolConformanceDescriptor final
 
   using ResilientWitnessesHeader = TargetResilientWitnessesHeader<Runtime>;
   using ResilientWitness = TargetResilientWitness<Runtime>;
+  using GenericWitnessTable = TargetGenericWitnessTable<Runtime>;
 
 private:
   /// The protocol being conformed to.
@@ -2375,6 +2378,14 @@ struct TargetProtocolConformanceDescriptor final
              numTrailingObjects(OverloadToken<ResilientWitness>()));
   }
 
+  ConstTargetPointer<Runtime, GenericWitnessTable>
+  getGenericWitnessTable() const {
+    if (!Flags.hasGenericWitnessTable())
+      return nullptr;
+
+    return this->template getTrailingObjects<GenericWitnessTable>();
+  }
+
 #if !defined(NDEBUG) && SWIFT_OBJC_INTEROP
   void dump() const;
 #endif
@@ -2409,6 +2420,10 @@ struct TargetProtocolConformanceDescriptor final
           ->NumWitnesses
       : 0;
   }
+
+  size_t numTrailingObjects(OverloadToken<GenericWitnessTable>) const {
+    return Flags.hasGenericWitnessTable() ? 1 : 0;
+  }
 };
 using ProtocolConformanceDescriptor
   = TargetProtocolConformanceDescriptor<InProcess>;

include/swift/ABI/MetadataValues.h

@@ -629,6 +629,7 @@ class ConformanceFlags {
     NumConditionalRequirementsShift = 8,
 
     HasResilientWitnessesMask = 0x01 << 16,
+    HasGenericWitnessTableMask = 0x01 << 17,
   };
 
   int_type Value;
@@ -668,6 +669,14 @@ class ConformanceFlags {
                                                     : 0));
   }
 
+  ConformanceFlags withHasGenericWitnessTable(
+                                           bool hasGenericWitnessTable) const {
+    return ConformanceFlags((Value & ~HasGenericWitnessTableMask)
+                            | (hasGenericWitnessTable
+                                 ? HasGenericWitnessTableMask
+                                 : 0));
+  }
+
   /// Retrieve the conformance kind.
   ConformanceKind getConformanceKind() const {
     return ConformanceKind(Value & ConformanceKindMask);
@@ -709,6 +718,12 @@ class ConformanceFlags {
     return Value & HasResilientWitnessesMask;
   }
 
+  /// Whether this conformance has a generic witness table that may need to
+  /// be instantiated.
+  bool hasGenericWitnessTable() const {
+    return Value & HasGenericWitnessTableMask;
+  }
+
   int_type getIntValue() const { return Value; }
 };
 

include/swift/AST/ASTMangler.h

@@ -46,12 +46,22 @@ class ASTMangler : public Mangler {
   /// If disabled, it is an error to try to mangle such an entity.
   bool AllowNamelessEntities = false;
 
-  /// If nonnull, provides a callback to encode symbolic references to
-  /// type contexts.
-  std::function<bool (const DeclContext *Context)>
-    CanSymbolicReference;
-  
-  std::vector<std::pair<const DeclContext *, unsigned>> SymbolicReferences;
+  /// If enabled, some entities will be emitted as symbolic reference
+  /// placeholders. The offsets of these references will be stored in the
+  /// `SymbolicReferences` vector, and it is up to the consumer of the mangling
+  /// to fill these in.
+  bool AllowSymbolicReferences = false;
+
+public:
+  using SymbolicReferent = llvm::PointerUnion<const NominalTypeDecl *,
+                                              const ProtocolConformance *>;
+protected:
+
+  /// If set, the mangler calls this function to determine whether to symbolic
+  /// reference a given entity. Defaults to always returning true.
+  std::function<bool (SymbolicReferent)> CanSymbolicReference;
+
+  std::vector<std::pair<SymbolicReferent, unsigned>> SymbolicReferences;
 
 public:
   enum class SymbolKind {
@@ -292,7 +302,7 @@ class ASTMangler : public Mangler {
 
   void appendOpParamForLayoutConstraint(LayoutConstraint Layout);
 
-  void appendSymbolicReference(const DeclContext *context);
+  void appendSymbolicReference(SymbolicReferent referent);
 
   std::string mangleTypeWithoutPrefix(Type type) {
     appendType(type);

include/swift/Demangling/Demangle.h

@@ -33,6 +33,8 @@ namespace llvm {
 namespace swift {
 namespace Demangle {
 
+enum class SymbolicReferenceKind : uint8_t;
+
 struct DemangleOptions {
   bool SynthesizeSugarOnTypes = false;
   bool DisplayDebuggerGeneratedModule = true;
@@ -473,7 +475,9 @@ void mangleIdentifier(const char *data, size_t length,
 /// This should always round-trip perfectly with demangleSymbolAsNode.
 std::string mangleNode(const NodePointer &root);
 
-using SymbolicResolver = llvm::function_ref<Demangle::NodePointer (const void *)>;
+using SymbolicResolver =
+  llvm::function_ref<Demangle::NodePointer (SymbolicReferenceKind,
+                                            const void *)>;
 
 /// \brief Remangle a demangled parse tree, using a callback to resolve
 /// symbolic references.
@@ -537,6 +541,8 @@ class DemanglerPrinter {
     return std::move(*this << std::forward<T>(x));
   }
 
+  DemanglerPrinter &writeHex(unsigned long long n) &;
+ 
   std::string &&str() && { return std::move(Stream); }
 
   llvm::StringRef getStringRef() const { return Stream; }

include/swift/Demangling/DemangleNodes.def

@@ -143,6 +143,7 @@ NODE(PrefixOperator)
 NODE(PrivateDeclName)
 NODE(PropertyDescriptor)
 CONTEXT_NODE(Protocol)
+CONTEXT_NODE(ProtocolSymbolicReference)
 NODE(ProtocolConformance)
 NODE(ProtocolDescriptor)
 NODE(ProtocolConformanceDescriptor)
@@ -172,13 +173,13 @@ NODE(SpecializationIsFragile)
 CONTEXT_NODE(Static)
 CONTEXT_NODE(Structure)
 CONTEXT_NODE(Subscript)
-CONTEXT_NODE(SymbolicReference)
 NODE(Suffix)
 NODE(ThinFunctionType)
 NODE(Tuple)
 NODE(TupleElement)
 NODE(TupleElementName)
 NODE(Type)
+CONTEXT_NODE(TypeSymbolicReference)
 CONTEXT_NODE(TypeAlias)
 NODE(TypeList)
 NODE(TypeMangling)
@@ -192,7 +193,6 @@ NODE(TypeMetadataLazyCache)
 NODE(UncurriedFunctionType)
 #define REF_STORAGE(Name, ...) NODE(Name)
 #include "swift/AST/ReferenceStorage.def"
-CONTEXT_NODE(UnresolvedSymbolicReference)
 CONTEXT_NODE(UnsafeAddressor)
 CONTEXT_NODE(UnsafeMutableAddressor)
 NODE(ValueWitness)

include/swift/Demangling/Demangler.h

@@ -280,6 +280,17 @@ class CharVector : public Vector<char> {
   }
 };
 
+/// Kinds of symbolic reference supported.
+enum class SymbolicReferenceKind : uint8_t {
+  /// A symbolic reference to a context descriptor, representing the
+  /// (unapplied generic) context.
+  Context,  
+};
+
+using SymbolicReferenceResolver_t = NodePointer (SymbolicReferenceKind,
+                                                 Directness,
+                                                 int32_t, const void *);
+
 /// The demangler.
 ///
 /// It de-mangles a string and it also owns the returned node-tree. This means
@@ -301,7 +312,7 @@ class Demangler : public NodeFactory {
   StringRef Words[MaxNumWords];
   int NumWords = 0;
 
-  std::function<NodePointer (int32_t, const void *)> SymbolicReferenceResolver;
+  std::function<SymbolicReferenceResolver_t> SymbolicReferenceResolver;
 
   bool nextIf(StringRef str) {
     if (!Text.substr(Pos).startswith(str)) return false;
@@ -472,7 +483,8 @@ class Demangler : public NodeFactory {
 
   NodePointer demangleObjCTypeName();
   NodePointer demangleTypeMangling();
-  NodePointer demangleSymbolicReference(const void *at);
+  NodePointer demangleSymbolicReference(unsigned char rawKind,
+                                        const void *at);
 
   void dump();
 
@@ -483,7 +495,7 @@ class Demangler : public NodeFactory {
 
   /// Install a resolver for symbolic references in a mangled string.
   void setSymbolicReferenceResolver(
-                  std::function<NodePointer (int32_t, const void*)> resolver) {
+                          std::function<SymbolicReferenceResolver_t> resolver) {
     SymbolicReferenceResolver = resolver;
   }