[NFC] Generate code for experimental nodes better

Initializers for nodes with experimental node children need to be marked `@_spi`. This PR:

• Adds that attribute.
• Generates an alternative which *doesn’t* use SPI as part of the compatibility layer.
• As a side effect, adds a `Child.Refactoring.introduced` case that can be used to generate compatibility `unexpected` properties.

No functional change in this commit, but it will affect the code generation in the next one.

Author: beccadax

CodeGeneration/Sources/SyntaxSupport/Child.swift

@@ -140,6 +140,19 @@ public class Child: NodeChoiceConvertible {
     }
   }
 
+  /// Should this child be hidden?
+  ///
+  /// A hidden child is one that is not accessible in any way at a specific point in the history, but still needs to be
+  /// (default) initialized. As always, its `newestChildPath` indicates the current way to access it.
+  ///
+  /// Hidden children are used for `Refactoring.introduced` and for the implicit changeset that creates
+  /// non-experimental APIs that ignore experimental children.
+  public let isHidden: Bool
+
+  /// True if this child was created by a `childHistory` change set. Such children
+  /// are part of the compatibility layer and are therefore deprecated.
+  public var isHistorical: Bool
+
   /// A name of this child as an identifier.
   public var identifier: TokenSyntax {
     return .identifier(lowercaseFirstWord(name: name))
@@ -161,8 +174,8 @@ public class Child: NodeChoiceConvertible {
     return "\(raw: newestName.withFirstCharacterUppercased)Options"
   }
 
-  /// If this child is deprecated, describes the sequence of accesses necessary
-  /// to reach the equivalent value using non-deprecated children; if the child
+  /// If this child is part of a compatibility layer, describes the sequence of accesses necessary
+  /// to reach the equivalent value using non-compatibility-layer children; if the child
   /// is not deprecated, this array is empty.
   ///
   /// Think of the elements of this array like components in a key path:
@@ -199,12 +212,6 @@ public class Child: NodeChoiceConvertible {
   ///         of the child. That information is not directly available anywhere.
   public let newestChildPath: [Child]
 
-  /// True if this child was created by a `Child.Refactoring`. Such children
-  /// are part of the compatibility layer and are therefore deprecated.
-  public var isHistorical: Bool {
-    !newestChildPath.isEmpty
-  }
-
   /// Replaces the nodes in `newerChildPath` with their own `newerChildPath`s,
   /// if any, to form a child path enitrely of non-historical nodes.
   static private func makeNewestChildPath(from newerChildPath: [Child]) -> [Child] {
@@ -214,7 +221,7 @@ public class Child: NodeChoiceConvertible {
     var workStack = Array(newerChildPath.reversed())
 
     while let elem = workStack.popLast() {
-      if elem.isHistorical {
+      if !elem.newestChildPath.isEmpty {
         // There's an even newer version. Start working on that.
         workStack.append(contentsOf: elem.newestChildPath.reversed())
       } else {
@@ -308,7 +315,8 @@ public class Child: NodeChoiceConvertible {
     documentation: String? = nil,
     isOptional: Bool = false,
     providesDefaultInitialization: Bool = true,
-    newerChildPath: [Child] = []
+    newerChildPath: [Child] = [],
+    isHistorical: Bool = false
   ) {
     precondition(name.first?.isLowercase ?? true, "The first letter of a child’s name should be lowercase")
     self.name = name
@@ -320,11 +328,18 @@ public class Child: NodeChoiceConvertible {
     self.documentationAbstract = String(documentation?.split(whereSeparator: \.isNewline).first ?? "")
     self.isOptional = isOptional
     self.providesDefaultInitialization = providesDefaultInitialization
+    self.isHidden = false
+    self.isHistorical = isHistorical
   }
 
   /// Create a node that is a copy of the last node in `newerChildPath`, but
   /// with modifications.
-  init(renamingTo replacementName: String? = nil, newerChildPath: [Child]) {
+  init(
+    renamingTo replacementName: String? = nil,
+    makingHistorical: Bool = false,
+    makingHidden: Bool = false,
+    newerChildPath: [Child]
+  ) {
     let other = newerChildPath.last!
 
     self.name = replacementName ?? other.name
@@ -336,6 +351,8 @@ public class Child: NodeChoiceConvertible {
     self.documentationAbstract = other.documentationAbstract
     self.isOptional = other.isOptional
     self.providesDefaultInitialization = other.providesDefaultInitialization
+    self.isHidden = makingHidden || other.isHidden
+    self.isHistorical = makingHistorical || other.isHistorical
   }
 
   /// Create a child for the unexpected nodes between two children (either or
@@ -361,7 +378,8 @@ public class Child: NodeChoiceConvertible {
       documentation: nil,
       isOptional: true,
       providesDefaultInitialization: true,
-      newerChildPath: newerChildPath
+      newerChildPath: newerChildPath,
+      isHistorical: (earlier?.isHistorical ?? false) || (later?.isHistorical ?? false)
     )
   }
 }
@@ -417,5 +435,8 @@ extension Child {
     /// point in the past, so deprecated aliases that flatten the other node's
     /// children into this node should be provided.
     case extracted
+
+    /// A new child was added (and it's important to preserve the names around it).
+    case introduced
   }
 }

CodeGeneration/Sources/SyntaxSupport/CompatibilityLayer.swift

@@ -18,8 +18,9 @@ public struct CompatibilityLayer {
   /// Deprecated members that the compatibility layer needs for each trait.
   public var deprecatedMembersByTrait: [String: DeprecatedMemberInfo] = [:]
 
-  /// Cache for `replacementChildren(for:by:)`. Ensures that we don't create two different replacement children even
-  /// if we refactor the same child twice, so we can reliably equate and hash `Child` objects by object identity.
+  /// Cache for `replacementChildren(for:by:historical:)`. Ensures that we don't create two different replacement
+  /// children even if we refactor the same child twice, so we can reliably equate and hash `Child` objects by
+  /// object identity.
   private var cachedReplacementChildren: [Child: [Child]] = [:]
 
   /// Returns the deprecated members that the compatibility layer needs for `node`.
@@ -46,13 +47,23 @@ public struct CompatibilityLayer {
 
   /// Returns the child or children that would have existed in place of this
   /// child before this refactoring was applied.
-  private mutating func replacementChildren(for newerChild: Child, by refactoring: Child.Refactoring) -> [Child] {
+  ///
+  /// - Parameters:
+  ///   - newerChild: The child which is being replaced.
+  ///   - refactoring: The refactoring which created that child and must be
+  ///     reversed.
+  fileprivate mutating func replacementChildren(
+    for newerChild: Child,
+    by refactoring: Child.Refactoring,
+    historical: Bool
+  ) -> [Child] {
     func make() -> [Child] {
       switch refactoring {
       case .renamed(from: let deprecatedName):
         return [
           Child(
             renamingTo: deprecatedName,
+            makingHistorical: historical,
             newerChildPath: [newerChild]
           )
         ]
@@ -72,8 +83,20 @@ public struct CompatibilityLayer {
         }
 
         return newerGrandchildren.map { newerGrandchild in
-          Child(newerChildPath: [newerChild, newerGrandchild])
+          Child(
+            makingHistorical: historical,
+            newerChildPath: [newerChild, newerGrandchild]
+          )
         }
+
+      case .introduced:
+        return [
+          Child(
+            makingHistorical: historical,
+            makingHidden: true,
+            newerChildPath: [newerChild]
+          )
+        ]
       }
     }
 
@@ -100,6 +123,7 @@ public struct CompatibilityLayer {
     deprecatedMembersByNode[node.syntaxNodeKind] = result
   }
 
+  /// Compute and cache compatibility layer information for the given children.
   private mutating func computeMembers(for trait: Trait) {
     guard deprecatedMembersByTrait[trait.traitName] == nil else {
       return
@@ -115,58 +139,146 @@ public struct CompatibilityLayer {
     deprecatedMembersByTrait[trait.traitName] = result
   }
 
-  /// Compute and cache compatibility layer information for the given children.
+  /// Compute compatibility layer information for the given children.
   private mutating func computeMembersFor(
     typeName: String,
     initialChildren: [Child],
     history: Child.History,
     areRequirements: Bool
   ) -> DeprecatedMemberInfo {
-    // The results that will ultimately be saved into the DeprecatedMemberInfo.
+    var builder = DeprecatedMemberInfo.Builder(
+      typeName: typeName,
+      children: initialChildren,
+      areRequirements: areRequirements
+    )
+
+    // If any of the children are experimental, apply an initial change set that hides them, ensuring that we generate
+    // APIs which aren't experimental.
+    let experimentalChildren = initialChildren.filter { $0.isExperimental && !$0.isUnexpectedNodes }
+    if !experimentalChildren.isEmpty {
+      let syntheticChangeSet = experimentalChildren.map { ($0.name, Child.Refactoring.introduced) }
+      builder.applyChangeSet(syntheticChangeSet, for: &self, historical: false)
+    }
+
+    // Apply changes in the history
+    for changeSet in history {
+      builder.applyChangeSet(changeSet, for: &self, historical: true)
+    }
+
+    return builder.make()
+  }
+}
+
+/// Describes the deprecated members of a given type that the compatibility layer ought to provide.
+public struct DeprecatedMemberInfo {
+  /// Properties that are needed in the compatibility layer, in the order they ought to appear in the generated file.
+  public var vars: [Child] = []
+
+  /// Initializer signatures that are needed in the compatibility layer, in the order they ought to appear in the generated file.
+  public var inits: [InitSignature] = []
+
+  /// Is there anything whatsoever that we ought to generate?
+  public var isEmpty: Bool {
+    return vars.isEmpty && inits.isEmpty
+  }
+
+  fileprivate struct Builder {
+    /// Properties that are needed in the compatibility layer, in the order they ought to appear in the generated file.
+    /// This becomes a property of the `DeprecatedMemberInfo`.
     var vars: [Child] = []
-    var initSignatures: [InitSignature] = []
 
-    // Temporary working state for the loop.
-    var children = initialChildren
-    var knownVars = Set(children)
+    /// Initializer signatures that are needed in the compatibility layer, in the order they ought to appear in the generated file.
+    /// This becomes a property of the `DeprecatedMemberInfo`.
+    var inits: [InitSignature] = []
 
-    func firstIndexOfChild(named targetName: String) -> Int {
-      guard let i = children.firstIndex(where: { $0.name == targetName }) else {
-        fatalError(
-          "couldn't find '\(targetName)' in current children of \(typeName): \(String(reflecting: children.map(\.name)))"
-        )
-      }
-      return i
+    /// Name of the type we're generating a compatibility layer for.
+    private let typeName: String
+
+    /// Are we building a compatibility layer for requirements of a trait? Traits don't have unexpected children or
+    /// initializers.
+    private let areRequirements: Bool
+
+    /// The current set of children after applying all of the change sets ever passed to `applyChangeSet(_:for:historical:)`.
+    /// This is working state.
+    private var children: [Child]
+
+    /// The set of all children that have ever been added to `vars`, plus the ones that were originally present.
+    /// Used to ensure duplicates aren't added to `vars`. This is working state.
+    private var knownVars: Set<Child>
+
+    /// Creates a builder with no deprecated members, but ready to start adding change sets.
+    init(typeName: String, children: [Child], areRequirements: Bool) {
+      self.typeName = typeName
+      self.areRequirements = areRequirements
+
+      self.children = children
+      self.knownVars = Set(children)
     }
 
-    for changeSet in history {
+    /// Creates a `DeprecatedMemberInfo` from all the change sets that have been passed to
+    /// `applyChangeSet(_:for:historical:)`.
+    func make() -> DeprecatedMemberInfo {
+      return DeprecatedMemberInfo(vars: vars, inits: inits)
+    }
+
+    /// Generate the new `vars` and `inits` that are required to maintain compatibility with `changeSet`.
+    ///
+    /// - Parameters:
+    ///   - changeSet: The changes to apply. This type is basically a generic form of `Child.ChangeSet`.
+    ///   - compatibilityLayer: The compatibility layer that these children will ultimately belong to.
+    ///   - historical: Should the children created by this change set be marked historical (and thus be deprecated)?
+    mutating func applyChangeSet(
+      _ changeSet: some RandomAccessCollection<(key: String, value: Child.Refactoring)>,
+      for compatibilityLayer: inout CompatibilityLayer,
+      historical: Bool
+    ) {
       var unexpectedChildrenWithNewNames: Set<Child> = []
 
       // First pass: Apply the changes explicitly specified in the change set.
       for (currentName, refactoring) in changeSet {
         let i = firstIndexOfChild(named: currentName)
 
-        let replacementChildren = replacementChildren(for: children[i], by: refactoring)
+        let replacementChildren = compatibilityLayer.replacementChildren(
+          for: children[i],
+          by: refactoring,
+          historical: historical
+        )
         children.replaceSubrange(i...i, with: replacementChildren)
 
         if !areRequirements {
+          func isDifferent(_ newChild: Child) -> Bool {
+            newChild.isHidden || currentName != newChild.name
+          }
+
           // Mark adjacent unexpected node children whose names have changed too.
-          if currentName != replacementChildren.first?.name {
+          if let firstNewChild = replacementChildren.first, isDifferent(firstNewChild) {
             unexpectedChildrenWithNewNames.insert(children[i - 1])
           }
-          if currentName != replacementChildren.last?.name {
+          if let lastNewChild = replacementChildren.last, isDifferent(lastNewChild) {
             unexpectedChildrenWithNewNames.insert(children[i + replacementChildren.count])
           }
         }
       }
 
       // Second pass: Update unexpected node children adjacent to those changes whose names have probably changed.
-      for unexpectedChild in unexpectedChildrenWithNewNames {
+      for unexpectedChild in unexpectedChildrenWithNewNames where !unexpectedChild.isHidden {
         precondition(unexpectedChild.isUnexpectedNodes)
         let i = firstIndexOfChild(named: unexpectedChild.name)
 
-        let earlier = children[checked: i - 1]
-        let later = children[checked: i + 1]
+        guard i == 0 || !children[i - 1].isHidden else {
+          // Special case: `unexpectedChild` follows a hidden child and should be hidden too.
+          children[i] = Child(makingHistorical: historical, makingHidden: true, newerChildPath: [unexpectedChild])
+          continue
+        }
+
+        // Find nearest expected, non-hidden node before `unexpectedChild`
+        let allEarlier = children.prefix(through: max(i - 1, children.startIndex))
+        let earlier = allEarlier.last { !$0.isHidden && !$0.isUnexpectedNodes }
+
+        // Find nearest expected, non-hidden node after `unexpectedChild`
+        let allLater = children.suffix(from: min(i + 1, children.endIndex))
+        let later = allLater.first { !$0.isHidden && !$0.isUnexpectedNodes }
+
         precondition(!(earlier?.isUnexpectedNodes ?? false) && !(later?.isUnexpectedNodes ?? false))
 
         let newChild = Child(forUnexpectedBetween: earlier, and: later, newerChildPath: [unexpectedChild])
@@ -176,33 +288,23 @@ public struct CompatibilityLayer {
         children[i] = newChild
       }
 
-      // Third pass: Append newly-created children to vars. We do this now so that changes from the first two passes are properly interleaved, preserving source order.
-      vars += children.filter { knownVars.insert($0).inserted }
+      // Third pass: Append newly-created children to vars. We do this now so that changes from the first two passes
+      // are properly interleaved, preserving source order.
+      self.vars += children.filter { !$0.isHidden && knownVars.insert($0).inserted }
 
       // We don't create compatibility layers for protocol requirement inits.
       if !areRequirements {
-        initSignatures.append(InitSignature(children: children))
+        self.inits.append(InitSignature(children: children))
       }
     }
 
-    return DeprecatedMemberInfo(vars: vars, inits: initSignatures)
-  }
-}
-
-/// Describes the deprecated members of a given type that the compatibility layer ought to provide.
-public struct DeprecatedMemberInfo {
-  /// Properties that are needed in the compatibility layer, in the order they ought to appear in the generated file.
-  public var vars: [Child] = []
-
-  /// Initializer signatures that are needed in the compatibility layer, in the order they ought to appear in the generated file.
-  public var inits: [InitSignature] = []
-}
-
-extension Array {
-  /// Returns `nil` if `i` is out of bounds, or the indicated element otherwise.
-  fileprivate subscript(checked i: Index) -> Element? {
-    get {
-      return indices.contains(i) ? self[i] : nil
+    private func firstIndexOfChild(named targetName: String) -> Int {
+      guard let i = children.firstIndex(where: { $0.name == targetName }) else {
+        fatalError(
+          "couldn't find '\(targetName)' in current children of \(typeName): \(String(reflecting: children.map(\.name)))"
+        )
+      }
+      return i
     }
   }
 }