Merge pull request swiftlang#81706 from eeckstein/mandatory-temprvalue-elimination

Optimizer: Improve performance of large InlineArrays at Onone

Author: eeckstein

SwiftCompilerSources/Sources/Optimizer/Analysis/AliasAnalysis.swift

@@ -622,6 +622,14 @@ private enum ImmutableScope {
       if beginAccess.isUnsafe {
         return nil
       }
+
+      // This is a workaround for a bug in the move-only checker: rdar://151841926.
+      // The move-only checker sometimes inserts destroy_addr within read-only static access scopes.
+      // TODO: remove this once the bug is fixed.
+      if beginAccess.isStatic {
+        return nil
+      }
+
       switch beginAccess.accessKind {
       case .read:
         self = .readAccess(beginAccess)

SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LifetimeDependenceDiagnostics.swift

@@ -69,7 +69,7 @@ let lifetimeDependenceDiagnosticsPass = FunctionPass(
       // mark_dependence will hanceforth be treated as an unknown use by the optimizer.  In the future, we should not
       // need to set this flag during diagnostics because, for escapable types, mark_dependence [unresolved] will all be
       // settled during an early LifetimeNormalization pass.
-      markDep.settleToEscaping()
+      markDep.settleToEscaping(context)
       continue
     }
     if let apply = instruction as? FullApplySite {

SwiftCompilerSources/Sources/Optimizer/FunctionPasses/TempRValueElimination.swift

@@ -36,7 +36,15 @@ import SIL
 ///
 let tempRValueElimination = FunctionPass(name: "temp-rvalue-elimination") {
   (function: Function, context: FunctionPassContext) in
+  removeTempRValues(in: function, keepDebugInfo: false, context)
+}
 
+let mandatoryTempRValueElimination = FunctionPass(name: "mandatory-temp-rvalue-elimination") {
+  (function: Function, context: FunctionPassContext) in
+  removeTempRValues(in: function, keepDebugInfo: true, context)
+}
+
+private func removeTempRValues(in function: Function, keepDebugInfo: Bool, _ context: FunctionPassContext) {
   for inst in function.instructions {
     switch inst {
     case let copy as CopyAddrInst:
@@ -45,12 +53,12 @@ let tempRValueElimination = FunctionPass(name: "temp-rvalue-elimination") {
         // copied the `alloc_stack` back to the source location.
         context.erase(instruction: copy)
       } else {
-        tryEliminate(copy: copy, context)
+        tryEliminate(copy: copy, keepDebugInfo: keepDebugInfo, context)
       }
     case let store as StoreInst:
       // Also handle `load`-`store` pairs which are basically the same thing as a `copy_addr`.
       if let load = store.source as? LoadInst, load.uses.isSingleUse, load.parentBlock == store.parentBlock {
-        tryEliminate(copy: store, context)
+        tryEliminate(copy: store, keepDebugInfo: keepDebugInfo, context)
       }
     default:
       break
@@ -82,9 +90,10 @@ extension StoreInst: CopyLikeInstruction {
   private var load: LoadInst { source as! LoadInst }
 }
 
-private func tryEliminate(copy: CopyLikeInstruction, _ context: FunctionPassContext) {
+private func tryEliminate(copy: CopyLikeInstruction, keepDebugInfo: Bool, _ context: FunctionPassContext) {
 
-  guard let (allocStack, lastUseOfAllocStack) = getRemovableAllocStackDestination(of: copy, context) else {
+  guard let (allocStack, lastUseOfAllocStack) =
+          getRemovableAllocStackDestination(of: copy, keepDebugInfo: keepDebugInfo, context) else {
     return
   }
 
@@ -127,6 +136,9 @@ private func tryEliminate(copy: CopyLikeInstruction, _ context: FunctionPassCont
     }
   }
   context.erase(instruction: allocStack)
+  if keepDebugInfo {
+    Builder(before: copy, context).createDebugStep()
+  }
   context.erase(instructionIncludingAllUsers: copy.loadingInstruction)
 }
 
@@ -138,18 +150,24 @@ private func tryEliminate(copy: CopyLikeInstruction, _ context: FunctionPassCont
 ///   %lastUseOfAllocStack = load %allocStack
 /// ```
 private func getRemovableAllocStackDestination(
-  of copy: CopyLikeInstruction, _ context: FunctionPassContext
+  of copy: CopyLikeInstruction, keepDebugInfo: Bool, _ context: FunctionPassContext
 ) -> (allocStack: AllocStackInst, lastUseOfAllocStack: Instruction)? {
   guard copy.isInitializationOfDestination,
         let allocStack = copy.destinationAddress as? AllocStackInst
   else {
     return nil
   }
 
-  // If the `allocStack` is lexical we can eliminate it if the source of the copy is lexical and
-  // it is live for longer than the `allocStack`.
-  if allocStack.isLexical && !copy.sourceAddress.accessBase.storageIsLexical {
-    return nil
+  if keepDebugInfo {
+    if allocStack.isFromVarDecl || allocStack.isLexical {
+      return nil
+    }
+  } else {
+    // If the `allocStack` is lexical we can eliminate it if the source of the copy is lexical and
+    // it is live for longer than the `allocStack`.
+    if allocStack.isLexical && !copy.sourceAddress.accessBase.storageIsLexical {
+      return nil
+    }
   }
 
   var allocStackUses = UseCollector(copy: copy, context)

SwiftCompilerSources/Sources/Optimizer/InstructionSimplification/CMakeLists.txt

@@ -29,6 +29,7 @@ swift_compiler_sources(Optimizer
   SimplifyInitEnumDataAddr.swift
   SimplifyKeyPath.swift
   SimplifyLoad.swift
+  SimplifyMarkDependence.swift
   SimplifyMisc.swift
   SimplifyPartialApply.swift
   SimplifyPointerToAddress.swift

SwiftCompilerSources/Sources/Optimizer/InstructionSimplification/SimplifyMarkDependence.swift

@@ -0,0 +1,112 @@
+//===--- SimplifyMarkDependence.swift -------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2025 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import SIL
+
+// Note: this simplification cannot run before dependency diagnostics.
+// See `var isRedundant` below.
+
+extension MarkDependenceInst : OnoneSimplifiable, SILCombineSimplifiable {
+  func simplify(_ context: SimplifyContext) {
+    if isRedundant ||
+       // A literal lives forever, so no mark_dependence is needed.
+       // This pattern can occur after StringOptimization when a utf8CString of a literal is replace
+       // by the string_literal itself.
+       value.isLiteral
+    {
+      replace(with: value, context)
+      return
+    }
+    simplifyBaseOperand(context)
+  }
+}
+
+extension MarkDependenceAddrInst : OnoneSimplifiable, SILCombineSimplifiable {
+  func simplify(_ context: SimplifyContext) {
+    if isRedundant {
+      context.erase(instruction: self)
+      return
+    }
+    simplifyBaseOperand(context)
+  }
+}
+
+private extension MarkDependenceInstruction {
+  var isRedundant: Bool {
+    if base.type.isObject && base.type.isTrivial(in: base.parentFunction) {
+      // Sometimes due to specialization/builtins, we can get a mark_dependence whose base is a trivial
+      // typed object. Trivial values live forever. Therefore the mark_dependence does not have a meaning.
+      // Note: the mark_dependence is still needed for lifetime diagnostics. So it's important that this
+      //       simplification does not run before the lifetime diagnostic pass.
+      return true
+    }
+    // If the value is an address projection from the base the mark_dependence is not needed because the
+    // base cannot be destroyed before the accessing the value, anyway.
+    if valueOrAddress.type.isAddress, base.type.isAddress,
+       // But we still need to keep the mark_dependence for non-escapable types because a non-escapable
+       // value can be copied and copies must not outlive the base.
+       valueOrAddress.type.isEscapable(in: parentFunction),
+       base.accessPath.isEqualOrContains(valueOrAddress.accessPath)
+    {
+      return true
+    }
+    return false
+  }
+
+  func simplifyBaseOperand(_ context: SimplifyContext) {
+    /// In OSSA, the `base` is a borrow introducing operand. It is pretty complicated to change the base.
+    /// So, for simplicity, we only do this optimization when OSSA is already lowered.
+    if parentFunction.hasOwnership {
+      return
+    }
+    // Replace the base operand with the operand of the base value if it's a certain kind of forwarding
+    // instruction.
+    let rootBase = base.lookThroughEnumAndExistentialRef
+    if rootBase != base {
+      baseOperand.set(to: rootBase, context)
+    }
+  }
+}
+
+private extension Value {
+  /// True, if this is a literal instruction or a struct of a literal instruction.
+  /// What we want to catch here is a `UnsafePointer<Int8>` of a string literal.
+  var isLiteral: Bool {
+    switch self {
+    case let s as StructInst:
+      if let singleOperand = s.operands.singleElement {
+        return singleOperand.value.isLiteral
+      }
+      return false
+    case  is IntegerLiteralInst, is FloatLiteralInst, is StringLiteralInst:
+      return true
+    default:
+      return false
+    }
+  }
+
+  var lookThroughEnumAndExistentialRef: Value {
+    switch self {
+    case let e as EnumInst:
+      if let payload = e.payload {
+        return payload.lookThroughEnumAndExistentialRef
+      }
+      return self
+    case let ier as InitExistentialRefInst:
+      return ier.instance.lookThroughEnumAndExistentialRef
+    case let oer as OpenExistentialRefInst:
+      return oer.existential.lookThroughEnumAndExistentialRef
+    default:
+      return self
+    }
+  }
+}

SwiftCompilerSources/Sources/Optimizer/PassManager/Context.swift

@@ -776,6 +776,20 @@ extension CopyAddrInst {
   }
 }
 
+extension MarkDependenceInstruction {
+  func resolveToNonEscaping(_ context: some MutatingContext) {
+    context.notifyInstructionsChanged()
+    bridged.MarkDependenceInstruction_resolveToNonEscaping()
+    context.notifyInstructionChanged(self)
+  }
+
+  func settleToEscaping(_ context: some MutatingContext) {
+    context.notifyInstructionsChanged()
+    bridged.MarkDependenceInstruction_settleToEscaping()
+    context.notifyInstructionChanged(self)
+  }
+}
+
 extension TermInst {
   func replaceBranchTarget(from fromBlock: BasicBlock, to toBlock: BasicBlock, _ context: some MutatingContext) {
     context.notifyBranchesChanged()

SwiftCompilerSources/Sources/Optimizer/PassManager/PassRegistration.swift

@@ -101,6 +101,7 @@ private func registerSwiftPasses() {
   registerPass(lifetimeDependenceScopeFixupPass, { lifetimeDependenceScopeFixupPass.run($0) })
   registerPass(copyToBorrowOptimization, { copyToBorrowOptimization.run($0) })
   registerPass(tempRValueElimination, { tempRValueElimination.run($0) })
+  registerPass(mandatoryTempRValueElimination, { mandatoryTempRValueElimination.run($0) })
   registerPass(generalClosureSpecialization, { generalClosureSpecialization.run($0) })
   registerPass(autodiffClosureSpecialization, { autodiffClosureSpecialization.run($0) })
 
@@ -122,6 +123,8 @@ private func registerSwiftPasses() {
   registerForSILCombine(DestructureTupleInst.self, { run(DestructureTupleInst.self, $0) })
   registerForSILCombine(TypeValueInst.self, { run(TypeValueInst.self, $0) })
   registerForSILCombine(ClassifyBridgeObjectInst.self, { run(ClassifyBridgeObjectInst.self, $0) })
+  registerForSILCombine(MarkDependenceInst.self,    { run(MarkDependenceInst.self, $0) })
+  registerForSILCombine(MarkDependenceAddrInst.self, { run(MarkDependenceAddrInst.self, $0) })
   registerForSILCombine(PointerToAddressInst.self,  { run(PointerToAddressInst.self, $0) })
   registerForSILCombine(UncheckedEnumDataInst.self, { run(UncheckedEnumDataInst.self, $0) })
   registerForSILCombine(WitnessMethodInst.self,     { run(WitnessMethodInst.self, $0) })

SwiftCompilerSources/Sources/Optimizer/Utilities/LifetimeDependenceUtils.swift

@@ -226,9 +226,9 @@ extension LifetimeDependence {
     return scope.computeRange(context)
   }
 
-  func resolve(_ context: some Context) {
+  func resolve(_ context: some MutatingContext) {
     if let mdi = markDepInst {
-      mdi.resolveToNonEscaping()
+      mdi.resolveToNonEscaping(context)
     }
   }
 }

SwiftCompilerSources/Sources/SIL/Instruction.swift

@@ -1124,58 +1124,39 @@ public enum MarkDependenceKind: Int32 {
 }
 
 public protocol MarkDependenceInstruction: Instruction {
-  var baseOperand: Operand { get }
-  var base: Value { get }
   var dependenceKind: MarkDependenceKind { get }
-  func resolveToNonEscaping()
-  func settleToEscaping()
 }
 
 extension MarkDependenceInstruction {
   public var isNonEscaping: Bool { dependenceKind == .NonEscaping }
   public var isUnresolved: Bool { dependenceKind == .Unresolved }
+
+  public var valueOrAddressOperand: Operand { operands[0] }
+  public var valueOrAddress: Value { valueOrAddressOperand.value }
+  public var baseOperand: Operand { operands[1] }
+  public var base: Value { baseOperand.value }
 }
 
 final public class MarkDependenceInst : SingleValueInstruction, MarkDependenceInstruction {
-  public var valueOperand: Operand { operands[0] }
-  public var baseOperand: Operand { operands[1] }
+  public var valueOperand: Operand { valueOrAddressOperand }
   public var value: Value { return valueOperand.value }
-  public var base: Value { return baseOperand.value }
 
   public var dependenceKind: MarkDependenceKind {
     MarkDependenceKind(rawValue: bridged.MarkDependenceInst_dependenceKind().rawValue)!
   }
 
-  public func resolveToNonEscaping() {
-    bridged.MarkDependenceInst_resolveToNonEscaping()
-  }
-
-  public func settleToEscaping() {
-    bridged.MarkDependenceInst_settleToEscaping()
-  }
-
   public var hasScopedLifetime: Bool {
     return isNonEscaping && type.isObject && ownership == .owned && type.isEscapable(in: parentFunction)
   }
 }
 
 final public class MarkDependenceAddrInst : Instruction, MarkDependenceInstruction {
-  public var addressOperand: Operand { operands[0] }
-  public var baseOperand: Operand { operands[1] }
+  public var addressOperand: Operand { valueOrAddressOperand }
   public var address: Value { return addressOperand.value }
-  public var base: Value { return baseOperand.value }
 
   public var dependenceKind: MarkDependenceKind {
     MarkDependenceKind(rawValue: bridged.MarkDependenceAddrInst_dependenceKind().rawValue)!
   }
-
-  public func resolveToNonEscaping() {
-    bridged.MarkDependenceAddrInst_resolveToNonEscaping()
-  }
-
-  public func settleToEscaping() {
-    bridged.MarkDependenceAddrInst_settleToEscaping()
-  }
 }
 
 final public class RefToBridgeObjectInst : SingleValueInstruction {

include/swift/SIL/SILBridging.h

@@ -803,11 +803,9 @@ struct BridgedInstruction {
   BRIDGED_INLINE SwiftInt StoreInst_getStoreOwnership() const;
   BRIDGED_INLINE SwiftInt AssignInst_getAssignOwnership() const;
   BRIDGED_INLINE MarkDependenceKind MarkDependenceInst_dependenceKind() const;
-  BRIDGED_INLINE void MarkDependenceInst_resolveToNonEscaping() const;
-  BRIDGED_INLINE void MarkDependenceInst_settleToEscaping() const;
+  BRIDGED_INLINE void MarkDependenceInstruction_resolveToNonEscaping() const;
+  BRIDGED_INLINE void MarkDependenceInstruction_settleToEscaping() const;
   BRIDGED_INLINE MarkDependenceKind MarkDependenceAddrInst_dependenceKind() const;
-  BRIDGED_INLINE void MarkDependenceAddrInst_resolveToNonEscaping() const;
-  BRIDGED_INLINE void MarkDependenceAddrInst_settleToEscaping() const;
   BRIDGED_INLINE SwiftInt BeginAccessInst_getAccessKind() const;
   BRIDGED_INLINE bool BeginAccessInst_isStatic() const;
   BRIDGED_INLINE bool BeginAccessInst_isUnsafe() const;