Ensure serially-scheduled tests run in source order. (#569)

This PR ensures that tests, when scheduled serially, run in declared
source order (or as close to it as suite layout allows.)

When tests are run in parallel, we don't bother to sort them because
they will run non-deterministically anyway. Although we don't explicitly
shuffle them, they are stored in a `Dictionary` and in Swift, hashcodes
are salted with a random value, so their iteration order will change on
every test run. (In other words, they're already effectively shuffled.)

Resolves #566.
Resolves rdar://132285155.

### Checklist:

- [x] Code and documentation should follow the style of the [Style
Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md).
- [x] If public symbols are renamed or modified, DocC references should
be updated.

Author: grynspan

Sources/Testing/Running/Runner.swift

@@ -222,6 +222,22 @@ extension Runner {
     }
   }
 
+  /// Get the source location corresponding to the root of a plan step graph.
+  ///
+  /// - Parameters:
+  ///   - stepGraph: The plan step graph whose root node is of interest.
+  ///
+  /// - Returns: The source location of the root node of `stepGraph`, or of the
+  ///   first descendant node thereof (sorted by source location.)
+  private func _sourceLocation(of stepGraph: Graph<String, Plan.Step?>) -> SourceLocation? {
+    if let result = stepGraph.value?.test.sourceLocation {
+      return result
+    }
+    return stepGraph.children.lazy
+      .compactMap { _sourceLocation(of: $0.value) }
+      .min()
+  }
+
   /// Recursively run the tests that are children of a given plan step.
   ///
   /// - Parameters:
@@ -243,8 +259,36 @@ extension Runner {
     // runnable steps' options.
     let stepOrAncestor = stepGraph.value ?? lastAncestorStep
 
-    // Run the children of this step.
-    let childGraphs = stepGraph.children.sorted { $0.key < $1.key }
+    let isParallelizationEnabled = stepOrAncestor?.action.isParallelizationEnabled ?? configuration.isParallelizationEnabled
+    let childGraphs = if isParallelizationEnabled {
+      // Explicitly shuffle the steps to help detect accidental dependencies
+      // between tests due to their ordering.
+      Array(stepGraph.children)
+    } else {
+      // Sort the children by source order. If a child node is empty but has
+      // descendants, the lowest-ordered child node is used. If a child node is
+      // empty and has no descendant nodes with source location information,
+      // then we sort it before nodes with source location information (though
+      // it won't end up doing anything in the test run.)
+      //
+      // FIXME: this operation is likely O(n log n) or worse when amortized
+      // across the entire test plan; Graph should adopt OrderedDictionary if
+      // possible so it can pre-sort its nodes once.
+      stepGraph.children.sorted { lhs, rhs in
+        switch (_sourceLocation(of: lhs.value), _sourceLocation(of: rhs.value)) {
+        case let (.some(lhs), .some(rhs)):
+          lhs < rhs
+        case (.some, _):
+          false // x < nil == false
+        case (_, .some):
+          true // nil < x == true
+        default:
+          false // stable ordering
+        }
+      }
+    }
+
+    // Run the child nodes.
     try await _forEach(in: childGraphs, for: stepOrAncestor) { _, childGraph in
       try await _runStep(atRootOf: childGraph, depth: depth + 1, lastAncestorStep: stepOrAncestor)
     }

Tests/TestingTests/RunnerTests.swift

@@ -878,5 +878,41 @@ final class RunnerTests: XCTestCase {
     await runTest(for: DeprecatedVersionTests.self, configuration: configuration)
     await fulfillment(of: [testStarted, testSkipped], timeout: 0.0)
   }
+
+  func testSerializedSortOrder() async {
+    OrderedTests.state.withLock { state in
+      state = 0
+    }
+    await runTest(for: OrderedTests.self, configuration: .init())
+  }
+}
+
+// MARK: - Fixtures
+
+extension OrderedTests.Inner {
+  @Test(.hidden) func s() { XCTAssertEqual(OrderedTests.state.increment(), 5) }
+}
+
+@Suite(.hidden, .serialized) struct OrderedTests {
+  static let state = Locked(rawValue: 0)
+
+  @Test(.hidden) func z() { XCTAssertEqual(Self.state.increment(), 1) }
+  @Test(.hidden) func y() { XCTAssertEqual(Self.state.increment(), 2) }
+  @Test(.hidden) func x() { XCTAssertEqual(Self.state.increment(), 3) }
+  @Test(.hidden) func w() { XCTAssertEqual(Self.state.increment(), 4) }
+  @Suite(.hidden) struct Inner {
+    // s() in extension above, numbered 5
+    @Test(.hidden) func t() { XCTAssertEqual(OrderedTests.state.increment(), 6) }
+    // u() in extension below, numbered 7
+  }
+
+  @Test(.hidden) func d() { XCTAssertEqual(Self.state.increment(), 8) }
+  @Test(.hidden) func c() { XCTAssertEqual(Self.state.increment(), 9) }
+  @Test(.hidden) func b() { XCTAssertEqual(Self.state.increment(), 10) }
+  @Test(.hidden) func a() { XCTAssertEqual(Self.state.increment(), 11) }
+}
+
+extension OrderedTests.Inner {
+  @Test(.hidden) func u() { XCTAssertEqual(OrderedTests.state.increment(), 7) }
 }
 #endif