Merge pull request #1382 from ahoppen/stream-index-log

Instead of sending a message to the index log when an indexing task finishes, stream results as they come in

Author: ahoppen

Sources/CAtomics/include/CAtomics.h

@@ -70,7 +70,7 @@ typedef struct {
 } AtomicUInt32;
 
 __attribute__((swift_name("AtomicUInt32.init(initialValue:)")))
-static inline AtomicUInt32 atomic_int_create(uint8_t initialValue) {
+static inline AtomicUInt32 atomic_int_create(uint32_t initialValue) {
   AtomicUInt32 atomic;
   atomic.value = initialValue;
   return atomic;

Sources/SKCore/BuildServerBuildSystem.swift

@@ -293,7 +293,7 @@ extension BuildServerBuildSystem: BuildSystem {
 
   public func prepare(
     targets: [ConfiguredTarget],
-    indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void
+    logMessageToIndexLog: @Sendable (_ taskID: IndexTaskID, _ message: String) -> Void
   ) async throws {
     throw PrepareNotSupportedError()
   }

Sources/SKCore/BuildSystem.swift

@@ -169,7 +169,7 @@ public protocol BuildSystem: AnyObject, Sendable {
   /// dependencies.
   func prepare(
     targets: [ConfiguredTarget],
-    indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void
+    logMessageToIndexLog: @escaping @Sendable (_ taskID: IndexTaskID, _ message: String) -> Void
   ) async throws
 
   /// If the build system has knowledge about the language that this document should be compiled in, return it.

Sources/SKCore/BuildSystemManager.swift

@@ -237,9 +237,9 @@ extension BuildSystemManager {
 
   public func prepare(
     targets: [ConfiguredTarget],
-    indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void
+    logMessageToIndexLog: @escaping @Sendable (_ taskID: IndexTaskID, _ message: String) -> Void
   ) async throws {
-    try await buildSystem?.prepare(targets: targets, indexProcessDidProduceResult: indexProcessDidProduceResult)
+    try await buildSystem?.prepare(targets: targets, logMessageToIndexLog: logMessageToIndexLog)
   }
 
   public func registerForChangeNotifications(for uri: DocumentURI, language: Language) async {

Sources/SKCore/CMakeLists.txt

@@ -11,7 +11,7 @@ add_library(SKCore STATIC
   ExperimentalFeatures.swift
   FallbackBuildSystem.swift
   FileBuildSettings.swift
-  IndexProcessResult.swift
+  IndexTaskID.swift
   MainFilesProvider.swift
   PathPrefixMapping.swift
   SplitShellCommand.swift

Sources/SKCore/CompilationDatabaseBuildSystem.swift

@@ -125,7 +125,7 @@ extension CompilationDatabaseBuildSystem: BuildSystem {
 
   public func prepare(
     targets: [ConfiguredTarget],
-    indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void
+    logMessageToIndexLog: @Sendable (_ taskID: IndexTaskID, _ message: String) -> Void
   ) async throws {
     throw PrepareNotSupportedError()
   }

Sources/SKCore/IndexProcessResult.swift

@@ -0,0 +1,46 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import struct TSCBasic.ProcessResult
+
+/// The ID of a preparation or update indexstore task. This allows us to log messages from multiple concurrently running
+/// indexing tasks to the index log while still being able to differentiate them.
+public enum IndexTaskID: Sendable {
+  case preparation(id: UInt32)
+  case updateIndexStore(id: UInt32)
+
+  private static func numberToEmojis(_ number: Int, numEmojis: Int) -> String {
+    let emojis = ["🟥", "🟩", "🟦", "🟧", "⬜️", "🟪", "⬛️", "🟨", "🟫"]
+    var number = abs(number)
+    var result = ""
+    for _ in 0..<numEmojis {
+      let (quotient, remainder) = number.quotientAndRemainder(dividingBy: emojis.count)
+      result += emojis[remainder]
+      number = quotient
+    }
+    return result
+  }
+
+  /// Returns a two-character emoji string that allows easy differentiation between different task IDs.
+  ///
+  /// This marker is prepended to every line in the index log.
+  public var emojiRepresentation: String {
+    // Multiply by 2 and optionally add 1 to make sure preparation and update index store have distinct IDs.
+    // Run .hashValue to make sure we semi-randomly pick new emoji markers for new tasks
+    switch self {
+    case .preparation(id: let id):
+      return Self.numberToEmojis((id * 2).hashValue, numEmojis: 2)
+    case .updateIndexStore(id: let id):
+      return Self.numberToEmojis((id * 2 + 1).hashValue, numEmojis: 2)
+    }
+  }
+}

Sources/SKCore/IndexTaskID.swift

@@ -11,6 +11,7 @@ add_library(SKSupport STATIC
   DocumentURI+CustomLogStringConvertible.swift
   FileSystem.swift
   LineTable.swift
+  PipeAsStringHandler.swift
   Process+LaunchWithWorkingDirectoryIfPossible.swift
   Process+WaitUntilExitWithCancellation.swift
   Random.swift

Sources/SKSupport/CMakeLists.txt

@@ -0,0 +1,49 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Foundation
+
+/// Gathers data from a stdout or stderr pipe. When it has accumulated a full line, calls the handler to handle the
+/// string.
+public actor PipeAsStringHandler {
+  /// Queue on which all data from the pipe will be handled. This allows us to have a
+  /// nonisolated `handle` function but ensure that data gets processed in order.
+  private let queue = AsyncQueue<Serial>()
+  private var buffer = Data()
+
+  /// The closure that actually handles
+  private let handler: @Sendable (String) -> Void
+
+  public init(handler: @escaping @Sendable (String) -> Void) {
+    self.handler = handler
+  }
+
+  private func handleDataFromPipeImpl(_ newData: Data) {
+    self.buffer += newData
+    while let newlineIndex = self.buffer.firstIndex(of: UInt8(ascii: "\n")) {
+      // Output a separate log message for every line in the pipe.
+      // The reason why we don't output multiple lines in a single log message is that
+      //  a) os_log truncates log messages at about 1000 bytes. The assumption is that a single line is usually less
+      //     than 1000 bytes long but if we merge multiple lines into one message, we might easily exceed this limit.
+      //  b) It might be confusing why sometimes a single log message contains one line while sometimes it contains
+      //     multiple.
+      handler(String(data: self.buffer[...newlineIndex], encoding: .utf8) ?? "<invalid UTF-8>")
+      buffer = buffer[buffer.index(after: newlineIndex)...]
+    }
+  }
+
+  public nonisolated func handleDataFromPipe(_ newData: Data) {
+    queue.async {
+      await self.handleDataFromPipeImpl(newData)
+    }
+  }
+}