[K/N] Make runtime .bc builds reproducible ^KT-81501

    * Make distro build depend on current host, so that runtime modules
      would not be reused accross different OS builds
    * Strip user-specific prefixes from absolute paths using
      `-ffile-preifx-map`
    * Sort llvm-link inputs


Merge-request: KT-MR-24296
Merged-by: Alexey Glushko <[email protected]>

Author: haitaka

kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/PlatformManagerProvider.kt

@@ -16,6 +16,7 @@ import org.gradle.api.tasks.PathSensitive
 import org.gradle.api.tasks.PathSensitivity
 import org.gradle.kotlin.dsl.newInstance
 import org.jetbrains.kotlin.konan.target.Distribution
+import org.jetbrains.kotlin.konan.target.HostManager
 import org.jetbrains.kotlin.konan.target.PlatformManager
 import org.jetbrains.kotlin.nativeDistribution.asProperties
 import org.jetbrains.kotlin.nativeDistribution.llvmDistributionSource
@@ -46,6 +47,16 @@ open class PlatformManagerProvider @Inject constructor(
     @get:Input
     val konanPropertiesOverride: Map<String, String> = project.llvmDistributionSource.asProperties
 
+    /**
+     * [PlatformManager] may depend on the current host, so the current host must be an input
+     *
+     * One example is `llvm`: we do not guarantee that llvm version is the same between Linux and macOS.
+     * There could be other problems as well.
+     */
+    @get:Input
+    @Suppress("UNUSED") // used by Gradle via reflection
+    protected val currentHost = HostManager.host
+
     @get:Input
     @get:Optional
     protected val konanDataDir = providerFactory.gradleProperty("konan.data.dir")

kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/bitcode/CompileToBitcodePlugin.kt

@@ -33,6 +33,7 @@ import org.jetbrains.kotlin.nativeDistribution.nativeProtoDistribution
 import org.jetbrains.kotlin.resolveLlvmUtility
 import org.jetbrains.kotlin.testing.native.GoogleTestExtension
 import org.jetbrains.kotlin.utils.capitalized
+import java.io.File
 import java.time.Duration
 import javax.inject.Inject
 
@@ -211,6 +212,12 @@ open class CompileToBitcodeExtension @Inject constructor(val project: Project) :
         private val platformManager = project.extensions.getByType<PlatformManager>()
         private val nativeDependencies = project.extensions.getByType<NativeDependenciesExtension>()
 
+        private val reproducibilityRootsMap: Map<File, String>
+            get() = mapOf(
+                    // This is the common root for native dependencies: sysroots, llvm, ...
+                    nativeDependencies.nativeDependenciesRoot to "NATIVE_DEPS",
+            )
+
         private val allCompilerArgs = module.compilerArgs.map {
             it + when (sanitizer) {
                 null -> emptyList()
@@ -233,6 +240,7 @@ open class CompileToBitcodeExtension @Inject constructor(val project: Project) :
             this.headersDirs.from(this@SourceSet.headersDirs)
             this.inputFiles.from(this@SourceSet.inputFiles)
             this.workingDirectory.set(module.compilerWorkingDirectory)
+            this.reproducibilityRootsMap.set(this@SourceSet.reproducibilityRootsMap)
             dependsOn(nativeDependencies.llvmDependency)
             dependsOn(nativeDependencies.targetDependency(_target))
             dependsOn(this@SourceSet.dependencies)
@@ -255,7 +263,7 @@ open class CompileToBitcodeExtension @Inject constructor(val project: Project) :
                         from(this@SourceSet.inputFiles.dir)
                         from(this@SourceSet.headersDirs)
                     }
-                    arguments.addAll(ClangFrontend.defaultCompilerFlags(headers))
+                    arguments.addAll(ClangFrontend.defaultCompilerFlags(headers, reproducibilityRootsMap))
                     arguments.addAll(allCompilerArgs)
                     arguments.addAll(platformManager.clangArgs(target, module.compiler.get()))
                     output.set(this@SourceSet.outputDirectory.map { it.asFile.absolutePath })
@@ -279,6 +287,7 @@ open class CompileToBitcodeExtension @Inject constructor(val project: Project) :
             this.inputFiles.from(compileTask)
             this.outputFile.set(this@SourceSet.outputFile)
             this.arguments.set(module.linkerArgs)
+            this.reproducibilityRootsMap.set(this@SourceSet.reproducibilityRootsMap)
             val specs = this@SourceSet.onlyIf
             val target = target
             onlyIf {

kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/cpp/ClangFrontend.kt

@@ -10,6 +10,7 @@ import org.gradle.api.file.*
 import org.gradle.api.internal.file.FileOperations
 import org.gradle.api.model.ObjectFactory
 import org.gradle.api.provider.ListProperty
+import org.gradle.api.provider.MapProperty
 import org.gradle.api.provider.Property
 import org.gradle.api.tasks.*
 import org.gradle.process.ExecOperations
@@ -123,6 +124,9 @@ open class ClangFrontend @Inject constructor(
     @get:Internal("Used to compute workUnits and headersPathsRelativeToWorkingDir")
     val workingDirectory: DirectoryProperty = objects.directoryProperty()
 
+    @get:Internal
+    val reproducibilityRootsMap: MapProperty<File, String> = objects.mapProperty(File::class.java, String::class.java)
+
     /**
      * Locations to search for headers.
      *
@@ -165,7 +169,7 @@ open class ClangFrontend @Inject constructor(
                 inputPathRelativeToWorkingDir.set(workUnit.inputPathRelativeToWorkingDir)
                 outputFile.set(workUnit.outputFile)
                 compilerExecutable.set(this@ClangFrontend.compiler)
-                arguments.set(defaultCompilerFlags(this@ClangFrontend.headersDirs))
+                arguments.set(defaultCompilerFlags(this@ClangFrontend.headersDirs, this@ClangFrontend.reproducibilityRootsMap.get()))
                 arguments.addAll(compilerSpecificArgs)
                 arguments.addAll(this@ClangFrontend.arguments)
                 this.platformManager.set(platformManager)
@@ -175,10 +179,14 @@ open class ClangFrontend @Inject constructor(
     }
 
     companion object {
-        internal fun defaultCompilerFlags(headersDirs: CppHeadersSet): List<String> = buildList {
+        internal fun defaultCompilerFlags(headersDirs: CppHeadersSet, reproducibilityRootsMap: Map<File, String>): List<String> = buildList {
             add("-c")
             add("-emit-llvm")
             addAll(headersDirs.asCompilerArguments.get())
+            // Prevent generated binaries from containing absolute paths
+            addAll(reproducibilityRootsMap.map {
+                "-ffile-prefix-map=${it.key}=${it.value}"
+            })
         }
     }
 }

kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/cpp/LlvmLink.kt

@@ -10,10 +10,12 @@ import org.gradle.api.file.ConfigurableFileCollection
 import org.gradle.api.file.RegularFileProperty
 import org.gradle.api.model.ObjectFactory
 import org.gradle.api.provider.ListProperty
+import org.gradle.api.provider.MapProperty
 import org.gradle.api.provider.Property
 import org.gradle.api.tasks.CacheableTask
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.InputFiles
+import org.gradle.api.tasks.Internal
 import org.gradle.api.tasks.Nested
 import org.gradle.api.tasks.OutputFile
 import org.gradle.api.tasks.PathSensitive
@@ -26,6 +28,8 @@ import org.gradle.workers.WorkerExecutor
 import org.jetbrains.kotlin.execLlvmUtility
 import org.jetbrains.kotlin.konan.target.PlatformManager
 import org.jetbrains.kotlin.platformManagerProvider
+import org.jetbrains.kotlin.utils.reproduciblySortedFilePaths
+import java.io.File
 import javax.inject.Inject
 
 private abstract class LlvmLinkJob : WorkAction<LlvmLinkJob.Parameters> {
@@ -34,6 +38,7 @@ private abstract class LlvmLinkJob : WorkAction<LlvmLinkJob.Parameters> {
         val outputFile: RegularFileProperty
         val arguments: ListProperty<String>
         val platformManager: Property<PlatformManager>
+        val reproducibilityRootsMap: MapProperty<File, String>
     }
 
     @get:Inject
@@ -42,7 +47,8 @@ private abstract class LlvmLinkJob : WorkAction<LlvmLinkJob.Parameters> {
     override fun execute() {
         with(parameters) {
             execOperations.execLlvmUtility(platformManager.get(), "llvm-link") {
-                args = listOf("-o", outputFile.asFile.get().absolutePath) + arguments.get() + inputFiles.map { it.absolutePath }
+                val sortedInputs = inputFiles.reproduciblySortedFilePaths(reproducibilityRootsMap.get()).map { it.absolutePath }
+                args = listOf("-o", outputFile.asFile.get().absolutePath) + arguments.get() + sortedInputs
             }
         }
     }
@@ -75,6 +81,9 @@ open class LlvmLink @Inject constructor(
     @get:Input
     val arguments: ListProperty<String> = objectFactory.listProperty(String::class.java)
 
+    @get:Internal
+    val reproducibilityRootsMap: MapProperty<File, String> = objectFactory.mapProperty(File::class.java, String::class.java)
+
     @get:Nested
     protected val platformManagerProvider = objectFactory.platformManagerProvider(project)
 
@@ -87,6 +96,7 @@ open class LlvmLink @Inject constructor(
             outputFile.set(this@LlvmLink.outputFile)
             arguments.set(this@LlvmLink.arguments)
             platformManager.set(this@LlvmLink.platformManagerProvider.platformManager)
+            reproducibilityRootsMap.set(this@LlvmLink.reproducibilityRootsMap)
         }
     }
 }

kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/tools/NativePlugin.kt

@@ -18,6 +18,7 @@ import org.jetbrains.kotlin.dependencies.NativeDependenciesExtension
 import org.jetbrains.kotlin.dependencies.NativeDependenciesPlugin
 import org.jetbrains.kotlin.konan.target.HostManager.Companion.hostIsMac
 import org.jetbrains.kotlin.konan.target.HostManager.Companion.hostIsMingw
+import org.jetbrains.kotlin.utils.reproduciblySortedFilePaths
 import java.io.File
 import javax.inject.Inject
 import kotlin.collections.List
@@ -232,17 +233,8 @@ open class NativeToolsExtension(val project: Project) {
     /**
      * Whenever a `FileCollection` is passed as arguments, it's order must be stable sorted for reproducibility.
      */
-    fun reproduciblySortedFilePaths(fileCollection: FileCollection): List<File> = fileCollection.files.map { file ->
-        // We cannot just sort absolute paths: they may change from machine to machine.
-        // So, apply `reproducbilityCompilerFlags` to the list of files manually and use the resulting paths as keys.
-        reproducibilityRootsMap.firstNotNullOfOrNull { (root, name) ->
-            if (file.startsWith(root)) {
-                "$name${File.separator}${file.toRelativeString(root)}" to file
-            } else {
-                null
-            }
-        } ?: (file.absolutePath to file) // TODO: consider erroring out instead
-    }.sortedBy { it.first }.map { it.second }
+    fun reproduciblySortedFilePaths(fileCollection: FileCollection): List<File> =
+            fileCollection.reproduciblySortedFilePaths(reproducibilityRootsMap)
 
     val sourceSets = SourceSets(project, this, mutableMapOf<String, SourceSet>())
     val toolPatterns = ToolConfigurationPatterns(this, mutableMapOf<Pair<String, String>, ToolPatternConfiguration>())

kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/utils/FileCollection.kt

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2010-2025 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.utils
+
+import org.gradle.api.file.FileCollection
+import java.io.File
+
+
+/**
+ * Whenever a `FileCollection` is passed as arguments, it's order must be stably sorted for reproducibility.
+ * @param reproducibilityRootsMap Map of volatile path prefixes to the replacements to be used instead.
+ */
+fun FileCollection.reproduciblySortedFilePaths(reproducibilityRootsMap: Map<File, String>): List<File> = files.map { file ->
+    // We cannot just sort absolute paths: they may change from machine to machine.
+    // So, apply `reproducibilityCompilerFlags` to the list of files manually and use the resulting paths as keys.
+    reproducibilityRootsMap.firstNotNullOfOrNull { (root, name) ->
+        if (file.startsWith(root)) {
+            "$name${File.separator}${file.toRelativeString(root)}" to file
+        } else if (file.isAbsolute) {
+            file.path to file
+        } else {
+            null
+        }
+    } ?: (file.absolutePath to file) // TODO: consider erroring out instead
+}.sortedBy { it.first }.map { it.second }