Merge pull request #134 from helmetcrest/add-placeholder-benchmarks

Author: ktoso

.github/workflows/pull_request.yml

@@ -29,7 +29,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Install System Dependencies
-        run: apt-get -qq update && apt-get -qq install -y make curl wget
+        run: apt-get -qq update && apt-get -qq install -y make curl wget libjemalloc2 libjemalloc-dev
       - name: Cache JDK
         id: cache-jdk
         uses: actions/cache@v4
@@ -69,6 +69,9 @@ jobs:
         run: |
           ./gradlew build -x test --no-daemon # just build
           ./gradlew build --info --no-daemon
+      - name: Gradle build (benchmarks)
+        run: |
+          ./gradlew compileJmh --info --no-daemon
 
   test-swift:
     name: Swift tests (swift:${{ matrix.swift_version }} jdk:${{matrix.jdk_vendor}} os:${{ matrix.os_version }})
@@ -86,7 +89,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Install System Dependencies
-        run: apt-get -qq update && apt-get -qq install -y make curl wget
+        run: apt-get -qq update && apt-get -qq install -y make curl wget libjemalloc2 libjemalloc-dev
       - name: Cache JDK
         id: cache-jdk
         uses: actions/cache@v4
@@ -130,3 +133,6 @@ jobs:
       - name: Build (Swift) Sample Apps
         run: |
           find Samples/ -name Package.swift -maxdepth 2 -exec swift build --package-path $(dirname {}) \;;
+      # TODO: Benchmark compile crashes in CI, enable when nightly toolchains in better shape.
+      # - name: Build (Swift) Benchmarks
+      #  run: "swift package --package-path Benchmarks/ benchmark list"

Benchmarks/Benchmarks/JavaApiCallBenchmarks/JavaApiCallBenchmarks.swift

@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2024 Apple Inc. and the Swift.org project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of Swift.org project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import Benchmark
+import Foundation
+import JavaKit
+import JavaKitNetwork
+
+@MainActor let benchmarks = {
+    var jvm: JavaVirtualMachine {
+        get throws {
+            try .shared()
+        }
+    }
+    Benchmark("Simple call to Java library") { benchmark in
+        for _ in benchmark.scaledIterations {
+            let environment = try jvm.environment()
+
+            let urlConnectionClass = try JavaClass<URLConnection>(environment: environment)
+            blackHole(urlConnectionClass.getDefaultAllowUserInteraction())
+        }
+    }
+}

Benchmarks/Package.swift

@@ -0,0 +1,69 @@
+// swift-tools-version: 6.0
+
+import PackageDescription
+
+import class Foundation.FileManager
+import class Foundation.ProcessInfo
+
+// Note: the JAVA_HOME environment variable must be set to point to where
+// Java is installed, e.g.,
+//   Library/Java/JavaVirtualMachines/openjdk-21.jdk/Contents/Home.
+func findJavaHome() -> String {
+  if let home = ProcessInfo.processInfo.environment["JAVA_HOME"] {
+    return home
+  }
+
+  // This is a workaround for envs (some IDEs) which have trouble with
+  // picking up env variables during the build process
+  let path = "\(FileManager.default.homeDirectoryForCurrentUser.path()).java_home"
+  if let home = try? String(contentsOfFile: path, encoding: .utf8) {
+    if let lastChar = home.last, lastChar.isNewline {
+      return String(home.dropLast())
+    }
+
+    return home
+  }
+
+  fatalError("Please set the JAVA_HOME environment variable to point to where Java is installed.")
+}
+let javaHome = findJavaHome()
+
+let javaIncludePath = "\(javaHome)/include"
+#if os(Linux)
+  let javaPlatformIncludePath = "\(javaIncludePath)/linux"
+#elseif os(macOS)
+  let javaPlatformIncludePath = "\(javaIncludePath)/darwin"
+#else
+  // TODO: Handle windows as well
+  #error("Currently only macOS and Linux platforms are supported, this may change in the future.")
+#endif
+
+let package = Package(
+  name: "benchmarks",
+  platforms: [
+    .macOS("15.03")
+  ],
+  dependencies: [
+    .package(path: "../"),
+    .package(url: "https://github.com/ordo-one/package-benchmark", .upToNextMajor(from: "1.4.0")),
+  ],
+  targets: [
+    .executableTarget(
+      name: "JavaApiCallBenchmarks",
+      dependencies: [
+        .product(name: "JavaRuntime", package: "swift-java"),
+        .product(name: "JavaKit", package: "swift-java"),
+        .product(name: "JavaKitNetwork", package: "swift-java"),
+        .product(name: "Benchmark", package: "package-benchmark"),
+      ],
+      path: "Benchmarks/JavaApiCallBenchmarks",
+      swiftSettings: [
+        .unsafeFlags(["-I\(javaIncludePath)", "-I\(javaPlatformIncludePath)"]),
+        .swiftLanguageMode(.v5),
+      ],
+      plugins: [
+        .plugin(name: "BenchmarkPlugin", package: "package-benchmark")
+      ]
+    )
+  ]
+)

Makefile

@@ -133,12 +133,12 @@ jextract-generate: jextract-swift generate-JExtract-interface-files
 	swift run jextract-swift  \
 		--package-name com.example.swift.generated \
 		--swift-module ExampleSwiftLibrary \
-		--output-directory ${SAMPLES_DIR}/SwiftKitSampleApp/src/generated/java \
+		--output-directory ${SAMPLES_DIR}/SwiftKitSampleApp/build/generated/sources/jextract/main \
 		$(BUILD_DIR)/jextract/ExampleSwiftLibrary/MySwiftLibrary.swiftinterface; \
 	swift run jextract-swift \
 		--package-name org.swift.swiftkit.generated \
 		--swift-module SwiftKitSwift \
-		--output-directory ${SAMPLES_DIR}/SwiftKitSampleApp/src/generated/java \
+		--output-directory ${SAMPLES_DIR}/SwiftKitSampleApp/build/generated/sources/jextract/main \
 		$(BUILD_DIR)/jextract/SwiftKitSwift/SwiftKit.swiftinterface
 
 

Package.swift

@@ -94,18 +94,18 @@ let package = Package(
 
     // ==== Plugin for building Java code
     .plugin(
-        name: "JavaCompilerPlugin",
-        targets: [
-            "JavaCompilerPlugin"
-        ]
+      name: "JavaCompilerPlugin",
+      targets: [
+        "JavaCompilerPlugin"
+      ]
     ),
 
     // ==== Plugin for wrapping Java classes in Swift
     .plugin(
-        name: "Java2SwiftPlugin",
-        targets: [
-            "Java2SwiftPlugin"
-        ]
+      name: "Java2SwiftPlugin",
+      targets: [
+        "Java2SwiftPlugin"
+      ]
     ),
 
     // ==== jextract-swift (extract Java accessors from Swift interface files)
@@ -140,6 +140,7 @@ let package = Package(
     .package(url: "https://github.com/swiftlang/swift-syntax.git", branch: "main"),
     .package(url: "https://github.com/apple/swift-argument-parser", from: "1.5.0"),
     .package(url: "https://github.com/apple/swift-collections.git", .upToNextMinor(from: "1.1.0")),
+    .package(url: "https://github.com/ordo-one/package-benchmark", .upToNextMajor(from: "1.4.0")),
   ],
   targets: [
     .macro(
@@ -225,16 +226,16 @@ let package = Package(
       ]
     ),
     .plugin(
-        name: "JavaCompilerPlugin",
-        capability: .buildTool()
+      name: "JavaCompilerPlugin",
+      capability: .buildTool()
     ),
 
     .plugin(
-        name: "Java2SwiftPlugin",
-        capability: .buildTool(),
-        dependencies: [
-          "Java2Swift"
-        ]
+      name: "Java2SwiftPlugin",
+      capability: .buildTool(),
+      dependencies: [
+        "Java2Swift"
+      ]
     ),
 
     .target(
@@ -373,6 +374,6 @@ let package = Package(
         .swiftLanguageMode(.v5),
         .unsafeFlags(["-I\(javaIncludePath)", "-I\(javaPlatformIncludePath)"])
       ]
-    ),
+    )
   ]
 )

README.md

@@ -111,6 +111,28 @@ To run a simple example app showcasing the jextract (Java calling Swift) approac
 This will also generate the necessary sources (by invoking jextract, extracting the `Sources/ExampleSwiftLibrary`) 
 and generating Java sources in `src/generated/java`.
 
+## Benchmarks
+
+You can run Swift [ordo-one/package-benchmark](https://github.com/ordo-one/package-benchmark) and OpenJDK [JMH](https://github.com/openjdk/jmh) benchmarks in this project.
+
+Swift benchmarks are located under `Benchmarks/` and JMH benchmarks are currently part of the SwiftKit sample project: `Samples/SwiftKitSampleApp/src/jmh` because they depend on generated sources from the sample.
+
+To run **Swift benchmarks** you can:
+
+```bash
+cd Benchmarks
+swift package benchmark
+```
+
+In order to run JMH benchmarks you can:
+
+```bash
+cd Samples/SwiftKitSampleApp
+gradle jmh
+```
+
+Please read documentation of both performance testing tools and understand that results must be interpreted and not just taken at face value. Benchmarking is tricky and environment sensitive task, so please be careful when constructing and reading benchmarks and their results. If in doubt, please reach out on the forums.
+
 ## User Guide
 
 More details about the project and how it can be used are available in [USER_GUIDE.md](USER_GUIDE.md)

Samples/SwiftKitSampleApp/build.gradle

@@ -16,6 +16,7 @@ import org.swift.swiftkit.gradle.BuildUtils
 
 plugins {
     id("build-logic.java-application-conventions")
+    id("me.champeau.jmh") version "0.7.2"
 }
 
 group = "org.swift.swiftkit"
@@ -31,43 +32,38 @@ java {
     }
 }
 
-sourceSets {
-    generated {
-        java.srcDir "${buildDir}/generated/src/java/"
-    }
 
-    // Make the 'main' and 'test' source sets depend on the generated sources
-    main {
-        compileClasspath += sourceSets.generated.output
-        runtimeClasspath += sourceSets.generated.output
-    }
-    test {
-        compileClasspath += sourceSets.main.output
-        runtimeClasspath += sourceSets.main.output
+def jextract = tasks.register("jextract", Exec) {
+    description = "Extracts Java accessor sources using jextract"
+
+    outputs.dir(layout.buildDirectory.dir("generated/sources/jextract/main"))
+    inputs.dir("$rootDir/Sources/ExampleSwiftLibrary") // monitored library
+
+    // any changes in the source generator sources also mean the resulting output might change
+    inputs.dir("$rootDir/Sources/JExtractSwift")
+    inputs.dir("$rootDir/Sources/JExtractSwiftTool")
 
-        compileClasspath += sourceSets.generated.output
-        runtimeClasspath += sourceSets.generated.output
+    workingDir = rootDir
+    commandLine "make"
+    args "jextract-generate"
+}
+
+sourceSets {
+    main {
+        java {
+            srcDir(jextract)
+        }
     }
 }
 
 dependencies {
     implementation(project(':SwiftKit'))
-    generatedImplementation(project(':SwiftKit'))
 
     testImplementation(platform("org.junit:junit-bom:5.10.0"))
     testImplementation("org.junit.jupiter:junit-jupiter")
 }
 
-configurations {
-    generatedImplementation.extendsFrom(mainImplementation)
-    generatedRuntimeOnly.extendsFrom(mainRuntimeOnly)
-}
-
-tasks.named("compileJava").configure {
-    dependsOn("jextract")
-}
-
-tasks.test {
+tasks.named('test', Test) {
     useJUnitPlatform()
 }
 
@@ -84,27 +80,15 @@ application {
         "--enable-native-access=ALL-UNNAMED",
 
         // Include the library paths where our dylibs are that we want to load and call
-        "-Djava.library.path=" + BuildUtils.javaLibraryPaths().join(":"),
+        "-Djava.library.path=" + BuildUtils.javaLibraryPaths(rootDir).join(":"),
 
         // Enable tracing downcalls (to Swift)
         "-Djextract.trace.downcalls=true"
     ]
 }
 
-task jextract(type: Exec) {
-    description = "Extracts Java accessor sources using jextract"
-    outputs.dir(layout.buildDirectory.dir("generated"))
-    inputs.dir("$rootDir/Sources/ExampleSwiftLibrary") // monitored library
-
-    // any changes in the source generator sources also mean the resulting output might change
-    inputs.dir("$rootDir/Sources/JExtractSwift")
-    inputs.dir("$rootDir/Sources/JExtractSwiftTool")
-
-    workingDir = rootDir
-    commandLine "make"
-    args "jextract-generate"
-}
-
-tasks.named("compileGeneratedJava").configure {
-    dependsOn jextract
+jmh {
+    jvmArgsAppend = [
+        "-Djava.library.path=" + BuildUtils.javaLibraryPaths(rootDir).join(":"),
+    ]
 }