Skip to content

Commit b0ec44d

Browse files
elringuselringus
authored
feat: add llvm bench (#174)
1 parent d6f036c commit b0ec44d

File tree

10 files changed

+195
-69
lines changed

10 files changed

+195
-69
lines changed

‎samples/bench/bench.mjs‎

Lines changed: 41 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
import { Bench, hrtimeNow } from "tinybench";
21
import { init as initBootsharp } from "./bootsharp/init.mjs";
32
import { init as initDotNet } from "./dotnet/init.mjs";
3+
import { init as initDotNetLLVM } from "./dotnet-llvm/init.mjs";
44
import { init as initGo } from "./go/init.mjs";
55
import { init as initRust } from "./rust/init.mjs";
66

@@ -11,19 +11,46 @@ import { init as initRust } from "./rust/init.mjs";
1111
* @property {(n: number) => number} fi
1212
*/
1313

14-
await run("Rust", await initRust());
15-
await run("DotNet", await initDotNet());
16-
await run("Bootsharp", await initBootsharp());
17-
await run("Go", await initGo());
14+
const lang = process.argv[2];
15+
const baseline = [];
1816

19-
/** @param {string} lang */
20-
/** @param {Exports} exports */
17+
if (!lang || lang.toLowerCase() === "rust")
18+
await run("Rust", await initRust());
19+
if (!lang || lang.toLowerCase() === "llvm")
20+
await run(".NET LLVM", await initDotNetLLVM());
21+
if (!lang || lang.toLowerCase() === "net")
22+
await run(".NET", await initDotNet());
23+
if (!lang || lang.toLowerCase() === "boot")
24+
await run("Bootsharp", await initBootsharp());
25+
if (!lang || lang.toLowerCase() === "go")
26+
await run("Go", await initGo());
27+
28+
/** @param {string} lang
29+
* @param {Exports} exports */
2130
async function run(lang, exports) {
22-
console.log(`Running ${lang}...`);
23-
const bench = new Bench({ hrtimeNow });
24-
bench.add("Echo Number", exports.echoNumber);
25-
bench.add("Echo Struct", exports.echoStruct);
26-
bench.add("Fibonacci", () => exports.fi(25));
27-
await bench.run();
28-
console.table(bench.table())
31+
console.log(`\n\nBenching ${lang}...\n`);
32+
console.log(`Echo number: ${iterate(0, exports.echoNumber, 100, 3, 1000)}`);
33+
console.log(`Echo struct: ${iterate(1, exports.echoStruct, 100, 3, 100)}`);
34+
console.log(`Fibonacci: ${iterate(2, () => exports.fi(33), 100, 3, 1)}`);
35+
}
36+
37+
function iterate(idx, fn, iterations, warms, loops) {
38+
const results = [];
39+
warms *= -1;
40+
for (let i = warms; i < iterations; i++) {
41+
const start = performance.now();
42+
for (let l = 0; l < loops; l++) fn();
43+
if (i >= 0) results.push(performance.now() - start);
44+
}
45+
let media = median(results);
46+
if (baseline[idx]) return `${(media / baseline[idx]).toFixed(1)}`;
47+
else baseline[idx] = media;
48+
return `${media.toFixed(3)} ms`;
49+
}
50+
51+
function median(numbers) {
52+
const sorted = [...numbers].sort((a, b) => a - b);
53+
const middle = Math.floor(sorted.length / 2);
54+
if (sorted.length % 2 === 1) return sorted[middle];
55+
return (sorted[middle - 1] + sorted[middle]) / 2;
2956
}

‎samples/bench/bootsharp/Boot.csproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
</PropertyGroup>
1111

1212
<ItemGroup>
13-
<PackageReference Include="Bootsharp" Version="0.5.0-alpha.32"/>
13+
<PackageReference Include="Bootsharp" Version="*-*"/>
1414
<PackageReference Include="Bootsharp.Inject" Version="*-*"/>
1515
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="*"/>
1616
</ItemGroup>

‎samples/bench/dotnet-llvm/DotNetLLVM.csproj‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<TargetFramework>net9.0-browser</TargetFramework>
5+
<Configuration>Release</Configuration>
6+
<RuntimeIdentifier>browser-wasm</RuntimeIdentifier>
7+
<OutputType>Exe</OutputType>
8+
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
9+
<PublishTrimmed>true</PublishTrimmed>
10+
<DotNetJsApi>true</DotNetJsApi>
11+
<UseAppHost>false</UseAppHost>
12+
<EmccFlags>$(EmccFlags) -O3</EmccFlags>
13+
<UsingBrowserRuntimeWorkload>false</UsingBrowserRuntimeWorkload>
14+
<UsingWasiRuntimeWorkload>false</UsingWasiRuntimeWorkload>
15+
<UsingEmscriptenWorkload>true</UsingEmscriptenWorkload>
16+
<ExecutableExtensionName Condition="'$([MSBuild]::IsOSPlatform(&quot;Windows&quot;))' == 'true'">.exe</ExecutableExtensionName>
17+
<IlcHostPackagePath Condition="'$([MSBuild]::IsOSPlatform(&quot;Windows&quot;))' == 'true'">$(Pkgruntime_win-x64_Microsoft_DotNet_ILCompiler_LLVM)</IlcHostPackagePath>
18+
<IlcHostPackagePath Condition="'$([MSBuild]::IsOSPlatform(&quot;Windows&quot;))' == 'false'">$(Pkgruntime_linux-x64_Microsoft_DotNet_ILCompiler_LLVM)</IlcHostPackagePath>
19+
<RestoreAdditionalProjectSources>
20+
https://api.nuget.org/v3/index.json;
21+
https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json;
22+
</RestoreAdditionalProjectSources>
23+
</PropertyGroup>
24+
25+
<ItemGroup>
26+
<PackageReference Include="Microsoft.DotNet.ILCompiler.LLVM" Version="10.0.0-*"/>
27+
<PackageReference Condition="'$([MSBuild]::IsOSPlatform(&quot;Windows&quot;))' == 'true'" Include="runtime.win-x64.Microsoft.DotNet.ILCompiler.LLVM" Version="10.0.0-*"/>
28+
<PackageReference Condition="'$([MSBuild]::IsOSPlatform(&quot;Windows&quot;))' == 'false'" Include="runtime.linux-x64.Microsoft.DotNet.ILCompiler.LLVM" Version="10.0.0-*"/>
29+
<EmscriptenEnvVars Include="DOTNET_EMSCRIPTEN_LLVM_ROOT=$(EmscriptenUpstreamBinPath)"/>
30+
<EmscriptenEnvVars Include="DOTNET_EMSCRIPTEN_BINARYEN_ROOT=$(EmscriptenSdkToolsPath)"/>
31+
<EmscriptenEnvVars Include="DOTNET_EMSCRIPTEN_NODE_JS=$(EmscriptenNodeBinPath)node$(ExecutableExtensionName)"/>
32+
<EmscriptenEnvVars Include="EM_CACHE=$(EmscriptenCacheSdkCacheDir)"/>
33+
</ItemGroup>
34+
35+
<PropertyGroup>
36+
<InvariantTimezone>true</InvariantTimezone>
37+
<InvariantGlobalization>true</InvariantGlobalization>
38+
<WasmEnableLegacyJsInterop>false</WasmEnableLegacyJsInterop>
39+
<TrimMode>full</TrimMode>
40+
<TrimmerRemoveSymbols>true</TrimmerRemoveSymbols>
41+
<_AggressiveAttributeTrimming>true</_AggressiveAttributeTrimming>
42+
<AutoreleasePoolSupport>false</AutoreleasePoolSupport>
43+
<DebuggerSupport>false</DebuggerSupport>
44+
<EnableCppCLIHostActivation>false</EnableCppCLIHostActivation>
45+
<EnableUnsafeBinaryFormatterSerialization>false</EnableUnsafeBinaryFormatterSerialization>
46+
<EnableUnsafeBinaryFormatterInDesigntimeLicenseContextSerialization>false</EnableUnsafeBinaryFormatterInDesigntimeLicenseContextSerialization>
47+
<EnableUnsafeUTF7Encoding>false</EnableUnsafeUTF7Encoding>
48+
<_EnableConsumingManagedCodeFromNativeHosting>false</_EnableConsumingManagedCodeFromNativeHosting>
49+
<EventSourceSupport>false</EventSourceSupport>
50+
<HttpActivityPropagationSupport>false</HttpActivityPropagationSupport>
51+
<MetadataUpdaterSupport>false</MetadataUpdaterSupport>
52+
<UseNativeHttpHandler>true</UseNativeHttpHandler>
53+
<UseSystemResourceKeys>true</UseSystemResourceKeys>
54+
<StartupHookSupport>false</StartupHookSupport>
55+
<CustomResourceTypesSupport>false</CustomResourceTypesSupport>
56+
<BuiltInComInteropSupport>false</BuiltInComInteropSupport>
57+
<WasmEmitSourceMap>false</WasmEmitSourceMap>
58+
<WasmNativeDebugSymbols>false</WasmNativeDebugSymbols>
59+
<JsonSerializerIsReflectionEnabledByDefault>false</JsonSerializerIsReflectionEnabledByDefault>
60+
<PredefinedCulturesOnly>true</PredefinedCulturesOnly>
61+
<MetricsSupport>false</MetricsSupport>
62+
<DisableDependencyInjectionDynamicEngine>true</DisableDependencyInjectionDynamicEngine>
63+
<NullabilityInfoContextSupport>false</NullabilityInfoContextSupport>
64+
<DynamicCodeSupport>false</DynamicCodeSupport>
65+
<XmlResolverIsNetworkingEnabledByDefault>false</XmlResolverIsNetworkingEnabledByDefault>
66+
<StackTraceSupport>false</StackTraceSupport>
67+
<DebugType>none</DebugType>
68+
</PropertyGroup>
69+
70+
</Project>

‎samples/bench/dotnet-llvm/Program.cs‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
using System.Runtime.InteropServices.JavaScript;
2+
using System.Text.Json;
3+
using System.Text.Json.Serialization;
4+
5+
public struct Data
6+
{
7+
public string Info;
8+
public bool Ok;
9+
public int Revision;
10+
public string[] Messages;
11+
}
12+
13+
14+
[JsonSerializable(typeof(Data))]
15+
internal partial class SourceGenerationContext : JsonSerializerContext;
16+
17+
public static partial class Program
18+
{
19+
public static void Main () { }
20+
21+
[JSExport]
22+
public static int EchoNumber () => GetNumber();
23+
24+
[JSExport]
25+
public static string EchoStruct ()
26+
{
27+
var json = GetStruct();
28+
var data = JsonSerializer.Deserialize(json, SourceGenerationContext.Default.Data);
29+
return JsonSerializer.Serialize(data, SourceGenerationContext.Default.Data);
30+
}
31+
32+
[JSExport]
33+
public static int Fi (int n) => n <= 1 ? n : Fi(n - 1) + Fi(n - 2);
34+
35+
[JSImport("getNumber", "x")]
36+
private static partial int GetNumber ();
37+
38+
[JSImport("getStruct", "x")]
39+
private static partial string GetStruct ();
40+
}

‎samples/bench/dotnet-llvm/init.mjs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
import { dotnet } from "./bin/Release/net9.0-browser/browser-wasm/publish/dotnet.js";
2+
import { getNumber, getStruct } from "../fixtures.mjs";
3+
4+
/** @returns {Promise<import("../bench.mjs").Exports>} */
5+
export async function init() {
6+
const runtime = await dotnet.withDiagnosticTracing(false).create();
7+
const asm = "DotNetLLVM";
8+
9+
runtime.setModuleImports("x", {
10+
getNumber,
11+
getStruct: () => JSON.stringify(getStruct())
12+
});
13+
14+
await runtime.runMain(asm, []);
15+
16+
const exports = await runtime.getAssemblyExports(asm);
17+
return {
18+
echoNumber: exports.Program.EchoNumber,
19+
echoStruct: exports.Program.EchoStruct,
20+
fi: exports.Program.Fi
21+
};
22+
}

‎samples/bench/dotnet-llvm/readme.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
1. Install .NET https://dotnet.microsoft.com/en-us/download
2+
2. Run `dotnet publish -c Release`
3+
4+
https://github.com/dotnet/runtime/issues/113979#issuecomment-2759220563

‎samples/bench/go/init.mjs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import { getNumber, getStruct } from "../fixtures.mjs";
55
/** @returns {Promise<import("../bench.mjs").Exports>} */
66
export async function init() {
77
global.getNumber = getNumber;
8-
global.getStruct = getStruct;
8+
global.getStruct = () => JSON.stringify(getStruct());
99

1010
const bin = await WebAssembly.compile(fs.readFileSync("./go/main.wasm"));
1111
const go = new Go();

‎samples/bench/package.json‎

Lines changed: 0 additions & 5 deletions
This file was deleted.

‎samples/bench/readme.md‎

Lines changed: 15 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,53 +1,21 @@
1-
1. Build each sub-dir (readme inside)
2-
2. Run `npm i`
3-
3. Run `npm bench.mjs`
4-
5-
## 2024 (.NET 9)
6-
7-
### Rust
1+
## Setup
82

9-
```
10-
┌─────────┬───────────────┬──────────────────┬───────────────────┬────────────────────────┬────────────────────────┬──────────┐
11-
│ (index) │ Task name │ Latency avg (ns) │ Latency med (ns) │ Throughput avg (ops/s) │ Throughput med (ops/s) │ Samples │
12-
├─────────┼───────────────┼──────────────────┼───────────────────┼────────────────────────┼────────────────────────┼──────────┤
13-
│ 0 │ 'Echo Number' │ '97.04 ± 0.04%' │ '100.00 ± 0.00' │ '9886346 ± 0.01%' │ '10000000 ± 0' │ 10304915 │
14-
│ 1 │ 'Echo Struct' │ '2399.2 ± 1.57%' │ '2300.0 ± 0.00' │ '425825 ± 0.02%' │ '434783 ± 0' │ 416802 │
15-
│ 2 │ 'Fibonacci' │ '298097 ± 0.12%' │ '296400 ± 200.00' │ '3358 ± 0.09%' │ '3374 ± 2' │ 3355 │
16-
└─────────┴───────────────┴──────────────────┴───────────────────┴────────────────────────┴────────────────────────┴──────────┘
17-
```
18-
19-
### DotNet
3+
1. Build each sub-dir (readme inside)
4+
2. Run `npm bench.mjs` to bench all
5+
3. Or `npm bench.mjs rust|llvm|net|boot|go`
206

21-
```
22-
┌─────────┬───────────────┬──────────────────┬───────────────────┬────────────────────────┬────────────────────────┬─────────┐
23-
│ (index) │ Task name │ Latency avg (ns) │ Latency med (ns) │ Throughput avg (ops/s) │ Throughput med (ops/s) │ Samples │
24-
├─────────┼───────────────┼──────────────────┼───────────────────┼────────────────────────┼────────────────────────┼─────────┤
25-
│ 0 │ 'Echo Number' │ '591.95 ± 0.49%' │ '600.00 ± 0.00' │ '1741653 ± 0.02%' │ '1666667 ± 0' │ 1689343 │
26-
│ 1 │ 'Echo Struct' │ '10813 ± 0.24%' │ '10700 ± 300.00' │ '93369 ± 0.04%' │ '93458 ± 2696' │ 92483 │
27-
│ 2 │ 'Fibonacci' │ '485910 ± 0.11%' │ '485200 ± 1500.0' │ '2059 ± 0.08%' │ '2061 ± 6' │ 2058 │
28-
└─────────┴───────────────┴──────────────────┴───────────────────┴────────────────────────┴────────────────────────┴─────────┘
29-
```
7+
## Benches
308

31-
### Bootsharp
9+
- `Echo Number` — interop with raw numbers
10+
- `Echo Struct` — interop with JSON-serialized structs
11+
- `Fibonacci` — compute performance
3212

33-
```
34-
┌─────────┬───────────────┬──────────────────┬───────────────────┬────────────────────────┬────────────────────────┬─────────┐
35-
│ (index) │ Task name │ Latency avg (ns) │ Latency med (ns) │ Throughput avg (ops/s) │ Throughput med (ops/s) │ Samples │
36-
├─────────┼───────────────┼──────────────────┼───────────────────┼────────────────────────┼────────────────────────┼─────────┤
37-
│ 0 │ 'Echo Number' │ '603.58 ± 0.30%' │ '600.00 ± 0.00' │ '1690103 ± 0.01%' │ '1666667 ± 0' │ 1656786 │
38-
│ 1 │ 'Echo Struct' │ '13243 ± 6.52%' │ '12400 ± 100.00' │ '79792 ± 0.04%' │ '80645 ± 656' │ 75513 │
39-
│ 2 │ 'Fibonacci' │ '521873 ± 0.04%' │ '521700 ± 1500.0' │ '1916 ± 0.03%' │ '1917 ± 6' │ 1917 │
40-
└─────────┴───────────────┴──────────────────┴───────────────────┴────────────────────────┴────────────────────────┴─────────┘
41-
```
13+
All results are relative to the Rust baseline (lower is better).
4214

43-
### Go
15+
## 2024 (.NET 9)
4416

45-
```
46-
┌─────────┬───────────────┬───────────────────┬────────────────────┬────────────────────────┬────────────────────────┬─────────┐
47-
│ (index) │ Task name │ Latency avg (ns) │ Latency med (ns) │ Throughput avg (ops/s) │ Throughput med (ops/s) │ Samples │
48-
├─────────┼───────────────┼───────────────────┼────────────────────┼────────────────────────┼────────────────────────┼─────────┤
49-
│ 0 │ 'Echo Number' │ '16583 ± 1.03%' │ '16300 ± 3600.0' │ '66140 ± 0.23%' │ '61350 ± 12806' │ 60303 │
50-
│ 1 │ 'Echo Struct' │ '22017 ± 13.76%' │ '18300 ± 2200.0' │ '55011 ± 0.15%' │ '54645 ± 6705' │ 45420 │
51-
│ 2 │ 'Fibonacci' │ '1254378 ± 0.23%' │ '1250500 ± 2200.0' │ '798 ± 0.15%' │ '800 ± 1' │ 798 │
52-
└─────────┴───────────────┴───────────────────┴────────────────────┴────────────────────────┴────────────────────────┴─────────┘
53-
```
17+
| | Rust | .NET LLVM | .NET | Bootsharp | Go |
18+
|-------------|-------|-----------|--------|-----------|---------|
19+
| Echo Number | `1.0` | `14.4` | `21.1` | `25.7` | `718.7` |
20+
| Echo Struct | `1.0` | `1.5` | `4.3` | `4.9` | `20.8` |
21+
| Fibonacci | `1.0` | `1.1` | `1.5` | `1.6` | `3.8` |

‎src/cs/Directory.Build.props‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
<Project>
22
<PropertyGroup>
3-
<Version>0.5.0-alpha.51</Version>
3+
<Version>0.5.0</Version>
44
<Authors>Elringus</Authors>
55
<PackageTags>javascript typescript ts js wasm node deno bun interop codegen</PackageTags>
66
<PackageProjectUrl>https://sharp.elringus.com</PackageProjectUrl>

0 commit comments

Comments
 (0)