Skip to content
This repository was archived by the owner on Jan 23, 2023. It is now read-only.

Commit c4c1394

Browse files
JosephTremouletJosephTremoulet
authored
Merge pull request #14095 from JosephTremoulet/BenchmarksGame/2.0.0
Port BenchmarksGame updates to release/2.0.0
2 parents 06c6241 + ccfa884 commit c4c1394

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

57 files changed

+3929
-1973
lines changed

‎.gitattributes‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,8 +69,10 @@ src/pal/tests/palsuite/paltestlist_to_be_reviewed.txt text eol=lf
6969

7070
tests/src/JIT/Performance/CodeQuality/BenchmarksGame/regexdna/regexdna-input25.txt text eol=lf
7171
tests/src/JIT/Performance/CodeQuality/BenchmarksGame/regexdna/regexdna-input25000.txt text eol=lf
72-
tests/src/JIT/Performance/CodeQuality/BenchmarksGame/revcomp/revcomp-input25.txt text eol=lf
73-
tests/src/JIT/Performance/CodeQuality/BenchmarksGame/revcomp/revcomp-input25000.txt text eol=lf
72+
tests/src/JIT/Performance/CodeQuality/BenchmarksGame/regex-redux/regexdna-input25.txt text eol=lf
73+
tests/src/JIT/Performance/CodeQuality/BenchmarksGame/regex-redux/regexdna-input25000.txt text eol=lf
74+
tests/src/JIT/Performance/CodeQuality/BenchmarksGame/reverse-complement/revcomp-input25.txt text eol=lf
75+
tests/src/JIT/Performance/CodeQuality/BenchmarksGame/reverse-complement/revcomp-input25000.txt text eol=lf
7476
tests/src/JIT/Performance/CodeQuality/BenchmarksGame/k-nucleotide/knucleotide-input.txt text eol=lf
7577
tests/src/JIT/Performance/CodeQuality/BenchmarksGame/k-nucleotide/knucleotide-input-big.txt text eol=lf
7678

‎tests/issues.targets‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -393,13 +393,22 @@
393393
<ExcludeList Include="$(XunitTestBinBase)\JIT\superpmi\superpmicollect\superpmicollect.cmd">
394394
<Issue>needs triage</Issue>
395395
</ExcludeList>
396-
<ExcludeList Include="$(XunitTestBinBase)\JIT\Performance\CodeQuality\BenchmarksGame\k-nucleotide\k-nucleotide\k-nucleotide.cmd">
396+
<ExcludeList Include="$(XunitTestBinBase)\JIT\Performance\CodeQuality\BenchmarksGame\k-nucleotide\k-nucleotide-1\k-nucleotide-1.cmd">
397397
<Issue>9314</Issue>
398398
</ExcludeList>
399-
<ExcludeList Include="$(XunitTestBinBase)\JIT\Performance\CodeQuality\BenchmarksGame\regexdna\regexdna\regexdna.cmd">
399+
<ExcludeList Include="$(XunitTestBinBase)\JIT\Performance\CodeQuality\BenchmarksGame\k-nucleotide\k-nucleotide-9\k-nucleotide-9.cmd">
400400
<Issue>9314</Issue>
401401
</ExcludeList>
402-
<ExcludeList Include="$(XunitTestBinBase)\JIT\Performance\CodeQuality\BenchmarksGame\revcomp\revcomp\revcomp.cmd">
402+
<ExcludeList Include="$(XunitTestBinBase)\JIT\Performance\CodeQuality\BenchmarksGame\regex-redux\regex-redux-1\regex-redux-1.cmd">
403+
<Issue>9314</Issue>
404+
</ExcludeList>
405+
<ExcludeList Include="$(XunitTestBinBase)\JIT\Performance\CodeQuality\BenchmarksGame\regex-redux\regex-redux-5\regex-redux-5.cmd">
406+
<Issue>9314</Issue>
407+
</ExcludeList>
408+
<ExcludeList Include="$(XunitTestBinBase)\JIT\Performance\CodeQuality\BenchmarksGame\reverse-complement\reverse-complement-1\reverse-complement-1.cmd">
409+
<Issue>9314</Issue>
410+
</ExcludeList>
411+
<ExcludeList Include="$(XunitTestBinBase)\JIT\Performance\CodeQuality\BenchmarksGame\reverse-complement\reverse-complement-6\reverse-complement-6.cmd">
403412
<Issue>9314</Issue>
404413
</ExcludeList>
405414
</ItemGroup>

‎tests/src/JIT/Performance/CodeQuality/BenchmarksGame/README.TXT‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,13 @@ The benchmarks in these sub-directories are based on
55

66
See the adjoining LICENSE.TXT file for license terms.
77

8-
Our intention with these tests is to provide interesting test cases
9-
for jit developers to use in daily development practice -- not to
10-
produce variants that give the maximum possible performance.
8+
Our intention with these tests is twofold:
9+
1 - To track .NET Core's performance on these benchmarks in the
10+
same benchmarking system used for other internal .NET Core
11+
performance benchmarks.
12+
2 - To make these available for daily JIT (and runtime) development,
13+
as a factor in assessing the performance impact of compiler
14+
(and runtime) changes.
1115

1216
The benchmarks have been modified to fit into the CoreCLR test and
1317
performance test framework, as follows:
@@ -19,19 +23,15 @@ performance test framework, as follows:
1923
xunit-performance iteration is approximately 1 second on modern x64
2024
hardware
2125
- reducing verbosity when run as a benchmark
22-
- reformatting (via the codeformatter tool)
26+
- calling different APIs in a few places to allow compiling against
27+
netstandard1.4
28+
- reformatting
29+
- in the case of pidigits, implementing on top of .NET's BigInteger
30+
type rather than p/invokes to the native GMP library
2331

2432
These benchmarks are just a subset of the benchmarks available in C# from
25-
the Benchmarks Game site. We've selected variants that do not rely on
26-
multiple threads to ensure relative benchmark stability across a
27-
variety of machines.
28-
29-
We've excluded two benchmarks that are inherently multitheaded:
30-
chamenosredux and threadring. We may revisit this as we improve our
31-
ability to harness threading tests in a stable way.
32-
33-
We've also excluded benchmarks that read in large input files:
34-
knucleotide, regexdna, revcomp.
35-
36-
37-
33+
the Benchmarks Game site. The highest-scoring C# .NET Core variant of each
34+
benchmark is included, and in the (common) case of benchmarks where the
35+
best-scoring variant uses multiple threads, we've also selected variants
36+
that do not rely on multiple threads, to ensure relative benchmark stability
37+
across a variety of machines.

‎tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees-2.cs‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
// Adapted from binary-trees C# .NET Core #2 program
6+
// http://benchmarksgame.alioth.debian.org/u64q/program.php?test=binarytrees&lang=csharpcore&id=2
7+
// aka (as of 2017-09-01) rev 1.3 of https://alioth.debian.org/scm/viewvc.php/benchmarksgame/bench/binarytrees/binarytrees.csharp-2.csharp?root=benchmarksgame&view=log
8+
// Best-scoring single-threaded C# .NET Core version as of 2017-09-01
9+
10+
/* The Computer Language Benchmarks Game
11+
http://benchmarksgame.alioth.debian.org/
12+
13+
contributed by Marek Safar
14+
*reset*
15+
*/
16+
17+
using System;
18+
using Microsoft.Xunit.Performance;
19+
20+
[assembly: OptimizeForBenchmarks]
21+
[assembly: MeasureGCCounts]
22+
23+
namespace BenchmarksGame
24+
{
25+
public class BinaryTrees_2
26+
{
27+
const int minDepth = 4;
28+
29+
public static int Main(String[] args)
30+
{
31+
int n = 0;
32+
if (args.Length > 0) n = Int32.Parse(args[0]);
33+
34+
int check = Bench(n, true);
35+
int expected = 4398;
36+
37+
// Return 100 on success, anything else on failure.
38+
return check - expected + 100;
39+
}
40+
41+
[Benchmark(InnerIterationCount = 7)]
42+
public static void RunBench()
43+
{
44+
Benchmark.Iterate(() => Bench(16, false));
45+
}
46+
47+
static int Bench(int n, bool verbose)
48+
{
49+
int maxDepth = Math.Max(minDepth + 2, n);
50+
int stretchDepth = maxDepth + 1;
51+
52+
int check = (TreeNode.bottomUpTree(stretchDepth)).itemCheck();
53+
int checkSum = check;
54+
if (verbose) Console.WriteLine("stretch tree of depth {0}\t check: {1}", stretchDepth, check);
55+
56+
TreeNode longLivedTree = TreeNode.bottomUpTree(maxDepth);
57+
58+
for (int depth = minDepth; depth <= maxDepth; depth += 2)
59+
{
60+
int iterations = 1 << (maxDepth - depth + minDepth);
61+
62+
check = 0;
63+
for (int i = 1; i <= iterations; i++)
64+
{
65+
check += (TreeNode.bottomUpTree(depth)).itemCheck();
66+
}
67+
checkSum += check;
68+
69+
if (verbose)
70+
Console.WriteLine("{0}\t trees of depth {1}\t check: {2}", iterations, depth, check);
71+
}
72+
73+
check = longLivedTree.itemCheck();
74+
checkSum += check;
75+
76+
if (verbose)
77+
Console.WriteLine("long lived tree of depth {0}\t check: {1}", maxDepth, check);
78+
79+
return checkSum;
80+
}
81+
82+
83+
struct TreeNode
84+
{
85+
class Next
86+
{
87+
public TreeNode left, right;
88+
}
89+
90+
private Next next;
91+
92+
internal static TreeNode bottomUpTree(int depth)
93+
{
94+
if (depth > 0)
95+
{
96+
return new TreeNode(
97+
bottomUpTree(depth - 1)
98+
, bottomUpTree(depth - 1)
99+
);
100+
}
101+
else
102+
{
103+
return new TreeNode();
104+
}
105+
}
106+
107+
TreeNode(TreeNode left, TreeNode right)
108+
{
109+
this.next = new Next();
110+
this.next.left = left;
111+
this.next.right = right;
112+
}
113+
114+
internal int itemCheck()
115+
{
116+
// if necessary deallocate here
117+
if (next == null) return 1;
118+
else return 1 + next.left.itemCheck() + next.right.itemCheck();
119+
}
120+
}
121+
}
122+
}

‎tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees.csproj‎ renamed to ‎tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees-2.csproj‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
<?xml version="1.0" encoding="utf-8"?>
1+
<?xml version="1.0" encoding="utf-8"?>
22
<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
33
<Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
44
<PropertyGroup>
@@ -31,7 +31,7 @@
3131
<Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
3232
</ItemGroup>
3333
<ItemGroup>
34-
<Compile Include="binarytrees.csharp.cs" />
34+
<Compile Include="$(MSBuildProjectName).cs" />
3535
</ItemGroup>
3636
<Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.targets))\dir.targets" />
3737
<PropertyGroup Condition=" '$(MsBuildProjectDirOverride)' != '' "></PropertyGroup>

‎tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees-5.cs‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
// Adapted from binary-trees C# .NET Core #5 program
6+
// http://benchmarksgame.alioth.debian.org/u64q/program.php?test=binarytrees&lang=csharpcore&id=5
7+
// aka (as of 2017-09-01) rev 1.1 of https://alioth.debian.org/scm/viewvc.php/benchmarksgame/bench/binarytrees/binarytrees.csharp-5.csharp?root=benchmarksgame&view=log
8+
// Best-scoring C# .NET Core version as of 2017-09-01
9+
10+
/* The Computer Language Benchmarks Game
11+
http://benchmarksgame.alioth.debian.org/
12+
13+
contributed by Marek Safar
14+
*reset*
15+
concurrency added by Peperud
16+
minor improvements by Alex Yakunin
17+
*/
18+
19+
using System;
20+
using System.Runtime.CompilerServices;
21+
using System.Threading.Tasks;
22+
using Microsoft.Xunit.Performance;
23+
24+
[assembly: OptimizeForBenchmarks]
25+
[assembly: MeasureGCCounts]
26+
27+
namespace BenchmarksGame
28+
{
29+
public sealed class BinaryTrees_5
30+
{
31+
public const int MinDepth = 4;
32+
33+
public static int Main(string[] args)
34+
{
35+
var n = args.Length == 0 ? 0 : int.Parse(args[0]);
36+
37+
int check = Bench(n, true);
38+
int expected = 4398;
39+
40+
// Return 100 on success, anything else on failure.
41+
return check - expected + 100;
42+
}
43+
44+
[Benchmark(InnerIterationCount = 7)]
45+
public static void RunBench()
46+
{
47+
Benchmark.Iterate(() => Bench(16, false));
48+
}
49+
50+
static int Bench(int n, bool verbose)
51+
{
52+
var maxDepth = n < (MinDepth + 2) ? MinDepth + 2 : n;
53+
var stretchDepth = maxDepth + 1;
54+
55+
var stretchDepthTask = Task.Run(() => TreeNode.CreateTree(stretchDepth).CountNodes());
56+
var maxDepthTask = Task.Run(() => TreeNode.CreateTree(maxDepth).CountNodes());
57+
58+
var tasks = new Task<string>[(maxDepth - MinDepth) / 2 + 1];
59+
for (int depth = MinDepth, ti = 0; depth <= maxDepth; depth += 2, ti++)
60+
{
61+
var iterationCount = 1 << (maxDepth - depth + MinDepth);
62+
var depthCopy = depth; // To make sure closure value doesn't change
63+
tasks[ti] = Task.Run(() =>
64+
{
65+
var count = 0;
66+
if (depthCopy >= 17)
67+
{
68+
// Parallelized computation for relatively large tasks
69+
var miniTasks = new Task<int>[iterationCount];
70+
for (var i = 0; i < iterationCount; i++)
71+
miniTasks[i] = Task.Run(() => TreeNode.CreateTree(depthCopy).CountNodes());
72+
Task.WaitAll(miniTasks);
73+
for (var i = 0; i < iterationCount; i++)
74+
count += miniTasks[i].Result;
75+
}
76+
else
77+
{
78+
// Sequential computation for smaller tasks
79+
for (var i = 0; i < iterationCount; i++)
80+
count += TreeNode.CreateTree(depthCopy).CountNodes();
81+
}
82+
return $"{iterationCount}\t trees of depth {depthCopy}\t check: {count}";
83+
});
84+
}
85+
Task.WaitAll(tasks);
86+
87+
if (verbose)
88+
{
89+
int count = 0;
90+
Action<string> printAndSum = (string s) =>
91+
{
92+
Console.WriteLine(s);
93+
count += int.Parse(s.Substring(s.LastIndexOf(':') + 1).TrimStart());
94+
};
95+
96+
printAndSum(String.Format("stretch tree of depth {0}\t check: {1}",
97+
stretchDepth, stretchDepthTask.Result));
98+
foreach (var task in tasks)
99+
printAndSum(task.Result);
100+
printAndSum(String.Format("long lived tree of depth {0}\t check: {1}",
101+
maxDepth, maxDepthTask.Result));
102+
103+
return count;
104+
}
105+
106+
return 0;
107+
}
108+
}
109+
110+
public struct TreeNode
111+
{
112+
public sealed class NodeData
113+
{
114+
public TreeNode Left, Right;
115+
116+
public NodeData(TreeNode left, TreeNode right)
117+
{
118+
Left = left;
119+
Right = right;
120+
}
121+
}
122+
123+
public NodeData Data;
124+
125+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
126+
public TreeNode(TreeNode left, TreeNode right)
127+
{
128+
Data = new NodeData(left, right);
129+
}
130+
131+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
132+
public static TreeNode CreateTree(int depth)
133+
{
134+
return depth <= 0
135+
? default(TreeNode)
136+
: new TreeNode(CreateTree(depth - 1), CreateTree(depth - 1));
137+
}
138+
139+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
140+
public int CountNodes()
141+
{
142+
if (ReferenceEquals(Data, null))
143+
return 1;
144+
return 1 + Data.Left.CountNodes() + Data.Right.CountNodes();
145+
}
146+
}
147+
}

‎tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees3.csproj‎ renamed to ‎tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees-5.csproj‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
<?xml version="1.0" encoding="utf-8"?>
1+
<?xml version="1.0" encoding="utf-8"?>
22
<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
33
<Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
44
<PropertyGroup>
@@ -31,7 +31,7 @@
3131
<Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
3232
</ItemGroup>
3333
<ItemGroup>
34-
<Compile Include="binarytrees.csharp3.cs" />
34+
<Compile Include="$(MSBuildProjectName).cs" />
3535
</ItemGroup>
3636
<Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.targets))\dir.targets" />
3737
<PropertyGroup Condition=" '$(MsBuildProjectDirOverride)' != '' "></PropertyGroup>

0 commit comments

Comments
 (0)