Skip to content

Commit 71d285c

Browse files
YoshiRulzYoshiRulz
committed
Set up SMS testroms project
1 parent 11bfce2 commit 71d285c

File tree

8 files changed

+317
-3
lines changed

8 files changed

+317
-3
lines changed

‎src/BizHawk.Tests.Testroms.SMS/BizHawk.Tests.Testroms.SMS.csproj‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
<PropertyGroup>
3+
<TargetFramework>net48</TargetFramework>
4+
</PropertyGroup>
5+
<Import Project="../Testroms.props" />
6+
<ItemGroup>
7+
<ProjectReference Include="$(ProjectDir)../BizHawk.Tests.Testroms/BizHawk.Tests.Testroms.csproj" />
8+
<EmbeddedResource Include="res/**/*" />
9+
</ItemGroup>
10+
</Project>

‎src/BizHawk.Tests.Testroms.SMS/SMSHelper.cs‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
using System.Collections.Generic;
2+
using System.Drawing;
3+
using System.IO;
4+
5+
using BizHawk.Emulation.Common;
6+
using BizHawk.Emulation.Cores;
7+
using BizHawk.Emulation.Cores.Consoles.Sega.gpgx;
8+
9+
using SMSHawk = BizHawk.Emulation.Cores.Sega.MasterSystem.SMS;
10+
11+
using static BizHawk.Emulation.Cores.Consoles.Sega.gpgx.GPGX;
12+
using static BizHawk.Emulation.Cores.Consoles.Sega.gpgx.LibGPGX.InitSettings;
13+
using static BizHawk.Emulation.Cores.Sega.MasterSystem.SMS;
14+
15+
namespace BizHawk.Tests.Testroms.SMS
16+
{
17+
public static class SMSHelper
18+
{
19+
public readonly struct CoreSetup
20+
{
21+
public const string GPGX_MAME = $"{CoreNames.Gpgx}_MAMESound";
22+
23+
public const string GPGX_NUKED = $"{CoreNames.Gpgx}_NukedSound";
24+
25+
public static IReadOnlyCollection<CoreSetup> ValidSetupsFor()
26+
=> new CoreSetup[]
27+
{
28+
new(GPGX_MAME),
29+
new(GPGX_MAME, useBios: false),
30+
new(GPGX_NUKED),
31+
new(GPGX_NUKED, useBios: false),
32+
new(CoreNames.SMSHawk),
33+
new(CoreNames.SMSHawk, useBios: false),
34+
};
35+
36+
public readonly string CoreName;
37+
38+
public readonly bool UseBIOS;
39+
40+
public CoreSetup(string coreName, bool useBios = true)
41+
{
42+
CoreName = coreName;
43+
UseBIOS = useBios;
44+
}
45+
46+
public readonly override string ToString()
47+
=> $"{CoreName}{(UseBIOS ? string.Empty : " (no BIOS)")}";
48+
}
49+
50+
private static readonly GPGXSyncSettings GPGXSyncSettings_MAME_NOBIOS = new() { LoadBIOS = false, SMSFMSoundChip = SMSFMSoundChipType.YM2413_MAME };
51+
52+
private static readonly GPGXSyncSettings GPGXSyncSettings_MAME_USEBIOS = new() { LoadBIOS = true, SMSFMSoundChip = SMSFMSoundChipType.YM2413_MAME };
53+
54+
private static readonly GPGXSyncSettings GPGXSyncSettings_NUKED_NOBIOS = new() { LoadBIOS = false, SMSFMSoundChip = SMSFMSoundChipType.YM2413_NUKED };
55+
56+
private static readonly GPGXSyncSettings GPGXSyncSettings_NUKED_USEBIOS = new() { LoadBIOS = true, SMSFMSoundChip = SMSFMSoundChipType.YM2413_NUKED };
57+
58+
private static readonly SmsSyncSettings SMSHawkSyncSettings_NOBIOS = new() { UseBios = false };
59+
60+
private static readonly SmsSyncSettings SMSHawkSyncSettings_USEBIOS = new() { UseBios = true };
61+
62+
private static bool AddEmbeddedSMSBIOS(this DummyFrontend.EmbeddedFirmwareProvider efp)
63+
=> efp.AddIfExists(new("SMS", "Export"), "res.fw.SMS__Export__U_1_3.bin");
64+
65+
public static GPGXSyncSettings GetGPGXSyncSettings(string coreVariant, bool biosAvailable)
66+
=> coreVariant is CoreSetup.GPGX_NUKED
67+
? biosAvailable ? GPGXSyncSettings_NUKED_USEBIOS : GPGXSyncSettings_NUKED_NOBIOS
68+
: biosAvailable ? GPGXSyncSettings_MAME_USEBIOS : GPGXSyncSettings_MAME_NOBIOS;
69+
70+
public static SmsSyncSettings GetSMSHawkSyncSettings(bool biosAvailable)
71+
=> biosAvailable ? SMSHawkSyncSettings_USEBIOS : SMSHawkSyncSettings_NOBIOS;
72+
73+
public static DummyFrontend.ClassInitCallbackDelegate InitSMSCore(CoreSetup setup, string romFilename, byte[] rom)
74+
=> (efp, _, coreComm) =>
75+
{
76+
efp.UseReflectionCache(ReflectionCache.EmbeddedResourceList(), ReflectionCache.EmbeddedResourceStream);
77+
if (setup.UseBIOS && !efp.AddEmbeddedSMSBIOS()) Assert.Inconclusive("BIOS not provided");
78+
var game = Database.GetGameInfo(rom, romFilename);
79+
IEmulator newCore = setup.CoreName switch
80+
{
81+
CoreSetup.GPGX_MAME or CoreSetup.GPGX_NUKED => new GPGX(new CoreLoadParameters<GPGX.GPGXSettings, GPGX.GPGXSyncSettings>
82+
{
83+
Comm = coreComm,
84+
DeterministicEmulationRequested = true,
85+
Game = game,
86+
Roms = { new DummyRomAsset(rom, VSystemID.Raw.SMS) },
87+
Settings = new(),
88+
SyncSettings = GetGPGXSyncSettings(setup.CoreName, setup.UseBIOS),
89+
}),
90+
CoreNames.SMSHawk => new SMSHawk(coreComm, game, rom, new(), GetSMSHawkSyncSettings(setup.UseBIOS)),
91+
_ => throw new InvalidOperationException("unknown SMS core"),
92+
};
93+
return (newCore, setup.UseBIOS ? 388 : 5);
94+
};
95+
96+
public static TestUtils.TestSuccessState SMSScreenshotsEqual(
97+
Stream expectFile,
98+
Image? actual,
99+
bool expectingNotEqual,
100+
CoreSetup setup,
101+
(string Suite, string Case) id,
102+
IReadOnlyDictionary<int, int>? extraPaletteMap = null)
103+
{
104+
if (actual is null)
105+
{
106+
Assert.Fail("actual screenshot was null");
107+
return TestUtils.TestSuccessState.Failure; // never hit
108+
}
109+
return ImageUtils.ScreenshotsEqualMagickDotNET(
110+
expectFile,
111+
extraPaletteMap is null ? actual : ImageUtils.PaletteSwap(actual, extraPaletteMap),
112+
expectingNotEqual,
113+
id);
114+
}
115+
}
116+
}

‎src/BizHawk.Tests.Testroms.SMS/SMSmemtest.cs‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
using System.Collections.Generic;
2+
using System.Drawing;
3+
using System.Linq;
4+
using System.Reflection;
5+
6+
using BizHawk.Common.IOExtensions;
7+
8+
using static BizHawk.Tests.Testroms.SMS.SMSHelper;
9+
10+
namespace BizHawk.Tests.Testroms.SMS
11+
{
12+
[TestClass]
13+
public sealed class SMSmemtest
14+
{
15+
[AttributeUsage(AttributeTargets.Method)]
16+
private sealed class SMSmemtestDataAttribute : Attribute, ITestDataSource
17+
{
18+
public static readonly IReadOnlyCollection<string> Subtests = [ "RAM", "VRAM", "SRAM" ];
19+
20+
public IEnumerable<object?[]> GetData(MethodInfo methodInfo)
21+
{
22+
var testCases = CoreSetup.ValidSetupsFor().ToList();
23+
// testCases.RemoveAll(static setup => setup.CoreName is not CoreSetup.GPGX_MAME); // uncomment and modify to run a subset of the test cases...
24+
foreach (var subTest in SMSmemtestDataAttribute.Subtests)
25+
{
26+
testCases.RemoveAll(setup => TestUtils.ShouldIgnoreCase(SUITE_ID, DisplayNameFor(setup, subTest))); // ...or use the global blocklist in TestUtils
27+
}
28+
return testCases.OrderBy(static setup => setup.ToString())
29+
.Select(static setup => new object?[] { setup });
30+
}
31+
32+
public string? GetDisplayName(MethodInfo methodInfo, object?[]? data)
33+
=> $"{methodInfo.Name}(\"{(CoreSetup) data![0]!}\")";
34+
}
35+
36+
private const string ROM_EMBED_PATH = "res.SMSmemtest_artifact.SMSmemtest.sms";
37+
38+
private const string SUITE_ID = "SMSmemtest";
39+
40+
private static readonly IReadOnlyList<string> KnownFailures = new[]
41+
{
42+
"", // none \o/
43+
};
44+
45+
private static readonly bool RomIsPresent = ReflectionCache.EmbeddedResourceList().Contains(ROM_EMBED_PATH);
46+
47+
[ClassCleanup]
48+
public static void AfterAll()
49+
=> TestUtils.WriteMetricsToDisk();
50+
51+
[ClassInitialize]
52+
public static void BeforeAll(TestContext ctx)
53+
{
54+
TestUtils.AssertKnownFailuresAreSorted(KnownFailures, suiteID: SUITE_ID);
55+
TestUtils.PrepareDBAndOutput(SUITE_ID);
56+
}
57+
58+
private static string DisplayNameFor(CoreSetup setup, string subTest)
59+
=> $"SMSmemtest.{subTest} on {setup}";
60+
61+
[DataTestMethod]
62+
[SMSmemtestData]
63+
public void RunSMSmemtest(CoreSetup setup)
64+
{
65+
TestUtils.ShortCircuitMissingRom(RomIsPresent);
66+
TestUtils.ShortCircuitKnownFailure(SMSmemtestDataAttribute.Subtests.Select(subTest => DisplayNameFor(setup, subTest)).All(KnownFailures.Contains));
67+
DummyFrontend fe = new(InitSMSCore(setup, "SMSmemtest.sms", ReflectionCache.EmbeddedResourceStream(ROM_EMBED_PATH).ReadAllBytes()));
68+
bool DoSubcaseAssertion(string subTest, Bitmap actual)
69+
{
70+
var caseStr = DisplayNameFor(setup, subTest);
71+
var knownFail = TestUtils.IsKnownFailure(caseStr, KnownFailures);
72+
var state = SMSScreenshotsEqual(
73+
ReflectionCache.EmbeddedResourceStream($"res.SMSmemtest_artifact.expected_{subTest.ToLowerInvariant()}.png"),
74+
actual,
75+
knownFail,
76+
setup,
77+
(SUITE_ID, caseStr));
78+
switch (state)
79+
{
80+
case TestUtils.TestSuccessState.ExpectedFailure:
81+
Console.WriteLine("expected failure, verified");
82+
return false;
83+
case TestUtils.TestSuccessState.Failure:
84+
Assert.Fail("expected and actual screenshots differ");
85+
return default;
86+
case TestUtils.TestSuccessState.Success:
87+
return true;
88+
case TestUtils.TestSuccessState.UnexpectedSuccess:
89+
Assert.Fail("expected and actual screenshots matched unexpectedly (this is a good thing)");
90+
return default;
91+
default:
92+
return default;
93+
}
94+
}
95+
fe.FrameAdvanceBy(273);
96+
// SMSHawk is done now
97+
fe.FrameAdvanceBy(18);
98+
var ramPassed = DoSubcaseAssertion("RAM", fe.Screenshot());
99+
fe.Dispose();
100+
if (!ramPassed) Assert.Inconclusive(); // for this to be false, it must have been an expected failure or execution would have stopped with an Assert.Fail call
101+
// Assert.Inconclusive("(other subtests aren't implemented)"); // uncommenting this causes tests to hang..?
102+
}
103+
}
104+
}

‎src/BizHawk.Tests.Testroms.SMS/readme.md‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
Before building, testroms and firmware need to be placed under `/res` in this project.
2+
You *should* be able to omit any suite or firmware and the relevant cases will be skipped.
3+
Firmware needs to be manually copied into a `fw` dir;
4+
testroms need to be copied into a separate dir per suite, with a hierarchy matching the CI artifacts of [this repo](https://gitlab.com/tasbot/libre-roms-ci).
5+
On Linux, run `/res/download_from_ci.sh` to automatically download and extract said artifacts.
6+
On Windows, run the same script in WSL, or do it manually (because Yoshi can't be bothered porting the script to PowerShell).
7+
All told, the expected directory structure is:
8+
```
9+
res
10+
├─ fw
11+
│ └─ SMS__Export__U_1_3.bin
12+
└─ SMSmemtest_artifact
13+
```
14+
15+
Like when building EmuHawk, the target framework and configuration for all the BizHawk project deps is dictated by this project.
16+
That means .NET Standard 2.0 for most libraries, or else .NET Framework 4.8 (which you can override with e.g. `-p:TestProjTargetFrameworkOverride=net8.0`).
17+
To build and run the tests in `Release` configuration (or `Debug` if you need that for some reason):
18+
- On Linux, run `run_tests_release.sh` or `run_tests_debug.sh`.
19+
- On Windows, pass `-c Release` to `dotnet test` (must `cd` to this project). Omitting `-c` will use `Debug`.
20+
21+
> You can at this point run the tests, but you should probably keep reading to see your options.
22+
23+
To run only some suites, comment out applications of the `[DataTestMethod]` attribute in the source. (Or applications of `[TestClass]`.)
24+
You can also disable individual test cases programmatically by modifying `TestUtils.ShouldIgnoreCase`—
25+
note that "ignored" here means cases are completely removed, and do not count as "skipped".
26+
27+
By default, known failures are counted as "skipped" *without actually running them*.
28+
Set the env. var `BIZHAWKTEST_RUN_KNOWN_FAILURES=1` to run them as well. They will count as "skipped" if they fail, or "failed" if they succeed unexpectedly.
29+
30+
On Linux, all cases for unavailable cores (N/A currently) are counted as "skipped".
31+
32+
Screenshots may be saved under `/test_output/<suite>` **in the repo**.
33+
For ease of typing, a random prefix is chosen for each case e.g. `DEADBEEF_{expected,actual}_*.png`. This is included in stdout (Windows users, see below for how to enable stdout).
34+
35+
The env. var `BIZHAWKTEST_SAVE_IMAGES` determines when to save screenshots (usually an expect/actual pair) to disk.
36+
- With `BIZHAWKTEST_SAVE_IMAGES=all`, all screenshots are saved.
37+
- With `BIZHAWKTEST_SAVE_IMAGES=failures` (the default), only screenshots of failed tests are saved.
38+
- With `BIZHAWKTEST_SAVE_IMAGES=none`, screenshots are never saved.
39+
40+
Test results are output using the logger(s) specified on the command-line.
41+
(Without the `console` logger, the results are summarised in the console, but prints to stdout are not shown.)
42+
- On Linux, the shell scripts add the `console` and `junit` (to file, for GitLab CI) loggers.
43+
- On Windows, pass `-l "console;verbosity=detailed"` to `dotnet`.
44+
45+
> Note that the results and stdout for each test case are not printed immediately.
46+
> Cases are grouped by test method, and once the set of test cases is finished executing, the outputs are sent to the console all at once.
47+
48+
Linux examples:
49+
```sh
50+
# default: simple regression testing, all test suites, saving failures to disk
51+
./run_tests_release.sh
52+
53+
# every test from every suite, not saving anything to disk (as might be used in CI)
54+
BIZHAWKTEST_RUN_KNOWN_FAILURES=1 BIZHAWKTEST_SAVE_IMAGES=none ./run_tests_release.sh
55+
```
56+
57+
Windows examples:
58+
```pwsh
59+
# reminder that if you have WSL, you can use that to run /res/download_from_ci.sh first
60+
61+
# default: simple regression testing, all test suites, saving failures to disk
62+
dotnet test -c Release -l "console;verbosity=detailed"
63+
64+
# same as Linux CI example
65+
$Env:BIZHAWKTEST_RUN_KNOWN_FAILURES = 1
66+
$Env:BIZHAWKTEST_SAVE_IMAGES = "all"
67+
dotnet test -c Release -l "console;verbosity=detailed"
68+
```
69+
70+
Summary of `BIZHAWKTEST_RUN_KNOWN_FAILURES=1 ./run_tests_release.sh` should read 6 passed / 0 skipped / 0 failed.

‎src/BizHawk.Tests.Testroms.SMS/res/download_from_ci.sh‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/bin/sh
2+
set -e
3+
cd "$(dirname "$(realpath "$0")")"
4+
exec ../../BizHawk.Tests.Testroms/.download_from_ci.sh \
5+
SMSmemtest \
6+
"$@"

‎src/BizHawk.Tests.Testroms.SMS/run_tests_debug.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/bin/sh
2+
set -e
3+
cd "$(dirname "$(realpath "$0")")"
4+
exec ../BizHawk.Tests.Testroms/.run_tests_with_configuration.sh "Debug" "$@"

‎src/BizHawk.Tests.Testroms.SMS/run_tests_release.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/bin/sh
2+
set -e
3+
cd "$(dirname "$(realpath "$0")")"
4+
exec ../BizHawk.Tests.Testroms/.run_tests_with_configuration.sh "Release" "$@"

‎src/BizHawk.Tests.Testroms/DummyRomAsset.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
namespace BizHawk.Tests.Testroms
55
{
6-
public readonly struct DummyRomAsset(byte[] fileData) : IRomAsset
6+
public readonly struct DummyRomAsset(byte[] fileData, string sysID = VSystemID.Raw.DEBUG) : IRomAsset
77
{
88
#pragma warning disable CA1065 // yes, really throw
99
public string? Extension
@@ -13,10 +13,10 @@ public byte[]? FileData
1313
=> fileData;
1414

1515
public GameInfo? Game
16-
=> throw new NotImplementedException();
16+
=> new() { System = sysID };
1717

1818
public byte[]? RomData
19-
=> throw new NotImplementedException();
19+
=> fileData;
2020

2121
public string? RomPath
2222
=> throw new NotImplementedException();

0 commit comments

Comments
 (0)