Skip to content

Commit 363c9aa

Browse files
authored
Merge pull request #178 from nowsprinting/feature/flipcomparer
Add FlipTexture2dEqualityComparer
2 parents 35345dc + 290a9b6 commit 363c9aa

17 files changed

+724
-119
lines changed

.github/workflows/test.yml

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,6 @@ jobs:
5050
include:
5151
- unityVersion: 6000.2.6f1
5252
octocov: true
53-
vrt: true
5453
- unityVersion: 6000.2.6f1
5554
testMode: Standalone # run tests on player
5655

@@ -91,12 +90,6 @@ jobs:
9190
openupm add -f com.cysharp.unitask
9291
working-directory: ${{ env.CREATED_PROJECT_PATH }}
9392

94-
- name: Install Graphics Test Framework
95-
run: |
96-
openupm add -f com.unity.testframework.graphics
97-
working-directory: ${{ env.CREATED_PROJECT_PATH }}
98-
if: ${{ matrix.vrt }}
99-
10093
- name: Install codecoverage package
10194
run: |
10295
openupm add -f com.unity.testtools.codecoverage

README.md

Lines changed: 46 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -334,7 +334,7 @@ public class MyTestClass
334334

335335
Default save path is "`Application.persistentDataPath`/TestHelper/Screenshots/`TestContext.Test.Name`.mp4".
336336
You can specify the save directory by arguments.
337-
Directory can also be specified by command line arguments `-testHelperScreenshotDirectory`.
337+
Directory can also be specified by command line argument `-testHelperScreenshotDirectory`.
338338

339339
This attribute can be placed on the test method only.
340340
Can be used with sync `Test`, async `Test`, and `UnityTest`.
@@ -370,7 +370,7 @@ public class MyTestClass
370370

371371
Default save path is "`Application.persistentDataPath`/TestHelper/Screenshots/`TestContext.Test.Name`.png".
372372
You can specify the save directory and/or filename by arguments.
373-
Directory can also be specified by command line arguments `-testHelperScreenshotDirectory`.
373+
Directory can also be specified by command line argument `-testHelperScreenshotDirectory`.
374374

375375
This attribute can be placed on the test method only.
376376
Can be used with sync `Test`, async `Test`, and `UnityTest`.
@@ -492,6 +492,42 @@ public class MyTestClass
492492

493493
### Comparers
494494

495+
#### FlipTexture2dEqualityComparer (optional)
496+
497+
`FlipTexture2dEqualityComparer` is a NUnit test comparer class that compares two `Texture2D` using [FLIP](https://github.com/NVlabs/flip).
498+
499+
Output error map image file if assertion fails.
500+
Default output path is "`Application.persistentDataPath`/TestHelper/Screenshots/`TestContext.Test.Name`.diff.png".
501+
You can specify the output directory and/or filename by constructor arguments.
502+
Directory can also be specified by command line argument `-testHelperScreenshotDirectory`.
503+
504+
Usage:
505+
506+
```csharp
507+
[TestFixture]
508+
public class MyTestClass
509+
{
510+
[Test]
511+
public async Task MyTestMethod()
512+
{
513+
await Awaitable.EndOfFrameAsync();
514+
var actual = ScreenCapture.CaptureScreenshotAsTexture();
515+
var expected = AssetDatabase.LoadAssetAtPath<Texture2D>(ExpectedImagePath);
516+
517+
var comparer = new FlipTexture2dEqualityComparer(meanErrorTolerance: 0.01f);
518+
Assert.That(actual, Is.EqualTo(expected).Using(comparer));
519+
}
520+
}
521+
```
522+
523+
> [!IMPORTANT]
524+
> `FlipTexture2dEqualityComparer` is an optional functionality. To use it, you need to install the [FlipBinding.CSharp](https://www.nuget.org/packages/FlipBinding.CSharp) NuGet package v1.0.0 or newer.
525+
> Also, add scripting define symbol `ENABLE_FLIP_BINDING` if not installed via OpenUPM (UnityNuGet).
526+
527+
> [!NOTE]
528+
> When running on the Ubuntu 22.04 image (e.g., [GameCI](https://game.ci/) provided images), the GLIBCXX_3.4.32 (GCC 13+) required by FLIP's native libraries is missing.
529+
> So, you will need to create a custom Docker image that includes libstdc++6 from GCC 13.
530+
495531
#### XmlComparer
496532

497533
`XmlComparer` is a NUnit test comparer class that compares two `string` as an XML document.
@@ -508,8 +544,8 @@ public class MyTestClass
508544
[Test]
509545
public void MyTestMethod()
510546
{
511-
var x = @"<root><child>value1</child><child attribute=""attr"">value2</child></root>";
512-
var y = @"<?xml version=""1.0"" encoding=""utf-8""?>
547+
var actual = @"<root><child>value1</child><child attribute=""attr"">value2</child></root>";
548+
var expected = @"<?xml version=""1.0"" encoding=""utf-8""?>
513549
<root>
514550
<!-- comment -->
515551
<child attribute=""attr"">
@@ -521,7 +557,7 @@ public class MyTestClass
521557
</child>
522558
</root>";
523559

524-
Assert.That(x, Is.EqualTo(y).Using(new XmlComparer()));
560+
Assert.That(actual, Is.EqualTo(expected).Using(new XmlComparer()));
525561
}
526562
}
527563
```
@@ -618,7 +654,7 @@ Experimental and Statistical Summary:
618654

619655
Default save path is "`Application.persistentDataPath`/TestHelper/Statistics/`TestContext.Test.Name`.png".
620656
You can specify the save directory and/or filename by arguments.
621-
Directory can also be specified by command line arguments `-testHelperStatisticsDirectory`.
657+
Directory can also be specified by command line argument `-testHelperStatisticsDirectory`.
622658

623659
Usage:
624660

@@ -715,7 +751,7 @@ public class MyTestClass
715751
Default save path is "`Application.persistentDataPath`/TestHelper/Screenshots/`TestContext.Test.Name`.png".
716752
(Replace `TestContext.Test.Name` to caller method name when called outside a test context.)
717753
You can specify the save directory and/or filename by arguments.
718-
Directory can also be specified by command line arguments `-testHelperScreenshotDirectory`.
754+
Directory can also be specified by command line argument `-testHelperScreenshotDirectory`.
719755

720756
Usage:
721757

@@ -908,8 +944,9 @@ UNITY_VERSION=2019.4.40f1 make -k test
908944
909945
> [!TIP]
910946
> To run all tests, you need to install the following packages in your project:
911-
> - [Graphics Test Framework](https://docs.unity3d.com/Packages/com.unity.testframework.graphics@latest)
912-
> - [UniTask](https://github.com/Cysharp/UniTask)
947+
> - [UniTask](https://github.com/Cysharp/UniTask) package v2.3.3 or newer.
948+
> - [FlipBinding.CSharp](https://www.nuget.org/packages/FlipBinding.CSharp) NuGet package v1.0.0 or newer.
949+
> - [Instant Replay for Unity](https://github.com/CyberAgentGameEntertainment/InstantReplay) package v1.0.0 or newer
913950
914951

915952
### Release workflow
Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
// Copyright (c) 2023-2025 Koji Hasegawa.
2+
// This software is released under the MIT License.
3+
4+
#if ENABLE_FLIP_BINDING
5+
using System.Collections.Generic;
6+
using System.IO;
7+
using FlipBinding.CSharp;
8+
using TestHelper.RuntimeInternals;
9+
using UnityEngine;
10+
11+
namespace TestHelper.Comparers
12+
{
13+
/// <summary>
14+
/// Compares two <see cref="Texture2D"/> using <see href="https://github.com/NVlabs/flip">FLIP</see>.
15+
/// </summary>
16+
public class FlipTexture2dEqualityComparer : IComparer<Texture2D>
17+
{
18+
private const float DefaultMeanErrorTolerance = 1E-05f;
19+
private const float DefaultPpd = 67.0206451f;
20+
21+
// Comparison parameters
22+
private readonly float _meanErrorTolerance;
23+
private readonly string _errorMapOutputDirectory;
24+
private readonly string _errorMapOutputFilename;
25+
private readonly bool _namespaceToDirectory;
26+
27+
// FLIP parameters
28+
private readonly bool _useHdr;
29+
private readonly float _ppd;
30+
private readonly Tonemapper _tonemapper;
31+
private readonly float _startExposure;
32+
private readonly float _stopExposure;
33+
private readonly int _numExposures;
34+
35+
/// <summary>
36+
/// Constructor.
37+
/// </summary>
38+
/// <param name="meanErrorTolerance">Mean FLIP error value in the range [0,1]. Lower values indicate more similar images.</param>
39+
/// <param name="errorMapOutputDirectory">Directory to output error map.
40+
/// If omitted, the directory specified by command line argument <c>-testHelperScreenshotDirectory</c> is used.
41+
/// If the command line argument is also omitted, <c>Application.persistentDataPath</c> + "/TestHelper/Screenshots/" is used.</param>
42+
/// <param name="errorMapOutputFilename">Filename to output error map.
43+
/// If omitted, default filename is <c>TestContext.Test.Name</c> + ".diff.png".</param>
44+
/// <param name="namespaceToDirectory">Insert subdirectory named from test namespace if true.</param>
45+
/// <param name="useHdr">Whether to use HDR mode. LDR: values in [0,1], HDR: values can exceed [0,1].</param>
46+
/// <param name="ppd">Pixels per degree. Default is 67 (4K display at 0.7m viewing distance). You can calculate PPD with <see cref="Flip.CalculatePpd"/></param>
47+
/// <param name="tonemapper">Tonemapper for HDR-FLIP processing. Ignored when useHdr is false.</param>
48+
/// <param name="startExposure">Start exposure for HDR-FLIP. Use float.PositiveInfinity for auto-calculation.</param>
49+
/// <param name="stopExposure">Stop exposure for HDR-FLIP. Use float.PositiveInfinity for auto-calculation.</param>
50+
/// <param name="numExposures">Number of exposures for HDR-FLIP. Use -1 for auto-calculation.</param>
51+
public FlipTexture2dEqualityComparer(
52+
float meanErrorTolerance = DefaultMeanErrorTolerance,
53+
string errorMapOutputDirectory = null,
54+
string errorMapOutputFilename = null,
55+
bool namespaceToDirectory = false,
56+
bool useHdr = false,
57+
float ppd = DefaultPpd,
58+
Tonemapper tonemapper = Tonemapper.Aces,
59+
float startExposure = float.PositiveInfinity,
60+
float stopExposure = float.PositiveInfinity,
61+
int numExposures = -1)
62+
{
63+
_meanErrorTolerance = meanErrorTolerance;
64+
_errorMapOutputDirectory = errorMapOutputDirectory;
65+
_errorMapOutputFilename = errorMapOutputFilename;
66+
_namespaceToDirectory = namespaceToDirectory;
67+
_useHdr = useHdr;
68+
_ppd = ppd;
69+
_tonemapper = tonemapper;
70+
_startExposure = startExposure;
71+
_stopExposure = stopExposure;
72+
_numExposures = numExposures;
73+
}
74+
75+
/// <inheritdoc/>
76+
public int Compare(Texture2D x, Texture2D y)
77+
{
78+
if (x!.width != y!.width || x.height != y.height)
79+
{
80+
Debug.Log("Texture sizes are different.\n" +
81+
$" Expected: {x.width}x{x.height}\n" +
82+
$" But was: {y.width}x{y.height}\n");
83+
return -1;
84+
}
85+
86+
var referenceRgb = ConvertToRgbArray(y);
87+
var testRgb = ConvertToRgbArray(x);
88+
89+
var result = Flip.Evaluate(
90+
referenceRgb,
91+
testRgb,
92+
y.width,
93+
y.height,
94+
_useHdr,
95+
_ppd,
96+
_tonemapper,
97+
_startExposure,
98+
_stopExposure,
99+
_numExposures,
100+
applyMagmaMap: true);
101+
102+
if (result.MeanError <= _meanErrorTolerance)
103+
{
104+
return 0;
105+
}
106+
107+
Debug.Log($"Mean FLIP error value: {result.MeanError}\n" +
108+
$"Exceeds the specified tolerance {_meanErrorTolerance}");
109+
SaveErrorMap(result);
110+
return -1;
111+
}
112+
113+
private static float[] ConvertToRgbArray(Texture2D texture)
114+
{
115+
var pixels = texture.GetPixels();
116+
var rgb = new float[pixels.Length * 3];
117+
118+
for (var i = 0; i < pixels.Length; i++)
119+
{
120+
var linear = pixels[i].linear;
121+
rgb[i * 3] = linear.r;
122+
rgb[i * 3 + 1] = linear.g;
123+
rgb[i * 3 + 2] = linear.b;
124+
}
125+
126+
return rgb;
127+
}
128+
129+
private void SaveErrorMap(FlipResult result)
130+
{
131+
var path = GetErrorMapPath();
132+
var directory = Path.GetDirectoryName(path);
133+
if (!string.IsNullOrEmpty(directory))
134+
{
135+
Directory.CreateDirectory(directory);
136+
}
137+
138+
var texture = new Texture2D(result.Width, result.Height, TextureFormat.RGB24, false);
139+
var colors = new Color32[result.Width * result.Height];
140+
141+
for (var i = 0; i < colors.Length; i++)
142+
{
143+
var (r, g, b) = result.GetPixelRgb(i % result.Width, i / result.Width);
144+
colors[i] = new Color(r, g, b);
145+
}
146+
147+
texture.SetPixels32(colors);
148+
texture.Apply();
149+
150+
var bytes = texture.EncodeToPNG();
151+
File.WriteAllBytes(path, bytes);
152+
153+
Debug.Log($"Error map saved to: {path}");
154+
Object.Destroy(texture);
155+
}
156+
157+
private string GetErrorMapPath()
158+
{
159+
string directory;
160+
if (_errorMapOutputDirectory != null)
161+
{
162+
directory = Path.GetFullPath(_errorMapOutputDirectory);
163+
}
164+
else
165+
{
166+
directory = CommandLineArgs.GetScreenshotDirectory();
167+
}
168+
169+
string path;
170+
if (_errorMapOutputFilename == null)
171+
{
172+
path = PathHelper.CreateFilePath(
173+
baseDirectory: directory,
174+
extension: ".diff.png",
175+
namespaceToDirectory: _namespaceToDirectory);
176+
}
177+
else
178+
{
179+
path = Path.Combine(directory, _errorMapOutputFilename);
180+
if (!path.EndsWith(".png"))
181+
{
182+
path += ".png";
183+
}
184+
}
185+
186+
return path;
187+
}
188+
}
189+
}
190+
#endif

Runtime/Comparers/FlipTexture2dEqualityComparer.cs.meta

Lines changed: 3 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Runtime/TestHelper.asmdef

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@
1313
"allowUnsafeCode": false,
1414
"overrideReferences": true,
1515
"precompiledReferences": [
16-
"nunit.framework.dll"
16+
"nunit.framework.dll",
17+
"FlipBinding.CSharp.dll"
1718
],
1819
"autoReferenced": false,
1920
"defineConstraints": [
@@ -29,6 +30,11 @@
2930
"name": "jp.co.cyberagent.instant-replay",
3031
"expression": "1.0.0",
3132
"define": "ENABLE_INSTANT_REPLAY"
33+
},
34+
{
35+
"name": "org.nuget.flipbinding.csharp",
36+
"expression": "1.0.0",
37+
"define": "ENABLE_FLIP_BINDING"
3238
}
3339
],
3440
"noEngineReferences": false
90.6 KB
Loading

0 commit comments

Comments
 (0)