Skip to content

Commit 6634711

Browse files
jiaweig-amznjiaweig-amzn
authored
Screenshots stored from nightly Jenkins ASV "Test GPU Profile" only contain Vulkan versions, not DX12 versions (#496)
* Add suffix to screenshot file name Signed-off-by: jiaweig <51759646+jiaweig-amzn@users.noreply.github.com> * Update screenshot scripts Signed-off-by: jiaweig <51759646+jiaweig-amzn@users.noreply.github.com> * Remove duplicated function Signed-off-by: jiaweig <51759646+jiaweig-amzn@users.noreply.github.com> * Change to add the env info to a folder instead of the file name Signed-off-by: jiaweig <51759646+jiaweig-amzn@users.noreply.github.com> * Optimize code Signed-off-by: jiaweig <51759646+jiaweig-amzn@users.noreply.github.com> Signed-off-by: jiaweig <51759646+jiaweig-amzn@users.noreply.github.com>
1 parent 00edc0e commit 6634711

35 files changed

+150
-54
lines changed

‎Gem/Code/Source/Automation/ScriptManager.cpp‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1089,6 +1089,7 @@ namespace AtomSampleViewer
10891089
behaviorContext->Method("ShowTool", &Script_ShowTool);
10901090

10911091
// Screenshots...
1092+
behaviorContext->Method("SetTestEnvPath", &Script_SetTestEnvPath);
10921093
behaviorContext->Method("SelectImageComparisonToleranceLevel", &Script_SelectImageComparisonToleranceLevel);
10931094
behaviorContext->Method("CaptureScreenshot", &Script_CaptureScreenshot);
10941095
behaviorContext->Method("CaptureScreenshotWithImGui", &Script_CaptureScreenshotWithImGui);
@@ -1405,9 +1406,8 @@ namespace AtomSampleViewer
14051406
s_instance->m_scriptOperations.push(AZStd::move(operation));
14061407
}
14071408

1408-
bool ScriptManager::PrepareForScreenCapture(const AZStd::string& path)
1409+
bool ScriptManager::PrepareForScreenCapture(const AZStd::string& path, const AZStd::string& envPath)
14091410
{
1410-
14111411
if (!Utils::IsFileUnderFolder(Utils::ResolvePath(path), ScreenshotPaths::GetScreenshotsFolder(true)))
14121412
{
14131413
// The main reason we require screenshots to be in a specific folder is to ensure we don't delete or replace some other important file.
@@ -1425,7 +1425,7 @@ namespace AtomSampleViewer
14251425
return false;
14261426
}
14271427

1428-
s_instance->m_scriptReporter.AddScreenshotTest(path);
1428+
s_instance->m_scriptReporter.AddScreenshotTest(path, envPath);
14291429

14301430
s_instance->m_isCapturePending = true;
14311431
s_instance->AZ::Render::FrameCaptureNotificationBus::Handler::BusConnect();
@@ -1434,6 +1434,11 @@ namespace AtomSampleViewer
14341434
return true;
14351435
}
14361436

1437+
void ScriptManager::Script_SetTestEnvPath(const AZStd::string& envPath)
1438+
{
1439+
s_instance->m_envPath = envPath;
1440+
}
1441+
14371442
void ScriptManager::Script_SelectImageComparisonToleranceLevel(const AZStd::string& presetName)
14381443
{
14391444
auto operation = [presetName]()
@@ -1451,7 +1456,7 @@ namespace AtomSampleViewer
14511456
auto operation = [filePath]()
14521457
{
14531458
// Note this will pause the script until the capture is complete
1454-
if (PrepareForScreenCapture(filePath))
1459+
if (PrepareForScreenCapture(filePath, s_instance->m_envPath))
14551460
{
14561461
AZ_Assert(s_instance->m_frameCaptureId == AZ::Render::FrameCaptureRequests::s_InvalidFrameCaptureId, "Attempting to start a capture while one is in progress");
14571462
uint32_t frameCaptureId = AZ::Render::FrameCaptureRequests::s_InvalidFrameCaptureId;
@@ -1484,7 +1489,7 @@ namespace AtomSampleViewer
14841489
auto operation = [filePath]()
14851490
{
14861491
// Note this will pause the script until the capture is complete
1487-
if (PrepareForScreenCapture(filePath))
1492+
if (PrepareForScreenCapture(filePath, s_instance->m_envPath))
14881493
{
14891494
AZ_Assert(s_instance->m_frameCaptureId == AZ::Render::FrameCaptureRequests::s_InvalidFrameCaptureId, "Attempting to start a capture while one is in progress");
14901495
uint32_t frameCaptureId = AZ::Render::FrameCaptureRequests::s_InvalidFrameCaptureId;
@@ -1514,7 +1519,7 @@ namespace AtomSampleViewer
15141519
auto operation = [filePath]()
15151520
{
15161521
// Note this will pause the script until the capture is complete
1517-
if (PrepareForScreenCapture(filePath))
1522+
if (PrepareForScreenCapture(filePath, s_instance->m_envPath))
15181523
{
15191524
AZ_Assert(s_instance->m_frameCaptureId == AZ::Render::FrameCaptureRequests::s_InvalidFrameCaptureId, "Attempting to start a capture while one is in progress");
15201525
uint32_t frameCaptureId = AZ::Render::FrameCaptureRequests::s_InvalidFrameCaptureId;
@@ -1611,7 +1616,7 @@ namespace AtomSampleViewer
16111616
auto operation = [passHierarchy, slot, outputFilePath, readbackOption]()
16121617
{
16131618
// Note this will pause the script until the capture is complete
1614-
if (PrepareForScreenCapture(outputFilePath))
1619+
if (PrepareForScreenCapture(outputFilePath, s_instance->m_envPath))
16151620
{
16161621
AZ_Assert(s_instance->m_frameCaptureId == AZ::Render::FrameCaptureRequests::s_InvalidFrameCaptureId, "Attempting to start a capture while one is in progress");
16171622
uint32_t frameCaptureId = AZ::Render::FrameCaptureRequests::s_InvalidFrameCaptureId;

‎Gem/Code/Source/Automation/ScriptManager.h‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,8 @@ namespace AtomSampleViewer
124124
static void Script_ShowTool(const AZStd::string& toolName, bool enable);
125125

126126
// Screenshots...
127+
// Store the test environment path of the screenshots. It will be used to figure out the baseline path.
128+
static void Script_SetTestEnvPath(const AZStd::string& envPath);
127129

128130
// Call this function before capturing screenshots to indicate which comparison tolerance level should be used.
129131
// The list of available tolerance levels can be found in "AtomSampleViewer/Config/ImageComparisonToleranceLevels.azasset".
@@ -233,7 +235,7 @@ namespace AtomSampleViewer
233235
// Validates the ScriptDataContext for ProfilingCapture script requests
234236
static bool ValidateProfilingCaptureScripContexts(AZ::ScriptDataContext& dc, AZStd::string& outputFilePath);
235237

236-
static bool PrepareForScreenCapture(const AZStd::string& path);
238+
static bool PrepareForScreenCapture(const AZStd::string& path, const AZStd::string& envPath);
237239

238240
// show/hide imgui
239241
void SetShowImGui(bool show);
@@ -249,6 +251,8 @@ namespace AtomSampleViewer
249251

250252
TestSuiteExecutionConfig m_testSuiteRunConfig;
251253

254+
AZStd::string m_envPath = "";
255+
252256
static constexpr float DefaultPauseTimeout = 5.0f;
253257

254258
int m_scriptIdleFrames = 0;

‎Gem/Code/Source/Automation/ScriptReporter.cpp‎

Lines changed: 18 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ namespace AtomSampleViewer
4545

4646
AZStd::string GetLocalBaselineFolder(bool resolvePath)
4747
{
48-
AZStd::string path = AZStd::string::format("@user@/scripts/screenshotslocalbaseline/%s", AZ::RHI::Factory::Get().GetName().GetCStr());
48+
AZStd::string path = AZStd::string::format("@user@/scripts/screenshotslocalbaseline/");
4949

5050
if (resolvePath)
5151
{
@@ -76,10 +76,10 @@ namespace AtomSampleViewer
7676
{
7777
newPath = "";
7878
}
79-
return newPath;
79+
return Utils::ResolvePath(newPath);
8080
}
8181

82-
AZStd::string GetOfficialBaseline(const AZStd::string& forScreenshotFile)
82+
AZStd::string GetOfficialBaseline(const AZStd::string& forScreenshotFile, const AZStd::string& envPath)
8383
{
8484
AZStd::string path = forScreenshotFile;
8585
const AZStd::string userPath = Utils::ResolvePath("@user@");
@@ -96,6 +96,10 @@ namespace AtomSampleViewer
9696
return "";
9797
}
9898

99+
if (!AzFramework::StringFunc::Replace(path, envPath.c_str(), ""))
100+
{
101+
return "";
102+
}
99103
// Turn it back into a full path
100104
path = Utils::ResolvePath("@projectroot@/" + path);
101105

@@ -204,12 +208,18 @@ namespace AtomSampleViewer
204208
return !m_currentScriptIndexStack.empty();
205209
}
206210

207-
bool ScriptReporter::AddScreenshotTest(const AZStd::string& path)
211+
ScriptReporter::ScreenshotTestInfo::ScreenshotTestInfo(const AZStd::string& screenshotFilePath, const AZStd::string& envPath)
212+
: m_screenshotFilePath(Utils::ResolvePath(screenshotFilePath))
213+
{
214+
m_officialBaselineScreenshotFilePath = ScreenshotPaths::GetOfficialBaseline(m_screenshotFilePath, envPath);
215+
m_localBaselineScreenshotFilePath = ScreenshotPaths::GetLocalBaseline(m_screenshotFilePath);
216+
}
217+
218+
bool ScriptReporter::AddScreenshotTest(const AZStd::string& path, const AZStd::string& envPath)
208219
{
209220
AZ_Assert(GetCurrentScriptReport(), "There is no active script");
210221

211-
ScreenshotTestInfo screenshotTestInfo;
212-
screenshotTestInfo.m_screenshotFilePath = path;
222+
ScreenshotTestInfo screenshotTestInfo(path, envPath);
213223
GetCurrentScriptReport()->m_screenshotTests.push_back(AZStd::move(screenshotTestInfo));
214224

215225
return true;
@@ -1000,7 +1010,7 @@ namespace AtomSampleViewer
10001010

10011011
bool ScriptReporter::UpdateLocalBaselineImage(ScreenshotTestInfo& screenshotTest, bool showResultDialog)
10021012
{
1003-
const AZStd::string destinationFile = ScreenshotPaths::GetLocalBaseline(screenshotTest.m_screenshotFilePath);
1013+
const AZStd::string destinationFile = screenshotTest.m_localBaselineScreenshotFilePath;
10041014

10051015
AZStd::string destinationFolder = destinationFile;
10061016
AzFramework::StringFunc::Path::StripFullName(destinationFolder);
@@ -1055,7 +1065,7 @@ namespace AtomSampleViewer
10551065
}
10561066

10571067
// Get official cache baseline file
1058-
const AZStd::string cacheFilePath = ScreenshotPaths::GetOfficialBaseline(screenshotTest.m_screenshotFilePath);
1068+
const AZStd::string cacheFilePath = screenshotTest.m_officialBaselineScreenshotFilePath;
10591069

10601070
// Divide cache file path into components to we can access the file name and the parent folder
10611071
AZStd::fixed_vector<AZ::IO::FixedMaxPathString, 16> reversePathComponents;
@@ -1160,7 +1170,6 @@ namespace AtomSampleViewer
11601170

11611171
screenshotTestInfo.m_toleranceLevel = *toleranceLevel;
11621172

1163-
screenshotTestInfo.m_officialBaselineScreenshotFilePath = ScreenshotPaths::GetOfficialBaseline(screenshotTestInfo.m_screenshotFilePath);
11641173
if (screenshotTestInfo.m_officialBaselineScreenshotFilePath.empty())
11651174
{
11661175
ReportScriptError(AZStd::string::format("Screenshot check failed. Could not determine expected screenshot path for '%s'", screenshotTestInfo.m_screenshotFilePath.c_str()));
@@ -1199,7 +1208,6 @@ namespace AtomSampleViewer
11991208
}
12001209
}
12011210

1202-
screenshotTestInfo.m_localBaselineScreenshotFilePath = ScreenshotPaths::GetLocalBaseline(screenshotTestInfo.m_screenshotFilePath);
12031211
if (screenshotTestInfo.m_localBaselineScreenshotFilePath.empty())
12041212
{
12051213
ReportScriptWarning(AZStd::string::format("Screenshot check failed. Could not determine local baseline screenshot path for '%s'", screenshotTestInfo.m_screenshotFilePath.c_str()));

‎Gem/Code/Source/Automation/ScriptReporter.h‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ namespace AtomSampleViewer
3838
AZStd::string GetLocalBaseline(const AZStd::string& forScreenshotFile);
3939

4040
//! Returns the path to the official baseline image that corresponds to @forScreenshotFile
41-
AZStd::string GetOfficialBaseline(const AZStd::string& forScreenshotFile);
41+
AZStd::string GetOfficialBaseline(const AZStd::string& forScreenshotFile, const AZStd::string& envPath);
4242
}
4343

4444
//! Collects data about each script run by the ScriptManager.
@@ -75,7 +75,7 @@ namespace AtomSampleViewer
7575
bool HasActiveScript() const;
7676

7777
//! Indicates that a new screenshot is about to be captured.
78-
bool AddScreenshotTest(const AZStd::string& path);
78+
bool AddScreenshotTest(const AZStd::string& path, const AZStd::string& envPath);
7979

8080
//! Check the latest screenshot using default thresholds.
8181
void CheckLatestScreenshot(const ImageComparisonToleranceLevel* comparisonPreset);
@@ -139,6 +139,8 @@ namespace AtomSampleViewer
139139
ImageComparisonToleranceLevel m_toleranceLevel; //!< Tolerance for checking against the official baseline image
140140
ImageComparisonResult m_officialComparisonResult; //!< Result of comparing against the official baseline image, for reporting test failure
141141
ImageComparisonResult m_localComparisonResult; //!< Result of comparing against a local baseline, for reporting warnings
142+
143+
ScreenshotTestInfo(const AZStd::string& screenshotFilePath, const AZStd::string& envPath);
142144
};
143145

144146
//! Records all the information about a single test script.

‎Scripts/AreaLightTest.bv.lua‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
--
1010
----------------------------------------------------------------------------------------------------
1111

12+
RunScript("scripts/TestEnvironment.luac")
13+
1214
function SetupPointLights()
1315
SetImguiValue('AreaLightSample/LightType/Point', true)
1416
SetImguiValue('AreaLightSample/Position Offset', Vector3(0.0, 0.0, 0.0))
@@ -109,7 +111,7 @@ function VaryMetallicOnly()
109111
SetImguiValue('AreaLightSample/Min Max Metallic', Vector2(0.0, 1.0))
110112
end
111113

112-
g_screenshotOutputFolder = ResolvePath('@user@/Scripts/Screenshots/AreaLights/')
114+
g_screenshotOutputFolder = ResolvePath('@user@/Scripts/Screenshots/' .. g_testEnv .. '/AreaLights/')
113115
Print('Saving screenshots to ' .. NormalizePath(g_screenshotOutputFolder))
114116
SelectImageComparisonToleranceLevel("Level E")
115117

‎Scripts/AuxGeom.bv.lua‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
--
1010
----------------------------------------------------------------------------------------------------
1111

12+
RunScript("scripts/TestEnvironment.luac")
13+
1214
function TakeScreenShotBoxes()
1315

1416
NoClipCameraController_SetFov(DegToRad(70))
@@ -29,7 +31,7 @@ function TakeScreenShotShapes()
2931
CaptureScreenshot(g_screenshotOutputFolder .. '/auxgeom_shapes.png')
3032
end
3133

32-
g_screenshotOutputFolder = ResolvePath('@user@/Scripts/Screenshots/AuxGeom/')
34+
g_screenshotOutputFolder = ResolvePath('@user@/Scripts/Screenshots/' .. g_testEnv .. '/AuxGeom/')
3335
Print('Saving screenshots to ' .. NormalizePath(g_screenshotOutputFolder))
3436

3537
OpenSample('RPI/AuxGeom')

‎Scripts/CheckerboardTest.bv.lua‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@
99
--
1010
----------------------------------------------------------------------------------------------------
1111

12-
g_screenshotOutputFolder = ResolvePath('@user@/Scripts/Screenshots/Checkerboard/')
12+
RunScript("scripts/TestEnvironment.luac")
13+
14+
g_screenshotOutputFolder = ResolvePath('@user@/Scripts/Screenshots/' .. g_testEnv .. '/Checkerboard/')
1315
Print('Saving screenshots to ' .. NormalizePath(g_screenshotOutputFolder))
1416

1517
OpenSample('Features/Checkerboard')

‎Scripts/CullingAndLod.bv.lua‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@
99
--
1010
----------------------------------------------------------------------------------------------------
1111

12-
g_screenshotOutputFolder = ResolvePath('@user@/Scripts/Screenshots/CullingAndLod/')
12+
RunScript("scripts/TestEnvironment.luac")
13+
14+
g_screenshotOutputFolder = ResolvePath('@user@/Scripts/Screenshots/' .. g_testEnv .. '/CullingAndLod/')
1315
Print('Saving screenshots to ' .. NormalizePath(g_screenshotOutputFolder))
1416

1517
OpenSample('RPI/CullingAndLod')

‎Scripts/Decals.bv.lua‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,13 @@
99
--
1010
----------------------------------------------------------------------------------------------------
1111

12+
RunScript("scripts/TestEnvironment.luac")
13+
1214
function TakeScreenshots()
1315
CaptureScreenshot(g_screenshotOutputFolder .. '/screenshot_decals.png')
1416
end
1517

16-
g_screenshotOutputFolder = ResolvePath('@user@/Scripts/Screenshots/Decals/')
18+
g_screenshotOutputFolder = ResolvePath('@user@/Scripts/Screenshots/' .. g_testEnv .. '/Decals/')
1719
Print('Saving screenshots to ' .. NormalizePath(g_screenshotOutputFolder))
1820

1921
OpenSample('RPI/Decals')

‎Scripts/DepthOfFieldTest.bv.lua‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@
99
--
1010
----------------------------------------------------------------------------------------------------
1111

12-
g_screenshotOutputFolder = ResolvePath('@user@/Scripts/Screenshots/DepthOfFieldTest/')
12+
RunScript("scripts/TestEnvironment.luac")
13+
14+
g_screenshotOutputFolder = ResolvePath('@user@/Scripts/Screenshots/' .. g_testEnv .. '/DepthOfFieldTest/')
1315
Print('Saving screenshots to ' .. NormalizePath(g_screenshotOutputFolder))
1416

1517
OpenSample('Features/DepthOfField')

0 commit comments

Comments
 (0)