Merge branch 'development' of https://github.com/o3de/o3de into daimini/dragAndDropAdjustment

Signed-off-by: Danilo Aimini <[email protected]>

Author: AMZN-daimini

Code/Framework/AzCore/AzCore/IO/Streamer/FileRequest.h

@@ -270,8 +270,12 @@ namespace AZ::IO::Requests
 
 namespace AZ::IO
 {
+    class Streamer_SchedulerTest_RequestSorting_Test;
+
     class FileRequest final
     {
+        friend Streamer_SchedulerTest_RequestSorting_Test;
+
     public:
         inline constexpr static AZStd::chrono::steady_clock::time_point s_noDeadlineTime = AZStd::chrono::steady_clock::time_point::max();
 
@@ -407,6 +411,7 @@ namespace AZ::IO
         friend class StreamerContext;
         friend class Scheduler;
         friend class Device;
+        friend class Streamer_SchedulerTest_RequestSorting_Test;
         friend bool operator==(const FileRequestHandle& lhs, const FileRequestPtr& rhs);
 
     public:

Code/Framework/AzCore/AzCore/IO/Streamer/Scheduler.cpp

@@ -501,6 +501,11 @@ namespace AZ::IO
 
     auto Scheduler::Thread_PrioritizeRequests(const FileRequest* first, const FileRequest* second) const -> Order
     {
+        if (first == second)
+        {
+            return Order::Equal;
+        }
+
         // Sort by order priority of the command in the request. This allows to for instance have cancel request
         // always happen before any other requests.
         auto order = [](auto&& args)
@@ -544,7 +549,12 @@ namespace AZ::IO
             }
 
             // If neither has started and have the same priority, prefer to start the closest deadline.
-            return firstRead->m_deadline <= secondRead->m_deadline ? Order::FirstRequest : Order::SecondRequest;
+            if (firstRead->m_deadline == secondRead->m_deadline)
+            {
+                return Order::Equal;
+            }
+
+            return firstRead->m_deadline < secondRead->m_deadline ? Order::FirstRequest : Order::SecondRequest;
         }
 
         // Check if one of the requests is in panic and prefer to prioritize that request
@@ -593,7 +603,13 @@ namespace AZ::IO
             s64 secondReadOffset = AZStd::visit(offset, second->GetCommand());
             s64 firstSeekDistance = abs(aznumeric_cast<s64>(m_threadData.m_lastFileOffset) - firstReadOffset);
             s64 secondSeekDistance = abs(aznumeric_cast<s64>(m_threadData.m_lastFileOffset) - secondReadOffset);
-            return firstSeekDistance <= secondSeekDistance ? Order::FirstRequest : Order::SecondRequest;
+
+            if (firstSeekDistance == secondSeekDistance)
+            {
+                return Order::Equal;
+            }
+            
+            return firstSeekDistance < secondSeekDistance ? Order::FirstRequest : Order::SecondRequest;
         }
 
         // Prefer to continue in the same file so prioritize the request that's in the same file
@@ -625,6 +641,13 @@ namespace AZ::IO
                 "Scheduler::Thread_ScheduleRequests - Sorting %i requests", m_context.GetNumPreparedRequests());
             auto sorter = [this](const FileRequest* lhs, const FileRequest* rhs) -> bool
             {
+                if (lhs == rhs)
+                {
+                    // AZStd::sort may compare an element to itself; 
+                    //   it's required this condition remain consistent and return false.
+                    return false;
+                }
+
                 Order order = Thread_PrioritizeRequests(lhs, rhs);
                 switch (order)
                 {

Code/Framework/AzCore/AzCore/IO/Streamer/Scheduler.h

@@ -24,6 +24,7 @@
 namespace AZ::IO
 {
     class FileRequest;
+    class Streamer_SchedulerTest_RequestSorting_Test;
 
     namespace Requests
     {
@@ -65,6 +66,7 @@ namespace AZ::IO
         void GetRecommendations(IStreamerTypes::Recommendations& recommendations) const;
 
     private:
+        friend class Streamer_SchedulerTest_RequestSorting_Test;
         inline static constexpr u32 ProfilerColor = 0x0080ffff; //!< A lite shade of blue. (See https://www.color-hex.com/color/0080ff).
 
         void Thread_MainLoop();

Code/Framework/AzCore/AzCore/IO/Streamer/Streamer.h

@@ -28,13 +28,15 @@ namespace AZ::IO::Requests
 namespace AZ::IO
 {
     class StreamStackEntry;
+    class Streamer_SchedulerTest_RequestSorting_Test;
 
     /**
      * Data streamer.
      */
     class Streamer final
         : public AZ::IO::IStreamer
     {
+        friend Streamer_SchedulerTest_RequestSorting_Test;
     public:
         AZ_RTTI(Streamer, "{3D880982-6E3F-4913-9947-55E01030D4AA}", IStreamer);
         AZ_CLASS_ALLOCATOR(Streamer, SystemAllocator);

Code/Framework/AzCore/Tests/Streamer/SchedulerTests.cpp

@@ -378,4 +378,54 @@ namespace AZ::IO
 
         EXPECT_EQ(Iterations + 1, counter);
     }
+
+    TEST_F(Streamer_SchedulerTest, RequestSorting)
+    {
+        //////////////////////////////////////////////////////////////
+        // Test equal priority requests that are past their deadlines (aka panic)
+        //////////////////////////////////////////////////////////////
+        IStreamerTypes::Deadline panicDeadline(IStreamerTypes::Deadline::min());
+        auto estimatedCompleteTime = AZStd::chrono::steady_clock::now();
+        char fakeBuffer[8];
+        FileRequestPtr panicRequest = m_streamer->Read("PanicRequest", fakeBuffer, sizeof(fakeBuffer), 8, panicDeadline);
+        panicRequest->m_request.SetEstimatedCompletion(estimatedCompleteTime);
+
+        // Passed deadline, same object (same pointer)
+        EXPECT_EQ(
+            m_streamer->m_streamStack->Thread_PrioritizeRequests(&panicRequest->m_request, &panicRequest->m_request),
+            Scheduler::Order::Equal);
+
+        // Passed deadline, different object
+        FileRequestPtr panicRequest2 = m_streamer->Read("PanicRequest2", fakeBuffer, sizeof(fakeBuffer), 8, panicDeadline);
+        panicRequest2->m_request.SetEstimatedCompletion(estimatedCompleteTime);
+        EXPECT_EQ(
+            m_streamer->m_streamStack->Thread_PrioritizeRequests(&panicRequest->m_request, &panicRequest2->m_request),
+            Scheduler::Order::Equal);
+
+
+        //////////////////////////////////////////////////////////////
+        // Test equal priority requests that are both reading the same file
+        //////////////////////////////////////////////////////////////
+        FileRequestPtr readRequest = m_streamer->Read("SameFile", fakeBuffer, sizeof(fakeBuffer), 8, panicDeadline);
+        FileRequestPtr sameFileRequest = m_streamer->CreateRequest();
+        sameFileRequest->m_request.CreateRead(&sameFileRequest->m_request, fakeBuffer, 8, RequestPath(), 0, 8);
+        sameFileRequest->m_request.m_parent = &readRequest->m_request;
+        sameFileRequest->m_request.m_dependencies = 0;
+
+        // Same file read, same object (same pointer)
+        EXPECT_EQ(
+            m_streamer->m_streamStack->Thread_PrioritizeRequests(&sameFileRequest->m_request, &sameFileRequest->m_request),
+            Scheduler::Order::Equal);
+
+        FileRequestPtr readRequest2 = m_streamer->Read("SameFile2", fakeBuffer, sizeof(fakeBuffer), 8, panicDeadline);
+        FileRequestPtr sameFileRequest2 = m_streamer->CreateRequest();
+        sameFileRequest2->m_request.CreateRead(&sameFileRequest2->m_request, fakeBuffer, 8, RequestPath(), 0, 8);
+        sameFileRequest2->m_request.m_parent = &readRequest2->m_request;
+        sameFileRequest2->m_request.m_dependencies = 0;
+
+        // Same file read, different objects
+        EXPECT_EQ(
+            m_streamer->m_streamStack->Thread_PrioritizeRequests(&sameFileRequest->m_request, &sameFileRequest2->m_request),
+            Scheduler::Order::Equal);
+    }
 } // namespace AZ::IO

Gems/AtomLyIntegration/CommonFeatures/Assets/Editor/Scripts/LegacyContentConversion/LegacyActorComponentConverter.py

@@ -37,7 +37,7 @@ def is_this_the_component_im_looking_for(self, xmlElement, parent):
 
     def gather_info_for_conversion(self, xmlElement, parent):
         # We don't need to modify the actor component, but we do need to add a material component
-        for possibleActorAssetComponent in xmlElement.getchildren():
+        for possibleActorAssetComponent in xmlElement.iter():
             if "field" in possibleActorAssetComponent.keys() and possibleActorAssetComponent.get("field") == "ActorAsset" and "value" in possibleActorAssetComponent.keys():
                 assetId = possibleActorAssetComponent.get("value")
 

Gems/AtomLyIntegration/CommonFeatures/Assets/Editor/Scripts/LegacyContentConversion/LegacyComponentConverter.py

@@ -15,14 +15,17 @@
 project and attempts to convert the following components into something
 reasonably similar that renders in Atom:
 Mesh
+Material
+PointLight
 Actor
 
 
 Do materials get carried over?
 ================================================
 For the mesh component, this script will attempt to create an atom mesh component
 that uses the same material, pre-supposing that you have already run the 
-LegacyMaterialConverter.py script sto generate Atom .material files out of legacy .mtl files
+LegacyAssetConverter script to generate Atom .material files out of legacy .mtl files.
+(Gems\AtomLyIntegration\TechnicalArt\DccScriptingInterface\SDK\Maya\Scripts\Python\legacy_asset_converter\main.py)
 For meshes that only have one sub-mesh, this is straightforward as the mesh will only
 have one material to apply, and this script will look for a material with the same
 name but a .material extension.
@@ -35,14 +38,26 @@
 re-named the submaterials, it should find a match. This applies to both the
 ActorComponent and the MeshComponent
 
+
+What does this script needs?
+================================================
+This script will parse the assetcatalog.xml from the cache folder to get asset ids and path.
+This file is in binary by default, you need to rebuild the engine with a modification to make it useable by this script.
+    - In lumberyard, modify dev\Code\Tools\AssetProcessor\native\AssetManager\AssetCatalog.cpp line 343 to use AZ::ObjectStream::ST_XML instead of ST_BINARY
+    - In o3de (if you use assetCatalogOverridePath), modify Code\Framework\AzFramework\AzFramework\Asset\AssetCatalog.cpp 
+      and Code\Tools\AssetProcessor\native\AssetManager\AssetCatalog.cpp in the same fashion
+
+Note that after the modification and rebuild, you will need to delete the cache folder and launch the engine for the catalog to be rebuilt
+
 How do I run this script from a command line?
 ================================================
 1) Check out any .slice, .layer, .ly, and .cry files you want to convert from source control
     - This script will remove the legacy components entirely, so make sure you have your files
       backed up before you run this script in case you want to run it again
-2) From the Lumberyard root folder, run LegacyComponentConverter.py project=<ProjectName> --include_gems
-    - --include_gems is optional
-    - if you include Gems, it will run all all Gems, not just the ones enabled by your project
+2) From the Lumberyard root folder, run LegacyComponentConverter.py -project=<ProjectName> -include_gems -assetCatalogOverridePath=<ExternalProjectPath\Cache\pc\assetcatalog.xml>
+    - -include_gems is optional. If you include Gems, it will run all all Gems, not just the ones enabled by your project
+    - -assetCatalogOverridePath is optional. It will replace mesh/material asset id from the source project to this target project
+      based on relative asset path from the project root (in general, Objects/Models/yourasset.fbx)
 
 
 What are the artifacts of this script?
@@ -59,12 +74,10 @@
 CONVERTED_LOG_NAME = "ComponentConversion_ConvertedLegacyFiles.log"
 UNCONVERTED_LOG_NAME = "ComponentConversion_UnsupportedLegacyFiles.log"
 STATS_LOG_NAME = "ComponentConversion_LegacyComponentStats.log"
-BUILD_PATH = None
-GEMS_PATH = None
 
 # Normal imports
 import sys
-import xml.etree.ElementTree
+import xml.etree.ElementTree as ET
 import time
 from zipfile import ZipFile
 import tempfile
@@ -76,8 +89,9 @@
 from LegacyMaterialComponentConverter import *
 from LegacyActorComponentConverter import *
 from LegacyPointLightComponentConverter import *
+from LegacyTransformComponentConverter import *
 
-BUILD_PATH = "./"
+BUILD_PATH = "./" # Use current working directory, we expect to be in lumberyard dev folder
 GEMS_PATH = os.path.join(BUILD_PATH, "Gems")
 
 class Component_File(object):
@@ -169,6 +183,7 @@ def gather_elements(self):
         componentConverters.append(Mesh_Component_Converter(self.assetCatalogHelper, self.statsCollector, self.normalizedProjectDir))
         componentConverters.append(Actor_Component_Converter(self.assetCatalogHelper, self.statsCollector, self.normalizedProjectDir))
         componentConverters.append(Point_Light_Component_Converter(self.assetCatalogHelper, self.statsCollector, self.normalizedProjectDir))
+        componentConverters.append(Transform_Component_Converter(self.assetCatalogHelper, self.statsCollector, self.normalizedProjectDir))
 
         if self.is_valid_xml():
             root = self.xml.getroot()
@@ -190,7 +205,10 @@ def gather_elements(self):
                                 # TODO - we're about to change the tree structure while iterating, which is apparently undefined but appears to work. Might be better to just build up a list of things that need to be modified, then do a second pass to replace the legacy component
                                 # Seems to be okay since we only change or add elements, never remove entirely
                                 componentConverter.convert(child, parent)
-            self.xml._setroot(root)                        
+            self.xml._setroot(root)    
+            # pretty print 
+            ET.indent(self.xml, space='\t')
+
         print("finished parsing {0}".format(self.filename))
 
     def can_be_converted(self):
@@ -265,12 +283,12 @@ def getUpdatedStatsCollector(self):
 def main():
     '''sys.__name__ wrapper function'''
 
-    msgStr = "This tool will scan all of your project's level/layer/slice files\n\
+    msgStr = "This tool will scan all of your lumberyard project's level/layer/slice files\n\
 convert any compatible legacy components into the equivalent Atom components\n\
 This script will overwrite the original files, and will remove the legacy components\n\
 upon conversion, decimating the previous contents of those components.\n"
 
-    commandLineOptions = Common_Command_Line_Options(sys.argv[0], sys.argv[1])
+    commandLineOptions = Common_Command_Line_Options(sys.argv)
     if commandLineOptions.isHelp:
         print (commandLineOptions.helpString)
         return
@@ -281,8 +299,13 @@ def main():
     extensionList = [".slice", ".layer", ".ly", ".cry"]
     fileList = get_file_list(commandLineOptions.projectName, commandLineOptions.includeGems, extensionList, BUILD_PATH, GEMS_PATH)
 
-    assetCatalogDictionaries = get_asset_catalog_dictionaries(BUILD_PATH, commandLineOptions.projectName)
-
+    if commandLineOptions.assetCatalogOverridePath:
+        assetCatalogPath = commandLineOptions.assetCatalogOverridePath
+        assetCatalogDictionaries = get_asset_catalog_dictionaries(assetCatalogPath)
+    else:
+        assetCatalogPath = os.path.join("Cache", commandLineOptions.projectName, get_default_asset_platform(), commandLineOptions.projectName, "assetcatalog.xml")
+        assetCatalogDictionaries = get_asset_catalog_dictionaries(assetCatalogPath)
+        
     # Create a log file to store converted component file filenames
     # and to check to see if the component file has already been converted.
     convertedLogFile = Log_File(filename="{0}\\{1}".format(BUILD_PATH, CONVERTED_LOG_NAME))

Gems/AtomLyIntegration/CommonFeatures/Assets/Editor/Scripts/LegacyContentConversion/LegacyConversionHelpers.py

@@ -8,6 +8,7 @@
 """
 
 import os
+import platform
 import xml.etree.ElementTree
 
 class Stats_Collector(object):
@@ -101,30 +102,36 @@ class Common_Command_Line_Options(object):
     """
     Some common options/parsing
     """
-    def __init__(self, argv0, argv1):
-        arguments = argv1.split('-')
-
+    def __init__(self, args):
         self.projectName = ""
+        self.assetCatalogOverridePath = ""
         self.includeGems = False
         self.useP4 = False
         self.endsWithStr = ""
         self.isHelp = False        
-        self.helpString = "usage: {0} -project=<project name> -include_gems -ends_with=<filter> -use_p4\n\
+        self.helpString = f"usage: {args[0]} -project=<project name> -include_gems -ends_with=<filter> -use_p4 -assetCatalogOverridePath=<path>\n\
             E.g.:\n\
-            {1} -project=StarterGame -include_gems\n\
-              -project is required.\n\
+              -project=StarterGame -include_gems\n\
+              -project is required. Path is relative\n\
               -include_gems is optional, and by default gems will not be included.\n\
               -ends_with is optional. It could be used to filter for a specific file (--ends_with=default.mtl)\n\
-              -use_p4 is optional. It will use the p4 command line to check out files that are edited in your default changelist".format(argv0, argv0)
-
-        for argument in arguments:
-            argument = argument.rstrip(" ")
-            if argument == "h" or argument == "help" or argv1 == "?":
+              -use_p4 is optional. It will use the p4 command line to check out files that are edited in your default changelist\n\
+              -assetCatalogOverridePath is optional. It will use asset ids from this file instead of using the current project asset catalog\n\
+               (match via relative asset path to the project root folder)"
+              
+
+        for argument in args:
+            argument = argument.rstrip(" ").lstrip("-")
+            if argument == "h" or argument == "help" or argument == "?":
                 self.isHelp = True
             elif argument.startswith("project"):
                 projectArgs = argument.split("=")
                 if len(projectArgs) > 1:
                     self.projectName = projectArgs[1]
+            elif argument.startswith("assetCatalogOverridePath"):
+                projectArgs = argument.split("=")
+                if len(projectArgs) > 1:
+                    self.assetCatalogOverridePath = projectArgs[1]
             elif argument == "include_gems":
                 self.includeGems = True
             elif argument == "use_p4":
@@ -204,7 +211,7 @@ def get_asset_id_from_relative_path(self, relativePath):
             return self.relativePathToAssetIdDict[relativePath]
         return ""
 
-def get_asset_catalog_dictionaries(buildPath, projectName):
+def get_asset_catalog_dictionaries(assetCatalogPath):
     """
     This function pre-supposes that you have modified AssetCatalog::SaveRegistry_Impl
     in Code/Tools/AssetProcessor/native/AssetManager/AssetCatalog.cpp
@@ -218,7 +225,6 @@ def get_asset_catalog_dictionaries(buildPath, projectName):
     relativePathToAssetIdDict[""] = "{00000000-0000-0000-0000-000000000000}:0"
     assetIdToRelativePathDict = {}
     assetUuidToAssetIdsDict = {}
-    assetCatalogPath = os.path.join(projectName, "Cache", "pc", "assetcatalog.xml")
     assetCatalogXml = xml.etree.ElementTree.parse(assetCatalogPath)
     for possibleAssetInfo in assetCatalogXml.getroot().iter('Class'):
         if "name" in possibleAssetInfo.keys() and possibleAssetInfo.get("name") == "AssetInfo":
@@ -278,6 +284,12 @@ def get_subid_from_assetId(assetId):
     separatorIndex = assetId.find(":") + 1
     return assetId[separatorIndex:]
 
+def get_default_asset_platform():
+    host_platform_to_asset_platform_map = { 'windows': 'pc',
+                                            'linux':   'linux',
+                                            'darwin':  'mac' }
+    return host_platform_to_asset_platform_map.get(platform.system().lower(), "")
+
 class Component_Converter(object):
     """
     Converter base class