new: Allow to use the LegacyComponentConverter between a lumberyard and o3de project (o3de#17397)

* fix: use the correct cache location and fix for python 3.10
* new: Allow to use specific asset catalog as new arg
* fix: Arguments parsing more than one arg
* clean: handle review comments
* clean: handle review comments 2
* fix: Remove default and unused material slot from entry
* new: Support non uniform scale conversion

---------

Signed-off-by: guillaume-haerinck <[email protected]>

Author: guillaume-haerinck

Gems/AtomLyIntegration/CommonFeatures/Assets/Editor/Scripts/LegacyContentConversion/LegacyActorComponentConverter.py

@@ -37,7 +37,7 @@ def is_this_the_component_im_looking_for(self, xmlElement, parent):
 
     def gather_info_for_conversion(self, xmlElement, parent):
         # We don't need to modify the actor component, but we do need to add a material component
-        for possibleActorAssetComponent in xmlElement.getchildren():
+        for possibleActorAssetComponent in xmlElement.iter():
             if "field" in possibleActorAssetComponent.keys() and possibleActorAssetComponent.get("field") == "ActorAsset" and "value" in possibleActorAssetComponent.keys():
                 assetId = possibleActorAssetComponent.get("value")
 

Gems/AtomLyIntegration/CommonFeatures/Assets/Editor/Scripts/LegacyContentConversion/LegacyComponentConverter.py

@@ -15,14 +15,17 @@
 project and attempts to convert the following components into something
 reasonably similar that renders in Atom:
 Mesh
+Material
+PointLight
 Actor
 
 
 Do materials get carried over?
 ================================================
 For the mesh component, this script will attempt to create an atom mesh component
 that uses the same material, pre-supposing that you have already run the 
-LegacyMaterialConverter.py script sto generate Atom .material files out of legacy .mtl files
+LegacyAssetConverter script to generate Atom .material files out of legacy .mtl files.
+(Gems\AtomLyIntegration\TechnicalArt\DccScriptingInterface\SDK\Maya\Scripts\Python\legacy_asset_converter\main.py)
 For meshes that only have one sub-mesh, this is straightforward as the mesh will only
 have one material to apply, and this script will look for a material with the same
 name but a .material extension.
@@ -35,14 +38,26 @@
 re-named the submaterials, it should find a match. This applies to both the
 ActorComponent and the MeshComponent
 
+
+What does this script needs?
+================================================
+This script will parse the assetcatalog.xml from the cache folder to get asset ids and path.
+This file is in binary by default, you need to rebuild the engine with a modification to make it useable by this script.
+    - In lumberyard, modify dev\Code\Tools\AssetProcessor\native\AssetManager\AssetCatalog.cpp line 343 to use AZ::ObjectStream::ST_XML instead of ST_BINARY
+    - In o3de (if you use assetCatalogOverridePath), modify Code\Framework\AzFramework\AzFramework\Asset\AssetCatalog.cpp 
+      and Code\Tools\AssetProcessor\native\AssetManager\AssetCatalog.cpp in the same fashion
+
+Note that after the modification and rebuild, you will need to delete the cache folder and launch the engine for the catalog to be rebuilt
+
 How do I run this script from a command line?
 ================================================
 1) Check out any .slice, .layer, .ly, and .cry files you want to convert from source control
     - This script will remove the legacy components entirely, so make sure you have your files
       backed up before you run this script in case you want to run it again
-2) From the Lumberyard root folder, run LegacyComponentConverter.py project=<ProjectName> --include_gems
-    - --include_gems is optional
-    - if you include Gems, it will run all all Gems, not just the ones enabled by your project
+2) From the Lumberyard root folder, run LegacyComponentConverter.py -project=<ProjectName> -include_gems -assetCatalogOverridePath=<ExternalProjectPath\Cache\pc\assetcatalog.xml>
+    - -include_gems is optional. If you include Gems, it will run all all Gems, not just the ones enabled by your project
+    - -assetCatalogOverridePath is optional. It will replace mesh/material asset id from the source project to this target project
+      based on relative asset path from the project root (in general, Objects/Models/yourasset.fbx)
 
 
 What are the artifacts of this script?
@@ -59,12 +74,10 @@
 CONVERTED_LOG_NAME = "ComponentConversion_ConvertedLegacyFiles.log"
 UNCONVERTED_LOG_NAME = "ComponentConversion_UnsupportedLegacyFiles.log"
 STATS_LOG_NAME = "ComponentConversion_LegacyComponentStats.log"
-BUILD_PATH = None
-GEMS_PATH = None
 
 # Normal imports
 import sys
-import xml.etree.ElementTree
+import xml.etree.ElementTree as ET
 import time
 from zipfile import ZipFile
 import tempfile
@@ -76,8 +89,9 @@
 from LegacyMaterialComponentConverter import *
 from LegacyActorComponentConverter import *
 from LegacyPointLightComponentConverter import *
+from LegacyTransformComponentConverter import *
 
-BUILD_PATH = "./"
+BUILD_PATH = "./" # Use current working directory, we expect to be in lumberyard dev folder
 GEMS_PATH = os.path.join(BUILD_PATH, "Gems")
 
 class Component_File(object):
@@ -169,6 +183,7 @@ def gather_elements(self):
         componentConverters.append(Mesh_Component_Converter(self.assetCatalogHelper, self.statsCollector, self.normalizedProjectDir))
         componentConverters.append(Actor_Component_Converter(self.assetCatalogHelper, self.statsCollector, self.normalizedProjectDir))
         componentConverters.append(Point_Light_Component_Converter(self.assetCatalogHelper, self.statsCollector, self.normalizedProjectDir))
+        componentConverters.append(Transform_Component_Converter(self.assetCatalogHelper, self.statsCollector, self.normalizedProjectDir))
 
         if self.is_valid_xml():
             root = self.xml.getroot()
@@ -190,7 +205,10 @@ def gather_elements(self):
                                 # TODO - we're about to change the tree structure while iterating, which is apparently undefined but appears to work. Might be better to just build up a list of things that need to be modified, then do a second pass to replace the legacy component
                                 # Seems to be okay since we only change or add elements, never remove entirely
                                 componentConverter.convert(child, parent)
-            self.xml._setroot(root)                        
+            self.xml._setroot(root)    
+            # pretty print 
+            ET.indent(self.xml, space='\t')
+
         print("finished parsing {0}".format(self.filename))
 
     def can_be_converted(self):
@@ -265,12 +283,12 @@ def getUpdatedStatsCollector(self):
 def main():
     '''sys.__name__ wrapper function'''
 
-    msgStr = "This tool will scan all of your project's level/layer/slice files\n\
+    msgStr = "This tool will scan all of your lumberyard project's level/layer/slice files\n\
 convert any compatible legacy components into the equivalent Atom components\n\
 This script will overwrite the original files, and will remove the legacy components\n\
 upon conversion, decimating the previous contents of those components.\n"
 
-    commandLineOptions = Common_Command_Line_Options(sys.argv[0], sys.argv[1])
+    commandLineOptions = Common_Command_Line_Options(sys.argv)
     if commandLineOptions.isHelp:
         print (commandLineOptions.helpString)
         return
@@ -281,8 +299,13 @@ def main():
     extensionList = [".slice", ".layer", ".ly", ".cry"]
     fileList = get_file_list(commandLineOptions.projectName, commandLineOptions.includeGems, extensionList, BUILD_PATH, GEMS_PATH)
 
-    assetCatalogDictionaries = get_asset_catalog_dictionaries(BUILD_PATH, commandLineOptions.projectName)
-
+    if commandLineOptions.assetCatalogOverridePath:
+        assetCatalogPath = commandLineOptions.assetCatalogOverridePath
+        assetCatalogDictionaries = get_asset_catalog_dictionaries(assetCatalogPath)
+    else:
+        assetCatalogPath = os.path.join("Cache", commandLineOptions.projectName, get_default_asset_platform(), commandLineOptions.projectName, "assetcatalog.xml")
+        assetCatalogDictionaries = get_asset_catalog_dictionaries(assetCatalogPath)
+        
     # Create a log file to store converted component file filenames
     # and to check to see if the component file has already been converted.
     convertedLogFile = Log_File(filename="{0}\\{1}".format(BUILD_PATH, CONVERTED_LOG_NAME))

Gems/AtomLyIntegration/CommonFeatures/Assets/Editor/Scripts/LegacyContentConversion/LegacyConversionHelpers.py

@@ -8,6 +8,7 @@
 """
 
 import os
+import platform
 import xml.etree.ElementTree
 
 class Stats_Collector(object):
@@ -101,30 +102,36 @@ class Common_Command_Line_Options(object):
     """
     Some common options/parsing
     """
-    def __init__(self, argv0, argv1):
-        arguments = argv1.split('-')
-
+    def __init__(self, args):
         self.projectName = ""
+        self.assetCatalogOverridePath = ""
         self.includeGems = False
         self.useP4 = False
         self.endsWithStr = ""
         self.isHelp = False        
-        self.helpString = "usage: {0} -project=<project name> -include_gems -ends_with=<filter> -use_p4\n\
+        self.helpString = f"usage: {args[0]} -project=<project name> -include_gems -ends_with=<filter> -use_p4 -assetCatalogOverridePath=<path>\n\
             E.g.:\n\
-            {1} -project=StarterGame -include_gems\n\
-              -project is required.\n\
+              -project=StarterGame -include_gems\n\
+              -project is required. Path is relative\n\
               -include_gems is optional, and by default gems will not be included.\n\
               -ends_with is optional. It could be used to filter for a specific file (--ends_with=default.mtl)\n\
-              -use_p4 is optional. It will use the p4 command line to check out files that are edited in your default changelist".format(argv0, argv0)
-
-        for argument in arguments:
-            argument = argument.rstrip(" ")
-            if argument == "h" or argument == "help" or argv1 == "?":
+              -use_p4 is optional. It will use the p4 command line to check out files that are edited in your default changelist\n\
+              -assetCatalogOverridePath is optional. It will use asset ids from this file instead of using the current project asset catalog\n\
+               (match via relative asset path to the project root folder)"
+              
+
+        for argument in args:
+            argument = argument.rstrip(" ").lstrip("-")
+            if argument == "h" or argument == "help" or argument == "?":
                 self.isHelp = True
             elif argument.startswith("project"):
                 projectArgs = argument.split("=")
                 if len(projectArgs) > 1:
                     self.projectName = projectArgs[1]
+            elif argument.startswith("assetCatalogOverridePath"):
+                projectArgs = argument.split("=")
+                if len(projectArgs) > 1:
+                    self.assetCatalogOverridePath = projectArgs[1]
             elif argument == "include_gems":
                 self.includeGems = True
             elif argument == "use_p4":
@@ -204,7 +211,7 @@ def get_asset_id_from_relative_path(self, relativePath):
             return self.relativePathToAssetIdDict[relativePath]
         return ""
 
-def get_asset_catalog_dictionaries(buildPath, projectName):
+def get_asset_catalog_dictionaries(assetCatalogPath):
     """
     This function pre-supposes that you have modified AssetCatalog::SaveRegistry_Impl
     in Code/Tools/AssetProcessor/native/AssetManager/AssetCatalog.cpp
@@ -218,7 +225,6 @@ def get_asset_catalog_dictionaries(buildPath, projectName):
     relativePathToAssetIdDict[""] = "{00000000-0000-0000-0000-000000000000}:0"
     assetIdToRelativePathDict = {}
     assetUuidToAssetIdsDict = {}
-    assetCatalogPath = os.path.join(projectName, "Cache", "pc", "assetcatalog.xml")
     assetCatalogXml = xml.etree.ElementTree.parse(assetCatalogPath)
     for possibleAssetInfo in assetCatalogXml.getroot().iter('Class'):
         if "name" in possibleAssetInfo.keys() and possibleAssetInfo.get("name") == "AssetInfo":
@@ -278,6 +284,12 @@ def get_subid_from_assetId(assetId):
     separatorIndex = assetId.find(":") + 1
     return assetId[separatorIndex:]
 
+def get_default_asset_platform():
+    host_platform_to_asset_platform_map = { 'windows': 'pc',
+                                            'linux':   'linux',
+                                            'darwin':  'mac' }
+    return host_platform_to_asset_platform_map.get(platform.system().lower(), "")
+
 class Component_Converter(object):
     """
     Converter base class

Gems/AtomLyIntegration/CommonFeatures/Assets/Editor/Scripts/LegacyContentConversion/LegacyMaterialComponentConverter.py

@@ -153,12 +153,9 @@ def create_material_component_with_material_assignments(self, atomMaterialInDefa
         componentConfig = create_xml_element_from_string("<Class name=\"ComponentConfig\" field=\"BaseClass1\" version=\"1\" type=\"{0A7929DF-2932-40EA-B2B3-79BC1C3490D0}\"/>")
         materialMap = create_xml_element_from_string("<Class name=\"AZStd::unordered_map\" field=\"materials\" type=\"{50F6716F-698B-5A6C-AACD-940597FDEC24}\">")
 
-        # {00000000-0000-0000-0000-000000000000}:0 is the default slot
-        defaultMaterialAssignmentMapEntry = self.create_material_map_entry("{00000000-0000-0000-0000-000000000000}:0", atomMaterialInDefaultSlotAssetId)
-        materialMap.append(defaultMaterialAssignmentMapEntry)
-
+        defaultSlot = "{00000000-0000-0000-0000-000000000000}:0"
         for atomMaterial in materialAssignmentList:
-            if atomMaterial.assignmentAssetId:
+            if atomMaterial.assignmentAssetId and atomMaterial.assignmentAssetId != defaultSlot:
                 materialMapElement = self.create_material_map_entry(atomMaterial.slotAssetId, atomMaterial.assignmentAssetId)
                 materialMap.append(materialMapElement)
 
@@ -169,7 +166,7 @@ def create_material_component_with_material_assignments(self, atomMaterialInDefa
         editorRenderComponentAdapter.append(editorComponentAdapter)
 
         isDefaultSlot = True
-        defaultMaterialComponentSlot = self.create_editor_material_assignment_slot("{00000000-0000-0000-0000-000000000000}:0", atomMaterialInDefaultSlotAssetId, isDefaultSlot)
+        defaultMaterialComponentSlot = self.create_editor_material_assignment_slot(defaultSlot, atomMaterialInDefaultSlotAssetId, isDefaultSlot)
 
         #   <Class name="AZStd::vector" field="materialSlots" type="{7FDDDE36-46C8-5DBC-8566-E792AA358BD9}">
         #        ... (slots)