Updates to fix crash in O3DE caused by the python relocation (o3de#17623)

1. Relocation of Python Packages and venv to ~/.o3de/Python (Independent of the 3rdParty folder)
2. Symlinking python3.10.so instead of doing a copy
3. Extra PhysX 4/5 Cleanup

Signed-off-by: Steve Pham <[email protected]>

Author: spham-amzn

Code/Framework/AzCore/AzCore/Utils/Utils.cpp

@@ -428,6 +428,14 @@ namespace AZ::Utils
         return AZ::Success(thirdPartyFolder);
     }
 
+    AZ::IO::FixedMaxPathString GetO3dePythonVenvRoot(AZ::SettingsRegistryInterface* settingsRegistry /*= nullptr*/)
+    {
+        // Locate the manifest directory
+        auto manifestDirectory = AZ::IO::FixedMaxPath(GetO3deManifestDirectory(settingsRegistry)) / "Python" / "venv";
+        return manifestDirectory.Native();
+    }
+
+
     template AZ::Outcome<AZStd::string, AZStd::string> ReadFile(AZStd::string_view filePath, size_t maxFileSize);
     template AZ::Outcome<AZStd::vector<int8_t>, AZStd::string> ReadFile(AZStd::string_view filePath, size_t maxFileSize);
     template AZ::Outcome<AZStd::vector<uint8_t>, AZStd::string> ReadFile(AZStd::string_view filePath, size_t maxFileSize);

Code/Framework/AzCore/AzCore/Utils/Utils.h

@@ -176,6 +176,11 @@ namespace AZ
         //! Returns the outcome of the request, the configured 3rd Party path if successful, the error message if not.
         AZ::Outcome<AZStd::string, AZStd::string> Get3rdPartyDirectory(AZ::SettingsRegistryInterface* settingsRegistry = nullptr);
 
+        //! Retrieves the full directory to the O3DE Python virtual environment (venv) folder. 
+        //! @param settingsRegistry pointer to the SettingsRegistry to use for lookup
+        //! If nullptr, the AZ::Interface instance of the SettingsRegistry is used
+        AZ::IO::FixedMaxPathString GetO3dePythonVenvRoot(AZ::SettingsRegistryInterface* settingsRegistry = nullptr);
+
 
         //! error code value returned when GetEnv fails
         enum class GetEnvErrorCode

Code/Framework/AzCore/Platform/Common/UnixLike/AzCore/Utils/Utils_UnixLike.cpp

@@ -39,16 +39,16 @@ namespace AZ::Utils
             }
         }
 
-        if (const char* homePath = std::getenv("HOME"); homePath != nullptr)
+        struct passwd* pass = getpwuid(getuid());
+        if (pass)
         {
-            AZ::IO::FixedMaxPath path{homePath};
+            AZ::IO::FixedMaxPath path{pass->pw_dir};
             return path.Native();
         }
 
-        struct passwd* pass = getpwuid(getuid());
-        if (pass)
+        if (const char* homePath = std::getenv("HOME"); homePath != nullptr)
         {
-            AZ::IO::FixedMaxPath path{pass->pw_dir};
+            AZ::IO::FixedMaxPath path{homePath};
             return path.Native();
         }
 

Code/Framework/AzCore/Platform/Windows/AzCore/Utils/Utils_Windows.cpp

@@ -7,10 +7,12 @@
  */
 
 #include <AzCore/Utils/Utils.h>
+#include <AzCore/IO/Path/Path.h>
 #include <AzCore/PlatformIncl.h>
 #include <AzCore/std/string/conversions.h>
 
 #include <stdlib.h>
+#include <shlobj_core.h>
 
 namespace AZ::Utils
 {
@@ -41,6 +43,14 @@ namespace AZ::Utils
             }
         }
 
+        wchar_t sysUserProfilePathW[MAX_PATH];
+        if (SUCCEEDED(SHGetFolderPath(0, CSIDL_PROFILE, 0, SHGFP_TYPE_DEFAULT, sysUserProfilePathW)))
+        {
+            AZ::IO::FixedMaxPathString sysUserProfilePathStr;
+            AZStd::to_string(sysUserProfilePathStr, sysUserProfilePathW);
+            return sysUserProfilePathStr;
+        }
+
         char userProfileBuffer[AZ::IO::MaxPathLength]{};
         size_t variableSize = 0;
         auto err = getenv_s(&variableSize, userProfileBuffer, AZ::IO::MaxPathLength, "USERPROFILE");

Code/Framework/AzToolsFramework/AzToolsFramework/API/PythonLoader.cpp

@@ -7,17 +7,18 @@
 */
 
 #include <AzToolsFramework/API/PythonLoader.h>
-#include <AzToolsFramework_Traits_Platform.h>
 #include <AzCore/Component/ComponentApplicationBus.h>
 #include <AzCore/IO/FileIO.h>
 #include <AzCore/IO/GenericStreams.h>
 #include <AzCore/IO/Path/Path.h>
+#include <AzCore/IO/SystemFile.h>
 #include <AzCore/Math/Sha1.h>
 #include <AzCore/std/containers/vector.h>
 #include <AzCore/std/string/string.h>
 #include <AzCore/std/string/string_view.h>
 #include <AzCore/std/string/conversions.h>
 #include <AzCore/std/string/tokenize.h>
+#include <AzCore/Serialization/Json/JsonUtils.h>
 #include <AzCore/Settings/ConfigParser.h>
 #include <AzCore/Utils/Utils.h>
 #include <AzFramework/IO/LocalFileIO.h>
@@ -26,82 +27,56 @@ namespace AzToolsFramework::EmbeddedPython
 {
     PythonLoader::PythonLoader()
     {
-        #if AZ_TRAIT_PYTHON_LOADER_ENABLE_EXPLICIT_LOADING
-        // PYTHON_SHARED_LIBRARY_PATH must be defined in the build scripts and referencing the path to the python shared library
-        #if !defined(PYTHON_SHARED_LIBRARY_PATH)
-        #error "PYTHON_SHARED_LIBRARY_PATH is not defined"
-        #endif
-
-        // Construct the path to the shared python library within the venv folder
-        AZ::IO::FixedMaxPath engineRoot = AZ::IO::FixedMaxPath(AZ::Utils::GetEnginePath());
-        AZ::IO::FixedMaxPath thirdPartyRoot = PythonLoader::GetDefault3rdPartyPath(false);
-        AZ::IO::FixedMaxPath pythonVenvPath = PythonLoader::GetPythonVenvPath(thirdPartyRoot, engineRoot);
-
-        AZ::IO::PathView libPythonName = AZ::IO::PathView(PYTHON_SHARED_LIBRARY_PATH).Filename();
-        AZ::IO::FixedMaxPath pythonVenvLibPath = pythonVenvPath / "lib" / libPythonName;
-
-        m_embeddedLibPythonModuleHandle = AZ::DynamicModuleHandle::Create(pythonVenvLibPath.StringAsPosix().c_str(), false);
-        bool loadResult = m_embeddedLibPythonModuleHandle->Load(false, true);
-        AZ_Error("PythonLoader", loadResult, "Failed to load %s.\n", libPythonName.StringAsPosix().c_str());
-        #endif // AZ_TRAIT_PYTHON_LOADER_ENABLE_EXPLICIT_LOADING
-    }
-
-    PythonLoader::~PythonLoader()
-    {
-        #if AZ_TRAIT_PYTHON_LOADER_ENABLE_EXPLICIT_LOADING
-        AZ_Assert(m_embeddedLibPythonModuleHandle, "DynamicModuleHandle for python was not created");
-        m_embeddedLibPythonModuleHandle->Unload();
-        #endif // AZ_TRAIT_PYTHON_LOADER_ENABLE_EXPLICIT_LOADING
-    }
+        #if defined(IMPLICIT_LOAD_PYTHON_SHARED_LIBRARY)
 
-    AZ::IO::FixedMaxPath PythonLoader::GetDefault3rdPartyPath(bool createOnDemand)
-    {
-        AZ::IO::FixedMaxPath thirdPartyEnvPathPath;
-
-        // The highest priority for the 3rd party path is the environment variable 'LY_3RDPARTY_PATH'
-        static constexpr const char* env3rdPartyKey = "LY_3RDPARTY_PATH";
-        char env3rdPartyPath[AZ::IO::MaxPathLength] = { '\0' };
-        auto envOutcome = AZ::Utils::GetEnv(AZStd::span(env3rdPartyPath), env3rdPartyKey);
-        if (envOutcome && (strlen(env3rdPartyPath) > 0))
-        {
-            // If so, then use the path that is set as the third party path
-            thirdPartyEnvPathPath = AZ::IO::FixedMaxPath(env3rdPartyPath).LexicallyNormal();
-        }
-        // The next priority is to read the 3rd party directory from the manifest file
-        else if (auto manifest3rdPartyResult = AZ::Utils::Get3rdPartyDirectory(); manifest3rdPartyResult.IsSuccess())
+        // Determine if this is an sdk-engine build. For SDK engines, we want to prevent implicit python module loading.
+        [[maybe_unused]] bool isSdkEngine{ false };
+        auto engineSettingsPath = AZ::IO::FixedMaxPath{ AZ::Utils::GetEnginePath() } / "engine.json";
+        if (AZ::IO::SystemFile::Exists(engineSettingsPath.c_str()))
         {
-            thirdPartyEnvPathPath = manifest3rdPartyResult.GetValue();
+            auto loadOutcome = AZ::JsonSerializationUtils::ReadJsonFile(engineSettingsPath.c_str());
+            if (loadOutcome.IsSuccess())
+            {
+                auto& doc = loadOutcome.GetValue();
+                rapidjson::Value::MemberIterator sdkEngineFieldIter = doc.FindMember("sdk_engine");
+                if (sdkEngineFieldIter != doc.MemberEnd())
+                {
+                    isSdkEngine = sdkEngineFieldIter->value.GetBool();
+                }
+            }
         }
-        // Fallback to the default 3rd Party path based on the location of the manifest folder
-        else
+        if (!isSdkEngine)
         {
-            auto manifestPath = AZ::Utils::GetO3deManifestDirectory();
-            thirdPartyEnvPathPath = AZ::IO::FixedMaxPath(manifestPath) / "3rdParty";
+            m_embeddedLibPythonModuleHandle = AZ::DynamicModuleHandle::Create(IMPLICIT_LOAD_PYTHON_SHARED_LIBRARY, false);
+            bool loadResult = m_embeddedLibPythonModuleHandle->Load(false, true);
+            AZ_Error("PythonLoader", loadResult, "Failed to load " IMPLICIT_LOAD_PYTHON_SHARED_LIBRARY "\n");
         }
+        #endif // IMPLICIT_LOAD_PYTHON_SHARED_LIBRARY
+    }
 
-        if ((!AZ::IO::SystemFile::IsDirectory(thirdPartyEnvPathPath.c_str())) && createOnDemand)
+    PythonLoader::~PythonLoader()
+    {
+        #if defined(IMPLICIT_LOAD_PYTHON_SHARED_LIBRARY)
+        if (m_embeddedLibPythonModuleHandle)
         {
-            auto createPathResult = AZ::IO::SystemFile::CreateDir(thirdPartyEnvPathPath.c_str());
-            AZ_Assert(createPathResult, "Unable to create missing 3rd Party Folder '%s'", thirdPartyEnvPathPath.c_str())
+            m_embeddedLibPythonModuleHandle->Unload();
         }
-        return thirdPartyEnvPathPath;
+        #endif // IMPLICIT_LOAD_PYTHON_SHARED_LIBRARY
     }
 
-    AZ::IO::FixedMaxPath PythonLoader::GetPythonHomePath(AZ::IO::PathView engineRoot)
+    AZ::IO::FixedMaxPath PythonLoader::GetPythonHomePath(AZ::IO::PathView engineRoot, const char* overridePythonBaseVenvPath /*= nullptr*/)
     {
-        AZ::IO::FixedMaxPath thirdPartyFolder = GetDefault3rdPartyPath(true);
-
         // The python HOME path relative to the executable depends on the host platform the package is created for
         #if AZ_TRAIT_PYTHON_LOADER_PYTHON_HOME_BIN_SUBPATH
-        AZ::IO::FixedMaxPath pythonHomePath = PythonLoader::GetPythonExecutablePath(thirdPartyFolder, engineRoot).ParentPath();
+        AZ::IO::FixedMaxPath pythonHomePath = PythonLoader::GetPythonExecutablePath(engineRoot, overridePythonBaseVenvPath).ParentPath();
         #else
-        AZ::IO::FixedMaxPath pythonHomePath = PythonLoader::GetPythonExecutablePath(thirdPartyFolder, engineRoot);
+        AZ::IO::FixedMaxPath pythonHomePath = PythonLoader::GetPythonExecutablePath(engineRoot, overridePythonBaseVenvPath);
         #endif // AZ_TRAIT_PYTHON_LOADER_PYTHON_HOME_BIN_SUBPATH
 
         return pythonHomePath;
     }
 
-    AZ::IO::FixedMaxPath PythonLoader::GetPythonVenvPath(AZ::IO::PathView thirdPartyRoot, AZ::IO::PathView engineRoot)
+    AZ::IO::FixedMaxPath PythonLoader::GetPythonVenvPath(AZ::IO::PathView engineRoot, const char* overridePythonBaseVenvPath /*= nullptr*/)
     {
         // Perform the same hash calculation as cmake/CalculateEnginePathId.cmake
         /////
@@ -118,16 +93,18 @@ namespace AzToolsFramework::EmbeddedPython
         hasher.GetDigest(digest);
 
         // Construct the path to where the python venv based on the engine path should be located
-        AZ::IO::FixedMaxPath libPath = thirdPartyRoot;
+        AZ::IO::FixedMaxPath libPath = (overridePythonBaseVenvPath != nullptr) ? AZ::IO::FixedMaxPath(overridePythonBaseVenvPath) : 
+                                                                                 AZ::Utils::GetO3dePythonVenvRoot();
+
         // The ID is based on the first 32 bits of the digest, and formatted to at least 8-character wide hexadecimal representation
-        libPath /= AZ::IO::FixedMaxPathString::format("venv/%08x", digest[0]);
+        libPath /= AZ::IO::FixedMaxPathString::format("%08x", digest[0]);
         libPath = libPath.LexicallyNormal();
         return libPath;
     }
 
-    AZ::IO::FixedMaxPath PythonLoader::GetPythonExecutablePath(AZ::IO::PathView thirdPartyRoot, AZ::IO::PathView engineRoot)
+    AZ::IO::FixedMaxPath PythonLoader::GetPythonExecutablePath(AZ::IO::PathView engineRoot, const char* overridePythonBaseVenvPath /*= nullptr*/)
     {
-        AZ::IO::FixedMaxPath pythonVenvConfig = PythonLoader::GetPythonVenvPath(thirdPartyRoot, engineRoot) / "pyvenv.cfg";
+        AZ::IO::FixedMaxPath pythonVenvConfig = PythonLoader::GetPythonVenvPath(engineRoot, overridePythonBaseVenvPath) / "pyvenv.cfg";
         AZ::IO::SystemFileStream systemFileStream;
         if (!systemFileStream.Open(pythonVenvConfig.c_str(), AZ::IO::OpenMode::ModeRead))
         {
@@ -151,11 +128,11 @@ namespace AzToolsFramework::EmbeddedPython
         return AZ::IO::FixedMaxPath(pythonHome);
     }
 
-    void PythonLoader::ReadPythonEggLinkPaths(AZ::IO::PathView thirdPartyRoot, AZ::IO::PathView engineRoot, EggLinkPathVisitor resultPathCallback)
+    void PythonLoader::ReadPythonEggLinkPaths(AZ::IO::PathView engineRoot, EggLinkPathVisitor resultPathCallback, const char* overridePythonBaseVenvPath /*= nullptr*/)
     {
         // Get the python venv path
         AZ::IO::FixedMaxPath pythonVenvSitePackages =
-            AZ::IO::FixedMaxPath(PythonLoader::GetPythonVenvPath(thirdPartyRoot, engineRoot)) / O3DE_PYTHON_SITE_PACKAGE_SUBPATH;
+            AZ::IO::FixedMaxPath(PythonLoader::GetPythonVenvPath(engineRoot, overridePythonBaseVenvPath)) / O3DE_PYTHON_SITE_PACKAGE_SUBPATH;
 
         // Always add the site-packages folder from the virtual environment into the path list
         resultPathCallback(pythonVenvSitePackages.LexicallyNormal());

Code/Framework/AzToolsFramework/AzToolsFramework/API/PythonLoader.h

@@ -26,35 +26,32 @@ namespace AzToolsFramework::EmbeddedPython
 
         //! Calculate the python home (PYTHONHOME) based on the engine root
         //! @param engineRoot The path to the engine root to locate the python home
+        //! @param overridePythonBaseVenvPath If set, the path override the base python venv folder
         //! @return The path of the python home path
-        static AZ::IO::FixedMaxPath GetPythonHomePath(AZ::IO::PathView engineRoot);
+        static AZ::IO::FixedMaxPath GetPythonHomePath(AZ::IO::PathView engineRoot, const char* overridePythonBaseVenvPath = nullptr);
 
         //! Collect the paths from all the egg-link files found in the python home
         //! paths used by the engine
-        //! @param thirdPartyRoot The root location of the O3DE 3rdParty folder
         //! @param engineRoot The path to the engine root to locate the python home
         //! @param eggLinkPathVisitor The callback visitor function to receive the egg-link paths that are discovered
+        //! @param overridePythonBaseVenvPath If set, the path override the base python venv folder
         using EggLinkPathVisitor = AZStd::function<void(AZ::IO::PathView)>;
-        static void ReadPythonEggLinkPaths(AZ::IO::PathView thirdPartyRoot, AZ::IO::PathView engineRoot, EggLinkPathVisitor eggLinkPathVisitor);
-
-        //! Get the default 3rd Party folder path.
-        //! @return The path of the 3rd Party root path
-        static AZ::IO::FixedMaxPath GetDefault3rdPartyPath(bool createOnDemand);
+        static void ReadPythonEggLinkPaths(AZ::IO::PathView engineRoot, EggLinkPathVisitor eggLinkPathVisitor, const char* overridePythonBaseVenvPath = nullptr);
 
         //! Calculate the path to the engine's python virtual environment used for
         //! python home (PYTHONHOME) based on the engine root
-        //! @param thirdPartyRoot The root location of the O3DE 3rdParty folder
         //! @param engineRoot The path to the engine root to locate the python venv path
+        //! @param overridePythonBaseVenvPath If set, the path override the base python venv folder
         //! @return The path of the python venv path
-        static AZ::IO::FixedMaxPath GetPythonVenvPath(AZ::IO::PathView thirdPartyRoot, AZ::IO::PathView engineRoot);
+        static AZ::IO::FixedMaxPath GetPythonVenvPath(AZ::IO::PathView engineRoot, const char* overridePythonBaseVenvPath = nullptr);
 
         //! Calculate the path to the where the python executable resides in. Note that this
         //! is not always the same path as the python home path
-        //! @param thirdPartyRoot The root location of the O3DE 3rdParty folder
         //! @param engineRoot The path to the engine root to
+        //! @param overridePythonBaseVenvPath If set, the path override the base python venv folder
         //! locate the python executable path
         //! @return The path of the python venv path
-        static AZ::IO::FixedMaxPath GetPythonExecutablePath(AZ::IO::PathView thirdPartyRoot, AZ::IO::PathView engineRoot);
+        static AZ::IO::FixedMaxPath GetPythonExecutablePath(AZ::IO::PathView engineRoot, const char* overridePythonBaseVenvPath = nullptr);
 
     private:
         AZStd::unique_ptr<AZ::DynamicModuleHandle> m_embeddedLibPythonModuleHandle;

Code/Framework/AzToolsFramework/Platform/Linux/AzToolsFramework_Traits_Linux.h

@@ -6,6 +6,8 @@
 #
 #
 
-get_target_property(libraries 3rdParty::Python INTERFACE_LINK_LIBRARIES)
-
-set(LY_COMPILE_DEFINITIONS PRIVATE PYTHON_SHARED_LIBRARY_PATH="${libraries}")
+set(LY_COMPILE_DEFINITIONS 
+    PRIVATE 
+        IMPLICIT_LOAD_PYTHON_SHARED_LIBRARY="${LY_PYTHON_SHARED_LIB}"
+        AZ_TRAIT_PYTHON_LOADER_PYTHON_HOME_BIN_SUBPATH
+)

Code/Framework/AzToolsFramework/Platform/Linux/AzToolsFramework_Traits_Platform.h

@@ -7,7 +7,5 @@
 #
 
 set(FILES
-    AzToolsFramework_Traits_Linux.h
-    AzToolsFramework_Traits_Platform.h
     AzToolsFramework/API/EditorAssetSystemAPI_Linux.cpp
 )