Publishing a module with python dependencies (#1021)

* Initial commit for checks checks to run

* Cleaning up changes to be ready for a PR to be reviewed

* Adding publishing test to ProcessPythonWheel suite

* Updating changelog to say -export-python-deps in publish is an addition as opposed to a fix

* Making sure python dependencies of dependency modules are also exported

* Updating changelog to keep up with main

* One more change to changelog to keep up with main

* These files shouldn't be here

* Fixing the bug

* Removing extraneous write statement

* Adding more test cases for publishing with python dependencies

* Commenting out failing package test case

* Re-adding publish-with-deps which I accidentally delted before

* Minor changes to start addressing review comments

* [In Progress] Adding checking for files in packaged tarball to tests

* [Completing] Adding checking for files in packaged tarball to tests

---------

Co-authored-by: James Lechtner <James.Lechtner@intersystems.com>

Author: isc-jlechtne

CHANGELOG.md

@@ -7,9 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [0.10.6] - Unreleased
 
+### Added
+- #1024: Added flag -export-python-deps to publish command
+
 ### Fixed
 - #996: Ensure COS commands execute in exec under a dedicated, isolated context
 - #1002: When listing configured repositories, only show the TokenAuthMethod when a token is defined.
+- #1024: Modules with PythonWheels have wheels packaged correctly under -export-python-deps
 
 ## [0.10.5] - 2026-01-15
 

src/cls/IPM/Lifecycle/Base.cls

@@ -1182,10 +1182,6 @@ Method %Export(
                 }
             }
         }
-        set exportPythonDependencies = $get(pParams("ExportPythonDependencies"), 0)
-        if exportPythonDependencies {
-            do ..ExportPythonDependencies(.pParams)
-        }
 
         // Deployed items should be exported as a studio project to a designated directory
         set tDeployedProjectName = ##class(%IPM.Utils.Module).GetDeployedProjectName(..Module.Name)
@@ -1198,11 +1194,18 @@ Method %Export(
         set tParams($$$DeployedProjectIndex) = tDeployedProject
 
         set tModuleName = ""
+        set exportPythonDependencies = $get(pParams("ExportPythonDependencies"), 0)
         for {
             set tModuleName = $order(tResourceArray(tModuleName))
             if (tModuleName = "") {
                 quit
             }
+            if exportPythonDependencies {
+                set tModule = ##class(%IPM.Storage.Module).NameOpen(tModuleName)
+                do ..ExportPythonDependencies(tModule, .pParams)
+                // Recalculate resource array to pick up any python dependencies added during export
+                $$$ThrowOnError(..Module.GetResolvedReferences(.tResourceArray,tExportDependencies,..PhaseList,.pDependencyGraph))
+            }
             kill tSingleModuleArray
             merge tSingleModuleArray = tResourceArray(tModuleName)
             do ..ExportSingleModule(.tSingleModuleArray, pTargetDirectory, .pDependencyGraph, .tParams, tVerbose)
@@ -1242,25 +1245,41 @@ Method %Export(
 }
 
 /// Downloads Python wheels for Python dependencies into the /root/wheels directory of the module. Also updates module.xml with the Python Wheel resources.
-Method ExportPythonDependencies(ByRef pParams)
+Method ExportPythonDependencies(
+	module As %IPM.Storage.Module,
+	ByRef pParams)
 {
-    set root = ##class(%File).NormalizeDirectory("", ..Module.Root)
+    set orderedResourceList = module.GetOrderedResourceList()
+    set key = ""
+    set resourceNameList = ""
+    for {
+        set resource = orderedResourceList.GetNext(.key)
+        quit:key=""
+        set resourceNameList = resourceNameList_$listbuild($zconvert(resource.Name, "l"))
+    }
+
+    set root = ##class(%File).NormalizeDirectory("", module.Root)
     $$$ThrowOnError(..InstallOrDownloadPythonRequirements(root, .pParams, 1))
-    set wheelsDir = ##class(%File).NormalizeDirectory("wheels", ..Module.Root)
+    set wheelsDir = ##class(%File).NormalizeDirectory("wheels", module.Root)
     if '##class(%File).DirectoryExists(wheelsDir) {
-        $$$ThrowStatus($$$ERROR($$$GeneralError, "Wheels directory: "_wheelsDir_" not found."))
+        // ExportPythonDependencies called on base module + dependencies so don't want to error if one of
+        // the modules doesn't have a wheels directory since that might actually be the case for the module
+        write !, "WARNING: No wheels directory for this module: "_wheelsDir_" not found"
+        quit
     }
     set stmt = ##class(%SQL.Statement).%New()
     $$$ThrowOnError(stmt.%PrepareClassQuery("%File", "FileSet"))
     set rset = stmt.%Execute(wheelsDir, "*.whl")
     $$$ThrowSQLIfError(rset.%SQLCODE,rset.%Message)
     while rset.%Next(.sc) {
-        set file = rset.%Get("ItemName")
+        set wheelFileName = rset.%Get("ItemName")
         set type = rset.%Get("Type")
-        if (type '= "F") || (file = "") {
+        if (type '= "F") || (wheelFileName = "") {
             continue
         }
-    do ##class(%IPM.StudioDocument.Module).AddPythonWheels(..Module.Name, file)
+        if '$listfind(resourceNameList, $zconvert(wheelFileName, "l")) {
+            do ##class(%IPM.StudioDocument.Module).AddPythonWheels(module.Name, wheelFileName)
+        }
     }
     $$$ThrowOnError(sc)
 }
@@ -1843,6 +1862,7 @@ Method ExportSingleModule(
 	pVerbose As %Boolean = 0)
 {
     merge tParams = pParams
+    set exportPythonDependencies = $get(tParams("ExportPythonDependencies"), 0)
 
     set tFullResourceName = ""
     for {
@@ -1880,6 +1900,8 @@ Method ExportSingleModule(
 
         #dim tProcessor As %IPM.ResourceProcessor.Abstract
         if $data(pResourceArray(tFullResourceName,"Processor"),tProcessor) && $isobject(tProcessor) {
+            // If this is a python resource but -export-python-deps was not specified, skip this resource
+            if 'exportPythonDependencies && (tProcessor.%ClassName(1) = "%IPM.ResourceProcessor.PythonWheel") continue
             kill tItemParams
             merge tItemParams = pResourceArray(tFullResourceName)
             set tItemHandled = 0

src/cls/IPM/Main.cls

@@ -140,6 +140,7 @@ This command is an alias for `module-action module-name publish`
 <parameter name="module" required="true" description="Name of module on which to perform publish actions" />
 <modifier name="repo" aliases="r" dataAlias="PublishTo" description="Namespace-unique name of repository for the module to publish to (if deployment is enabled)" />
 <modifier name="use-external-name" aliases="use-ext" dataAlias="UseExternalName" dataValue="1" description="Publish the package under the &lt;ExternalName&gt; of the package. If ExternalName is unspecified or illegal, an error will be thrown."/>
+<modifier name="export-python-deps" dataAlias="ExportPythonDependencies" dataValue="1" description="If specified, exports Python dependencies. If omitted, defaults to false." />
 </command>
 
 <command name="update" dataPrefix="D">

src/cls/IPM/StudioDocument/Module.cls

@@ -434,9 +434,9 @@ XData AppendWheelUnderModuleTransform [ XMLNamespace = "http://www.intersystems.
     <xsl:variable name="ns" select="namespace-uri(..)"/>
     <xsl:copy><xsl:apply-templates select="@*|node()"/></xsl:copy>
     <xsl:element name="PythonWheel" namespace="{$ns}">
-      <xsl:element name="Name" namespace="{$ns}">
+      <xsl:attribute name="Name" namespace="{$ns}">
         <xsl:value-of select="$wheelName"/>
-      </xsl:element>
+      </xsl:attribute>
     </xsl:element>
 
   </xsl:template>

tests/integration_tests/Test/PM/Integration/ProcessPythonWheel.cls

@@ -15,6 +15,20 @@ Parameter PackageWithPythonDepsLocation = "package-with-python-deps";
 
 Parameter ExportPackageWithPythonDepsLocation = "export-package-with-python-deps";
 
+Parameter PublishWithPythonDepsLocation = "publish-with-python-deps";
+
+Parameter ExportedPackagesDir = "/home/irisowner/zpm/tests/integration_tests/Test/PM/Integration/_data/python-deps-tests/export-package-with-python-deps/";
+
+Method OnAfterOneTest() As %Status
+{
+    // Make test environment clean by uninstalling all modules at the end of each test
+    set sc = ##class(%IPM.Main).Shell("uninstall -all")
+    do $$$AssertStatusOK(sc, "Successfully uninstalled all modules after test")
+    set sc = ##class(%IPM.Main).Shell("repo -delete-all")
+    do $$$AssertStatusOK(sc, "Successfully deleted all repositories after test")
+    return sc
+}
+
 /// The PythonWheel resource processor expects to install wheels from the dist directory, as opposed to source code
 ClassMethod GenerateWheel()
 {
@@ -178,25 +192,235 @@ Method TestPackageWithPythonDeps() As %Status
     set sc = ##class(%IPM.Main).Shell("install -v localrepo/package-with-python-deps")
     do $$$AssertStatusOK(sc, "Installed module from local filesystem repo")
 
-    set exportDir = ..GetModuleDir("python-deps-tests", ..#ExportPackageWithPythonDepsLocation)
-
-    set exportTarball = ..GetModuleDir("python-deps-tests", ..#ExportPackageWithPythonDepsLocation, "package-with-python-deps")
+    set exportPath = ..GetRandomExportPath()
+    set exportTarball = exportPath_".tgz"
 
     // Run package with the python-deps export flag
-    set sc = ##class(%IPM.Main).Shell("package package-with-python-deps -v -export-python-deps -path "_ exportTarball)
+    set sc = ##class(%IPM.Main).Shell("package package-with-python-deps -v -export-python-deps -path "_ exportPath)
     do $$$AssertStatusOK(sc, "Packaged module with -export-python-deps")
 
     set sc = ##class(%IPM.Main).Shell("uninstall package-with-python-deps")
     do $$$AssertStatusOK(sc, "Uninstalled module")
 
-    set sc = ##class(%IPM.Main).Shell("load -bypass-py-deps " _ exportDir _ "package-with-python-deps.tgz")
+
+    set wheels = { "wheels": ["ansible_core-2.19.4-py3-none-any.whl",
+                  "ansible_core-2.19.5-py3-none-any.whl",
+                  "ansible-12.2.0-py3-none-any.whl",
+                  "cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl",
+                  "cryptography-46.0.3-cp311-abi3-manylinux_2_34_x86_64.whl",
+                  "jinja2-3.1.6-py3-none-any.whl",
+                  "lune-1.6.4-py3-none-any.whl",
+                  "markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl",
+                  "packaging-25.0-py3-none-any.whl",
+                  "packaging-26.0-py3-none-any.whl",
+                  "pycparser-2.23-py3-none-any.whl",
+                  "pycparser-3.0-py3-none-any.whl",
+                  "pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl",
+                  "resolvelib-1.2.1-py3-none-any.whl"] }
+    do ..AssertWheelFilesExistInPackagedModule(exportTarball, wheels)
+
+    set sc = ##class(%IPM.Main).Shell("load -bypass-py-deps "_exportTarball)
     do $$$AssertStatusOK(sc, "Loaded packaged module")
 
     // Cleanup
     set sc = ##class(%IPM.Main).Shell("uninstall package-with-python-deps")
     do $$$AssertStatusOK(sc, "Uninstalled module")
+
     set sc = ##class(%IPM.Main).Shell("repo -delete -n localrepo")
     do $$$AssertStatusOK(sc, "Deleted local filesystem repo")
 }
 
+/// Omitting -export-python-deps flag should package without the python
+Method TestPackageWithoutPythonDeps() As %Status
+{
+    set dir = ..GetModuleDir("python-deps-tests", ..#PackageWithPythonDepsLocation)
+
+    // Use a local filesystem repo to install the test module into the namespace
+    set sc = ##class(%IPM.Main).Shell("repo -fs -name localrepo -path " _ dir)
+    do $$$AssertStatusOK(sc, "Configured local filesystem repo")
+
+    set sc = ##class(%IPM.Main).Shell("install -v localrepo/package-with-python-deps")
+    do $$$AssertStatusOK(sc, "Installed module from local filesystem repo")
+
+    set exportPath = ..GetRandomExportPath()
+    set exportTarball = exportPath_".tgz"
+
+    // Run package without the -export-python-deps flag
+    set sc = ##class(%IPM.Main).Shell("package package-with-python-deps -v -path "_ exportPath)
+    do $$$AssertStatusOK(sc, "Packaged module without -export-python-deps")
+
+    set sc = ##class(%IPM.Main).Shell("uninstall package-with-python-deps")
+    do $$$AssertStatusOK(sc, "Uninstalled module")
+
+
+    set wheels = { "wheels": ["ansible_core-2.19.4-py3-none-any.whl",
+                  "ansible_core-2.19.5-py3-none-any.whl",
+                  "ansible-12.2.0-py3-none-any.whl",
+                  "cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl",
+                  "cryptography-46.0.3-cp311-abi3-manylinux_2_34_x86_64.whl",
+                  "jinja2-3.1.6-py3-none-any.whl",
+                  "lune-1.6.4-py3-none-any.whl",
+                  "markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl",
+                  "packaging-25.0-py3-none-any.whl",
+                  "packaging-26.0-py3-none-any.whl",
+                  "pycparser-2.23-py3-none-any.whl",
+                  "pycparser-3.0-py3-none-any.whl",
+                  "pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl",
+                  "resolvelib-1.2.1-py3-none-any.whl"] }
+    do ..AssertWheelFilesExistInPackagedModule(exportTarball, wheels, 0)
+
+    set sc = ##class(%IPM.Main).Shell("repo -delete -n localrepo")
+    do $$$AssertStatusOK(sc, "Deleted local filesystem repo")
+}
+
+/// Test publishing with the -export-python-deps flag
+Method TestPublishWithPythonDeps() As %Status
+{
+
+    // 1. Test the case of python dependencies defined via requirements.txt
+    set moduleNameReq = "publish-with-python-just-reqs"
+    set wheelsReq = ["docx2md-1.0.5-py3-none-any.whl",
+                     "lxml-6.0.2-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl",
+                     "pillow-12.1.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl"]
+                     set wheels = {}
+    do wheels.%Set(moduleNameReq, wheelsReq)
+    do ..PublishTestHandler(moduleNameReq, wheels)
+
+    // 2. Test the case of python dependencies defined via wheel resources in module.xml
+    set moduleNameWhl = "publish-with-python-just-wheels"
+    set wheelsWhl = ["numpy-2.4.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl"]
+    set wheels = {}
+    do wheels.%Set(moduleNameWhl, wheelsWhl)
+    do ..PublishTestHandler(moduleNameWhl, wheels)
+
+    // 3. Test the case of python dependencies both defined in module.xml and requirements.txt
+    set moduleNameWhlReq = "publish-with-python-wheels-and-reqs"
+    set wheelsWhlReq = ["colorama-0.4.6-py2.py3-none-any.whl",
+                        "six-1.17.0-py2.py3-none-any.whl"]
+    set wheels = {}
+    do wheels.%Set(moduleNameWhlReq, wheelsWhlReq)
+    do ..PublishTestHandler(moduleNameWhlReq, wheels)
+
+
+
+    // 4. Test the three cases above but as dependencies of a base module being installed
+    set moduleNameDeps = "publish-with-python-deps-base"
+    // There are no wheels defined for this base module but each of the dependencies do have wheels to check
+    set wheels = {}
+    do wheels.%Set(moduleNameReq, wheelsReq)
+    do wheels.%Set(moduleNameWhl, wheelsWhl)
+    do wheels.%Set(moduleNameWhlReq, wheelsWhlReq)
+    do ..PublishTestHandler(moduleNameDeps, wheels)
+}
+
+/// Handler method for publish test suite
+Method PublishTestHandler(
+	moduleName As %String,
+	wheels As %DynamicObject)
+{
+    // Create a filesystem repository
+    set dir = ..GetModuleDir("python-deps-tests", ..#PublishWithPythonDepsLocation)
+    set sc = ##class(%IPM.Main).Shell("repo -fs -name localrepo -path "_dir)
+    do $$$AssertStatusOK(sc, "Configured local filesystem repo")
+
+    // Install module from local filesystem repo
+    set sc = ##class(%IPM.Main).Shell("install -v localrepo/"_moduleName)
+    do $$$AssertStatusOK(sc, "Installed module from local filesystem repo")
+
+    // Create a remote repository for publishing
+    set sc = ##class(%IPM.Main).Shell("repo -r -name remote -url http://registry:52773/registry -username admin -password SYS")
+    do $$$AssertStatusOK(sc, "Configured remote repo")
+
+    // Publish module to remote repository
+    set sc = ##class(%IPM.Main).Shell("publish -v "_moduleName_" -r remote -export-python-deps -export-deps 1")
+    do $$$AssertStatusOK(sc, "Published module "_moduleName_" to remote repository")
+
+    // Uninstall module that came from filesystem repository
+    set sc = ##class(%IPM.Main).Shell("uninstall "_moduleName)
+    do $$$AssertStatusOK(sc, "Uninstalled module successfully")
+
+    // Install published module from remote repository
+    set sc = ##class(%IPM.Main).Shell("install -v remote/"_moduleName)
+    do $$$AssertStatusOK(sc, "Successfully installed published module "_moduleName_" from remote repository")
+
+    // Confirm wheels were included in packaging/publishing
+    do ..AssertWheelResourcesExistForModule(wheels)
+
+    // Uninstall the module used for this test
+    set sc = ##class(%IPM.Main).Shell("uninstall "_moduleName)
+    do $$$AssertStatusOK(sc, "Uninstalled module "_moduleName_" successfully")
+
+    // Remove the repositories used for this test
+    set sc = ##class(%IPM.Main).Shell("repo -delete -n localrepo")
+    do $$$AssertStatusOK(sc, "Removed local filesystem repo")
+    set sc = ##class(%IPM.Main).Shell("repo -delete -n remote")
+    do $$$AssertStatusOK(sc, "Removed remote repo")
+}
+
+/// Used in package and publish tests to get a known, random directory path for where module gets packaged to
+Method GetRandomExportPath()
+{
+    set randomDir = $translate($system.Encryption.Base64Encode($system.Encryption.GenCryptRand(6)),"+/=")
+    return ..#ExportedPackagesDir_randomDir_"/package"
+}
+
+/// Checks to see if the physical files exist in a packaged module
+/// 
+/// Arguments:
+///     - exportTarball: path to the packaged tarball
+///     - wheels: Object of <relative path>-<wheel name> pairs array of file names to check for in the unpackaged module
+///               Ex: { "/": ["wheel-1.whl","wheel-2.whl"], "wheels":["wheel-3.whl","wheel-4.whl"] }
+///     - doesExist: set to 0 if we want to check that files explicitly do not exist (in the case -export-python-deps was not provided)
+Method AssertWheelFilesExistInPackagedModule(
+	exportTarball As %String,
+	wheels As %DynamicObject,
+	doesExist As %Boolean = 1)
+{
+    set exportDir = ##class(%File).GetDirectory(exportTarball)
+    set sc = ##class(%IPM.General.Archive).Extract(exportTarball, exportDir)
+    do $$$AssertStatusOK(sc, "Extracted packaged tarball contents to scan for wheel files")
+
+
+    set wheelsIter = wheels.%GetIterator()
+    while wheelsIter.%GetNext(.relativePath, .wheelArr) {
+        set directory = ##class(%File).NormalizeDirectory(relativePath, exportDir)
+        set wheelArrIter = wheelArr.%GetIterator()
+        while wheelArrIter.%GetNext(, .wheelName) {
+            set fileName = ##class(%File).NormalizeFilename(wheelName, directory)
+            set exists = ##class(%File).Exists(fileName)
+            if (doesExist) {
+                do $$$AssertTrue(exists, wheelName_" exists in packaged contents")
+            }
+            else {
+                do $$$AssertNotTrue(exists, wheelName_" not included in packaged contents")
+            }
+        }
+    }
+}
+
+/// Given an object of <module name>-<wheel list> pairs,
+/// compiles a list of resources for the module and iterates over the wheel list to check if that wheel is a resource of the module
+/// 
+/// Ex: wheels = { "module1": ["wheel-1.whl","wheel-2.whl"], "module2":["wheel-3.whl","wheel-4.whl"] }
+Method AssertWheelResourcesExistForModule(wheels As %DynamicObject)
+{
+    set wheelsIter = wheels.%GetIterator()
+    while wheelsIter.%GetNext(.moduleName, .moduleWheels) {
+        set module = ##class(%IPM.Storage.Module).NameOpen(moduleName)
+        set resourceList = ""
+        set key = ""
+        for {
+            set resource = module.Resources.GetNext(.key)
+            quit:key=""
+            set resourceList = resourceList_$listbuild($zconvert(resource.Name, "l"))
+        }
+
+        set moduleWheelsIter = moduleWheels.%GetIterator()
+        while moduleWheelsIter.%GetNext(, .wheel) {
+            set wheelIsResource = ($listfind(resourceList, wheel) > 0)
+            do $$$AssertTrue(wheelIsResource, "Wheel resource "_wheel_" is a part of the installed module "_moduleName)
+        }
+    }
+}
+
 }