Packaging: Add venv and packs to deb/rpm packages (#6328)

Author: cognifloyd

CHANGELOG.rst

@@ -79,7 +79,7 @@ Added
   to pants' use of PEX lockfiles. This is not a user-facing addition.
   #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244 #6251 #6253
   #6254 #6258 #6259 #6260 #6269 #6275 #6279 #6278 #6282 #6283 #6273 #6287 #6306 #6307
-  #6311 #6314 #6315 #6317 #6319 #6312 #6320 #6321 #6323 #6324 #6325 #6326 #6327
+  #6311 #6314 #6315 #6317 #6319 #6312 #6320 #6321 #6323 #6324 #6325 #6326 #6327 #6328
   Contributed by @cognifloyd
 * Build of ST2 EL9 packages #6153
   Contributed by @amanda11

packaging/BUILD

@@ -22,6 +22,15 @@ _common_pkg_metadata = dict(
 _maintainer = "StackStorm Engineering <[email protected]>"  # TODO: update this
 
 
+def _st2_venv_deps() -> list[str]:
+    """Return a list of deps required to build the st2 venv.
+
+    The :st2_venv parametrizations depend on parametrizations of :st2.pex,
+    so, make sure to keep these 3 lists of parametrizations in sync.
+    """
+    return [f":st2_venv@parametrize=py3{m}" for m in ("8", "9", "10", "11")]
+
+
 def _distro(distro_id: str, **kwargs):
     return parametrize(
         distro_id,
@@ -38,6 +47,9 @@ nfpm_deb_package(
         "./common/systemd:generators",
         "./common:dirs",
         "./common:symlinks",
+        *_st2_venv_deps(),
+        ":default_packs",
+        ":extra_packs",
     ],
     scripts=dict(
         preinstall="deb/scripts/pre-install.sh",
@@ -70,6 +82,12 @@ nfpm_rpm_package(
         "./common/systemd:generators",
         "./common:dirs",
         "./common:symlinks",
+        *_st2_venv_deps(),
+        ":default_packs",
+        ":extra_packs",
+    ],
+    ghost_contents=[
+        "/opt/stackstorm/install/st2.pex",  # a symlink (default in post-install; users may modify)
     ],
     scripts=dict(
         preinstall="rpm/scripts/pre-install.sh",
@@ -83,7 +101,8 @@ nfpm_rpm_package(
     vendor="The StackStorm Project",
     packager=_maintainer,
     # group="System/Management",  # was only useful for EL 5 and earlier
+    compression="zstd:best",  # xz and lzma are ~3x slower than gzip or zstd
     **_common_pkg_metadata,
-    **_distro("el8", compression="xz"),
-    **_distro("el9", compression="zstd:default"),
+    **_distro("el8"),
+    **_distro("el9"),
 )

packaging/BUILD.packs

@@ -0,0 +1,25 @@
+# These are packs that should be installed with the default ST2 install.
+_DEFAULT_PACKS = (
+    "chatops",
+    "core",
+    "default",
+    "linux",
+    "packs",
+)
+
+# These are packs that should NOT be installed with the default ST2 install.
+_EXTRA_PACKS = (
+    "debug",
+    "examples",
+    "hello_st2",
+)
+
+# :archive_for_npfm targets created by st2_pack_archive() macro (see pants-plugins/macros.py)
+target(
+    name="default_packs",
+    dependencies=[f"//contrib/{pack}:archive_for_nfpm" for pack in _DEFAULT_PACKS],
+)
+target(
+    name="extra_packs",
+    dependencies=[f"//contrib/{pack}:archive_for_nfpm" for pack in _EXTRA_PACKS],
+)

packaging/BUILD.venv

@@ -23,6 +23,7 @@ python_sources()
 
 
 def _pex_py3(minor: str, constraint: str = ""):
+    """Parametrize :st2.pex for a given python minor version."""
     if not constraint:
         constraint = f"CPython==3.{minor}.*"
     return parametrize(
@@ -71,3 +72,26 @@ pex_binary(
     **_pex_py3("10"),
     **_pex_py3("11"),
 )
+
+
+def _venv_py3(minor: str):
+    """Parametrize :st2_venv for a given python minor version."""
+    return parametrize(
+        f"py3{minor}",
+        dependencies=[f":st2.pex@parametrize=py3{minor}"],
+        src=f"st2-py3{minor}.pex",  # relative to this BUILD file
+        dst=f"/opt/stackstorm/install/st2-py3{minor}.pex",
+    )
+
+
+nfpm_content_file(
+    name="st2_venv",
+    description="Pex file that system packages can use to generate /opt/stackstorm/st2",
+    file_owner="root",
+    file_group="root",
+    file_mode="rwxr-x---",
+    **_venv_py3("8"),
+    **_venv_py3("9"),
+    **_venv_py3("10"),
+    **_venv_py3("11"),
+)

packaging/deb/scripts/post-install.sh

@@ -29,6 +29,9 @@ set -e
 # https://www.mankier.com/5/deb-triggers
 # https://stackoverflow.com/questions/15276535/dpkg-how-to-use-trigger
 
+# The supported minor versions of python3 (python3.{minor}) in reverse order.
+_ST2_PY3_MINOR="11 10 9 8"
+
 # The default set of packs installed with st2.
 _ST2_PACKS="
 chatops
@@ -93,7 +96,24 @@ systemd_enable_and_restart() {
 }
 
 rebuild_st2_venv() {
-    /opt/stackstorm/install/st2.pex
+    _pex="/opt/stackstorm/install/st2.pex"
+    if [ ! -e "${_pex}" ]; then
+        # symlink does not exist or does not point to a valid file
+        # (the symlink target might not exist any more if upgrading
+        # to an st2 version that drops support for an older python)
+        rm -f "${_pex}"
+        for minor in ${_ST2_PY3_MINOR}; do
+            if [ -x "/usr/bin/python3.${minor}" ]; then
+                ln -s "st2-py3${minor}.pex" "${_pex}"
+                break
+            fi
+        done
+        if [ ! -e "${_pex}" ]; then
+            # symlink creation failed; the python dep is somehow missing
+            return 42
+        fi
+    fi
+    "${_pex}"
 }
 
 extract_st2_pack() {

packaging/rpm/scripts/post-install.sh

@@ -7,6 +7,9 @@ set -e
 #   * on upgrade: $1 > 1
 # https://docs.fedoraproject.org/en-US/packaging-guidelines/Scriptlets/#_syntax
 
+# The supported minor versions of python3 (python3.{minor}) in reverse order.
+_ST2_PY3_MINOR="11 10 9 8"
+
 # The default set of packs installed with st2.
 _ST2_PACKS="
 chatops
@@ -31,7 +34,24 @@ st2workflowengine
 "
 
 rebuild_st2_venv() {
-    /opt/stackstorm/install/st2.pex
+    _pex="/opt/stackstorm/install/st2.pex"
+    if [ ! -e "${_pex}" ]; then
+        # symlink does not exist or does not point to a valid file
+        # (the symlink target might not exist any more if upgrading
+        # to an st2 version that drops support for an older python)
+        rm -f "${_pex}"
+        for minor in ${_ST2_PY3_MINOR}; do
+            if [ -x "/usr/bin/python3.${minor}" ]; then
+                ln -s "st2-py3${minor}.pex" "${_pex}"
+                break
+            fi
+        done
+        if [ ! -e "${_pex}" ]; then
+            # symlink creation failed; the python dep is somehow missing
+            return 42
+        fi
+    fi
+    "${_pex}"
 }
 
 extract_st2_pack() {

pants-plugins/macros.py

@@ -170,6 +170,16 @@ def st2_pack_archive(**kwargs):
         output_path=f"packaging/packs/{pack_name}.tgz.run",
     )
 
+    nfpm_content_file(  # noqa: F821
+        name="archive_for_nfpm",
+        dependencies=[":archive"],
+        src=f"packaging/packs/{pack_name}.tgz.run",
+        dst=f"/opt/stackstorm/install/packs/{pack_name}.tgz.run",
+        file_owner="root",
+        file_group=ST2_PACKS_GROUP,
+        file_mode="rwxr-x---",
+    )
+
 
 def st2_shell_sources_and_resources(**kwargs):
     """This creates a shell_sources and a resources target.

pants-plugins/release/packagecloud_rules.py

@@ -74,7 +74,7 @@
 }
 
 
-@dataclass
+@dataclass(frozen=True)
 class PackageCloudNextReleaseRequest:
     nfpm_arch: str
     distro_id: str
@@ -83,7 +83,7 @@ class PackageCloudNextReleaseRequest:
     production: bool
 
 
-@dataclass
+@dataclass(frozen=True)
 class PackageCloudNextRelease:
     value: Optional[int] = None
 

pants.toml

@@ -109,7 +109,9 @@ root_patterns = [
 # DEFAULT has values that we can reuse/interpolate below
 [DEFAULT]
 # This is the range of python versions that we support.
-# On update, make sure to also update parametrizations in packaging/BUILD*.
+# On update, make sure to also update:
+#   - the parametrizations in packaging/BUILD and packaging/BUILD.venv
+#   - the list of python minor versions in packaging/*/scripts/post-install.sh
 st2_interpreter_constraints = "CPython>=3.8.1,<3.12"
 
 # This should match the pants interpreter_constraints: