Pants+Packaging: Add makeself_archive for each pack to facilitate installation

CHANGELOG.rst

@@ -79,6 +79,7 @@ Added
   to pants' use of PEX lockfiles. This is not a user-facing addition.
   #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244 #6251 #6253
   #6254 #6258 #6259 #6260 #6269 #6275 #6279 #6278 #6282 #6283 #6273 #6287 #6306 #6307
+  #6311
   Contributed by @cognifloyd
 * Build of ST2 EL9 packages #6153
   Contributed by @amanda11

contrib/chatops/BUILD

@@ -3,3 +3,11 @@ __defaults__(all=dict(inject_pack_python_path=True))
 pack_metadata(
     name="metadata",
 )
+
+st2_pack_archive(
+    dependencies=[
+        ":metadata",
+        "./actions",
+        "./tests",
+    ],
+)

contrib/chatops/tests/BUILD

@@ -1,5 +1,5 @@
 # tests can only be dependencies of other tests in this directory
-__dependents_rules__(("*", "/**", "!*"))
+__dependents_rules__(("*", "/**", f"//{build_file_dir().parent}:files", "!*"))
 
 __defaults__(
     {python_test: dict(tags=["pack"])},

contrib/core/BUILD

@@ -10,9 +10,20 @@ python_requirements(
 )
 
 python_sources(
+    # this is for fixture.py
     dependencies=[
         ":metadata",
         "./actions",
         "./actions/send_mail:send_mail_resources",
     ],
 )
+
+st2_pack_archive(
+    dependencies=[
+        ":metadata",
+        ":reqs",
+        "./actions",
+        "./actions/send_mail:send_mail_resources",
+        "./tests",
+    ],
+)

contrib/core/tests/BUILD

@@ -1,5 +1,5 @@
 # tests can only be dependencies of other tests in this directory
-__dependents_rules__(("*", "/**", "!*"))
+__dependents_rules__(("*", "/**", f"//{build_file_dir().parent}:files", "!*"))
 
 __defaults__(
     {python_test: dict(tags=["pack"])},

contrib/debug/BUILD

@@ -3,3 +3,10 @@ __defaults__(all=dict(inject_pack_python_path=True))
 pack_metadata(
     name="metadata",
 )
+
+st2_pack_archive(
+    dependencies=[
+        ":metadata",
+        "./actions",
+    ],
+)

contrib/default/BUILD

@@ -3,3 +3,9 @@ __defaults__(all=dict(inject_pack_python_path=True))
 pack_metadata(
     name="metadata",
 )
+
+st2_pack_archive(
+    dependencies=[
+        ":metadata",
+    ],
+)

contrib/examples/BUILD

@@ -18,3 +18,29 @@ python_requirement(
     name="flask",
     requirements=["flask"],
 )
+
+# Also capture the requirements file for distribution in the pack archive.
+files(
+    name="reqs",
+    sources=["requirements*.txt"],
+)
+
+st2_pack_archive(
+    # we need to list targets of all files because transitive dep targets are NOT included
+    dependencies=[
+        ":metadata",
+        ":reqs",
+        "./actions",
+        "./actions:shell",
+        "./actions/bash_exit_code",
+        "./actions/bash_ping",
+        "./actions/bash_random",
+        "./actions/pythonactions",
+        "./actions/ubuntu_pkg_info",
+        "./actions/ubuntu_pkg_info/lib",
+        "./actions/windows",
+        "./lib",
+        "./sensors",
+        "./tests",
+    ],
+)

contrib/examples/actions/windows/BUILD

@@ -0,0 +1,3 @@
+files(
+    sources=["*.ps1"],
+)

contrib/examples/tests/BUILD

@@ -1,5 +1,5 @@
 # tests can only be dependencies of other tests in this directory
-__dependents_rules__(("*", "/**", "!*"))
+__dependents_rules__(("*", "/**", f"//{build_file_dir().parent}:files", "!*"))
 
 __defaults__(
     {python_test: dict(tags=["pack"])},

contrib/hello_st2/BUILD

@@ -3,3 +3,19 @@ __defaults__(all=dict(inject_pack_python_path=True))
 pack_metadata(
     name="metadata",
 )
+
+# Capture the requirements file for distribution in the pack archive;
+# we do not need to use `python_requirements()` for this sample file.
+files(
+    name="reqs",
+    sources=["requirements*.txt"],
+)
+
+st2_pack_archive(
+    dependencies=[
+        ":metadata",
+        ":reqs",
+        "./actions",
+        "./sensors",
+    ],
+)

contrib/linux/BUILD

@@ -7,3 +7,15 @@ pack_metadata(
 python_requirements(
     name="reqs",
 )
+
+st2_pack_archive(
+    dependencies=[
+        ":metadata",
+        ":reqs",
+        "./actions",
+        "./actions/checks",
+        # "./actions/lib",  # nothing in dir
+        "./sensors",
+        "./tests",
+    ],
+)

contrib/linux/tests/BUILD

@@ -1,5 +1,5 @@
 # tests can only be dependencies of other tests in this directory
-__dependents_rules__(("*", "/**", "!*"))
+__dependents_rules__(("*", "/**", f"//{build_file_dir().parent}:files", "!*"))
 
 __defaults__(
     {python_test: dict(tags=["pack"])},

contrib/packs/BUILD

@@ -3,3 +3,12 @@ __defaults__(all=dict(inject_pack_python_path=True))
 pack_metadata(
     name="metadata",
 )
+
+st2_pack_archive(
+    dependencies=[
+        ":metadata",
+        "./actions",
+        "./actions/pack_mgmt",
+        "./tests",
+    ],
+)

contrib/packs/tests/BUILD

@@ -1,5 +1,5 @@
 # tests can only be dependencies of other tests in this directory
-__dependents_rules__(("*", "/**", "!*"))
+__dependents_rules__(("*", "/**", f"//{build_file_dir().parent}:files", "!*"))
 
 __defaults__(
     {python_test: dict(tags=["pack"])},

pants-plugins/macros.py

@@ -112,6 +112,65 @@ def st2_component_python_distribution(**kwargs):
     python_distribution(**kwargs)  # noqa: F821
 
 
+# Default copied from PEX (which uses zipfile standard MS-DOS epoch).
+# https://github.com/pex-tool/pex/blob/v2.1.137/pex/common.py#L39-L45
+MTIME = "1980-01-01T00:00:00Z"
+
+# These are used for system packages (rpm/deb)
+ST2_PACKS_GROUP = "st2packs"
+ST2_SVC_USER = "st2"
+
+
+def st2_pack_archive(**kwargs):
+    """Create a makeself_archive using files from the given dependencies.
+
+    This macro should be used in the same BUILD file as the pack_metadata target.
+    """
+    build_file_path = build_file_dir()  # noqa: F821
+    if "st2tests" == build_file_path.parts[0]:
+        # avoid creating duplicate archive for the core pack
+        # which is also located under st2tests/st2tests/fixtures/packs
+        return
+    pack_name = build_file_path.name  # noqa: F821
+
+    dependencies = kwargs.pop("dependencies", [])
+    if ":metadata" not in dependencies:
+        dependencies = [":metadata", *dependencies]
+
+    # This is basically a "wrap_as_files" target (which does not exist yet)
+    shell_command(  # noqa: F821
+        name="files",
+        execution_dependencies=dependencies,
+        command="true",
+        output_directories=["."],
+        root_output_directory=".",
+    )
+
+    # https://www.pantsbuild.org/stable/docs/shell/self-extractable-archives
+    # https://www.pantsbuild.org/stable/reference/targets/makeself_archive
+    makeself_archive(  # noqa: F821
+        name="archive",
+        label=f"{pack_name} StackStorm pack",
+        files=[
+            ":files",  # archive contents
+            "//:license",  # LICENSE file included in archive header, excluded from contents
+        ],
+        args=(  # see: https://makeself.io/#usage
+            # Makeself expects '--arg value' (space) not '--arg=value' (equals) for cmdline
+            "--license",
+            "__archive/LICENSE",
+            "--target",
+            f"/opt/stackstorm/packs/{pack_name}",
+            # reproducibility flags:
+            "--tar-extra",  # extra tar args: '--arg=value' (equals delimited) space separated
+            f"--owner=root --group={ST2_PACKS_GROUP} --mtime={MTIME} --exclude=LICENSE",
+            "--packaging-date",
+            MTIME,
+        ),
+        output_path=f"packaging/packs/{pack_name}.tgz.run",
+    )
+
+
 def st2_shell_sources_and_resources(**kwargs):
     """This creates a shell_sources and a resources target.
 

pants-plugins/pack_metadata/python_rules/python_pack_content_test.py

@@ -36,13 +36,14 @@
         ((PackContentResourceTypes.pack_config_schema,), 4, "config.schema.yaml"),
         ((PackContentResourceTypes.pack_config_example,), 4, "config.yaml.example"),
         ((PackContentResourceTypes.pack_icon,), 4, "icon.png"),
+        ((PackContentResourceTypes.pack_doc,), 4, ".md"),
         ((PackContentResourceTypes.action_metadata,), 5, ".yaml"),
         ((PackContentResourceTypes.sensor_metadata,), 1, ".yaml"),
         ((PackContentResourceTypes.rule_metadata,), 0, ""),
         ((PackContentResourceTypes.policy_metadata,), 0, ""),
         ((PackContentResourceTypes.unknown,), 0, ""),
         # all content types
-        ((), 22, ""),
+        ((), 26, ""),
         # some content types
         (
             (

pants-plugins/pack_metadata/target_types.py

@@ -43,6 +43,7 @@ class PackContentResourceTypes(Enum):
     pack_config_schema = "pack_config_schema"
     pack_config_example = "pack_config_example"
     pack_icon = "pack_icon"
+    pack_doc = "pack_doc"
     # in subdirectory (see _content_type_by_path_parts below
     action_metadata = "action_metadata"
     action_chain_workflow = "action_chain_workflow"
@@ -86,6 +87,8 @@ def compute_value(
         if value is not None:
             return PackContentResourceTypes(value)
         path = PurePath(address.relative_file_path)
+        if path.suffix == ".md":
+            return PackContentResourceTypes.pack_doc
         _yaml_suffixes = (".yaml", ".yml")
         if len(path.parent.parts) == 0:
             # in the pack root
@@ -123,8 +126,7 @@ class PackMetadataSourcesField(ResourcesGeneratingSourcesField):
         "**/*.yml",
         "icon.png",  # used in st2web ui
         # "requirements*.txt",  # including this causes target conflicts
-        # "README.md",
-        # "HISTORY.md",
+        "**/*.md",  # including README.md, HISTORY.md
         # exclude yaml files under tests
         "!tests/**/*.yml",
         "!tests/**/*.yaml",

pants.toml

@@ -27,6 +27,9 @@ backend_packages = [
   "pants.backend.shell",
   "pants.backend.shell.lint.shellcheck",
 
+  # packaging
+  "pants.backend.experimental.makeself",
+
   # internal plugins in pants-plugins/
   "pants.backend.plugin_development",
   "api_spec",
@@ -43,8 +46,6 @@ pants_ignore.add = [
   "test_dist_utils.py",
   "setup.py",
   # keep tailor from using legacy requirements files (not for pants)
-  "contrib/examples/requirements.txt",
-  "contrib/hello_st2/requirements.txt",
   "contrib/runners/*/in-requirements.txt",
   "contrib/runners/*/requirements.txt",
   "st2*/in-requirements.txt",