diff --git a/CHANGELOG.rst b/CHANGELOG.rst index aa78a80b33..2b25f64e1e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -79,6 +79,7 @@ Added to pants' use of PEX lockfiles. This is not a user-facing addition. #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244 #6251 #6253 #6254 #6258 #6259 #6260 #6269 #6275 #6279 #6278 #6282 #6283 #6273 #6287 #6306 #6307 + #6311 Contributed by @cognifloyd * Build of ST2 EL9 packages #6153 Contributed by @amanda11 diff --git a/contrib/chatops/BUILD b/contrib/chatops/BUILD index 888be3a426..c9f2a2d183 100644 --- a/contrib/chatops/BUILD +++ b/contrib/chatops/BUILD @@ -3,3 +3,11 @@ __defaults__(all=dict(inject_pack_python_path=True)) pack_metadata( name="metadata", ) + +st2_pack_archive( + dependencies=[ + ":metadata", + "./actions", + "./tests", + ], +) diff --git a/contrib/chatops/tests/BUILD b/contrib/chatops/tests/BUILD index ead8561daa..63c5ba6b03 100644 --- a/contrib/chatops/tests/BUILD +++ b/contrib/chatops/tests/BUILD @@ -1,5 +1,5 @@ # tests can only be dependencies of other tests in this directory -__dependents_rules__(("*", "/**", "!*")) +__dependents_rules__(("*", "/**", f"//{build_file_dir().parent}:files", "!*")) __defaults__( {python_test: dict(tags=["pack"])}, diff --git a/contrib/core/BUILD b/contrib/core/BUILD index 9df7a372c9..e175664166 100644 --- a/contrib/core/BUILD +++ b/contrib/core/BUILD @@ -10,9 +10,20 @@ python_requirements( ) python_sources( + # this is for fixture.py dependencies=[ ":metadata", "./actions", "./actions/send_mail:send_mail_resources", ], ) + +st2_pack_archive( + dependencies=[ + ":metadata", + ":reqs", + "./actions", + "./actions/send_mail:send_mail_resources", + "./tests", + ], +) diff --git a/contrib/core/tests/BUILD b/contrib/core/tests/BUILD index 1a4583beb6..c84e095e16 100644 --- a/contrib/core/tests/BUILD +++ b/contrib/core/tests/BUILD @@ -1,5 +1,5 @@ # tests can only be dependencies of other tests in this directory -__dependents_rules__(("*", "/**", "!*")) +__dependents_rules__(("*", "/**", f"//{build_file_dir().parent}:files", "!*")) __defaults__( {python_test: dict(tags=["pack"])}, diff --git a/contrib/debug/BUILD b/contrib/debug/BUILD index 888be3a426..4713a9b1cc 100644 --- a/contrib/debug/BUILD +++ b/contrib/debug/BUILD @@ -3,3 +3,10 @@ __defaults__(all=dict(inject_pack_python_path=True)) pack_metadata( name="metadata", ) + +st2_pack_archive( + dependencies=[ + ":metadata", + "./actions", + ], +) diff --git a/contrib/default/BUILD b/contrib/default/BUILD index 888be3a426..fc15b27bea 100644 --- a/contrib/default/BUILD +++ b/contrib/default/BUILD @@ -3,3 +3,9 @@ __defaults__(all=dict(inject_pack_python_path=True)) pack_metadata( name="metadata", ) + +st2_pack_archive( + dependencies=[ + ":metadata", + ], +) diff --git a/contrib/examples/BUILD b/contrib/examples/BUILD index ab10cd1c85..22d0401478 100644 --- a/contrib/examples/BUILD +++ b/contrib/examples/BUILD @@ -18,3 +18,29 @@ python_requirement( name="flask", requirements=["flask"], ) + +# Also capture the requirements file for distribution in the pack archive. +files( + name="reqs", + sources=["requirements*.txt"], +) + +st2_pack_archive( + # we need to list targets of all files because transitive dep targets are NOT included + dependencies=[ + ":metadata", + ":reqs", + "./actions", + "./actions:shell", + "./actions/bash_exit_code", + "./actions/bash_ping", + "./actions/bash_random", + "./actions/pythonactions", + "./actions/ubuntu_pkg_info", + "./actions/ubuntu_pkg_info/lib", + "./actions/windows", + "./lib", + "./sensors", + "./tests", + ], +) diff --git a/contrib/examples/actions/windows/BUILD b/contrib/examples/actions/windows/BUILD new file mode 100644 index 0000000000..47dc872204 --- /dev/null +++ b/contrib/examples/actions/windows/BUILD @@ -0,0 +1,3 @@ +files( + sources=["*.ps1"], +) diff --git a/contrib/examples/tests/BUILD b/contrib/examples/tests/BUILD index 25a2e7cc4b..0ddb50de63 100644 --- a/contrib/examples/tests/BUILD +++ b/contrib/examples/tests/BUILD @@ -1,5 +1,5 @@ # tests can only be dependencies of other tests in this directory -__dependents_rules__(("*", "/**", "!*")) +__dependents_rules__(("*", "/**", f"//{build_file_dir().parent}:files", "!*")) __defaults__( {python_test: dict(tags=["pack"])}, diff --git a/contrib/hello_st2/BUILD b/contrib/hello_st2/BUILD index 888be3a426..6938233485 100644 --- a/contrib/hello_st2/BUILD +++ b/contrib/hello_st2/BUILD @@ -3,3 +3,19 @@ __defaults__(all=dict(inject_pack_python_path=True)) pack_metadata( name="metadata", ) + +# Capture the requirements file for distribution in the pack archive; +# we do not need to use `python_requirements()` for this sample file. +files( + name="reqs", + sources=["requirements*.txt"], +) + +st2_pack_archive( + dependencies=[ + ":metadata", + ":reqs", + "./actions", + "./sensors", + ], +) diff --git a/contrib/linux/BUILD b/contrib/linux/BUILD index 201435eecc..e4619b05cb 100644 --- a/contrib/linux/BUILD +++ b/contrib/linux/BUILD @@ -7,3 +7,15 @@ pack_metadata( python_requirements( name="reqs", ) + +st2_pack_archive( + dependencies=[ + ":metadata", + ":reqs", + "./actions", + "./actions/checks", + # "./actions/lib", # nothing in dir + "./sensors", + "./tests", + ], +) diff --git a/contrib/linux/tests/BUILD b/contrib/linux/tests/BUILD index 25a2e7cc4b..0ddb50de63 100644 --- a/contrib/linux/tests/BUILD +++ b/contrib/linux/tests/BUILD @@ -1,5 +1,5 @@ # tests can only be dependencies of other tests in this directory -__dependents_rules__(("*", "/**", "!*")) +__dependents_rules__(("*", "/**", f"//{build_file_dir().parent}:files", "!*")) __defaults__( {python_test: dict(tags=["pack"])}, diff --git a/contrib/packs/BUILD b/contrib/packs/BUILD index 888be3a426..cb69238387 100644 --- a/contrib/packs/BUILD +++ b/contrib/packs/BUILD @@ -3,3 +3,12 @@ __defaults__(all=dict(inject_pack_python_path=True)) pack_metadata( name="metadata", ) + +st2_pack_archive( + dependencies=[ + ":metadata", + "./actions", + "./actions/pack_mgmt", + "./tests", + ], +) diff --git a/contrib/packs/tests/BUILD b/contrib/packs/tests/BUILD index c8265214ca..6326aced98 100644 --- a/contrib/packs/tests/BUILD +++ b/contrib/packs/tests/BUILD @@ -1,5 +1,5 @@ # tests can only be dependencies of other tests in this directory -__dependents_rules__(("*", "/**", "!*")) +__dependents_rules__(("*", "/**", f"//{build_file_dir().parent}:files", "!*")) __defaults__( {python_test: dict(tags=["pack"])}, diff --git a/pants-plugins/macros.py b/pants-plugins/macros.py index 11131f20ee..0279408d08 100644 --- a/pants-plugins/macros.py +++ b/pants-plugins/macros.py @@ -112,6 +112,65 @@ def st2_component_python_distribution(**kwargs): python_distribution(**kwargs) # noqa: F821 +# Default copied from PEX (which uses zipfile standard MS-DOS epoch). +# https://github.com/pex-tool/pex/blob/v2.1.137/pex/common.py#L39-L45 +MTIME = "1980-01-01T00:00:00Z" + +# These are used for system packages (rpm/deb) +ST2_PACKS_GROUP = "st2packs" +ST2_SVC_USER = "st2" + + +def st2_pack_archive(**kwargs): + """Create a makeself_archive using files from the given dependencies. + + This macro should be used in the same BUILD file as the pack_metadata target. + """ + build_file_path = build_file_dir() # noqa: F821 + if "st2tests" == build_file_path.parts[0]: + # avoid creating duplicate archive for the core pack + # which is also located under st2tests/st2tests/fixtures/packs + return + pack_name = build_file_path.name # noqa: F821 + + dependencies = kwargs.pop("dependencies", []) + if ":metadata" not in dependencies: + dependencies = [":metadata", *dependencies] + + # This is basically a "wrap_as_files" target (which does not exist yet) + shell_command( # noqa: F821 + name="files", + execution_dependencies=dependencies, + command="true", + output_directories=["."], + root_output_directory=".", + ) + + # https://www.pantsbuild.org/stable/docs/shell/self-extractable-archives + # https://www.pantsbuild.org/stable/reference/targets/makeself_archive + makeself_archive( # noqa: F821 + name="archive", + label=f"{pack_name} StackStorm pack", + files=[ + ":files", # archive contents + "//:license", # LICENSE file included in archive header, excluded from contents + ], + args=( # see: https://makeself.io/#usage + # Makeself expects '--arg value' (space) not '--arg=value' (equals) for cmdline + "--license", + "__archive/LICENSE", + "--target", + f"/opt/stackstorm/packs/{pack_name}", + # reproducibility flags: + "--tar-extra", # extra tar args: '--arg=value' (equals delimited) space separated + f"--owner=root --group={ST2_PACKS_GROUP} --mtime={MTIME} --exclude=LICENSE", + "--packaging-date", + MTIME, + ), + output_path=f"packaging/packs/{pack_name}.tgz.run", + ) + + def st2_shell_sources_and_resources(**kwargs): """This creates a shell_sources and a resources target. diff --git a/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py b/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py index 33c1389bb3..97bcb0ccfc 100644 --- a/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py +++ b/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py @@ -36,13 +36,14 @@ ((PackContentResourceTypes.pack_config_schema,), 4, "config.schema.yaml"), ((PackContentResourceTypes.pack_config_example,), 4, "config.yaml.example"), ((PackContentResourceTypes.pack_icon,), 4, "icon.png"), + ((PackContentResourceTypes.pack_doc,), 4, ".md"), ((PackContentResourceTypes.action_metadata,), 5, ".yaml"), ((PackContentResourceTypes.sensor_metadata,), 1, ".yaml"), ((PackContentResourceTypes.rule_metadata,), 0, ""), ((PackContentResourceTypes.policy_metadata,), 0, ""), ((PackContentResourceTypes.unknown,), 0, ""), # all content types - ((), 22, ""), + ((), 26, ""), # some content types ( ( diff --git a/pants-plugins/pack_metadata/target_types.py b/pants-plugins/pack_metadata/target_types.py index 01d80c24ad..8fa0c8843b 100644 --- a/pants-plugins/pack_metadata/target_types.py +++ b/pants-plugins/pack_metadata/target_types.py @@ -43,6 +43,7 @@ class PackContentResourceTypes(Enum): pack_config_schema = "pack_config_schema" pack_config_example = "pack_config_example" pack_icon = "pack_icon" + pack_doc = "pack_doc" # in subdirectory (see _content_type_by_path_parts below action_metadata = "action_metadata" action_chain_workflow = "action_chain_workflow" @@ -86,6 +87,8 @@ def compute_value( if value is not None: return PackContentResourceTypes(value) path = PurePath(address.relative_file_path) + if path.suffix == ".md": + return PackContentResourceTypes.pack_doc _yaml_suffixes = (".yaml", ".yml") if len(path.parent.parts) == 0: # in the pack root @@ -123,8 +126,7 @@ class PackMetadataSourcesField(ResourcesGeneratingSourcesField): "**/*.yml", "icon.png", # used in st2web ui # "requirements*.txt", # including this causes target conflicts - # "README.md", - # "HISTORY.md", + "**/*.md", # including README.md, HISTORY.md # exclude yaml files under tests "!tests/**/*.yml", "!tests/**/*.yaml", diff --git a/pants.toml b/pants.toml index a2ae99b718..092dc72028 100644 --- a/pants.toml +++ b/pants.toml @@ -27,6 +27,9 @@ backend_packages = [ "pants.backend.shell", "pants.backend.shell.lint.shellcheck", + # packaging + "pants.backend.experimental.makeself", + # internal plugins in pants-plugins/ "pants.backend.plugin_development", "api_spec", @@ -43,8 +46,6 @@ pants_ignore.add = [ "test_dist_utils.py", "setup.py", # keep tailor from using legacy requirements files (not for pants) - "contrib/examples/requirements.txt", - "contrib/hello_st2/requirements.txt", "contrib/runners/*/in-requirements.txt", "contrib/runners/*/requirements.txt", "st2*/in-requirements.txt",