diff --git a/changelogs/fragments/2231-zos_fetch-interface-update.yml b/changelogs/fragments/2231-zos_fetch-interface-update.yml new file mode 100644 index 000000000..4f603783f --- /dev/null +++ b/changelogs/fragments/2231-zos_fetch-interface-update.yml @@ -0,0 +1,3 @@ +breaking_changes: + - zos_fetch - Return value ``file`` is replaced by ``src``. Return value ``note`` is deprecated, the messages thrown in ``note`` are now returned in ``msg``. + (https://github.com/ansible-collections/ibm_zos_core/pull/2231). diff --git a/plugins/action/zos_fetch.py b/plugins/action/zos_fetch.py index 91707f61f..2da09c0d1 100644 --- a/plugins/action/zos_fetch.py +++ b/plugins/action/zos_fetch.py @@ -1,4 +1,4 @@ -# Copyright (c) IBM Corporation 2019, 2024 +# Copyright (c) IBM Corporation 2019, 2025 # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -57,7 +57,7 @@ def _update_result(result, src, dest, ds_type="USS", is_binary=False): updated_result = dict((k, v) for k, v in result.items()) updated_result.update( { - "file": src, + "src": src, "dest": dest, "data_set_type": data_set_types[ds_type], "is_binary": is_binary, @@ -121,6 +121,7 @@ def run(self, tmp=None, task_vars=None): dest = self._task.args.get('dest') encoding = self._task.args.get('encoding', None) flat = _process_boolean(self._task.args.get('flat'), default=False) + fail_on_missing = _process_boolean(self._task.args.get('fail_on_missing'), default=True) is_binary = _process_boolean(self._task.args.get('is_binary')) ignore_sftp_stderr = _process_boolean( self._task.args.get("ignore_sftp_stderr"), default=True @@ -186,29 +187,55 @@ def run(self, tmp=None, task_vars=None): task_vars=task_vars ) ds_type = fetch_res.get("ds_type") - src = fetch_res.get("file") + src = fetch_res.get("src") remote_path = fetch_res.get("remote_path") - - if fetch_res.get("msg"): - result["msg"] = fetch_res.get("msg") + # Create a dictionary that is a schema for the return values + result = dict( + src="", + dest="", + is_binary=False, + checksum="", + changed=False, + data_set_type="", + msg="", + stdout="", + stderr="", + stdout_lines=[], + stderr_lines=[], + rc=0, + encoding=new_module_args.get("encoding"), + ) + # Populate it with the modules response + result["src"] = fetch_res.get("src") + result["dest"] = fetch_res.get("dest") + result["is_binary"] = fetch_res.get("is_binary", False) + result["checksum"] = fetch_res.get("checksum") + result["changed"] = fetch_res.get("changed", False) + result["data_set_type"] = fetch_res.get("data_set_type") + result["msg"] = fetch_res.get("msg") + result["stdout"] = fetch_res.get("stdout") + result["stderr"] = fetch_res.get("stderr") + result["stdout_lines"] = fetch_res.get("stdout_lines") + result["stderr_lines"] = fetch_res.get("stderr_lines") + result["rc"] = fetch_res.get("rc", 0) + result["encoding"] = fetch_res.get("encoding") + + if fetch_res.get("failed", False): result["stdout"] = fetch_res.get("stdout") or fetch_res.get( "module_stdout" ) result["stderr"] = fetch_res.get("stderr") or fetch_res.get( "module_stderr" ) - result["stdout_lines"] = fetch_res.get("stdout_lines") - result["stderr_lines"] = fetch_res.get("stderr_lines") - result["rc"] = fetch_res.get("rc") result["failed"] = True return result - - elif fetch_res.get("note"): - result["note"] = fetch_res.get("note") + if "No data was fetched." in result["msg"]: + if fail_on_missing: + result["failed"] = True return result except Exception as err: - result["msg"] = "Failure during module execution" + result["msg"] = f"Failure during module execution {msg}" result["stderr"] = str(err) result["stderr_lines"] = str(err).splitlines() result["failed"] = True @@ -229,7 +256,6 @@ def run(self, tmp=None, task_vars=None): # For instance: If src is: USER.TEST.PROCLIB(DATA) # # and dest is: /tmp/, then updated dest would be /tmp/DATA # # ********************************************************** # - if os.path.sep not in self._connection._shell.join_path("a", ""): src = self._connection._shell._unquote(src) source_local = src.replace("\\", "/") @@ -290,15 +316,11 @@ def run(self, tmp=None, task_vars=None): try: if ds_type in SUPPORTED_DS_TYPES: if ds_type == "PO" and os.path.isfile(dest) and not fetch_member: - result[ - "msg" - ] = "Destination must be a directory to fetch a partitioned data set" + result["msg"] = "Destination must be a directory to fetch a partitioned data set" result["failed"] = True return result if ds_type == "GDG" and os.path.isfile(dest): - result[ - "msg" - ] = "Destination must be a directory to fetch a generation data group" + result["msg"] = "Destination must be a directory to fetch a generation data group" result["failed"] = True return result @@ -309,7 +331,8 @@ def run(self, tmp=None, task_vars=None): ignore_stderr=ignore_sftp_stderr, ) if fetch_content.get("msg"): - return fetch_content + result.update(fetch_content) + return result if validate_checksum and ds_type != "GDG" and ds_type != "PO" and not is_binary: new_checksum = _get_file_checksum(dest) diff --git a/plugins/modules/zos_fetch.py b/plugins/modules/zos_fetch.py index 62004698a..cb270ab31 100644 --- a/plugins/modules/zos_fetch.py +++ b/plugins/modules/zos_fetch.py @@ -240,8 +240,10 @@ """ RETURN = r""" -file: - description: The source file path or data set on the remote machine. +src: + description: + - The source file path or data set on the remote machine. + - If the source is not found, then src will be empty. returned: success type: str sample: SOME.DATA.SET @@ -266,14 +268,9 @@ returned: success type: str sample: PDSE -note: - description: Notice of module failure when C(fail_on_missing) is false. - returned: failure and fail_on_missing=false - type: str - sample: The data set USER.PROCLIB does not exist. No data was fetched. msg: - description: Message returned on failure. - returned: failure + description: Any important messages from the module. + returned: always type: str sample: The source 'TEST.DATA.SET' does not exist or is uncataloged. stdout: @@ -921,8 +918,23 @@ def run_module(): # ********************************************************** # # Check for data set existence and determine its type # # ********************************************************** # - - res_args = dict() + encoding_dict = {"from": encoding.get("from"), "to": encoding.get("to")} + result = dict( + src=src, + dest="", + is_binary=is_binary, + checksum="", + changed=False, + data_set_type="", + remote_path="", + msg="", + stdout="", + stderr="", + stdout_lines=[], + stderr_lines=[], + rc=0, + encoding=encoding_dict, + ) src_data_set = None ds_type = None @@ -963,7 +975,7 @@ def run_module(): ) else: module.exit_json( - note=("Source '{0}' was not found. No data was fetched.".format(src)) + msg=("Source '{0}' was not found. No data was fetched.".format(src)) ) if "/" in src: @@ -992,7 +1004,7 @@ def run_module(): is_binary, encoding=encoding ) - res_args["remote_path"] = file_path + result["remote_path"] = file_path # ********************************************************** # # Fetch a partitioned data set or one of its members # @@ -1005,9 +1017,9 @@ def run_module(): is_binary, encoding=encoding ) - res_args["remote_path"] = file_path + result["remote_path"] = file_path else: - res_args["remote_path"] = fetch_handler._fetch_pdse( + result["remote_path"] = fetch_handler._fetch_pdse( src_data_set.name, is_binary, encoding=encoding @@ -1027,7 +1039,7 @@ def run_module(): is_binary, encoding=encoding ) - res_args["remote_path"] = file_path + result["remote_path"] = file_path # ********************************************************** # # Fetch a VSAM data set # @@ -1039,32 +1051,32 @@ def run_module(): is_binary, encoding=encoding ) - res_args["remote_path"] = file_path + result["remote_path"] = file_path # ********************************************************** # # Fetch a GDG # # ********************************************************** # elif ds_type == "GDG": - res_args["remote_path"] = fetch_handler._fetch_gdg( + result["remote_path"] = fetch_handler._fetch_gdg( src_data_set.name, is_binary, encoding=encoding ) if ds_type == "USS": - res_args["file"] = src + result["src"] = src else: - res_args["file"] = src_data_set.name + result["src"] = src_data_set.name # Removing the HLQ since the user is probably not expecting it. The module # hasn't returned it ever since it was originally written. Changes made to # add GDG/GDS support started leaving the HLQ behind in the file name. if hlq: - res_args["file"] = res_args["file"].replace(f"{hlq}.", "") + result["src"] = result["src"].replace(f"{hlq}.", "") - res_args["ds_type"] = ds_type - module.exit_json(**res_args) + result["ds_type"] = ds_type + module.exit_json(**result) class ZOSFetchError(Exception): @@ -1094,7 +1106,7 @@ def __init__(self, msg, rc="", stdout="", stderr="", stdout_lines="", stderr_lin stdout_lines=stdout_lines, stderr_lines=stderr_lines, ) - super().__init__(self.msg) + super().__init__(msg) def main(): diff --git a/tests/functional/modules/test_zos_fetch_func.py b/tests/functional/modules/test_zos_fetch_func.py index 222bc5888..e6673c136 100644 --- a/tests/functional/modules/test_zos_fetch_func.py +++ b/tests/functional/modules/test_zos_fetch_func.py @@ -182,6 +182,13 @@ def test_fetch_uss_file_not_present_on_local_machine(ansible_zos_module): assert result.get("data_set_type") == "USS" assert result.get("module_stderr") is None assert os.path.exists(dest_path) + assert "msg" in result.keys() + assert "stdout" in result.keys() + assert "stdout_lines" in result.keys() + assert "stderr" in result.keys() + assert "stderr_lines" in result.keys() + assert "rc" is not None + assert isinstance(result.get("encoding"), dict) finally: if os.path.exists(dest_path): os.remove(dest_path) @@ -191,8 +198,9 @@ def test_fetch_uss_file_replace_on_local_machine(ansible_zos_module): with open("/tmp/profile", "w",encoding="utf-8") as file: file.close() hosts = ansible_zos_module + src = "/etc/profile" params = { - "src":"/etc/profile", + "src": src, "dest":"/tmp/", "flat":True } @@ -206,14 +214,17 @@ def test_fetch_uss_file_replace_on_local_machine(ansible_zos_module): assert result.get("checksum") != local_checksum assert result.get("module_stderr") is None assert os.path.exists(dest_path) + assert "msg" in result.keys() + assert result.get("src") is not None finally: os.remove(dest_path) def test_fetch_uss_file_present_on_local_machine(ansible_zos_module): hosts = ansible_zos_module + src = "/etc/profile" params = { - "src":"/etc/profile", + "src": src, "dest": "/tmp/", "flat":True } @@ -227,6 +238,8 @@ def test_fetch_uss_file_present_on_local_machine(ansible_zos_module): assert result.get("changed") is False assert result.get("checksum") == local_checksum assert result.get("module_stderr") is None + assert "msg" in result.keys() + assert result.get("src") is not None finally: os.remove(dest_path) @@ -256,6 +269,8 @@ def test_fetch_sequential_data_set_fixed_block(ansible_zos_module): assert result.get("module_stderr") is None assert result.get("dest") == dest_path assert os.path.exists(dest_path) + assert "msg" in result.keys() + assert result.get("src") is not None finally: hosts.all.zos_data_set(name=TEST_PS, state="absent") if os.path.exists(dest_path): @@ -280,6 +295,8 @@ def test_fetch_sequential_data_set_variable_block(ansible_zos_module): assert result.get("module_stderr") is None assert result.get("dest") == dest_path assert os.path.exists(dest_path) + assert "msg" in result.keys() + assert result.get("src") is not None finally: if os.path.exists(dest_path): os.remove(dest_path) @@ -308,6 +325,8 @@ def test_fetch_partitioned_data_set(ansible_zos_module): assert result.get("dest") == dest_path assert os.path.exists(dest_path) assert os.path.isdir(dest_path) + assert "msg" in result.keys() + assert result.get("src") is not None finally: hosts.all.zos_data_set(name=TEST_PDS, state="absent") if os.path.exists(dest_path): @@ -329,7 +348,7 @@ def test_fetch_vsam_data_set(ansible_zos_module, volumes_on_systems): cmd=f"echo {quote(KSDS_CREATE_JCL.format(volume_1, test_vsam))} > {temp_jcl_path}/SAMPLE" ) hosts.all.zos_job_submit( - src=f"{temp_jcl_path}/SAMPLE", location="uss", wait_time_s=30 + src=f"{temp_jcl_path}/SAMPLE", remote_src=True, wait_time=30 ) hosts.all.shell(cmd=f"echo \"{TEST_DATA}\\c\" > {uss_file}") hosts.all.zos_encode( @@ -357,6 +376,8 @@ def test_fetch_vsam_data_set(ansible_zos_module, volumes_on_systems): file = open(dest_path, 'r',encoding="utf-8") read_file = file.read() assert read_file == TEST_DATA + assert "msg" in result.keys() + assert result.get("src") is not None finally: if os.path.exists(dest_path): @@ -384,6 +405,8 @@ def test_fetch_vsam_empty_data_set(ansible_zos_module): assert result.get("module_stderr") is None assert result.get("dest") == dest_path assert os.path.exists(dest_path) + assert "msg" in result.keys() + assert result.get("src") is not None finally: hosts.all.zos_data_set(name=src_ds, state="absent") if os.path.exists(dest_path): @@ -414,6 +437,8 @@ def test_fetch_partitioned_data_set_member_in_binary_mode(ansible_zos_module): assert result.get("is_binary") is True assert os.path.exists(dest_path) assert os.path.isfile(dest_path) + assert "msg" in result.keys() + assert result.get("src") is not None finally: hosts.all.zos_data_set(name=TEST_PDS, state="absent") if os.path.exists(dest_path): @@ -446,6 +471,8 @@ def test_fetch_sequential_data_set_in_binary_mode(ansible_zos_module): assert result.get("module_stderr") is None assert result.get("is_binary") is True assert os.path.exists(dest_path) + assert "msg" in result.keys() + assert result.get("src") is not None finally: hosts.all.zos_data_set(name=TEST_PS, state="absent") if os.path.exists(dest_path): @@ -475,6 +502,8 @@ def test_fetch_partitioned_data_set_binary_mode(ansible_zos_module): assert result.get("is_binary") is True assert os.path.exists(dest_path) assert os.path.isdir(dest_path) + assert "msg" in result.keys() + assert result.get("src") is not None finally: hosts.all.zos_data_set(name=TEST_PDS, state="absent") if os.path.exists(dest_path): @@ -500,6 +529,8 @@ def test_fetch_sequential_data_set_empty(ansible_zos_module): assert result.get("dest") == dest_path assert os.path.exists(dest_path) assert os.stat(dest_path).st_size == 0 + assert "msg" in result.keys() + assert result.get("src") is not None finally: if os.path.exists(dest_path): os.remove(dest_path) @@ -558,6 +589,8 @@ def test_fetch_partitioned_data_set_member_empty(ansible_zos_module): assert result.get("module_stderr") is None assert os.path.exists(dest_path) assert os.stat(dest_path).st_size == 0 + assert "msg" in result.keys() + assert result.get("src") is not None finally: if os.path.exists(dest_path): os.remove(dest_path) @@ -576,7 +609,7 @@ def test_fetch_missing_uss_file_does_not_fail(ansible_zos_module): results = hosts.all.zos_fetch(**params) for result in results.contacted.values(): assert result.get("changed") is False - assert "note" in result.keys() + assert "msg" in result.keys() assert result.get("module_stderr") is None except Exception: raise @@ -610,7 +643,7 @@ def test_fetch_missing_mvs_data_set_does_not_fail(ansible_zos_module): results = hosts.all.zos_fetch(**params) for result in results.contacted.values(): assert result.get("changed") is False - assert "note" in result.keys() + assert "msg" in result.keys() assert result.get("module_stderr") is None assert not os.path.exists("/tmp/FETCH.TEST.DATA.SET") except Exception: @@ -679,6 +712,7 @@ def test_fetch_sequential_data_set_replace_on_local_machine(ansible_zos_module): assert result.get("changed") is True assert result.get("module_stderr") is None assert checksum(dest_path, hash_func=sha256) != local_checksum + assert result.get("src") is not None finally: hosts.all.zos_data_set(name=TEST_PS, state="absent") if os.path.exists(dest_path): @@ -718,6 +752,7 @@ def test_fetch_partitioned_data_set_replace_on_local_machine(ansible_zos_module) assert result.get("changed") is True assert result.get("module_stderr") is None assert os.path.getmtime(dest_path) != prev_timestamp + assert result.get("src") is not None finally: if os.path.exists(dest_path): shutil.rmtree(dest_path) @@ -736,6 +771,7 @@ def test_fetch_uss_file_insufficient_write_permission_fails(ansible_zos_module): results = hosts.all.zos_fetch(**params) for result in results.contacted.values(): assert "msg" in result.keys() + assert result.get("src") is not None dest_path.close() @@ -776,11 +812,12 @@ def test_fetch_use_data_set_qualifier(ansible_zos_module): try: results = hosts.all.zos_fetch(**params) for result in results.contacted.values(): - print(result) assert result.get("changed") is True assert result.get("data_set_type") == "Sequential" assert result.get("module_stderr") is None assert os.path.exists(dest_path) + assert "msg" in result.keys() + assert result.get("src") is not None finally: if os.path.exists(dest_path): os.remove(dest_path) @@ -807,6 +844,8 @@ def test_fetch_flat_create_dirs(ansible_zos_module, z_python_interpreter): for result in results.contacted.values(): assert result.get("changed") is True assert result.get("module_stderr") is None + assert "msg" in result.keys() + assert result.get("src") is not None assert os.path.exists(dest_path) finally: if os.path.exists(dest_path): @@ -836,6 +875,8 @@ def test_fetch_sequential_data_set_with_special_chars(ansible_zos_module): assert result.get("module_stderr") is None assert result.get("dest") == dest_path assert os.path.exists(dest_path) + assert "msg" in result.keys() + assert result.get("src") is not None finally: hosts.all.zos_data_set(name=TEST_PS, state="absent") if os.path.exists(dest_path): @@ -862,6 +903,8 @@ def test_fetch_gds_from_gdg(ansible_zos_module, generation): assert result.get("changed") is True assert result.get("data_set_type") == "Sequential" assert result.get("module_stderr") is None + assert "msg" in result.keys() + assert result.get("src") is not None # Checking that we got a dest of the form: ANSIBLE.DATA.SET.G0001V01. dest_path = result.get("dest", "") @@ -894,6 +937,7 @@ def test_error_fetch_inexistent_gds(ansible_zos_module): for result in results.contacted.values(): assert result.get("changed") is False assert result.get("failed") is True + assert "msg" in result.keys() assert "does not exist" in result.get("msg", "") finally: @@ -920,6 +964,7 @@ def test_fetch_gdg(ansible_zos_module): assert result.get("changed") is True assert result.get("data_set_type") == "Generation Data Group" assert result.get("module_stderr") is None + assert "msg" in result.keys() # Checking that we got a dest of the form: ANSIBLE.DATA.SET.G0001V01. dest_path = result.get("dest", "") @@ -981,6 +1026,7 @@ def test_fetch_uss_file_relative_path_not_present_on_local_machine(ansible_zos_m assert result.get("module_stderr") is None assert dest == result.get("dest") dest = result.get("dest") + assert "msg" in result.keys() finally: if os.path.exists(dest):