[Enabler] [Various modules] Remove use of deprecated pipes library (#1565)

* Replace use of pipes for shlex

* Remove import of pipes due to use of Python 2

* Add changelog fragment

* Remove import of PY3

* Add special symbols workaround

* Fix syntax error

* Fix another syntax error

* Fix pep8 issues

* Add ZOAUResponse to keep compatibility

* Fix ZOAU exception

* Update changed attribute in response

* Add cmd to result

* Add single quotes to data set names

* Add workaround for batches

* Fix pylint issue

* Fix cat issue

Author: rexemin

changelogs/fragments/1565-remove-deprecated-pipes-library.yml

@@ -0,0 +1,11 @@
+trivial:
+  - zos_find - remove deprecated library pipes in favor of shlex.
+    (https://github.com/ansible-collections/ibm_zos_core/pull/1565).
+  - zos_mvs_raw - remove deprecated library pipes in favor of shlex.
+    (https://github.com/ansible-collections/ibm_zos_core/pull/1565).
+  - module_utils/backup.py - remove deprecated library pipes in favor of shlex.
+    (https://github.com/ansible-collections/ibm_zos_core/pull/1565).
+  - module_utils/copy.py - remove deprecated library pipes in favor of shlex.
+    (https://github.com/ansible-collections/ibm_zos_core/pull/1565).
+  - module_utils/encode.py - remove deprecated library pipes in favor of shlex.
+    (https://github.com/ansible-collections/ibm_zos_core/pull/1565).

plugins/module_utils/backup.py

@@ -15,7 +15,6 @@
 __metaclass__ = type
 
 import os
-from ansible.module_utils.six import PY3
 from ansible_collections.ibm.ibm_zos_core.plugins.module_utils.ansible_module import (
     AnsibleModuleHelper,
 )
@@ -44,10 +43,8 @@
 except Exception:
     datasets = ZOAUImportError(traceback.format_exc())
     exceptions = ZOAUImportError(traceback.format_exc())
-if PY3:
-    from shlex import quote
-else:
-    from pipes import quote
+
+from shlex import quote
 
 
 def _validate_data_set_name(ds):

plugins/module_utils/copy.py

@@ -17,7 +17,6 @@
 
 import traceback
 from os import path
-from ansible.module_utils.six import PY3
 from ansible_collections.ibm.ibm_zos_core.plugins.module_utils.ansible_module import (
     AnsibleModuleHelper,
 )
@@ -30,10 +29,7 @@
 from ansible_collections.ibm.ibm_zos_core.plugins.module_utils.import_handler import \
     ZOAUImportError
 
-if PY3:
-    from shlex import quote
-else:
-    from pipes import quote
+from shlex import quote
 
 try:
     from zoautil_py import datasets, gdgs

plugins/module_utils/encode.py

@@ -17,7 +17,6 @@
 from tempfile import NamedTemporaryFile, mkstemp, mkdtemp
 from math import floor, ceil
 from os import path, walk, makedirs, unlink
-from ansible.module_utils.six import PY3
 
 import shutil
 import errno
@@ -42,11 +41,7 @@
 except Exception:
     datasets = ZOAUImportError(traceback.format_exc())
 
-
-if PY3:
-    from shlex import quote
-else:
-    from pipes import quote
+from shlex import quote
 
 
 class Defaults:

plugins/modules/zos_apf.py

@@ -305,8 +305,10 @@
 
 try:
     from zoautil_py import zsystem
+    from zoautil_py import ztypes
 except Exception:
     zsystem = ZOAUImportError(traceback.format_exc())
+    ztypes = ZOAUImportError(traceback.format_exc())
 
 
 # supported data set types
@@ -361,6 +363,107 @@ def backupOper(module, src, backup, tmphlq=None):
     return backup_name
 
 
+def make_apf_command(library, opt, volume=None, sms=None, force_dynamic=None, persistent=None):
+    """Returns a string that can run an APF command in a shell.
+
+    Parameters
+    ----------
+    library : str
+        Name of the data set that will be operated on.
+    opt : str
+        APF operation (either add or del)
+    volume : str
+        Volume of library.
+    sms : bool
+        Whether library is managed by SMS.
+    force_dynamic : bool
+        Whether the APF list format should be dynamic.
+    persistent : dict
+        Options for persistent entries that should be modified by APF.
+
+    Returns
+    -------
+    str
+        APF command.
+    """
+    operation = "-A" if opt == "add" else "-D"
+    operation_args = library
+
+    if volume:
+        operation_args = f"{operation_args},{volume}"
+    elif sms:
+        operation_args = f"{operation_args},SMS"
+
+    command = f"apfadm {operation} '{operation_args}'"
+
+    if force_dynamic:
+        command = f"{command} -f"
+
+    if persistent:
+        if opt == "add":
+            persistent_args = f""" -P '{persistent.get("addDataset")}' """
+        else:
+            persistent_args = f""" -R '{persistent.get("delDataset")}' """
+
+        if persistent.get("marker"):
+            persistent_args = f""" {persistent_args} -M '{persistent.get("marker")}' """
+
+        command = f"{command} {persistent_args}"
+
+    return command
+
+
+def make_apf_batch_command(batch, force_dynamic=None, persistent=None):
+    """Returns a string that can run an APF command for multiple operations
+    in a shell.
+
+    Parameters
+    ----------
+    batch : list
+        List of dicts containing different APF add/del operations.
+    force_dynamic : bool
+        Whether the APF list format should be dynamic.
+    persistent : dict
+        Options for persistent entries that should be modified by APF.
+
+    Returns
+    -------
+    str
+        APF command.
+    """
+    command = "apfadm"
+
+    for item in batch:
+        operation = "-A" if item["opt"] == "add" else "-D"
+        operation_args = item["dsname"]
+
+        volume = item.get("volume")
+        sms = item.get("sms")
+
+        if volume:
+            operation_args = f"{operation_args},{volume}"
+        elif sms:
+            operation_args = f"{operation_args},SMS"
+
+        command = f"{command} {operation} '{operation_args}'"
+
+    if force_dynamic:
+        command = f"{command} -f"
+
+    if persistent:
+        if persistent.get("addDataset"):
+            persistent_args = f""" -P '{persistent.get("addDataset")}' """
+        else:
+            persistent_args = f""" -R '{persistent.get("delDataset")}' """
+
+        if persistent.get("marker"):
+            persistent_args = f""" {persistent_args} -M '{persistent.get("marker")}' """
+
+        command = f"{command} {persistent_args}"
+
+    return command
+
+
 def main():
     """Initialize the module.
 
@@ -551,18 +654,32 @@ def main():
                 item['opt'] = opt
                 item['dsname'] = item.get('library')
                 del item['library']
-            ret = zsystem.apf(batch=batch, forceDynamic=force_dynamic, persistent=persistent)
+            # Commenting this line to implement a workaround for names with '$'. ZOAU should
+            # release a fix soon so we can uncomment this Python API call.
+            # ret = zsystem.apf(batch=batch, forceDynamic=force_dynamic, persistent=persistent)
+            apf_command = make_apf_batch_command(batch, force_dynamic=force_dynamic, persistent=persistent)
+            rc, out, err = module.run_command(apf_command)
+            ret = ztypes.ZOAUResponse(rc, out, err, apf_command, 'utf-8')
         else:
             if not library:
                 module.fail_json(msg='library is required')
-            ret = zsystem.apf(opt=opt, dsname=library, volume=volume, sms=sms, forceDynamic=force_dynamic, persistent=persistent)
+            # Commenting this line to implement a workaround for names with '$'. ZOAU should
+            # release a fix soon so we can uncomment this Python API call.
+            # ret = zsystem.apf(opt=opt, dsname=library, volume=volume, sms=sms, forceDynamic=force_dynamic, persistent=persistent)
+            apf_command = make_apf_command(library, opt, volume=volume, sms=sms, force_dynamic=force_dynamic, persistent=persistent)
+            rc, out, err = module.run_command(apf_command)
+            ret = ztypes.ZOAUResponse(rc, out, err, apf_command, 'utf-8')
 
     operOut = ret.stdout_response
     operErr = ret.stderr_response
     operRc = ret.rc
     result['stderr'] = operErr
     result['rc'] = operRc
     result['stdout'] = operOut
+
+    if operation != 'list' and operRc == 0:
+        result['changed'] = True
+
     if operation == 'list':
         try:
             data = json.loads(operOut)

plugins/modules/zos_find.py

@@ -304,7 +304,6 @@
 from re import match as fullmatch
 
 
-from ansible.module_utils.six import PY3
 from ansible.module_utils.basic import AnsibleModule
 
 from ansible_collections.ibm.ibm_zos_core.plugins.module_utils import (
@@ -319,10 +318,7 @@
     AnsibleModuleHelper
 )
 
-if PY3:
-    from shlex import quote
-else:
-    from pipes import quote
+from shlex import quote
 
 
 def content_filter(module, patterns, content):

plugins/modules/zos_mvs_raw.py

@@ -1645,12 +1645,8 @@
 )
 import re
 import traceback
-from ansible.module_utils.six import PY3
 
-if PY3:
-    from shlex import quote
-else:
-    from pipes import quote
+from shlex import quote
 
 try:
     from zoautil_py import datasets

tests/functional/modules/test_module_security.py

@@ -17,7 +17,7 @@
 
 import pytest
 from pprint import pprint
-from pipes import quote
+from shlex import quote
 import unittest
 
 # TODO: remove some of the logic from tests and make pytest fixtures