[Enabler] [conftest.py] Hot fix for testing the collection in Ansible 2.17.1 (#1772)

* Hijacking ansible-pytest to pass env vars

* Decorate environment computation function

* Update python used in test

* Add playbook env vars debugging

* Fix typo

* Add Python version to env vars

* Fix Python version in test

* Add changelog fragment

* Update index used by test

* Fix zos_copy test

* Escape dataset name in command calling drm

* Use shlex.quote only on dataset name

* Escape $ on dataset name

* Fix dataset name escape

* Fix escaping issues in tests

---------

Co-authored-by: Fernando Flores <[email protected]>

Author: rexemin

changelogs/fragments/1772-fix-testing-ansible-2.17.1.yml

@@ -0,0 +1,5 @@
+trivial:
+  - tests/conftest.py - The testing suite could not load environment
+    variables when executing modules. Fix now correctly passes all
+    environment variables to the tests.
+    (https://github.com/ansible-collections/ibm_zos_core/pull/1772).

tests/conftest.py

@@ -15,11 +15,25 @@
 import pytest
 from ibm_zos_core.tests.helpers.ztest import ZTestHelper
 from ibm_zos_core.tests.helpers.volumes import get_volumes, get_volumes_with_vvds
+from ansible.plugins.action import ActionBase
 import sys
 from mock import MagicMock
 import importlib
 
 
+def add_vars_to_compute_environment(env_vars):
+    """This decorator injects the environment variables defined in a config
+    file to the Ansible method responsible for constructing the environment
+    string used by the SSH connection plugin."""
+    def wrapper_compute_env(compute_environment_string):
+        def wrapped_compute_environment_string(self, *args, **kwargs):
+            self._task.environment = env_vars
+            env_string = compute_environment_string(self, *args, **kwargs)
+            return env_string
+        return wrapped_compute_environment_string
+    return wrapper_compute_env
+
+
 def pytest_addoption(parser):
     """
     Add CLI options and modify options for pytest-ansible where needed.
@@ -57,7 +71,7 @@ def z_python_interpreter(request):
     interpreter_str = helper.build_interpreter_string()
     inventory = helper.get_inventory_info()
     python_path = helper.get_python_path()
-    yield (interpreter_str, inventory, python_path)
+    yield (helper._environment, interpreter_str, inventory, python_path)
 
 
 def clean_logs(adhoc):
@@ -81,7 +95,7 @@ def clean_logs(adhoc):
 def ansible_zos_module(request, z_python_interpreter):
     """ Initialize pytest-ansible plugin with values from
     our YAML config and inject interpreter path into inventory. """
-    interpreter, inventory, python_path = z_python_interpreter
+    env_vars, interpreter, inventory, python_path = z_python_interpreter
 
     # next two lines perform similar action to ansible_adhoc fixture
     plugin = request.config.pluginmanager.getplugin("ansible")
@@ -93,9 +107,11 @@ def ansible_zos_module(request, z_python_interpreter):
     # Courtesy, pass along the python_path for some test cases need this information
     adhoc["options"]["ansible_python_path"] = python_path
 
+    # Adding the environment variables decorator to the Ansible engine.
+    ActionBase._compute_environment_string = add_vars_to_compute_environment(env_vars)(ActionBase._compute_environment_string)
+
     for host in hosts.values():
-        host.vars["ansible_python_interpreter"] = interpreter
-        # host.vars["ansible_connection"] = "zos_ssh"
+        host.vars["ansible_python_interpreter"] = python_path
     yield adhoc
     try:
         clean_logs(adhoc)

tests/functional/modules/test_zos_archive_func.py

@@ -400,7 +400,7 @@ def test_mvs_archive_single_dataset(ansible_zos_module, format, data_set, record
             assert result.get("changed") is True
             assert result.get("dest") == archive_data_set
             assert src_data_set in result.get("archived")
-            cmd_result = hosts.all.shell(cmd = "dls {0}.*".format(HLQ))
+            cmd_result = hosts.all.shell(cmd = """dls "{0}.*" """.format(HLQ))
             for c_result in cmd_result.contacted.values():
                 assert archive_data_set in c_result.get("stdout")
     finally:
@@ -480,7 +480,7 @@ def test_mvs_archive_single_dataset_use_adrdssu(ansible_zos_module, format, data
             assert result.get("changed") is True
             assert result.get("dest") == archive_data_set
             assert src_data_set in result.get("archived")
-            cmd_result = hosts.all.shell(cmd = "dls {0}.*".format(HLQ))
+            cmd_result = hosts.all.shell(cmd = """dls "{0}.*" """.format(HLQ))
             for c_result in cmd_result.contacted.values():
                 assert archive_data_set in c_result.get("stdout")
     finally:
@@ -550,7 +550,7 @@ def test_mvs_archive_single_data_set_remove_target(ansible_zos_module, format, d
             assert result.get("changed") is True
             assert result.get("dest") == archive_data_set
             assert src_data_set in result.get("archived")
-            cmd_result = hosts.all.shell(cmd = "dls {0}.*".format(HLQ))
+            cmd_result = hosts.all.shell(cmd = """dls "{0}.*" """.format(HLQ))
             for c_result in cmd_result.contacted.values():
                 assert archive_data_set in c_result.get("stdout")
                 assert src_data_set != c_result.get("stdout")
@@ -614,7 +614,7 @@ def test_mvs_archive_multiple_data_sets(ansible_zos_module, format, data_set):
             assert result.get("dest") == archive_data_set
             for ds in target_ds_list:
                 assert ds.get("name") in result.get("archived")
-            cmd_result = hosts.all.shell(cmd = "dls {0}.*".format(HLQ))
+            cmd_result = hosts.all.shell(cmd = """dls "{0}.*" """.format(HLQ))
             for c_result in cmd_result.contacted.values():
                 assert archive_data_set in c_result.get("stdout")
     finally:
@@ -682,7 +682,7 @@ def test_mvs_archive_multiple_data_sets_with_exclusion(ansible_zos_module, forma
                     assert exclude not in result.get("archived")
                 else:
                     assert ds.get("name") in result.get("archived")
-            cmd_result = hosts.all.shell(cmd = "dls {0}.*".format(HLQ))
+            cmd_result = hosts.all.shell(cmd = """dls "{0}.*" """.format(HLQ))
             for c_result in cmd_result.contacted.values():
                 assert archive_data_set in c_result.get("stdout")
     finally:
@@ -744,7 +744,7 @@ def test_mvs_archive_multiple_data_sets_and_remove(ansible_zos_module, format, d
         for result in archive_result.contacted.values():
             assert result.get("changed") is True
             assert result.get("dest") == archive_data_set
-            cmd_result = hosts.all.shell(cmd = "dls {0}.*".format(HLQ))
+            cmd_result = hosts.all.shell(cmd = """dls "{0}.*" """.format(HLQ))
             for c_result in cmd_result.contacted.values():
                 assert archive_data_set in c_result.get("stdout")
                 for ds in target_ds_list:
@@ -820,7 +820,7 @@ def test_mvs_archive_multiple_data_sets_with_missing(ansible_zos_module, format,
                     assert ds.get("name") not in result.get("archived")
                 else:
                     assert ds.get("name") in result.get("archived")
-            cmd_result = hosts.all.shell(cmd = "dls {0}.*".format(HLQ))
+            cmd_result = hosts.all.shell(cmd = """dls "{0}.*" """.format(HLQ))
             for c_result in cmd_result.contacted.values():
                 assert archive_data_set in c_result.get("stdout")
 
@@ -902,7 +902,7 @@ def test_mvs_archive_single_dataset_force_lock(ansible_zos_module, format, data_
             assert result.get("changed") is True
             assert result.get("dest") == archive_data_set
             assert src_data_set in result.get("archived")
-            cmd_result = hosts.all.shell(cmd = "dls {0}.*".format(HLQ))
+            cmd_result = hosts.all.shell(cmd = """dls "{0}.*" """.format(HLQ))
             for c_result in cmd_result.contacted.values():
                 assert archive_data_set in c_result.get("stdout")
 
@@ -958,7 +958,7 @@ def test_gdg_archive(ansible_zos_module, dstype, format):
             assert result.get("dest") == archive_data_set
             assert f"{data_set_name}.G0001V00" in result.get("archived")
             assert f"{data_set_name}.G0002V00" in result.get("archived")
-            cmd_result = hosts.all.shell(cmd = "dls {0}.*".format(HLQ))
+            cmd_result = hosts.all.shell(cmd = """dls "{0}.*" """.format(HLQ))
             for c_result in cmd_result.contacted.values():
                 assert archive_data_set in c_result.get("stdout")
     finally:
@@ -999,7 +999,7 @@ def test_archive_into_gds(ansible_zos_module, dstype, format):
         for result in archive_result.contacted.values():
             assert result.get("changed") is True
             assert data_set_name in result.get("archived")
-            cmd_result = hosts.all.shell(cmd = "dls {0}.*".format(HLQ))
+            cmd_result = hosts.all.shell(cmd = """dls "{0}.*" """.format(HLQ))
             for c_result in cmd_result.contacted.values():
                 assert archive_data_set in c_result.get("stdout")
     finally:

tests/functional/modules/test_zos_backup_restore.py

@@ -897,7 +897,8 @@ def test_backup_into_gds(ansible_zos_module, dstype):
         for result in results.contacted.values():
             assert result.get("changed") is True
             assert result.get("module_stderr") is None
-        results = hosts.all.shell(cmd=f"drm \"{ds_name}\"")
+        escaped_ds_name = ds_name.replace('$', '\$')
+        results = hosts.all.shell(cmd=f"drm \"{escaped_ds_name}\"")
         for result in results.contacted.values():
             assert result.get("changed") is True
             assert result.get("module_stderr") is None

tests/functional/modules/test_zos_copy_func.py

@@ -2046,7 +2046,7 @@ def test_copy_dest_lock(ansible_zos_module, ds_type, f_lock ):
         hosts.all.zos_data_set(name=data_set_2, state="absent")
 
 
-def test_copy_dest_lock_test_with_no_opercmd_access_pds_without_force_lock(ansible_zos_module):
+def test_copy_dest_lock_test_with_no_opercmd_access_pds_without_force_lock(ansible_zos_module, z_python_interpreter):
     """
     This tests the module exeception raised 'msg="Unable to determine if the source {0} is in use.".format(dataset_name)'. 
     This this a wrapper for the actual test case `managed_user_copy_dest_lock_test_with_no_opercmd_access`.
@@ -2055,7 +2055,7 @@ def test_copy_dest_lock_test_with_no_opercmd_access_pds_without_force_lock(ansib
     managed_user_test_case_name = "managed_user_copy_dest_lock_test_with_no_opercmd_access"
     try:
         # Initialize the Managed user API from the pytest fixture.
-        managed_user = ManagedUser.from_fixture(ansible_zos_module)
+        managed_user = ManagedUser.from_fixture(ansible_zos_module, z_python_interpreter)
 
         # Important: Execute the test case with the managed users execution utility.
         managed_user.execute_managed_user_test(

tests/functional/modules/test_zos_fetch_func.py

@@ -788,7 +788,7 @@ def test_fetch_use_data_set_qualifier(ansible_zos_module):
 def test_fetch_flat_create_dirs(ansible_zos_module, z_python_interpreter):
     z_int = z_python_interpreter
     hosts = ansible_zos_module
-    remote_host = z_int[1].get("inventory").strip(",")
+    remote_host = z_int[2].get("inventory").strip(",")
     dest_path = f"/tmp/{remote_host}/etc/ssh/ssh_config"
     params = {
         "src":"/etc/ssh/ssh_config",

tests/functional/modules/test_zos_operator_func.py

@@ -36,7 +36,7 @@
   environment:
     _BPXK_AUTOCVT: "ON"
     ZOAU_HOME: "{0}"
-    PYTHONPATH: "{0}/lib"
+    PYTHONPATH: "{0}/lib/{2}"
     LIBPATH: "{0}/lib:{1}/lib:/lib:/usr/lib:."
     PATH: "{0}/bin:/bin:/usr/lpp/rsusr/ported/bin:/var/bin:/usr/lpp/rsusr/ported/bin:/usr/lpp/java/java180/J8.0_64/bin:{1}/bin:"
     _CEE_RUNOPTS: "FILETAG(AUTOCVT,AUTOTAG) POSIX(ON)"
@@ -62,7 +62,7 @@
       ansible_host: {0}
       ansible_ssh_private_key_file: {1}
       ansible_user: {2}
-      ansible_python_interpreter: /allpython/3.9/usr/lpp/IBM/cyp/v3r9/pyz/bin/python3.9"""
+      ansible_python_interpreter: /allpython/3.11/usr/lpp/IBM/cyp/v3r11/pyz/bin/python3"""
 
 
 def test_zos_operator_various_command(ansible_zos_module):
@@ -212,12 +212,15 @@ def test_zos_operator_parallel_terminal(get_config):
     python_path = enviroment["python_path"]
     cut_python_path = python_path[:python_path.find('/bin')].strip()
     zoau = enviroment["environment"]["ZOAU_ROOT"]
+    python_version = cut_python_path.split('/')[2]
+
     try:
         playbook = "playbook.yml"
         inventory = "inventory.yml"
         os.system("echo {0} > {1}".format(quote(PARALLEL_RUNNING.format(
             zoau,
             cut_python_path,
+            python_version
         )), playbook))
         os.system("echo {0} > {1}".format(quote(INVENTORY.format(
             hosts,

tests/functional/modules/test_zos_unarchive_func.py

@@ -1158,7 +1158,7 @@ def test_gdg_unarchive(ansible_zos_module, dstype, format):
             assert result.get("dest") == archive_data_set
             assert f"{data_set_name}.G0001V00" in result.get("archived")
             assert f"{data_set_name}.G0002V00" in result.get("archived")
-            cmd_result = hosts.all.shell(cmd = "dls {0}.*".format(HLQ))
+            cmd_result = hosts.all.shell(cmd = """dls "{0}.*" """.format(HLQ))
             for c_result in cmd_result.contacted.values():
                 assert archive_data_set in c_result.get("stdout")
 
@@ -1178,10 +1178,10 @@ def test_gdg_unarchive(ansible_zos_module, dstype, format):
             assert len(result.get("missing")) == 0
             assert f"{data_set_name}.G0001V00" in result.get("targets")
             assert f"{data_set_name}.G0002V00" in result.get("targets")
-            cmd_result = hosts.all.shell(cmd = "dls {0}.*".format(HLQ))
+            cmd_result = hosts.all.shell(cmd = """dls "{0}.*" """.format(HLQ))
             for c_result in cmd_result.contacted.values():
                 assert f"{data_set_name}.G0001V00" in c_result.get("stdout")
                 assert f"{data_set_name}.G0002V00" in c_result.get("stdout")
     finally:
-        hosts.all.shell(cmd=f"drm {HLQ}.*")
+        hosts.all.shell(cmd=f'drm "{HLQ}.*"')
 

tests/helpers/users.py

@@ -183,20 +183,22 @@ def __init__(self, model_user: str = None, remote_host: str = None, zoau_path: s
         self._create_ssh_config_and_directory()
 
     @classmethod
-    def from_fixture(cls, pytest_fixture):
-
-        remote_host = pytest_fixture["options"]["inventory"].replace(",", "")
-        model_user = pytest_fixture["options"]["user"]
-        inventory_hosts = pytest_fixture["options"]["inventory_manager"]._inventory.hosts
+    def from_fixture(cls, pytest_module_fixture, pytest_interpreter_fixture):
+        remote_host = pytest_module_fixture["options"]["inventory"].replace(",", "")
+        model_user = pytest_module_fixture["options"]["user"]
+        inventory_hosts = pytest_module_fixture["options"]["inventory_manager"]._inventory.hosts
         inventory_list = list(inventory_hosts.values())[0].vars.get('ansible_python_interpreter').split(";")
-        zoau_path = [v for v in inventory_list if f"ZOAU_HOME=" in v][0].split('=')[1].strip() or None
-        pythonpath = [v for v in inventory_list if f"PYTHONPATH=" in v][0].split('=')[1].strip() or None
+        environment_vars = pytest_interpreter_fixture[0]
+
+        zoau_path = environment_vars.get("ZOAU_HOME")
+        pythonpath = environment_vars.get("PYTHONPATH")
         pyz_path = [v for v in inventory_list if f"bin/python" in v][0].split('/bin')[0].strip() or None
+
         # TODO: To make this dynamic, we need to update AC and then also test with the new fixture because
         # the legacy fixture is using a VOLUMES keyword while raw fixture uses extra_args. Best to move
         # volumes to extra_args. 
         volumes = "000000,222222"
-        hostpattern = pytest_fixture["options"]["host_pattern"]
+        hostpattern = pytest_module_fixture["options"]["host_pattern"]
         return cls(model_user, remote_host, zoau_path, pyz_path, pythonpath, volumes, hostpattern)