Skip to content

Commit b79ce3d

Browse files
AndreMarcel99AndreMarcel99
committed
Merge branch 'dev' into enabler/1589/volume_init_portability
2 parents 4e444c6 + 1c64254 commit b79ce3d

File tree

6 files changed

+156
-41
lines changed

6 files changed

+156
-41
lines changed

‎changelogs/fragments/1816-migrate_shell_call_cp_to_python_module.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
trivial:
2+
- zos_fetch - Migrate calls from cp to python API copy.
3+
(https://github.com/ansible-collections/ibm_zos_core/pull/1816).

‎changelogs/fragments/1822-zos_find-vsam-disp-old.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
bugfixes:
2+
- zos_find - Module would not find a VSAM cluster resource type if it was in use with DISP=OLD. Fix now finds the VSAM cluster.
3+
(https://github.com/ansible-collections/ibm_zos_core/pull/1822).
4+
- zos_find - Module would not find VSAM data and index resource types. Fix now finds the data and index resource types.
5+
(https://github.com/ansible-collections/ibm_zos_core/pull/1822).

‎plugins/modules/zos_fetch.py

Lines changed: 72 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -320,6 +320,11 @@
320320
mvscmd = ZOAUImportError(traceback.format_exc())
321321
ztypes = ZOAUImportError(traceback.format_exc())
322322

323+
try:
324+
from zoautil_py import exceptions as zoau_exceptions
325+
except Exception:
326+
zoau_exceptions = ZOAUImportError(traceback.format_exc())
327+
323328

324329
class FetchHandler:
325330
def __init__(self, module):
@@ -621,29 +626,37 @@ def _fetch_pdse(self, src, is_binary, temp_dir=None, encoding=None):
621626
622627
Raises
623628
------
624-
fail_json
629+
ZOSFetchError
625630
Error copying partitioned dataset to USS.
626631
fail_json
627632
Error converting encoding of the member.
628633
"""
629634
dir_path = tempfile.mkdtemp(dir=temp_dir)
630-
cmd = "cp -B \"//'{0}'\" {1}"
631-
if not is_binary:
632-
cmd = cmd.replace(" -B", "")
633-
rc, out, err = self._run_command(cmd.format(src, dir_path))
634-
if rc != 0:
635+
636+
copy_args = {
637+
"options": ""
638+
}
639+
640+
if is_binary:
641+
copy_args["options"] = "-B"
642+
643+
try:
644+
datasets.copy(source=src, target=dir_path, **copy_args)
645+
646+
except zoau_exceptions.ZOAUException as copy_exception:
635647
rmtree(dir_path)
636-
self._fail_json(
648+
raise ZOSFetchError(
637649
msg=(
638650
"Error copying partitioned data set {0} to USS. Make sure it is"
639651
" not empty".format(src)
640652
),
641-
stdout=out,
642-
stderr=err,
643-
stdout_lines=out.splitlines(),
644-
stderr_lines=err.splitlines(),
645-
rc=rc,
653+
rc=copy_exception.response.rc,
654+
stdout=copy_exception.response.stdout_response,
655+
stderr=copy_exception.response.stderr_response,
656+
stdout_lines=copy_exception.response.stdout_response.splitlines(),
657+
stderr_lines=copy_exception.response.stderr_response.splitlines(),
646658
)
659+
647660
if (not is_binary) and encoding:
648661
enc_utils = encode.EncodeUtils()
649662
from_code_set = encoding.get("from")
@@ -740,7 +753,7 @@ def _fetch_mvs_data(self, src, is_binary, temp_dir=None, file_override=None, enc
740753
741754
Raises
742755
------
743-
fail_json
756+
ZOSFetchError
744757
Unable to copy to USS.
745758
fail_json
746759
Error converting encoding of the dataset.
@@ -754,22 +767,27 @@ def _fetch_mvs_data(self, src, is_binary, temp_dir=None, file_override=None, enc
754767
fd, file_path = tempfile.mkstemp(dir=temp_dir)
755768
os.close(fd)
756769

757-
cmd = "cp -B \"//'{0}'\" {1}"
758-
if not is_binary:
759-
cmd = cmd.replace(" -B", "")
770+
copy_args = {
771+
"options": ""
772+
}
760773

761-
rc, out, err = self._run_command(cmd.format(src, file_path))
774+
if is_binary:
775+
copy_args["options"] = "-B"
762776

763-
if rc != 0:
777+
try:
778+
datasets.copy(source=src, target=file_path, **copy_args)
779+
780+
except zoau_exceptions.ZOAUException as copy_exception:
764781
os.remove(file_path)
765-
self._fail_json(
782+
raise ZOSFetchError(
766783
msg="Unable to copy {0} to USS".format(src),
767-
stdout=str(out),
768-
stderr=str(err),
769-
rc=rc,
770-
stdout_lines=str(out).splitlines(),
771-
stderr_lines=str(err).splitlines(),
784+
rc=copy_exception.response.rc,
785+
stdout=copy_exception.response.stdout_response,
786+
stderr=copy_exception.response.stderr_response,
787+
stdout_lines=copy_exception.response.stdout_response.splitlines(),
788+
stderr_lines=copy_exception.response.stderr_response.splitlines(),
772789
)
790+
773791
if (not is_binary) and encoding:
774792
enc_utils = encode.EncodeUtils()
775793
from_code_set = encoding.get("from")
@@ -1036,6 +1054,36 @@ def run_module():
10361054
module.exit_json(**res_args)
10371055

10381056

1057+
class ZOSFetchError(Exception):
1058+
def __init__(self, msg, rc="", stdout="", stderr="", stdout_lines="", stderr_lines=""):
1059+
"""Error in a copy operation.
1060+
1061+
Parameters
1062+
----------
1063+
msg : str
1064+
Human readable string describing the exception.
1065+
rc : int
1066+
Result code.
1067+
stdout : str
1068+
Standart output.
1069+
stderr : str
1070+
Standart error.
1071+
stdout_lines : str
1072+
Standart output divided in lines.
1073+
stderr_lines : str
1074+
Standart error divided in lines.
1075+
"""
1076+
self.json_args = dict(
1077+
msg=msg,
1078+
rc=rc,
1079+
stdout=stdout,
1080+
stderr=stderr,
1081+
stdout_lines=stdout_lines,
1082+
stderr_lines=stderr_lines,
1083+
)
1084+
super().__init__(self.msg)
1085+
1086+
10391087
def main():
10401088
run_module()
10411089

‎plugins/modules/zos_find.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -503,7 +503,8 @@ def vsam_filter(module, patterns, resource_type, age=None):
503503
filtered_data_sets = set()
504504
now = time.time()
505505
for pattern in patterns:
506-
rc, out, err = _vls_wrapper(pattern, details=True)
506+
request_details = age is not None
507+
rc, out, err = _vls_wrapper(pattern, details=request_details)
507508
if rc > 4:
508509
module.fail_json(
509510
msg="Non-zero return code received while executing ZOAU shell command 'vls'",
@@ -1069,7 +1070,6 @@ def run_module(module):
10691070
size = int(m.group(1)) * bytes_per_unit.get(m.group(2), 1)
10701071
else:
10711072
module.fail_json(size=size, msg="failed to process size")
1072-
10731073
if resource_type == "NONVSAM":
10741074
if contains:
10751075
init_filtered_data_sets = content_filter(
@@ -1104,7 +1104,7 @@ def run_module(module):
11041104

11051105
res_args['examined'] = init_filtered_data_sets.get("searched")
11061106

1107-
elif resource_type == "CLUSTER":
1107+
elif resource_type in ["CLUSTER", "DATA", "INDEX"]:
11081108
filtered_data_sets = vsam_filter(module, patterns, resource_type, age=age)
11091109
res_args['examined'] = len(filtered_data_sets)
11101110
elif resource_type == "GDG":

‎scripts/mounts.env

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -28,19 +28,19 @@
2828
# data_set - the z/OS data set containing the binaries to mount
2929
# space - must be a space before the closing quote
3030
# ------------------------------------------------------------------------------
31-
zoau_mount_list_str="1:1.1.0-ga:/zoau/v1.1.0-ga:IMSTESTU.ZOAU.V110.GA.ZFS "\
32-
"2:1.1.1-ptf1:/zoau/v1.1.1-ptf1:IMSTESTU.ZOAU.V111.PTF1.ZFS "\
33-
"3:1.2.0:/zoau/v1.2.0:IMSTESTU.ZOAU.V120.ZFS "\
34-
"4:1.2.1:/zoau/v1.2.1:IMSTESTU.ZOAU.V121.ZFS "\
35-
"5:1.2.2:/zoau/v1.2.2:IMSTESTU.ZOAU.V122.ZFS "\
36-
"6:1.2.3:/zoau/v1.2.3:IMSTESTU.ZOAU.V123.ZFS "\
37-
"7:1.2.4:/zoau/v1.2.4:IMSTESTU.ZOAU.V124.ZFS "\
38-
"8:1.2.5.8:/zoau/v1.2.5.8:IMSTESTU.ZOAU.V102.GA.ZFS "\
39-
"9:1.2.5.10:/zoau/v1.2.5.10:IMSTESTU.ZOAU.V103.PTF2.ZFS "\
40-
"10:1.3.0:/zoau/v1.3.0:IMSTESTU.ZOAU.V103.GA5.ZFS "\
41-
"11:1.3.1:/zoau/v1.3.1:IMSTESTU.ZOAU.V130.ZFS "\
42-
"12:1.3.2:/zoau/v1.3.2.0:IMSTESTU.ZOAU.V100.GA.ZFS "\
43-
"13:1.3.3:/zoau/v1.3.3:IMSTESTU.ZOAU.V101.GA.ZFS "\
31+
zoau_mount_list_str="1:1.1.1-ptf1:/zoau/v1.1.1-ptf1:IMSTESTU.ZOAU.V111.PTF1.ZFS "\
32+
"2:1.2.0:/zoau/v1.2.0:IMSTESTU.ZOAU.V120.ZFS "\
33+
"3:1.2.1:/zoau/v1.2.1:IMSTESTU.ZOAU.V121.ZFS "\
34+
"4:1.2.2:/zoau/v1.2.2:IMSTESTU.ZOAU.V122.ZFS "\
35+
"5:1.2.3:/zoau/v1.2.3:IMSTESTU.ZOAU.V123.ZFS "\
36+
"6:1.2.4:/zoau/v1.2.4:IMSTESTU.ZOAU.V124.ZFS "\
37+
"7:1.2.5.8:/zoau/v1.2.5.8:IMSTESTU.ZOAU.V102.GA.ZFS "\
38+
"8:1.2.5.10:/zoau/v1.2.5.10:IMSTESTU.ZOAU.V103.PTF2.ZFS "\
39+
"9:1.3.0:/zoau/v1.3.0:IMSTESTU.ZOAU.V103.GA5.ZFS "\
40+
"10:1.3.1:/zoau/v1.3.1:IMSTESTU.ZOAU.V130.ZFS "\
41+
"11:1.3.2:/zoau/v1.3.2.0:IMSTESTU.ZOAU.V100.GA.ZFS "\
42+
"12:1.3.3:/zoau/v1.3.3:IMSTESTU.ZOAU.V101.GA.ZFS "\
43+
"13:1.3.4:/zoau/v1.3.4:IMSTESTU.ZOAU.V110.GA.ZFS "\
4444
"14:latest:/zoau/latest:IMSTESTU.ZOAU.LATEST.ZFS "
4545

4646
# ------------------------------------------------------------------------------

‎tests/functional/modules/test_zos_find_func.py

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,13 @@
4242

4343
DATASET_TYPES = ['seq', 'pds', 'pdse']
4444

45+
LOCK_VSAM_JCL = """//SLEEP JOB (T043JM,JM00,1,0,0,0),'SLEEP - JRM',CLASS=R,
46+
// MSGCLASS=X,MSGLEVEL=1,NOTIFY=S0JM
47+
//STEP1 EXEC PGM=BPXBATCH,PARM='SH sleep 60'
48+
//VSAM1 DD DISP=OLD,DSN={0}
49+
//STDOUT DD SYSOUT=*
50+
//STDERR DD SYSOUT=*
51+
"""
4552

4653
def create_vsam_ksds(ds_name, ansible_zos_module, volume):
4754
hosts = ansible_zos_module
@@ -391,7 +398,6 @@ def test_find_data_sets_in_volume(ansible_zos_module, volumes_on_systems):
391398
hosts.all.zos_data_set(name=data_set_name, state="absent")
392399

393400

394-
395401
def test_find_vsam_pattern(ansible_zos_module, volumes_on_systems):
396402
hosts = ansible_zos_module
397403
try:
@@ -401,13 +407,34 @@ def test_find_vsam_pattern(ansible_zos_module, volumes_on_systems):
401407
volume = volumes.get_available_vol()
402408
create_vsam_ksds(vsam, hosts, volume)
403409

410+
# A KSDS VSAM has 3 different components, cluster, data and index
411+
# This test should find all three
404412
find_res = hosts.all.zos_find(
405413
patterns=[f'{TEST_SUITE_HLQ}.FIND.VSAM.FUNCTEST.*'],
406414
resource_type='cluster'
407415
)
408416
for val in find_res.contacted.values():
409417
assert len(val.get('data_sets')) == 1
410418
assert val.get('matched') == len(val.get('data_sets'))
419+
assert val.get('data_sets')[0].get("name", None) == VSAM_NAMES[0]
420+
421+
find_res = hosts.all.zos_find(
422+
patterns=[f'{TEST_SUITE_HLQ}.FIND.VSAM.FUNCTEST.*'],
423+
resource_type='data'
424+
)
425+
for val in find_res.contacted.values():
426+
assert len(val.get('data_sets')) == 1
427+
assert val.get('matched') == len(val.get('data_sets'))
428+
assert val.get('data_sets')[0].get("name", None) == f"{VSAM_NAMES[0]}.DATA"
429+
430+
find_res = hosts.all.zos_find(
431+
patterns=[f'{TEST_SUITE_HLQ}.FIND.VSAM.FUNCTEST.*'],
432+
resource_type='index'
433+
)
434+
for val in find_res.contacted.values():
435+
assert len(val.get('data_sets')) == 1
436+
assert val.get('matched') == len(val.get('data_sets'))
437+
assert val.get('data_sets')[0].get("name", None) == f"{VSAM_NAMES[0]}.INDEX"
411438
finally:
412439
hosts.all.zos_data_set(
413440
batch=[
@@ -419,6 +446,38 @@ def test_find_vsam_pattern(ansible_zos_module, volumes_on_systems):
419446
)
420447

421448

449+
def test_find_vsam_pattern_disp_old(ansible_zos_module, volumes_on_systems):
450+
"""
451+
Creates a VSAM cluster and runs a JCL to lock the data set with DISP=OLD.
452+
Then make sure that we can query the VSAM. Currently, if using age + cluster
453+
resource_type the module will not find the vsam.
454+
"""
455+
hosts = ansible_zos_module
456+
try:
457+
volumes = Volume_Handler(volumes_on_systems)
458+
jcl_ds = get_tmp_ds_name()
459+
for vsam in VSAM_NAMES:
460+
volume = volumes.get_available_vol()
461+
create_vsam_ksds(vsam, hosts, volume)
462+
463+
hosts.all.shell(cmd=f"decho \"{LOCK_VSAM_JCL.format(VSAM_NAMES[0])}\" '{jcl_ds}'; jsub '{jcl_ds}'")
464+
find_res = hosts.all.zos_find(
465+
patterns=[f'{TEST_SUITE_HLQ}.FIND.VSAM.FUNCTEST.*'],
466+
resource_type='cluster'
467+
)
468+
for val in find_res.contacted.values():
469+
assert len(val.get('data_sets')) == 1
470+
assert val.get('matched') == len(val.get('data_sets'))
471+
finally:
472+
hosts.all.zos_data_set(
473+
batch=[
474+
{
475+
"name":i,
476+
"state":'absent'
477+
} for i in VSAM_NAMES
478+
]
479+
)
480+
422481
def test_find_vsam_in_volume(ansible_zos_module, volumes_on_systems):
423482
hosts = ansible_zos_module
424483
volumes = Volume_Handler(volumes_on_systems)

0 commit comments

Comments
 (0)