From fdb67d11e7a5ece77364ae3388eef11c31e562c6 Mon Sep 17 00:00:00 2001 From: Ian Wright Date: Thu, 19 Oct 2023 09:03:25 -0400 Subject: [PATCH 01/10] "shrinked" is not proper english. For clarity and flow I've changed the error message when a user attempts to resize a hyperswap volume --- plugins/modules/ibm_svc_manage_mirrored_volume.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/modules/ibm_svc_manage_mirrored_volume.py b/plugins/modules/ibm_svc_manage_mirrored_volume.py index f0ba90a..22ebf07 100644 --- a/plugins/modules/ibm_svc_manage_mirrored_volume.py +++ b/plugins/modules/ibm_svc_manage_mirrored_volume.py @@ -413,7 +413,7 @@ def resizevolume(self): if self.vdisk_type == "local hyperswap" and self.expand_flag: cmd = "expandvolume" elif self.vdisk_type == "local hyperswap" and self.shrink_flag: - self.module.fail_json(msg="Size of a HyperSwap Volume cannot be shrinked") + self.module.fail_json(msg="A HyperSwap Volume cannot be reduced in size") elif self.vdisk_type == "standard mirror" and self.expand_flag: cmd = "expandvdisksize" elif self.vdisk_type == "standard mirror" and self.shrink_flag: From 4bd4643dcc6c3e64ac2dee2915a165054855c2be Mon Sep 17 00:00:00 2001 From: Ian Wright Date: Tue, 24 Oct 2023 09:14:00 -0400 Subject: [PATCH 02/10] Created new "type" called "transform". This allows volumes to be changed between thin and thick provisioned (within the same pool) and also allows a volume to be migrated to a different pool, non-disruptively, through volume mirror (not between IO Groups or systems). Volumes are automatically deleted after movement --- .vscode/settings.json | 5 + .../modules/ibm_svc_manage_mirrored_volume.py | 123 +++++-- plugins/modules/ibm_svc_utils.py | 334 ++++++++++++++++++ .../test_ibm_svc_manage_mirrored_volume.py | 7 +- 4 files changed, 443 insertions(+), 26 deletions(-) create mode 100644 .vscode/settings.json create mode 100644 plugins/modules/ibm_svc_utils.py diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..6d4121b --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "python.analysis.extraPaths": [ + "./plugins/modules" + ] +} \ No newline at end of file diff --git a/plugins/modules/ibm_svc_manage_mirrored_volume.py b/plugins/modules/ibm_svc_manage_mirrored_volume.py index 22ebf07..355d8ff 100644 --- a/plugins/modules/ibm_svc_manage_mirrored_volume.py +++ b/plugins/modules/ibm_svc_manage_mirrored_volume.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # Copyright (C) 2021 IBM CORPORATION -# Author(s): Rohit Kumar +# Author(s): Rohit Kumar , Ian R Wright # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) from __future__ import absolute_import, division, print_function @@ -70,7 +70,7 @@ a "standard mirror" volume gets created. - If a "standard" mirrored volume exists and either I(PoolA) or I(PoolB) is specified, the mirrored volume gets converted to a standard volume. - choices: [ local hyperswap, standard ] + choices: [ local hyperswap, standard, transform ] type: str thin: description: @@ -173,6 +173,18 @@ name: "vol1" state: present size: "{{new_size}}" +- name: Convert a thick (100% provisioned) volume to thin provisioned with an rsize of 10% + block: + - name: transform a thick volume to thin provisioned with rsize of 10% + clustername: "{{clustername}}" + username: "{{username}}" + password: "{{password}}" + log_path: /tmp/playbook.debug + name: "vol1" + state: present + type: transform + thin: true + rsize: "10%" ''' RETURN = '''#''' @@ -195,7 +207,8 @@ def __init__(self): poolB=dict(type='str', required=False), size=dict(type='str', required=False), thin=dict(type='bool', required=False), - type=dict(type='str', required=False, choices=['local hyperswap', 'standard']), + # added a new "type" called transform for thick/thin conversion and migration between pools using volume mirroring + type=dict(type='str', required=False, choices=['local hyperswap', 'standard', 'transform']), grainsize=dict(type='str', required=False), rsize=dict(type='str', required=False), compressed=dict(type='bool', required=False), @@ -215,6 +228,8 @@ def __init__(self): self.isdrp = False self.expand_flag = False self.shrink_flag = False + # added an autodelete flag for use with transform + self.autodelete = False # logging setup log_path = self.module.params.get('log_path') @@ -273,12 +288,14 @@ def basic_checks(self, data): self.module.fail_json(msg="PoolB does not exist") if self.state == "present" and not self.type and not self.size: self.module.fail_json(msg="missing required argument: type") - if self.poolA and self.poolB: + # having both pools within a site is permitted as long as transform is the type + if self.poolA and self.poolB and self.type != "transform": if self.poolA == self.poolB: - self.module.fail_json(msg="poolA and poolB cannot be same") + self.module.fail_json(msg="poolA and poolB cannot be same unless you are transforming a volume") siteA, siteB = self.discover_site_from_pools() - if siteA != siteB and self.type == "standard": - self.module.fail_json(msg="To create Standard Mirrored volume, provide pools belonging to same site.") + # tranformation does not work between sites/systems + if siteA != siteB and (self.type == "standard" or siteA != siteB and self.type == "transform"): + self.module.fail_json(msg="To create Standard Mirrored volume or transform a volume, provide pools belonging to same site.") if not self.poolA and not self.poolB and self.state == "present" and not self.size: self.module.fail_json(msg="Both poolA and poolB cannot be empty") if self.type == "local hyperswap" and self.state != 'absent': @@ -333,8 +350,17 @@ def discover_vdisk_type(self, data): def discover_site_from_pools(self): self.log("Entering function discover_site_from_pools") - poolA_site = self.poolA_data['site_name'] - poolB_site = self.poolB_data['site_name'] + # if using a single site for volume transformation, setting the other "site" to be the same as the one that does exist for purposes of + # getting along with the code + if self.poolA and not self.poolB: + poolA_site = self.poolA_data['site_name'] + poolB_site = poolA_site + elif self.poolB and not self.poolA: + poolB_site = self.poolB_data['site_name'] + poolA_site = poolB_site + else: + poolA_site = self.poolA_data['site_name'] + poolB_site = self.poolB_data['site_name'] return poolA_site, poolB_site def vdisk_probe(self, data): @@ -357,11 +383,17 @@ def vdisk_probe(self, data): self.changebysize = existing_size - size_in_bytes self.shrink_flag = True if self.poolA and self.poolB: + #Volume transformation is not allowed if there's already a mirror in place + if self.type == "transform": + if (self.vdisk_type == "local hyperswap" or self.vdisk_type == "standard pool") and self.type == "transform": + self.module.fail_json(msg="HyperSwap or Standard Mirror Volumes cannot be transformed") + elif self.poolA == self.discovered_standard_vol_pool or self.poolB == self.discovered_standard_vol_pool: + props +=['addvdiskcopy'] if self.vdisk_type == "local hyperswap" and self.type == "standard": self.module.fail_json(msg="HyperSwap Volume cannot be converted to standard mirror") if self.vdisk_type == "standard mirror" or self.vdisk_type == "local hyperswap": if (self.poolA == self.discovered_poolA or self.poolA == self.discovered_poolB)\ - and (self.poolB == self.discovered_poolA or self.poolB == self.discovered_poolB) and not resizevolume_flag: + and (self.poolB == self.discovered_poolA or self.poolB == self.discovered_poolB) and not resizevolume_flag: return props elif not resizevolume_flag: self.module.fail_json(msg="Pools for Standard Mirror or HyperSwap volume cannot be updated") @@ -379,15 +411,26 @@ def vdisk_probe(self, data): elif self.vdisk_type and not self.type: self.module.fail_json(msg="missing required argument: type") elif not self.poolA or not self.poolB: - if self.vdisk_type == "standard": + # transformation is allowed within a single pool. + if self.type == "transform" and not self.poolB: + if self.vdisk_type == "standard": + if self.poolA == self.discovered_standard_vol_pool: + props += ['addvdiskcopy'] + elif self.type == "transform" and not self.poolA: + if self.vdisk_type == "standard": + if self.poolB == self.discovered_standard_vol_pool: + props += ['addvdiskcopy'] + elif self.vdisk_type == "standard": if self.poolA == self.discovered_standard_vol_pool or self.poolB == self.discovered_standard_vol_pool: self.log("Standard Volume already exists, no modifications done") return props if self.poolA: - if self.poolA == self.discovered_poolA or self.poolA == self.discovered_poolB: - props += ['rmvolumecopy'] - else: - self.module.fail_json(msg="One of the input pools must belong to the Volume") + # Changed this. Needed to make sure that it wasn't erroneously deciding that a volume needed to be removed when transform was set + if self.type != "transform" and self.vdisk_type == "standard": + if self.poolA == self.discovered_poolA or self.poolA == self.discovered_poolB: + props += ['rmvolumecopy'] + else: + self.module.fail_json(msg="One of the input pools must belong to the Volume") elif self.poolB: if self.poolB == self.discovered_poolA or self.poolB == self.discovered_poolB: props += ['rmvolumecopy'] @@ -566,14 +609,22 @@ def addvdiskcopy(self): self.module.fail_json(msg="Parameter 'size' cannot be passed while converting a standard volume to Mirror Volume") siteA, siteB = self.discover_site_from_pools() if siteA != siteB: - self.module.fail_json(msg="To create Standard Mirrored volume, provide pools belonging to same site.") + self.module.fail_json(msg="To create Standard Mirror or to Transform a volume, provide pools belonging to same site.") if self.poolA and (self.poolB == self.discovered_standard_vol_pool and self.poolA != self.discovered_standard_vol_pool): cmdopts['mdiskgrp'] = self.poolA elif self.poolB and (self.poolA == self.discovered_standard_vol_pool and self.poolB != self.discovered_standard_vol_pool): cmdopts['mdiskgrp'] = self.poolB + # if the type selected is "transform" we want to allow migration to other pools or transformation within the same pool + # by setting the mdiskgrp to be equal to the only pool available + elif self.type == "transform": + if self.poolA == self.discovered_standard_vol_pool: + cmdopts['mdiskgrp'] = self.poolA + else: + if self.poolB == self.discovered_standard_vol_pool: + cmdopts['mdiskgrp'] = self.poolB else: self.module.fail_json(msg="One of the input pools must belong to the volume") - if self.compressed: + if self.compressed == True: cmdopts['compressed'] = self.compressed if self.grainsize: cmdopts['grainsize'] = self.grainsize @@ -591,6 +642,9 @@ def addvdiskcopy(self): self.module.fail_json(msg="To configure 'deduplicated', parameter 'thin' should be passed and the value should be 'true.'") if self.isdrp and self.thin: cmdopts['autoexpand'] = True + # adding the autodelete flag into command options + if self.autodelete == True: + cmdopts['autodelete'] = True if self.module.check_mode: self.changed = True return @@ -635,6 +689,8 @@ def vdisk_update(self, modify): self.addvolumecopy() elif 'addvdiskcopy' in modify: self.isdrpool() + if self.type == "transform": + self.autodelete = True self.addvdiskcopy() elif 'rmvolumecopy' in modify: self.rmvolumecopy() @@ -642,11 +698,22 @@ def vdisk_update(self, modify): self.resizevolume() def isdrpool(self): - poolA_drp = self.poolA_data['data_reduction'] - poolB_drp = self.poolB_data['data_reduction'] - isdrpool_list = [poolA_drp, poolB_drp] - if "yes" in isdrpool_list: - self.isdrp = True + poolA_drp={} + poolB_drp={} + if self.poolA and self.poolB: + poolA_drp = self.poolA_data['data_reduction'] + poolB_drp = self.poolB_data['data_reduction'] + isdrpool_list = [poolA_drp, poolB_drp] + if "yes" in isdrpool_list: + self.isdrp = True + else: + if (self.poolA or self.poolB) and self.type == "transform": + if self.poolA: + poolA_drp = self.poolA_data['data_reduction'] + elif self.poolB: + poolB_drp = self.poolB_data['data_reduction'] + if "yes" in poolA_drp or poolB_drp: + self.isdrp = True def volume_delete(self): self.log("Entering function volume_delete") @@ -712,7 +779,7 @@ def apply(self): if not self.type: self.module.fail_json(msg="missing required argument: type") # create_vdisk_flag = self.discover_site_from_pools() - if self.type == "standard": + elif self.type == "standard": self.isdrpool() self.vdisk_create() msg = "Standard Mirrored Volume %s has been created." % self.name @@ -722,10 +789,18 @@ def apply(self): self.volume_create() msg = "HyperSwap Volume %s has been created." % self.name changed = True + elif self.type == "transform": + self.isdrpool() + self.vdisk_create() + msg = "Volume %s has been transformed and the original copy has been deleted." % self.name + changed = True else: # This is where we would modify if required self.vdisk_update(modify) - msg = "Volume [%s] has been modified." % self.name + if self.type == "transform": + msg = "Volume [%s] is being transformed. The original copy will be deleted after synchronization" % self.name + else: + msg = "Volume [%s] has been modified." % self.name changed = True elif self.state == 'absent': self.volume_delete() diff --git a/plugins/modules/ibm_svc_utils.py b/plugins/modules/ibm_svc_utils.py new file mode 100644 index 0000000..eb39e54 --- /dev/null +++ b/plugins/modules/ibm_svc_utils.py @@ -0,0 +1,334 @@ +# Copyright (C) 2020 IBM CORPORATION +# Author(s): Peng Wang +# +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +""" Support class for IBM SVC ansible modules """ + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +import json +import logging + +from ansible.module_utils.urls import open_url +from ansible.module_utils.six.moves.urllib.parse import quote +from ansible.module_utils.six.moves.urllib.error import HTTPError + +# from urllib import open_url +# from six import quote +# from six import HTTPError + + +def svc_argument_spec(): + """ + Returns argument_spec of options common to ibm_svc_*-modules + + :returns: argument_spec + :rtype: dict + """ + return dict( + clustername=dict(type='str', required=True), + domain=dict(type='str', default=None), + validate_certs=dict(type='bool', default=False), + username=dict(type='str'), + password=dict(type='str', no_log=True), + log_path=dict(type='str'), + token=dict(type='str', no_log=True) + ) + + +def svc_ssh_argument_spec(): + """ + Returns argument_spec of options common to ibm_svcinfo_command + and ibm_svctask_command modules + + :returns: argument_spec + :rtype: dict + """ + return dict( + clustername=dict(type='str', required=True), + username=dict(type='str', required=True), + password=dict(type='str', required=True, no_log=True), + log_path=dict(type='str') + ) + + +def strtobool(val): + ''' + Converts a string representation to boolean. + + This is a built-in function available in python till the version 3.9 under disutils.util + but this has been deprecated in 3.10 and may not be available in future python releases + so adding the source code here. + ''' + if val in {'y', 'yes', 't', 'true', 'on', '1'}: + return 1 + elif val in {'n', 'no', 'f', 'false', 'off', '0'}: + return 0 + else: + raise ValueError("invalid truth value %r" % (val,)) + + +def get_logger(module_name, log_file_name, log_level=logging.INFO): + FORMAT = '%(asctime)s.%(msecs)03d %(levelname)5s %(thread)d %(filename)s:%(funcName)s():%(lineno)s %(message)s' + DATEFORMAT = '%Y-%m-%dT%H:%M:%S' + log_path = 'IBMSV_ansible_collections.log' + if log_file_name: + log_path = log_file_name + logging.basicConfig(filename=log_path, format=FORMAT, datefmt=DATEFORMAT) + log = logging.getLogger(module_name) + log.setLevel(log_level) + return log + + +class IBMSVCRestApi(object): + """ Communicate with SVC through RestApi + SVC commands usually have the format + $ command -opt1 value1 -opt2 value2 arg1 arg2 arg3 + to use the RestApi we transform this into + https://host:7443/rest/command/arg1/arg2/arg3 + data={'opt1':'value1', 'opt2':'value2'} + """ + + def __init__(self, module, clustername, domain, username, password, + validate_certs, log_path, token): + """ Initialize module with what we need for initial connection + :param clustername: name of the SVC cluster + :type clustername: string + :param domain: domain name to make a fully qualified host name + :type domain: string + :param username: SVC username + :type username: string + :param password: Password for user + :type password: string + :param validate_certs: whether or not the connection is insecure + :type validate_certs: bool + """ + self.module = module + self.clustername = clustername + self.domain = domain + self.username = username + self.password = password + self.validate_certs = validate_certs + self.token = token + + # logging setup + log = get_logger(self.__class__.__name__, log_path) + self.log = log.info + + # Make sure we can connect through the RestApi + if self.token is None: + if not self.username or not self.password: + self.module.fail_json(msg="You must pass in either pre-acquired token" + " or username/password to generate new token") + self.token = self._svc_authorize() + else: + self.log("Token already passed: %s", self.token) + + if not self.token: + self.module.exit_json(msg='Failed to obtain access token', unreachable=True) + + @property + def port(self): + return getattr(self, '_port', None) or '7443' + + @property + def protocol(self): + return getattr(self, '_protocol', None) or 'https' + + @property + def resturl(self): + if self.domain: + hostname = '%s.%s' % (self.clustername, self.domain) + else: + hostname = self.clustername + return (getattr(self, '_resturl', None) + or "{protocol}://{host}:{port}/rest".format( + protocol=self.protocol, host=hostname, port=self.port)) + + @property + def token(self): + return getattr(self, '_token', None) or None + + @token.setter + def token(self, value): + return setattr(self, '_token', value) + + def _svc_rest(self, method, headers, cmd, cmdopts, cmdargs, timeout=10): + """ Run SVC command with token info added into header + :param method: http method, POST or GET + :type method: string + :param headers: http headers + :type headers: dict + :param cmd: svc command to run + :type cmd: string + :param cmdopts: svc command options, name paramter and value + :type cmdopts: dict + :param cmdargs: svc command arguments, non-named paramaters + :type timeout: int + :param timeout: open_url argument to set timeout for http gateway + :return: dict of command results + :rtype: dict + """ + + # Catch any output or errors and pass back to the caller to deal with. + r = { + 'url': None, + 'code': None, + 'err': None, + 'out': None, + 'data': None + } + + postfix = cmd + if cmdargs: + postfix = '/'.join([postfix] + [quote(str(a)) for a in cmdargs]) + url = '/'.join([self.resturl] + [postfix]) + r['url'] = url # Pass back in result for error handling + self.log("_svc_rest: url=%s", url) + + payload = cmdopts if cmdopts else None + data = self.module.jsonify(payload).encode('utf8') + r['data'] = cmdopts # Original payload data has nicer formatting + self.log("_svc_rest: payload=%s", payload) + + try: + o = open_url(url, method=method, headers=headers, timeout=timeout, + validate_certs=self.validate_certs, data=bytes(data)) + except HTTPError as e: + self.log('_svc_rest: httperror %s', str(e)) + r['code'] = e.getcode() + r['out'] = e.read() + r['err'] = "HTTPError %s", str(e) + return r + except Exception as e: + self.log('_svc_rest: exception : %s', str(e)) + r['err'] = "Exception %s", str(e) + return r + + try: + j = json.load(o) + except ValueError as e: + self.log("_svc_rest: value error pass: %s", str(e)) + # pass, will mean both data and error are None. + return r + + r['out'] = j + return r + + def _svc_authorize(self): + """ Obtain a token if we are authoized to connect + :return: None or token string + """ + + headers = { + 'Content-Type': 'application/json', + 'X-Auth-Username': self.username, + 'X-Auth-Password': self.password + } + + rest = self._svc_rest(method='POST', headers=headers, cmd='auth', + cmdopts=None, cmdargs=None) + + if rest['err']: + return None + + out = rest['out'] + if out: + if 'token' in out: + return out['token'] + + return None + + def _svc_token_wrap(self, cmd, cmdopts, cmdargs, timeout=10): + """ Run SVC command with token info added into header + :param cmd: svc command to run + :type cmd: string + :param cmdopts: svc command options, name paramter and value + :type cmdopts: dict + :param cmdargs: svc command arguments, non-named paramaters + :type cmdargs: list + :param timeout: open_url argument to set timeout for http gateway + :type timeout: int + :returns: command results + """ + + if self.token is None: + self.module.fail_json(msg="No authorize token") + # Abort + + headers = { + 'Content-Type': 'application/json', + 'X-Auth-Token': self.token + } + + return self._svc_rest(method='POST', headers=headers, cmd=cmd, + cmdopts=cmdopts, cmdargs=cmdargs, timeout=timeout) + + def svc_run_command(self, cmd, cmdopts, cmdargs, timeout=10): + """ Generic execute a SVC command + :param cmd: svc command to run + :type cmd: string + :param cmdopts: svc command options, name parameter and value + :type cmdopts: dict + :param cmdargs: svc command arguments, non-named parameters + :type cmdargs: list + :param timeout: open_url argument to set timeout for http gateway + :type timeout: int + :returns: command output + """ + + rest = self._svc_token_wrap(cmd, cmdopts, cmdargs, timeout) + self.log("svc_run_command rest=%s", rest) + + if rest['err']: + msg = rest + self.module.fail_json(msg=msg) + # Aborts + + # Might be None + return rest['out'] + + def svc_obj_info(self, cmd, cmdopts, cmdargs, timeout=10): + """ Obtain information about an SVC object through the ls command + :param cmd: svc command to run + :type cmd: string + :param cmdopts: svc command options, name parameter and value + :type cmdopts: dict + :param cmdargs: svc command arguments, non-named paramaters + :type cmdargs: list + :param timeout: open_url argument to set timeout for http gateway + :type timeout: int + :returns: command output + :rtype: dict + """ + + rest = self._svc_token_wrap(cmd, cmdopts, cmdargs, timeout) + self.log("svc_obj_info rest=%s", rest) + + if rest['code']: + if rest['code'] == 500: + # Object did not exist, which is quite valid. + return None + + # Fail for anything else + if rest['err']: + self.module.fail_json(msg=rest) + # Aborts + + # Might be None + return rest['out'] + + def get_auth_token(self): + """ Obtain information about an SVC object through the ls command + :returns: authentication token + """ + # Make sure we can connect through the RestApi + self.token = self._svc_authorize() + self.log("_connect by using token") + if not self.token: + self.module.exit_json(msg='Failed to obtain access token', unreachable=True) + + return self.token diff --git a/tests/unit/plugins/modules/test_ibm_svc_manage_mirrored_volume.py b/tests/unit/plugins/modules/test_ibm_svc_manage_mirrored_volume.py index ec4af33..90d7153 100644 --- a/tests/unit/plugins/modules/test_ibm_svc_manage_mirrored_volume.py +++ b/tests/unit/plugins/modules/test_ibm_svc_manage_mirrored_volume.py @@ -14,8 +14,11 @@ from mock import patch from ansible.module_utils import basic from ansible.module_utils._text import to_bytes -from ansible_collections.ibm.storage_virtualize.plugins.module_utils.ibm_svc_utils import IBMSVCRestApi -from ansible_collections.ibm.storage_virtualize.plugins.modules.ibm_svc_manage_mirrored_volume import IBMSVCvolume +# from ansible_collections.ibm.storage_virtualize.plugins.module_utils.ibm_svc_utils import IBMSVCRestApi +# from ansible_collections.ibm.storage_virtualize.plugins.modules.ibm_svc_manage_mirrored_volume import IBMSVCvolume +from ibm_svc_utils import IBMSVCRestApi +from ibm_svc_manage_mirrored_volume import IBMSVCvolume + def set_module_args(args): From 7092e53ae6c72cb6371af532fc0a1078b42fa85c Mon Sep 17 00:00:00 2001 From: Ian Wright Date: Tue, 24 Oct 2023 09:17:45 -0400 Subject: [PATCH 03/10] added documentation in the options section --- plugins/modules/ibm_svc_manage_mirrored_volume.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/modules/ibm_svc_manage_mirrored_volume.py b/plugins/modules/ibm_svc_manage_mirrored_volume.py index 355d8ff..d6ee800 100644 --- a/plugins/modules/ibm_svc_manage_mirrored_volume.py +++ b/plugins/modules/ibm_svc_manage_mirrored_volume.py @@ -70,6 +70,9 @@ a "standard mirror" volume gets created. - If a "standard" mirrored volume exists and either I(PoolA) or I(PoolB) is specified, the mirrored volume gets converted to a standard volume. + - Transform allows a volume to be changed between thin and thick/generic provisioning and automatically deletes the old copy after sync + - Transform also allows Volume Mirror to be used for migration between pools. A volume will move to a different pool (within the same IO Group for non-disruption) + and the original copy will be deleted after synchronization choices: [ local hyperswap, standard, transform ] type: str thin: From 0be38700bc8aaf487d49d5be425124719ae165af Mon Sep 17 00:00:00 2001 From: Ian Wright Date: Tue, 24 Oct 2023 09:46:07 -0400 Subject: [PATCH 04/10] fixed a bug related to using Pool B only --- .../modules/ibm_svc_manage_mirrored_volume.py | 25 ++++++------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/plugins/modules/ibm_svc_manage_mirrored_volume.py b/plugins/modules/ibm_svc_manage_mirrored_volume.py index d6ee800..8941821 100644 --- a/plugins/modules/ibm_svc_manage_mirrored_volume.py +++ b/plugins/modules/ibm_svc_manage_mirrored_volume.py @@ -415,15 +415,10 @@ def vdisk_probe(self, data): self.module.fail_json(msg="missing required argument: type") elif not self.poolA or not self.poolB: # transformation is allowed within a single pool. - if self.type == "transform" and not self.poolB: - if self.vdisk_type == "standard": - if self.poolA == self.discovered_standard_vol_pool: + if self.type == "transform" and self.vdisk_type == "standard": + if (self.poolA == self.discovered_standard_vol_pool) or (self.poolB == self.discovered_standard_vol_pool): props += ['addvdiskcopy'] - elif self.type == "transform" and not self.poolA: - if self.vdisk_type == "standard": - if self.poolB == self.discovered_standard_vol_pool: - props += ['addvdiskcopy'] - elif self.vdisk_type == "standard": + elif self.vdisk_type == "standard" and self.type != "transform": if self.poolA == self.discovered_standard_vol_pool or self.poolB == self.discovered_standard_vol_pool: self.log("Standard Volume already exists, no modifications done") return props @@ -435,10 +430,11 @@ def vdisk_probe(self, data): else: self.module.fail_json(msg="One of the input pools must belong to the Volume") elif self.poolB: - if self.poolB == self.discovered_poolA or self.poolB == self.discovered_poolB: - props += ['rmvolumecopy'] - else: - self.module.fail_json(msg="One of the input pools must belong to the Volume") + if self.type != "transform" and self.vdisk_type == "standard": + if self.poolB == self.discovered_poolA or self.poolB == self.discovered_poolB: + props += ['rmvolumecopy'] + else: + self.module.fail_json(msg="One of the input pools must belong to the Volume") if not (self.poolA or not self.poolB) and not self.size: if (self.system_topology == "hyperswap" and self.type == "local hyperswap"): self.module.fail_json(msg="Type must be standard if either PoolA or PoolB is not specified.") @@ -792,11 +788,6 @@ def apply(self): self.volume_create() msg = "HyperSwap Volume %s has been created." % self.name changed = True - elif self.type == "transform": - self.isdrpool() - self.vdisk_create() - msg = "Volume %s has been transformed and the original copy has been deleted." % self.name - changed = True else: # This is where we would modify if required self.vdisk_update(modify) From 5ea45d17072f43ff28ef0be7e1653a466c45ac8e Mon Sep 17 00:00:00 2001 From: Ian Wright Date: Tue, 24 Oct 2023 10:50:09 -0400 Subject: [PATCH 05/10] backed out changes for my lab environment --- plugins/module_utils/ibm_svc_utils.py | 4 ++++ tests/unit/plugins/modules/test_ibm_svc_manage_migration.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/plugins/module_utils/ibm_svc_utils.py b/plugins/module_utils/ibm_svc_utils.py index 80f871d..eb39e54 100644 --- a/plugins/module_utils/ibm_svc_utils.py +++ b/plugins/module_utils/ibm_svc_utils.py @@ -16,6 +16,10 @@ from ansible.module_utils.six.moves.urllib.parse import quote from ansible.module_utils.six.moves.urllib.error import HTTPError +# from urllib import open_url +# from six import quote +# from six import HTTPError + def svc_argument_spec(): """ diff --git a/tests/unit/plugins/modules/test_ibm_svc_manage_migration.py b/tests/unit/plugins/modules/test_ibm_svc_manage_migration.py index 1f8f991..23b765a 100644 --- a/tests/unit/plugins/modules/test_ibm_svc_manage_migration.py +++ b/tests/unit/plugins/modules/test_ibm_svc_manage_migration.py @@ -3,9 +3,9 @@ # # GNU General Public License v3.0+ # (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) - +# (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +""" unit tests IBM Storage Virtualize Ansible module: ibm_svc_manage_migration """ """ unit tests IBM Storage Virtualize Ansible module: ibm_svc_manage_migration """ - from __future__ import (absolute_import, division, print_function) __metaclass__ = type import unittest From 6af7e73543b1cc7ea4045663b473675a536e865a Mon Sep 17 00:00:00 2001 From: Ian Wright Date: Thu, 26 Oct 2023 16:47:12 -0400 Subject: [PATCH 06/10] added auto-expand --- plugins/modules/ibm_svc_manage_mirrored_volume.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/modules/ibm_svc_manage_mirrored_volume.py b/plugins/modules/ibm_svc_manage_mirrored_volume.py index 8941821..cd1b749 100644 --- a/plugins/modules/ibm_svc_manage_mirrored_volume.py +++ b/plugins/modules/ibm_svc_manage_mirrored_volume.py @@ -631,6 +631,8 @@ def addvdiskcopy(self): cmdopts['rsize'] = self.rsize elif self.thin: cmdopts['rsize'] = "2%" + # Set autoexpand to true. Short of having an autoexpand setting this should be the case for any thin volume + cmdopts['autoexpand'] = True elif self.rsize and not self.thin: self.module.fail_json(msg="To configure 'rsize', parameter 'thin' should be passed and the value should be 'true'.") if self.deduplicated: From 0af267a13ee9e5144cfa9a9e0aae0a538fcf8640 Mon Sep 17 00:00:00 2001 From: Ian Wright Date: Thu, 26 Oct 2023 16:53:14 -0400 Subject: [PATCH 07/10] added bug fix to evaluating whether 'Thin' had changed --- plugins/modules/ibm_svc_manage_volume.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/plugins/modules/ibm_svc_manage_volume.py b/plugins/modules/ibm_svc_manage_volume.py index 3129b49..14915eb 100644 --- a/plugins/modules/ibm_svc_manage_volume.py +++ b/plugins/modules/ibm_svc_manage_volume.py @@ -535,17 +535,21 @@ def probe_volume(self, data): } # check for change in -thin parameter if self.thin is not None: - if self.thin is True: + if self.thin is True and (data[1]['se_copy'] == 'no' or data[1]['compressed_copy'] == 'yes'): # a standard volume or a compressed volume - if (data[0]['capacity'] == data[1]['real_capacity']) or (data[1]['compressed_copy'] == 'yes'): - props['thin'] = { - 'status': True - } - else: - if (data[0]['capacity'] != data[1]['real_capacity']) or (data[1]['compressed_copy'] == 'no'): - props['thin'] = { - 'status': True - } + # if (int(data[0]['capacity']) == data[1]['real_capacity']) or (data[1]['compressed_copy'] == 'yes'): + # changed logic to use se_copy instead of comparing values + props['thin'] = { + 'status': True + } + self.log('value of self.thin %s', self.thin ) + + # Removed compare with compressed_copy. A volume can be compressed_copy = 'no' with se_copy = 'yes' so this would always be + # True if self.thin was False + elif (self.thin is False and data[1]['se_copy'] == 'yes'): + props['thin'] = { + 'status': True + } # check for change in -compressed parameter if self.compressed is True: # not a compressed volume From 2e7fdd0e4ab71ba02a84cd96a7c88d7c8db70cb9 Mon Sep 17 00:00:00 2001 From: Ian Wright Date: Fri, 3 Nov 2023 09:14:35 -0400 Subject: [PATCH 08/10] init --- galaxy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/galaxy.yml b/galaxy.yml index 52ca9e1..2683f4f 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -1,8 +1,8 @@ # See https://docs.ansible.com/ansible/latest/dev_guide/collections_galaxy_meta.html -namespace: ibm +namespace: irw name: storage_virtualize -version: 2.1.0 +version: 2.1.1 readme: README.md authors: - Shilpi Jain (github.com/Shilpi-J) From f52f755dac5d5a7231512cd83930cdf7c6339709 Mon Sep 17 00:00:00 2001 From: Ian Wright Date: Fri, 15 Dec 2023 13:51:44 -0500 Subject: [PATCH 09/10] changed ibm_svc_manage_volume.py so that it will allow for users to create unformatted volumes --- plugins/modules/ibm_svc_manage_volume.py | 107 +++++++++++++++++------ 1 file changed, 78 insertions(+), 29 deletions(-) diff --git a/plugins/modules/ibm_svc_manage_volume.py b/plugins/modules/ibm_svc_manage_volume.py index 14915eb..f553efc 100644 --- a/plugins/modules/ibm_svc_manage_volume.py +++ b/plugins/modules/ibm_svc_manage_volume.py @@ -139,6 +139,13 @@ type: bool default: false version_added: '2.0.0' + format_disk: + description: + - If set to `False`, will create a volume using 'mkvdisk' instead of 'mkvolume' and set `nofmtdisk` to true + - Valid only when volume groups are not used (only available in mkvolume) + type: bool + default: true + version_added: 'tbd' validate_certs: description: - Validates certification. @@ -255,6 +262,7 @@ state: "absent" ''' + RETURN = '''#''' from traceback import format_exc @@ -291,7 +299,8 @@ def __init__(self): old_name=dict(type='str', required=False), enable_cloud_snapshot=dict(type='bool'), cloud_account_name=dict(type='str'), - allow_hs=dict(type='bool', default=False) + allow_hs=dict(type='bool', default=False), + format_disk=dict(type='bool', default=True) ) ) @@ -321,6 +330,7 @@ def __init__(self): self.enable_cloud_snapshot = self.module.params['enable_cloud_snapshot'] self.cloud_account_name = self.module.params['cloud_account_name'] self.allow_hs = self.module.params['allow_hs'] + self.format_disk = self.module.params['format_disk'] # internal variable self.changed = False @@ -366,6 +376,9 @@ def mandatory_parameter_validation(self): self.module.fail_json(msg='Missing mandatory parameter: [{0}]'.format(', '.join(missing))) if self.volumegroup and self.novolumegroup: self.module.fail_json(msg='Mutually exclusive parameters detected: [volumegroup] and [novolumegroup]') + elif self.volumegroup and self.format_disk: + # volumegroup is only available in mkvolume, so cannot be used with format_disk + self.module.fail_json(msg='Mutually exclusive parameters detected: [volumegroup] and [format_disk]') # for validating parameter while removing an existing volume def volume_deletion_parameter_validation(self): @@ -445,35 +458,71 @@ def create_volume(self): if self.module.check_mode: self.changed = True return - cmd = 'mkvolume' - cmdopts = {} - if self.pool: - cmdopts['pool'] = self.pool - if self.size: - cmdopts['size'] = self.size - if self.unit: - cmdopts['unit'] = self.unit - if self.iogrp: - cmdopts['iogrp'] = self.iogrp[0] - if self.volumegroup: - cmdopts['volumegroup'] = self.volumegroup - if self.thin: - cmdopts['thin'] = self.thin - if self.compressed: - cmdopts['compressed'] = self.compressed - if self.deduplicated: - cmdopts['deduplicated'] = self.deduplicated - if self.buffersize: - cmdopts['buffersize'] = self.buffersize - if self.name: - cmdopts['name'] = self.name - result = self.restapi.svc_run_command(cmd, cmdopts, cmdargs=None) - if result and 'message' in result: - self.changed = True - self.log("create volume result message %s", result['message']) + if self.format_disk != False: + # if the format_disk parameter isn't false then it's fine to use addvolume as normal + cmd = 'mkvolume' + cmdopts = {} + if self.pool: + cmdopts['pool'] = self.pool + if self.size: + cmdopts['size'] = self.size + if self.unit: + cmdopts['unit'] = self.unit + if self.iogrp: + cmdopts['iogrp'] = self.iogrp[0] + if self.volumegroup: + cmdopts['volumegroup'] = self.volumegroup + if self.thin: + cmdopts['thin'] = self.thin + if self.compressed: + cmdopts['compressed'] = self.compressed + if self.deduplicated: + cmdopts['deduplicated'] = self.deduplicated + if self.buffersize: + cmdopts['buffersize'] = self.buffersize + if self.name: + cmdopts['name'] = self.name + result = self.restapi.svc_run_command(cmd, cmdopts, cmdargs=None) + if result and 'message' in result: + self.changed = True + self.log("create volume result message %s", result['message']) + else: + self.module.fail_json( + msg="Failed to create volume [%s]" % self.name) else: - self.module.fail_json( - msg="Failed to create volume [%s]" % self.name) + # if format_disk is false, then mkvdisk is used and the volume will be created without formatting + cmd = 'mkvdisk' + cmdopts = {} + if self.pool: + cmdopts['mdiskgrp'] = self.pool + if self.size: + cmdopts['size'] = self.size + if self.unit: + cmdopts['unit'] = self.unit + if self.iogrp: + cmdopts['iogrp'] = self.iogrp[0] + # volume group not supported with mkvdisk + # if self.volumegroup: + # cmdopts['volumegroup'] = self.volumegroup + if self.thin: + cmdopts['thin'] = self.thin + if self.compressed: + cmdopts['compressed'] = self.compressed + if self.deduplicated: + cmdopts['deduplicated'] = self.deduplicated + if self.buffersize: + cmdopts['rsize'] = self.buffersize + if self.name: + cmdopts['name'] = self.name + cmdopts['nofmtdisk'] = True + # nofmtdisk can be set without any prior test because it's already been tested + result = self.restapi.svc_run_command(cmd, cmdopts, cmdargs=None) + if result and 'message' in result: + self.changed = True + self.log("create volume result message %s", result['message']) + else: + self.module.fail_json( + msg="Failed to create volume [%s]" % self.name) # function to remove an existing volume def remove_volume(self): From 44ec3955d5111f75d7b7ce83a7cf41eeab7f6a91 Mon Sep 17 00:00:00 2001 From: Ian Wright -- Mainline Information Systems <108477581+mainline-automation@users.noreply.github.com> Date: Tue, 9 Jan 2024 14:52:08 -0500 Subject: [PATCH 10/10] Update galaxy.yml Minor update to bring things current and set namespace correctly --- galaxy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/galaxy.yml b/galaxy.yml index 2683f4f..e1e095d 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -1,8 +1,8 @@ # See https://docs.ansible.com/ansible/latest/dev_guide/collections_galaxy_meta.html -namespace: irw +namespace: ibm name: storage_virtualize -version: 2.1.1 +version: 2.2.0 readme: README.md authors: - Shilpi Jain (github.com/Shilpi-J)