Add support for new model format

*  Modify to_argpsec filter to handle suboptions and conditional
   metadata (for eg: required_together, required_if etc) for suboptions

*  Add to_doc filter that renders the compelete module doc in Ansible
   module format

*  Additional changes in template file to work with new model format

Author: ganeshrn

roles/init/tasks/main.yml

@@ -15,6 +15,14 @@
     file: "{{ model }}"
     name: rm
 
+- name: Set the module documentation variable
+  set_fact:
+    rm_docmentation: "{{ rm['DOCUMENTATION']|from_yaml }}"
+
+- name: Set the module ansible_metada variable
+  set_fact:
+    rm_ansible_metadata: "{{ rm['ANSIBLE_METADATA']|from_yaml }}"
+
 - name: "Create the {{ structure }} directory structure"
   file:
     path: "{{ parent }}/{{ item }}"
@@ -23,7 +31,7 @@
 
 - name: Copy the license file to the parent directory
   copy:
-    src: "{{ rm['metadata']['license'] }}"
+    src: "{{ rm['LICENSE'] | default('gpl-3.0.txt') }}"
     dest: "{{ parent }}/LICENSE.txt"
 
 - name: Ensure the 'collection_org' is set when 'structure' is set to collection

roles/resource_module/filter_plugins/to_argspec.py

@@ -1,54 +1,58 @@
-# Copyright (c) 2018 Ansible Project
+# Copyright (c) 2019 Ansible Project
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 
 from __future__ import (absolute_import, division, print_function)
-import jsonref
-import json
+__metaclass__ = type
+
+import yaml
 import pprint
+
 from ansible.module_utils.six import iteritems
+from ansible.module_utils.six import string_types
+from ansible.utils.display import Display
+from ansible.errors import AnsibleFilterError
 
-__metaclass__ = type
+OPTIONS_METADATA = ('type', 'elements', 'default', 'choices', 'required')
+SUBOPTIONS_METADATA = ('mutually_exclusive', 'required_together', 'required_one_of', 'supports_check_mode', 'required_if')
 
-from ansible.errors import AnsibleFilterError
+display = Display()
+
+
+def retrieve_metadata(values, out):
+    for key in OPTIONS_METADATA:
+        if key in values:
+            data = values.get(key, None)
+            if data:
+                out[key] = data
 
-def dive(obj, required=False):
+
+def dive(obj, result):
+    for k, v in iteritems(obj):
+        result[k] = dict()
+        retrieve_metadata(v, result[k])
+        suboptions = v.get('suboptions')
+        if suboptions:
+            for item in SUBOPTIONS_METADATA:
+                if item in v:
+                    result[k][item] = v[item]
+            result[k]['options'] = dict()
+            dive(suboptions, result[k]['options'])
+
+
+def to_argspec(spec):
+    if 'DOCUMENTATION' not in spec:
+        raise AnsibleFilterError("missing required element 'DOCUMENTATION' in model")
+
+    if not isinstance(spec['DOCUMENTATION'], string_types):
+        raise AnsibleFilterError("value of element 'DOCUMENTATION' should be of type string")
     result = {}
-    if not 'type' in obj:
-        raise AnsibleFilterError('missing type key')
-    if obj['type'] == 'object':
-        result['options'] = {}
-        if not 'properties' in obj:
-            raise AnsibleFilterError('missing properties key')
-        for propkey, propval in iteritems(obj['properties']):
-            required = bool('required' in obj and propkey in obj['required'])
-            result['options'][propkey] = dive(propval, required)
-    elif obj['type'] == 'array':
-        result['options'] = {}
-        if obj.get('elements'):
-            result['elements'] = obj['elements']
-        if not 'items' in obj:
-            raise AnsibleFilterError('missing items key in array')
-        if not 'properties' in obj['items']:
-            raise AnsibleFilterError('missing properties in items')
-        for propkey, propval in iteritems(obj['items']['properties']):
-            required = bool('required' in obj['items'] and propkey in obj['items']['required'])
-            result['options'][propkey] = dive(propval, required)
-            result['type'] = 'list'
-    elif obj['type'] in ['str', 'bool', 'int']:
-        if 'default' in obj:
-            result['default'] = obj['default']
-        if 'enum' in obj:
-            result['choices'] = obj['enum']
-        if 'version_added' in obj:
-            result['version_added'] = obj['version_added']
-        result['required'] = required
-        result['type'] = obj['type']
-    return result
+    doc = yaml.safe_load(spec['DOCUMENTATION'])
+
+    dive(doc['options'], result)
 
-def to_argspec(value):
-    data = jsonref.loads(json.dumps(value))
-    result = dive(data['schema'])
-    return str(result['options'])
+    result = pprint.pformat(result, indent=1)
+    display.debug("Arguments: %s" % result)
+    return result
 
 
 class FilterModule(object):

roles/resource_module/filter_plugins/to_doc.py

@@ -0,0 +1,170 @@
+# Copyright (c) 2019 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import os
+import shutil
+import json
+from subprocess import Popen, PIPE
+
+from copy import deepcopy
+
+from ansible.module_utils.six import StringIO, string_types
+from ansible.errors import AnsibleError, AnsibleFilterError
+from ansible.utils.display import Display
+from ansible.utils.path import unfrackpath, makedirs_safe
+
+display = Display()
+
+SECTIONS = ('ANSIBLE_METADATA', 'DOCUMENTATION', 'EXAMPLES', 'RETURN')
+
+DOC_SECTION_SANITIZE = ('mutually_exclusive', 'required_together', 'required_one_of', 'supports_check_mode', 'required_if')
+
+DEFAULT_RETURN = """
+before:
+  description: The configuration prior to the model invocation.
+  returned: always
+  sample: The configuration returned will always be in the same format of the parameters above.
+after:
+  description: The resulting configuration model invocation.
+  returned: when changed
+  sample: The configuration returned will always be in the same format of the parameters above.
+commands:
+  description: The set of commands pushed to the remote device.
+  returned: always
+  type: list
+  sample: ['command 1', 'command 2', 'command 3']
+"""
+
+output = StringIO()
+RM_DIR_PATH = "~/.ansible/tmp/resource_model"
+
+
+def to_list(val):
+    if isinstance(val, (list, tuple, set)):
+        return list(val)
+    elif val is not None:
+        return [val]
+    else:
+        return list()
+
+
+def add(line, spaces=0, newline=True):
+    line = line.rjust(len(line)+spaces, ' ')
+    if newline:
+        output.write(line + '\n')
+    else:
+        output.write(line)
+
+
+def get_ansible_metadata(spec, path):
+    # write ansible metadata
+    if 'ANSIBLE_METADATA' not in spec:
+        raise AnsibleFilterError("missing required element 'ANSIBLE_METADATA' in model")
+
+    metadata = spec['ANSIBLE_METADATA']
+    if not isinstance(metadata, string_types):
+        raise AnsibleFilterError("value of element 'ANSIBLE_METADATA' should be of type string")
+
+    add('ANSIBLE_METADATA = %s' % metadata, newline=True)
+    #add(metadata)
+
+
+def get_documentation(spec, path):
+    # write documentation
+    if 'DOCUMENTATION' not in spec:
+        raise AnsibleFilterError("missing required element 'DOCUMENTATION' in model")
+
+    doc = spec['DOCUMENTATION']
+    if not isinstance(doc, string_types):
+        raise AnsibleFilterError("value of element 'DOCUMENTATION' should be of type string")
+
+    add('DOCUMENTATION = """')
+    add('---')
+    add('%s' % doc)
+    add('"""')
+
+
+def get_examples(spec, path):
+    # write examples
+    if 'EXAMPLES' not in spec:
+        raise AnsibleFilterError("missing required element 'EXAMPLES' in model")
+
+    add('EXAMPLES = """')
+    dir_name = os.path.dirname(path)
+    for item in to_list(spec['EXAMPLES']):
+        with open(os.path.join(dir_name, item)) as fp:
+            add(fp.read().strip("\n"))
+        add("\n")
+    add('"""')
+
+
+def get_return(spec, path):
+    # write return
+    ret = spec.get('RETURN')
+    add('RETURN = """')
+    add(ret) if ret else add(DEFAULT_RETURN.strip())
+    add('"""')
+
+
+def validate_model(model, contents):
+    try:
+        resource_module_dir = unfrackpath(RM_DIR_PATH)
+        makedirs_safe(resource_module_dir)
+        module_name = "%s_%s" % (model['NETWORK_OS'], model['RESOURCE'])
+        module_file_path = os.path.join(RM_DIR_PATH, '%s.%s' % (module_name, 'py'))
+        module_file_path = os.path.realpath(os.path.expanduser(module_file_path))
+        with open(module_file_path, 'w+') as fp:
+            fp.write(contents)
+
+        display.debug("Module file: %s" % module_file_path)
+
+        # validate the model
+        cmd = ["ansible-doc", "-M", os.path.dirname(module_file_path), module_name]
+        p = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
+        out, err = p.communicate()
+        if err:
+            raise AnsibleError("Error while parsing module: %s" % err)
+        display.debug("Module output:\n%s" % out)
+    except Exception as e:
+        raise AnsibleError('Failed to validate the model with error: %s\n%s' % (e, contents))
+    finally:
+        shutil.rmtree(os.path.realpath(os.path.expanduser(resource_module_dir)), ignore_errors=True)
+
+
+def _sanitize_documentation(doc):
+    sanitize_doc =[]
+    for line in doc.splitlines():
+        for item in DOC_SECTION_SANITIZE:
+            if line.strip().startswith(item):
+                break
+        else:
+            sanitize_doc.append(line)
+    return "\n".join(sanitize_doc)
+
+
+def to_doc(rm, path):
+    model = deepcopy(rm)
+    model['DOCUMENTATION'] = _sanitize_documentation(rm['DOCUMENTATION'])
+    path = os.path.realpath(os.path.expanduser(path))
+    if not os.path.isfile(path):
+        raise AnsibleFilterError("model file %s does not exist" % path)
+
+    for name in SECTIONS:
+        func = globals().get('get_%s' % name.lower())
+        func(model, path)
+
+    contents = output.getvalue()
+    display.debug("%s" % contents)
+    validate_model(model, contents)
+
+    return contents
+
+
+class FilterModule(object):
+    def filters(self):
+        return {
+            'to_doc': to_doc,
+        }

roles/resource_module/filter_plugins/to_docoptions.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
-# {{ rm['metadata']['copyright_str'] }}
+# {{ rm['COPYRIGHT'] }}
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 """
 The module file for {{ network_os }}_facts
@@ -12,23 +12,23 @@ from ansible.module_utils.connection import Connection
 from {{ import_path }}. \
      {{ network_os }}.facts.facts import Facts
 
-ANSIBLE_METADATA = {'metadata_version': '1.1',
-                    'status': {{ rm['metadata']['status'] }},
-                    'supported_by': {{ rm['metadata']['supported_by'] }}}
+ANSIBLE_METADATA = {'metadata_version': '{{rm_ansible_metadata['metadata_version']}}',
+                    'status': {{rm_ansible_metadata['status']}},
+                    'supported_by': '{{rm_ansible_metadata['supported_by']}}',}
 
 
 DOCUMENTATION = """
 ---
 module: {{ network_os }}_facts
-version_added: {{ rm['info']['version_added'] }}
+version_added: {{ rm_docmentation['version_added'] }}
 short_description: Get facts about {{ network_os }} devices.
 description:
   - Collects facts from network devices running the {{ network_os }} operating
     system. This module places the facts gathered in the fact tree keyed by the
     respective resource name.  The facts module will always collect a
     base set of facts from the device and can enable or disable
     collection of additional facts.
-author: {{ rm['info']['authors'] }}
+author: {{ rm_docmentation['author'] }}
 options:
   gather_subset:
     description: