qa/tasks: remove generic template stuff from cephadm.py

Remove the generic template stuff from cephadm.py now that the
template.py file is usable and various yamls have been updated to
use the new module now.

Signed-off-by: John Mulligan <[email protected]>

Author: phlogistonjohn

qa/tasks/cephadm.py

@@ -4,7 +4,6 @@
 import argparse
 import configobj
 import contextlib
-import functools
 import json
 import logging
 import os
@@ -13,8 +12,6 @@
 import uuid
 import yaml
 
-import jinja2
-
 from copy import deepcopy
 from io import BytesIO, StringIO
 from tarfile import ReadError
@@ -30,6 +27,7 @@
 from tasks.cephfs.filesystem import MDSCluster, Filesystem
 from tasks.daemonwatchdog import DaemonWatchdog
 from tasks.util import chacra
+from tasks import template
 
 # these items we use from ceph.py should probably eventually move elsewhere
 from tasks.ceph import get_mons, healthy
@@ -39,73 +37,6 @@
 log = logging.getLogger(__name__)
 
 
-def _convert_strs_in(o, conv):
-    """A function to walk the contents of a dict/list and recurisvely apply
-    a conversion function (`conv`) to the strings within.
-    """
-    if isinstance(o, str):
-        return conv(o)
-    if isinstance(o, dict):
-        for k in o:
-            o[k] = _convert_strs_in(o[k], conv)
-    if isinstance(o, list):
-        o[:] = [_convert_strs_in(v, conv) for v in o]
-    return o
-
-
-def _apply_template(jinja_env, rctx, template):
-    """Apply jinja2 templating to the template string `template` via the jinja
-    environment `jinja_env`, passing a dictionary containing top-level context
-    to render into the template.
-    """
-    if '{{' in template or '{%' in template:
-        return jinja_env.from_string(template).render(**rctx)
-    return template
-
-
-def _template_transform(ctx, config, target):
-    """Apply jinja2 based templates to strings within the target object,
-    returning a transformed target. Target objects may be a list or dict or
-    str.
-
-    Note that only string values in the list or dict objects are modified.
-    Therefore one can read & parse yaml or json that contain templates in
-    string values without the risk of changing the structure of the yaml/json.
-    """
-    jenv = getattr(ctx, '_jinja_env', None)
-    if jenv is None:
-        loader = jinja2.BaseLoader()
-        jenv = jinja2.Environment(loader=loader)
-        jenv.filters['role_to_remote'] = _role_to_remote
-        setattr(ctx, '_jinja_env', jenv)
-    rctx = dict(ctx=ctx, config=config, cluster_name=config.get('cluster', ''))
-    _vip_vars(rctx)
-    conv = functools.partial(_apply_template, jenv, rctx)
-    return _convert_strs_in(target, conv)
-
-
-def _vip_vars(rctx):
-    """For backwards compat with the previous subst_vip function."""
-    ctx = rctx['ctx']
-    if 'vnet' in getattr(ctx, 'vip', {}):
-        rctx['VIPPREFIXLEN'] = str(ctx.vip["vnet"].prefixlen)
-        rctx['VIPSUBNET'] = str(ctx.vip["vnet"].network_address)
-    if 'vips' in getattr(ctx, 'vip', {}):
-        vips = ctx.vip['vips']
-        for idx, vip in enumerate(vips):
-            rctx[f'VIP{idx}'] = str(vip)
-
-
-@jinja2.pass_context
-def _role_to_remote(rctx, role):
-    """Return the first remote matching the given role."""
-    ctx = rctx['ctx']
-    for remote, roles in ctx.cluster.remotes.items():
-        if role in roles:
-            return remote
-    return None
-
-
 def _shell(ctx, cluster_name, remote, args, extra_cephadm_args=[], **kwargs):
     teuthology.get_testdir(ctx)
     return remote.run(
@@ -1513,22 +1444,6 @@ def stop(ctx, config):
     yield
 
 
-def _expand_roles(ctx, config):
-    if 'all-roles' in config and len(config) == 1:
-        a = config['all-roles']
-        roles = teuthology.all_roles(ctx.cluster)
-        config = dict((id_, a) for id_ in roles if not id_.startswith('host.'))
-    elif 'all-hosts' in config and len(config) == 1:
-        a = config['all-hosts']
-        roles = teuthology.all_roles(ctx.cluster)
-        config = dict((id_, a) for id_ in roles if id_.startswith('host.'))
-    elif 'all-roles' in config or 'all-hosts' in config:
-        raise ValueError(
-            'all-roles/all-hosts may not be combined with any other roles'
-        )
-    return config
-
-
 def shell(ctx, config):
     """
     Execute (shell) commands
@@ -1541,8 +1456,8 @@ def shell(ctx, config):
     for k in config.pop('volumes', []):
         args.extend(['-v', k])
 
-    config = _expand_roles(ctx, config)
-    config = _template_transform(ctx, config, config)
+    config = template.expand_roles(ctx, config)
+    config = template.transform(ctx, config, config)
     for role, cmd in config.items():
         (remote,) = ctx.cluster.only(role).remotes.keys()
         log.info('Running commands on role %s host %s', role, remote.name)
@@ -1575,31 +1490,6 @@ def _shell_command(obj):
     raise ValueError(f'invalid command item: {obj!r}')
 
 
-def exec(ctx, config):
-    """
-    This is similar to the standard 'exec' task, but does template substitutions.
-
-    TODO: this should probably be moved out of cephadm.py as it's pretty generic.
-    """
-    assert isinstance(config, dict), "task exec got invalid config"
-    testdir = teuthology.get_testdir(ctx)
-    config = _expand_roles(ctx, config)
-    for role, ls in config.items():
-        (remote,) = ctx.cluster.only(role).remotes.keys()
-        log.info('Running commands on role %s host %s', role, remote.name)
-        for c in ls:
-            c.replace('$TESTDIR', testdir)
-            remote.run(
-                args=[
-                    'sudo',
-                    'TESTDIR={tdir}'.format(tdir=testdir),
-                    'bash',
-                    '-ex',
-                    '-c',
-                    _template_transform(ctx, config, c)],
-                )
-
-
 def apply(ctx, config):
     """
     Apply spec
@@ -1622,7 +1512,7 @@ def apply(ctx, config):
     cluster_name = config.get('cluster', 'ceph')
 
     specs = config.get('specs', [])
-    specs = _template_transform(ctx, config, specs)
+    specs = template.transform(ctx, config, specs)
     y = yaml.dump_all(specs)
 
     log.info(f'Applying spec(s):\n{y}')