test/rgw/dedup: add dedup test suite to teuthology

yuvalif · yuvalif · commit 895121fb3d31 · 2025-05-23T13:26:09.000Z
Signed-off-by: Yuval Lifshitz &lt;ylifshit@ibm.com&gt;
diff --git a/qa/suites/rgw/dedup/% b/qa/suites/rgw/dedup/%
diff --git a/qa/suites/rgw/dedup/.qa b/qa/suites/rgw/dedup/.qa
@@ -0,0 +1 @@
+../.qa/
diff --git a/qa/suites/rgw/dedup/beast.yaml b/qa/suites/rgw/dedup/beast.yaml
@@ -0,0 +1 @@
+.qa/rgw_frontend/beast.yaml
diff --git a/qa/suites/rgw/dedup/bluestore-bitmap.yaml b/qa/suites/rgw/dedup/bluestore-bitmap.yaml
@@ -0,0 +1 @@
+.qa/objectstore/bluestore-bitmap.yaml
diff --git a/qa/suites/rgw/dedup/fixed-3-rgw.yaml b/qa/suites/rgw/dedup/fixed-3-rgw.yaml
@@ -0,0 +1,13 @@
+roles:
+- [mon.a, mon.c, mgr.y, osd.0, osd.1, osd.2, osd.3, client.0]
+- [mon.b, mgr.x, osd.4, osd.5, osd.6, osd.7, client.1]
+- [client.2]
+openstack:
+- volumes: # attached to each instance
+    count: 4
+    size: 10 # GB
+overrides:
+  ceph:
+    conf:
+      osd:
+        osd shutdown pgref assert: true
diff --git a/qa/suites/rgw/dedup/ignore-pg-availability.yaml b/qa/suites/rgw/dedup/ignore-pg-availability.yaml
@@ -0,0 +1 @@
+.qa/rgw/ignore-pg-availability.yaml
diff --git a/qa/suites/rgw/dedup/overrides.yaml b/qa/suites/rgw/dedup/overrides.yaml
@@ -0,0 +1,12 @@
+overrides:
+  ceph:
+    conf:
+      client:
+        setuser: ceph
+        setgroup: ceph
+        debug rgw: 20
+        debug rgw dedup: 20
+  rgw:
+    storage classes:
+      LUKEWARM:
+      FROZEN:
diff --git a/qa/suites/rgw/dedup/supported-distros b/qa/suites/rgw/dedup/supported-distros
@@ -0,0 +1 @@
+.qa/distros/supported-random-distro$/
diff --git a/qa/suites/rgw/dedup/tasks/+ b/qa/suites/rgw/dedup/tasks/+
diff --git a/qa/suites/rgw/dedup/tasks/.qa b/qa/suites/rgw/dedup/tasks/.qa
@@ -0,0 +1 @@
+../.qa/
diff --git a/qa/suites/rgw/dedup/tasks/0-install.yaml b/qa/suites/rgw/dedup/tasks/0-install.yaml
@@ -0,0 +1,13 @@
+tasks:
+- install:
+- ceph:
+- openssl_keys:
+- rgw: [client.0, client.1, client.2]
+- tox: [client.0]
+
+overrides:
+  ceph:
+    conf:
+      global:
+        osd_min_pg_log_entries: 10
+        osd_max_pg_log_entries: 10
diff --git a/qa/suites/rgw/dedup/tasks/test_dedup.yaml b/qa/suites/rgw/dedup/tasks/test_dedup.yaml
@@ -0,0 +1,5 @@
+tasks:
+- tox: [client.0]
+- dedup-tests:
+    client.0:
+      rgw_server: client.0
diff --git a/qa/tasks/dedup_tests.py b/qa/tasks/dedup_tests.py
@@ -0,0 +1,254 @@
+"""
+Run a set of dedup tests on rgw.
+"""
+from io import BytesIO
+from configobj import ConfigObj
+import base64
+import contextlib
+import logging
+import os
+import random
+import string
+
+from teuthology import misc as teuthology
+from teuthology import contextutil
+from teuthology.orchestra import run
+
+log = logging.getLogger(__name__)
+
+@contextlib.contextmanager
+def download(ctx, config):
+    assert isinstance(config, dict)
+    log.info('Downloading dedup-tests...')
+    testdir = teuthology.get_testdir(ctx)
+    branch = ctx.config.get('suite_branch')
+    repo = ctx.config.get('suite_repo')
+    log.info('Using branch %s from %s for dedup tests', branch, repo)
+    for (client, client_config) in config.items():
+
+        ctx.cluster.only(client).run(
+            args=['git', 'clone', '-b', branch, repo, '{tdir}/ceph'.format(tdir=testdir)],
+            )
+
+        sha1 = client_config.get('sha1')
+
+        if sha1 is not None:
+            ctx.cluster.only(client).run(
+                args=[
+                    'cd', '{tdir}/ceph'.format(tdir=testdir),
+                    run.Raw('&&'),
+                    'git', 'reset', '--hard', sha1,
+                    ],
+                )
+
+    try:
+        yield
+    finally:
+        log.info('Removing dedup-tests...')
+        testdir = teuthology.get_testdir(ctx)
+        for client in config:
+            ctx.cluster.only(client).run(
+                args=[
+                    'rm',
+                    '-rf',
+                    '{tdir}/ceph'.format(tdir=testdir),
+                    ],
+                )
+
+
+def _config_user(deduptests_conf, section, user):
+    """
+    Configure users for this section by stashing away keys, ids, and
+    email addresses.
+    """
+    deduptests_conf[section].setdefault('user_id', user)
+    deduptests_conf[section].setdefault('email', '{user}+test@test.test'.format(user=user))
+    deduptests_conf[section].setdefault('display_name', 'Mr. {user}'.format(user=user))
+    deduptests_conf[section].setdefault('access_key',
+        ''.join(random.choice(string.ascii_uppercase) for i in range(20)))
+    deduptests_conf[section].setdefault('secret_key',
+        base64.b64encode(os.urandom(40)).decode())
+
+
+@contextlib.contextmanager
+def create_users(ctx, config):
+    """
+    Create a main and an alternate s3 user.
+    """
+    assert isinstance(config, dict)
+    log.info('Creating rgw user...')
+    testdir = teuthology.get_testdir(ctx)
+
+    users = {'s3 main': 'foo'}
+    for client in config['clients']:
+        deduptests_conf = config['deduptests_conf'][client]
+        for section, user in users.items():
+            _config_user(deduptests_conf, section, '{user}.{client}'.format(user=user, client=client))
+            log.debug('Creating user {user} on {host}'.format(user=deduptests_conf[section]['user_id'], host=client))
+            cluster_name, daemon_type, client_id = teuthology.split_role(client)
+            client_with_id = daemon_type + '.' + client_id
+            ctx.cluster.only(client).run(
+                args=[
+                    'adjust-ulimits',
+                    'ceph-coverage',
+                    '{tdir}/archive/coverage'.format(tdir=testdir),
+                    'radosgw-admin',
+                    '-n', client_with_id,
+                    'user', 'create',
+                    '--uid', deduptests_conf[section]['user_id'],
+                    '--display-name', deduptests_conf[section]['display_name'],
+                    '--access-key', deduptests_conf[section]['access_key'],
+                    '--secret', deduptests_conf[section]['secret_key'],
+                    '--cluster', cluster_name,
+                    ],
+                )
+
+    try:
+        yield
+    finally:
+        for client in config['clients']:
+            for user in users.values():
+                uid = '{user}.{client}'.format(user=user, client=client)
+                cluster_name, daemon_type, client_id = teuthology.split_role(client)
+                client_with_id = daemon_type + '.' + client_id
+                ctx.cluster.only(client).run(
+                    args=[
+                        'adjust-ulimits',
+                        'ceph-coverage',
+                        '{tdir}/archive/coverage'.format(tdir=testdir),
+                        'radosgw-admin',
+                        '-n', client_with_id,
+                        'user', 'rm',
+                        '--uid', uid,
+                        '--purge-data',
+                        '--cluster', cluster_name,
+                        ],
+                    )
+
+
+@contextlib.contextmanager
+def configure(ctx, config):
+    assert isinstance(config, dict)
+    log.info('Configuring dedup-tests...')
+    testdir = teuthology.get_testdir(ctx)
+    for client, properties in config['clients'].items():
+        (remote,) = ctx.cluster.only(client).remotes.keys()
+        deduptests_conf = config['deduptests_conf'][client]
+
+        conf_fp = BytesIO()
+        deduptests_conf.write(conf_fp)
+        remote.write_file(
+            path='{tdir}/ceph/src/test/rgw/dedup/deduptests.{client}.conf'.format(tdir=testdir, client=client),
+            data=conf_fp.getvalue(),
+            )
+
+    try:
+        yield
+    finally:
+        log.info('Removing dedup-tests.conf file...')
+        testdir = teuthology.get_testdir(ctx)
+        for client, properties in config['clients'].items():
+            (remote,) = ctx.cluster.only(client).remotes.keys()
+            remote.run(
+                 args=['rm', '-f',
+                       '{tdir}/ceph/src/test/rgw/dedup/deduptests.{client}.conf'.format(tdir=testdir,client=client),
+                 ],
+                 )
+
+
+def get_toxvenv_dir(ctx):
+    return ctx.tox.venv_path
+
+
+def toxvenv_sh(ctx, remote, args, **kwargs):
+    activate = get_toxvenv_dir(ctx) + '/bin/activate'
+    return remote.sh(['source', activate, run.Raw('&&')] + args, **kwargs)
+
+
+@contextlib.contextmanager
+def run_tests(ctx, config):
+    """
+    Run the dedup tests after everything is set up.
+    :param ctx: Context passed to task
+    :param config: specific configuration information
+    """
+    assert isinstance(config, dict)
+    log.info('Running dedup-tests...')
+    testdir = teuthology.get_testdir(ctx)
+    for client, client_config in config.items():
+        (remote,) = ctx.cluster.only(client).remotes.keys()
+
+        # test marks to use by default
+        attr = ['basic_test', 'request_test', 'example_test']
+
+        if 'extra_attr' in client_config:
+            attr = client_config.get('extra_attr')
+
+        args = ['cd', '{tdir}/ceph/src/test/rgw/dedup/'.format(tdir=testdir), run.Raw('&&'),
+            'DEDUPTESTS_CONF=./deduptests.{client}.conf'.format(client=client),
+            'tox', '--', '-v', '-m', ' or '.join(attr)]
+
+        toxvenv_sh(ctx, remote, args, label="dedup tests against rgw")
+
+    yield
+
+
+@contextlib.contextmanager
+def task(ctx,config):
+    """
+
+    If you want to run the tests against your changes pushed to your remote repo you can provide 'suite_branch' and 'suite_repo'
+    parameters in your teuthology-suite command. Example command for this is as follows::
+
+    teuthology-suite --ceph-repo https://github.com/ceph/ceph-ci.git -s rgw:dedup --ceph your_ceph_branch_name -m smithi --suite-repo https://github.com/your_name/ceph.git --suite-branch your_branch_name
+
+    """
+    assert hasattr(ctx, 'rgw'), 's3tests must run after the rgw task'
+    assert hasattr(ctx, 'tox'), 's3tests must run after the tox task'
+    assert config is None or isinstance(config, list) \
+        or isinstance(config, dict), \
+        "task only supports a list or dictionary for configuration"
+
+    all_clients = ['client.{id}'.format(id=id_)
+                   for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
+    if config is None:
+        config = all_clients
+    if isinstance(config, list):
+        config = dict.fromkeys(config)
+    clients=config.keys()
+
+    log.debug('config is %s', config)
+
+    deduptests_conf = {}
+
+    for client in clients:
+        endpoint = ctx.rgw.role_endpoints.get(client)
+        assert endpoint, 'deduptests: no rgw endpoint for {}'.format(client)
+
+        deduptests_conf[client] = ConfigObj(
+            indent_type='',
+            infile={
+                'DEFAULT':
+                    {
+                    'port':endpoint.port,
+                    'host':endpoint.dns_name,
+                    },
+                's3 main':{}
+            }
+        )
+
+    with contextutil.nested(
+        lambda: download(ctx=ctx, config=config),
+        lambda: create_users(ctx=ctx, config=dict(
+                clients=clients,
+                deduptests_conf=deduptests_conf,
+                )),
+        lambda: configure(ctx=ctx, config=dict(
+                clients=config,
+                deduptests_conf=deduptests_conf,
+                )),
+        lambda: run_tests(ctx=ctx, config=config),
+        ):
+        pass
+    yield
+
