Merge pull request ceph#61930 from thotz/s3-tests-verify-restore

qa/rgw: add new teuthology for cloud restore s3 tests

 Reviewed-by:  Casey Bodley <[email protected]>
 Reviewed-by:  Adam Emerson <[email protected]>
 Reviewed-by:  Soumya Koduri <[email protected]>

Author: soumyakoduri

qa/suites/rgw/cloud-transition/+ qa/suites/rgw/cloud-transition/%qa/suites/rgw/cloud-transition/+ renamed to qa/suites/rgw/cloud-transition/%

@@ -11,6 +11,7 @@ overrides:
         rgw sts key: abcdefghijklmnop
         rgw s3 auth use sts: true
         rgw lc debug interval: 10
+        rgw_restore_debug_interval: 20
   rgw:
     storage classes: LUKEWARM, FROZEN
     frontend: beast

qa/suites/rgw/cloud-transition/overrides.yaml

@@ -0,0 +1,33 @@
+tasks:
+- install:
+- ceph:
+- rgw:
+    storage classes: LUKEWARM, FROZEN
+    client.0:
+      port: 8000
+    client.1:
+      port: 8001
+- rgw-cloudtier:
+    client.0:
+      cloud_storage_class: CLOUDTIER-CLIENT0
+      cloud_client: client.1
+      cloud_regular_storage_class: LUKEWARM
+      cloud_target_storage_class: FROZEN
+      cloud_retain_head_object: "true"
+      cloud_target_path: "teuthology-client0"
+      cloud_allow_read_through: "true"
+      cloud_read_through_restore_days: "1"
+      cloudtier_user:
+        # cloud-user creds to be created on cloud-client
+        cloud_secret: "abcefgh"
+        cloud_access_key: "12345678"
+- tox: [client.0]
+- s3tests:
+    client.0:
+      rgw_server: client.0
+      storage classes: LUKEWARM, FROZEN
+      extra_attrs: ["cloud_restore"]
+      lc_debug_interval: 10
+      rgw_restore_debug_interval: 20
+      lifecycle_tests: True
+      cloudtier_tests: True

qa/suites/rgw/cloud-transition/tasks/restore/cloud_restore_s3tests.yaml

@@ -28,6 +28,8 @@ class RGWCloudTier(Task):
               cloud_target_storage_class:
               cloud_retain_head_object:
               cloud_target_path:
+              cloud_allow_read_through:
+              cloud_read_through_restore_days:
               cloudtier_user:
                 cloud_secret:
                 cloud_access_key:
@@ -64,6 +66,8 @@ def setup(self):
                 cloud_target_path = client_config.get('cloud_target_path')
                 cloud_target_storage_class = client_config.get('cloud_target_storage_class')
                 cloud_retain_head_object = client_config.get('cloud_retain_head_object')
+                cloud_allow_read_through = client_config.get('cloud_allow_read_through')
+                cloud_read_through_restore_days = client_config.get('cloud_read_through_restore_days')
 
                 cloudtier_user = client_config.get('cloudtier_user')
                 cloud_access_key = cloudtier_user.get('cloud_access_key')
@@ -85,6 +89,10 @@ def setup(self):
                     tier_config_params += ",target_path=" + cloud_target_path
                 if (cloud_target_storage_class != None):
                     tier_config_params += ",target_storage_class=" + cloud_target_storage_class
+                if (cloud_allow_read_through != None):
+                    tier_config_params += ",allow_read_through=" + cloud_allow_read_through
+                if (cloud_read_through_restore_days != None):
+                    tier_config_params += ",read_through_restore_days=" + cloud_read_through_restore_days
 
                 log.info('Configuring cloud-s3 tier storage class type = %s', cloud_storage_class)
 

…tion/tasks/cloud_transition_s3tests.yaml …transition/cloud_transition_s3tests.yamlqa/suites/rgw/cloud-transition/tasks/cloud_transition_s3tests.yaml renamed to qa/suites/rgw/cloud-transition/tasks/transition/cloud_transition_s3tests.yaml qa/suites/rgw/cloud-transition/tasks/cloud_transition_s3tests.yaml renamed to qa/suites/rgw/cloud-transition/tasks/transition/cloud_transition_s3tests.yaml

@@ -368,6 +368,10 @@ def configure(ctx, config):
         if lc_debug_interval:
             s3tests_conf['s3 main']['lc_debug_interval'] = lc_debug_interval
 
+        rgw_restore_debug_interval = properties.get('rgw_restore_debug_interval')
+        if rgw_restore_debug_interval:
+            s3tests_conf['s3 main']['rgw_restore_debug_interval'] = rgw_restore_debug_interval
+
         if ctx.rgw_cloudtier is not None:
             log.info(' ctx.rgw_cloudtier config  is %s ...', ctx.rgw_cloudtier.config)
             client_rgw_config = ctx.rgw_cloudtier.config.get(client)
@@ -389,6 +393,12 @@ def configure(ctx, config):
                     s3tests_conf['s3 cloud']['target_path'] = cloud_target_path
                 if (cloud_target_storage_class != None):
                     s3tests_conf['s3 cloud']['target_storage_class'] = cloud_target_storage_class
+                cloud_allow_read_through = client_rgw_config.get('cloud_allow_read_through')
+                if (cloud_allow_read_through != None):
+                    s3tests_conf['s3 cloud']['allow_read_through'] = cloud_allow_read_through
+                cloud_read_through_restore_days = client_rgw_config.get('cloud_read_through_restore_days')
+                if (cloud_read_through_restore_days != None):
+                    s3tests_conf['s3 cloud']['read_through_restore_days'] = cloud_read_through_restore_days
 
         (remote,) = ctx.cluster.only(client).remotes.keys()
         conf_fp = BytesIO()