mgr/rgw: Adding a retry config while calling zone_create()

Fixes https://tracker.ceph.com/issues/66750

Signed-off-by: Kritik Sachdeva <[email protected]>

Author: Kritik Sachdeva

src/pybind/mgr/rgw/module.py

@@ -101,7 +101,14 @@ def wrapper(self: Any, *args: Any, **kwargs: Any) -> HandleCommandResult:
 
 
 class Module(orchestrator.OrchestratorClientMixin, MgrModule):
-    MODULE_OPTIONS: List[Option] = []
+    MODULE_OPTIONS: List[Option] = [
+        Option(
+            'secondary_zone_period_retry_limit',
+            type='int',
+            default=5,
+            desc='RGW module period update retry limit for secondary site'
+        ),
+    ]
 
     # These are "native" Ceph options that this module cares about.
     NATIVE_OPTIONS: List[Option] = []
@@ -115,6 +122,9 @@ def __init__(self, *args: Any, **kwargs: Any):
         # ensure config options members are initialized; see config_notify()
         self.config_notify()
 
+        if TYPE_CHECKING:
+            self.secondary_zone_period_retry_limit = 5
+
         with self.lock:
             self.inited = True
             self.env = EnvArgs(RGWAMOrchMgr(self))
@@ -312,7 +322,7 @@ def _cmd_rgw_zone_create(self,
 
         try:
             created_zones = self.rgw_zone_create(zone_name, realm_token, port, placement,
-                                                 start_radosgw, zone_endpoints, inbuf)
+                                                 start_radosgw, zone_endpoints, self.secondary_zone_period_retry_limit, inbuf)
             return HandleCommandResult(retval=0, stdout=f"Zones {', '.join(created_zones)} created successfully")
         except RGWAMException as e:
             return HandleCommandResult(retval=e.retcode, stderr=f'Failed to create zone: {str(e)}')
@@ -324,6 +334,7 @@ def rgw_zone_create(self,
                         placement: Optional[Union[str, Dict[str, Any]]] = None,
                         start_radosgw: Optional[bool] = True,
                         zone_endpoints: Optional[str] = None,
+                        secondary_zone_period_retry_limit: Optional[int] = None,
                         inbuf: Optional[str] = None) -> List[str]:
 
         if inbuf:
@@ -350,7 +361,7 @@ def rgw_zone_create(self,
         try:
             created_zones = []
             for rgw_spec in rgw_specs:
-                RGWAM(self.env).zone_create(rgw_spec, start_radosgw)
+                RGWAM(self.env).zone_create(rgw_spec, start_radosgw, secondary_zone_period_retry_limit)
                 if rgw_spec.rgw_zone is not None:
                     created_zones.append(rgw_spec.rgw_zone)
                     return created_zones

src/python-common/ceph/rgw/rgwam_core.py

@@ -786,7 +786,7 @@ def get_realms_info(self):
                                 "secret": secret})
         return realms_info
 
-    def zone_create(self, rgw_spec, start_radosgw):
+    def zone_create(self, rgw_spec, start_radosgw, secondary_zone_period_retry_limit=5):
 
         if not rgw_spec.rgw_realm_token:
             raise RGWAMException('missing realm token')
@@ -823,7 +823,25 @@ def zone_create(self, rgw_spec, start_radosgw):
         zone = self.create_zone(realm, zonegroup, rgw_spec.rgw_zone,
                                 False,  # secondary zone
                                 access_key, secret, endpoints=rgw_spec.zone_endpoints)
-        self.update_period(realm, zonegroup, zone)
+
+        # Adding a retry limit for period update in case the default 10s timeout is not sufficient
+        rgw_limit = 0
+
+        while rgw_limit != int(secondary_zone_period_retry_limit):
+            try:
+                self.update_period(realm, zonegroup, zone)
+                break
+            except RGWAMException as e:
+                logging.info(f'Failed to update Period in 10s. Retrying with current limit \
+                             & retry-limit values {rgw_limit} {secondary_zone_period_retry_limit}')
+                rgw_limit += 1
+                if rgw_limit == secondary_zone_period_retry_limit:
+                    raise RGWAMException(f'Period Update failed for zone {zone}. \
+                                          Exception raised while period update {e.message}')
+                continue
+
+        # By default the above operation is expected to be completed in 10s timeout but if we
+        # updating this for secondary site it would take some time because of pool creation
 
         period = RGWPeriod(period_info)
         logging.debug(period.to_json())