Log event queue exception after max attempts (#1263)

* Log event queue exception after max attempts

* address comments

Author: HaijingFu

axlearn/cloud/common/event_queue.py

@@ -203,7 +203,6 @@ def publish(self, event: Event):
             try:
                 # Ensure connection is established before publishing.
                 if not self._channel or not self._connection:
-                    logging.error("RabbitMQ publisher channel is closed, reconnecting...")
                     self.connect()
 
                 # Setting durable=True ensures that the queue will survive.
@@ -230,25 +229,30 @@ def publish(self, event: Event):
                 # Only retry on recoverable exceptions.
                 # AMQPConnectionError is assumed to be related to network issues,
                 # or temporary unavailable host.
-                logging.error(
-                    "Failed to publish event: %s. Error: %s. Attempt: %d",
-                    message,
-                    str(e),
-                    attempt,
-                )
                 self._handle_publish_error()
                 attempt += 1
-                if attempt <= self._num_tries:
+                if attempt < self._num_tries:
                     time.sleep(2**attempt)
+                else:
+                    logging.error(
+                        "Failed to publish event: %s after %d attempts. Error: %s.",
+                        message,
+                        attempt,
+                        str(e),
+                    )
             except Exception as e:  # pylint: disable=broad-except
-                # Unknown errors. Don't retry. Log to avoid crashing clients.
-                logging.error(
-                    "Unknown error. Failed to publish event: %s. Error: %s.", message, str(e)
-                )
                 self._handle_publish_error()
                 attempt += 1
-                if attempt <= self._num_tries:
+                if attempt < self._num_tries:
                     time.sleep(2**attempt)
+                else:
+                    # Unknown errors. Don't retry. Log to avoid crashing clients.
+                    logging.error(
+                        "Unknown error. Failed to publish event: %s after %d attempts. Error: %s.",
+                        message,
+                        attempt,
+                        str(e),
+                    )
 
     def _handle_publish_error(self):
         """Handle publish errors with retrying on connection issue."""