added attempts

Author: Andrei Neagu

packages/models-library/src/models_library/progress_bar.py

@@ -2,14 +2,12 @@
 
 from pydantic import BaseModel, ConfigDict
 
-from .basic_types import IDStr
-
 # NOTE: keep a list of possible unit, and please use correct official unit names
 ProgressUnit: TypeAlias = Literal["Byte"]
 
 
 class ProgressStructuredMessage(BaseModel):
-    description: IDStr
+    description: str
     current: float
     total: int
     unit: str | None = None
@@ -51,6 +49,7 @@ class ProgressStructuredMessage(BaseModel):
 class ProgressReport(BaseModel):
     actual_value: float
     total: float = 1.0
+    attempt: int = 0
     unit: ProgressUnit | None = UNITLESS
     message: ProgressStructuredMessage | None = None
 

packages/service-library/src/servicelib/docker_utils.py

@@ -22,7 +22,7 @@
 )
 from settings_library.docker_registry import RegistrySettings
 from tenacity import (
-    AsyncRetrying,
+    retry,
     retry_if_exception_type,
     stop_after_attempt,
     wait_random_exponential,
@@ -237,6 +237,7 @@ async def pull_image(
         image_information -- the image layer information. If this is None, then no fine progress will be retrieved.
         retry_upon_error_count -- number of tries if there is a TimeoutError. Usually cased by networking issues.
     """
+
     registry_auth = None
     if registry_settings.REGISTRY_URL and registry_settings.REGISTRY_URL in image:
         registry_auth = {
@@ -262,56 +263,59 @@ async def pull_image(
 
         client = await exit_stack.enter_async_context(aiodocker.Docker())
 
-        async for attempt in AsyncRetrying(
+        def _reset_progress_from_previous_attempt() -> None:
+            for pulled_status in layer_id_to_size.values():
+                pulled_status.downloaded = 0
+                pulled_status.extracted = 0
+
+        @retry(
             wait=wait_random_exponential(),
             stop=stop_after_attempt(retry_upon_error_count),
             reraise=True,
             retry=retry_if_exception_type(asyncio.TimeoutError),
-        ):
-            # each time there is an error progress starts from zero
+        )
+        async def _pull_image_with_retry() -> None:
+            # for each attempt rest the progress
             progress_bar.reset_progress()
-            _logger.info(
-                "attempt='%s' to pull image='%s'",
-                attempt.retry_state.attempt_number,
-                image,
-            )
+            _reset_progress_from_previous_attempt()
+
+            _logger.info("trying to pull image='%s'", image)
 
-            with attempt:
-                reported_progress = 0.0
-                async for pull_progress in client.images.pull(
-                    image, stream=True, auth=registry_auth
-                ):
-                    try:
-                        parsed_progress = TypeAdapter(_DockerPullImage).validate_python(
-                            pull_progress
-                        )
-                    except ValidationError:
-                        _logger.exception(
-                            "Unexpected error while validating '%s'. "
-                            "TIP: This is probably an unforeseen pull status text that shall be added to the code. "
-                            "The pulling process will still continue.",
-                            f"{pull_progress=}",
-                        )
-                    else:
-                        await _parse_pull_information(
-                            parsed_progress, layer_id_to_size=layer_id_to_size
-                        )
-
-                    # compute total progress
-                    total_downloaded_size = sum(
-                        layer.downloaded for layer in layer_id_to_size.values()
+            reported_progress = 0.0
+            async for pull_progress in client.images.pull(
+                image, stream=True, auth=registry_auth
+            ):
+                try:
+                    parsed_progress = TypeAdapter(_DockerPullImage).validate_python(
+                        pull_progress
                     )
-                    total_extracted_size = sum(
-                        layer.extracted for layer in layer_id_to_size.values()
+                except ValidationError:
+                    _logger.exception(
+                        "Unexpected error while validating '%s'. "
+                        "TIP: This is probably an unforeseen pull status text that shall be added to the code. "
+                        "The pulling process will still continue.",
+                        f"{pull_progress=}",
                     )
-                    total_progress = (
-                        total_downloaded_size + total_extracted_size
-                    ) / 2.0
-                    progress_to_report = total_progress - reported_progress
-                    await progress_bar.update(progress_to_report)
-                    reported_progress = total_progress
-
-                    await log_cb(
-                        f"pulling {image_short_name}: {pull_progress}...",
-                        logging.DEBUG,
+                else:
+                    await _parse_pull_information(
+                        parsed_progress, layer_id_to_size=layer_id_to_size
                     )
+
+                # compute total progress
+                total_downloaded_size = sum(
+                    layer.downloaded for layer in layer_id_to_size.values()
+                )
+                total_extracted_size = sum(
+                    layer.extracted for layer in layer_id_to_size.values()
+                )
+                total_progress = (total_downloaded_size + total_extracted_size) / 2.0
+                progress_to_report = total_progress - reported_progress
+                await progress_bar.update(progress_to_report)
+                reported_progress = total_progress
+
+                await log_cb(
+                    f"pulling {image_short_name}: {pull_progress}...",
+                    logging.DEBUG,
+                )
+
+        await _pull_image_with_retry()

packages/service-library/src/servicelib/progress_bar.py

@@ -87,6 +87,7 @@ async def main_fct():
     progress_unit: ProgressUnit | None = None
     progress_report_cb: AsyncReportCB | ReportCB | None = None
     _current_steps: float = _INITIAL_VALUE
+    _currnet_attempt: int = 0
     _children: list["ProgressBarData"] = field(default_factory=list)
     _parent: Optional["ProgressBarData"] = None
     _continuous_value_lock: asyncio.Lock = field(init=False)
@@ -146,6 +147,7 @@ async def _report_external(self, value: float) -> None:
                         # NOTE: here we convert back to actual value since this is possibly weighted
                         actual_value=value * self.num_steps,
                         total=self.num_steps,
+                        attempt=self._currnet_attempt,
                         unit=self.progress_unit,
                         message=self.compute_report_message_stuct(),
                     ),
@@ -197,7 +199,9 @@ async def update(self, steps: float = 1) -> None:
         await self._report_external(new_progress_value)
 
     def reset_progress(self) -> None:
+        self._currnet_attempt += 1
         self._current_steps = _INITIAL_VALUE
+        self._last_report_value = _INITIAL_VALUE
 
     async def set_(self, new_value: float) -> None:
         await self.update(new_value - self._current_steps)

services/static-webserver/client/source/class/osparc/data/model/NodeProgressSequence.js

@@ -186,8 +186,11 @@ qx.Class.define("osparc.data.model.NodeProgressSequence", {
 
     getProgress: function(report) {
       if (report.unit) {
+        const attempt = ("attempt" in report && report["attempt"] > 1) ? `(attempt ${report["attempt"]}) ` : "";
+        const current_value = osparc.utils.Utils.bytesToSize(report["actual_value"], 1, false);
+        const total_value = osparc.utils.Utils.bytesToSize(report["total"], 1, false)
         return {
-          progressLabel: `${osparc.utils.Utils.bytesToSize(report["actual_value"], 1, false)} / ${osparc.utils.Utils.bytesToSize(report["total"], 1, false)}`,
+          progressLabel: `${attempt}${current_value} / ${total_value}`,
           value: report["actual_value"] / report["total"]
         }
       }