fix: retry 'snap watch' (#899)

Signed-off-by: Callahan Kovacs <callahan.kovacs@canonical.com>
Co-authored-by: Alex Lowe <alex.lowe@canonical.com>

Author: mr-cal

craft_providers/base.py

@@ -612,12 +612,23 @@ def _disable_and_wait_for_snap_refresh(self, executor: Executor) -> None:
 
         # a refresh may have started before the hold was set
         logger.debug("Waiting for pending snap refreshes to complete.")
-        try:
+
+        def snap_watch(timeout: float) -> None:
             executor.execute_run(
                 ["snap", "watch", "--last=auto-refresh?"],
                 capture_output=True,
                 check=True,
-                timeout=self._timeout_simple,
+                timeout=timeout,
+            )
+
+        try:
+            # There is a small time window after restarting the snapd service where snapd
+            # claims it's ready but actually isn't (SNAPDENG-36387). The workaround is to
+            # retry.
+            retry.retry_until_timeout(
+                self._timeout_complex or TIMEOUT_COMPLEX,
+                self._retry_wait,
+                snap_watch,
             )
         except subprocess.CalledProcessError as error:
             raise BaseConfigurationError(

docs/reference/changelog.rst

@@ -4,7 +4,7 @@ Changelog
 See the `Releases page`_ on GitHub for a complete list of commits that are
 included in each version.
 
-3.3.0 (Unreleased)
+3.3.0 (2026-02-09)
 ------------------
 
 Breaking changes:
@@ -14,6 +14,7 @@ Breaking changes:
 Bug fixes:
 
 - Do not fail if the build host is an unknown Ubuntu release.
+- Fix a bug where snapd would fail while waiting for snap refreshes to complete.
 
 3.2.0 (2025-11-13)
 ------------------

tests/unit/bases/test_almalinux.py

@@ -838,6 +838,8 @@ def test_setup_snapd_failures(fake_process, fake_executor):
 @pytest.mark.parametrize("fail_index", list(range(0, 8)))
 def test_post_setup_snapd_failures(fake_process, fake_executor, fail_index):
     base_config = almalinux.AlmaLinuxBase(alias=almalinux.AlmaLinuxBaseAlias.NINE)
+    base_config._retry_wait = 0.01
+    base_config._timeout_complex = 0.01
 
     return_codes = [0] * 8
     return_codes[fail_index] = 1
@@ -1607,6 +1609,8 @@ def test_disable_and_wait_for_snap_refresh_hold_error(fake_process, fake_executo
 def test_disable_and_wait_for_snap_refresh_wait_error(fake_process, fake_executor):
     """Raise BaseConfigurationError when the `snap watch` command fails."""
     base_config = almalinux.AlmaLinuxBase(alias=almalinux.AlmaLinuxBaseAlias.NINE)
+    base_config._retry_wait = 0.01
+    base_config._timeout_complex = 0.01
     fake_process.register_subprocess([*DEFAULT_FAKE_CMD, "snap", "refresh", "--hold"])
     fake_process.register_subprocess(
         [*DEFAULT_FAKE_CMD, "snap", "watch", "--last=auto-refresh?"],

tests/unit/bases/test_centos_7.py

@@ -776,6 +776,8 @@ def test_setup_snapd_failures(fake_process, fake_executor):
 @pytest.mark.parametrize("fail_index", list(range(0, 8)))
 def test_post_setup_snapd_failures(fake_process, fake_executor, fail_index):
     base_config = centos.CentOSBase(alias=centos.CentOSBaseAlias.SEVEN)
+    base_config._retry_wait = 0.01
+    base_config._timeout_complex = 0.01
     return_codes = [0] * 8
     return_codes[fail_index] = 1
     fake_process.register_subprocess(
@@ -1543,6 +1545,8 @@ def test_disable_and_wait_for_snap_refresh_hold_error(fake_process, fake_executo
 def test_disable_and_wait_for_snap_refresh_wait_error(fake_process, fake_executor):
     """Raise BaseConfigurationError when the `snap watch` command fails."""
     base_config = centos.CentOSBase(alias=centos.CentOSBaseAlias.SEVEN)
+    base_config._retry_wait = 0.01
+    base_config._timeout_complex = 0.01
     fake_process.register_subprocess([*DEFAULT_FAKE_CMD, "snap", "refresh", "--hold"])
     fake_process.register_subprocess(
         [*DEFAULT_FAKE_CMD, "snap", "watch", "--last=auto-refresh?"],

tests/unit/bases/test_ubuntu_buildd.py

@@ -969,6 +969,8 @@ def test_setup_snapd_failures(fake_process, fake_executor):
 @pytest.mark.parametrize("fail_index", list(range(0, 8)))
 def test_post_setup_snapd_failures(fake_process, fake_executor, fail_index):
     base_config = ubuntu.BuilddBase(alias=ubuntu.BuilddBaseAlias.JAMMY)
+    base_config._retry_wait = 0.01
+    base_config._timeout_complex = 0.01
     return_codes = [0] * 8
     return_codes[fail_index] = 1
 
@@ -1721,6 +1723,8 @@ def test_disable_and_wait_for_snap_refresh_hold_error(fake_process, fake_executo
 def test_disable_and_wait_for_snap_refresh_wait_error(fake_process, fake_executor):
     """Raise BaseConfigurationError when the `snap watch` command fails."""
     base_config = ubuntu.BuilddBase(alias=ubuntu.BuilddBaseAlias.JAMMY)
+    base_config._retry_wait = 0.01
+    base_config._timeout_complex = 0.01
     fake_process.register_subprocess([*DEFAULT_FAKE_CMD, "snap", "refresh", "--hold"])
     fake_process.register_subprocess(
         [*DEFAULT_FAKE_CMD, "snap", "watch", "--last=auto-refresh?"],

tests/unit/test_base.py

@@ -327,3 +327,73 @@ def test_set_hostname(fake_base):
     bad_name = "bad_123-ABC%-"
     fake_base._set_hostname(bad_name)
     assert fake_base._hostname == "bad123-ABC"
+
+
+def test_snap_refresh(fake_process, fake_executor, fake_base):
+    """Disable and wait for snap refreshes."""
+    fake_process.register_subprocess(
+        [*DEFAULT_FAKE_CMD, "snap", "refresh", "--hold"],
+        returncode=0,
+    )
+    fake_process.register_subprocess(
+        [*DEFAULT_FAKE_CMD, "snap", "watch", "--last=auto-refresh?"],
+        returncode=0,
+    )
+
+    fake_base._disable_and_wait_for_snap_refresh(executor=fake_executor)
+
+
+def test_snap_refresh_hold_error(fake_process, fake_executor, fake_base):
+    """Error on failure to hold refreshes."""
+    fake_process.register_subprocess(
+        [*DEFAULT_FAKE_CMD, "snap", "refresh", "--hold"],
+        stderr="test error",
+        returncode=1,
+    )
+    fake_process.register_subprocess(
+        [*DEFAULT_FAKE_CMD, "snap", "watch", "--last=auto-refresh?"],
+        returncode=0,
+    )
+
+    with pytest.raises(BaseConfigurationError) as err:
+        fake_base._disable_and_wait_for_snap_refresh(executor=fake_executor)
+
+    assert err.value.brief == "Failed to hold snap refreshes."
+    assert err.value.details
+    assert "test error" in err.value.details
+
+
+def test_snap_refresh_watch_error(fake_process, fake_executor, fake_base):
+    """Error on failure of 'snap watch'."""
+    stderr = "error: daemon is stopping to wait for socket activation"
+    fake_base._timeout_complex = 0.01
+    fake_process.register_subprocess(
+        [*DEFAULT_FAKE_CMD, "snap", "refresh", "--hold"],
+        returncode=0,
+    )
+    fake_process.register_subprocess(
+        [*DEFAULT_FAKE_CMD, "snap", "watch", "--last=auto-refresh?"],
+        stderr=stderr,
+        returncode=1,
+    )
+
+    with pytest.raises(BaseConfigurationError) as err:
+        fake_base._disable_and_wait_for_snap_refresh(executor=fake_executor)
+
+    assert err.value.brief == "Failed to wait for snap refreshes to complete."
+    assert err.value.details
+    assert stderr in err.value.details
+
+
+def test_snap_refresh_watch_retry(fake_process, fake_executor, fake_base):
+    """Retry when 'snap watch' fails."""
+    fake_process.register_subprocess(
+        [*DEFAULT_FAKE_CMD, "snap", "refresh", "--hold"],
+        returncode=0,
+    )
+    # fail twice then succeed
+    snap_watch = [*DEFAULT_FAKE_CMD, "snap", "watch", "--last=auto-refresh?"]
+    for returncode in (1, 1, 0):
+        fake_process.register_subprocess(snap_watch, returncode=returncode)
+
+    fake_base._disable_and_wait_for_snap_refresh(executor=fake_executor)