diff --git a/src/_zkapauthorizer/recover.py b/src/_zkapauthorizer/recover.py index 842eb269..b045548f 100644 --- a/src/_zkapauthorizer/recover.py +++ b/src/_zkapauthorizer/recover.py @@ -119,7 +119,11 @@ async def recover( :param cursor: A database cursor which can be used to populate the database with recovered state. """ - if self._state.stage != RecoveryStages.inactive: + if self._state.stage not in { + RecoveryStages.inactive, + RecoveryStages.download_failed, + RecoveryStages.import_failed, + }: return self._set_state(RecoveryState(stage=RecoveryStages.started)) diff --git a/src/_zkapauthorizer/resource.py b/src/_zkapauthorizer/resource.py index c956cf8b..5b7ad364 100644 --- a/src/_zkapauthorizer/resource.py +++ b/src/_zkapauthorizer/resource.py @@ -336,9 +336,14 @@ def err(f): ) ) disconnect_clients() + self.recovering_d = None def happy(_): + # note that the StatefulRecoverer eats download / + # import errors so we'll exit here sometimes even + # though an error message has been sent... disconnect_clients() + self.recovering_d = None self.recovering_d.addCallbacks(happy, err) diff --git a/src/_zkapauthorizer/tests/test_client_resource.py b/src/_zkapauthorizer/tests/test_client_resource.py index d5730fde..2918db65 100644 --- a/src/_zkapauthorizer/tests/test_client_resource.py +++ b/src/_zkapauthorizer/tests/test_client_resource.py @@ -961,6 +961,107 @@ def create_proto(): ), ) + @given( + tahoe_configs(), + api_auth_tokens(), + ) + def test_recover_retry(self, get_config, api_auth_token): + """ + If at first our download fails, we can still retry using the API. + """ + downloads = [] + fails = [RuntimeError("downloader fails")] + + def get_sometimes_fail_downloader(cap): + async def do_download(set_state): + nonlocal downloads, fails + if fails: + raise fails.pop(0) + downloads.append(set_state) + return ( + lambda: BytesIO(statements_to_snapshot([])), + [], # no event-streams + ) + + return do_download + + clock = MemoryReactorClockResolver() + store = self.useFixture(TemporaryVoucherStore(aware_now, get_config)).store + factory = RecoverFactory(store, get_sometimes_fail_downloader) + pumper = create_pumper() + self.addCleanup(pumper.stop) + + def create_proto(): + addr = IPv4Address("TCP", "127.0.0.1", "0") + proto = factory.buildProtocol(addr) + return proto + + agent = create_memory_agent(clock, pumper, create_proto) + pumper.start() + + # first recovery will fail + d0 = Deferred.fromCoroutine( + recover( + agent, + DecodedURL.from_text("ws://127.0.0.1:1/"), + api_auth_token, + self.GOOD_CAPABILITY, + ) + ) + pumper._flush() + + # try to recover again (this one should work, as we only fail + # once in the test-provided downloader) + d1 = Deferred.fromCoroutine( + recover( + agent, + DecodedURL.from_text("ws://127.0.0.1:1/"), + api_auth_token, + self.GOOD_CAPABILITY, + ) + ) + pumper._flush() + + self.assertThat( + d0, + succeeded( + Equals( + [ + { + "stage": "started", + "failure-reason": None, + }, + # "our" downloader (above) doesn't set any downloading etc + # state-updates + { + "stage": "download_failed", + "failure-reason": "downloader fails", + }, + ] + ) + ), + ) + # second attempt should succeed + self.assertThat( + d1, + succeeded( + Equals( + [ + { + "stage": "started", + "failure-reason": None, + }, + # "our" downloader (above) doesn't set any downloading etc + # state-updates + { + "stage": "succeeded", + "failure-reason": None, + }, + ] + ) + ), + ) + def maybe_extra_tokens(): """