Bump retries and timeouts considerably

As LDK currently will still panic and crash if we'd ever fail a `write`
operation, we here considerably bump the defaults set in `VssClient` to
at least make this less likely. Longer term, we still hope to mitigate
the issue by moving to the async-persist flow.

Author: tnull

Cargo.toml

@@ -66,7 +66,7 @@ serde_json = { version = "1.0.128", default-features = false, features = ["std"]
 log = { version = "0.4.22", default-features = false, features = ["std"]}
 
 #vss-client-ng = "0.3"
-vss-client-ng = { git = "https://github.com/tnull/vss-client", rev = "e6d8e57a949bff3cf511aef258b7aa2c681e35c9" }
+vss-client-ng = { git = "https://github.com/tnull/vss-client", rev = "7cf661b4ba45983ecad0f59e6d74050e2c84212f" }
 prost = { version = "0.11.6", default-features = false}
 
 [target.'cfg(windows)'.dependencies]

src/io/vss_store.rs

@@ -35,6 +35,9 @@ use crate::io::utils::check_namespace_key_validity;
 // would hit a blocking case
 const INTERNAL_RUNTIME_WORKERS: usize = 2;
 
+const HTTP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
+const HTTP_RETRIES: u32 = 10;
+
 /// A [`KVStoreSync`] implementation that writes to and reads from a [VSS](https://github.com/lightningdevkit/vss-server/blob/main/README.md) backend.
 pub struct VssStore {
 	inner: Arc<VssStoreInner>,
@@ -284,10 +287,11 @@ impl VssStoreInner {
 		let (data_encryption_key, obfuscation_master_key) =
 			derive_data_encryption_and_obfuscation_keys(&vss_seed);
 		let key_obfuscator = KeyObfuscator::new(obfuscation_master_key);
-		let client = VssClient::new_with_headers(base_url, header_provider).map_err(|e| {
-			let msg = format!("Failed to setup VssClient: {}", e);
+		let reqwest_client = build_client(&base_url).map_err(|_| {
+			let msg = format!("Failed to setup HTTP client: invalid URL");
 			Error::new(ErrorKind::Other, msg)
 		})?;
+		let client = VssClient::from_client_and_headers(base_url, reqwest_client, header_provider);
 		let locks = Mutex::new(HashMap::new());
 		Ok(Self { client, store_id, data_encryption_key, key_obfuscator, locks })
 	}
@@ -524,6 +528,32 @@ impl VssStoreInner {
 	}
 }
 
+fn build_client(base_url: &str) -> Result<reqwest::Client, ()> {
+	let url = reqwest::Url::parse(base_url).map_err(|_| ())?;
+	let host_str = url.host_str().ok_or(())?.to_string();
+	let retry = reqwest::retry::for_host(host_str)
+		.max_retries_per_request(HTTP_RETRIES)
+		.classify_fn(|req_rep| match req_rep.status() {
+			// VSS uses INTERNAL_SERVER_ERROR when sending back error repsonses. These are
+			// currently still covered by our `RetryPolicy`, so we tell `reqwest` not to retry them.
+			Some(reqwest::StatusCode::INTERNAL_SERVER_ERROR) => req_rep.success(),
+			Some(reqwest::StatusCode::BAD_REQUEST) => req_rep.success(),
+			Some(reqwest::StatusCode::UNAUTHORIZED) => req_rep.success(),
+			Some(reqwest::StatusCode::NOT_FOUND) => req_rep.success(),
+			Some(reqwest::StatusCode::CONFLICT) => req_rep.success(),
+			Some(reqwest::StatusCode::OK) => req_rep.success(),
+			_ => req_rep.retryable(),
+		});
+	let client = reqwest::Client::builder()
+		.timeout(HTTP_TIMEOUT)
+		.connect_timeout(HTTP_TIMEOUT)
+		.read_timeout(HTTP_TIMEOUT)
+		.retry(retry)
+		.build()
+		.unwrap();
+	Ok(client)
+}
+
 fn derive_data_encryption_and_obfuscation_keys(vss_seed: &[u8; 32]) -> ([u8; 32], [u8; 32]) {
 	let hkdf = |initial_key_material: &[u8], salt: &[u8]| -> [u8; 32] {
 		let mut engine = HmacEngine::<sha256::Hash>::new(salt);