Skip to content

Commit 453bcea

Browse files
isumisum
authored
ipfs: improve ipfs client selection (#5723)
1 parent 1d29605 commit 453bcea

File tree

5 files changed

+110
-78
lines changed

5 files changed

+110
-78
lines changed

‎graph/src/ipfs/client.rs

Lines changed: 47 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -36,16 +36,19 @@ pub trait IpfsClient: Send + Sync + 'static {
3636
timeout: Option<Duration>,
3737
retry_policy: RetryPolicy,
3838
) -> IpfsResult<BoxStream<'static, IpfsResult<Bytes>>> {
39-
let fut = retry_policy.create("IPFS.cat_stream", self.logger()).run({
40-
let path = path.to_owned();
39+
let fut = retry_policy
40+
.create("IPFS.cat_stream", self.logger())
41+
.no_timeout()
42+
.run({
43+
let path = path.to_owned();
4144

42-
move || {
43-
let path = path.clone();
44-
let client = self.clone();
45+
move || {
46+
let path = path.clone();
47+
let client = self.clone();
4548

46-
async move { client.call(IpfsRequest::Cat(path)).await }
47-
}
48-
});
49+
async move { client.call(IpfsRequest::Cat(path)).await }
50+
}
51+
});
4952

5053
let resp = run_with_optional_timeout(path, fut, timeout).await?;
5154

@@ -63,22 +66,25 @@ pub trait IpfsClient: Send + Sync + 'static {
6366
timeout: Option<Duration>,
6467
retry_policy: RetryPolicy,
6568
) -> IpfsResult<Bytes> {
66-
let fut = retry_policy.create("IPFS.cat", self.logger()).run({
67-
let path = path.to_owned();
68-
69-
move || {
70-
let path = path.clone();
71-
let client = self.clone();
72-
73-
async move {
74-
client
75-
.call(IpfsRequest::Cat(path))
76-
.await?
77-
.bytes(Some(max_size))
78-
.await
69+
let fut = retry_policy
70+
.create("IPFS.cat", self.logger())
71+
.no_timeout()
72+
.run({
73+
let path = path.to_owned();
74+
75+
move || {
76+
let path = path.clone();
77+
let client = self.clone();
78+
79+
async move {
80+
client
81+
.call(IpfsRequest::Cat(path))
82+
.await?
83+
.bytes(Some(max_size))
84+
.await
85+
}
7986
}
80-
}
81-
});
87+
});
8288

8389
run_with_optional_timeout(path, fut, timeout).await
8490
}
@@ -93,22 +99,25 @@ pub trait IpfsClient: Send + Sync + 'static {
9399
timeout: Option<Duration>,
94100
retry_policy: RetryPolicy,
95101
) -> IpfsResult<Bytes> {
96-
let fut = retry_policy.create("IPFS.get_block", self.logger()).run({
97-
let path = path.to_owned();
98-
99-
move || {
100-
let path = path.clone();
101-
let client = self.clone();
102-
103-
async move {
104-
client
105-
.call(IpfsRequest::GetBlock(path))
106-
.await?
107-
.bytes(None)
108-
.await
102+
let fut = retry_policy
103+
.create("IPFS.get_block", self.logger())
104+
.no_timeout()
105+
.run({
106+
let path = path.to_owned();
107+
108+
move || {
109+
let path = path.clone();
110+
let client = self.clone();
111+
112+
async move {
113+
client
114+
.call(IpfsRequest::GetBlock(path))
115+
.await?
116+
.bytes(None)
117+
.await
118+
}
109119
}
110-
}
111-
});
120+
});
112121

113122
run_with_optional_timeout(path, fut, timeout).await
114123
}

‎graph/src/ipfs/gateway_client.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,8 @@ impl IpfsGatewayClient {
8080

8181
let fut = RetryPolicy::NonDeterministic
8282
.create("IPFS.Gateway.send_test_request", &self.logger)
83+
.no_logging()
84+
.no_timeout()
8385
.run(move || {
8486
let req = req.try_clone().expect("request can be cloned");
8587

‎graph/src/ipfs/mod.rs

Lines changed: 52 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
use std::sync::Arc;
22

33
use anyhow::anyhow;
4+
use futures03::future::BoxFuture;
5+
use futures03::stream::FuturesUnordered;
6+
use futures03::stream::StreamExt;
47
use slog::info;
58
use slog::Logger;
69

@@ -55,40 +58,7 @@ where
5558
SafeDisplay(server_address)
5659
);
5760

58-
match IpfsGatewayClient::new(server_address, logger).await {
59-
Ok(client) => {
60-
info!(
61-
logger,
62-
"Successfully connected to IPFS gateway at: '{}'",
63-
SafeDisplay(server_address)
64-
);
65-
66-
clients.push(Arc::new(client));
67-
continue;
68-
}
69-
Err(err) if err.is_invalid_server() => {}
70-
Err(err) => return Err(err),
71-
};
72-
73-
match IpfsRpcClient::new(server_address, logger).await {
74-
Ok(client) => {
75-
info!(
76-
logger,
77-
"Successfully connected to IPFS RPC API at: '{}'",
78-
SafeDisplay(server_address)
79-
);
80-
81-
clients.push(Arc::new(client));
82-
continue;
83-
}
84-
Err(err) if err.is_invalid_server() => {}
85-
Err(err) => return Err(err),
86-
};
87-
88-
return Err(IpfsError::InvalidServer {
89-
server_address: server_address.parse()?,
90-
reason: anyhow!("unknown server kind"),
91-
});
61+
clients.push(use_first_valid_api(server_address, logger).await?);
9262
}
9363

9464
match clients.len() {
@@ -106,3 +76,51 @@ where
10676
}
10777
}
10878
}
79+
80+
async fn use_first_valid_api(
81+
server_address: &str,
82+
logger: &Logger,
83+
) -> IpfsResult<Arc<dyn IpfsClient>> {
84+
let supported_apis: Vec<BoxFuture<IpfsResult<Arc<dyn IpfsClient>>>> = vec![
85+
Box::pin(async {
86+
IpfsGatewayClient::new(server_address, logger)
87+
.await
88+
.map(|client| {
89+
info!(
90+
logger,
91+
"Successfully connected to IPFS gateway at: '{}'",
92+
SafeDisplay(server_address)
93+
);
94+
95+
Arc::new(client) as Arc<dyn IpfsClient>
96+
})
97+
}),
98+
Box::pin(async {
99+
IpfsRpcClient::new(server_address, logger)
100+
.await
101+
.map(|client| {
102+
info!(
103+
logger,
104+
"Successfully connected to IPFS RPC API at: '{}'",
105+
SafeDisplay(server_address)
106+
);
107+
108+
Arc::new(client) as Arc<dyn IpfsClient>
109+
})
110+
}),
111+
];
112+
113+
let mut stream = supported_apis.into_iter().collect::<FuturesUnordered<_>>();
114+
while let Some(result) = stream.next().await {
115+
match result {
116+
Ok(client) => return Ok(client),
117+
Err(err) if err.is_invalid_server() => {}
118+
Err(err) => return Err(err),
119+
};
120+
}
121+
122+
Err(IpfsError::InvalidServer {
123+
server_address: server_address.parse()?,
124+
reason: anyhow!("unknown server kind"),
125+
})
126+
}

‎graph/src/ipfs/retry_policy.rs

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ use slog::Logger;
22

33
use crate::ipfs::error::IpfsError;
44
use crate::util::futures::retry;
5-
use crate::util::futures::RetryConfigNoTimeout;
5+
use crate::util::futures::RetryConfig;
66

77
/// This is a safety mechanism to prevent infinite spamming of IPFS servers
88
/// in the event of logical or unhandled deterministic errors.
@@ -24,14 +24,11 @@ pub enum RetryPolicy {
2424

2525
impl RetryPolicy {
2626
/// Creates a retry policy for every request sent to IPFS servers.
27-
///
28-
/// Note: It is expected that retries will be wrapped in timeouts
29-
/// when necessary to make them more flexible.
3027
pub(super) fn create<O: Send + Sync + 'static>(
3128
self,
3229
operation_name: impl ToString,
3330
logger: &Logger,
34-
) -> RetryConfigNoTimeout<O, IpfsError> {
31+
) -> RetryConfig<O, IpfsError> {
3532
retry(operation_name, logger)
3633
.limit(DEFAULT_MAX_ATTEMPTS)
3734
.when(move |result: &Result<O, IpfsError>| match result {
@@ -42,7 +39,6 @@ impl RetryPolicy {
4239
Self::NonDeterministic => !err.is_deterministic(),
4340
},
4441
})
45-
.no_timeout()
4642
}
4743
}
4844

@@ -69,6 +65,7 @@ mod tests {
6965

7066
let err = RetryPolicy::None
7167
.create::<()>("test", &discard())
68+
.no_timeout()
7269
.run({
7370
let counter = counter.clone();
7471
move || {
@@ -92,6 +89,7 @@ mod tests {
9289

9390
let err = RetryPolicy::Networking
9491
.create("test", &discard())
92+
.no_timeout()
9593
.run({
9694
let counter = counter.clone();
9795
move || {
@@ -126,6 +124,7 @@ mod tests {
126124

127125
RetryPolicy::Networking
128126
.create("test", &discard())
127+
.no_timeout()
129128
.run({
130129
let counter = counter.clone();
131130
move || {
@@ -159,6 +158,7 @@ mod tests {
159158

160159
let err = RetryPolicy::NonDeterministic
161160
.create::<()>("test", &discard())
161+
.no_timeout()
162162
.run({
163163
let counter = counter.clone();
164164
move || {
@@ -190,6 +190,7 @@ mod tests {
190190

191191
RetryPolicy::NonDeterministic
192192
.create("test", &discard())
193+
.no_timeout()
193194
.run({
194195
let counter = counter.clone();
195196
move || {

‎graph/src/ipfs/rpc_client.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,8 @@ impl IpfsRpcClient {
6969
async fn send_test_request(&self) -> anyhow::Result<()> {
7070
let fut = RetryPolicy::NonDeterministic
7171
.create("IPFS.RPC.send_test_request", &self.logger)
72+
.no_logging()
73+
.no_timeout()
7274
.run({
7375
let client = self.to_owned();
7476

0 commit comments

Comments
 (0)