Skip to content

Commit 998fcaf

Browse files
authored
feat: add random walk component (#2501)
To allow finding network services, add a random walk component that lets services find random network peers in a scalable way. If two services try to random walk at the same time, they will share the results.
1 parent 90cfd25 commit 998fcaf

File tree

6 files changed

+419
-2
lines changed

6 files changed

+419
-2
lines changed
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
export * from './address-manager/index.js'
22
export * from './connection-manager/index.js'
3+
export * from './random-walk/index.js'
34
export * from './record/index.js'
45
export * from './registrar/index.js'
56
export * from './transport-manager/index.js'
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
import type { AbortOptions, PeerInfo } from '@libp2p/interface'
2+
3+
/**
4+
* RandomWalk finds random peers on the network and dials them. Use this after
5+
* registering a Topology if you need to discover common network services.
6+
*/
7+
export interface RandomWalk {
8+
/**
9+
* Begin or join an existing walk. Abort the passed signal if you wish to
10+
* abort the walk early.
11+
*/
12+
walk(options?: AbortOptions): AsyncGenerator<PeerInfo>
13+
}

packages/libp2p/package.json

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,9 @@
105105
"it-parallel": "^3.0.6",
106106
"merge-options": "^3.0.4",
107107
"multiformats": "^13.1.0",
108+
"p-defer": "^4.0.1",
109+
"race-event": "^1.3.0",
110+
"race-signal": "^1.0.2",
108111
"uint8arrays": "^5.0.3"
109112
},
110113
"devDependencies": {
@@ -121,11 +124,11 @@
121124
"delay": "^6.0.0",
122125
"it-all": "^3.0.4",
123126
"it-drain": "^3.0.5",
124-
"it-map": "^3.0.5",
127+
"it-map": "^3.1.0",
125128
"it-pipe": "^3.0.1",
126129
"it-pushable": "^3.2.3",
127130
"it-stream-types": "^2.0.1",
128-
"p-defer": "^4.0.1",
131+
"it-take": "^3.0.4",
129132
"p-event": "^6.0.1",
130133
"p-wait-for": "^5.0.2",
131134
"sinon": "^17.0.1",

packages/libp2p/src/libp2p.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import { DefaultConnectionManager } from './connection-manager/index.js'
1717
import { CompoundContentRouting } from './content-routing.js'
1818
import { codes } from './errors.js'
1919
import { DefaultPeerRouting } from './peer-routing.js'
20+
import { RandomWalk } from './random-walk.js'
2021
import { DefaultRegistrar } from './registrar.js'
2122
import { DefaultTransportManager } from './transport-manager.js'
2223
import { DefaultUpgrader } from './upgrader.js'
@@ -137,6 +138,9 @@ export class Libp2pNode<T extends ServiceMap = Record<string, unknown>> extends
137138
routers: contentRouters
138139
}))
139140

141+
// Random walk
142+
this.configureComponent('randomWalk', new RandomWalk(this.components))
143+
140144
// Discovery modules
141145
;(init.peerDiscovery ?? []).forEach((fn, index) => {
142146
const service = this.configureComponent(`peer-discovery-${index}`, fn(this.components))

packages/libp2p/src/random-walk.ts

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
import { randomBytes } from '@libp2p/crypto'
2+
import { TypedEventEmitter, setMaxListeners } from '@libp2p/interface'
3+
import { anySignal } from 'any-signal'
4+
import pDefer, { type DeferredPromise } from 'p-defer'
5+
import { raceEvent } from 'race-event'
6+
import { raceSignal } from 'race-signal'
7+
import type { AbortOptions, ComponentLogger, Logger, PeerInfo, PeerRouting, Startable } from '@libp2p/interface'
8+
import type { RandomWalk as RandomWalkInterface } from '@libp2p/interface-internal'
9+
10+
export interface RandomWalkComponents {
11+
peerRouting: PeerRouting
12+
logger: ComponentLogger
13+
}
14+
15+
interface RandomWalkEvents {
16+
'walk:peer': CustomEvent<PeerInfo>
17+
'walk:error': CustomEvent<Error>
18+
}
19+
20+
export class RandomWalk extends TypedEventEmitter<RandomWalkEvents> implements RandomWalkInterface, Startable {
21+
private readonly peerRouting: PeerRouting
22+
private readonly log: Logger
23+
private walking: boolean
24+
private walkers: number
25+
private shutdownController: AbortController
26+
private walkController?: AbortController
27+
private needNext?: DeferredPromise<void>
28+
29+
constructor (components: RandomWalkComponents) {
30+
super()
31+
32+
this.log = components.logger.forComponent('libp2p:random-walk')
33+
this.peerRouting = components.peerRouting
34+
this.walkers = 0
35+
this.walking = false
36+
37+
// stops any in-progress walks when the node is shut down
38+
this.shutdownController = new AbortController()
39+
setMaxListeners(Infinity, this.shutdownController.signal)
40+
}
41+
42+
start (): void {
43+
this.shutdownController = new AbortController()
44+
setMaxListeners(Infinity, this.shutdownController.signal)
45+
}
46+
47+
stop (): void {
48+
this.shutdownController.abort()
49+
}
50+
51+
async * walk (options?: AbortOptions): AsyncGenerator<PeerInfo> {
52+
if (!this.walking) {
53+
// start the query that causes walk:peer events to be emitted
54+
this.startWalk()
55+
}
56+
57+
this.walkers++
58+
const signal = anySignal([this.shutdownController.signal, options?.signal])
59+
setMaxListeners(Infinity, signal)
60+
61+
try {
62+
while (true) {
63+
// if another consumer has paused the query, start it again
64+
this.needNext?.resolve()
65+
this.needNext = pDefer()
66+
67+
// wait for a walk:peer or walk:error event
68+
const event = await raceEvent<CustomEvent<PeerInfo>>(this, 'walk:peer', signal, {
69+
errorEvent: 'walk:error'
70+
})
71+
72+
yield event.detail
73+
}
74+
} finally {
75+
signal.clear()
76+
this.walkers--
77+
78+
// stop the walk if no more consumers are interested
79+
if (this.walkers === 0) {
80+
this.walkController?.abort()
81+
this.walkController = undefined
82+
}
83+
}
84+
}
85+
86+
private startWalk (): void {
87+
this.walking = true
88+
89+
// the signal for this controller will be aborted if no more random peers
90+
// are required
91+
this.walkController = new AbortController()
92+
setMaxListeners(Infinity, this.walkController.signal)
93+
94+
const signal = anySignal([this.walkController.signal, this.shutdownController.signal])
95+
setMaxListeners(Infinity, signal)
96+
97+
const start = Date.now()
98+
let found = 0
99+
100+
Promise.resolve().then(async () => {
101+
this.log('start walk')
102+
103+
// find peers until no more consumers are interested
104+
while (this.walkers > 0) {
105+
try {
106+
for await (const peer of this.peerRouting.getClosestPeers(randomBytes(32), { signal })) {
107+
signal.throwIfAborted()
108+
109+
this.log('found peer %p', peer.id)
110+
found++
111+
this.safeDispatchEvent('walk:peer', {
112+
detail: peer
113+
})
114+
115+
// if we only have one consumer, pause the query until they request
116+
// another random peer or they signal they are no longer interested
117+
if (this.walkers === 1 && this.needNext != null) {
118+
await raceSignal(this.needNext.promise, signal)
119+
}
120+
}
121+
} catch (err) {
122+
this.log.error('randomwalk errored', err)
123+
124+
this.safeDispatchEvent('walk:error', {
125+
detail: err
126+
})
127+
}
128+
}
129+
})
130+
.catch(err => {
131+
this.log.error('randomwalk errored', err)
132+
})
133+
.finally(() => {
134+
this.log('finished walk, found %d peers after %dms', found, Date.now() - start)
135+
this.walking = false
136+
})
137+
}
138+
}

0 commit comments

Comments
 (0)