Skip to content

Commit 35119d9

Browse files
AmeanAsadguanzo
AmeanAsad
and
guanzo
authored
feat: log node ids with smart client (#35)
* change origin domains * improve test setup * enhancement: add node ids to logging * add id to test-utils * Update src/client.js Co-authored-by: Eric Guan <[email protected]> * fix typo * Update src/client.js Co-authored-by: Eric Guan <[email protected]> * log parsing --------- Co-authored-by: Eric Guan <[email protected]>
1 parent 76c5802 commit 35119d9

File tree

4 files changed

+65
-48
lines changed

4 files changed

+65
-48
lines changed

‎src/client.js

Lines changed: 28 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,8 @@ export class Saturn {
6363
*
6464
* @param {string} cidPath
6565
* @param {object} [opts={}]
66+
* @param {Node[]} [opts.nodes]
67+
* @param {Node} [opts.node]
6668
* @param {('car'|'raw')} [opts.format]
6769
* @param {number} [opts.connectTimeout=5000]
6870
* @param {number} [opts.downloadTimeout=0]
@@ -83,17 +85,16 @@ export class Saturn {
8385
}
8486
}
8587

86-
let origins = options.origins
87-
if (!origins || origins.length === 0) {
88-
const replacementUrl = options.url ?? options.cdnURL
89-
origins = [replacementUrl]
88+
let nodes = options.nodes
89+
if (!nodes || nodes.length === 0) {
90+
const replacementNode = options.node ?? { url: this.opts.cdnURL }
91+
nodes = [replacementNode]
9092
}
9193
const controllers = []
9294

93-
const createFetchPromise = async (origin) => {
94-
const fetchOptions = { ...options, url: origin }
95+
const createFetchPromise = async (node) => {
96+
const fetchOptions = { ...options, url: node.url }
9597
const url = this.createRequestURL(cidPath, fetchOptions)
96-
9798
const controller = new AbortController()
9899
controllers.push(controller)
99100
const connectTimeout = setTimeout(() => {
@@ -103,11 +104,10 @@ export class Saturn {
103104
try {
104105
res = await fetch(parseUrl(url), { signal: controller.signal, ...options })
105106
clearTimeout(connectTimeout)
106-
return { res, url, controller }
107+
return { res, url, node, controller }
107108
} catch (err) {
108-
throw new Error(
109-
`Non OK response received: ${res.status} ${res.statusText}`
110-
)
109+
err.node = node
110+
throw err
111111
}
112112
}
113113

@@ -119,17 +119,18 @@ export class Saturn {
119119
})
120120
}
121121

122-
const fetchPromises = Promise.any(origins.map((origin) => createFetchPromise(origin)))
122+
const fetchPromises = Promise.any(nodes.map((node) => createFetchPromise(node)))
123123

124124
let log = {
125125
startTime: new Date()
126126
}
127127

128-
let res, url, controller
128+
let res, url, controller, node
129129
try {
130-
({ res, url, controller } = await fetchPromises)
130+
({ res, url, controller, node } = await fetchPromises)
131131

132132
abortRemainingFetches(controller, controllers)
133+
log.nodeId = node.id
133134
log = Object.assign(log, this._generateLog(res, log), { url })
134135
if (!res.ok) {
135136
const error = new Error(
@@ -142,6 +143,8 @@ export class Saturn {
142143
if (!res) {
143144
log.error = err.message
144145
}
146+
if (err.node) log.nodeId = err.node.id
147+
145148
// Report now if error, otherwise report after download is done.
146149
this._finalizeLog(log)
147150

@@ -156,6 +159,7 @@ export class Saturn {
156159
* @param {string} cidPath
157160
* @param {object} [opts={}]
158161
* @param {('car'|'raw')} [opts.format]
162+
* @param {Node} [opts.node]
159163
* @param {number} [opts.connectTimeout=5000]
160164
* @param {number} [opts.downloadTimeout=0]
161165
* @returns {Promise<object>}
@@ -167,11 +171,15 @@ export class Saturn {
167171
const jwt = await getJWT(this.opts, this.storage)
168172

169173
const options = Object.assign({}, this.opts, { format: 'car', jwt }, opts)
170-
const url = this.createRequestURL(cidPath, options)
174+
const node = options.node
175+
const origin = node?.url ?? this.opts.cdnURL
176+
const url = this.createRequestURL(cidPath, { ...options, url: origin })
177+
171178
let log = {
172179
url,
173180
startTime: new Date()
174181
}
182+
if (node?.id) log.nodeId = node.id
175183

176184
const controller = options.controller ?? new AbortController()
177185
const connectTimeout = setTimeout(() => {
@@ -220,7 +228,7 @@ export class Saturn {
220228
const { headers } = res
221229
log.httpStatusCode = res.status
222230
log.cacheHit = headers.get('saturn-cache-status') === 'HIT'
223-
log.nodeId = headers.get('saturn-node-id')
231+
log.nodeId = log.nodeId ?? headers.get('saturn-node-id')
224232
log.requestId = headers.get('saturn-transfer-id')
225233
log.httpProtocol = headers.get('quic-status')
226234

@@ -294,10 +302,9 @@ export class Saturn {
294302
return
295303
}
296304
if (opts.raceNodes) {
297-
const origins = nodes.slice(i, i + Saturn.defaultRaceCount).map((node) => node.url)
298-
opts.origins = origins
305+
opts.nodes = nodes.slice(i, i + Saturn.defaultRaceCount)
299306
} else {
300-
opts.url = nodes[i].url
307+
opts.node = nodes[i]
301308
}
302309

303310
try {
@@ -376,10 +383,11 @@ export class Saturn {
376383
*
377384
* @param {string} cidPath
378385
* @param {object} [opts={}]
386+
* @param {string} [opts.url]
379387
* @returns {URL}
380388
*/
381389
createRequestURL (cidPath, opts) {
382-
let origin = opts.url || (opts.origins && opts.origins[0]) || opts.cdnURL
390+
let origin = opts.url ?? this.opts.cdnURL
383391
origin = addHttpPrefix(origin)
384392
const url = new URL(`${origin}/ipfs/${cidPath}`)
385393

‎src/types.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
/**
66
*
77
* @typedef {object} Node
8+
* @property {string} id
89
* @property {string} ip
910
* @property {number} weight
1011
* @property {number} distance

‎test/fallback.spec.js

Lines changed: 34 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,20 @@ import { describe, mock, test } from 'node:test'
55
import { Saturn } from '#src/index.js'
66
import { concatChunks, generateNodes, getMockServer, HTTP_STATUS_GONE, mockJWT, mockNodesHandlers, mockOrchHandler, mockSaturnOriginHandler, MSW_SERVER_OPTS } from './test-utils.js'
77

8-
const TEST_DEFAULT_ORCH = 'https://orchestrator.strn.pl/nodes'
8+
const TEST_DEFAULT_ORCH = 'https://orchestrator.strn.pl.test/nodes'
99
const TEST_NODES_LIST_KEY = 'saturn-nodes'
10-
const TEST_AUTH = 'https://fz3dyeyxmebszwhuiky7vggmsu0rlkoy.lambda-url.us-west-2.on.aws/'
11-
const TEST_ORIGIN_DOMAIN = 'saturn.ms'
10+
const TEST_AUTH = 'https://auth.test/'
11+
const TEST_ORIGIN_DOMAIN = 'l1s.saturn.test'
1212
const CLIENT_KEY = 'key'
1313

14-
const experimental = true
14+
const options = {
15+
cdnURL: TEST_ORIGIN_DOMAIN,
16+
orchURL: TEST_DEFAULT_ORCH,
17+
authURL: TEST_AUTH,
18+
experimental: true,
19+
clientKey: CLIENT_KEY,
20+
clientId: 'test'
21+
}
1522

1623
describe('Client Fallback', () => {
1724
test('Nodes are loaded from the orchestrator if no storage is passed', async (t) => {
@@ -24,7 +31,7 @@ describe('Client Fallback', () => {
2431
const expectedNodes = generateNodes(2, TEST_ORIGIN_DOMAIN)
2532

2633
// No Storage is injected
27-
const saturn = new Saturn({ clientKey: CLIENT_KEY, experimental })
34+
const saturn = new Saturn({ ...options })
2835
const mockOpts = { orchURL: TEST_DEFAULT_ORCH }
2936

3037
await saturn._loadNodes(mockOpts)
@@ -37,7 +44,7 @@ describe('Client Fallback', () => {
3744

3845
test('Storage is invoked correctly when supplied', async (t) => {
3946
const handlers = [
40-
mockOrchHandler(2, TEST_DEFAULT_ORCH, 'saturn.ms')
47+
mockOrchHandler(2, TEST_DEFAULT_ORCH, TEST_ORIGIN_DOMAIN)
4148
]
4249
const server = getMockServer(handlers)
4350
server.listen(MSW_SERVER_OPTS)
@@ -53,7 +60,7 @@ describe('Client Fallback', () => {
5360
t.mock.method(mockStorage, 'get')
5461
t.mock.method(mockStorage, 'set')
5562

56-
const saturn = new Saturn({ storage: mockStorage, clientKey: CLIENT_KEY, experimental })
63+
const saturn = new Saturn({ storage: mockStorage, ...options })
5764

5865
// Mocking options
5966
const mockOpts = { orchURL: TEST_DEFAULT_ORCH }
@@ -75,7 +82,7 @@ describe('Client Fallback', () => {
7582

7683
test('Storage is loaded first when the orch is slower', async (t) => {
7784
const handlers = [
78-
mockOrchHandler(2, TEST_DEFAULT_ORCH, 'saturn.ms', 1000)
85+
mockOrchHandler(2, TEST_DEFAULT_ORCH, TEST_ORIGIN_DOMAIN, 500)
7986
]
8087
const server = getMockServer(handlers)
8188
server.listen(MSW_SERVER_OPTS)
@@ -90,7 +97,7 @@ describe('Client Fallback', () => {
9097
t.mock.method(mockStorage, 'get')
9198
t.mock.method(mockStorage, 'set')
9299

93-
const saturn = new Saturn({ storage: mockStorage, clientKey: CLIENT_KEY, experimental })
100+
const saturn = new Saturn({ storage: mockStorage, ...options })
94101

95102
// Mocking options
96103
const mockOpts = { orchURL: TEST_DEFAULT_ORCH }
@@ -111,7 +118,7 @@ describe('Client Fallback', () => {
111118

112119
test('Content Fallback fetches a cid properly', async (t) => {
113120
const handlers = [
114-
mockOrchHandler(2, TEST_DEFAULT_ORCH, 'saturn.ms'),
121+
mockOrchHandler(2, TEST_DEFAULT_ORCH, TEST_ORIGIN_DOMAIN),
115122
mockJWT(TEST_AUTH),
116123
mockSaturnOriginHandler(TEST_ORIGIN_DOMAIN, 0, true),
117124
...mockNodesHandlers(2, TEST_ORIGIN_DOMAIN)
@@ -129,7 +136,7 @@ describe('Client Fallback', () => {
129136
t.mock.method(mockStorage, 'get')
130137
t.mock.method(mockStorage, 'set')
131138

132-
const saturn = new Saturn({ storage: mockStorage, clientKey: CLIENT_KEY, clientId: 'test', experimental })
139+
const saturn = new Saturn({ storage: mockStorage, ...options })
133140

134141
const cid = saturn.fetchContentWithFallback('bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4')
135142

@@ -144,7 +151,7 @@ describe('Client Fallback', () => {
144151

145152
test('Content Fallback fetches a cid properly with race', async (t) => {
146153
const handlers = [
147-
mockOrchHandler(5, TEST_DEFAULT_ORCH, 'saturn.ms'),
154+
mockOrchHandler(5, TEST_DEFAULT_ORCH, TEST_ORIGIN_DOMAIN),
148155
mockJWT(TEST_AUTH),
149156
mockSaturnOriginHandler(TEST_ORIGIN_DOMAIN, 0, true),
150157
...mockNodesHandlers(5, TEST_ORIGIN_DOMAIN)
@@ -162,8 +169,7 @@ describe('Client Fallback', () => {
162169
t.mock.method(mockStorage, 'get')
163170
t.mock.method(mockStorage, 'set')
164171

165-
const saturn = new Saturn({ storage: mockStorage, clientKey: CLIENT_KEY, clientId: 'test', experimental })
166-
// const origins =
172+
const saturn = new Saturn({ ...options })
167173

168174
const cid = saturn.fetchContentWithFallback('bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4', { raceNodes: true })
169175

@@ -178,7 +184,7 @@ describe('Client Fallback', () => {
178184

179185
test('Content Fallback with race fetches from consecutive nodes on failure', async (t) => {
180186
const handlers = [
181-
mockOrchHandler(5, TEST_DEFAULT_ORCH, 'saturn.ms'),
187+
mockOrchHandler(5, TEST_DEFAULT_ORCH, TEST_ORIGIN_DOMAIN),
182188
mockJWT(TEST_AUTH),
183189
mockSaturnOriginHandler(TEST_ORIGIN_DOMAIN, 0, true),
184190
...mockNodesHandlers(5, TEST_ORIGIN_DOMAIN, 2)
@@ -196,7 +202,7 @@ describe('Client Fallback', () => {
196202
t.mock.method(mockStorage, 'get')
197203
t.mock.method(mockStorage, 'set')
198204

199-
const saturn = new Saturn({ storage: mockStorage, clientKey: CLIENT_KEY, clientId: 'test', experimental })
205+
const saturn = new Saturn({ storage: mockStorage, ...options })
200206

201207
const cid = saturn.fetchContentWithFallback('bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4', { raceNodes: true })
202208

@@ -211,14 +217,14 @@ describe('Client Fallback', () => {
211217

212218
test('should fetch content from the first node successfully', async () => {
213219
const handlers = [
214-
mockOrchHandler(2, TEST_DEFAULT_ORCH, 'saturn.ms'),
220+
mockOrchHandler(2, TEST_DEFAULT_ORCH, TEST_ORIGIN_DOMAIN),
215221
mockJWT(TEST_AUTH),
216222
...mockNodesHandlers(2, TEST_ORIGIN_DOMAIN)
217223
]
218224

219225
const server = getMockServer(handlers)
220226
server.listen(MSW_SERVER_OPTS)
221-
const saturn = new Saturn({ clientKey: CLIENT_KEY, clientId: 'test', experimental })
227+
const saturn = new Saturn({ ...options })
222228

223229
const fetchContentMock = mock.fn(async function * (cidPath, opts) {
224230
yield Buffer.from('chunk1')
@@ -239,14 +245,14 @@ describe('Client Fallback', () => {
239245
test('should try all nodes and fail if all nodes fail', async () => {
240246
const numNodes = 3
241247
const handlers = [
242-
mockOrchHandler(numNodes, TEST_DEFAULT_ORCH, 'saturn.ms'),
248+
mockOrchHandler(numNodes, TEST_DEFAULT_ORCH, TEST_ORIGIN_DOMAIN),
243249
mockJWT(TEST_AUTH),
244250
...mockNodesHandlers(numNodes, TEST_ORIGIN_DOMAIN)
245251
]
246252

247253
const server = getMockServer(handlers)
248254
server.listen(MSW_SERVER_OPTS)
249-
const saturn = new Saturn({ clientKey: CLIENT_KEY, clientId: 'test', experimental })
255+
const saturn = new Saturn({ ...options })
250256

251257
const fetchContentMock = mock.fn(async function * (cidPath, opts) { throw new Error('Fetch error') }) // eslint-disable-line
252258
saturn.fetchContent = fetchContentMock
@@ -270,14 +276,14 @@ describe('Client Fallback', () => {
270276
test('Should abort fallback on 410s', async () => {
271277
const numNodes = 3
272278
const handlers = [
273-
mockOrchHandler(numNodes, TEST_DEFAULT_ORCH, 'saturn.ms'),
279+
mockOrchHandler(numNodes, TEST_DEFAULT_ORCH, TEST_ORIGIN_DOMAIN),
274280
mockJWT(TEST_AUTH),
275281
...mockNodesHandlers(numNodes, TEST_ORIGIN_DOMAIN, 3, HTTP_STATUS_GONE)
276282
]
277283

278284
const server = getMockServer(handlers)
279285
server.listen(MSW_SERVER_OPTS)
280-
const saturn = new Saturn({ clientKey: CLIENT_KEY, clientId: 'test', experimental })
286+
const saturn = new Saturn({ ...options })
281287
await saturn.loadNodesPromise
282288

283289
let error
@@ -299,14 +305,14 @@ describe('Client Fallback', () => {
299305
test('Should abort fallback on specific errors', async () => {
300306
const numNodes = 3
301307
const handlers = [
302-
mockOrchHandler(numNodes, TEST_DEFAULT_ORCH, 'saturn.ms'),
308+
mockOrchHandler(numNodes, TEST_DEFAULT_ORCH, TEST_ORIGIN_DOMAIN),
303309
mockJWT(TEST_AUTH),
304310
...mockNodesHandlers(numNodes, TEST_ORIGIN_DOMAIN, 3, HTTP_STATUS_GONE)
305311
]
306312

307313
const server = getMockServer(handlers)
308314
server.listen(MSW_SERVER_OPTS)
309-
const saturn = new Saturn({ clientKey: CLIENT_KEY, clientId: 'test', experimental })
315+
const saturn = new Saturn({ ...options })
310316
await saturn.loadNodesPromise
311317

312318
let callCount = 0
@@ -334,14 +340,14 @@ describe('Client Fallback', () => {
334340
test('Handles fallback with chunk overlap correctly', async () => {
335341
const numNodes = 3
336342
const handlers = [
337-
mockOrchHandler(numNodes, TEST_DEFAULT_ORCH, 'saturn.ms'),
343+
mockOrchHandler(numNodes, TEST_DEFAULT_ORCH, TEST_ORIGIN_DOMAIN),
338344
mockJWT(TEST_AUTH),
339345
...mockNodesHandlers(numNodes, TEST_ORIGIN_DOMAIN)
340346
]
341347

342348
const server = getMockServer(handlers)
343349
server.listen(MSW_SERVER_OPTS)
344-
const saturn = new Saturn({ clientKey: CLIENT_KEY, clientId: 'test', experimental })
350+
const saturn = new Saturn({ ...options })
345351

346352
let callCount = 0
347353
const fetchContentMock = mock.fn(async function * (cidPath, opts) {
@@ -373,14 +379,14 @@ describe('Client Fallback', () => {
373379
test('should handle byte chunk overlaps correctly', async () => {
374380
const numNodes = 3
375381
const handlers = [
376-
mockOrchHandler(numNodes, TEST_DEFAULT_ORCH, 'saturn.ms'),
382+
mockOrchHandler(numNodes, TEST_DEFAULT_ORCH, TEST_ORIGIN_DOMAIN),
377383
mockJWT(TEST_AUTH),
378384
...mockNodesHandlers(numNodes, TEST_ORIGIN_DOMAIN)
379385
]
380386

381387
const server = getMockServer(handlers)
382388
server.listen(MSW_SERVER_OPTS)
383-
const saturn = new Saturn({ clientKey: CLIENT_KEY, clientId: 'test', experimental })
389+
const saturn = new Saturn({ ...options })
384390

385391
let callCount = 0
386392
let fetchContentMock = mock.fn(async function * (cidPath, opts) {

‎test/test-utils.js

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ export function generateNodes (count, originDomain) {
3333
const nodeIp = `node${i}`
3434
const node = {
3535
ip: nodeIp,
36+
id: nodeIp,
3637
weight: 50,
3738
distance: 100,
3839
url: `https://${nodeIp}.${originDomain}`
@@ -52,6 +53,7 @@ export function generateNodes (count, originDomain) {
5253
*/
5354
export function mockSaturnOriginHandler (cdnURL, delay = 0, error = false) {
5455
cdnURL = addHttpPrefix(cdnURL)
56+
cdnURL = `${cdnURL}/ipfs/:cid`
5557
return rest.get(cdnURL, (req, res, ctx) => {
5658
if (error) {
5759
throw Error('Simulated Error')

0 commit comments

Comments
 (0)