Skip to content
This repository was archived by the owner on Feb 12, 2024. It is now read-only.

Commit 649b755

Browse files
alanshawalanshaw
authored
feat: preload on content fetch requests (#1475)
When JS IPFS requests content not stored locally it needs to ask the peers it knows about to provide it if they have it. The peers are not relays so if they don't have it, they won't find it and provide it to JS IPFS. However, if we issue a preload request prior to these requests we prompt the preload nodes to fetch the content using their DHT and they can then provide it to JS IPFS. License: MIT Signed-off-by: Alan Shaw <[email protected]> * chore: update dependencies (#1473) * chore: update libp2p and is-ipfs dependencies License: MIT Signed-off-by: Alan Shaw <[email protected]> * chore: update all the deps * fix: rm non used boostrappers from tests * test: increase mfs preload timeout * test: pin-set timeout increase * test: add preload tests for content retrieval License: MIT Signed-off-by: Alan Shaw <[email protected]> * fix: GET request not HEAD License: MIT Signed-off-by: Alan Shaw <[email protected]>
1 parent 90e9f68 commit 649b755

File tree

15 files changed

+223
-45
lines changed

15 files changed

+223
-45
lines changed

‎package.json‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@
5858
},
5959
"homepage": "https://github.com/ipfs/js-ipfs#readme",
6060
"devDependencies": {
61-
"aegir": "^15.0.0",
61+
"aegir": "^15.1.0",
6262
"buffer-loader": "~0.0.1",
6363
"chai": "^4.1.2",
6464
"delay": "^3.0.0",
@@ -70,8 +70,8 @@
7070
"expose-loader": "~0.7.5",
7171
"form-data": "^2.3.2",
7272
"hat": "0.0.3",
73-
"interface-ipfs-core": "~0.72.0",
74-
"ipfsd-ctl": "~0.37.5",
73+
"interface-ipfs-core": "~0.72.1",
74+
"ipfsd-ctl": "~0.38.0",
7575
"mocha": "^5.2.0",
7676
"ncp": "^2.0.0",
7777
"nexpect": "~0.5.0",
@@ -93,7 +93,7 @@
9393
"byteman": "^1.3.5",
9494
"cids": "~0.5.3",
9595
"debug": "^3.1.0",
96-
"file-type": "^8.0.0",
96+
"file-type": "^8.1.0",
9797
"filesize": "^3.6.1",
9898
"fnv1a": "^1.0.1",
9999
"fsm-event": "^2.1.0",
@@ -104,30 +104,30 @@
104104
"hoek": "^5.0.3",
105105
"human-to-milliseconds": "^1.0.0",
106106
"interface-datastore": "~0.4.2",
107-
"ipfs-api": "^22.2.1",
108-
"ipfs-bitswap": "~0.20.2",
107+
"ipfs-api": "^22.2.4",
108+
"ipfs-bitswap": "~0.20.3",
109109
"ipfs-block": "~0.7.1",
110110
"ipfs-block-service": "~0.14.0",
111111
"ipfs-http-response": "~0.1.2",
112-
"ipfs-mfs": "~0.2.2",
112+
"ipfs-mfs": "~0.2.3",
113113
"ipfs-multipart": "~0.1.0",
114114
"ipfs-repo": "~0.22.1",
115115
"ipfs-unixfs": "~0.1.15",
116116
"ipfs-unixfs-engine": "~0.31.3",
117117
"ipld": "~0.17.3",
118118
"ipld-dag-cbor": "~0.12.1",
119-
"ipld-dag-pb": "~0.14.5",
120-
"is-ipfs": "~0.3.2",
119+
"ipld-dag-pb": "~0.14.6",
120+
"is-ipfs": "~0.4.2",
121121
"is-pull-stream": "~0.0.0",
122122
"is-stream": "^1.1.0",
123123
"joi": "^13.4.0",
124124
"joi-browser": "^13.4.0",
125125
"joi-multiaddr": "^2.0.0",
126-
"libp2p": "~0.22.0",
126+
"libp2p": "~0.23.0",
127127
"libp2p-bootstrap": "~0.9.3",
128128
"libp2p-circuit": "~0.2.0",
129129
"libp2p-floodsub": "~0.15.0",
130-
"libp2p-kad-dht": "~0.10.0",
130+
"libp2p-kad-dht": "~0.10.1",
131131
"libp2p-keychain": "~0.3.1",
132132
"libp2p-mdns": "~0.12.0",
133133
"libp2p-mplex": "~0.8.0",
@@ -138,7 +138,7 @@
138138
"libp2p-websockets": "~0.12.0",
139139
"lodash": "^4.17.10",
140140
"mafmt": "^6.0.0",
141-
"mime-types": "^2.1.18",
141+
"mime-types": "^2.1.19",
142142
"mkdirp": "~0.5.1",
143143
"multiaddr": "^5.0.0",
144144
"multiaddr-to-uri": "^4.0.0",

‎src/core/components/block.js‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,20 @@ const promisify = require('promisify-es6')
99

1010
module.exports = function block (self) {
1111
return {
12-
get: promisify((cid, callback) => {
12+
get: promisify((cid, options, callback) => {
13+
if (typeof options === 'function') {
14+
callback = options
15+
options = {}
16+
}
17+
18+
options = options || {}
19+
1320
cid = cleanCid(cid)
21+
22+
if (options.preload !== false) {
23+
self._preload(cid)
24+
}
25+
1426
self._blockService.get(cid, callback)
1527
}),
1628
put: promisify((block, options, callback) => {
@@ -65,9 +77,18 @@ module.exports = function block (self) {
6577
cid = cleanCid(cid)
6678
self._blockService.delete(cid, callback)
6779
}),
68-
stat: promisify((cid, callback) => {
80+
stat: promisify((cid, options, callback) => {
81+
if (typeof options === 'function') {
82+
callback = options
83+
options = {}
84+
}
85+
6986
cid = cleanCid(cid)
7087

88+
if (options.preload !== false) {
89+
self._preload(cid)
90+
}
91+
7192
self._blockService.get(cid, (err, block) => {
7293
if (err) {
7394
return callback(err)

‎src/core/components/dag.js‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,10 @@ module.exports = function dag (self) {
6666
}
6767
}
6868

69+
if (options.preload !== false) {
70+
self._preload(cid)
71+
}
72+
6973
self._ipld.get(cid, path, options, callback)
7074
}),
7175

@@ -100,17 +104,28 @@ module.exports = function dag (self) {
100104
}
101105
}
102106

107+
if (options.preload !== false) {
108+
self._preload(cid)
109+
}
110+
103111
pull(
104112
self._ipld.treeStream(cid, path, options),
105113
pull.collect(callback)
106114
)
107115
}),
108116

109117
// TODO - use IPLD selectors once they are implemented
110-
_getRecursive: promisify((multihash, callback) => {
118+
_getRecursive: promisify((multihash, options, callback) => {
111119
// gets flat array of all DAGNodes in tree given by multihash
112120

113-
self.dag.get(new CID(multihash), (err, res) => {
121+
if (typeof options === 'function') {
122+
callback = options
123+
options = {}
124+
}
125+
126+
options = options || {}
127+
128+
self.dag.get(new CID(multihash), '', options, (err, res) => {
114129
if (err) { return callback(err) }
115130

116131
mapAsync(res.value.links, (link, cb) => {

‎src/core/components/files.js‎

Lines changed: 50 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -182,11 +182,17 @@ module.exports = function files (self) {
182182
throw new Error('You must supply an ipfsPath')
183183
}
184184

185+
options = options || {}
186+
185187
ipfsPath = normalizePath(ipfsPath)
186188
const pathComponents = ipfsPath.split('/')
187189
const restPath = normalizePath(pathComponents.slice(1).join('/'))
188190
const filterFile = (file) => (restPath && file.path === restPath) || (file.path === ipfsPath)
189191

192+
if (options.preload !== false) {
193+
self._preload(pathComponents[0])
194+
}
195+
190196
const d = deferred.source()
191197

192198
pull(
@@ -213,16 +219,21 @@ module.exports = function files (self) {
213219
}
214220

215221
function _lsPullStreamImmutable (ipfsPath, options) {
222+
options = options || {}
223+
216224
const path = normalizePath(ipfsPath)
217-
const recursive = options && options.recursive
218-
const pathDepth = path.split('/').length
225+
const recursive = options.recursive
226+
const pathComponents = path.split('/')
227+
const pathDepth = pathComponents.length
219228
const maxDepth = recursive ? global.Infinity : pathDepth
220-
const opts = Object.assign({}, {
221-
maxDepth: maxDepth
222-
}, options)
229+
options.maxDepth = options.maxDepth || maxDepth
230+
231+
if (options.preload !== false) {
232+
self._preload(pathComponents[0])
233+
}
223234

224235
return pull(
225-
exporter(ipfsPath, self._ipld, opts),
236+
exporter(ipfsPath, self._ipld, options),
226237
pull.filter(node =>
227238
recursive ? node.depth >= pathDepth : node.depth === pathDepth
228239
),
@@ -334,8 +345,11 @@ module.exports = function files (self) {
334345
options = {}
335346
}
336347

337-
if (typeof callback !== 'function') {
338-
throw new Error('Please supply a callback to ipfs.files.get')
348+
options = options || {}
349+
350+
if (options.preload !== false) {
351+
const pathComponents = normalizePath(ipfsPath).split('/')
352+
self._preload(pathComponents[0])
339353
}
340354

341355
pull(
@@ -359,6 +373,13 @@ module.exports = function files (self) {
359373
}),
360374

361375
getReadableStream: (ipfsPath, options) => {
376+
options = options || {}
377+
378+
if (options.preload !== false) {
379+
const pathComponents = normalizePath(ipfsPath).split('/')
380+
self._preload(pathComponents[0])
381+
}
382+
362383
return toStream.source(
363384
pull(
364385
exporter(ipfsPath, self._ipld, options),
@@ -375,6 +396,13 @@ module.exports = function files (self) {
375396
},
376397

377398
getPullStream: (ipfsPath, options) => {
399+
options = options || {}
400+
401+
if (options.preload !== false) {
402+
const pathComponents = normalizePath(ipfsPath).split('/')
403+
self._preload(pathComponents[0])
404+
}
405+
378406
return exporter(ipfsPath, self._ipld, options)
379407
},
380408

@@ -384,6 +412,13 @@ module.exports = function files (self) {
384412
options = {}
385413
}
386414

415+
options = options || {}
416+
417+
if (options.preload !== false) {
418+
const pathComponents = normalizePath(ipfsPath).split('/')
419+
self._preload(pathComponents[0])
420+
}
421+
387422
pull(
388423
_lsPullStreamImmutable(ipfsPath, options),
389424
pull.collect((err, values) => {
@@ -397,6 +432,13 @@ module.exports = function files (self) {
397432
}),
398433

399434
lsReadableStreamImmutable: (ipfsPath, options) => {
435+
options = options || {}
436+
437+
if (options.preload !== false) {
438+
const pathComponents = normalizePath(ipfsPath).split('/')
439+
self._preload(pathComponents[0])
440+
}
441+
400442
return toStream.source(_lsPullStreamImmutable(ipfsPath, options))
401443
},
402444

‎src/core/components/mfs.js‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,23 @@ module.exports = self => {
77
const mfsSelf = Object.assign({}, self)
88

99
// A patched dag API to ensure preload doesn't happen for MFS operations
10+
// (MFS is preloaded periodically)
1011
mfsSelf.dag = Object.assign({}, self.dag, {
12+
get: promisify((cid, path, opts, cb) => {
13+
if (typeof path === 'function') {
14+
cb = path
15+
path = undefined
16+
}
17+
18+
if (typeof opts === 'function') {
19+
cb = opts
20+
opts = {}
21+
}
22+
23+
opts = Object.assign({}, opts, { preload: false })
24+
25+
return self.dag.get(cid, path, opts, cb)
26+
}),
1127
put: promisify((node, opts, cb) => {
1228
if (typeof opts === 'function') {
1329
cb = opts

‎src/core/components/object.js‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
'use strict'
22

33
const waterfall = require('async/waterfall')
4+
const setImmediate = require('async/setImmediate')
45
const promisify = require('promisify-es6')
56
const dagPB = require('ipld-dag-pb')
67
const DAGNode = dagPB.DAGNode
78
const DAGLink = dagPB.DAGLink
89
const CID = require('cids')
910
const mh = require('multihashes')
1011
const Unixfs = require('ipfs-unixfs')
11-
const assert = require('assert')
1212

1313
function normalizeMultihash (multihash, enc) {
1414
if (typeof multihash === 'string') {
@@ -116,7 +116,9 @@ module.exports = function object (self) {
116116
let data
117117

118118
if (template) {
119-
assert(template === 'unixfs-dir', 'unkown template')
119+
if (template !== 'unixfs-dir') {
120+
return setImmediate(() => callback(new Error('unknown template')))
121+
}
120122
data = (new Unixfs('directory')).marshal()
121123
} else {
122124
data = Buffer.alloc(0)
@@ -221,6 +223,10 @@ module.exports = function object (self) {
221223
cid = cid.toV1()
222224
}
223225

226+
if (options.preload !== false) {
227+
self._preload(cid)
228+
}
229+
224230
self._ipld.get(cid, (err, result) => {
225231
if (err) {
226232
return callback(err)

‎src/core/components/pin-set.js‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ exports = module.exports = function (dag) {
7474

7575
seen[bs58Link] = true
7676

77-
dag.get(multihash, (err, res) => {
77+
dag.get(multihash, '', { preload: false }, (err, res) => {
7878
if (err) { return someCb(err) }
7979
searchChildren(res.value, someCb)
8080
})
@@ -184,7 +184,7 @@ exports = module.exports = function (dag) {
184184
return callback(new Error('No link found with name ' + name))
185185
}
186186

187-
dag.get(link.multihash, (err, res) => {
187+
dag.get(link.multihash, '', { preload: false }, (err, res) => {
188188
if (err) { return callback(err) }
189189
const keys = []
190190
const step = link => keys.push(link.multihash)
@@ -211,7 +211,7 @@ exports = module.exports = function (dag) {
211211

212212
if (!emptyKey.equals(linkHash)) {
213213
// walk the links of this fanout bin
214-
return dag.get(linkHash, (err, res) => {
214+
return dag.get(linkHash, '', { preload: false }, (err, res) => {
215215
if (err) { return eachCb(err) }
216216
pinSet.walkItems(res.value, step, eachCb)
217217
})

‎src/core/components/pin.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -360,7 +360,7 @@ module.exports = (self) => {
360360
(_, cb) => repo.datastore.has(pinDataStoreKey, cb),
361361
(has, cb) => has ? cb() : cb(new Error('No pins to load')),
362362
(cb) => repo.datastore.get(pinDataStoreKey, cb),
363-
(mh, cb) => dag.get(new CID(mh), cb)
363+
(mh, cb) => dag.get(new CID(mh), '', { preload: false }, cb)
364364
], (err, pinRoot) => {
365365
if (err) {
366366
if (err.message === 'No pins to load') {

‎src/core/runtime/preload-browser.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ module.exports = function preload (url, callback) {
1111

1212
const req = new self.XMLHttpRequest()
1313

14-
req.open('HEAD', url)
14+
req.open('GET', url)
1515

1616
req.onreadystatechange = function () {
1717
if (this.readyState !== this.DONE) {

0 commit comments

Comments
 (0)