fix: remove non default ipld formats in the browser (#1980)

BREAKING CHANGE: Browser application bundles now include only `ipld-dag-pb`, `ipld-dag-cbor` and `ipld-raw` IPLD codecs. Other codecs should be added manually, see https://github.com/ipfs/js-ipfs/blob/master/README.md#optionsipld for details.

* In Node.js `require('ipfs')`
    * all IPLD formats included
* In browser application bundle `require('ipfs')` bundled with webpack/browserify/etc.
    * only `ipld-dag-pb`, `ipld-dag-cbor` and `ipld-raw` included
* CDN bundle `<script src="https://unpkg.com/ipfs/dist/index.min.js"></script>`
    * all IPLD formats included

Co-Authored-By: hugomrdias <[email protected]>

Author: hugomrdias

.aegir.js

@@ -10,7 +10,8 @@ const preloadNode = MockPreloadNode.createNode()
 module.exports = {
   webpack: {
     resolve: {
-      mainFields: ['browser', 'main']
+      mainFields: ['browser', 'main'],
+      aliasFields: ['browser', 'browser-all-ipld-formats'],
     }
   },
   karma: {

README.md

@@ -326,6 +326,146 @@ Enable and configure experimental features.
 
 Modify the default IPFS node config. This object will be *merged* with the default config; it will not replace it.
 
+##### `options.ipld`
+
+ | Type | Default |
+|------|---------|
+| object |  [`ipld-nodejs.js`](https://github.com/ipfs/js-ipfs/tree/master/src/core/runtime/ipld-nodejs.js) in Node.js, [`ipld-browser.js`](https://github.com/ipfs/js-ipfs/tree/master/src/core/runtime/ipld-browser.js) in browsers |
+
+ Modify the default IPLD config. This object will be *merged* with the default config; it will not replace it. Check IPLD [docs](https://github.com/ipld/js-ipld#ipld-constructor) for more information on the available options.
+
+ > Browser config does **NOT** include by default all the IPLD formats. Only `ipld-dag-pb`, `ipld-dag-cbor` and `ipld-raw` are included.
+
+ To add support for other formats we provide two options, one sync and another async.
+
+ Examples for the sync option:
+
+<details><summary>ESM Environments</summary>
+
+```js
+import ipldGit from 'ipld-git'
+import ipldBitcoin from 'ipld-bitcoin'
+
+const node = new IPFS(
+  {
+    ipld: {
+      formats: [ipldGit, ipldBitcoin]
+    }
+  }
+)
+```
+</details>
+<details><summary>Commonjs Environments</summary>
+
+```js
+const node = new IPFS(
+  {
+    ipld: {
+      formats: [require('ipld-git'), require('ipld-bitcoin')]
+    }
+  }
+)
+```
+</details>
+<details><summary>Using script tags</summary>
+
+```html
+<script src="https://unpkg.com/ipfs"></script>
+<script src="https://unpkg.com/ipld-git"></script>
+<script src="https://unpkg.com/ipld-bitcoin"></script>
+<script>
+const node = new self.IPFS(
+  {
+    ipld: {
+      formats: [self.ipldGit, self.ipldBitcoin]
+    }
+  }
+)
+</script>
+```
+</details>
+
+ Examples for the async option:
+
+
+<details><summary>ESM Environments</summary>
+
+```js
+const node = new IPFS(
+  {
+    ipld: {
+      async loadFormat (codec) {
+        if (codec === 'git-raw') {
+          return import('ipld-git') // This is a dynamic import
+        } else {
+          throw new Error('unable to load format ' + multicodec.print[codec])
+        }
+      }
+    }
+  }
+)
+```
+> For more information about dynamic imports please check [webpack docs](https://webpack.js.org/guides/code-splitting/#dynamic-imports) or search your bundler documention.
+
+Using dynamic imports will tell your bundler to create a separate file (normally called *chunk*) that will **only** be requested by the browser if it's really needed. This strategy will reduce your bundle size and load times without removing any functionality.
+
+With Webpack IPLD formats can even be grouped together using magic comments `import(/* webpackChunkName: "ipld-formats" */ 'ipld-git')` to produce a single file with all of them.
+
+</details>
+<details><summary>Commonjs Environments</summary>
+
+```js
+const node = new IPFS(
+  {
+    ipld: {
+      async loadFormat (codec) {
+        if (codec === 'git-raw') {
+          return require('ipld-git')
+        } else {
+          throw new Error('unable to load format ' + multicodec.print[codec])
+        }
+      }
+    }
+  }
+)
+```
+</details>
+
+<details><summary>Using Script tags</summary>
+
+```js
+<script src="https://unpkg.com/ipfs"></script>
+<script>
+
+const load = (url, cb) => {
+  const script = document.createElement('script')
+  script.src = url
+  script.onload = () => cb()
+  script.onerror = () => cb(new Error('Unable to load script'))
+  document.body.appendChild(script);
+};
+
+const node = new self.IPFS(
+  {
+    ipld: {
+      loadFormat (codec, cb) {
+        switch (codec) {
+          case 'git-raw':
+            return load('https://unpkg.com/ipld-git', cb)
+          case 'bitcoin-block':
+            return load('https://unpkg.com/ipld-bitcoin', cb)
+          default:
+            throw new Error('unable to load format ' + multicodec.print[codec])
+        }
+      }
+    }
+  }
+)
+</script>
+```
+</details>
+
+
 ##### `options.libp2p`
 
 | Type | Default |

package.json

@@ -16,10 +16,14 @@
     "./src/core/runtime/libp2p-nodejs.js": "./src/core/runtime/libp2p-browser.js",
     "./src/core/runtime/preload-nodejs.js": "./src/core/runtime/preload-browser.js",
     "./src/core/runtime/repo-nodejs.js": "./src/core/runtime/repo-browser.js",
+    "./src/core/runtime/ipld-nodejs.js": "./src/core/runtime/ipld-browser.js",
     "./test/utils/create-repo-nodejs.js": "./test/utils/create-repo-browser.js",
     "stream": "readable-stream",
     "joi": "joi-browser"
   },
+  "browser-all-ipld-formats": {
+    "./src/core/runtime/ipld-browser.js": "./src/core/runtime/ipld-browser-all.js"
+  },
   "engines": {
     "node": ">=10.0.0",
     "npm": ">=6.0.0"
@@ -115,9 +119,11 @@
     "ipfs-unixfs-importer": "~0.38.5",
     "ipld": "~0.21.1",
     "ipld-bitcoin": "~0.1.8",
+    "ipld-dag-cbor": "~0.13.1",
     "ipld-dag-pb": "~0.15.3",
     "ipld-ethereum": "^2.0.1",
     "ipld-git": "~0.3.0",
+    "ipld-raw": "^2.0.1",
     "ipld-zcash": "~0.1.6",
     "ipns": "~0.5.0",
     "is-ipfs": "~0.6.0",

src/core/index.js

@@ -25,40 +25,7 @@ const components = require('./components')
 const defaultRepo = require('./runtime/repo-nodejs')
 const preload = require('./preload')
 const mfsPreload = require('./mfs-preload')
-
-// All known (non-default) IPLD formats
-const IpldFormats = {
-  get 'bitcoin-block' () {
-    return require('ipld-bitcoin')
-  },
-  get 'eth-account-snapshot' () {
-    return require('ipld-ethereum').ethAccountSnapshot
-  },
-  get 'eth-block' () {
-    return require('ipld-ethereum').ethBlock
-  },
-  get 'eth-block-list' () {
-    return require('ipld-ethereum').ethBlockList
-  },
-  get 'eth-state-trie' () {
-    return require('ipld-ethereum').ethStateTrie
-  },
-  get 'eth-storage-trie' () {
-    return require('ipld-ethereum').ethStorageTrie
-  },
-  get 'eth-tx' () {
-    return require('ipld-ethereum').ethTx
-  },
-  get 'eth-tx-trie' () {
-    return require('ipld-ethereum').ethTxTrie
-  },
-  get 'git-raw' () {
-    return require('ipld-git')
-  },
-  get 'zcash-block' () {
-    return require('ipld-zcash')
-  }
-}
+const ipldOptions = require('./runtime/ipld-nodejs')
 
 class IPFS extends EventEmitter {
   constructor (options) {
@@ -106,14 +73,7 @@ class IPFS extends EventEmitter {
     this._peerInfo = undefined
     this._bitswap = undefined
     this._blockService = new BlockService(this._repo)
-    this._ipld = new Ipld({
-      blockService: this._blockService,
-      loadFormat: (codec, callback) => {
-        this.log('Loading IPLD format', codec)
-        if (IpldFormats[codec]) return callback(null, IpldFormats[codec])
-        callback(new Error(`Missing IPLD format "${codec}"`))
-      }
-    })
+    this._ipld = new Ipld(ipldOptions(this._blockService, this._options.ipld, this.log))
     this._preload = preload(this)
     this._mfsPreload = mfsPreload(this)
     this._ipns = undefined

src/core/runtime/ipld-browser-all.js

@@ -0,0 +1,26 @@
+'use strict'
+const mergeOptions = require('merge-options')
+
+module.exports = (blockService, options = {}) => {
+  return mergeOptions.call(
+    // ensure we have the defaults formats even if the user overrides `formats: []`
+    { concatArrays: true },
+    {
+      blockService: blockService,
+      formats: [
+        require('ipld-dag-cbor'),
+        require('ipld-dag-pb'),
+        require('ipld-raw'),
+        require('ipld-bitcoin'),
+        require('ipld-ethereum').ethAccountSnapshot,
+        require('ipld-ethereum').ethBlock,
+        require('ipld-ethereum').ethBlockList,
+        require('ipld-ethereum').ethStateTrie,
+        require('ipld-ethereum').ethStorageTrie,
+        require('ipld-ethereum').ethTx,
+        require('ipld-ethereum').ethTxTrie,
+        require('ipld-git'),
+        require('ipld-zcash')
+      ]
+    }, options)
+}

src/core/runtime/ipld-browser.js

@@ -0,0 +1,15 @@
+'use strict'
+const mergeOptions = require('merge-options')
+const ipldDagCbor = require('ipld-dag-cbor')
+const ipldDagPb = require('ipld-dag-pb')
+const ipldRaw = require('ipld-raw')
+
+module.exports = (blockService, options = {}) => {
+  return mergeOptions.call(
+    // ensure we have the defaults formats even if the user overrides `formats: []`
+    { concatArrays: true },
+    {
+      blockService: blockService,
+      formats: [ipldDagCbor, ipldDagPb, ipldRaw]
+    }, options)
+}

src/core/runtime/ipld-nodejs.js

@@ -0,0 +1,54 @@
+'use strict'
+const mergeOptions = require('merge-options')
+const ipldDagCbor = require('ipld-dag-cbor')
+const ipldDagPb = require('ipld-dag-pb')
+const ipldRaw = require('ipld-raw')
+
+// All known (non-default) IPLD formats
+const IpldFormats = {
+  get 'bitcoin-block' () {
+    return require('ipld-bitcoin')
+  },
+  get 'eth-account-snapshot' () {
+    return require('ipld-ethereum').ethAccountSnapshot
+  },
+  get 'eth-block' () {
+    return require('ipld-ethereum').ethBlock
+  },
+  get 'eth-block-list' () {
+    return require('ipld-ethereum').ethBlockList
+  },
+  get 'eth-state-trie' () {
+    return require('ipld-ethereum').ethStateTrie
+  },
+  get 'eth-storage-trie' () {
+    return require('ipld-ethereum').ethStorageTrie
+  },
+  get 'eth-tx' () {
+    return require('ipld-ethereum').ethTx
+  },
+  get 'eth-tx-trie' () {
+    return require('ipld-ethereum').ethTxTrie
+  },
+  get 'git-raw' () {
+    return require('ipld-git')
+  },
+  get 'zcash-block' () {
+    return require('ipld-zcash')
+  }
+}
+
+module.exports = (blockService, options = {}, log) => {
+  return mergeOptions.call(
+    // ensure we have the defaults formats even if the user overrides `formats: []`
+    { concatArrays: true },
+    {
+      blockService: blockService,
+      formats: [ipldDagCbor, ipldDagPb, ipldRaw],
+      loadFormat: (codec, callback) => {
+        log('Loading IPLD format', codec)
+        if (IpldFormats[codec]) return callback(null, IpldFormats[codec])
+        callback(new Error(`Missing IPLD format "${codec}"`))
+      }
+    }, options)
+}