feat: enable custom formats for dag put and get (#3347)

Ensures we can use custom/extra IPLD formats with `ipfs.dag.get` and `ipfs.dag.put` over HTTP as well as directly into core.

Adds an example to show how to do it.

The basic idea is you configure your node with the extra formats and pass that node into the http server, so instead of having the IPLD format logic in the http method endpoint and in core it's just in core.  The http client works the same as it did before.

Co-authored-by: Vasco Santos <[email protected]>
Co-authored-by: Janko Simonovic <[email protected]>

Author: 3 people

src/dag/get.js

@@ -1,40 +1,29 @@
 'use strict'
 
-const dagPB = require('ipld-dag-pb')
-const dagCBOR = require('ipld-dag-cbor')
-const raw = require('ipld-raw')
 const configure = require('../lib/configure')
+const multicodec = require('multicodec')
+const loadFormat = require('../lib/ipld-formats')
 
-const resolvers = {
-  'dag-cbor': dagCBOR.resolver,
-  'dag-pb': dagPB.resolver,
-  raw: raw.resolver
-}
-
-module.exports = configure((api, options) => {
-  const getBlock = require('../block/get')(options)
-  const dagResolve = require('./resolve')(options)
+module.exports = configure((api, opts) => {
+  const getBlock = require('../block/get')(opts)
+  const dagResolve = require('./resolve')(opts)
+  const load = loadFormat(opts.ipld)
 
   /**
    * @type {import('..').Implements<import('ipfs-core/src/components/dag/get')>}
    */
   const get = async (cid, options = {}) => {
     const resolved = await dagResolve(cid, options)
     const block = await getBlock(resolved.cid, options)
-    const dagResolver = resolvers[resolved.cid.codec]
 
-    if (!dagResolver) {
-      throw Object.assign(
-        new Error(`Missing IPLD format "${resolved.cid.codec}"`),
-        { missingMulticodec: resolved.cid.codec }
-      )
-    }
+    const codecName = multicodec.getName(resolved.cid.code)
+    const format = await load(codecName)
 
-    if (resolved.cid.codec === 'raw' && !resolved.remainderPath) {
+    if (resolved.cid.code === multicodec.RAW && !resolved.remainderPath) {
       resolved.remainderPath = '/'
     }
 
-    return dagResolver.resolve(block.data, resolved.remainderPath)
+    return format.resolver.resolve(block.data, resolved.remainderPath)
   }
 
   return get

src/dag/put.js

@@ -1,8 +1,5 @@
 'use strict'
 
-const dagCBOR = require('ipld-dag-cbor')
-const dagPB = require('ipld-dag-pb')
-const ipldRaw = require('ipld-raw')
 const CID = require('cids')
 const multihash = require('multihashes')
 const configure = require('../lib/configure')
@@ -11,19 +8,10 @@ const toUrlSearchParams = require('../lib/to-url-search-params')
 const { anySignal } = require('any-signal')
 const AbortController = require('native-abort-controller')
 const multicodec = require('multicodec')
+const loadFormat = require('../lib/ipld-formats')
 
 module.exports = configure((api, opts) => {
-  const formats = {
-    [multicodec.DAG_PB]: dagPB,
-    [multicodec.DAG_CBOR]: dagCBOR,
-    [multicodec.RAW]: ipldRaw
-  }
-
-  const ipldOptions = (opts && opts.ipld) || {}
-  const configuredFormats = (ipldOptions && ipldOptions.formats) || []
-  configuredFormats.forEach(format => {
-    formats[format.codec] = format
-  })
+  const load = loadFormat(opts.ipld)
 
   /**
    * @type {import('..').Implements<import('ipfs-core/src/components/dag/put')>}
@@ -39,7 +27,7 @@ module.exports = configure((api, opts) => {
       const cid = new CID(options.cid)
       options = {
         ...options,
-        format: cid.codec,
+        format: multicodec.getName(cid.code),
         hashAlg: multihash.decode(cid.multihash).name
       }
       delete options.cid
@@ -52,24 +40,7 @@ module.exports = configure((api, opts) => {
       ...options
     }
 
-    const number = multicodec.getNumber(settings.format)
-    let format = formats[number]
-
-    if (!format) {
-      if (opts && opts.ipld && opts.ipld.loadFormat) {
-        // @ts-ignore - loadFormat expect string but it could be a number
-        format = await opts.ipld.loadFormat(settings.format)
-      }
-
-      if (!format) {
-        throw new Error('Format unsupported - please add support using the options.ipld.formats or options.ipld.loadFormat options')
-      }
-    }
-
-    if (!format.util || !format.util.serialize) {
-      throw new Error('Format does not support utils.serialize function')
-    }
-
+    const format = await load(settings.format)
     const serialized = format.util.serialize(dagNode)
 
     // allow aborting requests on body errors

src/lib/ipld-formats.js

@@ -0,0 +1,57 @@
+'use strict'
+
+const dagPB = require('ipld-dag-pb')
+const dagCBOR = require('ipld-dag-cbor')
+const raw = require('ipld-raw')
+const multicodec = require('multicodec')
+
+const noop = () => {}
+
+/**
+ * @typedef {import('cids')} CID
+ */
+
+/**
+ * Return an object containing supported IPLD Formats
+ *
+ * @param {object} [options] - IPLD options passed to the http client constructor
+ * @param {Array} [options.formats] - A list of IPLD Formats to use
+ * @param {Function} [options.loadFormat] - An async function that can load a format when passed a codec number
+ * @returns {Function}
+ */
+module.exports = ({ formats = [], loadFormat = noop } = {}) => {
+  formats = formats || []
+  loadFormat = loadFormat || noop
+
+  const configuredFormats = {
+    [multicodec.DAG_PB]: dagPB,
+    [multicodec.DAG_CBOR]: dagCBOR,
+    [multicodec.RAW]: raw
+  }
+
+  formats.forEach(format => {
+    configuredFormats[format.codec] = format
+  })
+
+  /**
+   * Attempts to load an IPLD format for the passed CID
+   *
+   * @param {string} codec - The code to load the format for
+   * @returns {Promise<object>} - An IPLD format
+   */
+  const loadResolver = async (codec) => {
+    const number = multicodec.getNumber(codec)
+    const format = configuredFormats[number] || await loadFormat(codec)
+
+    if (!format) {
+      throw Object.assign(
+        new Error(`Missing IPLD format "${codec}"`),
+        { missingMulticodec: codec }
+      )
+    }
+
+    return format
+  }
+
+  return loadResolver
+}

test/dag.spec.js

@@ -73,7 +73,7 @@ describe('.dag', function () {
   it('should error when putting node with esoteric format', () => {
     const node = uint8ArrayFromString('some data')
 
-    return expect(ipfs.dag.put(node, { format: 'git-raw', hashAlg: 'sha2-256' })).to.eventually.be.rejectedWith(/Format unsupported/)
+    return expect(ipfs.dag.put(node, { format: 'git-raw', hashAlg: 'sha2-256' })).to.eventually.be.rejectedWith(/Missing IPLD format/)
   })
 
   it('should attempt to load an unsupported format', async () => {