feat!: update http-client DAG API to match [email protected]

Fixes: #3914
Ref: https://github.com/ipfs/go-ipfs/blob/master/CHANGELOG.md#v0100-2021-09-30

--format and --input-enc have been replaced with --input-codec and
--store-codec and mean something a little different. You now supply
raw input and instruct the server which --input-codec that data is
in which it will decode, then re-encode with --store-codec before
storing it and providing you with the CID.

We accept plain JavaScript objects to encode with --store-codec via
the API here, defaulting to dag-cbor, and send that to the server as
encoded bytes using that codec, to be stored using that codec.

If you supply an --input-codec then we assume you're supplying raw,
encoded bytes using that codec and we pass that directly on to the
server to handle.

Author: rvagg

packages/ipfs-core-types/src/dag/index.ts

@@ -132,7 +132,12 @@ export interface GetResult {
 
 export interface PutOptions extends AbortOptions, PreloadOptions {
   /**
-   * The codec to use to create the CID (defaults to 'dag-cbor')
+   * The codec that the input object is encoded with (defaults to 'dag-json')
+   */
+  inputCodec?: string
+
+  /**
+   * The codec that the stored object will be encoded with (defaults to 'dag-cbor')
    */
   format?: string
 

packages/ipfs-http-client/package.json

@@ -55,6 +55,7 @@
   },
   "dependencies": {
     "@ipld/dag-cbor": "^6.0.5",
+    "@ipld/dag-json": "^8.0.1",
     "@ipld/dag-pb": "^2.1.3",
     "abort-controller": "^3.0.0",
     "any-signal": "^2.1.2",
@@ -76,7 +77,7 @@
   "devDependencies": {
     "aegir": "^35.1.1",
     "delay": "^5.0.0",
-    "go-ipfs": "0.9.1",
+    "go-ipfs": "0.10.0",
     "ipfsd-ctl": "^10.0.4",
     "it-all": "^1.0.4",
     "it-first": "^1.0.4",

packages/ipfs-http-client/src/dag/put.js

@@ -21,14 +21,31 @@ export const createPut = (codecs, options) => {
      */
     const put = async (dagNode, options = {}) => {
       const settings = {
-        format: 'dag-cbor',
+        storeCodec: 'dag-cbor',
         hashAlg: 'sha2-256',
-        inputEnc: 'raw',
         ...options
       }
 
-      const codec = await codecs.getCodec(settings.format)
-      const serialized = codec.encode(dagNode)
+      let serialized
+
+      if (settings.inputCodec) {
+        // if you supply an inputCodec, we assume you're passing in a raw, encoded
+        // block using that codec, so we'll just pass that on to the server and let
+        // it deal with the decode/encode/store cycle
+        if (!(dagNode instanceof Uint8Array)) {
+          throw new Error('Can only inputCodec on raw bytes that can be decoded')
+        }
+        serialized = dagNode
+      } else {
+        // if you don't supply an inputCodec, we assume you've passed in a JavaScript
+        // object you want to have encoded using storeCodec, so we'll prepare it for
+        // you if we have the codec
+        const storeCodec = await codecs.getCodec(settings.storeCodec)
+        serialized = storeCodec.encode(dagNode)
+        // now we have a serialized form, the server should be told to receive it
+        // in that format
+        settings.inputCodec = settings.storeCodec
+      }
 
       // allow aborting requests on body errors
       const controller = new AbortController()

packages/ipfs-http-client/src/index.js

@@ -5,6 +5,7 @@ import { Multicodecs } from 'ipfs-core-utils/multicodecs'
 import { Multihashes } from 'ipfs-core-utils/multihashes'
 import * as dagPB from '@ipld/dag-pb'
 import * as dagCBOR from '@ipld/dag-cbor'
+import * as dagJSON from '@ipld/dag-json'
 import { identity } from 'multiformats/hashes/identity'
 import { bases, hashes, codecs } from 'multiformats/basics'
 import { createBitswap } from './bitswap/index.js'
@@ -79,7 +80,7 @@ export function create (options = {}) {
   /** @type {BlockCodec[]} */
   const blockCodecs = Object.values(codecs);
 
-  [dagPB, dagCBOR, id].concat((options.ipld && options.ipld.codecs) || []).forEach(codec => blockCodecs.push(codec))
+  [dagPB, dagCBOR, dagJSON, id].concat((options.ipld && options.ipld.codecs) || []).forEach(codec => blockCodecs.push(codec))
 
   const multicodecs = new Multicodecs({
     codecs: blockCodecs,

packages/ipfs-http-client/test/dag.spec.js

@@ -6,7 +6,6 @@ import { expect } from 'aegir/utils/chai.js'
 import * as dagPB from '@ipld/dag-pb'
 import * as dagCBOR from '@ipld/dag-cbor'
 import * as raw from 'multiformats/codecs/raw'
-import { base58btc } from 'multiformats/bases/base58'
 import { base32 } from 'multiformats/bases/base32'
 import { create as httpClient } from '../src/index.js'
 import { factory } from './utils/factory.js'
@@ -22,25 +21,25 @@ describe('.dag', function () {
 
   after(() => f.clean())
 
-  it('should be able to put and get a DAG node with format dag-pb', async () => {
+  it('should be able to put and get a DAG node with dag-pb codec', async () => {
     const data = uint8ArrayFromString('some data')
     const node = {
       Data: data,
       Links: []
     }
 
-    const cid = await ipfs.dag.put(node, { format: 'dag-pb', hashAlg: 'sha2-256', cidVersion: 0 })
+    const cid = await ipfs.dag.put(node, { storeCodec: 'dag-pb', hashAlg: 'sha2-256' })
     expect(cid.code).to.equal(dagPB.code)
-    expect(cid.toString(base58btc)).to.equal('Qmd7xRhW5f29QuBFtqu3oSD27iVy35NRB91XFjmKFhtgMr')
+    expect(cid.toV0().toString()).to.equal('Qmd7xRhW5f29QuBFtqu3oSD27iVy35NRB91XFjmKFhtgMr')
 
     const result = await ipfs.dag.get(cid)
 
     expect(result.value.Data).to.deep.equal(data)
   })
 
-  it('should be able to put and get a DAG node with format dag-cbor', async () => {
+  it('should be able to put and get a DAG node with dag-cbor codec', async () => {
     const cbor = { foo: 'dag-cbor-bar' }
-    const cid = await ipfs.dag.put(cbor, { format: 'dag-cbor', hashAlg: 'sha2-256' })
+    const cid = await ipfs.dag.put(cbor, { storeCodec: 'dag-cbor', hashAlg: 'sha2-256' })
 
     expect(cid.code).to.equal(dagCBOR.code)
     expect(cid.toString(base32)).to.equal('bafyreic6f672hnponukaacmk2mmt7vs324zkagvu4hcww6yba6kby25zce')
@@ -50,9 +49,9 @@ describe('.dag', function () {
     expect(result.value).to.deep.equal(cbor)
   })
 
-  it('should be able to put and get a DAG node with format raw', async () => {
+  it('should be able to put and get a DAG node with raw codec', async () => {
     const node = uint8ArrayFromString('some data')
-    const cid = await ipfs.dag.put(node, { format: 'raw', hashAlg: 'sha2-256' })
+    const cid = await ipfs.dag.put(node, { storeCodec: 'raw', hashAlg: 'sha2-256' })
 
     expect(cid.code).to.equal(raw.code)
     expect(cid.toString(base32)).to.equal('bafkreiata6mq425fzikf5m26temcvg7mizjrxrkn35swuybmpah2ajan5y')
@@ -70,19 +69,28 @@ describe('.dag', function () {
     await expect(ipfs.dag.get(cid)).to.eventually.be.rejectedWith(/No codec found/)
   })
 
-  it('should error when putting node with esoteric format', () => {
+  it('should error when putting node with esoteric codec', () => {
     const node = uint8ArrayFromString('some data')
 
-    return expect(ipfs.dag.put(node, { format: 'git-raw', hashAlg: 'sha2-256' })).to.eventually.be.rejectedWith(/No codec found/)
+    return expect(ipfs.dag.put(node, { storeCodec: 'git-raw', hashAlg: 'sha2-256' })).to.eventually.be.rejectedWith(/No codec found/)
   })
 
-  it('should attempt to load an unsupported format', async () => {
-    let askedToLoadFormat
+  it('should pass through raw bytes with inputCodec', async () => {
+    const node = uint8ArrayFromString('blob 9\0some data')
+    // we don't support git-raw in the HTTP client, but inputCodec and a Uint8Array should make
+    // the raw data pass through to go-ipfs, which does talk git-raw
+    const cid = await ipfs.dag.put(node, { inputCodec: 'git-raw', storeCodec: 'git-raw', hashAlg: 'sha1' })
+    expect(cid.code).to.equal(0x78)
+    expect(cid.toString(base32)).to.equal('baf4bcfd4azdl7vj4d4hnix75qfld6mabo4l4uwa')
+  })
+
+  it('should attempt to load an unsupported codec', async () => {
+    let askedToLoadCodec
     const ipfs2 = httpClient({
       url: `http://${ipfs.apiHost}:${ipfs.apiPort}`,
       ipld: {
-        loadCodec: (format) => {
-          askedToLoadFormat = format === 'git-raw'
+        loadCodec: (codec) => {
+          askedToLoadCodec = codec === 'boop'
           return {
             encode: (buf) => buf
           }
@@ -93,9 +101,9 @@ describe('.dag', function () {
     const node = uint8ArrayFromString('some data')
 
     // error is from go-ipfs, this means the client serialized it ok
-    await expect(ipfs2.dag.put(node, { format: 'git-raw', hashAlg: 'sha2-256' })).to.eventually.be.rejectedWith(/no parser for format "git-raw"/)
+    await expect(ipfs2.dag.put(node, { storeCodec: 'boop', hashAlg: 'sha2-256' })).to.eventually.be.rejectedWith(/unknown multicodec: "boop"/)
 
-    expect(askedToLoadFormat).to.be.true()
+    expect(askedToLoadCodec).to.be.true()
   })
 
   it('should allow formats to be specified without overwriting others', async () => {
@@ -115,7 +123,7 @@ describe('.dag', function () {
       hello: 'world'
     }
     const cid1 = await ipfs2.dag.put(dagCborNode, {
-      format: 'dag-cbor',
+      storeCodec: 'dag-cbor',
       hashAlg: 'sha2-256'
     })
 
@@ -124,7 +132,7 @@ describe('.dag', function () {
       Links: []
     }
     const cid2 = await ipfs2.dag.put(dagPbNode, {
-      format: 'dag-pb',
+      storeCodec: 'dag-pb',
       hashAlg: 'sha2-256'
     })