fix: use ipfs-core-types instead of redefining ipld (#121)

Adds timeout and signal options to exporter

Author: achingbrain

packages/ipfs-unixfs-exporter/README.md

@@ -159,7 +159,8 @@ Entries with a `dag-cbor` codec `CID` return JavaScript object entries:
   name: 'foo.txt',
   path: 'Qmbar/foo.txt',
   cid: CID, // see https://github.com/multiformats/js-cid
-  node: Object, // see https://github.com/ipld/js-ipld-dag-cbor
+  node: Uint8Array,
+  content: function // returns an async iterator that yields a single object - see https://github.com/ipld/js-ipld-dag-cbor
 }
 ```
 

packages/ipfs-unixfs-exporter/package.json

@@ -13,7 +13,7 @@
     "clean": "rimraf ./dist",
     "lint": "aegir lint",
     "coverage": "nyc -s npm run test -t node && nyc report --reporter=html",
-    "depcheck": "aegir dep-check -i @types/mocha -i @types/sinon -i nyc -i abort-controller -i rimraf"
+    "depcheck": "aegir dep-check -i @types/mocha -i @types/sinon -i nyc -i abort-controller -i rimraf -i ipfs-core-types"
   },
   "repository": {
     "type": "git",
@@ -37,6 +37,7 @@
     "abort-controller": "^3.0.0",
     "aegir": "^30.3.0",
     "detect-node": "^2.0.4",
+    "ipfs-core-types": "^0.3.0",
     "ipfs-unixfs-importer": "^6.0.0",
     "ipld": "^0.28.0",
     "ipld-dag-pb": "^0.21.0",

packages/ipfs-unixfs-exporter/src/index.js

@@ -8,6 +8,7 @@ const last = require('it-last')
 /**
  * @typedef {import('ipfs-unixfs')} UnixFS
  * @typedef {import('ipld-dag-pb').DAGNode} DAGNode
+ * @typedef {import('ipfs-core-types/src/ipld').IPLD} IPLD
  *
  * @typedef {object} UnixFSFile
  * @property {'file'} type
@@ -35,7 +36,8 @@ const last = require('it-last')
  * @property {string} path
  * @property {CID} cid
  * @property {number} depth
- * @property {any} node
+ * @property {Uint8Array} node
+ * @property {(options?: ExporterOptions) => AsyncIterable<any>} content
  *
  * @typedef {object} RawNode
  * @property {'raw'} type
@@ -59,13 +61,11 @@ const last = require('it-last')
  */
 
 /**
- * @typedef {object} IPLDResolver
- * @property {(cid: CID, options?: any) => Promise<any>} get
- * @property {(node: any, codec: number, options?: any) => Promise<CID>} put
- *
  * @typedef {object} ExporterOptions
  * @property {number} [offset=0]
  * @property {number} [length]
+ * @property {AbortSignal} [signal]
+ * @property {number} [timeout]
  */
 
 const toPathComponents = (path = '') => {
@@ -112,7 +112,7 @@ const cidAndRest = (path) => {
 
 /**
  * @param {string | CID} path
- * @param {IPLDResolver} ipld
+ * @param {IPLD} ipld
  * @param {ExporterOptions} [options]
  */
 const walkPath = async function * (path, ipld, options = {}) {
@@ -149,7 +149,7 @@ const walkPath = async function * (path, ipld, options = {}) {
 
 /**
  * @param {string | CID} path
- * @param {IPLDResolver} ipld
+ * @param {IPLD} ipld
  * @param {ExporterOptions} [options]
  */
 const exporter = async (path, ipld, options = {}) => {
@@ -164,7 +164,7 @@ const exporter = async (path, ipld, options = {}) => {
 
 /**
  * @param {string | CID} path
- * @param {IPLDResolver} ipld
+ * @param {IPLD} ipld
  * @param {ExporterOptions} [options]
  */
 const recursive = async function * (path, ipld, options = {}) {

packages/ipfs-unixfs-exporter/src/resolvers/dag-cbor.js

@@ -7,8 +7,9 @@ const errCode = require('err-code')
  * @type {import('./').Resolver}
  */
 const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options) => {
-  const node = await ipld.get(cid, options)
-  let subObject = node
+  const object = await ipld.get(cid, options)
+  const block = await ipld.get(new CID(1, 'raw', cid.multihash))
+  let subObject = object
   let subPath = path
 
   while (toResolve.length) {
@@ -26,8 +27,11 @@ const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options
             name,
             path,
             cid,
-            node,
-            depth
+            node: block,
+            depth,
+            content: async function * () {
+              yield object
+            }
           },
           next: {
             cid: subObject[prop],
@@ -51,8 +55,11 @@ const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options
       name,
       path,
       cid,
-      node,
-      depth
+      node: block,
+      depth,
+      content: async function * () {
+        yield object
+      }
     }
   }
 }

packages/ipfs-unixfs-exporter/src/resolvers/index.js

@@ -3,7 +3,7 @@
 const errCode = require('err-code')
 
 /**
- * @typedef {import('../').IPLDResolver} IPLDResolver
+ * @typedef {import('ipfs-core-types/src/ipld').IPLD} IPLD
  * @typedef {import('../').ExporterOptions} ExporterOptions
  * @typedef {import('../').UnixFSEntry} UnixFSEntry
  * @typedef {import('cids')} CID
@@ -23,9 +23,9 @@ const errCode = require('err-code')
 
 /**
  *
- * @typedef {(cid: CID, name: string, path: string, toResolve: string[], depth: number, ipld: IPLDResolver, options: ExporterOptions) => Promise<ResolveResult>} Resolve
+ * @typedef {(cid: CID, name: string, path: string, toResolve: string[], depth: number, ipld: IPLD, options: ExporterOptions) => Promise<ResolveResult>} Resolve
  *
- * @typedef {(cid: CID, name: string, path: string, toResolve: string[], resolve: Resolve, depth: number, ipld: IPLDResolver, options: ExporterOptions) => Promise<ResolveResult>} Resolver
+ * @typedef {(cid: CID, name: string, path: string, toResolve: string[], resolve: Resolve, depth: number, ipld: IPLD, options: ExporterOptions) => Promise<ResolveResult>} Resolver
  *
  * @type {{ [ key: string ]: Resolver }}
  */

packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.js

@@ -7,10 +7,10 @@ const errCode = require('err-code')
 
 /**
  * @typedef {import('../../../').ExporterOptions} ExporterOptions
- * @typedef {import('../../../').IPLDResolver} IPLDResolver
+ * @typedef {import('ipfs-core-types/src/ipld').IPLD} IPLD
  * @typedef {import('ipld-dag-pb').DAGNode} DAGNode
  *
- * @param {IPLDResolver} ipld
+ * @param {IPLD} ipld
  * @param {DAGNode} node
  * @param {number} start
  * @param {number} end
@@ -61,7 +61,10 @@ async function * emitBytes (ipld, node, start, end, streamPosition = 0, options)
     if ((start >= childStart && start < childEnd) || // child has offset byte
         (end > childStart && end <= childEnd) || // child has end byte
         (start < childStart && end > childEnd)) { // child is between offset and end bytes
-      const child = await ipld.get(childLink.Hash, options)
+      const child = await ipld.get(childLink.Hash, {
+        signal: options.signal,
+        timeout: options.timeout
+      })
 
       for await (const buf of emitBytes(ipld, child, start, end, streamPosition, options)) {
         streamPosition += buf.length

packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.js

@@ -4,7 +4,7 @@
  * @typedef {import('../../../').ExporterOptions} ExporterOptions
  * @typedef {import('ipld-dag-pb').DAGNode} DAGNode
  * @typedef {import('../../').Resolve} Resolve
- * @typedef {import('../../../').IPLDResolver} IPLDResolver
+ * @typedef {import('ipfs-core-types/src/ipld').IPLD} IPLD
  * @typedef {import('../').UnixfsV1DirectoryContent} UnixfsV1DirectoryContent
  *
  * @type {import('../').UnixfsV1Resolver}
@@ -26,7 +26,7 @@ const hamtShardedDirectoryContent = (cid, node, unixfs, path, resolve, depth, ip
  * @param {string} path
  * @param {Resolve} resolve
  * @param {number} depth
- * @param {IPLDResolver} ipld
+ * @param {IPLD} ipld
  * @param {ExporterOptions} options
  *
  * @returns {UnixfsV1DirectoryContent}

packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.js

@@ -6,7 +6,7 @@ const findShardCid = require('../../utils/find-cid-in-shard')
 
 /**
  * @typedef {import('../../').ExporterOptions} ExporterOptions
- * @typedef {import('../../').IPLDResolver} IPLDResolver
+ * @typedef {import('ipfs-core-types/src/ipld').IPLD} IPLD
  * @typedef {import('../').UnixFSEntry} UnixFSEntry
  * @typedef {import('cids')} CID
  * @typedef {import('ipld-dag-pb').DAGNode} DAGNode
@@ -28,7 +28,7 @@ const findLinkCid = (node, name) => {
  * @typedef {AsyncIterable<UnixFSEntry> | Iterable<UnixFSEntry>} UnixfsV1DirectoryContent
  *
  * @typedef {UnixfsV1FileContent | UnixfsV1DirectoryContent} UnixfsV1Content
- * @typedef {(cid: CID, node: DAGNode, unixfs: UnixFS, path: string, resolve: Resolve, depth: number, ipld: IPLDResolver) => (options: ExporterOptions) => UnixfsV1Content } UnixfsV1Resolver
+ * @typedef {(cid: CID, node: DAGNode, unixfs: UnixFS, path: string, resolve: Resolve, depth: number, ipld: IPLD) => (options: ExporterOptions) => UnixfsV1Content } UnixfsV1Resolver
  *
  * @type {{ [key: string]: UnixfsV1Resolver }}
  */

packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.js

@@ -5,7 +5,7 @@ const multihashing = require('multihashing-async')
 
 /**
  * @typedef {import('../').ExporterOptions} ExporterOptions
- * @typedef {import('../').IPLDResolver} IPLDResolver
+ * @typedef {import('ipfs-core-types/src/ipld').IPLD} IPLD
  * @typedef {import('cids')} CID
  */
 
@@ -90,7 +90,7 @@ const toBucketPath = (position) => {
  *
  * @param {import('ipld-dag-pb').DAGNode} node
  * @param {string} name
- * @param {IPLDResolver} ipld
+ * @param {IPLD} ipld
  * @param {ShardTraversalContext} [context]
  * @param {ExporterOptions} [options]
  * @returns {Promise<CID|null>}

packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.js

@@ -30,7 +30,7 @@ const SHARD_SPLIT_THRESHOLD = 10
 describe('exporter sharded', function () {
   this.timeout(30000)
 
-  /** @type {import('../src').IPLDResolver} */
+  /** @type {import('ipfs-core-types/src/ipld').IPLD} */
   let ipld
   /** @type {import('ipfs-unixfs-importer').BlockAPI} */
   let block

packages/ipfs-unixfs-exporter/test/exporter-subtree.spec.js

@@ -18,7 +18,7 @@ const ONE_MEG = Math.pow(1024, 2)
 const exporter = require('./../src')
 
 describe('exporter subtree', () => {
-  /** @type {import('../src').IPLDResolver} */
+  /** @type {import('ipfs-core-types/src/ipld').IPLD} */
   let ipld
   /** @type {import('ipfs-unixfs-importer').BlockAPI} */
   let block

packages/ipfs-unixfs-exporter/test/exporter.spec.js

@@ -29,7 +29,7 @@ const uint8ArrayConcat = require('uint8arrays/concat')
 const ONE_MEG = Math.pow(1024, 2)
 
 describe('exporter', () => {
-  /** @type {import('../src').IPLDResolver} */
+  /** @type {import('ipfs-core-types/src/ipld').IPLD} */
   let ipld
   /** @type {import('ipfs-unixfs-importer').BlockAPI} */
   let block
@@ -134,7 +134,7 @@ describe('exporter', () => {
   }
 
   /**
-   * @param {import('../src').IPLDResolver} ipld
+   * @param {import('ipfs-core-types/src/ipld').IPLD} ipld
    * @param {'file' | 'directory' | 'raw'} type
    * @param {Uint8Array | ArrayLike<number> | undefined} data
    * @param {{ node: DAGNode, cid: CID }[]} children
@@ -959,7 +959,7 @@ describe('exporter', () => {
       throw new Error('Unexpected type')
     }
 
-    expect(exported.node).to.deep.equal(node)
+    return expect(first(exported.content())).to.eventually.deep.equal(node)
   })
 
   it('errors when exporting a node with no resolver', async () => {

packages/ipfs-unixfs-exporter/test/helpers/block.js

@@ -8,7 +8,7 @@ const multicodec = require('multicodec')
 const mh = require('multihashing-async').multihash
 
 /**
- * @param {import('../../src/').IPLDResolver} ipld
+ * @param {import('ipfs-core-types/src/ipld').IPLD} ipld
  */
 function createBlockApi (ipld) {
   // make ipld behave like the block api, some tests need to pull

packages/ipfs-unixfs-exporter/test/helpers/collect-leaf-cids.js

@@ -2,7 +2,7 @@
 
 /**
  * @param {import('cids')} cid
- * @param {import('../../src').IPLDResolver} ipld
+ * @param {import('ipfs-core-types/src/ipld').IPLD} ipld
  */
 module.exports = function (cid, ipld) {
   /**

packages/ipfs-unixfs-exporter/test/import-export-dir-sharding.spec.js

@@ -22,7 +22,7 @@ const uint8ArrayConcat = require('uint8arrays/concat')
  */
 
 describe('builder: directory sharding', () => {
-  /** @type {import('../src').IPLDResolver} */
+  /** @type {import('ipfs-core-types/src/ipld').IPLD} */
   let ipld
   /** @type {import('ipfs-unixfs-importer').BlockAPI} */
   let block

packages/ipfs-unixfs-exporter/test/import-export-nested-dir.spec.js

@@ -16,7 +16,7 @@ const uint8ArrayConcat = require('uint8arrays/concat')
 
 describe('import and export: directory', () => {
   const rootHash = 'QmdCrquDwd7RfZ6GCZFEVADwe8uyyw1YmF9mtAB7etDgmK'
-  /** @type {import('../src').IPLDResolver} */
+  /** @type {import('ipfs-core-types/src/ipld').IPLD} */
   let ipld
   /** @type {import('ipfs-unixfs-importer').BlockAPI} */
   let block

packages/ipfs-unixfs-exporter/test/import-export.spec.js

@@ -30,7 +30,7 @@ describe('import and export', function () {
     const importerOptions = { strategy: strategy }
 
     describe('using builder: ' + strategy, () => {
-      /** @type {import('../src').IPLDResolver} */
+      /** @type {import('ipfs-core-types/src/ipld').IPLD} */
       let ipld
       /** @type {import('ipfs-unixfs-importer').BlockAPI} */
       let block

packages/ipfs-unixfs-exporter/test/importer.spec.js

@@ -30,7 +30,7 @@ const last = require('it-last')
 const CID = require('cids')
 
 /**
- * @typedef {import('../src').IPLDResolver} IPLDResolver
+ * @typedef {import('ipfs-core-types/src/ipld').IPLD} IPLD
  * @typedef {import('ipfs-unixfs-importer').BlockAPI} BlockAPI
  * @typedef {import('ipld-dag-pb').DAGNode} DAGNode
  */
@@ -200,7 +200,7 @@ const strategyOverrides = {
 
 /**
  * @param {BlockAPI} block
- * @param {IPLDResolver} ipld
+ * @param {IPLD} ipld
  * @param {import('ipfs-unixfs-importer').UserImporterOptions} options
  * @param {*} expected
  */
@@ -233,7 +233,7 @@ const checkLeafNodeTypes = async (block, ipld, options, expected) => {
 
 /**
  * @param {BlockAPI} block
- * @param {IPLDResolver} ipld
+ * @param {IPLD} ipld
  * @param {import('ipfs-unixfs-importer').UserImporterOptions} options
  * @param {*} expected
  */
@@ -355,7 +355,7 @@ strategies.forEach((strategy) => {
   describe('importer: ' + strategy, function () {
     this.timeout(30 * 1000)
 
-    /** @type {IPLDResolver} */
+    /** @type {IPLD} */
     let ipld
     /** @type {BlockAPI} */
     let block
@@ -1076,7 +1076,7 @@ strategies.forEach((strategy) => {
 })
 
 describe('configuration', () => {
-  /** @type {IPLDResolver} */
+  /** @type {IPLD} */
   let ipld
   /** @type {BlockAPI} */
   let block

packages/ipfs-unixfs-exporter/tsconfig.json

@@ -1,7 +1,8 @@
 {
   "extends": "aegir/src/config/tsconfig.aegir.json",
   "compilerOptions": {
-    "outDir": "dist"
+    "outDir": "dist",
+    "importsNotUsedAsValues": "preserve"
   },
   "include": [
     "src",

packages/ipfs-unixfs-importer/test/benchmark.spec.js

@@ -18,7 +18,7 @@ const CHUNK_SIZE = 65536
 describe.skip('benchmark', function () {
   this.timeout(30 * 1000)
 
-  /** @type {import('./helpers/block').IPLDResolver} */
+  /** @type {import('./helpers/block').IPLD} */
   let ipld
   /** @type {import('../src').BlockAPI} */
   let block

packages/ipfs-unixfs-importer/test/builder-only-hash.spec.js

@@ -12,7 +12,7 @@ const blockApi = require('./helpers/block')
 const defaultOptions = require('../src/options')
 
 describe('builder: onlyHash', () => {
-  /** @type {import('./helpers/block').IPLDResolver} */
+  /** @type {import('./helpers/block').IPLD} */
   let ipld
   /** @type {import('../src').BlockAPI} */
   let block

packages/ipfs-unixfs-importer/test/builder.spec.js

@@ -15,7 +15,7 @@ const uint8ArrayFromString = require('uint8arrays/from-string')
 const defaultOptions = require('../src/options')
 
 describe('builder', () => {
-  /** @type {import('./helpers/block').IPLDResolver} */
+  /** @type {import('./helpers/block').IPLD} */
   let ipld
   /** @type {import('../src').BlockAPI} */
   let block