migrate importer to async IPLD format interface

Author: daviddias

README.md

@@ -25,6 +25,14 @@ IPFS unixFS Engine
 - [Contribute](#contribute)
 - [License](#license)
 
+## BEWARE BEWARE BEWARE there might be 🐉
+
+This module has passed through several iterations and still is far from a nice and easy understandable codebase. Currently missing features:
+
+- tar importer
+- trickle dag exporter
+- sharding
+
 ## Install
 
 With [npm](https://npmjs.org/) installed, run

package.json

@@ -2,7 +2,7 @@
   "name": "ipfs-unixfs-engine",
   "version": "0.11.4",
   "description": "JavaScript implementation of the unixfs Engine used by IPFS",
-  "main": "src/index.js",
+  "main": "lib/index.js",
   "jsnext:main": "src/index.js",
   "scripts": {
     "lint": "aegir-lint",
@@ -34,31 +34,33 @@
   },
   "homepage": "https://github.com/ipfs/js-ipfs-unixfs-engineg#readme",
   "devDependencies": {
-    "aegir": "^8.0.1",
+    "aegir": "^8.1.2",
     "buffer-loader": "0.0.1",
     "chai": "^3.5.0",
-    "fs-pull-blob-store": "^0.3.0",
+    "fs-pull-blob-store": "^0.4.1",
     "idb-pull-blob-store": "^0.5.1",
-    "ipfs-block-service": "^0.5.0",
-    "ipfs-repo": "^0.9.0",
+    "ipfs-block-service": "^0.6.0",
+    "ipfs-repo": "^0.10.0",
     "ncp": "^2.0.0",
     "pre-commit": "^1.1.3",
-    "pull-zip": "^2.0.0",
+    "pull-zip": "^2.0.1",
     "raw-loader": "^0.5.1",
     "rimraf": "^2.5.4",
     "run-series": "^1.1.4"
   },
   "dependencies": {
-    "ipld-dag-pb": "^0.0.1",
+    "cids": "^0.2.0",
     "ipfs-unixfs": "^0.1.4",
-    "is-ipfs": "^0.2.0",
+    "ipld-dag-pb": "^0.1.3",
+    "ipld-resolver": "^0.1.1",
+    "is-ipfs": "^0.2.1",
     "multihashes": "^0.2.2",
     "pull-block": "^1.0.2",
-    "pull-paramap": "^1.1.6",
+    "pull-paramap": "^1.2.0",
     "pull-pushable": "^2.0.1",
     "pull-stream": "^3.4.5",
     "pull-traverse": "^1.0.3",
-    "pull-write": "^1.1.0",
+    "pull-write": "^1.1.1",
     "run-parallel": "^1.1.6"
   },
   "contributors": [

src/chunker-fixed-size.js

@@ -0,0 +1,7 @@
+'use strict'
+
+const pullBlock = require('pull-block')
+
+module.exports = (size) => {
+  return pullBlock(size, { zeroPadding: false })
+}

src/chunker/fixed-size.js

@@ -4,12 +4,12 @@ const traverse = require('pull-traverse')
 const pull = require('pull-stream')
 const CID = require('cids')
 
-const util = require('./util')
+const util = require('./../util')
 const switchType = util.switchType
 const cleanMultihash = util.cleanMultihash
 
-const dirExporter = require('./exporters/dir')
-const fileExporter = require('./exporters/file')
+const dirExporter = require('./dir')
+const fileExporter = require('./file')
 
 module.exports = (hash, ipldResolver, options) => {
   hash = cleanMultihash(hash)

src/exporters/dir.js renamed to src/exporter/dir.js

@@ -0,0 +1,167 @@
+'use strict'
+
+const mh = require('multihashes')
+const UnixFS = require('ipfs-unixfs')
+const CID = require('cids')
+const dagPB = require('ipld-dag-pb')
+const mapValues = require('async/mapValues')
+const parallel = require('async/parallel')
+
+const DAGLink = dagPB.DAGLink
+const DAGNode = dagPB.DAGNode
+
+module.exports = (files, ipldResolver, source, callback) => {
+  // 1) convert files to a tree
+  const fileTree = createTree(files)
+
+  if (Object.keys(fileTree).length === 0) {
+    return callback()// no dirs to be created
+  }
+
+  // 2) create sizeIndex
+  const sizeIndex = createSizeIndex(files)
+
+  // 3) bottom up flushing
+  traverse(fileTree, sizeIndex, null, ipldResolver, source, callback)
+}
+
+/*
+ * createTree
+ *
+ * received an array of files with the format:
+ * {
+ *    path: // full path
+ *    multihash: // multihash of the dagNode
+ *    size: // cumulative size
+ * }
+ *
+ * returns a JSON object that represents a tree where branches are the paths
+ * and the leaves are objects with file names and respective multihashes, such
+ * as:
+ *   {
+ *     foo: {
+ *       bar: {
+ *         baz.txt: <multihash>
+ *       }
+ *     }
+ *   }
+ */
+function createTree (files) {
+  const fileTree = {}
+
+  files.forEach((file) => {
+    let splitted = file.path.split('/')
+    if (splitted.length === 1) {
+      return // adding just one file
+    }
+    if (splitted[0] === '') {
+      splitted = splitted.slice(1)
+    }
+    var tmpTree = fileTree
+
+    for (var i = 0; i < splitted.length; i++) {
+      if (!tmpTree[splitted[i]]) {
+        tmpTree[splitted[i]] = {}
+      }
+      if (i === splitted.length - 1) {
+        tmpTree[splitted[i]] = file.multihash
+      } else {
+        tmpTree = tmpTree[splitted[i]]
+      }
+    }
+  })
+
+  return fileTree
+}
+
+/*
+ * create a size index that goes like:
+ * { <multihash>: <size> }
+ */
+function createSizeIndex (files) {
+  const sizeIndex = {}
+
+  files.forEach((file) => {
+    sizeIndex[mh.toB58String(file.multihash)] = file.size
+  })
+
+  return sizeIndex
+}
+
+/*
+ * expand the branches recursively (depth first), flush them first
+ * and then traverse through the bottoum up, flushing everynode
+ *
+ * Algorithm tl;dr;
+ *  create a dirNode
+ *  Object.keys
+ *    If the value is an Object
+ *      create a dir Node
+ *      Object.keys
+ *    Once finished, add the result as a link to the dir node
+ *  If the value is not an object
+ *    add as a link to the dirNode
+ */
+function traverse (tree, sizeIndex, path, ipldResolver, source, done) {
+  mapValues(tree, (node, key, cb) => {
+    if (isLeaf(node)) {
+      return cb(null, node)
+    }
+
+    traverse(node, sizeIndex, path ? `${path}/${key}` : key, ipldResolver, source, cb)
+  }, (err, tree) => {
+    if (err) {
+      return done(err)
+    }
+
+    // at this stage, all keys are multihashes
+    // create a dir node
+    // add all the multihashes as links
+    // return this new node multihash
+
+    const keys = Object.keys(tree)
+
+    const ufsDir = new UnixFS('directory')
+    const node = new DAGNode(ufsDir.marshal())
+
+    keys.forEach((key) => {
+      const b58mh = mh.toB58String(tree[key])
+      const link = new DAGLink(key, sizeIndex[b58mh], tree[key])
+      node.addRawLink(link)
+    })
+
+    parallel([
+      (cb) => node.multihash(cb),
+      (cb) => node.size(cb)
+    ], (err, res) => {
+      if (err) {
+        return done(err)
+      }
+
+      const multihash = res[0]
+      const size = res[1]
+
+      sizeIndex[mh.toB58String(multihash)] = size
+      ipldResolver.put({
+        node: node,
+        cid: new CID(multihash)
+      }, (err) => {
+        if (err) {
+          source.push(new Error('failed to store dirNode'))
+        } else if (path) {
+          source.push({
+            path: path,
+            multihash: multihash,
+            size: size
+          })
+        }
+
+        done(null, multihash)
+      })
+    })
+  })
+}
+
+function isLeaf (value) {
+  return !(typeof value === 'object' && !Buffer.isBuffer(value))
+}