feat: add gateway to ipfs daemon

* feat: add HTTP Gateway to the js-ipfs daemon

License: MIT
Signed-off-by: Yahya <[email protected]>

* apply remaining CR

* chore: update deps

* test: fix failing gateway tests with this one simple trick! (#1006)

* fix failing tests with this one simple trick!

* adding files to make tests self-contained

* changes requested by VictorBjelkholm #968

* chore: fix circle CI chrome build (#1008) (#1009)

Thanks to mkg20001

Author: daviddias

gulpfile.js

@@ -4,7 +4,7 @@ const gulp = require('gulp')
 const parallel = require('async/parallel')
 const series = require('async/series')
 const createTempRepo = require('./test/utils/create-repo-nodejs.js')
-const HTTPAPI = require('./src/http-api')
+const HTTPAPI = require('./src/http')
 const leftPad = require('left-pad')
 
 let nodes = []

package.json

@@ -27,6 +27,7 @@
     "test:unit:node": "gulp test:node",
     "test:unit:node:core": "TEST=core npm run test:unit:node",
     "test:unit:node:http": "TEST=http npm run test:unit:node",
+    "test:unit:node:gateway": "TEST=gateway npm run test:unit:node",
     "test:unit:node:cli": "TEST=cli npm run test:unit:node",
     "test:unit:browser": "gulp test:browser",
     "test:interop": "npm run test:interop:node",
@@ -92,31 +93,33 @@
     "async": "^2.5.0",
     "bl": "^1.2.1",
     "boom": "^5.2.0",
-    "cids": "~0.5.1",
     "debug": "^3.0.1",
+    "cids": "^0.5.1",
+    "file-type": "^6.1.0",
+    "filesize": "^3.5.10",
     "fsm-event": "^2.1.0",
     "glob": "^7.1.2",
     "hapi": "^16.5.2",
     "hapi-set-header": "^1.0.2",
     "hoek": "^4.2.0",
-    "ipfs-api": "^14.3.3",
+    "ipfs-api": "^14.3.4",
     "ipfs-bitswap": "~0.17.2",
     "ipfs-block": "~0.6.0",
     "ipfs-block-service": "~0.12.0",
     "ipfs-multipart": "~0.1.0",
     "ipfs-repo": "~0.17.0",
     "ipfs-unixfs": "~0.1.13",
-    "ipfs-unixfs-engine": "~0.22.3",
+    "ipfs-unixfs-engine": "~0.22.4",
     "ipld-resolver": "~0.13.2",
     "is-ipfs": "^0.3.0",
     "is-stream": "^1.1.0",
     "joi": "^10.6.0",
-    "libp2p": "~0.12.3",
-    "libp2p-floodsub": "~0.11.0",
-    "libp2p-kad-dht": "~0.5.0",
-    "libp2p-mdns": "~0.9.0",
+    "libp2p": "~0.12.4",
+    "libp2p-floodsub": "~0.11.1",
+    "libp2p-kad-dht": "~0.5.1",
+    "libp2p-mdns": "~0.9.1",
     "libp2p-multiplex": "~0.5.0",
-    "libp2p-railing": "~0.7.0",
+    "libp2p-railing": "~0.7.1",
     "libp2p-secio": "~0.8.1",
     "libp2p-tcp": "~0.11.0",
     "libp2p-webrtc-star": "~0.13.1",
@@ -126,13 +129,14 @@
     "lodash.sortby": "^4.7.0",
     "lodash.values": "^4.3.0",
     "mafmt": "^3.0.1",
+    "mime-types": "^2.1.17",
     "mkdirp": "~0.5.1",
     "multiaddr": "^3.0.1",
     "multihashes": "~0.4.9",
     "once": "^1.4.0",
     "path-exists": "^3.0.0",
-    "peer-book": "~0.5.0",
-    "peer-id": "~0.10.0",
+    "peer-book": "~0.5.1",
+    "peer-id": "~0.10.1",
     "peer-info": "~0.11.0",
     "promisify-es6": "^1.0.3",
     "pull-file": "^1.0.0",

src/cli/commands/daemon.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const HttpAPI = require('../../http-api')
+const HttpAPI = require('../../http')
 const utils = require('../utils')
 const print = utils.print
 

src/http-api/resources/bitswap.js renamed to src/http/api/resources/bitswap.js

@@ -0,0 +1,86 @@
+'use strict'
+
+const filesize = require('filesize')
+
+const mainStyle = require('./style')
+const pathUtil = require('../utils/path')
+
+function getParentDirectoryURL (originalParts) {
+  const parts = originalParts.slice()
+
+  if (parts.length > 1) {
+    parts.pop()
+  }
+
+  return [ '', 'ipfs' ].concat(parts).join('/')
+}
+
+function buildFilesList (path, links) {
+  const rows = links.map((link) => {
+    let row = [
+      `<div class="ipfs-icon ipfs-_blank">&nbsp;</div>`,
+      `<a href="${pathUtil.joinURLParts(path, link.name)}">${link.name}</a>`,
+      filesize(link.size)
+    ]
+
+    row = row.map((cell) => `<td>${cell}</td>`).join('')
+
+    return `<tr>${row}</tr>`
+  })
+
+  return rows.join('')
+}
+
+function buildTable (path, links) {
+  const parts = pathUtil.splitPath(path)
+  const parentDirectoryURL = getParentDirectoryURL(parts)
+
+  return `
+    <table class="table table-striped">
+      <tbody>
+        <tr>
+          <td class="narrow">
+            <div class="ipfs-icon ipfs-_blank">&nbsp;</div>
+          </td>
+          <td class="padding">
+            <a href="${parentDirectoryURL}">..</a>
+          </td>
+          <td></td>
+        </tr>
+        ${buildFilesList(path, links)}
+      </tbody>
+    </table>
+  `
+}
+
+function render (path, links) {
+  return `
+    <!DOCTYPE html>
+    <html>
+    <head>
+      <meta charset="utf-8">
+      <title>${path}</title>
+      <style>${mainStyle}</style>
+    </head>
+    <body>
+      <div id="header" class="row">
+        <div class="col-xs-2">
+          <div id="logo" class="ipfs-logo"></div>
+        </div>
+      </div>
+      <br>
+      <div class="col-xs-12">
+        <div class="panel panel-default">
+          <div class="panel-heading">
+            <strong>Index of ${path}</strong>
+          </div>
+          ${buildTable(path, links)}
+        </div>
+      </div>
+    </body>
+    </html>
+  `
+}
+
+exports = module.exports
+exports.render = render

src/http-api/resources/block.js renamed to src/http/api/resources/block.js

@@ -0,0 +1,110 @@
+'use strict'
+
+const mh = require('multihashes')
+const promisify = require('promisify-es6')
+const reduce = require('async/reduce')
+const CID = require('cids')
+const Unixfs = require('ipfs-unixfs')
+const debug = require('debug')
+const log = debug('jsipfs:http-gateway:resolver')
+log.error = debug('jsipfs:http-gateway:resolver:error')
+
+const dirView = require('./dir-view')
+const pathUtil = require('./utils/path')
+
+function getIndexFiles (links) {
+  const INDEX_HTML_FILES = [
+    'index.html',
+    'index.htm',
+    'index.shtml'
+  ]
+
+  return links.filter((link) => INDEX_HTML_FILES.indexOf(link.name) !== -1)
+}
+
+const resolveDirectory = promisify((ipfs, path, multihash, callback) => {
+  mh.validate(mh.fromB58String(multihash))
+
+  ipfs.object.get(multihash, { enc: 'base58' }, (err, dagNode) => {
+    if (err) { return callback(err) }
+
+    const indexFiles = getIndexFiles(dagNode.links)
+
+    if (indexFiles.length > 0) {
+      return callback(null, indexFiles)
+    }
+
+    return callback(null, dirView.render(path, dagNode.links))
+  })
+})
+
+const resolveMultihash = promisify((ipfs, path, callback) => {
+  const parts = pathUtil.splitPath(path)
+  let firstMultihash = parts.shift()
+  let currentCid
+
+  reduce(parts, firstMultihash, (memo, item, next) => {
+    try {
+      currentCid = new CID(mh.fromB58String(memo))
+    } catch (err) {
+      return next(err)
+    }
+
+    log('memo: ', memo)
+    log('item: ', item)
+
+    ipfs.dag.get(currentCid, (err, result) => {
+      if (err) { return next(err) }
+
+      let dagNode = result.value
+      // find multihash of requested named-file in current dagNode's links
+      let multihashOfNextFile
+      let nextFileName = item
+
+      const links = dagNode.links
+
+      for (let link of links) {
+        if (link.name === nextFileName) {
+          // found multihash of requested named-file
+          multihashOfNextFile = mh.toB58String(link.multihash)
+          log('found multihash: ', multihashOfNextFile)
+          break
+        }
+      }
+
+      if (!multihashOfNextFile) {
+        return next(new Error(`no link named "${nextFileName}" under ${memo}`))
+      }
+
+      next(null, multihashOfNextFile)
+    })
+  }, (err, result) => {
+    if (err) { return callback(err) }
+
+    let cid
+    try {
+      cid = new CID(mh.fromB58String(result))
+    } catch (err) {
+      return callback(err)
+    }
+
+    ipfs.dag.get(cid, (err, dagResult) => {
+      if (err) return callback(err)
+
+      let dagDataObj = Unixfs.unmarshal(dagResult.value.data)
+      if (dagDataObj.type === 'directory') {
+        let isDirErr = new Error('This dag node is a directory')
+        // add memo (last multihash) as a fileName so it can be used by resolveDirectory
+        isDirErr.fileName = result
+        return callback(isDirErr)
+      }
+
+      callback(null, { multihash: result })
+    })
+  })
+})
+
+module.exports = {
+  resolveDirectory: resolveDirectory,
+  resolveMultihash: resolveMultihash
+}