diff --git a/.aegir.js b/.aegir.js
index 979ffde3df..5d87aa66d7 100644
--- a/.aegir.js
+++ b/.aegir.js
@@ -1,8 +1,11 @@
'use strict'
-const createServer = require('ipfsd-ctl').createServer
+const IPFSFactory = require('ipfsd-ctl')
+const parallel = require('async/parallel')
+const MockPreloadNode = require('./test/utils/mock-preload-node')
-const server = createServer()
+const ipfsdServer = IPFSFactory.createServer()
+const preloadNode = MockPreloadNode.createNode()
module.exports = {
webpack: {
@@ -21,9 +24,29 @@ module.exports = {
singleRun: true
},
hooks: {
+ node: {
+ pre: (cb) => preloadNode.start(cb),
+ post: (cb) => preloadNode.stop(cb)
+ },
browser: {
- pre: server.start.bind(server),
- post: server.stop.bind(server)
+ pre: (cb) => {
+ parallel([
+ (cb) => {
+ ipfsdServer.start()
+ cb()
+ },
+ (cb) => preloadNode.start(cb)
+ ], cb)
+ },
+ post: (cb) => {
+ parallel([
+ (cb) => {
+ ipfsdServer.stop()
+ cb()
+ },
+ (cb) => preloadNode.stop(cb)
+ ], cb)
+ }
}
}
}
diff --git a/README.md b/README.md
index 0c20f35cdb..c2dbefa5b3 100644
--- a/README.md
+++ b/README.md
@@ -231,6 +231,9 @@ Creates and returns an instance of an IPFS node. Use the `options` argument to s
- `enabled` (boolean): Make this node a relay (other nodes can connect *through* it). (Default: `false`)
- `active` (boolean): Make this an *active* relay node. Active relay nodes will attempt to dial a destination peer even if that peer is not yet connected to the relay. (Default: `false`)
+- `preload` (object): Configure external nodes that will preload content added to this node
+ - `enabled` (boolean): Enable content preloading (Default: `true`)
+ - `addresses` (array): Multiaddr API addresses of nodes that should preload content. NOTE: nodes specified here should also be added to your node's bootstrap address list at `config.Boostrap`
- `EXPERIMENTAL` (object): Enable and configure experimental features.
- `pubsub` (boolean): Enable libp2p pub-sub. (Default: `false`)
- `sharding` (boolean): Enable directory sharding. Directories that have many child objects will be represented by multiple DAG nodes instead of just one. It can improve lookup performance when a directory has several thousand files or more. (Default: `false`)
diff --git a/package.json b/package.json
index 2cd7bcb477..e98cd5a278 100644
--- a/package.json
+++ b/package.json
@@ -11,6 +11,7 @@
"./src/core/components/init-assets.js": false,
"./src/core/runtime/config-nodejs.js": "./src/core/runtime/config-browser.js",
"./src/core/runtime/libp2p-nodejs.js": "./src/core/runtime/libp2p-browser.js",
+ "./src/core/runtime/preload-nodejs.js": "./src/core/runtime/preload-browser.js",
"./src/core/runtime/repo-nodejs.js": "./src/core/runtime/repo-browser.js",
"./src/core/runtime/dns-nodejs.js": "./src/core/runtime/dns-browser.js",
"./test/utils/create-repo-nodejs.js": "./test/utils/create-repo-browser.js",
@@ -140,6 +141,7 @@
"mime-types": "^2.1.18",
"mkdirp": "~0.5.1",
"multiaddr": "^5.0.0",
+ "multiaddr-to-uri": "^4.0.0",
"multibase": "~0.4.0",
"multihashes": "~0.4.13",
"once": "^1.4.0",
diff --git a/src/core/components/dag.js b/src/core/components/dag.js
index 88a80bdcce..105985df68 100644
--- a/src/core/components/dag.js
+++ b/src/core/components/dag.js
@@ -24,7 +24,11 @@ module.exports = function dag (self) {
options = options.cid ? options : Object.assign({}, optionDefaults, options)
- self._ipld.put(dagNode, options, callback)
+ self._ipld.put(dagNode, options, (err, cid) => {
+ if (err) return callback(err)
+ if (options.preload !== false) self._preload(cid)
+ callback(null, cid)
+ })
}),
get: promisify((cid, path, options, callback) => {
diff --git a/src/core/components/files.js b/src/core/components/files.js
index f69868bd77..12a1fcdce0 100644
--- a/src/core/components/files.js
+++ b/src/core/components/files.js
@@ -89,6 +89,20 @@ function normalizeContent (opts, content) {
})
}
+function preloadFile (self, opts, file) {
+ const isRootFile = opts.wrapWithDirectory
+ ? file.path === ''
+ : !file.path.includes('/')
+
+ const shouldPreload = isRootFile && !opts.onlyHash && opts.preload !== false
+
+ if (shouldPreload) {
+ self._preload(file.hash)
+ }
+
+ return file
+}
+
function pinFile (self, opts, file, cb) {
// Pin a file if it is the root dir of a recursive add or the single file
// of a direct add.
@@ -158,6 +172,7 @@ module.exports = function files (self) {
pull.flatten(),
importer(self._ipld, opts),
pull.asyncMap(prepareFile.bind(null, self, opts)),
+ pull.map(preloadFile.bind(null, self, opts)),
pull.asyncMap(pinFile.bind(null, self, opts))
)
}
diff --git a/src/core/components/index.js b/src/core/components/index.js
index 9eb36ad4c3..1f6f084dee 100644
--- a/src/core/components/index.js
+++ b/src/core/components/index.js
@@ -26,4 +26,4 @@ exports.dht = require('./dht')
exports.dns = require('./dns')
exports.key = require('./key')
exports.stats = require('./stats')
-exports.mfs = require('ipfs-mfs/core')
+exports.mfs = require('./mfs')
diff --git a/src/core/components/mfs.js b/src/core/components/mfs.js
new file mode 100644
index 0000000000..9f033545b4
--- /dev/null
+++ b/src/core/components/mfs.js
@@ -0,0 +1,24 @@
+'use strict'
+
+const promisify = require('promisify-es6')
+const mfs = require('ipfs-mfs/core')
+
+module.exports = self => {
+ const mfsSelf = Object.assign({}, self)
+
+ // A patched dag API to ensure preload doesn't happen for MFS operations
+ mfsSelf.dag = Object.assign({}, self.dag, {
+ put: promisify((node, opts, cb) => {
+ if (typeof opts === 'function') {
+ cb = opts
+ opts = {}
+ }
+
+ opts = Object.assign({}, opts, { preload: false })
+
+ return self.dag.put(node, opts, cb)
+ })
+ })
+
+ return mfs(mfsSelf, mfsSelf._options)
+}
diff --git a/src/core/components/pin-set.js b/src/core/components/pin-set.js
index f18a248604..806df5f05f 100644
--- a/src/core/components/pin-set.js
+++ b/src/core/components/pin-set.js
@@ -90,7 +90,7 @@ exports = module.exports = function (dag) {
pinSet.storeItems(pins, (err, rootNode) => {
if (err) { return callback(err) }
- const opts = { cid: new CID(rootNode.multihash) }
+ const opts = { cid: new CID(rootNode.multihash), preload: false }
dag.put(rootNode, opts, (err, cid) => {
if (err) { return callback(err) }
callback(null, rootNode)
@@ -168,7 +168,8 @@ exports = module.exports = function (dag) {
function storeChild (err, child, binIdx, cb) {
if (err) { return cb(err) }
- dag.put(child, { cid: new CID(child._multihash) }, err => {
+ const opts = { cid: new CID(child._multihash), preload: false }
+ dag.put(child, opts, err => {
if (err) { return cb(err) }
fanoutLinks[binIdx] = new DAGLink('', child.size, child.multihash)
cb(null)
diff --git a/src/core/components/pin.js b/src/core/components/pin.js
index d2bd670e14..ce0cd72d84 100644
--- a/src/core/components/pin.js
+++ b/src/core/components/pin.js
@@ -80,14 +80,14 @@ module.exports = (self) => {
// the pin-set nodes link to a special 'empty' node, so make sure it exists
cb => DAGNode.create(Buffer.alloc(0), (err, empty) => {
if (err) { return cb(err) }
- dag.put(empty, { cid: new CID(empty.multihash) }, cb)
+ dag.put(empty, { cid: new CID(empty.multihash), preload: false }, cb)
}),
// create a root node with DAGLinks to the direct and recursive DAGs
cb => DAGNode.create(Buffer.alloc(0), [dLink, rLink], (err, node) => {
if (err) { return cb(err) }
root = node
- dag.put(root, { cid: new CID(root.multihash) }, cb)
+ dag.put(root, { cid: new CID(root.multihash), preload: false }, cb)
}),
// hack for CLI tests
diff --git a/src/core/components/start.js b/src/core/components/start.js
index fd4832e35a..3a7a5716ce 100644
--- a/src/core/components/start.js
+++ b/src/core/components/start.js
@@ -42,7 +42,9 @@ module.exports = (self) => {
self._bitswap.start()
self._blockService.setExchange(self._bitswap)
- cb()
+
+ self._preload.start()
+ self._mfsPreload.start(cb)
}
], done)
})
diff --git a/src/core/components/stop.js b/src/core/components/stop.js
index 4d35190d21..cf97b6ec6a 100644
--- a/src/core/components/stop.js
+++ b/src/core/components/stop.js
@@ -30,8 +30,10 @@ module.exports = (self) => {
self.state.stop()
self._blockService.unsetExchange()
self._bitswap.stop()
+ self._preload.stop()
series([
+ (cb) => self._mfsPreload.stop(cb),
(cb) => self.libp2p.stop(cb),
(cb) => self._repo.close(cb)
], done)
diff --git a/src/core/config.js b/src/core/config.js
index 1b04d10a2f..7b16c17d06 100644
--- a/src/core/config.js
+++ b/src/core/config.js
@@ -8,6 +8,10 @@ const schema = Joi.object().keys({
Joi.string()
).allow(null),
repoOwner: Joi.boolean().default(true),
+ preload: Joi.object().keys({
+ enabled: Joi.boolean().default(true),
+ addresses: Joi.array().items(Joi.multiaddr().options({ convert: false }))
+ }).allow(null),
init: Joi.alternatives().try(
Joi.boolean(),
Joi.object().keys({ bits: Joi.number().integer() })
diff --git a/src/core/index.js b/src/core/index.js
index 36f7c3a118..52266b7b41 100644
--- a/src/core/index.js
+++ b/src/core/index.js
@@ -22,6 +22,8 @@ const boot = require('./boot')
const components = require('./components')
// replaced by repo-browser when running in the browser
const defaultRepo = require('./runtime/repo-nodejs')
+const preload = require('./preload')
+const mfsPreload = require('./mfs-preload')
class IPFS extends EventEmitter {
constructor (options) {
@@ -30,7 +32,14 @@ class IPFS extends EventEmitter {
this._options = {
init: true,
start: true,
- EXPERIMENTAL: {}
+ EXPERIMENTAL: {},
+ preload: {
+ enabled: true,
+ addresses: [
+ '/dnsaddr/node0.preload.ipfs.io/https',
+ '/dnsaddr/node1.preload.ipfs.io/https'
+ ]
+ }
}
options = config.validate(options || {})
@@ -78,6 +87,8 @@ class IPFS extends EventEmitter {
this._blockService = new BlockService(this._repo)
this._ipld = new Ipld(this._blockService)
this._pubsub = undefined
+ this._preload = preload(this)
+ this._mfsPreload = mfsPreload(this)
// IPFS Core exposed components
// - for booting up a node
@@ -134,7 +145,7 @@ class IPFS extends EventEmitter {
}
// ipfs.files
- const mfs = components.mfs(this, this._options)
+ const mfs = components.mfs(this)
Object.keys(mfs).forEach(key => {
this.files[key] = mfs[key]
diff --git a/src/core/mfs-preload.js b/src/core/mfs-preload.js
new file mode 100644
index 0000000000..fc75f25a0a
--- /dev/null
+++ b/src/core/mfs-preload.js
@@ -0,0 +1,49 @@
+'use strict'
+
+const debug = require('debug')
+
+const log = debug('jsipfs:mfs-preload')
+log.error = debug('jsipfs:mfs-preload:error')
+
+module.exports = (self, options) => {
+ options = options || {}
+ options.interval = options.interval || 30 * 1000
+
+ let rootCid
+ let timeoutId
+
+ const preloadMfs = () => {
+ self.files.stat('/', (err, stats) => {
+ if (err) {
+ timeoutId = setTimeout(preloadMfs, options.interval)
+ return log.error('failed to stat MFS root for preload', err)
+ }
+
+ if (rootCid !== stats.hash) {
+ log(`preloading updated MFS root ${rootCid} -> ${stats.hash}`)
+
+ self._preload(stats.hash, (err) => {
+ timeoutId = setTimeout(preloadMfs, options.interval)
+ if (err) return log.error(`failed to preload MFS root ${stats.hash}`, err)
+ rootCid = stats.hash
+ })
+ }
+ })
+ }
+
+ return {
+ start (cb) {
+ self.files.stat('/', (err, stats) => {
+ if (err) return cb(err)
+ rootCid = stats.hash
+ log(`monitoring MFS root ${rootCid}`)
+ timeoutId = setTimeout(preloadMfs, options.interval)
+ cb()
+ })
+ },
+ stop (cb) {
+ clearTimeout(timeoutId)
+ cb()
+ }
+ }
+}
diff --git a/src/core/preload.js b/src/core/preload.js
new file mode 100644
index 0000000000..d99a9d8f20
--- /dev/null
+++ b/src/core/preload.js
@@ -0,0 +1,88 @@
+'use strict'
+
+const setImmediate = require('async/setImmediate')
+const retry = require('async/retry')
+const toUri = require('multiaddr-to-uri')
+const debug = require('debug')
+const CID = require('cids')
+const preload = require('./runtime/preload-nodejs')
+
+const log = debug('jsipfs:preload')
+log.error = debug('jsipfs:preload:error')
+
+const noop = (err) => { if (err) log.error(err) }
+
+module.exports = self => {
+ const options = self._options.preload || {}
+ options.enabled = Boolean(options.enabled)
+ options.addresses = options.addresses || []
+
+ if (!options.enabled || !options.addresses.length) {
+ return (_, callback) => {
+ if (callback) {
+ setImmediate(() => callback())
+ }
+ }
+ }
+
+ let stopped = true
+ let requests = []
+ const apiUris = options.addresses.map(apiAddrToUri)
+
+ const api = (cid, callback) => {
+ callback = callback || noop
+
+ if (typeof cid !== 'string') {
+ try {
+ cid = new CID(cid).toBaseEncodedString()
+ } catch (err) {
+ return setImmediate(() => callback(err))
+ }
+ }
+
+ const fallbackApiUris = Array.from(apiUris)
+ let request
+ const now = Date.now()
+
+ retry({ times: fallbackApiUris.length }, (cb) => {
+ if (stopped) return cb(new Error(`preload aborted for ${cid}`))
+
+ // Remove failed request from a previous attempt
+ requests = requests.filter(r => r !== request)
+
+ const apiUri = fallbackApiUris.shift()
+
+ request = preload(`${apiUri}/api/v0/refs?r=true&arg=${cid}`, cb)
+ requests = requests.concat(request)
+ }, (err) => {
+ requests = requests.filter(r => r !== request)
+
+ if (err) {
+ return callback(err)
+ }
+
+ log(`preloaded ${cid} in ${Date.now() - now}ms`)
+ callback()
+ })
+ }
+
+ api.start = () => {
+ stopped = false
+ }
+
+ api.stop = () => {
+ stopped = true
+ log(`canceling ${requests.length} pending preload request(s)`)
+ requests.forEach(r => r.cancel())
+ requests = []
+ }
+
+ return api
+}
+
+function apiAddrToUri (addr) {
+ if (!(addr.endsWith('http') || addr.endsWith('https'))) {
+ addr = addr + '/http'
+ }
+ return toUri(addr)
+}
diff --git a/src/core/runtime/config-browser.js b/src/core/runtime/config-browser.js
index 9819c04aaa..f7662420cb 100644
--- a/src/core/runtime/config-browser.js
+++ b/src/core/runtime/config-browser.js
@@ -24,6 +24,8 @@ module.exports = () => ({
'/dns4/nyc-1.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmSoLueR4xBeUbY9WZ9xGUUxunbKWcrNFTDAadQJmocnWm',
'/dns4/nyc-2.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64',
'/dns4/wss0.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic',
- '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6'
+ '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6',
+ '/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic',
+ '/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6'
]
})
diff --git a/src/core/runtime/config-nodejs.js b/src/core/runtime/config-nodejs.js
index 995f66261d..d56550e181 100644
--- a/src/core/runtime/config-nodejs.js
+++ b/src/core/runtime/config-nodejs.js
@@ -37,6 +37,8 @@ module.exports = () => ({
'/ip6/2a03:b0c0:1:d0::e7:1/tcp/4001/ipfs/QmSoLMeWqB7YGVLJN3pNLQpmmEk35v6wYtsMGLzSr5QBU3',
'/ip6/2604:a880:1:20::1d9:6001/tcp/4001/ipfs/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx',
'/dns4/wss0.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic',
- '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6'
+ '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6',
+ '/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic',
+ '/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6'
]
})
diff --git a/src/core/runtime/preload-browser.js b/src/core/runtime/preload-browser.js
new file mode 100644
index 0000000000..8d123a12be
--- /dev/null
+++ b/src/core/runtime/preload-browser.js
@@ -0,0 +1,37 @@
+/* eslint-env browser */
+'use strict'
+
+const debug = require('debug')
+
+const log = debug('jsipfs:preload')
+log.error = debug('jsipfs:preload:error')
+
+module.exports = function preload (url, callback) {
+ log(url)
+
+ const req = new self.XMLHttpRequest()
+
+ req.open('HEAD', url)
+
+ req.onreadystatechange = function () {
+ if (this.readyState !== this.DONE) {
+ return
+ }
+
+ if (this.status < 200 || this.status >= 300) {
+ log.error('failed to preload', url, this.status, this.statusText)
+ return callback(new Error(`failed to preload ${url}`))
+ }
+
+ callback()
+ }
+
+ req.send()
+
+ return {
+ cancel: () => {
+ req.abort()
+ callback(new Error('request aborted'))
+ }
+ }
+}
diff --git a/src/core/runtime/preload-nodejs.js b/src/core/runtime/preload-nodejs.js
new file mode 100644
index 0000000000..405798ca34
--- /dev/null
+++ b/src/core/runtime/preload-nodejs.js
@@ -0,0 +1,64 @@
+'use strict'
+
+const http = require('http')
+const https = require('https')
+const { URL } = require('url')
+const debug = require('debug')
+const setImmediate = require('async/setImmediate')
+
+const log = debug('jsipfs:preload')
+log.error = debug('jsipfs:preload:error')
+
+module.exports = function preload (url, callback) {
+ log(url)
+
+ try {
+ url = new URL(url)
+ } catch (err) {
+ return setImmediate(() => callback(err))
+ }
+
+ const transport = url.protocol === 'https:' ? https : http
+
+ const req = transport.get({
+ hostname: url.hostname,
+ port: url.port,
+ path: url.pathname + url.search
+ }, (res) => {
+ if (res.statusCode < 200 || res.statusCode >= 300) {
+ res.resume()
+ log.error('failed to preload', url.href, res.statusCode, res.statusMessage)
+ return callback(new Error(`failed to preload ${url}`))
+ }
+
+ res.on('data', chunk => log(`data ${chunk}`))
+
+ res.on('abort', () => {
+ callback(new Error('request aborted'))
+ })
+
+ res.on('error', err => {
+ log.error('response error preloading', url.href, err)
+ callback(err)
+ })
+
+ res.on('end', () => {
+ // If aborted, callback is called in the abort handler
+ if (!res.aborted) callback()
+ })
+ })
+
+ req.on('error', err => {
+ log.error('request error preloading', url.href, err)
+ callback(err)
+ })
+
+ return {
+ cancel: () => {
+ // No need to call callback here
+ // before repsonse - called in req error handler
+ // after response - called in res abort hander
+ req.abort()
+ }
+ }
+}
diff --git a/test/cli/bootstrap.js b/test/cli/bootstrap.js
index 8807a12411..a301bca3f2 100644
--- a/test/cli/bootstrap.js
+++ b/test/cli/bootstrap.js
@@ -31,7 +31,9 @@ describe('bootstrap', () => runOnAndOff((thing) => {
'/ip6/2a03:b0c0:1:d0::e7:1/tcp/4001/ipfs/QmSoLMeWqB7YGVLJN3pNLQpmmEk35v6wYtsMGLzSr5QBU3',
'/ip6/2604:a880:1:20::1d9:6001/tcp/4001/ipfs/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx',
'/dns4/wss0.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic',
- '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6'
+ '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6',
+ '/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic',
+ '/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6'
]
const updatedList = [
@@ -54,6 +56,8 @@ describe('bootstrap', () => runOnAndOff((thing) => {
'/ip6/2604:a880:1:20::1d9:6001/tcp/4001/ipfs/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx',
'/dns4/wss0.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic',
'/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6',
+ '/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic',
+ '/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6',
'/ip4/111.111.111.111/tcp/1001/ipfs/QmcyFFKfLDGJKwufn2GeitxvhricsBQyNKTkrD14psikoD'
]
diff --git a/test/core/bootstrap.spec.js b/test/core/bootstrap.spec.js
index f092765354..d95d622f18 100644
--- a/test/core/bootstrap.spec.js
+++ b/test/core/bootstrap.spec.js
@@ -59,7 +59,9 @@ describe('bootstrap', () => {
'/ip6/2a03:b0c0:1:d0::e7:1/tcp/4001/ipfs/QmSoLMeWqB7YGVLJN3pNLQpmmEk35v6wYtsMGLzSr5QBU3',
'/ip6/2604:a880:1:20::1d9:6001/tcp/4001/ipfs/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx',
'/dns4/wss0.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic',
- '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6'
+ '/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6',
+ '/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic',
+ '/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6'
]
const updatedList = [
@@ -82,6 +84,8 @@ describe('bootstrap', () => {
'/ip6/2604:a880:1:20::1d9:6001/tcp/4001/ipfs/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx',
'/dns4/wss0.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic',
'/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6',
+ '/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic',
+ '/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6',
'/ip4/111.111.111.111/tcp/1001/ipfs/QmXFX2P5ammdmXQgfqGkfswtEVFsZUJ5KeHRXQYCTdiTAb'
]
diff --git a/test/core/preload.spec.js b/test/core/preload.spec.js
new file mode 100644
index 0000000000..54fbed45cf
--- /dev/null
+++ b/test/core/preload.spec.js
@@ -0,0 +1,130 @@
+/* eslint max-nested-callbacks: ["error", 8] */
+/* eslint-env mocha */
+'use strict'
+
+const hat = require('hat')
+const chai = require('chai')
+const dirtyChai = require('dirty-chai')
+const expect = chai.expect
+chai.use(dirtyChai)
+
+const MockPreloadNode = require('../utils/mock-preload-node')
+const IPFS = require('../../src')
+
+describe('preload', () => {
+ let ipfs
+
+ before((done) => {
+ ipfs = new IPFS({
+ config: {
+ Addresses: {
+ Swarm: []
+ }
+ },
+ preload: {
+ enabled: true,
+ addresses: [MockPreloadNode.defaultAddr]
+ }
+ })
+
+ ipfs.on('ready', done)
+ })
+
+ afterEach((done) => MockPreloadNode.clearPreloadCids(done))
+
+ after((done) => ipfs.stop(done))
+
+ it('should preload content added with ipfs.files.add', (done) => {
+ ipfs.files.add(Buffer.from(hat()), (err, res) => {
+ expect(err).to.not.exist()
+
+ // Wait for preloading to finish
+ setTimeout(() => {
+ MockPreloadNode.getPreloadCids((err, cids) => {
+ expect(err).to.not.exist()
+ expect(cids.length).to.equal(1)
+ expect(cids[0]).to.equal(res[0].hash)
+ done()
+ })
+ }, 100)
+ })
+ })
+
+ it('should preload multiple content added with ipfs.files.add', (done) => {
+ ipfs.files.add([{
+ content: Buffer.from(hat())
+ }, {
+ content: Buffer.from(hat())
+ }, {
+ content: Buffer.from(hat())
+ }], (err, res) => {
+ expect(err).to.not.exist()
+
+ // Wait for preloading to finish
+ setTimeout(() => {
+ MockPreloadNode.getPreloadCids((err, cids) => {
+ expect(err).to.not.exist()
+ expect(cids.length).to.equal(res.length)
+ res.forEach(file => expect(cids).to.include(file.hash))
+ done()
+ })
+ }, 100)
+ })
+ })
+
+ it('should preload multiple content and intermediate dirs added with ipfs.files.add', (done) => {
+ ipfs.files.add([{
+ path: 'dir0/dir1/file0',
+ content: Buffer.from(hat())
+ }, {
+ path: 'dir0/dir1/file1',
+ content: Buffer.from(hat())
+ }, {
+ path: 'dir0/file2',
+ content: Buffer.from(hat())
+ }], (err, res) => {
+ expect(err).to.not.exist()
+
+ const rootDir = res.find(file => file.path === 'dir0')
+ expect(rootDir).to.exist()
+
+ // Wait for preloading to finish
+ setTimeout(() => {
+ MockPreloadNode.getPreloadCids((err, cids) => {
+ expect(err).to.not.exist()
+ expect(cids.length).to.equal(1)
+ expect(cids[0]).to.equal(rootDir.hash)
+ done()
+ })
+ }, 100)
+ })
+ })
+
+ it('should preload multiple content and wrapping dir for content added with ipfs.files.add and wrapWithDirectory option', (done) => {
+ ipfs.files.add([{
+ path: 'dir0/dir1/file0',
+ content: Buffer.from(hat())
+ }, {
+ path: 'dir0/dir1/file1',
+ content: Buffer.from(hat())
+ }, {
+ path: 'dir0/file2',
+ content: Buffer.from(hat())
+ }], { wrapWithDirectory: true }, (err, res) => {
+ expect(err).to.not.exist()
+
+ const wrappingDir = res.find(file => file.path === '')
+ expect(wrappingDir).to.exist()
+
+ // Wait for preloading to finish
+ setTimeout(() => {
+ MockPreloadNode.getPreloadCids((err, cids) => {
+ expect(err).to.not.exist()
+ expect(cids.length).to.equal(1)
+ expect(cids[0]).to.equal(wrappingDir.hash)
+ done()
+ })
+ })
+ })
+ })
+})
diff --git a/test/fixtures/go-ipfs-repo/config b/test/fixtures/go-ipfs-repo/config
index 00f467f95f..9843d866a8 100644
--- a/test/fixtures/go-ipfs-repo/config
+++ b/test/fixtures/go-ipfs-repo/config
@@ -65,7 +65,9 @@
"/ip6/2a03:b0c0:1:d0::e7:1/tcp/4001/ipfs/QmSoLMeWqB7YGVLJN3pNLQpmmEk35v6wYtsMGLzSr5QBU3",
"/ip6/2604:a880:1:20::1d9:6001/tcp/4001/ipfs/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx",
"/dns4/wss0.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic",
- "/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6"
+ "/dns4/wss1.bootstrap.libp2p.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6",
+ "/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic",
+ "/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6"
],
"Tour": {
"Last": ""
@@ -106,4 +108,4 @@
"hash": "sha2-512"
}
}
-}
\ No newline at end of file
+}
diff --git a/test/utils/mock-preload-node.js b/test/utils/mock-preload-node.js
new file mode 100644
index 0000000000..0f08c8d61d
--- /dev/null
+++ b/test/utils/mock-preload-node.js
@@ -0,0 +1,118 @@
+/* eslint-env browser */
+'use strict'
+
+const http = require('http')
+const toUri = require('multiaddr-to-uri')
+const URL = require('url').URL || self.URL
+
+const defaultPort = 1138
+const defaultAddr = `/dnsaddr/localhost/tcp/${defaultPort}`
+
+module.exports.defaultAddr = defaultAddr
+
+// Create a mock preload IPFS node with a gateway that'll respond 200 to a
+// request for /api/v0/refs?arg=*. It remembers the preload CIDs it has been
+// called with, and you can ask it for them and also clear them by issuing a
+// GET/DELETE request to /cids.
+module.exports.createNode = () => {
+ let cids = []
+
+ const server = http.createServer((req, res) => {
+ if (req.url.startsWith('/api/v0/refs')) {
+ const arg = new URL(`https://ipfs.io${req.url}`).searchParams.get('arg')
+ cids = cids.concat(arg)
+ } else if (req.method === 'DELETE' && req.url === '/cids') {
+ res.statusCode = 204
+ cids = []
+ } else if (req.method === 'GET' && req.url === '/cids') {
+ res.setHeader('Content-Type', 'application/json')
+ res.write(JSON.stringify(cids))
+ } else {
+ res.statusCode = 500
+ }
+
+ res.end()
+ })
+
+ server.start = (opts, cb) => {
+ if (typeof opts === 'function') {
+ cb = opts
+ opts = {}
+ }
+ return server.listen(Object.assign({ port: defaultPort }, opts), cb)
+ }
+
+ server.stop = (cb) => server.close(cb)
+
+ return server
+}
+
+function parseMultiaddr (addr) {
+ if (!(addr.endsWith('http') || addr.endsWith('https'))) {
+ addr = addr + '/http'
+ }
+ return new URL(toUri(addr))
+}
+
+// Get the stored preload CIDs for the server at `addr`
+module.exports.getPreloadCids = (addr, cb) => {
+ if (typeof addr === 'function') {
+ cb = addr
+ addr = defaultAddr
+ }
+
+ const { protocol, hostname, port } = parseMultiaddr(addr)
+
+ const req = http.get({ protocol, hostname, port, path: '/cids' }, (res) => {
+ if (res.statusCode !== 200) {
+ res.resume()
+ return cb(new Error('failed to get preloaded CIDs from mock preload node'))
+ }
+
+ let data = ''
+
+ res.on('error', cb)
+ res.on('data', chunk => { data += chunk })
+
+ res.on('end', () => {
+ let obj
+ try {
+ obj = JSON.parse(data)
+ } catch (err) {
+ return cb(err)
+ }
+ cb(null, obj)
+ })
+ })
+
+ req.on('error', cb)
+}
+
+// Clear the stored preload URLs for the server at `addr`
+module.exports.clearPreloadCids = (addr, cb) => {
+ if (typeof addr === 'function') {
+ cb = addr
+ addr = defaultAddr
+ }
+
+ const { protocol, hostname, port } = parseMultiaddr(addr)
+
+ const req = http.request({
+ method: 'DELETE',
+ protocol,
+ hostname,
+ port,
+ path: '/cids'
+ }, (res) => {
+ res.resume()
+
+ if (res.statusCode !== 204) {
+ return cb(new Error('failed to clear CIDs from mock preload node'))
+ }
+
+ cb()
+ })
+
+ req.on('error', cb)
+ req.end()
+}