From c8cdb6378b61641e2b8159e12db1822322630d1c Mon Sep 17 00:00:00 2001 From: Stephen Whitmore Date: Mon, 23 May 2016 14:09:15 -0700 Subject: [PATCH 1/6] Make ipfs.files.add return DAGNodes. --- package.json | 3 ++- src/api/add.js | 21 ++++++++++++++++++ src/request-api.js | 39 ++++++++++++++++++++++++++++++++- test/api/add.spec.js | 51 +++++++++++++++++++++++++++----------------- 4 files changed, 93 insertions(+), 21 deletions(-) diff --git a/package.json b/package.json index ac57b0fe4..0cfa01255 100644 --- a/package.json +++ b/package.json @@ -5,8 +5,9 @@ "main": "lib/index.js", "jsnext:main": "src/index.js", "dependencies": { - "bl": "^1.1.2", + "async": "^2.0.0-rc.5", "babel-runtime": "^6.6.1", + "bl": "^1.1.2", "bs58": "^3.0.0", "detect-node": "^2.0.3", "flatmap": "0.0.3", diff --git a/src/api/add.js b/src/api/add.js index 4b0d4e6b8..21a1d192c 100644 --- a/src/api/add.js +++ b/src/api/add.js @@ -1,6 +1,8 @@ 'use strict' const Wreck = require('wreck') +const async = require('async') +const DAGNode = require('ipfs-merkle-dag').DAGNode module.exports = (send) => { return function add (files, opts, cb) { @@ -9,6 +11,8 @@ module.exports = (send) => { opts = {} } + send = send.withTransform(transform) + if (typeof files === 'string' && files.startsWith('http')) { return Wreck.request('GET', files, null, (err, res) => { if (err) return cb(err) @@ -18,5 +22,22 @@ module.exports = (send) => { } return send('add', null, opts, files, cb) + + // transform returned objects into DAGNodes + function transform (err, res, done) { + if (err) return done(err) + + async.map(res, + function map (entry, fin) { + send('object/get', entry.Hash, null, null, function (err, result) { + if (err) return done(err) + const node = new DAGNode(result.Data, result.Links) + fin(err, node) + }) + }, + function complete (err, results) { + if (done) return done(err, results) + }) + } } } diff --git a/src/request-api.js b/src/request-api.js index 7b2d73c53..700f68260 100644 --- a/src/request-api.js +++ b/src/request-api.js @@ -110,7 +110,7 @@ function requestAPI (config, path, args, qs, files, buffer, cb) { // -- Interface exports = module.exports = function getRequestAPI (config) { - return function (path, args, qs, files, buffer, cb) { + var send = function (path, args, qs, files, buffer, cb) { if (typeof buffer === 'function') { cb = buffer buffer = false @@ -127,4 +127,41 @@ exports = module.exports = function getRequestAPI (config) { return requestAPI(config, path, args, qs, files, buffer, cb) } + + // Wraps the 'send' function such that an asynchronous transform may be + // applied to its result before passing it on to either its callback or + // promise. + send.withTransform = function (transform) { + return function (path, args, qs, files, buffer, cb) { + if (typeof buffer === 'function') { + cb = buffer + buffer = false + } + + var p = send(path, args, qs, files, buffer, wrap(cb)) + + if (p instanceof Promise) { + return p.then((res) => { + return new Promise(function (resolve, reject) { + transform(null, res, function (err, res) { + if (err) reject(err) + else resolve(res) + }) + }) + }) + } else { + return p + } + + function wrap (done) { + if (done) { + return function (err, res) { + transform(err, res, done) + } + } + } + } + } + + return send } diff --git a/test/api/add.spec.js b/test/api/add.spec.js index 6c09509ce..eb2212791 100644 --- a/test/api/add.spec.js +++ b/test/api/add.spec.js @@ -7,6 +7,7 @@ const Readable = require('stream').Readable const path = require('path') const isNode = require('detect-node') const fs = require('fs') +const bs58 = require('bs58') let testfile let testfileBig @@ -37,8 +38,9 @@ describe('.add', () => { expect(err).to.not.exist const added = res[0] != null ? res[0] : res - expect(added).to.have.property('Hash', 'Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP') - expect(added).to.have.property('Name', 'testfile.txt') + const mh = bs58.encode(added.multihash()).toString() + expect(mh).to.equal('Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP') + expect(added.links).to.have.length(0) done() }) }) @@ -49,7 +51,9 @@ describe('.add', () => { expect(err).to.not.exist expect(res).to.have.length(1) - expect(res[0]).to.have.property('Hash', 'Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP') + const mh = bs58.encode(res[0].multihash()).toString() + expect(mh).to.equal('Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP') + expect(res[0].links).to.have.length(0) done() }) }) @@ -63,7 +67,9 @@ describe('.add', () => { expect(err).to.not.exist expect(res).to.have.length(1) - expect(res[0]).to.have.a.property('Hash', 'Qme79tX2bViL26vNjPsF3DP1R9rMKMvnPYJiKTTKPrXJjq') + const mh = bs58.encode(res[0].multihash()).toString() + expect(mh).to.equal('Qmcx5werSWQPdrGVap7LARHB4QUSPRPJwxhFuHvdoXqQXT') + expect(res[0].links).to.have.length(58) done() }) }) @@ -77,7 +83,9 @@ describe('.add', () => { expect(err).to.not.exist const added = res[0] != null ? res[0] : res - expect(added).to.have.property('Hash', 'Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP') + const mh = bs58.encode(added.multihash()).toString() + expect(mh).to.equal('Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP') + expect(added.links).to.have.length(0) done() }) }) @@ -88,14 +96,9 @@ describe('.add', () => { expect(err).to.not.exist const added = res[res.length - 1] - expect(added).to.have.property('Hash', 'QmRNjDeKStKGTQXnJ2NFqeQ9oW23WcpbmvCVrpDHgDg3T6') - - // check that the symlink was replaced by the target file - const linkPath = 'test-folder/hello-link' - const filePath = 'test-folder/files/hello.txt' - const linkHash = res.filter((e) => e.Name === linkPath)[0].Hash - const fileHash = res.filter((e) => e.Name === filePath)[0].Hash - expect(linkHash).to.equal(fileHash) + const mh = bs58.encode(added.multihash()).toString() + expect(mh).to.equal('QmRNjDeKStKGTQXnJ2NFqeQ9oW23WcpbmvCVrpDHgDg3T6') + expect(added.links).to.have.length(7) done() } else { @@ -112,7 +115,9 @@ describe('.add', () => { const added = res[res.length - 1] // same hash as the result from the cli (ipfs add test/test-folder -r) - expect(added).to.have.property('Hash', 'QmRArDYd8Rk7Zb7K2699KqmQM1uUoejn1chtEAcqkvjzGg') + const mh = bs58.encode(added.multihash()).toString() + expect(mh).to.equal('QmRArDYd8Rk7Zb7K2699KqmQM1uUoejn1chtEAcqkvjzGg') + expect(added.links).to.have.length(7) done() } else { expect(err.message).to.be.equal('Recursive uploads are not supported in the browser') @@ -147,7 +152,9 @@ describe('.add', () => { expect(err).to.not.exist const added = res[res.length - 1] - expect(added).to.have.property('Hash', 'QmTDH2RXGn8XyDAo9YyfbZAUXwL1FCr44YJCN9HBZmL9Gj') + const mh = bs58.encode(added.multihash()).toString() + expect(mh).to.equal('QmTDH2RXGn8XyDAo9YyfbZAUXwL1FCr44YJCN9HBZmL9Gj') + expect(added.links).to.have.length(6) done() }) }) @@ -161,7 +168,9 @@ describe('.add', () => { expect(err).to.not.exist const added = res[0] != null ? res[0] : res - expect(added).to.have.a.property('Hash', 'QmNRCQWfgze6AbBCaT1rkrkV5tJ2aP4oTNPb5JZcXYywve') + const mh = bs58.encode(added.multihash()).toString() + expect(mh).to.equal('QmNRCQWfgze6AbBCaT1rkrkV5tJ2aP4oTNPb5JZcXYywve') + expect(added.links).to.have.length(0) done() }) }) @@ -172,7 +181,9 @@ describe('.add', () => { expect(err).to.not.exist const added = res[0] != null ? res[0] : res - expect(added).to.have.a.property('Hash', 'QmZmHgEX9baxUn3qMjsEXQzG6DyNcrVnwieQQTrpDdrFvt') + const mh = bs58.encode(added.multihash()).toString() + expect(mh).to.equal('QmRzvSX35JpzQ2Lyn55r3YwWqdVP6PPxYHFpiWpwQTff8A') + expect(added.links).to.have.length(0) done() }) }) @@ -182,8 +193,10 @@ describe('.add', () => { let buf = new Buffer(testfile) return apiClients.a.add(buf) .then((res) => { - expect(res).to.have.length(1) - expect(res[0]).to.have.property('Hash', 'Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP') + const added = res[0] != null ? res[0] : res + const mh = bs58.encode(added.multihash()).toString() + expect(mh).to.equal('Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP') + expect(added.links).to.have.length(0) }) }) }) From 6c22cd52edcf91aa03c79e87d8206c6023d76bac Mon Sep 17 00:00:00 2001 From: Stephen Whitmore Date: Tue, 24 May 2016 11:16:01 -0700 Subject: [PATCH 2/6] Add "addUrl" API. --- package.json | 1 + src/add-to-dagnode-transform.js | 12 +++++++++++ src/api/add-url.js | 27 +++++++++++++++++++++++++ src/api/add.js | 35 +++++++++------------------------ src/get-dagnode.js | 9 +++++++++ src/load-commands.js | 1 + src/request-api.js | 4 ++-- test/api/add.spec.js | 2 +- 8 files changed, 62 insertions(+), 29 deletions(-) create mode 100644 src/add-to-dagnode-transform.js create mode 100644 src/api/add-url.js create mode 100644 src/get-dagnode.js diff --git a/package.json b/package.json index 0cfa01255..90e68d2bb 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "flatmap": "0.0.3", "glob": "^7.0.3", "ipfs-merkle-dag": "^0.6.0", + "isstream": "^0.1.2", "multiaddr": "^2.0.0", "multipart-stream": "^2.0.1", "ndjson": "^1.4.3", diff --git a/src/add-to-dagnode-transform.js b/src/add-to-dagnode-transform.js new file mode 100644 index 000000000..109740b99 --- /dev/null +++ b/src/add-to-dagnode-transform.js @@ -0,0 +1,12 @@ +const async = require('async') +const getDagNode = require('./get-dagnode') + +// transform { Hash: '...' } objects into DAGNodes async +module.exports = function (err, res, send, done) { + if (err) return done(err) + async.map(res, function map (entry, next) { + getDagNode(send, entry.Hash, next) + }, function (err, res) { + done(err, res) + }) +} diff --git a/src/api/add-url.js b/src/api/add-url.js new file mode 100644 index 000000000..1aeb2cd47 --- /dev/null +++ b/src/api/add-url.js @@ -0,0 +1,27 @@ +'use strict' + +const Wreck = require('wreck') +const async = require('async') +const DAGNode = require('ipfs-merkle-dag').DAGNode +const addToDagNodesTransform = require('../add-to-dagnode-transform') + +module.exports = (send) => { + return function add (url, opts, cb) { + if (typeof (opts) === 'function' && cb === undefined) { + cb = opts + opts = {} + } + + if (typeof url !== 'string' || !url.startsWith('http')) { + return cb(new Error('"url" param must be an http(s) url')) + } + + var sendWithTransform = send.withTransform(addToDagNodesTransform) + + Wreck.request('GET', url, null, (err, res) => { + if (err) return cb(err) + + sendWithTransform('add', null, opts, res, cb) + }) + } +} diff --git a/src/api/add.js b/src/api/add.js index 21a1d192c..7e96a4145 100644 --- a/src/api/add.js +++ b/src/api/add.js @@ -1,8 +1,8 @@ 'use strict' +const isStream = require('isstream') const Wreck = require('wreck') -const async = require('async') -const DAGNode = require('ipfs-merkle-dag').DAGNode +const addToDagNodesTransform = require('../add-to-dagnode-transform') module.exports = (send) => { return function add (files, opts, cb) { @@ -11,33 +11,16 @@ module.exports = (send) => { opts = {} } - send = send.withTransform(transform) + var good = Buffer.isBuffer(files) || + isStream.isReadable(files) || + Array.isArray(files) - if (typeof files === 'string' && files.startsWith('http')) { - return Wreck.request('GET', files, null, (err, res) => { - if (err) return cb(err) - - send('add', null, opts, res, cb) - }) + if (!good) { + return cb(new Error('"files" must be a buffer, readable stream, or array of objects')) } - return send('add', null, opts, files, cb) - - // transform returned objects into DAGNodes - function transform (err, res, done) { - if (err) return done(err) + var sendWithTransform = send.withTransform(addToDagNodesTransform) - async.map(res, - function map (entry, fin) { - send('object/get', entry.Hash, null, null, function (err, result) { - if (err) return done(err) - const node = new DAGNode(result.Data, result.Links) - fin(err, node) - }) - }, - function complete (err, results) { - if (done) return done(err, results) - }) - } + return sendWithTransform('add', null, opts, files, cb) } } diff --git a/src/get-dagnode.js b/src/get-dagnode.js new file mode 100644 index 000000000..1b7c2a193 --- /dev/null +++ b/src/get-dagnode.js @@ -0,0 +1,9 @@ +const DAGNode = require('ipfs-merkle-dag').DAGNode + +module.exports = function (send, hash, cb) { + send('object/get', hash, null, null, function (err, result) { + if (err) return cb(err) + const node = new DAGNode(result.Data, result.Links) + cb(err, node) + }) +} diff --git a/src/load-commands.js b/src/load-commands.js index fc5ee614b..d8e98e436 100644 --- a/src/load-commands.js +++ b/src/load-commands.js @@ -3,6 +3,7 @@ function requireCommands () { return { add: require('./api/add'), + addUrl: require('./api/add-url'), bitswap: require('./api/bitswap'), block: require('./api/block'), cat: require('./api/cat'), diff --git a/src/request-api.js b/src/request-api.js index 700f68260..753305bc0 100644 --- a/src/request-api.js +++ b/src/request-api.js @@ -143,7 +143,7 @@ exports = module.exports = function getRequestAPI (config) { if (p instanceof Promise) { return p.then((res) => { return new Promise(function (resolve, reject) { - transform(null, res, function (err, res) { + transform(null, res, send, function (err, res) { if (err) reject(err) else resolve(res) }) @@ -156,7 +156,7 @@ exports = module.exports = function getRequestAPI (config) { function wrap (done) { if (done) { return function (err, res) { - transform(err, res, done) + transform(err, res, send, done) } } } diff --git a/test/api/add.spec.js b/test/api/add.spec.js index eb2212791..4505fd9e1 100644 --- a/test/api/add.spec.js +++ b/test/api/add.spec.js @@ -177,7 +177,7 @@ describe('.add', () => { it('add url', (done) => { const url = 'https://raw.githubusercontent.com/ipfs/js-ipfs-api/2a9cc63d7427353f2145af6b1a768a69e67c0588/README.md' - apiClients.a.add(url, (err, res) => { + apiClients.a.addUrl(url, (err, res) => { expect(err).to.not.exist const added = res[0] != null ? res[0] : res From f942e76df8c90e46876f227f7911569c637ba36a Mon Sep 17 00:00:00 2001 From: Stephen Whitmore Date: Tue, 24 May 2016 11:33:04 -0700 Subject: [PATCH 3/6] Add "addFiles" API. --- src/add-to-dagnode-transform.js | 2 ++ src/api/add-files.js | 20 ++++++++++++++++++++ src/api/add-url.js | 2 -- src/api/add.js | 1 - src/get-dagnode.js | 2 ++ src/load-commands.js | 1 + test/api/add.spec.js | 14 +++++++------- 7 files changed, 32 insertions(+), 10 deletions(-) create mode 100644 src/api/add-files.js diff --git a/src/add-to-dagnode-transform.js b/src/add-to-dagnode-transform.js index 109740b99..844e5761a 100644 --- a/src/add-to-dagnode-transform.js +++ b/src/add-to-dagnode-transform.js @@ -1,3 +1,5 @@ +'use strict' + const async = require('async') const getDagNode = require('./get-dagnode') diff --git a/src/api/add-files.js b/src/api/add-files.js new file mode 100644 index 000000000..c5362f643 --- /dev/null +++ b/src/api/add-files.js @@ -0,0 +1,20 @@ +'use strict' + +const addToDagNodesTransform = require('../add-to-dagnode-transform') + +module.exports = (send) => { + return function add (path, opts, cb) { + if (typeof (opts) === 'function' && cb === undefined) { + cb = opts + opts = {} + } + + if (typeof (path) !== 'string') { + return cb(new Error('"path" must be a string')) + } + + var sendWithTransform = send.withTransform(addToDagNodesTransform) + + return sendWithTransform('add', null, opts, path, cb) + } +} diff --git a/src/api/add-url.js b/src/api/add-url.js index 1aeb2cd47..fccb96e75 100644 --- a/src/api/add-url.js +++ b/src/api/add-url.js @@ -1,8 +1,6 @@ 'use strict' const Wreck = require('wreck') -const async = require('async') -const DAGNode = require('ipfs-merkle-dag').DAGNode const addToDagNodesTransform = require('../add-to-dagnode-transform') module.exports = (send) => { diff --git a/src/api/add.js b/src/api/add.js index 7e96a4145..0b465ef7b 100644 --- a/src/api/add.js +++ b/src/api/add.js @@ -1,7 +1,6 @@ 'use strict' const isStream = require('isstream') -const Wreck = require('wreck') const addToDagNodesTransform = require('../add-to-dagnode-transform') module.exports = (send) => { diff --git a/src/get-dagnode.js b/src/get-dagnode.js index 1b7c2a193..717456805 100644 --- a/src/get-dagnode.js +++ b/src/get-dagnode.js @@ -1,3 +1,5 @@ +'use strict' + const DAGNode = require('ipfs-merkle-dag').DAGNode module.exports = function (send, hash, cb) { diff --git a/src/load-commands.js b/src/load-commands.js index d8e98e436..a91a4fe94 100644 --- a/src/load-commands.js +++ b/src/load-commands.js @@ -3,6 +3,7 @@ function requireCommands () { return { add: require('./api/add'), + addFiles: require('./api/add-files'), addUrl: require('./api/add-url'), bitswap: require('./api/bitswap'), block: require('./api/block'), diff --git a/test/api/add.spec.js b/test/api/add.spec.js index 4505fd9e1..eede5d280 100644 --- a/test/api/add.spec.js +++ b/test/api/add.spec.js @@ -24,7 +24,7 @@ if (isNode) { } describe('.add', () => { - it('add file', (done) => { + it('add buffer as tuple', (done) => { if (!isNode) { return done() } @@ -74,12 +74,12 @@ describe('.add', () => { }) }) - it('add path', (done) => { + it('local fs: add file', (done) => { if (!isNode) { return done() } - apiClients.a.add(testfilePath, (err, res) => { + apiClients.a.addFiles(testfilePath, (err, res) => { expect(err).to.not.exist const added = res[0] != null ? res[0] : res @@ -90,8 +90,8 @@ describe('.add', () => { }) }) - it('add a nested dir following symlinks', (done) => { - apiClients.a.add(path.join(__dirname, '/../test-folder'), { recursive: true }, (err, res) => { + it('local fs: add nested dir (follow symlinks)', (done) => { + apiClients.a.addFiles(path.join(__dirname, '/../test-folder'), { recursive: true }, (err, res) => { if (isNode) { expect(err).to.not.exist @@ -108,8 +108,8 @@ describe('.add', () => { }) }) - it('add a nested dir without following symlinks', (done) => { - apiClients.a.add(path.join(__dirname, '/../test-folder'), { recursive: true, followSymlinks: false }, (err, res) => { + it('local fs: add nested dir (don\'t follow symlinks)', (done) => { + apiClients.a.addFiles(path.join(__dirname, '/../test-folder'), { recursive: true, followSymlinks: false }, (err, res) => { if (isNode) { expect(err).to.not.exist From 8ae764eab7e8c25fd75b8cb385c7adb5c5f84141 Mon Sep 17 00:00:00 2001 From: Stephen Whitmore Date: Tue, 24 May 2016 13:16:51 -0700 Subject: [PATCH 4/6] Update 'files' and 'util' command paths. --- src/load-commands.js | 24 +++++++++++++++++++----- test/api/add.spec.js | 20 ++++++++++---------- 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/src/load-commands.js b/src/load-commands.js index a91a4fe94..3c451018c 100644 --- a/src/load-commands.js +++ b/src/load-commands.js @@ -1,10 +1,7 @@ 'use strict' function requireCommands () { - return { - add: require('./api/add'), - addFiles: require('./api/add-files'), - addUrl: require('./api/add-url'), + var cmds = { bitswap: require('./api/bitswap'), block: require('./api/block'), cat: require('./api/cat'), @@ -13,7 +10,6 @@ function requireCommands () { dht: require('./api/dht'), diag: require('./api/diag'), id: require('./api/id'), - files: require('./api/files'), log: require('./api/log'), ls: require('./api/ls'), mount: require('./api/mount'), @@ -26,6 +22,24 @@ function requireCommands () { update: require('./api/update'), version: require('./api/version') } + + // TODO: crowding the 'files' namespace temporarily for interface-ipfs-core + // compatibility, until 'files vs mfs' naming decision is resolved. + cmds.files = function (send) { + var files = require('./api/files')(send) + files.add = require('./api/add')(send) + return files + } + + cmds.util = function (send) { + var util = { + addFiles: require('./api/add-files')(send), + addUrl: require('./api/add-url')(send) + } + return util + } + + return cmds } function loadCommands (send) { diff --git a/test/api/add.spec.js b/test/api/add.spec.js index eede5d280..6b5a58072 100644 --- a/test/api/add.spec.js +++ b/test/api/add.spec.js @@ -34,7 +34,7 @@ describe('.add', () => { content: new Buffer(testfile) } - apiClients.a.add([file], (err, res) => { + apiClients.a.files.add([file], (err, res) => { expect(err).to.not.exist const added = res[0] != null ? res[0] : res @@ -47,7 +47,7 @@ describe('.add', () => { it('add buffer', (done) => { let buf = new Buffer(testfile) - apiClients.a.add(buf, (err, res) => { + apiClients.a.files.add(buf, (err, res) => { expect(err).to.not.exist expect(res).to.have.length(1) @@ -63,7 +63,7 @@ describe('.add', () => { return done() } - apiClients.a.add(testfileBig, (err, res) => { + apiClients.a.files.add(testfileBig, (err, res) => { expect(err).to.not.exist expect(res).to.have.length(1) @@ -79,7 +79,7 @@ describe('.add', () => { return done() } - apiClients.a.addFiles(testfilePath, (err, res) => { + apiClients.a.util.addFiles(testfilePath, (err, res) => { expect(err).to.not.exist const added = res[0] != null ? res[0] : res @@ -91,7 +91,7 @@ describe('.add', () => { }) it('local fs: add nested dir (follow symlinks)', (done) => { - apiClients.a.addFiles(path.join(__dirname, '/../test-folder'), { recursive: true }, (err, res) => { + apiClients.a.util.addFiles(path.join(__dirname, '/../test-folder'), { recursive: true }, (err, res) => { if (isNode) { expect(err).to.not.exist @@ -109,7 +109,7 @@ describe('.add', () => { }) it('local fs: add nested dir (don\'t follow symlinks)', (done) => { - apiClients.a.addFiles(path.join(__dirname, '/../test-folder'), { recursive: true, followSymlinks: false }, (err, res) => { + apiClients.a.util.addFiles(path.join(__dirname, '/../test-folder'), { recursive: true, followSymlinks: false }, (err, res) => { if (isNode) { expect(err).to.not.exist @@ -148,7 +148,7 @@ describe('.add', () => { } ] - apiClients.a.add(dirs, { recursive: true }, (err, res) => { + apiClients.a.files.add(dirs, { recursive: true }, (err, res) => { expect(err).to.not.exist const added = res[res.length - 1] @@ -164,7 +164,7 @@ describe('.add', () => { stream.push('Hello world') stream.push(null) - apiClients.a.add(stream, (err, res) => { + apiClients.a.files.add(stream, (err, res) => { expect(err).to.not.exist const added = res[0] != null ? res[0] : res @@ -177,7 +177,7 @@ describe('.add', () => { it('add url', (done) => { const url = 'https://raw.githubusercontent.com/ipfs/js-ipfs-api/2a9cc63d7427353f2145af6b1a768a69e67c0588/README.md' - apiClients.a.addUrl(url, (err, res) => { + apiClients.a.util.addUrl(url, (err, res) => { expect(err).to.not.exist const added = res[0] != null ? res[0] : res @@ -191,7 +191,7 @@ describe('.add', () => { describe('promise', () => { it('add buffer', () => { let buf = new Buffer(testfile) - return apiClients.a.add(buf) + return apiClients.a.files.add(buf) .then((res) => { const added = res[0] != null ? res[0] : res const mh = bs58.encode(added.multihash()).toString() From 50dab2b66e0dfff0f2a59db77d19ecfda655554c Mon Sep 17 00:00:00 2001 From: Stephen Whitmore Date: Tue, 24 May 2016 15:29:43 -0700 Subject: [PATCH 5/6] Move "add" tests to interface-ipfs-core. --- test/api/add.spec.js | 207 +++---------------------------------------- test/api/ls.spec.js | 4 +- 2 files changed, 12 insertions(+), 199 deletions(-) diff --git a/test/api/add.spec.js b/test/api/add.spec.js index 6b5a58072..67c190235 100644 --- a/test/api/add.spec.js +++ b/test/api/add.spec.js @@ -2,202 +2,15 @@ /* globals apiClients */ 'use strict' -const expect = require('chai').expect -const Readable = require('stream').Readable -const path = require('path') -const isNode = require('detect-node') -const fs = require('fs') -const bs58 = require('bs58') - -let testfile -let testfileBig -const testfilePath = path.join(__dirname, '/../testfile.txt') - -if (isNode) { - testfile = fs.readFileSync(testfilePath) - testfileBig = fs.createReadStream(path.join(__dirname, '/../15mb.random'), { bufferSize: 128 }) - // testfileBig = fs.createReadStream(path.join(__dirname, '/../100mb.random'), { bufferSize: 128 }) -} else { - testfile = require('raw!../testfile.txt') - // browser goes nuts with a 100mb in memory - // testfileBig = require('raw!../100mb.random') +const test = require('interface-ipfs-core') + +const common = { + setup: function (cb) { + cb(null, apiClients.a) + }, + teardown: function (cb) { + cb() + } } -describe('.add', () => { - it('add buffer as tuple', (done) => { - if (!isNode) { - return done() - } - - const file = { - path: 'testfile.txt', - content: new Buffer(testfile) - } - - apiClients.a.files.add([file], (err, res) => { - expect(err).to.not.exist - - const added = res[0] != null ? res[0] : res - const mh = bs58.encode(added.multihash()).toString() - expect(mh).to.equal('Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP') - expect(added.links).to.have.length(0) - done() - }) - }) - - it('add buffer', (done) => { - let buf = new Buffer(testfile) - apiClients.a.files.add(buf, (err, res) => { - expect(err).to.not.exist - - expect(res).to.have.length(1) - const mh = bs58.encode(res[0].multihash()).toString() - expect(mh).to.equal('Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP') - expect(res[0].links).to.have.length(0) - done() - }) - }) - - it('add BIG buffer', (done) => { - if (!isNode) { - return done() - } - - apiClients.a.files.add(testfileBig, (err, res) => { - expect(err).to.not.exist - - expect(res).to.have.length(1) - const mh = bs58.encode(res[0].multihash()).toString() - expect(mh).to.equal('Qmcx5werSWQPdrGVap7LARHB4QUSPRPJwxhFuHvdoXqQXT') - expect(res[0].links).to.have.length(58) - done() - }) - }) - - it('local fs: add file', (done) => { - if (!isNode) { - return done() - } - - apiClients.a.util.addFiles(testfilePath, (err, res) => { - expect(err).to.not.exist - - const added = res[0] != null ? res[0] : res - const mh = bs58.encode(added.multihash()).toString() - expect(mh).to.equal('Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP') - expect(added.links).to.have.length(0) - done() - }) - }) - - it('local fs: add nested dir (follow symlinks)', (done) => { - apiClients.a.util.addFiles(path.join(__dirname, '/../test-folder'), { recursive: true }, (err, res) => { - if (isNode) { - expect(err).to.not.exist - - const added = res[res.length - 1] - const mh = bs58.encode(added.multihash()).toString() - expect(mh).to.equal('QmRNjDeKStKGTQXnJ2NFqeQ9oW23WcpbmvCVrpDHgDg3T6') - expect(added.links).to.have.length(7) - - done() - } else { - expect(err.message).to.be.equal('Recursive uploads are not supported in the browser') - done() - } - }) - }) - - it('local fs: add nested dir (don\'t follow symlinks)', (done) => { - apiClients.a.util.addFiles(path.join(__dirname, '/../test-folder'), { recursive: true, followSymlinks: false }, (err, res) => { - if (isNode) { - expect(err).to.not.exist - - const added = res[res.length - 1] - // same hash as the result from the cli (ipfs add test/test-folder -r) - const mh = bs58.encode(added.multihash()).toString() - expect(mh).to.equal('QmRArDYd8Rk7Zb7K2699KqmQM1uUoejn1chtEAcqkvjzGg') - expect(added.links).to.have.length(7) - done() - } else { - expect(err.message).to.be.equal('Recursive uploads are not supported in the browser') - done() - } - }) - }) - - it('add a nested dir as array', (done) => { - if (!isNode) return done() - const fs = require('fs') - const base = path.join(__dirname, '../test-folder') - const content = (name) => ({ - path: `test-folder/${name}`, - content: fs.readFileSync(path.join(base, name)) - }) - const dirs = [ - content('add.js'), - content('cat.js'), - content('ls.js'), - content('ipfs-add.js'), - content('version.js'), - content('files/hello.txt'), - content('files/ipfs.txt'), - { - path: 'test-folder', - dir: true - } - ] - - apiClients.a.files.add(dirs, { recursive: true }, (err, res) => { - expect(err).to.not.exist - - const added = res[res.length - 1] - const mh = bs58.encode(added.multihash()).toString() - expect(mh).to.equal('QmTDH2RXGn8XyDAo9YyfbZAUXwL1FCr44YJCN9HBZmL9Gj') - expect(added.links).to.have.length(6) - done() - }) - }) - - it('add stream', (done) => { - const stream = new Readable() - stream.push('Hello world') - stream.push(null) - - apiClients.a.files.add(stream, (err, res) => { - expect(err).to.not.exist - - const added = res[0] != null ? res[0] : res - const mh = bs58.encode(added.multihash()).toString() - expect(mh).to.equal('QmNRCQWfgze6AbBCaT1rkrkV5tJ2aP4oTNPb5JZcXYywve') - expect(added.links).to.have.length(0) - done() - }) - }) - - it('add url', (done) => { - const url = 'https://raw.githubusercontent.com/ipfs/js-ipfs-api/2a9cc63d7427353f2145af6b1a768a69e67c0588/README.md' - apiClients.a.util.addUrl(url, (err, res) => { - expect(err).to.not.exist - - const added = res[0] != null ? res[0] : res - const mh = bs58.encode(added.multihash()).toString() - expect(mh).to.equal('QmRzvSX35JpzQ2Lyn55r3YwWqdVP6PPxYHFpiWpwQTff8A') - expect(added.links).to.have.length(0) - done() - }) - }) - - describe('promise', () => { - it('add buffer', () => { - let buf = new Buffer(testfile) - return apiClients.a.files.add(buf) - .then((res) => { - const added = res[0] != null ? res[0] : res - const mh = bs58.encode(added.multihash()).toString() - expect(mh).to.equal('Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP') - expect(added.links).to.have.length(0) - }) - }) - }) -}) +test.files(common) diff --git a/test/api/ls.spec.js b/test/api/ls.spec.js index 97692dbca..2dbcd075a 100644 --- a/test/api/ls.spec.js +++ b/test/api/ls.spec.js @@ -30,7 +30,7 @@ describe('ls', function () { it('should correctly handle a nonexisting path', function (done) { if (!isNode) return done() - apiClients.a.ls('QmRNjDeKStKGTQXnJ2NFqeQ9oW23WcpbmvCVrpDHgDg3T6/folder_that_isnt_there', (err, res) => { + apiClients.a.ls('QmRNjDeKStKGTQXnJ2NFqeQ9oW/folder_that_isnt_there', (err, res) => { expect(err).to.exist expect(res).to.not.exist done() @@ -59,7 +59,7 @@ describe('ls', function () { it('should correctly handle a nonexisting path', () => { if (!isNode) return - return apiClients.a.ls('QmRNjDeKStKGTQXnJ2NFqeQ9oW23WcpbmvCVrpDHgDg3T6/folder_that_isnt_there') + return apiClients.a.ls('QmRNjDeKStKGTQXnJ3NFqeQ9oW/folder_that_isnt_there') .catch((err) => { expect(err).to.exist }) From 29695390fafbd5901433dfa5662889abe77a3060 Mon Sep 17 00:00:00 2001 From: Stephen Whitmore Date: Wed, 25 May 2016 11:38:04 -0700 Subject: [PATCH 6/6] Make ipfs.files.add return wrapped path+node. --- src/add-to-dagnode-transform.js | 10 ++++++++-- src/get-dagnode.js | 35 ++++++++++++++++++++++++++++----- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/src/add-to-dagnode-transform.js b/src/add-to-dagnode-transform.js index 844e5761a..61d38b4bd 100644 --- a/src/add-to-dagnode-transform.js +++ b/src/add-to-dagnode-transform.js @@ -3,11 +3,17 @@ const async = require('async') const getDagNode = require('./get-dagnode') -// transform { Hash: '...' } objects into DAGNodes async +// transform { Hash: '...' } objects into { path: 'string', node: DAGNode } module.exports = function (err, res, send, done) { if (err) return done(err) async.map(res, function map (entry, next) { - getDagNode(send, entry.Hash, next) + getDagNode(send, entry.Hash, function (err, node) { + var obj = { + path: entry.Name, + node: node + } + next(null, obj) + }) }, function (err, res) { done(err, res) }) diff --git a/src/get-dagnode.js b/src/get-dagnode.js index 717456805..25dd3aaf2 100644 --- a/src/get-dagnode.js +++ b/src/get-dagnode.js @@ -1,11 +1,36 @@ 'use strict' const DAGNode = require('ipfs-merkle-dag').DAGNode +const bl = require('bl') +const async = require('async') module.exports = function (send, hash, cb) { - send('object/get', hash, null, null, function (err, result) { - if (err) return cb(err) - const node = new DAGNode(result.Data, result.Links) - cb(err, node) - }) + + // Retrieve the object and its data in parallel, then produce a DAGNode + // instance using this information. + async.parallel([ + function get (done) { + send('object/get', hash, null, null, done) + }, + + function data (done) { + // WORKAROUND: request the object's data separately, since raw bits in JSON + // are interpreted as UTF-8 and corrupt the data. + // See https://github.com/ipfs/go-ipfs/issues/1582 for more details. + send('object/data', hash, null, null, done) + }], + + function done (err, res) { + if (err) return cb(err) + + var object = res[0] + var stream = res[1] + + stream.pipe(bl(function (err, data) { + if (err) return cb(err) + + cb(err, new DAGNode(data, object.Links)) + })) + }) + }