From e2dfe3be1c997bb29da1e6b33d68c25e9a19c77d Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 28 Nov 2018 15:42:14 +0000 Subject: [PATCH] docs: add mfs stream ls methods The only mfs ls method at the moment buffers the output into an array before returning it to the user. This PR adds two new methods, lsPullStream and lsReadableStream to allow the user to either buffer the output themseleves (in case they need sorting, etc) or just stream it on to an output of some sort. N.b the http API will not actually do any streaming until https://github.com/ipfs/go-ipfs/pull/5611 is released. --- SPEC/FILES.md | 72 +++++++++++++++++ js/src/files-mfs/index.js | 2 + js/src/files-mfs/ls-pull-stream.js | 107 +++++++++++++++++++++++++ js/src/files-mfs/ls-readable-stream.js | 107 +++++++++++++++++++++++++ js/src/files-mfs/ls.js | 20 ++--- 5 files changed, 298 insertions(+), 10 deletions(-) create mode 100644 js/src/files-mfs/ls-pull-stream.js create mode 100644 js/src/files-mfs/ls-readable-stream.js diff --git a/SPEC/FILES.md b/SPEC/FILES.md index 05ad0494..dab6dfc7 100644 --- a/SPEC/FILES.md +++ b/SPEC/FILES.md @@ -22,6 +22,8 @@ - [files.cp](#filescp) - [files.flush](#filesflush) - [files.ls](#filesls) + - [files.lsReadableStream](#fileslsreadablestream) + - [files.lsPullStream](#fileslspullstream) - [files.mkdir](#filesmkdir) - [files.mv](#filesmv) - [files.read](#filesread) @@ -1090,6 +1092,7 @@ Where: - `options` is an optional Object that might contain the following keys: - `long` is a Boolean value to decide whether or not to populate `type`, `size` and `hash` (default: false) - `cidBase` is which number base to use to format hashes - e.g. `base32`, `base64` etc (default: `base58btc`) + - `sort` is a Boolean value, if true entries will be sorted by filename (default: false) - `callback` is an optional function with the signature `function (error, files) {}`, where `error` may be an Error that occured if the operation was not successful and `files` is an array containing Objects that contain the following keys: - `name` which is the file's name @@ -1112,6 +1115,75 @@ ipfs.files.ls('/screenshots', function (err, files) { // 2018-01-22T18:08:49.184Z.png ``` +#### `files.lsReadableStream` + +> Lists a directory from the local mutable namespace that is addressed by a valid IPFS Path. The list will be yielded as Readable Streams. + +##### `Go` **WIP** + +##### `JavaScript` - ipfs.files.lsReadableStream([path], [options]) -> [Readable Stream][rs] + +Where: + +- `path` is an optional string to show listing for (default: `/`) +- `options` is an optional Object that might contain the following keys: + - `long` is a Boolean value to decide whether or not to populate `type`, `size` and `hash` (default: false) + - `cidBase` is which number base to use to format hashes - e.g. `base32`, `base64` etc (default: `base58btc`) + +It returns a [Readable Stream][rs] in [Object mode](https://nodejs.org/api/stream.html#stream_object_mode) that will yield objects containing the following keys: + + - `name` which is the file's name + - `type` which is the object's type (`directory` or `file`) + - `size` the size of the file in bytes + - `hash` the hash of the file + +**Example:** + +```JavaScript +const stream = ipfs.lsReadableStream('/some-dir') + +stream.on('data', (file) => { + // write the file's path and contents to standard out + console.log(file.name) +}) +``` + +#### `files.lsPullStream` + +> Fetch a file or an entire directory tree from IPFS that is addressed by a valid IPFS Path. The files will be yielded through a Pull Stream. + +##### `Go` **WIP** + +##### `JavaScript` - ipfs.lsPullStream([path], [options]) -> [Pull Stream][ps] + +Where: + +- `path` is an optional string to show listing for (default: `/`) +- `options` is an optional Object that might contain the following keys: + - `long` is a Boolean value to decide whether or not to populate `type`, `size` and `hash` (default: false) + - `cidBase` is which number base to use to format hashes - e.g. `base32`, `base64` etc (default: `base58btc`) + +It returns a [Pull Stream][os] that will yield objects containing the following keys: + + - `name` which is the file's name + - `type` which is the object's type (`directory` or `file`) + - `size` the size of the file in bytes + - `hash` the hash of the file + +**Example:** + +```JavaScript +pull( + ipfs.lsPullStream('/some-dir'), + pull.through(file => { + console.log(file.name) + }) + pull.onEnd(...) +) +``` + +A great source of [examples][] can be found in the tests for this API. + [examples]: https://github.com/ipfs/interface-ipfs-core/blob/master/js/src/files [b]: https://www.npmjs.com/package/buffer [rs]: https://www.npmjs.com/package/readable-stream diff --git a/js/src/files-mfs/index.js b/js/src/files-mfs/index.js index c315291f..6f70a80d 100644 --- a/js/src/files-mfs/index.js +++ b/js/src/files-mfs/index.js @@ -13,6 +13,8 @@ const tests = { readReadableStream: require('./read-readable-stream'), readPullStream: require('./read-pull-stream'), ls: require('./ls'), + lsReadableStream: require('./ls-readable-stream'), + lsPullStream: require('./ls-pull-stream'), flush: require('./flush') } diff --git a/js/src/files-mfs/ls-pull-stream.js b/js/src/files-mfs/ls-pull-stream.js new file mode 100644 index 00000000..0da05999 --- /dev/null +++ b/js/src/files-mfs/ls-pull-stream.js @@ -0,0 +1,107 @@ +/* eslint-env mocha */ +'use strict' + +const series = require('async/series') +const hat = require('hat') +const { getDescribe, getIt, expect } = require('../utils/mocha') +const pull = require('pull-stream/pull') +const onEnd = require('pull-stream/sinks/on-end') +const collect = require('pull-stream/sinks/collect') + +module.exports = (createCommon, options) => { + const describe = getDescribe(options) + const it = getIt(options) + const common = createCommon() + + describe('.files.lsPullStream', function () { + this.timeout(40 * 1000) + + let ipfs + + before(function (done) { + // CI takes longer to instantiate the daemon, so we need to increase the + // timeout for the before step + this.timeout(60 * 1000) + + common.setup((err, factory) => { + expect(err).to.not.exist() + factory.spawnNode((err, node) => { + expect(err).to.not.exist() + ipfs = node + done() + }) + }) + }) + + after((done) => common.teardown(done)) + + it('should not ls not found file/dir, expect error', (done) => { + const testDir = `/test-${hat()}` + + pull( + ipfs.files.lsPullStream(`${testDir}/404`), + onEnd((err) => { + expect(err).to.exist() + expect(err.message).to.include('does not exist') + done() + }) + ) + }) + + it('should ls directory', (done) => { + const testDir = `/test-${hat()}` + + series([ + (cb) => ipfs.files.mkdir(`${testDir}/lv1`, { p: true }, cb), + (cb) => ipfs.files.write(`${testDir}/b`, Buffer.from('Hello, world!'), { create: true }, cb) + ], (err) => { + expect(err).to.not.exist() + + pull( + ipfs.files.lsPullStream(testDir), + collect((err, entries) => { + expect(err).to.not.exist() + expect(entries.sort((a, b) => a.name.localeCompare(b.name))).to.eql([ + { name: 'b', type: 0, size: 0, hash: '' }, + { name: 'lv1', type: 0, size: 0, hash: '' } + ]) + done() + }) + ) + }) + }) + + it('should ls -l directory', (done) => { + const testDir = `/test-${hat()}` + + series([ + (cb) => ipfs.files.mkdir(`${testDir}/lv1`, { p: true }, cb), + (cb) => ipfs.files.write(`${testDir}/b`, Buffer.from('Hello, world!'), { create: true }, cb) + ], (err) => { + expect(err).to.not.exist() + + pull( + ipfs.files.lsPullStream(testDir, { l: true }), + collect((err, entries) => { + expect(err).to.not.exist() + expect(entries.sort((a, b) => a.name.localeCompare(b.name))).to.eql([ + { + name: 'b', + type: 0, + size: 13, + hash: 'QmcZojhwragQr5qhTeFAmELik623Z21e3jBTpJXoQ9si1T' + }, + { + name: 'lv1', + type: 1, + size: 0, + hash: 'QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn' + } + ]) + done() + }) + ) + }) + }) + }) +} diff --git a/js/src/files-mfs/ls-readable-stream.js b/js/src/files-mfs/ls-readable-stream.js new file mode 100644 index 00000000..4a70d641 --- /dev/null +++ b/js/src/files-mfs/ls-readable-stream.js @@ -0,0 +1,107 @@ +/* eslint-env mocha */ +'use strict' + +const series = require('async/series') +const hat = require('hat') +const { getDescribe, getIt, expect } = require('../utils/mocha') + +module.exports = (createCommon, options) => { + const describe = getDescribe(options) + const it = getIt(options) + const common = createCommon() + + describe('.files.lsReadableStream', function () { + this.timeout(40 * 1000) + + let ipfs + + before(function (done) { + // CI takes longer to instantiate the daemon, so we need to increase the + // timeout for the before step + this.timeout(60 * 1000) + + common.setup((err, factory) => { + expect(err).to.not.exist() + factory.spawnNode((err, node) => { + expect(err).to.not.exist() + ipfs = node + done() + }) + }) + }) + + after((done) => common.teardown(done)) + + it('should not ls not found file/dir, expect error', (done) => { + const testDir = `/test-${hat()}` + + const stream = ipfs.files.lsReadableStream(`${testDir}/404`) + + stream.once('error', (err) => { + expect(err).to.exist() + expect(err.message).to.include('does not exist') + done() + }) + }) + + it('should ls directory', (done) => { + const testDir = `/test-${hat()}` + + series([ + (cb) => ipfs.files.mkdir(`${testDir}/lv1`, { p: true }, cb), + (cb) => ipfs.files.write(`${testDir}/b`, Buffer.from('Hello, world!'), { create: true }, cb) + ], (err) => { + expect(err).to.not.exist() + + const stream = ipfs.files.lsReadableStream(testDir) + + let entries = [] + + stream.on('data', entry => entries.push(entry)) + + stream.once('end', () => { + expect(entries.sort((a, b) => a.name.localeCompare(b.name))).to.eql([ + { name: 'b', type: 0, size: 0, hash: '' }, + { name: 'lv1', type: 0, size: 0, hash: '' } + ]) + done() + }) + }) + }) + + it('should ls -l directory', (done) => { + const testDir = `/test-${hat()}` + + series([ + (cb) => ipfs.files.mkdir(`${testDir}/lv1`, { p: true }, cb), + (cb) => ipfs.files.write(`${testDir}/b`, Buffer.from('Hello, world!'), { create: true }, cb) + ], (err) => { + expect(err).to.not.exist() + + const stream = ipfs.files.lsReadableStream(testDir, { l: true }) + + let entries = [] + + stream.on('data', entry => entries.push(entry)) + + stream.once('end', () => { + expect(entries.sort((a, b) => a.name.localeCompare(b.name))).to.eql([ + { + name: 'b', + type: 0, + size: 13, + hash: 'QmcZojhwragQr5qhTeFAmELik623Z21e3jBTpJXoQ9si1T' + }, + { + name: 'lv1', + type: 1, + size: 0, + hash: 'QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn' + } + ]) + done() + }) + }) + }) + }) +} diff --git a/js/src/files-mfs/ls.js b/js/src/files-mfs/ls.js index ff7fd4d0..57a85da5 100644 --- a/js/src/files-mfs/ls.js +++ b/js/src/files-mfs/ls.js @@ -53,9 +53,9 @@ module.exports = (createCommon, options) => { ipfs.files.ls(testDir, (err, info) => { expect(err).to.not.exist() - expect(info).to.eql([ - { name: 'lv1', type: 0, size: 0, hash: '' }, - { name: 'b', type: 0, size: 0, hash: '' } + expect(info.sort((a, b) => a.name.localeCompare(b.name))).to.eql([ + { name: 'b', type: 0, size: 0, hash: '' }, + { name: 'lv1', type: 0, size: 0, hash: '' } ]) done() }) @@ -73,18 +73,18 @@ module.exports = (createCommon, options) => { ipfs.files.ls(testDir, { l: true }, (err, info) => { expect(err).to.not.exist() - expect(info).to.eql([ - { - name: 'lv1', - type: 1, - size: 0, - hash: 'QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn' - }, + expect(info.sort((a, b) => a.name.localeCompare(b.name))).to.eql([ { name: 'b', type: 0, size: 13, hash: 'QmcZojhwragQr5qhTeFAmELik623Z21e3jBTpJXoQ9si1T' + }, + { + name: 'lv1', + type: 1, + size: 0, + hash: 'QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn' } ]) done()