From 582dd017c9017c945f81e8a65e05e01c300f940f Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Mon, 29 Jul 2019 20:43:09 +0100 Subject: [PATCH 01/18] feat: add, addPullStream, addFromFs and addFromStream License: MIT Signed-off-by: Alan Shaw --- package.json | 5 +- src/add-from-url.js | 25 +++++ src/add/form-data.browser.js | 30 ++++++ src/add/form-data.js | 42 ++++++++ src/add/index.js | 56 +++++++++++ src/add/normalise-input.js | 130 +++++++++++++++++++++++++ src/files-regular/add-from-url.js | 70 ------------- src/files-regular/add-pull-stream.js | 13 --- src/files-regular/add.js | 55 ----------- src/lib/blob-to-async-iterable.js | 30 ++++++ src/lib/file-data-to-async-iterable.js | 54 ++++++++++ src/lib/iterable-to-readable-stream.js | 101 +++++++++++++++++++ src/lib/iterable.js | 14 +++ src/lib/object-to-camel.js | 21 ++++ src/utils/load-commands.js | 43 +++++++- test/interface.spec.js | 12 +++ 16 files changed, 558 insertions(+), 143 deletions(-) create mode 100644 src/add-from-url.js create mode 100644 src/add/form-data.browser.js create mode 100644 src/add/form-data.js create mode 100644 src/add/index.js create mode 100644 src/add/normalise-input.js delete mode 100644 src/files-regular/add-from-url.js delete mode 100644 src/files-regular/add-pull-stream.js delete mode 100644 src/files-regular/add.js create mode 100644 src/lib/blob-to-async-iterable.js create mode 100644 src/lib/file-data-to-async-iterable.js create mode 100644 src/lib/iterable-to-readable-stream.js create mode 100644 src/lib/iterable.js create mode 100644 src/lib/object-to-camel.js diff --git a/package.json b/package.json index cc37fe15e..b13d6a069 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,8 @@ "glob": false, "fs": false, "stream": "readable-stream", - "ky-universal": "ky/umd" + "ky-universal": "ky/umd", + "./src/add/form-data.js": "./src/add/form-data.browser.js" }, "repository": "github:ipfs/js-ipfs-http-client", "scripts": { @@ -40,6 +41,7 @@ "dependencies": { "abort-controller": "^3.0.0", "async": "^2.6.1", + "async-iterator-to-pull-stream": "^1.3.0", "bignumber.js": "^9.0.0", "bl": "^3.0.0", "bs58": "^4.0.1", @@ -82,6 +84,7 @@ "promisify-es6": "^1.0.3", "pull-defer": "~0.2.3", "pull-stream": "^3.6.9", + "pull-stream-to-async-iterator": "^1.0.2", "pull-to-stream": "~0.1.1", "pump": "^3.0.0", "qs": "^6.5.2", diff --git a/src/add-from-url.js b/src/add-from-url.js new file mode 100644 index 000000000..b3e21af57 --- /dev/null +++ b/src/add-from-url.js @@ -0,0 +1,25 @@ +'use strict' + +const configure = require('./lib/configure') +const { ok, toIterable } = require('./lib/fetch') + +module.exports = configure(({ fetch, apiAddr, apiPath, headers }) => { + const add = require('./add')({ fetch, apiAddr, apiPath, headers }) + + return (url, options) => (async function * () { + options = options || {} + const res = await ok(fetch(url, { + signal: options.signal, + headers: options.headers || headers + })) + + const input = { + path: decodeURIComponent(new URL(url).pathname.split('/').pop() || ''), + content: toIterable(res.body) + } + + for await (const file of add(input, options)) { + yield file + } + })() +}) diff --git a/src/add/form-data.browser.js b/src/add/form-data.browser.js new file mode 100644 index 000000000..46f0254d5 --- /dev/null +++ b/src/add/form-data.browser.js @@ -0,0 +1,30 @@ +'use strict' +/* eslint-env browser */ + +const normaliseInput = require('./normalise-input') + +exports.toFormData = async (input) => { + const files = normaliseInput(input) + const formData = new FormData() + let i = 0 + + for await (const file of files) { + if (file.content) { + // In the browser there's _currently_ no streaming upload, buffer up our + // async iterator chunks and append a big Blob :( + // One day, this will be browser streams + const bufs = [] + for await (const chunk of file.content) { + bufs.push(Buffer.isBuffer(chunk) ? chunk.buffer : chunk) + } + + formData.append(`file-${i}`, new Blob(bufs, { type: 'application/octet-stream' }), file.path) + } else { + formData.append(`dir-${i}`, new Blob([], { type: 'application/x-directory' }), file.path) + } + + i++ + } + + return formData +} diff --git a/src/add/form-data.js b/src/add/form-data.js new file mode 100644 index 000000000..1ffa66f1d --- /dev/null +++ b/src/add/form-data.js @@ -0,0 +1,42 @@ +'use strict' + +const FormData = require('form-data') +const { Buffer } = require('buffer') +const normaliseInput = require('./normalise-input') +const toStream = require('../lib/iterable-to-readable-stream') + +exports.toFormData = async (input) => { + const files = normaliseInput(input) + const formData = new FormData() + let i = 0 + + for await (const file of files) { + if (file.content) { + // In Node.js, FormData can be passed a stream so no need to buffer + formData.append( + `file-${i}`, + // FIXME: add a `path` property to the stream so `form-data` doesn't set + // a Content-Length header that is only the sum of the size of the + // header/footer when knownLength option (below) is null. + Object.assign( + toStream(file.content), + { path: file.path || `file-${i}` } + ), + { + filepath: file.path, + contentType: 'application/octet-stream', + knownLength: file.content.length // Send Content-Length header if known + } + ) + } else { + formData.append(`dir-${i}`, Buffer.alloc(0), { + filepath: file.path, + contentType: 'application/x-directory' + }) + } + + i++ + } + + return formData +} diff --git a/src/add/index.js b/src/add/index.js new file mode 100644 index 000000000..1abd1a098 --- /dev/null +++ b/src/add/index.js @@ -0,0 +1,56 @@ +'use strict' + +const ndjson = require('iterable-ndjson') +const { objectToQuery } = require('../lib/querystring') +const configure = require('../lib/configure') +const { ok, toIterable } = require('../lib/fetch') +const { toFormData } = require('./form-data') +const toCamel = require('../lib/object-to-camel') + +module.exports = configure(({ fetch, apiAddr, apiPath, headers }) => { + return (input, options) => (async function * () { + options = options || {} + + const qs = objectToQuery({ + 'stream-channels': true, + chunker: options.chunker, + 'cid-version': options.cidVersion, + 'cid-base': options.cidBase, + 'enable-sharding-experiment': options.enableShardingExperiment, + hash: options.hashAlg, + 'only-hash': options.onlyHash, + pin: options.pin, + progress: options.progress ? true : null, + quiet: options.quiet, + quieter: options.quieter, + 'raw-leaves': options.rawLeaves, + 'shard-split-threshold': options.shardSplitThreshold, + silent: options.silent, + trickle: options.trickle, + 'wrap-with-directory': options.wrapWithDirectory, + ...(options.qs || {}) + }) + + const url = `${apiAddr}${apiPath}/add${qs}` + const res = await ok(fetch(url, { + method: 'POST', + signal: options.signal, + headers: options.headers || headers, + body: await toFormData(input) + })) + + for await (let file of ndjson(toIterable(res.body))) { + file = toCamel(file) + // console.log(file) + if (options.progress && file.bytes) { + options.progress(file.bytes) + } else { + yield toCoreInterface(file) + } + } + })() +}) + +function toCoreInterface ({ name, hash, size }) { + return { path: name, hash, size: parseInt(size) } +} diff --git a/src/add/normalise-input.js b/src/add/normalise-input.js new file mode 100644 index 000000000..561324838 --- /dev/null +++ b/src/add/normalise-input.js @@ -0,0 +1,130 @@ +'use strict' +/* eslint-env browser */ + +const { Buffer } = require('buffer') +const errCode = require('err-code') +const toAsyncIterable = require('../lib/file-data-to-async-iterable') + +/* +Transform one of: + +Buffer|ArrayBuffer|TypedArray +Blob|File +{ path, content: Buffer } +{ path, content: Blob } +{ path, content: Iterable } +{ path, content: AsyncIterable } +{ path, content: PullStream } +Iterable +Iterable<{ path, content: Buffer }> +Iterable<{ path, content: Blob }> +Iterable<{ path, content: Iterable }> +Iterable<{ path, content: AsyncIterable }> +Iterable<{ path, content: PullStream }> +AsyncIterable +AsyncIterable<{ path, content: Buffer }> +AsyncIterable<{ path, content: Blob }> +AsyncIterable<{ path, content: Iterable }> +AsyncIterable<{ path, content: AsyncIterable }> +AsyncIterable<{ path, content: PullStream }> +PullStream + +Into: + +AsyncIterable<{ path, content: AsyncIterable }> +*/ + +module.exports = function normalizeInput (input) { + // Buffer|ArrayBuffer|TypedArray + if (Buffer.isBuffer(input) || ArrayBuffer.isView(input) || input instanceof ArrayBuffer) { + return (async function * () { // eslint-disable-line require-await + yield normalizeTuple({ path: '', content: input }) + })() + } + + // Blob|File + if (typeof Blob !== 'undefined' && input instanceof Blob) { + return (async function * () { // eslint-disable-line require-await + yield normalizeTuple({ path: '', content: input }) + })() + } + + // Iterable + // Iterable<{ path, content: Buffer }> + // Iterable<{ path, content: Blob }> + // Iterable<{ path, content: Iterable }> + // Iterable<{ path, content: AsyncIterable }> + // Iterable<{ path, content: PullStream }> + if (input[Symbol.iterator]) { + return (async function * () { // eslint-disable-line require-await + for (const chunk of input) { + if (typeof chunk === 'object' && (chunk.path || chunk.content)) { + yield normalizeTuple(chunk) + } else if (Number.isInteger(chunk)) { // Must be an Iterable i.e. Buffer/ArrayBuffer/Array of bytes + yield normalizeTuple({ path: '', content: input }) + return + } else { + throw errCode(new Error('Unexpected input: ' + typeof chunk), 'ERR_UNEXPECTED_INPUT') + } + } + })() + } + + // AsyncIterable + // AsyncIterable<{ path, content: Buffer }> + // AsyncIterable<{ path, content: Blob }> + // AsyncIterable<{ path, content: Iterable }> + // AsyncIterable<{ path, content: AsyncIterable }> + // AsyncIterable<{ path, content: PullStream }> + if (input[Symbol.asyncIterator]) { + return (async function * () { + for await (const chunk of input) { + if (typeof chunk === 'object' && (chunk.path || chunk.content)) { + yield normalizeTuple(chunk) + } else { // Must be an AsyncIterable i.e. a Stream + let path = '' + + // fs.createReadStream will create a stream with a `path` prop + // If available, use it here! + if (input.path && input.path.split) { + path = input.path.split(/[/\\]/).pop() || '' + } + + yield normalizeTuple({ + path, + content: (async function * () { + yield chunk + for await (const restChunk of input) { + yield restChunk + } + })() + }) + return + } + } + })() + } + + // { path, content: Buffer } + // { path, content: Blob } + // { path, content: Iterable } + // { path, content: AsyncIterable } + // { path, content: PullStream } + if (typeof input === 'object' && (input.path || input.content)) { + // eslint-disable-next-line require-await + return (async function * () { yield normalizeTuple(input) })() + } + + // PullStream + if (typeof input === 'function') { + return (async function * () { // eslint-disable-line require-await + yield normalizeTuple({ path: '', content: input }) + })() + } + + throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') +} + +function normalizeTuple ({ path, content }) { + return { path: path || '', content: content ? toAsyncIterable(content) : null } +} diff --git a/src/files-regular/add-from-url.js b/src/files-regular/add-from-url.js deleted file mode 100644 index f1065f2de..000000000 --- a/src/files-regular/add-from-url.js +++ /dev/null @@ -1,70 +0,0 @@ -'use strict' - -const promisify = require('promisify-es6') -const { URL } = require('iso-url') -const { getRequest } = require('iso-stream-http') -const SendOneFile = require('../utils/send-one-file-multiple-results') -const FileResultStreamConverter = require('../utils/file-result-stream-converter') - -module.exports = (send) => { - const sendOneFile = SendOneFile(send, 'add') - - return promisify((url, opts, callback) => { - if (typeof (opts) === 'function' && - callback === undefined) { - callback = opts - opts = {} - } - - // opts is the real callback -- - // 'callback' is being injected by promisify - if (typeof opts === 'function' && - typeof callback === 'function') { - callback = opts - opts = {} - } - - if (!validUrl(url)) { - return callback(new Error('"url" param must be an http(s) url')) - } - - requestWithRedirect(url, opts, sendOneFile, callback) - }) -} - -const validUrl = (url) => typeof url === 'string' && url.startsWith('http') - -const requestWithRedirect = (url, opts, sendOneFile, callback) => { - const parsedUrl = new URL(url) - - const req = getRequest(parsedUrl, (res) => { - if (res.statusCode >= 400) { - return callback(new Error(`Failed to download with ${res.statusCode}`)) - } - - const redirection = res.headers.location - - if (res.statusCode >= 300 && res.statusCode < 400 && redirection) { - if (!validUrl(redirection)) { - return callback(new Error('redirection url must be an http(s) url')) - } - - requestWithRedirect(redirection, opts, sendOneFile, callback) - } else { - const requestOpts = { - qs: opts, - converter: FileResultStreamConverter - } - const fileName = decodeURIComponent(parsedUrl.pathname.split('/').pop()) - - sendOneFile({ - content: res, - path: fileName - }, requestOpts, callback) - } - }) - - req.once('error', callback) - - req.end() -} diff --git a/src/files-regular/add-pull-stream.js b/src/files-regular/add-pull-stream.js deleted file mode 100644 index 2076ffa8d..000000000 --- a/src/files-regular/add-pull-stream.js +++ /dev/null @@ -1,13 +0,0 @@ -'use strict' - -const SendFilesStream = require('../utils/send-files-stream') -const FileResultStreamConverter = require('../utils/file-result-stream-converter') -const toPull = require('stream-to-pull-stream') - -module.exports = (send) => { - return (options) => { - options = options || {} - options.converter = FileResultStreamConverter - return toPull(SendFilesStream(send, 'add')({ qs: options })) - } -} diff --git a/src/files-regular/add.js b/src/files-regular/add.js deleted file mode 100644 index cb5898265..000000000 --- a/src/files-regular/add.js +++ /dev/null @@ -1,55 +0,0 @@ -'use strict' - -const promisify = require('promisify-es6') -const ConcatStream = require('concat-stream') -const once = require('once') -const { isSource } = require('is-pull-stream') -const FileResultStreamConverter = require('../utils/file-result-stream-converter') -const SendFilesStream = require('../utils/send-files-stream') -const validateAddInput = require('ipfs-utils/src/files/add-input-validation') - -module.exports = (send) => { - const createAddStream = SendFilesStream(send, 'add') - - const add = promisify((_files, options, _callback) => { - if (typeof options === 'function') { - _callback = options - options = null - } - const callback = once(_callback) - - if (!options) { - options = {} - } - options.converter = FileResultStreamConverter - - try { - validateAddInput(_files) - } catch (err) { - return callback(err) - } - - const files = [].concat(_files) - - const stream = createAddStream({ qs: options }) - const concat = ConcatStream((result) => callback(null, result)) - stream.once('error', callback) - stream.pipe(concat) - - files.forEach((file) => stream.write(file)) - stream.end() - }) - - return function () { - const args = Array.from(arguments) - - // If we files.add(), then promisify thinks the pull stream is - // a callback! Add an empty options object in this case so that a promise - // is returned. - if (args.length === 1 && isSource(args[0])) { - args.push({}) - } - - return add.apply(null, args) - } -} diff --git a/src/lib/blob-to-async-iterable.js b/src/lib/blob-to-async-iterable.js new file mode 100644 index 000000000..3f9e982c0 --- /dev/null +++ b/src/lib/blob-to-async-iterable.js @@ -0,0 +1,30 @@ +'use strict' +/* eslint-env browser */ + +// Convert a Blob into an AsyncIterable +module.exports = (blob, options) => (async function * () { + options = options || {} + + const reader = new FileReader() + const chunkSize = options.chunkSize || 1024 * 1024 + let offset = options.offset || 0 + + const getNextChunk = () => new Promise((resolve, reject) => { + reader.onloadend = e => { + const data = e.target.result + resolve(data.byteLength === 0 ? null : data) + } + reader.onerror = reject + + const end = offset + chunkSize + const slice = blob.slice(offset, end) + reader.readAsArrayBuffer(slice) + offset = end + }) + + while (true) { + const data = await getNextChunk() + if (data == null) return + yield data + } +})() diff --git a/src/lib/file-data-to-async-iterable.js b/src/lib/file-data-to-async-iterable.js new file mode 100644 index 000000000..ffa0f8d11 --- /dev/null +++ b/src/lib/file-data-to-async-iterable.js @@ -0,0 +1,54 @@ +'use strict' +/* eslint-env browser */ + +const toIterator = require('pull-stream-to-async-iterator') +const { Buffer } = require('buffer') +const blobToAsyncIterable = require('../lib/blob-to-async-iterable') + +/* +Transform one of: + +Buffer|ArrayBuffer|TypedArray +Blob|File +Iterable +AsyncIterable +PullStream + +Into: + +AsyncIterable +*/ +module.exports = function toAsyncIterable (input) { + // Buffer|ArrayBuffer|TypedArray|array of bytes + if (input[Symbol.iterator]) { + const buf = Buffer.from(input) + return Object.assign( + (async function * () { yield buf })(), // eslint-disable-line require-await + { length: buf.length } + ) + } + + // Blob|File + if (typeof Blob !== 'undefined' && input instanceof Blob) { + return Object.assign( + blobToAsyncIterable(input), + { length: input.size } + ) + } + + // AsyncIterable + if (input[Symbol.asyncIterator]) { + return (async function * () { + for await (const chunk of input) { + yield Buffer.from(chunk) + } + })() + } + + // PullStream + if (typeof input === 'function') { + return toIterator(input) + } + + throw new Error('Unexpected input: ' + typeof input) +} diff --git a/src/lib/iterable-to-readable-stream.js b/src/lib/iterable-to-readable-stream.js new file mode 100644 index 000000000..06b3783dd --- /dev/null +++ b/src/lib/iterable-to-readable-stream.js @@ -0,0 +1,101 @@ +'use strict' + +const { Readable, Writable } = require('stream') + +function toReadable (source) { + let reading = false + return new Readable({ + async read (size) { + if (reading) return + reading = true + + try { + while (true) { + const { value, done } = await source.next(size) + if (done) return this.push(null) + if (!this.push(value)) break + } + } catch (err) { + this.emit('error', err) + if (source.return) source.return() + } finally { + reading = false + } + } + }) +} + +module.exports = toReadable +module.exports.readable = toReadable + +function toWritable (sink) { + const END_CHUNK = {} + + class Buf { + constructor () { + this._buffer = [] + this._waitingConsumers = [] + this._consuming = false + } + + push (chunk) { + let resolver + const pushPromise = new Promise((resolve, reject) => { + resolver = { resolve, reject } + }) + this._buffer.push({ chunk, resolver }) + this._consume() + return pushPromise + } + + _consume () { + if (this._consuming) return + this._consuming = true + + while (this._waitingConsumers.length && this._buffer.length) { + const nextConsumer = this._waitingConsumers.shift() + const nextChunk = this._buffer.shift() + nextConsumer.resolver.resolve(nextChunk) + nextChunk.resolver.resolve() + } + + this._consuming = false + } + + consume () { + let resolver + const consumePromise = new Promise((resolve, reject) => { + resolver = { resolve, reject } + }) + this._waitingConsumers.push({ resolver }) + this._consume() + return consumePromise + } + } + + const buf = new Buf() + + const it = { + async next () { + const chunk = await buf.consume() + return chunk === END_CHUNK ? { done: true } : { value: chunk } + } + } + + sink({ + [Symbol.asyncIterator] () { + return it + } + }) + + return new Writable({ + write (chunk, enc, cb) { + buf.push(chunk).then(() => cb(), cb) + }, + final (cb) { + buf.push(END_CHUNK).then(() => cb(), cb) + } + }) +} + +module.exports.toWritable = toWritable diff --git a/src/lib/iterable.js b/src/lib/iterable.js new file mode 100644 index 000000000..4c59ff510 --- /dev/null +++ b/src/lib/iterable.js @@ -0,0 +1,14 @@ +'use strict' + +const toPull = require('async-iterator-to-pull-stream') + +exports.collectify = fn => async (...args) => { + const items = [] + for await (const item of fn(...args)) items.push(item) + return items +} + +exports.pullify = { + source: fn => (...args) => toPull(fn(...args)), + transform: fn => (...args) => toPull.transform(source => fn(source, ...args)) +} diff --git a/src/lib/object-to-camel.js b/src/lib/object-to-camel.js new file mode 100644 index 000000000..f13b2b6a1 --- /dev/null +++ b/src/lib/object-to-camel.js @@ -0,0 +1,21 @@ +'use strict' + +// Convert object properties to camel case. +// NOT recursive! +// e.g. +// AgentVersion => agentVersion +// ID => id +module.exports = obj => { + if (obj == null) return obj + const caps = /^[A-Z]+$/ + return Object.keys(obj).reduce((camelObj, k) => { + if (caps.test(k)) { // all caps + camelObj[k.toLowerCase()] = obj[k] + } else if (caps.test(k[0])) { // pascal + camelObj[k[0].toLowerCase() + k.slice(1)] = obj[k] + } else { + camelObj[k] = obj[k] + } + return camelObj + }, {}) +} diff --git a/src/utils/load-commands.js b/src/utils/load-commands.js index 467af3cea..837f6ce02 100644 --- a/src/utils/load-commands.js +++ b/src/utils/load-commands.js @@ -1,14 +1,49 @@ 'use strict' +const nodeify = require('promise-nodeify') +const { collectify, pullify } = require('../lib/iterable') + function requireCommands () { return { // Files Regular (not MFS) - add: require('../files-regular/add'), + add: (_, config) => { + const add = collectify(require('../add')(config)) + return (input, options, callback) => { + if (typeof options === 'function') { + callback = options + options = {} + } + return nodeify(add(input, options), callback) + } + }, + // TODO: convert addReadableStream: require('../files-regular/add-readable-stream'), - addPullStream: require('../files-regular/add-pull-stream'), + addPullStream: (_, config) => { + const add = require('../add')(config) + return pullify.transform(add) + }, + // TODO: convert addFromFs: require('../files-regular/add-from-fs'), - addFromURL: require('../files-regular/add-from-url'), - addFromStream: require('../files-regular/add'), + addFromURL: (_, config) => { + const addFromURL = collectify(require('../add-from-url')(config)) + return (url, options, callback) => { + if (typeof options === 'function') { + callback = options + options = {} + } + return nodeify(addFromURL(url, options), callback) + } + }, + addFromStream: (_, config) => { + const add = collectify(require('../add')(config)) + return (input, options, callback) => { + if (typeof options === 'function') { + callback = options + options = {} + } + return nodeify(add(input, options), callback) + } + }, _addAsyncIterator: require('../files-regular/add-async-iterator'), cat: require('../files-regular/cat'), catReadableStream: require('../files-regular/cat-readable-stream'), diff --git a/test/interface.spec.js b/test/interface.spec.js index fb54c422b..6ea21e485 100644 --- a/test/interface.spec.js +++ b/test/interface.spec.js @@ -116,6 +116,14 @@ describe('interface-ipfs-core tests', () => { name: 'should add a nested directory as array of tupples with progress', reason: 'FIXME https://github.com/ipfs/js-ipfs-http-client/issues/339' }, + { + name: 'should not be able to add a string', + reason: 'FIXME test needs to change to inspect error code ERR_UNEXPECTED_INPUT' + }, + { + name: 'should not be able to add a non-Buffer TypedArray', + reason: 'TODO remove test, this should be supported' + }, // .addPullStream isNode ? null : { name: 'should add pull stream of valid files and dirs', @@ -131,6 +139,10 @@ describe('interface-ipfs-core tests', () => { name: 'addFromStream', reason: 'Not designed to run in the browser' }, + { + name: 'should add from a stream', + reason: 'TODO change test to use readable-stream@3 with async iterator support' + }, // .addFromFs isNode ? null : { name: 'addFromFs', From fd5079f0883b101a01260ae0eab19de6e87b9667 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 30 Jul 2019 20:26:00 +0100 Subject: [PATCH 02/18] refactor: use ky License: MIT Signed-off-by: Alan Shaw --- src/add-from-url.js | 15 ++++++------- src/add/index.js | 52 ++++++++++++++++++++++----------------------- 2 files changed, 32 insertions(+), 35 deletions(-) diff --git a/src/add-from-url.js b/src/add-from-url.js index b3e21af57..e98169eed 100644 --- a/src/add-from-url.js +++ b/src/add-from-url.js @@ -1,21 +1,20 @@ 'use strict' +const kyDefault = require('ky-universal').default const configure = require('./lib/configure') -const { ok, toIterable } = require('./lib/fetch') +const toIterable = require('./lib/stream-to-iterable') -module.exports = configure(({ fetch, apiAddr, apiPath, headers }) => { - const add = require('./add')({ fetch, apiAddr, apiPath, headers }) +module.exports = configure(({ ky }) => { + const add = require('./add')({ ky }) return (url, options) => (async function * () { options = options || {} - const res = await ok(fetch(url, { - signal: options.signal, - headers: options.headers || headers - })) + + const { body } = await kyDefault.get(url) const input = { path: decodeURIComponent(new URL(url).pathname.split('/').pop() || ''), - content: toIterable(res.body) + content: toIterable(body) } for await (const file of add(input, options)) { diff --git a/src/add/index.js b/src/add/index.js index 1abd1a098..34a5d33bd 100644 --- a/src/add/index.js +++ b/src/add/index.js @@ -1,43 +1,41 @@ 'use strict' const ndjson = require('iterable-ndjson') -const { objectToQuery } = require('../lib/querystring') const configure = require('../lib/configure') -const { ok, toIterable } = require('../lib/fetch') +const toIterable = require('../lib/stream-to-iterable') const { toFormData } = require('./form-data') const toCamel = require('../lib/object-to-camel') -module.exports = configure(({ fetch, apiAddr, apiPath, headers }) => { +module.exports = configure(({ ky }) => { return (input, options) => (async function * () { options = options || {} - const qs = objectToQuery({ - 'stream-channels': true, - chunker: options.chunker, - 'cid-version': options.cidVersion, - 'cid-base': options.cidBase, - 'enable-sharding-experiment': options.enableShardingExperiment, - hash: options.hashAlg, - 'only-hash': options.onlyHash, - pin: options.pin, - progress: options.progress ? true : null, - quiet: options.quiet, - quieter: options.quieter, - 'raw-leaves': options.rawLeaves, - 'shard-split-threshold': options.shardSplitThreshold, - silent: options.silent, - trickle: options.trickle, - 'wrap-with-directory': options.wrapWithDirectory, - ...(options.qs || {}) - }) + const searchParams = new URLSearchParams(options.searchParams) + + searchParams.set('stream-channels', true) + if (options.chunker) searchParams.set('chunker', options.chunker) + if (options.cidVersion) searchParams.set('cid-version', options.cidVersion) + if (options.cidBase) searchParams.set('cid-base', options.cidBase) + if (options.enableShardingExperiment != null) searchParams.set('enable-sharding-experiment', options.enableShardingExperiment) + if (options.hashAlg) searchParams.set('hash', options.hashAlg) + if (options.onlyHash != null) searchParams.set('only-hash', options.onlyHash) + if (options.pin != null) searchParams.set('pin', options.pin) + if (options.progress) searchParams.set('progress', true) + if (options.quiet != null) searchParams.set('quiet', options.quiet) + if (options.quieter != null) searchParams.set('quieter', options.quieter) + if (options.rawLeaves != null) searchParams.set('raw-leaves', options.rawLeaves) + if (options.shardSplitThreshold) searchParams.set('shard-split-threshold', options.shardSplitThreshold) + if (options.silent) searchParams.set('silent', options.silent) + if (options.trickle != null) searchParams.set('trickle', options.trickle) + if (options.wrapWithDirectory != null) searchParams.set('wrap-with-directory', options.wrapWithDirectory) - const url = `${apiAddr}${apiPath}/add${qs}` - const res = await ok(fetch(url, { - method: 'POST', + const res = await ky.post('add', { + timeout: options.timeout, signal: options.signal, - headers: options.headers || headers, + headers: options.headers, + searchParams, body: await toFormData(input) - })) + }) for await (let file of ndjson(toIterable(res.body))) { file = toCamel(file) From 261b21bfcf6c3866336c92eed6bec211835fa9fd Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Sat, 3 Aug 2019 13:45:25 +0100 Subject: [PATCH 03/18] refactor: convert addReadableStream License: MIT Signed-off-by: Alan Shaw --- package.json | 1 + src/lib/iterable.js | 5 +++++ src/utils/load-commands.js | 8 +++++--- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index b13d6a069..87c627124 100644 --- a/package.json +++ b/package.json @@ -65,6 +65,7 @@ "is-stream": "^2.0.0", "iso-stream-http": "~0.1.2", "iso-url": "~0.4.6", + "it-to-stream": "^0.1.1", "iterable-ndjson": "^1.1.0", "just-kebab-case": "^1.1.0", "just-map-keys": "^1.1.0", diff --git a/src/lib/iterable.js b/src/lib/iterable.js index 4c59ff510..78de19f5e 100644 --- a/src/lib/iterable.js +++ b/src/lib/iterable.js @@ -1,6 +1,7 @@ 'use strict' const toPull = require('async-iterator-to-pull-stream') +const toStream = require('it-to-stream') exports.collectify = fn => async (...args) => { const items = [] @@ -12,3 +13,7 @@ exports.pullify = { source: fn => (...args) => toPull(fn(...args)), transform: fn => (...args) => toPull.transform(source => fn(source, ...args)) } + +exports.streamify = { + transform: fn => (...args) => toStream.transform(source => fn(source, ...args), { objectMode: true }) +} diff --git a/src/utils/load-commands.js b/src/utils/load-commands.js index 837f6ce02..d8df38b9b 100644 --- a/src/utils/load-commands.js +++ b/src/utils/load-commands.js @@ -1,7 +1,7 @@ 'use strict' const nodeify = require('promise-nodeify') -const { collectify, pullify } = require('../lib/iterable') +const { collectify, pullify, streamify } = require('../lib/iterable') function requireCommands () { return { @@ -16,8 +16,10 @@ function requireCommands () { return nodeify(add(input, options), callback) } }, - // TODO: convert - addReadableStream: require('../files-regular/add-readable-stream'), + addReadableStream: (_, config) => { + const add = require('../add')(config) + return streamify.transform(add) + }, addPullStream: (_, config) => { const add = require('../add')(config) return pullify.transform(add) From fcf55743846ec2d94766111be0be924126778690 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Sat, 3 Aug 2019 14:25:46 +0100 Subject: [PATCH 04/18] refactor: convert addFromFs License: MIT Signed-off-by: Alan Shaw --- package.json | 4 +- src/add-from-fs/glob-source.js | 94 ++++++++++++++++++++++++ src/add-from-fs/index.browser.js | 3 + src/add-from-fs/index.js | 9 +++ src/files-regular/add-from-fs.js | 40 ---------- src/files-regular/add-readable-stream.js | 12 --- src/utils/load-commands.js | 12 ++- 7 files changed, 119 insertions(+), 55 deletions(-) create mode 100644 src/add-from-fs/glob-source.js create mode 100644 src/add-from-fs/index.browser.js create mode 100644 src/add-from-fs/index.js delete mode 100644 src/files-regular/add-from-fs.js delete mode 100644 src/files-regular/add-readable-stream.js diff --git a/package.json b/package.json index 87c627124..bb35dde09 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,8 @@ "fs": false, "stream": "readable-stream", "ky-universal": "ky/umd", - "./src/add/form-data.js": "./src/add/form-data.browser.js" + "./src/add/form-data.js": "./src/add/form-data.browser.js", + "./src/add-from-fs/index.js": "./src/add-from-fs/index.browser.js" }, "repository": "github:ipfs/js-ipfs-http-client", "scripts": { @@ -65,6 +66,7 @@ "is-stream": "^2.0.0", "iso-stream-http": "~0.1.2", "iso-url": "~0.4.6", + "it-pushable": "^1.2.1", "it-to-stream": "^0.1.1", "iterable-ndjson": "^1.1.0", "just-kebab-case": "^1.1.0", diff --git a/src/add-from-fs/glob-source.js b/src/add-from-fs/glob-source.js new file mode 100644 index 000000000..b48a31cdc --- /dev/null +++ b/src/add-from-fs/glob-source.js @@ -0,0 +1,94 @@ +'use strict' + +const Fs = require('fs') +const Path = require('path') +const glob = require('glob') +const pushable = require('it-pushable') +const errCode = require('err-code') + +/** +* Create an AsyncIterable that can be passed to ipfs.add for the +* provided file paths. +* +* @param {String} ...paths File system path(s) to glob from +* @param {Object} [options] Optional options +* @param {Boolean} [options.recursive] Recursively glob all paths in directories +* @param {Boolean} [options.hidden] Include .dot files in matched paths +* @param {Array} [options.ignore] Glob paths to ignore +* @param {Boolean} [options.followSymlinks] follow symlinks +* @returns {AsyncIterable} +*/ +module.exports = (...args) => (async function * () { + const options = typeof args[args.length - 1] === 'string' ? {} : args.pop() + const paths = args + + const globSourceOptions = { + recursive: options.recursive, + glob: { + dot: Boolean(options.hidden), + ignore: Array.isArray(options.ignore) ? options.ignore : [], + follow: options.followSymlinks != null ? options.followSymlinks : true + } + } + + // Check the input paths comply with options.recursive and convert to glob sources + const results = await Promise.all(paths.map(pathAndType)) + const globSources = results.map(r => toGlobSource(r, globSourceOptions)) + + for (const globSource of globSources) { + for await (const { path, contentPath } of globSource) { + yield { path, content: Fs.createReadStream(contentPath) } + } + } +})() + +function toGlobSource ({ path, type }, options) { + return (async function * () { + options = options || {} + + const baseName = Path.basename(path) + + if (type === 'file') { + yield { path: baseName, contentPath: path } + return + } + + if (type === 'dir' && !options.recursive) { + throw errCode( + new Error(`'${path}' is a directory and recursive option not set`), + 'ERR_DIR_NON_RECURSIVE', + { path } + ) + } + + const globOptions = Object.assign({}, options.glob, { + cwd: path, + nodir: true, + realpath: false, + absolute: false + }) + + // TODO: want to use pull-glob but it doesn't have the features... + const pusher = pushable() + + glob('**/*', globOptions) + .on('match', m => pusher.push(m)) + .on('end', () => pusher.end()) + .on('abort', () => pusher.end()) + .on('error', err => pusher.end(err)) + + for await (const p of pusher) { + yield { + path: `${baseName}/${toPosix(p)}`, + contentPath: Path.join(path, p) + } + } + })() +} + +async function pathAndType (path) { + const stat = await Fs.promises.stat(path) + return { path, type: stat.isDirectory() ? 'dir' : 'file' } +} + +const toPosix = path => path.replace(/\\/g, '/') diff --git a/src/add-from-fs/index.browser.js b/src/add-from-fs/index.browser.js new file mode 100644 index 000000000..0a33f4c2e --- /dev/null +++ b/src/add-from-fs/index.browser.js @@ -0,0 +1,3 @@ +'use strict' + +module.exports = () => { throw new Error('unavailable in the browser') } diff --git a/src/add-from-fs/index.js b/src/add-from-fs/index.js new file mode 100644 index 000000000..225acf9c3 --- /dev/null +++ b/src/add-from-fs/index.js @@ -0,0 +1,9 @@ +'use strict' + +const configure = require('../lib/configure') +const globSource = require('./glob-source') + +module.exports = configure(({ ky }) => { + const add = require('../add')({ ky }) + return (path, options) => add(globSource(path, options), options) +}) diff --git a/src/files-regular/add-from-fs.js b/src/files-regular/add-from-fs.js deleted file mode 100644 index 2320fc537..000000000 --- a/src/files-regular/add-from-fs.js +++ /dev/null @@ -1,40 +0,0 @@ -'use strict' - -const isNode = require('detect-node') -const promisify = require('promisify-es6') -const SendOneFile = require('../utils/send-one-file-multiple-results') -const FileResultStreamConverter = require('../utils/file-result-stream-converter') - -module.exports = (send) => { - const sendOneFile = SendOneFile(send, 'add') - - return promisify((path, opts, callback) => { - if (typeof opts === 'function' && - callback === undefined) { - callback = opts - opts = {} - } - - // opts is the real callback -- - // 'callback' is being injected by promisify - if (typeof opts === 'function' && - typeof callback === 'function') { - callback = opts - opts = {} - } - - if (!isNode) { - return callback(new Error('fsAdd does not work in the browser')) - } - - if (typeof path !== 'string') { - return callback(new Error('"path" must be a string')) - } - - const requestOpts = { - qs: opts, - converter: FileResultStreamConverter - } - sendOneFile(path, requestOpts, callback) - }) -} diff --git a/src/files-regular/add-readable-stream.js b/src/files-regular/add-readable-stream.js deleted file mode 100644 index 320abe692..000000000 --- a/src/files-regular/add-readable-stream.js +++ /dev/null @@ -1,12 +0,0 @@ -'use strict' - -const SendFilesStream = require('../utils/send-files-stream') -const FileResultStreamConverter = require('../utils/file-result-stream-converter') - -module.exports = (send) => { - return (options) => { - options = options || {} - options.converter = FileResultStreamConverter - return SendFilesStream(send, 'add')(options) - } -} diff --git a/src/utils/load-commands.js b/src/utils/load-commands.js index d8df38b9b..1c36702d1 100644 --- a/src/utils/load-commands.js +++ b/src/utils/load-commands.js @@ -24,8 +24,16 @@ function requireCommands () { const add = require('../add')(config) return pullify.transform(add) }, - // TODO: convert - addFromFs: require('../files-regular/add-from-fs'), + addFromFs: (_, config) => { + const addFromFs = collectify(require('../add-from-fs')(config)) + return (path, options, callback) => { + if (typeof options === 'function') { + callback = options + options = {} + } + return nodeify(addFromFs(path, options), callback) + } + }, addFromURL: (_, config) => { const addFromURL = collectify(require('../add-from-url')(config)) return (url, options, callback) => { From 3f83cbe9136cab803e5f75542b17410bff1280a9 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Sat, 3 Aug 2019 14:44:35 +0100 Subject: [PATCH 05/18] refactor: remove unused code License: MIT Signed-off-by: Alan Shaw --- src/add/form-data.js | 4 +- src/files-mfs/write.js | 4 +- src/lib/iterable-to-readable-stream.js | 101 ---------------------- src/utils/file-result-stream-converter.js | 44 ---------- 4 files changed, 3 insertions(+), 150 deletions(-) delete mode 100644 src/lib/iterable-to-readable-stream.js delete mode 100644 src/utils/file-result-stream-converter.js diff --git a/src/add/form-data.js b/src/add/form-data.js index 1ffa66f1d..1d61d9a83 100644 --- a/src/add/form-data.js +++ b/src/add/form-data.js @@ -2,8 +2,8 @@ const FormData = require('form-data') const { Buffer } = require('buffer') +const toStream = require('it-to-stream') const normaliseInput = require('./normalise-input') -const toStream = require('../lib/iterable-to-readable-stream') exports.toFormData = async (input) => { const files = normaliseInput(input) @@ -19,7 +19,7 @@ exports.toFormData = async (input) => { // a Content-Length header that is only the sum of the size of the // header/footer when knownLength option (below) is null. Object.assign( - toStream(file.content), + toStream.readable(file.content), { path: file.path || `file-${i}` } ), { diff --git a/src/files-mfs/write.js b/src/files-mfs/write.js index 0485406bd..33f1ec973 100644 --- a/src/files-mfs/write.js +++ b/src/files-mfs/write.js @@ -3,7 +3,6 @@ const promisify = require('promisify-es6') const concatStream = require('concat-stream') const once = require('once') -const FileResultStreamConverter = require('../utils/file-result-stream-converter') const SendFilesStream = require('../utils/send-files-stream') module.exports = (send) => { @@ -29,8 +28,7 @@ module.exports = (send) => { const options = { args: pathDst, - qs: opts, - converter: FileResultStreamConverter + qs: opts } const stream = sendFilesStream({ qs: options }) diff --git a/src/lib/iterable-to-readable-stream.js b/src/lib/iterable-to-readable-stream.js deleted file mode 100644 index 06b3783dd..000000000 --- a/src/lib/iterable-to-readable-stream.js +++ /dev/null @@ -1,101 +0,0 @@ -'use strict' - -const { Readable, Writable } = require('stream') - -function toReadable (source) { - let reading = false - return new Readable({ - async read (size) { - if (reading) return - reading = true - - try { - while (true) { - const { value, done } = await source.next(size) - if (done) return this.push(null) - if (!this.push(value)) break - } - } catch (err) { - this.emit('error', err) - if (source.return) source.return() - } finally { - reading = false - } - } - }) -} - -module.exports = toReadable -module.exports.readable = toReadable - -function toWritable (sink) { - const END_CHUNK = {} - - class Buf { - constructor () { - this._buffer = [] - this._waitingConsumers = [] - this._consuming = false - } - - push (chunk) { - let resolver - const pushPromise = new Promise((resolve, reject) => { - resolver = { resolve, reject } - }) - this._buffer.push({ chunk, resolver }) - this._consume() - return pushPromise - } - - _consume () { - if (this._consuming) return - this._consuming = true - - while (this._waitingConsumers.length && this._buffer.length) { - const nextConsumer = this._waitingConsumers.shift() - const nextChunk = this._buffer.shift() - nextConsumer.resolver.resolve(nextChunk) - nextChunk.resolver.resolve() - } - - this._consuming = false - } - - consume () { - let resolver - const consumePromise = new Promise((resolve, reject) => { - resolver = { resolve, reject } - }) - this._waitingConsumers.push({ resolver }) - this._consume() - return consumePromise - } - } - - const buf = new Buf() - - const it = { - async next () { - const chunk = await buf.consume() - return chunk === END_CHUNK ? { done: true } : { value: chunk } - } - } - - sink({ - [Symbol.asyncIterator] () { - return it - } - }) - - return new Writable({ - write (chunk, enc, cb) { - buf.push(chunk).then(() => cb(), cb) - }, - final (cb) { - buf.push(END_CHUNK).then(() => cb(), cb) - } - }) -} - -module.exports.toWritable = toWritable diff --git a/src/utils/file-result-stream-converter.js b/src/utils/file-result-stream-converter.js deleted file mode 100644 index 7f5b19aeb..000000000 --- a/src/utils/file-result-stream-converter.js +++ /dev/null @@ -1,44 +0,0 @@ -'use strict' - -const TransformStream = require('readable-stream').Transform - -/* - Transforms a stream of {Name, Hash} objects to include size - of the DAG object. - - Usage: inputStream.pipe(new FileResultStreamConverter()) - - Input object format: - { - Name: '/path/to/file/foo.txt', - Hash: 'Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP' - Size: '20' - } - - Output object format: - { - path: '/path/to/file/foo.txt', - hash: 'Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP', - size: 20 - } -*/ -class FileResultStreamConverter extends TransformStream { - constructor (options) { - const opts = Object.assign({}, options || {}, { objectMode: true }) - super(opts) - } - - _transform (obj, enc, callback) { - if (!obj.Hash) { - return callback() - } - - callback(null, { - path: obj.Name, - hash: obj.Hash, - size: parseInt(obj.Size, 10) - }) - } -} - -module.exports = FileResultStreamConverter From b5e0dc8a2353530f433d84fbf7f81e90bd3e5e71 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Sun, 4 Aug 2019 13:35:14 +0100 Subject: [PATCH 06/18] fix: tests License: MIT Signed-off-by: Alan Shaw --- src/add-from-fs/index.browser.js | 2 +- src/files-regular/index.js | 7 ------- test/files-mfs.spec.js | 6 +++--- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/src/add-from-fs/index.browser.js b/src/add-from-fs/index.browser.js index 0a33f4c2e..81d551294 100644 --- a/src/add-from-fs/index.browser.js +++ b/src/add-from-fs/index.browser.js @@ -1,3 +1,3 @@ 'use strict' -module.exports = () => { throw new Error('unavailable in the browser') } +module.exports = () => () => { throw new Error('unavailable in the browser') } diff --git a/src/files-regular/index.js b/src/files-regular/index.js index e5b49e495..408f76494 100644 --- a/src/files-regular/index.js +++ b/src/files-regular/index.js @@ -6,13 +6,6 @@ module.exports = (arg) => { const send = moduleConfig(arg) return { - add: require('../files-regular/add')(send), - addReadableStream: require('../files-regular/add-readable-stream')(send), - addPullStream: require('../files-regular/add-pull-stream')(send), - addFromFs: require('../files-regular/add-from-fs')(send), - addFromURL: require('../files-regular/add-from-url')(send), - addFromStream: require('../files-regular/add')(send), - _addAsyncIterator: require('../files-regular/add-async-iterator')(send), cat: require('../files-regular/cat')(send), catReadableStream: require('../files-regular/cat-readable-stream')(send), catPullStream: require('../files-regular/cat-pull-stream')(send), diff --git a/test/files-mfs.spec.js b/test/files-mfs.spec.js index 311659200..d1c2e0900 100644 --- a/test/files-mfs.spec.js +++ b/test/files-mfs.spec.js @@ -91,7 +91,7 @@ describe('.files (the MFS API part)', function () { it('.add with cid-version=1 and raw-leaves=false', async () => { const expectedCid = 'bafybeifogzovjqrcxvgt7g36y7g63hvwvoakledwk4b2fr2dl4wzawpnny' - const options = { 'cid-version': 1, 'raw-leaves': false } + const options = { cidVersion: 1, rawLeaves: false } const res = await ipfs.add(testfile, options) @@ -149,7 +149,7 @@ describe('.files (the MFS API part)', function () { path: content + '.txt', content: Buffer.from(content) } - const options = { hash: name, 'raw-leaves': false } + const options = { hashAlg: name, rawLeaves: false } const res = await ipfs.add([file], options) @@ -222,7 +222,7 @@ describe('.files (the MFS API part)', function () { path: content + '.txt', content: Buffer.from(content) } - const options = { hash: name, 'raw-leaves': false } + const options = { hashAlg: name, rawLeaves: false } const res = await ipfs.add([file], options) From 623be3553d01e8980ba891acdfc4706a3985c261 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Sun, 4 Aug 2019 13:44:10 +0100 Subject: [PATCH 07/18] fix: tests License: MIT Signed-off-by: Alan Shaw --- src/add/form-data.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/add/form-data.js b/src/add/form-data.js index 1d61d9a83..be268320f 100644 --- a/src/add/form-data.js +++ b/src/add/form-data.js @@ -23,14 +23,14 @@ exports.toFormData = async (input) => { { path: file.path || `file-${i}` } ), { - filepath: file.path, + filepath: encodeURIComponent(file.path), contentType: 'application/octet-stream', knownLength: file.content.length // Send Content-Length header if known } ) } else { formData.append(`dir-${i}`, Buffer.alloc(0), { - filepath: file.path, + filepath: encodeURIComponent(file.path), contentType: 'application/x-directory' }) } From ac8442d7395e56530b6a87635d95b5c461147d43 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Sun, 4 Aug 2019 14:11:08 +0100 Subject: [PATCH 08/18] fix: normalize input for iterable of buffer or blob License: MIT Signed-off-by: Alan Shaw --- src/add/normalise-input.js | 29 +++++++++++++++----------- src/lib/file-data-to-async-iterable.js | 3 ++- src/lib/is-bloby.js | 7 +++++++ src/lib/is-bytes.js | 8 +++++++ 4 files changed, 34 insertions(+), 13 deletions(-) create mode 100644 src/lib/is-bloby.js create mode 100644 src/lib/is-bytes.js diff --git a/src/add/normalise-input.js b/src/add/normalise-input.js index 561324838..e38ba3a31 100644 --- a/src/add/normalise-input.js +++ b/src/add/normalise-input.js @@ -1,9 +1,9 @@ 'use strict' -/* eslint-env browser */ -const { Buffer } = require('buffer') const errCode = require('err-code') const toAsyncIterable = require('../lib/file-data-to-async-iterable') +const isBytes = require('../lib/is-bytes') +const isBloby = require('../lib/is-bloby') /* Transform one of: @@ -16,6 +16,8 @@ Blob|File { path, content: AsyncIterable } { path, content: PullStream } Iterable +Iterable +Iterable Iterable<{ path, content: Buffer }> Iterable<{ path, content: Blob }> Iterable<{ path, content: Iterable }> @@ -36,20 +38,16 @@ AsyncIterable<{ path, content: AsyncIterable }> module.exports = function normalizeInput (input) { // Buffer|ArrayBuffer|TypedArray - if (Buffer.isBuffer(input) || ArrayBuffer.isView(input) || input instanceof ArrayBuffer) { - return (async function * () { // eslint-disable-line require-await - yield normalizeTuple({ path: '', content: input }) - })() - } - // Blob|File - if (typeof Blob !== 'undefined' && input instanceof Blob) { + if (isBytes(input) || isBloby(input)) { return (async function * () { // eslint-disable-line require-await yield normalizeTuple({ path: '', content: input }) })() } // Iterable + // Iterable + // Iterable // Iterable<{ path, content: Buffer }> // Iterable<{ path, content: Blob }> // Iterable<{ path, content: Iterable }> @@ -58,7 +56,9 @@ module.exports = function normalizeInput (input) { if (input[Symbol.iterator]) { return (async function * () { // eslint-disable-line require-await for (const chunk of input) { - if (typeof chunk === 'object' && (chunk.path || chunk.content)) { + if (isBytes(chunk) || isBloby(chunk)) { + yield normalizeTuple({ path: '', content: chunk }) + } else if (isFileObject(chunk)) { yield normalizeTuple(chunk) } else if (Number.isInteger(chunk)) { // Must be an Iterable i.e. Buffer/ArrayBuffer/Array of bytes yield normalizeTuple({ path: '', content: input }) @@ -79,7 +79,7 @@ module.exports = function normalizeInput (input) { if (input[Symbol.asyncIterator]) { return (async function * () { for await (const chunk of input) { - if (typeof chunk === 'object' && (chunk.path || chunk.content)) { + if (isFileObject(chunk)) { yield normalizeTuple(chunk) } else { // Must be an AsyncIterable i.e. a Stream let path = '' @@ -110,7 +110,7 @@ module.exports = function normalizeInput (input) { // { path, content: Iterable } // { path, content: AsyncIterable } // { path, content: PullStream } - if (typeof input === 'object' && (input.path || input.content)) { + if (isFileObject(input)) { // eslint-disable-next-line require-await return (async function * () { yield normalizeTuple(input) })() } @@ -128,3 +128,8 @@ module.exports = function normalizeInput (input) { function normalizeTuple ({ path, content }) { return { path: path || '', content: content ? toAsyncIterable(content) : null } } + +// An object with a path or content property +function isFileObject (obj) { + return typeof obj === 'object' && (obj.path || obj.content) +} diff --git a/src/lib/file-data-to-async-iterable.js b/src/lib/file-data-to-async-iterable.js index ffa0f8d11..32614324e 100644 --- a/src/lib/file-data-to-async-iterable.js +++ b/src/lib/file-data-to-async-iterable.js @@ -4,6 +4,7 @@ const toIterator = require('pull-stream-to-async-iterator') const { Buffer } = require('buffer') const blobToAsyncIterable = require('../lib/blob-to-async-iterable') +const isBloby = require('../lib/is-bloby') /* Transform one of: @@ -29,7 +30,7 @@ module.exports = function toAsyncIterable (input) { } // Blob|File - if (typeof Blob !== 'undefined' && input instanceof Blob) { + if (isBloby(input)) { return Object.assign( blobToAsyncIterable(input), { length: input.size } diff --git a/src/lib/is-bloby.js b/src/lib/is-bloby.js new file mode 100644 index 000000000..a8682f315 --- /dev/null +++ b/src/lib/is-bloby.js @@ -0,0 +1,7 @@ +'use strict' +/* eslint-env browser */ + +// Blob|File +module.exports = function isBloby (obj) { + return typeof Blob !== 'undefined' && obj instanceof Blob +} diff --git a/src/lib/is-bytes.js b/src/lib/is-bytes.js new file mode 100644 index 000000000..adc996835 --- /dev/null +++ b/src/lib/is-bytes.js @@ -0,0 +1,8 @@ +'use strict' + +const { Buffer } = require('buffer') + +// Buffer|ArrayBuffer|TypedArray +module.exports = function isBytes (obj) { + return Buffer.isBuffer(obj) || ArrayBuffer.isView(obj) || obj instanceof ArrayBuffer +} From 52acf479e274e83c5b8ef78d7964e4d759640d9e Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Sun, 1 Sep 2019 14:44:57 +0100 Subject: [PATCH 09/18] refactor: use it-glob License: MIT Signed-off-by: Alan Shaw --- package.json | 3 +- src/add-from-fs/glob-source.js | 89 ++++++++++++++-------------------- src/add-from-url.js | 4 +- 3 files changed, 40 insertions(+), 56 deletions(-) diff --git a/package.json b/package.json index bb35dde09..cd38e5b32 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,7 @@ "err-code": "^2.0.0", "explain-error": "^1.0.4", "flatmap": "0.0.3", + "fs-extra": "^8.1.0", "glob": "^7.1.3", "ipfs-block": "~0.8.1", "ipfs-utils": "~0.0.3", @@ -66,7 +67,7 @@ "is-stream": "^2.0.0", "iso-stream-http": "~0.1.2", "iso-url": "~0.4.6", - "it-pushable": "^1.2.1", + "it-glob": "0.0.1", "it-to-stream": "^0.1.1", "iterable-ndjson": "^1.1.0", "just-kebab-case": "^1.1.0", diff --git a/src/add-from-fs/glob-source.js b/src/add-from-fs/glob-source.js index b48a31cdc..376b794f3 100644 --- a/src/add-from-fs/glob-source.js +++ b/src/add-from-fs/glob-source.js @@ -1,9 +1,8 @@ 'use strict' -const Fs = require('fs') +const fs = require('fs-extra') +const glob = require('it-glob') const Path = require('path') -const glob = require('glob') -const pushable = require('it-pushable') const errCode = require('err-code') /** @@ -16,9 +15,9 @@ const errCode = require('err-code') * @param {Boolean} [options.hidden] Include .dot files in matched paths * @param {Array} [options.ignore] Glob paths to ignore * @param {Boolean} [options.followSymlinks] follow symlinks -* @returns {AsyncIterable} +* @returns {Iterable} source iterable */ -module.exports = (...args) => (async function * () { +module.exports = async function * globSource (...args) { const options = typeof args[args.length - 1] === 'string' ? {} : args.pop() const paths = args @@ -32,63 +31,49 @@ module.exports = (...args) => (async function * () { } // Check the input paths comply with options.recursive and convert to glob sources - const results = await Promise.all(paths.map(pathAndType)) - const globSources = results.map(r => toGlobSource(r, globSourceOptions)) - - for (const globSource of globSources) { - for await (const { path, contentPath } of globSource) { - yield { path, content: Fs.createReadStream(contentPath) } - } + for (const path of paths) { + const stat = await fs.stat(path) + const prefix = Path.dirname(path) + yield * toGlobSource({ path, type: stat.isDirectory() ? 'dir' : 'file', prefix }, globSourceOptions)) } -})() - -function toGlobSource ({ path, type }, options) { - return (async function * () { - options = options || {} +} - const baseName = Path.basename(path) +async function * toGlobSource ({ path, type, prefix }, options) { + options = options || {} - if (type === 'file') { - yield { path: baseName, contentPath: path } - return - } + const baseName = Path.basename(path) - if (type === 'dir' && !options.recursive) { - throw errCode( - new Error(`'${path}' is a directory and recursive option not set`), - 'ERR_DIR_NON_RECURSIVE', - { path } + if (type === 'file') { + yield { + path: baseName.replace(prefix, ''), + content: fs.createReadStream( + Path.isAbsolute(path) ? path : Path.join(process.cwd(), path) ) } + return + } - const globOptions = Object.assign({}, options.glob, { - cwd: path, - nodir: true, - realpath: false, - absolute: false - }) - - // TODO: want to use pull-glob but it doesn't have the features... - const pusher = pushable() + if (type === 'dir' && !options.recursive) { + throw errCode( + new Error(`'${path}' is a directory and recursive option not set`), + 'ERR_DIR_NON_RECURSIVE', + { path } + ) + } - glob('**/*', globOptions) - .on('match', m => pusher.push(m)) - .on('end', () => pusher.end()) - .on('abort', () => pusher.end()) - .on('error', err => pusher.end(err)) + const globOptions = Object.assign({}, options.glob, { + cwd: path, + nodir: true, + realpath: false, + absolute: false + }) - for await (const p of pusher) { - yield { - path: `${baseName}/${toPosix(p)}`, - contentPath: Path.join(path, p) - } + for await (const p of glob(path, '**/*', globOptions)) { + yield { + path: toPosix(p.replace(prefix, '')), + content: fs.createReadStream(p) } - })() -} - -async function pathAndType (path) { - const stat = await Fs.promises.stat(path) - return { path, type: stat.isDirectory() ? 'dir' : 'file' } + } } const toPosix = path => path.replace(/\\/g, '/') diff --git a/src/add-from-url.js b/src/add-from-url.js index e98169eed..336906ea1 100644 --- a/src/add-from-url.js +++ b/src/add-from-url.js @@ -17,8 +17,6 @@ module.exports = configure(({ ky }) => { content: toIterable(body) } - for await (const file of add(input, options)) { - yield file - } + yield * add(input, options) })() }) From 440d622c04fb9f1b5431815145b2056710134085 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Sun, 1 Sep 2019 14:50:41 +0100 Subject: [PATCH 10/18] refactor: use normalise from ipfs-utils License: MIT Signed-off-by: Alan Shaw --- package.json | 2 +- src/add/form-data.browser.js | 4 +- src/add/form-data.js | 4 +- src/add/normalise-input.js | 135 ------------------------- src/lib/blob-to-async-iterable.js | 30 ------ src/lib/file-data-to-async-iterable.js | 55 ---------- src/lib/is-bloby.js | 7 -- src/lib/is-bytes.js | 8 -- 8 files changed, 5 insertions(+), 240 deletions(-) delete mode 100644 src/add/normalise-input.js delete mode 100644 src/lib/blob-to-async-iterable.js delete mode 100644 src/lib/file-data-to-async-iterable.js delete mode 100644 src/lib/is-bloby.js delete mode 100644 src/lib/is-bytes.js diff --git a/package.json b/package.json index cd38e5b32..41cea5d84 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,7 @@ "fs-extra": "^8.1.0", "glob": "^7.1.3", "ipfs-block": "~0.8.1", - "ipfs-utils": "~0.0.3", + "ipfs-utils": "github:ipfs/js-ipfs-utils#normalise-input", "ipld-dag-cbor": "~0.15.0", "ipld-dag-pb": "~0.17.3", "ipld-raw": "^4.0.0", diff --git a/src/add/form-data.browser.js b/src/add/form-data.browser.js index 46f0254d5..497a4f397 100644 --- a/src/add/form-data.browser.js +++ b/src/add/form-data.browser.js @@ -1,9 +1,9 @@ 'use strict' /* eslint-env browser */ -const normaliseInput = require('./normalise-input') +const normaliseInput = require('ipfs-utils/src/files/normalise-input') -exports.toFormData = async (input) => { +exports.toFormData = async input => { const files = normaliseInput(input) const formData = new FormData() let i = 0 diff --git a/src/add/form-data.js b/src/add/form-data.js index be268320f..2465ce2e3 100644 --- a/src/add/form-data.js +++ b/src/add/form-data.js @@ -3,9 +3,9 @@ const FormData = require('form-data') const { Buffer } = require('buffer') const toStream = require('it-to-stream') -const normaliseInput = require('./normalise-input') +const normaliseInput = require('ipfs-utils/src/files/normalise-input') -exports.toFormData = async (input) => { +exports.toFormData = async input => { const files = normaliseInput(input) const formData = new FormData() let i = 0 diff --git a/src/add/normalise-input.js b/src/add/normalise-input.js deleted file mode 100644 index e38ba3a31..000000000 --- a/src/add/normalise-input.js +++ /dev/null @@ -1,135 +0,0 @@ -'use strict' - -const errCode = require('err-code') -const toAsyncIterable = require('../lib/file-data-to-async-iterable') -const isBytes = require('../lib/is-bytes') -const isBloby = require('../lib/is-bloby') - -/* -Transform one of: - -Buffer|ArrayBuffer|TypedArray -Blob|File -{ path, content: Buffer } -{ path, content: Blob } -{ path, content: Iterable } -{ path, content: AsyncIterable } -{ path, content: PullStream } -Iterable -Iterable -Iterable -Iterable<{ path, content: Buffer }> -Iterable<{ path, content: Blob }> -Iterable<{ path, content: Iterable }> -Iterable<{ path, content: AsyncIterable }> -Iterable<{ path, content: PullStream }> -AsyncIterable -AsyncIterable<{ path, content: Buffer }> -AsyncIterable<{ path, content: Blob }> -AsyncIterable<{ path, content: Iterable }> -AsyncIterable<{ path, content: AsyncIterable }> -AsyncIterable<{ path, content: PullStream }> -PullStream - -Into: - -AsyncIterable<{ path, content: AsyncIterable }> -*/ - -module.exports = function normalizeInput (input) { - // Buffer|ArrayBuffer|TypedArray - // Blob|File - if (isBytes(input) || isBloby(input)) { - return (async function * () { // eslint-disable-line require-await - yield normalizeTuple({ path: '', content: input }) - })() - } - - // Iterable - // Iterable - // Iterable - // Iterable<{ path, content: Buffer }> - // Iterable<{ path, content: Blob }> - // Iterable<{ path, content: Iterable }> - // Iterable<{ path, content: AsyncIterable }> - // Iterable<{ path, content: PullStream }> - if (input[Symbol.iterator]) { - return (async function * () { // eslint-disable-line require-await - for (const chunk of input) { - if (isBytes(chunk) || isBloby(chunk)) { - yield normalizeTuple({ path: '', content: chunk }) - } else if (isFileObject(chunk)) { - yield normalizeTuple(chunk) - } else if (Number.isInteger(chunk)) { // Must be an Iterable i.e. Buffer/ArrayBuffer/Array of bytes - yield normalizeTuple({ path: '', content: input }) - return - } else { - throw errCode(new Error('Unexpected input: ' + typeof chunk), 'ERR_UNEXPECTED_INPUT') - } - } - })() - } - - // AsyncIterable - // AsyncIterable<{ path, content: Buffer }> - // AsyncIterable<{ path, content: Blob }> - // AsyncIterable<{ path, content: Iterable }> - // AsyncIterable<{ path, content: AsyncIterable }> - // AsyncIterable<{ path, content: PullStream }> - if (input[Symbol.asyncIterator]) { - return (async function * () { - for await (const chunk of input) { - if (isFileObject(chunk)) { - yield normalizeTuple(chunk) - } else { // Must be an AsyncIterable i.e. a Stream - let path = '' - - // fs.createReadStream will create a stream with a `path` prop - // If available, use it here! - if (input.path && input.path.split) { - path = input.path.split(/[/\\]/).pop() || '' - } - - yield normalizeTuple({ - path, - content: (async function * () { - yield chunk - for await (const restChunk of input) { - yield restChunk - } - })() - }) - return - } - } - })() - } - - // { path, content: Buffer } - // { path, content: Blob } - // { path, content: Iterable } - // { path, content: AsyncIterable } - // { path, content: PullStream } - if (isFileObject(input)) { - // eslint-disable-next-line require-await - return (async function * () { yield normalizeTuple(input) })() - } - - // PullStream - if (typeof input === 'function') { - return (async function * () { // eslint-disable-line require-await - yield normalizeTuple({ path: '', content: input }) - })() - } - - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') -} - -function normalizeTuple ({ path, content }) { - return { path: path || '', content: content ? toAsyncIterable(content) : null } -} - -// An object with a path or content property -function isFileObject (obj) { - return typeof obj === 'object' && (obj.path || obj.content) -} diff --git a/src/lib/blob-to-async-iterable.js b/src/lib/blob-to-async-iterable.js deleted file mode 100644 index 3f9e982c0..000000000 --- a/src/lib/blob-to-async-iterable.js +++ /dev/null @@ -1,30 +0,0 @@ -'use strict' -/* eslint-env browser */ - -// Convert a Blob into an AsyncIterable -module.exports = (blob, options) => (async function * () { - options = options || {} - - const reader = new FileReader() - const chunkSize = options.chunkSize || 1024 * 1024 - let offset = options.offset || 0 - - const getNextChunk = () => new Promise((resolve, reject) => { - reader.onloadend = e => { - const data = e.target.result - resolve(data.byteLength === 0 ? null : data) - } - reader.onerror = reject - - const end = offset + chunkSize - const slice = blob.slice(offset, end) - reader.readAsArrayBuffer(slice) - offset = end - }) - - while (true) { - const data = await getNextChunk() - if (data == null) return - yield data - } -})() diff --git a/src/lib/file-data-to-async-iterable.js b/src/lib/file-data-to-async-iterable.js deleted file mode 100644 index 32614324e..000000000 --- a/src/lib/file-data-to-async-iterable.js +++ /dev/null @@ -1,55 +0,0 @@ -'use strict' -/* eslint-env browser */ - -const toIterator = require('pull-stream-to-async-iterator') -const { Buffer } = require('buffer') -const blobToAsyncIterable = require('../lib/blob-to-async-iterable') -const isBloby = require('../lib/is-bloby') - -/* -Transform one of: - -Buffer|ArrayBuffer|TypedArray -Blob|File -Iterable -AsyncIterable -PullStream - -Into: - -AsyncIterable -*/ -module.exports = function toAsyncIterable (input) { - // Buffer|ArrayBuffer|TypedArray|array of bytes - if (input[Symbol.iterator]) { - const buf = Buffer.from(input) - return Object.assign( - (async function * () { yield buf })(), // eslint-disable-line require-await - { length: buf.length } - ) - } - - // Blob|File - if (isBloby(input)) { - return Object.assign( - blobToAsyncIterable(input), - { length: input.size } - ) - } - - // AsyncIterable - if (input[Symbol.asyncIterator]) { - return (async function * () { - for await (const chunk of input) { - yield Buffer.from(chunk) - } - })() - } - - // PullStream - if (typeof input === 'function') { - return toIterator(input) - } - - throw new Error('Unexpected input: ' + typeof input) -} diff --git a/src/lib/is-bloby.js b/src/lib/is-bloby.js deleted file mode 100644 index a8682f315..000000000 --- a/src/lib/is-bloby.js +++ /dev/null @@ -1,7 +0,0 @@ -'use strict' -/* eslint-env browser */ - -// Blob|File -module.exports = function isBloby (obj) { - return typeof Blob !== 'undefined' && obj instanceof Blob -} diff --git a/src/lib/is-bytes.js b/src/lib/is-bytes.js deleted file mode 100644 index adc996835..000000000 --- a/src/lib/is-bytes.js +++ /dev/null @@ -1,8 +0,0 @@ -'use strict' - -const { Buffer } = require('buffer') - -// Buffer|ArrayBuffer|TypedArray -module.exports = function isBytes (obj) { - return Buffer.isBuffer(obj) || ArrayBuffer.isView(obj) || obj instanceof ArrayBuffer -} From 75764325f04144df3490df2cbff006aa3c1f1bd7 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 3 Sep 2019 14:09:24 +0100 Subject: [PATCH 11/18] fix: tweaks and fixes License: MIT Signed-off-by: Alan Shaw --- package.json | 7 ++++--- src/add-from-fs/glob-source.js | 3 ++- src/files-regular/add-async-iterator.js | 23 ----------------------- src/lib/{iterable.js => converters.js} | 7 ++----- src/utils/load-commands.js | 4 ++-- test/interface.spec.js | 21 ++++----------------- 6 files changed, 14 insertions(+), 51 deletions(-) delete mode 100644 src/files-regular/add-async-iterator.js rename src/lib/{iterable.js => converters.js} (73%) diff --git a/package.json b/package.json index 41cea5d84..647fd32fe 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "dependencies": { "abort-controller": "^3.0.0", "async": "^2.6.1", + "async-iterator-all": "^1.0.0", "async-iterator-to-pull-stream": "^1.3.0", "bignumber.js": "^9.0.0", "bl": "^3.0.0", @@ -58,7 +59,7 @@ "fs-extra": "^8.1.0", "glob": "^7.1.3", "ipfs-block": "~0.8.1", - "ipfs-utils": "github:ipfs/js-ipfs-utils#normalise-input", + "ipfs-utils": "github:ipfs/js-ipfs-utils#norm-in-pullstream", "ipld-dag-cbor": "~0.15.0", "ipld-dag-pb": "~0.17.3", "ipld-raw": "^4.0.0", @@ -67,7 +68,7 @@ "is-stream": "^2.0.0", "iso-stream-http": "~0.1.2", "iso-url": "~0.4.6", - "it-glob": "0.0.1", + "it-glob": "0.0.3", "it-to-stream": "^0.1.1", "iterable-ndjson": "^1.1.0", "just-kebab-case": "^1.1.0", @@ -105,7 +106,7 @@ "cross-env": "^5.2.0", "dirty-chai": "^2.0.1", "go-ipfs-dep": "^0.4.22", - "interface-ipfs-core": "^0.111.0", + "interface-ipfs-core": "^0.112.0", "ipfsd-ctl": "~0.45.0", "nock": "^10.0.2", "stream-equal": "^1.1.1" diff --git a/src/add-from-fs/glob-source.js b/src/add-from-fs/glob-source.js index 376b794f3..5828bd410 100644 --- a/src/add-from-fs/glob-source.js +++ b/src/add-from-fs/glob-source.js @@ -34,7 +34,8 @@ module.exports = async function * globSource (...args) { for (const path of paths) { const stat = await fs.stat(path) const prefix = Path.dirname(path) - yield * toGlobSource({ path, type: stat.isDirectory() ? 'dir' : 'file', prefix }, globSourceOptions)) + const type = stat.isDirectory() ? 'dir' : 'file' + yield * toGlobSource({ path, type, prefix }, globSourceOptions) } } diff --git a/src/files-regular/add-async-iterator.js b/src/files-regular/add-async-iterator.js deleted file mode 100644 index 3fa2b23ed..000000000 --- a/src/files-regular/add-async-iterator.js +++ /dev/null @@ -1,23 +0,0 @@ -'use strict' - -const SendFilesStream = require('../utils/send-files-stream') -const FileResultStreamConverter = require('../utils/file-result-stream-converter') - -module.exports = (send) => { - return async function * (source, options) { - options = options || {} - options.converter = FileResultStreamConverter - - const stream = SendFilesStream(send, 'add')(options) - - for await (const entry of source) { - stream.write(entry) - } - - stream.end() - - for await (const entry of stream) { - yield entry - } - } -} diff --git a/src/lib/iterable.js b/src/lib/converters.js similarity index 73% rename from src/lib/iterable.js rename to src/lib/converters.js index 78de19f5e..96f4148fd 100644 --- a/src/lib/iterable.js +++ b/src/lib/converters.js @@ -1,13 +1,10 @@ 'use strict' const toPull = require('async-iterator-to-pull-stream') +const all = require('async-iterator-all') const toStream = require('it-to-stream') -exports.collectify = fn => async (...args) => { - const items = [] - for await (const item of fn(...args)) items.push(item) - return items -} +exports.collectify = fn => async (...args) => all(fn(...args)) exports.pullify = { source: fn => (...args) => toPull(fn(...args)), diff --git a/src/utils/load-commands.js b/src/utils/load-commands.js index 1c36702d1..39ce58b8e 100644 --- a/src/utils/load-commands.js +++ b/src/utils/load-commands.js @@ -1,7 +1,7 @@ 'use strict' const nodeify = require('promise-nodeify') -const { collectify, pullify, streamify } = require('../lib/iterable') +const { collectify, pullify, streamify } = require('../lib/converters') function requireCommands () { return { @@ -54,7 +54,7 @@ function requireCommands () { return nodeify(add(input, options), callback) } }, - _addAsyncIterator: require('../files-regular/add-async-iterator'), + _addAsyncIterator: (_, config) => require('../add')(config), cat: require('../files-regular/cat'), catReadableStream: require('../files-regular/cat-readable-stream'), catPullStream: require('../files-regular/cat-pull-stream'), diff --git a/test/interface.spec.js b/test/interface.spec.js index 6ea21e485..3fabf2c56 100644 --- a/test/interface.spec.js +++ b/test/interface.spec.js @@ -116,14 +116,6 @@ describe('interface-ipfs-core tests', () => { name: 'should add a nested directory as array of tupples with progress', reason: 'FIXME https://github.com/ipfs/js-ipfs-http-client/issues/339' }, - { - name: 'should not be able to add a string', - reason: 'FIXME test needs to change to inspect error code ERR_UNEXPECTED_INPUT' - }, - { - name: 'should not be able to add a non-Buffer TypedArray', - reason: 'TODO remove test, this should be supported' - }, // .addPullStream isNode ? null : { name: 'should add pull stream of valid files and dirs', @@ -134,20 +126,15 @@ describe('interface-ipfs-core tests', () => { name: 'should add readable stream of valid files and dirs', reason: 'FIXME https://github.com/ipfs/js-ipfs-http-client/issues/339' }, - // .addFromStream - isNode ? null : { - name: 'addFromStream', - reason: 'Not designed to run in the browser' - }, - { - name: 'should add from a stream', - reason: 'TODO change test to use readable-stream@3 with async iterator support' - }, // .addFromFs isNode ? null : { name: 'addFromFs', reason: 'Not designed to run in the browser' }, + { + name: 'should ignore a directory from the file system', + reason: 'FIXME bug in it-glob' + }, // .addFromURL isNode ? null : { name: 'addFromURL', From 0598651e816785d563cc2464bc13500a901ed95d Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 3 Sep 2019 14:25:22 +0100 Subject: [PATCH 12/18] chore: appease linter License: MIT Signed-off-by: Alan Shaw --- src/lib/converters.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/converters.js b/src/lib/converters.js index 96f4148fd..7c5965feb 100644 --- a/src/lib/converters.js +++ b/src/lib/converters.js @@ -4,7 +4,7 @@ const toPull = require('async-iterator-to-pull-stream') const all = require('async-iterator-all') const toStream = require('it-to-stream') -exports.collectify = fn => async (...args) => all(fn(...args)) +exports.collectify = fn => (...args) => all(fn(...args)) exports.pullify = { source: fn => (...args) => toPull(fn(...args)), From 2e7e9698289b892f9c13e4b4539849d8322f745f Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 3 Sep 2019 14:27:38 +0100 Subject: [PATCH 13/18] fix: import buffer when needed License: MIT Signed-off-by: Alan Shaw --- src/add/form-data.browser.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/add/form-data.browser.js b/src/add/form-data.browser.js index 497a4f397..8e228c164 100644 --- a/src/add/form-data.browser.js +++ b/src/add/form-data.browser.js @@ -1,6 +1,7 @@ 'use strict' /* eslint-env browser */ +const { Buffer } = require('buffer') const normaliseInput = require('ipfs-utils/src/files/normalise-input') exports.toFormData = async input => { From 956a5afd6fe9d9dbbe3bdffb34efba083311e908 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 3 Sep 2019 14:28:33 +0100 Subject: [PATCH 14/18] chore: temporarily increase bundle size limit This will reduce as we switch more APIs to using fetch. License: MIT Signed-off-by: Alan Shaw --- .aegir.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.aegir.js b/.aegir.js index d9a727161..20aa93b74 100644 --- a/.aegir.js +++ b/.aegir.js @@ -5,7 +5,7 @@ const createServer = require('ipfsd-ctl').createServer const server = createServer() module.exports = { - bundlesize: { maxSize: '237kB' }, + bundlesize: { maxSize: '240kB' }, webpack: { resolve: { mainFields: ['browser', 'main'] From fee0eed1f8b6de465a8f40db2f6088294fbf80de Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 3 Sep 2019 15:10:23 +0100 Subject: [PATCH 15/18] fix: tests License: MIT Signed-off-by: Alan Shaw --- src/files-regular/index.js | 51 ++++++++++++++++++++++++++++++ src/utils/load-commands.js | 65 +------------------------------------- 2 files changed, 52 insertions(+), 64 deletions(-) diff --git a/src/files-regular/index.js b/src/files-regular/index.js index 408f76494..20812d749 100644 --- a/src/files-regular/index.js +++ b/src/files-regular/index.js @@ -1,11 +1,62 @@ 'use strict' +const nodeify = require('promise-nodeify') const moduleConfig = require('../utils/module-config') +const { collectify, pullify, streamify } = require('../lib/converters') module.exports = (arg) => { const send = moduleConfig(arg) return { + add: (_, config) => { + const add = collectify(require('../add')(config)) + return (input, options, callback) => { + if (typeof options === 'function') { + callback = options + options = {} + } + return nodeify(add(input, options), callback) + } + }, + addReadableStream: (_, config) => { + const add = require('../add')(config) + return streamify.transform(add) + }, + addPullStream: (_, config) => { + const add = require('../add')(config) + return pullify.transform(add) + }, + addFromFs: (_, config) => { + const addFromFs = collectify(require('../add-from-fs')(config)) + return (path, options, callback) => { + if (typeof options === 'function') { + callback = options + options = {} + } + return nodeify(addFromFs(path, options), callback) + } + }, + addFromURL: (_, config) => { + const addFromURL = collectify(require('../add-from-url')(config)) + return (url, options, callback) => { + if (typeof options === 'function') { + callback = options + options = {} + } + return nodeify(addFromURL(url, options), callback) + } + }, + addFromStream: (_, config) => { + const add = collectify(require('../add')(config)) + return (input, options, callback) => { + if (typeof options === 'function') { + callback = options + options = {} + } + return nodeify(add(input, options), callback) + } + }, + _addAsyncIterator: (_, config) => require('../add')(config), cat: require('../files-regular/cat')(send), catReadableStream: require('../files-regular/cat-readable-stream')(send), catPullStream: require('../files-regular/cat-pull-stream')(send), diff --git a/src/utils/load-commands.js b/src/utils/load-commands.js index 39ce58b8e..f23f6130d 100644 --- a/src/utils/load-commands.js +++ b/src/utils/load-commands.js @@ -1,72 +1,9 @@ 'use strict' -const nodeify = require('promise-nodeify') -const { collectify, pullify, streamify } = require('../lib/converters') - function requireCommands () { return { // Files Regular (not MFS) - add: (_, config) => { - const add = collectify(require('../add')(config)) - return (input, options, callback) => { - if (typeof options === 'function') { - callback = options - options = {} - } - return nodeify(add(input, options), callback) - } - }, - addReadableStream: (_, config) => { - const add = require('../add')(config) - return streamify.transform(add) - }, - addPullStream: (_, config) => { - const add = require('../add')(config) - return pullify.transform(add) - }, - addFromFs: (_, config) => { - const addFromFs = collectify(require('../add-from-fs')(config)) - return (path, options, callback) => { - if (typeof options === 'function') { - callback = options - options = {} - } - return nodeify(addFromFs(path, options), callback) - } - }, - addFromURL: (_, config) => { - const addFromURL = collectify(require('../add-from-url')(config)) - return (url, options, callback) => { - if (typeof options === 'function') { - callback = options - options = {} - } - return nodeify(addFromURL(url, options), callback) - } - }, - addFromStream: (_, config) => { - const add = collectify(require('../add')(config)) - return (input, options, callback) => { - if (typeof options === 'function') { - callback = options - options = {} - } - return nodeify(add(input, options), callback) - } - }, - _addAsyncIterator: (_, config) => require('../add')(config), - cat: require('../files-regular/cat'), - catReadableStream: require('../files-regular/cat-readable-stream'), - catPullStream: require('../files-regular/cat-pull-stream'), - get: require('../files-regular/get'), - getReadableStream: require('../files-regular/get-readable-stream'), - getPullStream: require('../files-regular/get-pull-stream'), - ls: require('../files-regular/ls'), - lsReadableStream: require('../files-regular/ls-readable-stream'), - lsPullStream: require('../files-regular/ls-pull-stream'), - refs: require('../files-regular/refs'), - refsReadableStream: require('../files-regular/refs-readable-stream'), - refsPullStream: require('../files-regular/refs-pull-stream'), + ...require('../files-regular'), // Files MFS (Mutable Filesystem) files: require('../files-mfs'), From 732f063e6a1a2a9af570157960d2f12e881d4c7d Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 3 Sep 2019 15:53:07 +0100 Subject: [PATCH 16/18] fix: more fix License: MIT Signed-off-by: Alan Shaw --- package.json | 2 +- src/add-from-fs/glob-source.js | 6 +-- src/files-regular/index.js | 67 +++++++++++++--------------------- src/utils/load-commands.js | 25 +++++-------- test/interface.spec.js | 4 -- 5 files changed, 40 insertions(+), 64 deletions(-) diff --git a/package.json b/package.json index 647fd32fe..cacd556ea 100644 --- a/package.json +++ b/package.json @@ -68,7 +68,7 @@ "is-stream": "^2.0.0", "iso-stream-http": "~0.1.2", "iso-url": "~0.4.6", - "it-glob": "0.0.3", + "it-glob": "0.0.4", "it-to-stream": "^0.1.1", "iterable-ndjson": "^1.1.0", "just-kebab-case": "^1.1.0", diff --git a/src/add-from-fs/glob-source.js b/src/add-from-fs/glob-source.js index 5828bd410..465b45494 100644 --- a/src/add-from-fs/glob-source.js +++ b/src/add-from-fs/glob-source.js @@ -46,7 +46,7 @@ async function * toGlobSource ({ path, type, prefix }, options) { if (type === 'file') { yield { - path: baseName.replace(prefix, ''), + path: baseName, content: fs.createReadStream( Path.isAbsolute(path) ? path : Path.join(process.cwd(), path) ) @@ -71,8 +71,8 @@ async function * toGlobSource ({ path, type, prefix }, options) { for await (const p of glob(path, '**/*', globOptions)) { yield { - path: toPosix(p.replace(prefix, '')), - content: fs.createReadStream(p) + path: toPosix(Path.join(baseName, p)), + content: fs.createReadStream(Path.join(path, p)) } } } diff --git a/src/files-regular/index.js b/src/files-regular/index.js index 20812d749..d098516f1 100644 --- a/src/files-regular/index.js +++ b/src/files-regular/index.js @@ -6,57 +6,42 @@ const { collectify, pullify, streamify } = require('../lib/converters') module.exports = (arg) => { const send = moduleConfig(arg) + const add = require('../add')(arg) + const addFromFs = require('../add-from-fs')(arg) + const addFromURL = require('../add-from-url')(arg) return { - add: (_, config) => { - const add = collectify(require('../add')(config)) - return (input, options, callback) => { - if (typeof options === 'function') { - callback = options - options = {} - } - return nodeify(add(input, options), callback) + add: (input, options, callback) => { + if (typeof options === 'function') { + callback = options + options = {} } + return nodeify(collectify(add)(input, options), callback) }, - addReadableStream: (_, config) => { - const add = require('../add')(config) - return streamify.transform(add) - }, - addPullStream: (_, config) => { - const add = require('../add')(config) - return pullify.transform(add) - }, - addFromFs: (_, config) => { - const addFromFs = collectify(require('../add-from-fs')(config)) - return (path, options, callback) => { - if (typeof options === 'function') { - callback = options - options = {} - } - return nodeify(addFromFs(path, options), callback) + addReadableStream: streamify.transform(add), + addPullStream: pullify.transform(add), + addFromFs: (path, options, callback) => { + if (typeof options === 'function') { + callback = options + options = {} } + return nodeify(collectify(addFromFs)(path, options), callback) }, - addFromURL: (_, config) => { - const addFromURL = collectify(require('../add-from-url')(config)) - return (url, options, callback) => { - if (typeof options === 'function') { - callback = options - options = {} - } - return nodeify(addFromURL(url, options), callback) + addFromURL: (url, options, callback) => { + if (typeof options === 'function') { + callback = options + options = {} } + return nodeify(collectify(addFromURL)(url, options), callback) }, - addFromStream: (_, config) => { - const add = collectify(require('../add')(config)) - return (input, options, callback) => { - if (typeof options === 'function') { - callback = options - options = {} - } - return nodeify(add(input, options), callback) + addFromStream: (input, options, callback) => { + if (typeof options === 'function') { + callback = options + options = {} } + return nodeify(collectify(add)(input, options), callback) }, - _addAsyncIterator: (_, config) => require('../add')(config), + _addAsyncIterator: add, cat: require('../files-regular/cat')(send), catReadableStream: require('../files-regular/cat-readable-stream')(send), catPullStream: require('../files-regular/cat-pull-stream')(send), diff --git a/src/utils/load-commands.js b/src/utils/load-commands.js index f23f6130d..710b8396f 100644 --- a/src/utils/load-commands.js +++ b/src/utils/load-commands.js @@ -1,10 +1,12 @@ 'use strict' -function requireCommands () { - return { - // Files Regular (not MFS) - ...require('../files-regular'), +function requireCommands (send, config) { + const cmds = { + ...require('../files-regular')(config), + getEndpointConfig: require('../get-endpoint-config')(config) + } + const subCmds = { // Files MFS (Mutable Filesystem) files: require('../files-mfs'), @@ -42,21 +44,14 @@ function requireCommands () { stats: require('../stats'), update: require('../update'), version: require('../version'), - resolve: require('../resolve'), - // ipfs-http-client instance - getEndpointConfig: (send, config) => require('../get-endpoint-config')(config) + resolve: require('../resolve') } -} - -function loadCommands (send, config) { - const files = requireCommands() - const cmds = {} - Object.keys(files).forEach((file) => { - cmds[file] = files[file](send, config) + Object.keys(subCmds).forEach((file) => { + cmds[file] = subCmds[file](send, config) }) return cmds } -module.exports = loadCommands +module.exports = requireCommands diff --git a/test/interface.spec.js b/test/interface.spec.js index 3fabf2c56..32c6f6749 100644 --- a/test/interface.spec.js +++ b/test/interface.spec.js @@ -131,10 +131,6 @@ describe('interface-ipfs-core tests', () => { name: 'addFromFs', reason: 'Not designed to run in the browser' }, - { - name: 'should ignore a directory from the file system', - reason: 'FIXME bug in it-glob' - }, // .addFromURL isNode ? null : { name: 'addFromURL', From c9bf01e5556e9f85b60bc5dc79a8fded677d2850 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 3 Sep 2019 20:58:36 +0100 Subject: [PATCH 17/18] chore: update ipfs-utils dep --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index cacd556ea..011beaf2e 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,7 @@ "fs-extra": "^8.1.0", "glob": "^7.1.3", "ipfs-block": "~0.8.1", - "ipfs-utils": "github:ipfs/js-ipfs-utils#norm-in-pullstream", + "ipfs-utils": "github:ipfs/js-ipfs-utils#normalise-input", "ipld-dag-cbor": "~0.15.0", "ipld-dag-pb": "~0.17.3", "ipld-raw": "^4.0.0", From 24ad264781c187f71ac7948e7bf927a4bef0aba5 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 4 Sep 2019 11:14:07 +0100 Subject: [PATCH 18/18] refactor: use globSource from ipfs-utils License: MIT Signed-off-by: Alan Shaw --- package.json | 2 +- src/add-from-fs/glob-source.js | 80 ---------------------------------- src/add-from-fs/index.js | 2 +- 3 files changed, 2 insertions(+), 82 deletions(-) delete mode 100644 src/add-from-fs/glob-source.js diff --git a/package.json b/package.json index 011beaf2e..fd4d82c69 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,7 @@ "fs-extra": "^8.1.0", "glob": "^7.1.3", "ipfs-block": "~0.8.1", - "ipfs-utils": "github:ipfs/js-ipfs-utils#normalise-input", + "ipfs-utils": "^0.1.0", "ipld-dag-cbor": "~0.15.0", "ipld-dag-pb": "~0.17.3", "ipld-raw": "^4.0.0", diff --git a/src/add-from-fs/glob-source.js b/src/add-from-fs/glob-source.js deleted file mode 100644 index 465b45494..000000000 --- a/src/add-from-fs/glob-source.js +++ /dev/null @@ -1,80 +0,0 @@ -'use strict' - -const fs = require('fs-extra') -const glob = require('it-glob') -const Path = require('path') -const errCode = require('err-code') - -/** -* Create an AsyncIterable that can be passed to ipfs.add for the -* provided file paths. -* -* @param {String} ...paths File system path(s) to glob from -* @param {Object} [options] Optional options -* @param {Boolean} [options.recursive] Recursively glob all paths in directories -* @param {Boolean} [options.hidden] Include .dot files in matched paths -* @param {Array} [options.ignore] Glob paths to ignore -* @param {Boolean} [options.followSymlinks] follow symlinks -* @returns {Iterable} source iterable -*/ -module.exports = async function * globSource (...args) { - const options = typeof args[args.length - 1] === 'string' ? {} : args.pop() - const paths = args - - const globSourceOptions = { - recursive: options.recursive, - glob: { - dot: Boolean(options.hidden), - ignore: Array.isArray(options.ignore) ? options.ignore : [], - follow: options.followSymlinks != null ? options.followSymlinks : true - } - } - - // Check the input paths comply with options.recursive and convert to glob sources - for (const path of paths) { - const stat = await fs.stat(path) - const prefix = Path.dirname(path) - const type = stat.isDirectory() ? 'dir' : 'file' - yield * toGlobSource({ path, type, prefix }, globSourceOptions) - } -} - -async function * toGlobSource ({ path, type, prefix }, options) { - options = options || {} - - const baseName = Path.basename(path) - - if (type === 'file') { - yield { - path: baseName, - content: fs.createReadStream( - Path.isAbsolute(path) ? path : Path.join(process.cwd(), path) - ) - } - return - } - - if (type === 'dir' && !options.recursive) { - throw errCode( - new Error(`'${path}' is a directory and recursive option not set`), - 'ERR_DIR_NON_RECURSIVE', - { path } - ) - } - - const globOptions = Object.assign({}, options.glob, { - cwd: path, - nodir: true, - realpath: false, - absolute: false - }) - - for await (const p of glob(path, '**/*', globOptions)) { - yield { - path: toPosix(Path.join(baseName, p)), - content: fs.createReadStream(Path.join(path, p)) - } - } -} - -const toPosix = path => path.replace(/\\/g, '/') diff --git a/src/add-from-fs/index.js b/src/add-from-fs/index.js index 225acf9c3..01a82e166 100644 --- a/src/add-from-fs/index.js +++ b/src/add-from-fs/index.js @@ -1,7 +1,7 @@ 'use strict' const configure = require('../lib/configure') -const globSource = require('./glob-source') +const globSource = require('ipfs-utils/src/files/glob-source') module.exports = configure(({ ky }) => { const add = require('../add')({ ky })