From cd0a0d72d6b81884a9ede2e9020f0e6044057995 Mon Sep 17 00:00:00 2001 From: Dan Ordille Date: Wed, 25 Jul 2018 13:05:05 -0400 Subject: [PATCH 1/3] Allow chunker algorithm to be specified when adding files --- src/cli/commands/files/add.js | 14 +++++- src/core/components/files.js | 4 +- src/core/utils.js | 88 +++++++++++++++++++++++++++++++++ src/http/api/resources/files.js | 3 +- 4 files changed, 106 insertions(+), 3 deletions(-) diff --git a/src/cli/commands/files/add.js b/src/cli/commands/files/add.js index fe2a602a41..ae3d28da84 100644 --- a/src/cli/commands/files/add.js +++ b/src/cli/commands/files/add.js @@ -135,6 +135,17 @@ module.exports = { default: false, describe: 'Only chunk and hash, do not write' }, + 'chunker': { + default: 'default', + describe: 'Chunking algorithm to use', + choices: [ + "default", + "size-{size}", + "rabin", + "rabin-{avg}", + "rabin-{min}-{avg}-{max}" + ] + }, 'enable-sharding-experiment': { type: 'boolean', default: false @@ -194,7 +205,8 @@ module.exports = { onlyHash: argv.onlyHash, hashAlg: argv.hash, wrapWithDirectory: argv.wrapWithDirectory, - pin: argv.pin + pin: argv.pin, + chunker: argv.chunker } // Temporary restriction on raw-leaves: diff --git a/src/core/components/files.js b/src/core/components/files.js index f69868bd77..d15118a529 100644 --- a/src/core/components/files.js +++ b/src/core/components/files.js @@ -17,6 +17,7 @@ const Duplex = require('readable-stream').Duplex const OtherBuffer = require('buffer').Buffer const CID = require('cids') const toB58String = require('multihashes').toB58String +const parseChunkerString = require('../utils').parseChunkerString const WRAPPER = 'wrapper/' @@ -134,11 +135,12 @@ class AddHelper extends Duplex { module.exports = function files (self) { function _addPullStream (options) { + const chunkerOptions = parseChunkerString(options.chunker) const opts = Object.assign({}, { shardSplitThreshold: self._options.EXPERIMENTAL.sharding ? 1000 : Infinity - }, options) + }, options, chunkerOptions) if (opts.hashAlg && opts.cidVersion !== 1) { opts.cidVersion = 1 diff --git a/src/core/utils.js b/src/core/utils.js index a0d67e449a..9e05d9e640 100644 --- a/src/core/utils.js +++ b/src/core/utils.js @@ -107,5 +107,93 @@ const resolvePath = promisify(function (objectAPI, ipfsPaths, callback) { }, callback) }) +/** + * Parses chunker string into options used by DAGBuilder in ipfs-unixfs-engine + * + * + * @param {String} chunker Chunker algorithm supported formats: + * "default" ("") + * "size-{size}", + * "rabin" + * "rabin-{avg}" + * "rabin-{min}-{avg}-{max}" + * + * @return {Object} Chunker options for DAGBuilder + */ +function parseChunkerString(chunker) { + if (!chunker || chunker === '' || chunker === 'default') { + return { + chunker: 'fixed', + } + } else if (chunker.startsWith('size-')) { + const sizeStr = chunker.split("-")[1] + const size = parseInt(sizeStr) + if (isNaN(size)) { + throw new Error("Parameter avg must be an integer") + } + return { + chunker: 'fixed', + chunkerOptions: { + maxChunkSize: size + } + } + } else if (chunker.startsWith('rabin')) { + return { + chunker: 'rabin', + chunkerOptions: parseRabinString(chunker) + } + } else { + throw new Error(`unrecognized chunker option: ${chunker}`) + } +} + +/** + * Parses rabin chunker string + * + * @param {String} chunker Chunker algorithm supported formats: + * "rabin" + * "rabin-{avg}" + * "rabin-{min}-{avg}-{max}" + * + * @return {Object} rabin chunker options + */ +function parseRabinString(chunker) { + const options = {} + const parts = chunker.split("-") + switch (parts.length) { + case 1: + options.avgChunkSize = 262144 + break + case 2: + options.avgChunkSize = parseInt(parts[1]) + if (isNaN(options.avgBlockSize)) { + throw new Error("Parameter avg must be an integer") + } + break + case 4: + options.minChunkSize = parseSub(parts[1].split(":"), "min") + options.avgChunkSize = parseSub(parts[2].split(":"), "avg") + options.maxChunkSize = parseSub(parts[3].split(":"), "max") + break + default: + throw new Error("incorrect format (expected 'rabin' 'rabin-[avg]' or 'rabin-[min]-[avg]-[max]'") + } + + return options +} + +function parseSub(sub, name) { + if (sub.length > 1 && sub[0] !== name) { + throw new Error("Parameter order must be min:avg:max") + } + let size = parseInt(sub[sub.length-1]) + if (isNaN(size)) { + throw new Error(`Parameter ${name} must be an integer`) + } + + return size +} + exports.parseIpfsPath = parseIpfsPath exports.resolvePath = resolvePath +exports.parseChunkerString = parseChunkerString diff --git a/src/http/api/resources/files.js b/src/http/api/resources/files.js index a3b7e8eaed..34efb38955 100644 --- a/src/http/api/resources/files.js +++ b/src/http/api/resources/files.js @@ -233,7 +233,8 @@ exports.add = { onlyHash: request.query['only-hash'], hashAlg: request.query['hash'], wrapWithDirectory: request.query['wrap-with-directory'], - pin: request.query.pin + pin: request.query.pin, + chunker: request.query['chunker'] } const aborter = abortable() From f08a564b6d27e13de3db88b1d7acd78f029b4bdb Mon Sep 17 00:00:00 2001 From: Dan Ordille Date: Wed, 25 Jul 2018 13:24:17 -0400 Subject: [PATCH 2/3] Add test for parseChunkerString --- src/core/utils.js | 2 +- test/core/utils.js | 49 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/src/core/utils.js b/src/core/utils.js index 9e05d9e640..e95033cc19 100644 --- a/src/core/utils.js +++ b/src/core/utils.js @@ -166,7 +166,7 @@ function parseRabinString(chunker) { break case 2: options.avgChunkSize = parseInt(parts[1]) - if (isNaN(options.avgBlockSize)) { + if (isNaN(options.avgChunkSize)) { throw new Error("Parameter avg must be an integer") } break diff --git a/test/core/utils.js b/test/core/utils.js index b5c84b15c1..75a559d687 100644 --- a/test/core/utils.js +++ b/test/core/utils.js @@ -157,4 +157,53 @@ describe('utils', () => { }) }) }) + + describe('parseChunkerString', () => { + + it('handles an empty string', () => { + const options = utils.parseChunkerString("") + expect(options).to.have.property('chunker').to.equal("fixed") + }) + + it('handles a null chunker string', () => { + const options = utils.parseChunkerString(null) + expect(options).to.have.property('chunker').to.equal("fixed") + }) + + it('parses a fixed size string', () => { + const options = utils.parseChunkerString("size-512") + expect(options).to.have.property('chunker').to.equal("fixed") + expect(options) + .to.have.property('chunkerOptions') + .to.have.property('maxChunkSize') + .to.equal(512) + }) + + it('parses a rabin string without size', () => { + const options = utils.parseChunkerString("rabin") + expect(options).to.have.property('chunker').to.equal("rabin") + expect(options) + .to.have.property('chunkerOptions') + .to.have.property('avgChunkSize') + }) + + it('parses a rabin string with only avg size', () => { + const options = utils.parseChunkerString("rabin-512") + expect(options).to.have.property('chunker').to.equal("rabin") + expect(options) + .to.have.property('chunkerOptions') + .to.have.property('avgChunkSize') + .to.equal(512) + }) + + it('parses a rabin string with min, avg, and max', () => { + const options = utils.parseChunkerString("rabin-42-92-184") + expect(options).to.have.property('chunker').to.equal("rabin") + expect(options).to.have.property('chunkerOptions') + expect(options.chunkerOptions).to.have.property('minChunkSize').to.equal(42) + expect(options.chunkerOptions).to.have.property('avgChunkSize').to.equal(92) + expect(options.chunkerOptions).to.have.property('maxChunkSize').to.equal(184) + }) + + }) }) From 620cd5940014b6762804e0766d5db2a152748636 Mon Sep 17 00:00:00 2001 From: Dan Ordille Date: Wed, 25 Jul 2018 13:31:49 -0400 Subject: [PATCH 3/3] Remove choice option in add command --- src/cli/commands/files/add.js | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/cli/commands/files/add.js b/src/cli/commands/files/add.js index ae3d28da84..753169f630 100644 --- a/src/cli/commands/files/add.js +++ b/src/cli/commands/files/add.js @@ -137,14 +137,7 @@ module.exports = { }, 'chunker': { default: 'default', - describe: 'Chunking algorithm to use', - choices: [ - "default", - "size-{size}", - "rabin", - "rabin-{avg}", - "rabin-{min}-{avg}-{max}" - ] + describe: 'Chunking algorithm to use, formatted like [default, size-{size}, rabin, rabin-{avg}, rabin-{min}-{avg}-{max}]', }, 'enable-sharding-experiment': { type: 'boolean',