diff --git a/.travis.yml b/.travis.yml index e1d6320b..dd9e44ef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ language: node_js node_js: - 4 - 5 + - stable # Make sure we have new NPM. before_install: diff --git a/package.json b/package.json index e8442aa9..4cb384d5 100644 --- a/package.json +++ b/package.json @@ -2,17 +2,17 @@ "name": "ipfs-unixfs-engine", "version": "0.10.2", "description": "JavaScript implementation of the unixfs Engine used by IPFS", - "main": "src/index.js", + "main": "lib/index.js", "jsnext:main": "src/index.js", "scripts": { "lint": "aegir-lint", "build": "aegir-build", - "test": "aegir-test", - "test:node": "aegir-test node", - "test:browser": "aegir-test browser", - "release": "aegir-release", - "release-minor": "aegir-release --type minor", - "release-major": "aegir-release --type major", + "test": "PHANTOM=off aegir-test", + "test:node": "aegir-test --env node", + "test:browser": "PHANTOM=off aegir-test --env browser", + "release": "PHANTOM=off aegir-release", + "release-minor": "PHANTOM=off aegir-release --type minor", + "release-major": "PHANTOM=off aegir-release --type major", "coverage": "aegir-coverage", "coverage-publish": "aegir-coverage publish" }, @@ -22,7 +22,7 @@ ], "repository": { "type": "git", - "url": "git+https://github.com/ipfs/js-ipfs-data-importing.git" + "url": "git+https://github.com/ipfs/js-ipfs-unixfs-engine.git" }, "keywords": [ "IPFS" @@ -34,37 +34,31 @@ }, "homepage": "https://github.com/ipfs/js-ipfs-unixfs-engineg#readme", "devDependencies": { - "aegir": "^6.0.1", - "async": "^2.0.1", - "block-stream2": "^1.1.0", + "aegir": "^8.0.1", "buffer-loader": "0.0.1", "chai": "^3.5.0", - "concat-stream": "^1.5.1", - "fs-blob-store": "^5.2.1", - "idb-plus-blob-store": "^1.1.2", - "ipfs-repo": "^0.7.5", + "fs-pull-blob-store": "^0.3.0", + "idb-pull-blob-store": "^0.4.0", + "ipfs-block-service": "^0.5.0", + "ipfs-repo": "^0.9.0", "ncp": "^2.0.0", - "pre-commit": "^1.1.2", + "pre-commit": "^1.1.3", + "pull-zip": "^2.0.0", "raw-loader": "^0.5.1", - "rimraf": "^2.5.1", - "streamifier": "^0.1.1", - "ipfs-block-service": "^0.3.0", - "string-to-stream": "^1.0.1" + "rimraf": "^2.5.4", + "run-series": "^1.1.4" }, "dependencies": { - "block-stream2": "^1.1.0", - "bs58": "^3.0.0", - "debug": "^2.2.0", - "field-trip": "0.0.3", - "ipfs-merkle-dag": "^0.6.1", - "ipfs-unixfs": "^0.1.0", + "ipfs-merkle-dag": "^0.7.0", + "ipfs-unixfs": "^0.1.4", "is-ipfs": "^0.2.0", - "isstream": "^0.1.2", "multihashes": "^0.2.2", - "readable-stream": "^1.1.13", - "run-series": "^1.1.4", - "streamifier": "^0.1.1", - "through2": "^2.0.0" + "pull-block": "^1.0.2", + "pull-pushable": "^2.0.1", + "pull-stream": "^3.4.5", + "pull-traverse": "^1.0.3", + "pull-write": "^1.1.0", + "run-parallel": "^1.1.6" }, "contributors": [ "David Dias ", @@ -75,4 +69,4 @@ "greenkeeperio-bot ", "nginnever " ] -} \ No newline at end of file +} diff --git a/src/chunker-fixed-size.js b/src/chunker-fixed-size.js index 372ef519..2d0663ad 100644 --- a/src/chunker-fixed-size.js +++ b/src/chunker-fixed-size.js @@ -1,7 +1,7 @@ 'use strict' -const chunker = require('block-stream2') +const block = require('pull-block') exports = module.exports = function (size) { - return chunker({ size: size, zeroPadding: false }) + return block(size, {zeroPadding: false}) } diff --git a/src/clean-multihash.js b/src/clean-multihash.js deleted file mode 100644 index d30a3ecf..00000000 --- a/src/clean-multihash.js +++ /dev/null @@ -1,14 +0,0 @@ -'use strict' - -const mh = require('multihashes') -const isIPFS = require('is-ipfs') - -module.exports = function (multihash) { - if (!isIPFS.multihash(multihash)) { - throw new Error('not valid multihash') - } - if (Buffer.isBuffer(multihash)) { - return mh.toB58String(multihash) - } - return multihash -} diff --git a/src/exporter.js b/src/exporter.js index cabeb039..6489bd01 100644 --- a/src/exporter.js +++ b/src/exporter.js @@ -1,124 +1,39 @@ 'use strict' -const debug = require('debug') -const log = debug('unixfs') -log.err = debug('unixfs:error') -const isIPFS = require('is-ipfs') -const UnixFS = require('ipfs-unixfs') -const series = require('run-series') -const Readable = require('readable-stream').Readable -const pathj = require('path') -const util = require('util') -const fieldtrip = require('field-trip') -const cleanMultihash = require('./clean-multihash') +const traverse = require('pull-traverse') +const pull = require('pull-stream') -exports = module.exports = Exporter +const util = require('./util') +const switchType = util.switchType +const cleanMultihash = util.cleanMultihash -util.inherits(Exporter, Readable) +const dirExporter = require('./exporters/dir') +const fileExporter = require('./exporters/file') -function Exporter (hash, dagService, options) { - if (!(this instanceof Exporter)) { - return new Exporter(hash, dagService, options) - } - - // Sanitize hash - if (!isIPFS.multihash(hash)) { - throw new Error('not valid multihash') - } +module.exports = (hash, dagService, options) => { hash = cleanMultihash(hash) - - Readable.call(this, { objectMode: true }) - - this.options = options || {} - - this._read = (n) => {} - - let fileExporter = (node, name, done) => { - if (!done) { - throw new Error('done must be set') - } - - const contentRS = new Readable() - contentRS._read = () => {} - - // Logic to export a single (possibly chunked) unixfs file. - if (node.links.length === 0) { - const unmarshaledData = UnixFS.unmarshal(node.data) - contentRS.push(unmarshaledData.data) - contentRS.push(null) - this.push({ content: contentRS, path: name }) - done() - } else { - const array = node.links.map((link) => { - return (cb) => { - dagService.get(link.hash, (err, res) => { - if (err) { - return cb(err) - } - var unmarshaledData = UnixFS.unmarshal(res.data) - contentRS.push(unmarshaledData.data) - cb() - }) - } - }) - series(array, (err) => { - if (err) { - return contentRS.emit('error', err) - } - contentRS.push(null) - }) - this.push({ content: contentRS, path: name }) - done() - } - } - - // Logic to export a unixfs directory. - let dirExporter = (node, name, add, done) => { - if (!add) { - throw new Error('add must be set') - } - if (!done) { - throw new Error('done must be set') - } - - this.push({content: null, path: name}) - - // Directory has links - if (node.links.length > 0) { - node.links.forEach((link) => { - add({ path: pathj.join(name, link.name), hash: link.hash }) - }) - } - done() - } - - // Traverse the DAG asynchronously - fieldtrip([{path: hash, hash: hash}], visit.bind(this), (err) => { - if (err) { - return this.emit('error', err) - } - this.push(null) - }) - - // Visit function: called once per node in the exported graph - function visit (item, add, done) { - dagService.get(item.hash, (err, node) => { - if (err) { - return this.emit('error', err) - } - - const data = UnixFS.unmarshal(node.data) - const type = data.type - - if (type === 'directory') { - dirExporter(node, item.path, add, done) - } - - if (type === 'file') { - fileExporter(node, item.path, done) - } - }) + options = options || {} + + function visitor (item) { + return pull( + dagService.getStream(item.hash), + pull.map((node) => switchType( + node, + () => dirExporter(node, item.path, dagService), + () => fileExporter(node, item.path, dagService) + )), + pull.flatten() + ) } - return this + // Traverse the DAG + return pull( + dagService.getStream(hash), + pull.map((node) => switchType( + node, + () => traverse.widthFirst({path: hash, hash}, visitor), + () => fileExporter(node, hash, dagService) + )), + pull.flatten() + ) } diff --git a/src/exporters/dir.js b/src/exporters/dir.js new file mode 100644 index 00000000..98e01417 --- /dev/null +++ b/src/exporters/dir.js @@ -0,0 +1,28 @@ +'use strict' + +const path = require('path') +const pull = require('pull-stream') + +const fileExporter = require('./file') +const switchType = require('../util').switchType + +// Logic to export a unixfs directory. +module.exports = (node, name, dagService) => { + return pull( + pull.values(node.links), + pull.map((link) => ({ + path: path.join(name, link.name), + hash: link.hash + })), + pull.map((item) => pull( + dagService.getStream(item.hash), + pull.map((n) => switchType( + n, + () => pull.values([item]), + () => fileExporter(n, item.path, dagService) + )), + pull.flatten() + )), + pull.flatten() + ) +} diff --git a/src/exporters/file.js b/src/exporters/file.js new file mode 100644 index 00000000..c2993fcb --- /dev/null +++ b/src/exporters/file.js @@ -0,0 +1,30 @@ +'use strict' + +const UnixFS = require('ipfs-unixfs') +const pull = require('pull-stream') + +function extractContent (node) { + return UnixFS.unmarshal(node.data).data +} + +// Logic to export a single (possibly chunked) unixfs file. +module.exports = (node, name, ds) => { + let content + + if (node.links.length === 0) { + const c = extractContent(node) + content = pull.values([c]) + } else { + content = pull( + pull.values(node.links), + pull.map((link) => ds.getStream(link.hash)), + pull.flatten(), + pull.map(extractContent) + ) + } + + return pull.values([{ + content: content, + path: name + }]) +} diff --git a/src/importer.js b/src/importer.js index 019f454a..c832119f 100644 --- a/src/importer.js +++ b/src/importer.js @@ -1,304 +1,155 @@ 'use strict' -const debug = require('debug') -const log = debug('unixfs') -log.err = debug('unixfs:error') -const fsc = require('./chunker-fixed-size') -const through2 = require('through2') const merkleDAG = require('ipfs-merkle-dag') const UnixFS = require('ipfs-unixfs') -const util = require('util') -const bs58 = require('bs58') -const Duplex = require('readable-stream').Duplex -const isStream = require('isstream') -const streamifier = require('streamifier') - -exports = module.exports = Importer - -const CHUNK_SIZE = 262144 +const assert = require('assert') +const pull = require('pull-stream') +const pushable = require('pull-pushable') +const write = require('pull-write') +const parallel = require('run-parallel') -util.inherits(Importer, Duplex) +const fsc = require('./chunker-fixed-size') +const createAndStoreTree = require('./tree') -function Importer (dagService, options) { - Duplex.call(this, { objectMode: true }) +const DAGNode = merkleDAG.DAGNode - if (!(this instanceof Importer)) { - return new Importer(dagService) - } +const CHUNK_SIZE = 262144 - if (!dagService) { - return new Error('must specify a dagService') - } +module.exports = (dagService, options) => { + assert(dagService, 'Missing dagService') const files = [] - var counter = 0 - this._read = (n) => {} + const source = pushable() + const sink = write( + makeWriter(source, files, dagService), + null, + 100, + (err) => { + if (err) return source.end(err) - this._write = (fl, enc, next) => { - this.read() - counter++ - if (!fl.content) { - // 1. create the empty dir dag node - // 2. write it to the dag store - // 3. add to the files array {path: <>, hash: <>} - // 4. emit the path + hash - const d = new UnixFS('directory') - const n = new merkleDAG.DAGNode() - n.data = d.marshal() - dagService.add(n, (err) => { - if (err) { - this.emit('error', `Failed to store: ${fl.path}`) - return - } - const el = { - path: fl.path, - multihash: n.multihash(), - size: n.size(), - dataSize: d.fileSize() - } - files.push(el) - this.push(el) - counter-- - next() + createAndStoreTree(files, dagService, source, () => { + source.end() }) - return - } - - // Convert a buffer to a readable stream - if (Buffer.isBuffer(fl.content)) { - const r = streamifier.createReadStream(fl.content) - fl.content = r - } - - // Bail if 'content' is not readable - if (!isStream.isReadable(fl.content)) { - this.emit('error', new Error('"content" is not a Buffer nor Readable stream')) - return } + ) - const leaves = [] - fl.content - .pipe(fsc(CHUNK_SIZE)) - .pipe(through2((chunk, enc, cb) => { - // 1. create the unixfs merkledag node - // 2. add its hash and size to the leafs array - - // TODO - Support really large files - // a) check if we already reach max chunks if yes - // a.1) create a parent node for all of the current leaves - // b.2) clean up the leaves array and add just the parent node - - const l = new UnixFS('file', chunk) - const n = new merkleDAG.DAGNode(l.marshal()) - - dagService.add(n, (err) => { - if (err) { - this.push({error: 'Failed to store chunk of: ${fl.path}'}) - return cb(err) - } - - leaves.push({ - Hash: n.multihash(), - Size: n.size(), - leafSize: l.fileSize(), - Name: '' - }) + return {source, sink} +} +function makeWriter (source, files, dagService) { + return (items, cb) => { + parallel(items.map((item) => (cb) => { + if (!item.content) { + return createAndStoreDir(item, dagService, (err, node) => { + if (err) return cb(err) + source.push(node) + files.push(node) cb() }) - }, (cb) => { - if (leaves.length === 1) { - // 1. add to the files array {path: <>, hash: <>} - // 2. emit the path + hash - - const el = { - path: fl.path, - multihash: leaves[0].Hash, - size: leaves[0].Size, - dataSize: leaves[0].leafSize - } - - files.push(el) - this.push(el) - return done(cb) - } - // 1. create a parent node and add all the leafs - // 2. add to the files array {path: <>, hash: <>} - // 3. emit the path + hash of the parent node - - const f = new UnixFS('file') - const n = new merkleDAG.DAGNode() - - leaves.forEach((leaf) => { - f.addBlockSize(leaf.leafSize) - const l = new merkleDAG.DAGLink(leaf.Name, leaf.Size, leaf.Hash) - n.addRawLink(l) - }) + } - n.data = f.marshal() - dagService.add(n, (err) => { - if (err) { - // this.emit('error', `Failed to store: ${fl.path}`) - this.push({ error: 'Failed to store chunk of: ${fl.path}' }) - return cb() - } + createAndStoreFile(item, dagService, (err, node) => { + if (err) return cb(err) + source.push(node) + files.push(node) + cb() + }) + }), cb) + } +} - const el = { - path: fl.path, - multihash: n.multihash(), - size: n.size() - // dataSize: f.fileSize() - } +function createAndStoreDir (item, ds, cb) { + // 1. create the empty dir dag node + // 2. write it to the dag store + + const d = new UnixFS('directory') + const n = new DAGNode() + n.data = d.marshal() + + ds.put(n, (err) => { + if (err) return cb(err) + cb(null, { + path: item.path, + multihash: n.multihash(), + size: n.size() + // dataSize: d.fileSize() + }) + }) +} - files.push(el) - // this.emit('file', el) - this.push(el) - return done(cb) - }) - })) +function createAndStoreFile (file, ds, cb) { + if (Buffer.isBuffer(file.content)) { + file.content = pull.values([file.content]) + } - function done (cb) { - counter-- - next() - cb() - } + if (typeof file.content !== 'function') { + return cb(new Error('invalid content')) } - this.end = () => { - finish.call(this) + // 1. create the unixfs merkledag node + // 2. add its hash and size to the leafs array - function finish () { - if (counter > 0) { - return setTimeout(() => { - finish.call(this) - }, 200) - } - // file struct - // { - // path: // full path - // multihash: // multihash of the dagNode - // size: // cumulative size - // dataSize: // dagNode size - // } + // TODO - Support really large files + // a) check if we already reach max chunks if yes + // a.1) create a parent node for all of the current leaves + // b.2) clean up the leaves array and add just the parent node - // 1) convert files to a tree - // for each path, split, add to a json tree and in the end the name of the - // file points to an object that is has a key multihash and respective value - // { foo: { bar: { baz.txt: }}} - // the stop condition is if the value is not an object - const fileTree = {} - files.forEach((file) => { - let splitted = file.path.split('/') - if (splitted.length === 1) { - return // adding just one file - // fileTree[file.path] = bs58.encode(file.multihash).toString() - } - if (splitted[0] === '') { - splitted = splitted.slice(1) - } - var tmpTree = fileTree + pull( + file.content, + fsc(CHUNK_SIZE), + pull.asyncMap((chunk, cb) => { + const l = new UnixFS('file', Buffer(chunk)) + const n = new DAGNode(l.marshal()) - for (var i = 0; i < splitted.length; i++) { - if (!tmpTree[splitted[i]]) { - tmpTree[splitted[i]] = {} - } - if (i === splitted.length - 1) { - tmpTree[splitted[i]] = file.multihash - } else { - tmpTree = tmpTree[splitted[i]] - } + ds.put(n, (err) => { + if (err) { + return cb(new Error('Failed to store chunk')) } - }) - - if (Object.keys(fileTree).length === 0) { - this.push(null) - return // no dirs to be created - } - - // 2) create a index for multihash: { size, dataSize } so - // that we can fetch these when creating the merkle dag nodes - const mhIndex = {} - - files.forEach((file) => { - mhIndex[bs58.encode(file.multihash)] = { - size: file.size, - dataSize: file.dataSize - } + cb(null, { + Hash: n.multihash(), + Size: n.size(), + leafSize: l.fileSize(), + Name: '' + }) }) - - // 3) expand leaves recursively - // create a dirNode - // Object.keys - // If the value is an Object - // create a dir Node - // Object.keys - // Once finished, add the result as a link to the dir node - // If the value is not an object - // add as a link to the dirNode - - let pendingWrites = 0 - - function traverse (tree, path, done) { - const keys = Object.keys(tree) - let tmpTree = tree - keys.map((key) => { - if (typeof tmpTree[key] === 'object' && - !Buffer.isBuffer(tmpTree[key])) { - tmpTree[key] = traverse.call(this, tmpTree[key], path ? path + '/' + key : key, done) - } + }), + pull.collect((err, leaves) => { + if (err) return cb(err) + + if (leaves.length === 1) { + return cb(null, { + path: file.path, + multihash: leaves[0].Hash, + size: leaves[0].Size + // dataSize: leaves[0].leafSize }) + } - // at this stage, all keys are multihashes - // create a dir node - // add all the multihashes as links - // return this new node multihash - - const d = new UnixFS('directory') - const n = new merkleDAG.DAGNode() + // create a parent node and add all the leafs - keys.forEach((key) => { - const b58mh = bs58.encode(tmpTree[key]) - const l = new merkleDAG.DAGLink( - key, mhIndex[b58mh].size, tmpTree[key]) - n.addRawLink(l) - }) + const f = new UnixFS('file') + const n = new merkleDAG.DAGNode() - n.data = d.marshal() + for (let leaf of leaves) { + f.addBlockSize(leaf.leafSize) + n.addRawLink( + new merkleDAG.DAGLink(leaf.Name, leaf.Size, leaf.Hash) + ) + } - pendingWrites++ - dagService.add(n, (err) => { - pendingWrites-- - if (err) { - this.push({error: 'failed to store dirNode'}) - } else if (path) { - const el = { - path: path, - multihash: n.multihash(), - yes: 'no', - size: n.size() - } - this.push(el) - } + n.data = f.marshal() + ds.put(n, (err) => { + if (err) return cb(err) - if (pendingWrites <= 0) { - done() - } + cb(null, { + path: file.path, + multihash: n.multihash(), + size: n.size() + // dataSize: f.fileSize() }) - - if (!path) { - return - } - - mhIndex[bs58.encode(n.multihash())] = { size: n.size() } - return n.multihash() - } - - let self = this - /* const rootHash = */ traverse.call(this, fileTree, null, function () { - self.push(null) }) - } - } + }) + ) } diff --git a/src/tree.js b/src/tree.js new file mode 100644 index 00000000..45ff9539 --- /dev/null +++ b/src/tree.js @@ -0,0 +1,130 @@ +'use strict' + +const mh = require('multihashes') +const UnixFS = require('ipfs-unixfs') +const merkleDAG = require('ipfs-merkle-dag') + +const DAGLink = merkleDAG.DAGLink +const DAGNode = merkleDAG.DAGNode + +module.exports = (files, dagService, source, cb) => { + // file struct + // { + // path: // full path + // multihash: // multihash of the dagNode + // size: // cumulative size + // dataSize: // dagNode size + // } + + // 1) convert files to a tree + // for each path, split, add to a json tree and in the end the name of the + // file points to an object that is has a key multihash and respective value + // { foo: { bar: { baz.txt: }}} + // the stop condition is if the value is not an object + const fileTree = {} + files.forEach((file) => { + let splitted = file.path.split('/') + if (splitted.length === 1) { + return // adding just one file + // fileTree[file.path] = bs58.encode(file.multihash).toString() + } + if (splitted[0] === '') { + splitted = splitted.slice(1) + } + var tmpTree = fileTree + + for (var i = 0; i < splitted.length; i++) { + if (!tmpTree[splitted[i]]) { + tmpTree[splitted[i]] = {} + } + if (i === splitted.length - 1) { + tmpTree[splitted[i]] = file.multihash + } else { + tmpTree = tmpTree[splitted[i]] + } + } + }) + + if (Object.keys(fileTree).length === 0) { + return cb()// no dirs to be created + } + + // 2) create a index for multihash: { size, dataSize } so + // that we can fetch these when creating the merkle dag nodes + + const mhIndex = {} + + files.forEach((file) => { + mhIndex[mh.toB58String(file.multihash)] = { + size: file.size, + dataSize: file.dataSize + } + }) + + // 3) expand leaves recursively + // create a dirNode + // Object.keys + // If the value is an Object + // create a dir Node + // Object.keys + // Once finished, add the result as a link to the dir node + // If the value is not an object + // add as a link to the dirNode + + let pendingWrites = 0 + + function traverse (tree, path, done) { + const keys = Object.keys(tree) + let tmpTree = tree + keys.map((key) => { + if (typeof tmpTree[key] === 'object' && + !Buffer.isBuffer(tmpTree[key])) { + tmpTree[key] = traverse.call(this, tmpTree[key], path ? path + '/' + key : key, done) + } + }) + + // at this stage, all keys are multihashes + // create a dir node + // add all the multihashes as links + // return this new node multihash + + const d = new UnixFS('directory') + const n = new DAGNode() + + keys.forEach((key) => { + const b58mh = mh.toB58String(tmpTree[key]) + const l = new DAGLink( + key, mhIndex[b58mh].size, tmpTree[key]) + n.addRawLink(l) + }) + + n.data = d.marshal() + + pendingWrites++ + dagService.put(n, (err) => { + pendingWrites-- + if (err) { + source.push(new Error('failed to store dirNode')) + } else if (path) { + source.push({ + path: path, + multihash: n.multihash(), + size: n.size() + }) + } + + if (pendingWrites <= 0) { + done() + } + }) + + if (!path) { + return + } + + mhIndex[mh.toB58String(n.multihash())] = { size: n.size() } + return n.multihash() + } + + traverse(fileTree, null, cb) +} diff --git a/src/util.js b/src/util.js new file mode 100644 index 00000000..8f44ae01 --- /dev/null +++ b/src/util.js @@ -0,0 +1,28 @@ +'use strict' + +const UnixFS = require('ipfs-unixfs') +const pull = require('pull-stream') +const mh = require('multihashes') +const isIPFS = require('is-ipfs') + +exports.switchType = (node, dirHandler, fileHandler) => { + const data = UnixFS.unmarshal(node.data) + const type = data.type + + if (type === 'directory') return dirHandler() + if (type === 'file') return fileHandler() + + return pull.error(new Error('Unkown node type')) +} + +exports.cleanMultihash = (multihash) => { + if (!isIPFS.multihash(multihash)) { + throw new Error('not valid multihash') + } + + if (Buffer.isBuffer(multihash)) { + return mh.toB58String(multihash) + } + + return multihash +} diff --git a/test/browser.js b/test/browser.js index 70143995..60503760 100644 --- a/test/browser.js +++ b/test/browser.js @@ -1,11 +1,10 @@ /* eslint-env mocha */ 'use strict' -const eachSeries = require('async/eachSeries') -const store = require('idb-plus-blob-store') -const _ = require('lodash') +const Store = require('idb-pull-blob-store') const IPFSRepo = require('ipfs-repo') const repoContext = require.context('buffer!./repo-example', true) +const pull = require('pull-stream') const idb = window.indexedDB || window.mozIndexedDB || @@ -26,26 +25,31 @@ describe('IPFS data importing tests on the Browser', function () { }) }) - const mainBlob = store('ipfs') - const blocksBlob = store('ipfs/blocks') - - eachSeries(repoData, (file, cb) => { - if (_.startsWith(file.key, 'datastore/')) { - return cb() - } - - const blocks = _.startsWith(file.key, 'blocks/') - const blob = blocks ? blocksBlob : mainBlob - const key = blocks ? file.key.replace(/^blocks\//, '') : file.key - - blob.createWriteStream({ - key: key - }).end(file.value, cb) - }, done) + const mainBlob = new Store('ipfs') + const blocksBlob = new Store('ipfs/blocks') + + pull( + pull.values(repoData), + pull.asyncMap((file, cb) => { + if (file.key.indexOf('datastore/') === 0) { + return cb() + } + + const blocks = file.key.indexOf('blocks/') === 0 + const blob = blocks ? blocksBlob : mainBlob + const key = blocks ? file.key.replace(/^blocks\//, '') : file.key + + pull( + pull.values([file.value]), + blob.write(key, cb) + ) + }), + pull.onEnd(done) + ) }) // create the repo constant to be used in the import a small buffer test - const repo = new IPFSRepo('ipfs', {stores: store}) + const repo = new IPFSRepo('ipfs', {stores: Store}) require('./test-exporter')(repo) require('./test-importer')(repo) diff --git a/test/node.js b/test/node.js index 1d4a9edf..33c81a2a 100644 --- a/test/node.js +++ b/test/node.js @@ -1,13 +1,13 @@ /* eslint-env mocha */ 'use strict' -const fs = require('fs') const ncp = require('ncp').ncp const rimraf = require('rimraf') -const expect = require('chai').expect const path = require('path') const IPFSRepo = require('ipfs-repo') -const fsbs = require('fs-blob-store') +const Store = require('fs-pull-blob-store') +const mkdirp = require('mkdirp') +const series = require('run-series') describe('core', () => { const repoExample = path.join(process.cwd(), '/test/repo-example') @@ -16,34 +16,26 @@ describe('core', () => { before((done) => { ncp(repoExample, repoTests, (err) => { process.env.IPFS_PATH = repoTests - expect(err).to.equal(null) - done() + done(err) }) }) before((done) => { - fs.stat(path.join(__dirname, '/test-data/dir-nested/dir-another'), (err, exists) => { - if (err) { - fs.mkdirSync(path.join(__dirname, '/test-data/dir-nested/dir-another')) - } - }) + const paths = [ + 'test-data/dir-nested/dir-another', + 'test-data/dir-nested/level-1/level-2' + ] - fs.stat(path.join(__dirname, '/test-data/dir-nested/level-1/level-2'), (err, exists) => { - if (err) { - fs.mkdirSync(path.join(__dirname, '/test-data/dir-nested/level-1/level-2')) - } - done() - }) + series(paths.map((p) => (cb) => { + mkdirp(path.join(__dirname, p), cb) + }), done) }) after((done) => { - rimraf(repoTests, (err) => { - expect(err).to.equal(null) - done() - }) + rimraf(repoTests, done) }) - const repo = new IPFSRepo(repoTests, {stores: fsbs}) + const repo = new IPFSRepo(repoTests, {stores: Store}) require('./test-exporter')(repo) require('./test-importer')(repo) require('./test-fixed-size-chunker') diff --git a/test/repo-example/blocks/12200d06/12200d06d4afb85a411662dc882c52c9c79e7422bec62c066f8215705880b6d3a29c.data b/test/repo-example/blocks/CIQA2/CIQA2BWUV64FUQIWMLOIQLCSZHDZ45BCX3DCYBTPQIKXAWEAW3J2FHA.data similarity index 100% rename from test/repo-example/blocks/12200d06/12200d06d4afb85a411662dc882c52c9c79e7422bec62c066f8215705880b6d3a29c.data rename to test/repo-example/blocks/CIQA2/CIQA2BWUV64FUQIWMLOIQLCSZHDZ45BCX3DCYBTPQIKXAWEAW3J2FHA.data diff --git a/test/repo-example/blocks/12200e72/12200e725b1743efb7d00acec61eaf7ba84fafc2a0443cd606301d8018bb79d7b41e.data b/test/repo-example/blocks/CIQA4/CIQA44S3C5B67N6QBLHMMHVPPOUE7L6CUBCDZVQGGAOYAGF3PHL3IHQ.data similarity index 100% rename from test/repo-example/blocks/12200e72/12200e725b1743efb7d00acec61eaf7ba84fafc2a0443cd606301d8018bb79d7b41e.data rename to test/repo-example/blocks/CIQA4/CIQA44S3C5B67N6QBLHMMHVPPOUE7L6CUBCDZVQGGAOYAGF3PHL3IHQ.data diff --git a/test/repo-example/blocks/12200046/12200046ea932273bc69bf22ccd57a2837eee93896de64e70099724ae333c67eebae.data b/test/repo-example/blocks/CIQAA/CIQAARXKSMRHHPDJX4RMZVL2FA3652JYS3PGJZYATFZEVYZTYZ7OXLQ.data similarity index 100% rename from test/repo-example/blocks/12200046/12200046ea932273bc69bf22ccd57a2837eee93896de64e70099724ae333c67eebae.data rename to test/repo-example/blocks/CIQAA/CIQAARXKSMRHHPDJX4RMZVL2FA3652JYS3PGJZYATFZEVYZTYZ7OXLQ.data diff --git a/test/repo-example/blocks/122000e5/122000e508d684a83e258b5230e5791d6c35dc3c287dbcc8ea26bb3bcf3d7c4ad942.data b/test/repo-example/blocks/CIQAB/CIQABZII22CKQPRFRNJDBZLZDVWDLXB4FB63ZSHKE25TXTZ5PRFNSQQ.data similarity index 100% rename from test/repo-example/blocks/122000e5/122000e508d684a83e258b5230e5791d6c35dc3c287dbcc8ea26bb3bcf3d7c4ad942.data rename to test/repo-example/blocks/CIQAB/CIQABZII22CKQPRFRNJDBZLZDVWDLXB4FB63ZSHKE25TXTZ5PRFNSQQ.data diff --git a/test/repo-example/blocks/1220016d/1220016d22c790b4e50495df66eff6361276442eff53b5457cf8de5c2c6726325ca7.data b/test/repo-example/blocks/CIQAC/CIQAC3JCY6ILJZIESXPWN37WGYJHMRBO75J3KRL47DPFYLDHEYZFZJY.data similarity index 100% rename from test/repo-example/blocks/1220016d/1220016d22c790b4e50495df66eff6361276442eff53b5457cf8de5c2c6726325ca7.data rename to test/repo-example/blocks/CIQAC/CIQAC3JCY6ILJZIESXPWN37WGYJHMRBO75J3KRL47DPFYLDHEYZFZJY.data diff --git a/test/repo-example/blocks/1220016d/1220016dad5f5c19d1cad0382bdc72ee3c35ac5216bce4d54d875151dd67333a700c.data b/test/repo-example/blocks/CIQAC/CIQAC3NNL5OBTUOK2A4CXXDS5Y6DLLCSC26OJVKNQ5IVDXLHGM5HADA.data similarity index 100% rename from test/repo-example/blocks/1220016d/1220016dad5f5c19d1cad0382bdc72ee3c35ac5216bce4d54d875151dd67333a700c.data rename to test/repo-example/blocks/CIQAC/CIQAC3NNL5OBTUOK2A4CXXDS5Y6DLLCSC26OJVKNQ5IVDXLHGM5HADA.data diff --git a/test/repo-example/blocks/1220039c/1220039c0842ef4f653b86630496e6d686b63271de581cedfb1de84bdea6f504ec12.data b/test/repo-example/blocks/CIQAH/CIQAHHAIILXU6ZJ3QZRQJFXG22DLMMTR3ZMBZ3P3DXUEXXVG6UCOYEQ.data similarity index 100% rename from test/repo-example/blocks/1220039c/1220039c0842ef4f653b86630496e6d686b63271de581cedfb1de84bdea6f504ec12.data rename to test/repo-example/blocks/CIQAH/CIQAHHAIILXU6ZJ3QZRQJFXG22DLMMTR3ZMBZ3P3DXUEXXVG6UCOYEQ.data diff --git a/test/repo-example/blocks/12200975/12200975fdafa3ecdb026118837fe67a9ed6ed11ef5aacd61a516cddf519b1cb56e1.data b/test/repo-example/blocks/CIQAS/CIQAS5P5V6R6ZWYCMEMIG77GPKPNN3IR55NKZVQ2KFWN35IZWHFVNYI.data similarity index 100% rename from test/repo-example/blocks/12200975/12200975fdafa3ecdb026118837fe67a9ed6ed11ef5aacd61a516cddf519b1cb56e1.data rename to test/repo-example/blocks/CIQAS/CIQAS5P5V6R6ZWYCMEMIG77GPKPNN3IR55NKZVQ2KFWN35IZWHFVNYI.data diff --git a/test/repo-example/blocks/12200b0a/12200b0a5998ef9632814f333e81fc82bed3690292d0d0058833abb2af46af5a1c91.data b/test/repo-example/blocks/CIQAW/CIQAWCSZTDXZMMUBJ4ZT5AP4QK7NG2ICSLINABMIGOV3FL2GV5NBZEI.data similarity index 100% rename from test/repo-example/blocks/12200b0a/12200b0a5998ef9632814f333e81fc82bed3690292d0d0058833abb2af46af5a1c91.data rename to test/repo-example/blocks/CIQAW/CIQAWCSZTDXZMMUBJ4ZT5AP4QK7NG2ICSLINABMIGOV3FL2GV5NBZEI.data diff --git a/test/repo-example/blocks/12201dab/12201dab1bf33e76651d0ae2f63dbf5b9e71e979b34c1cc7a19b8276d8c11966027c.data b/test/repo-example/blocks/CIQB3/CIQB3KY36M7HMZI5BLRPMPN7LOPHD2LZWNGBZR5BTOBHNWGBDFTAE7A.data similarity index 100% rename from test/repo-example/blocks/12201dab/12201dab1bf33e76651d0ae2f63dbf5b9e71e979b34c1cc7a19b8276d8c11966027c.data rename to test/repo-example/blocks/CIQB3/CIQB3KY36M7HMZI5BLRPMPN7LOPHD2LZWNGBZR5BTOBHNWGBDFTAE7A.data diff --git a/test/repo-example/blocks/1220120f/1220120f6af601d46e10b2d2e11ed71c55d25f3042c22501e41d1246e7a1e9d3d8ec.data b/test/repo-example/blocks/CIQBE/CIQBED3K6YA5I3QQWLJOCHWXDRK5EXZQILBCKAPEDUJENZ5B5HJ5R3A.data similarity index 100% rename from test/repo-example/blocks/1220120f/1220120f6af601d46e10b2d2e11ed71c55d25f3042c22501e41d1246e7a1e9d3d8ec.data rename to test/repo-example/blocks/CIQBE/CIQBED3K6YA5I3QQWLJOCHWXDRK5EXZQILBCKAPEDUJENZ5B5HJ5R3A.data diff --git a/test/repo-example/blocks/1220141a/1220141a2aa747a6b67ff33e0f68be055ed1dde2f90350090a781b6bed84cf4ae810.data b/test/repo-example/blocks/CIQBI/CIQBIGRKU5D2NNT76M7A62F6AVPNDXPC7EBVACIKPANWX3MEZ5FOQEA.data similarity index 100% rename from test/repo-example/blocks/1220141a/1220141a2aa747a6b67ff33e0f68be055ed1dde2f90350090a781b6bed84cf4ae810.data rename to test/repo-example/blocks/CIQBI/CIQBIGRKU5D2NNT76M7A62F6AVPNDXPC7EBVACIKPANWX3MEZ5FOQEA.data diff --git a/test/repo-example/blocks/1220184e/1220184e2152a396caf16f41a1cc4bec30817fb2ab953ca645c57e83ac431cfc8a6a.data b/test/repo-example/blocks/CIQBQ/CIQBQTRBKKRZNSXRN5A2DTCL5QYIC75SVOKTZJSFYV7IHLCDDT6IU2Q.data similarity index 100% rename from test/repo-example/blocks/1220184e/1220184e2152a396caf16f41a1cc4bec30817fb2ab953ca645c57e83ac431cfc8a6a.data rename to test/repo-example/blocks/CIQBQ/CIQBQTRBKKRZNSXRN5A2DTCL5QYIC75SVOKTZJSFYV7IHLCDDT6IU2Q.data diff --git a/test/repo-example/blocks/12202d19/12202d198f2071c2e3cb56d40ac3ca78f5129291807dfd1f9083ce6b3be74e408534.data b/test/repo-example/blocks/CIQC2/CIQC2GMPEBY4FY6LK3KAVQ6KPD2RFEURQB672H4QQPHGWO7HJZAIKNA.data similarity index 100% rename from test/repo-example/blocks/12202d19/12202d198f2071c2e3cb56d40ac3ca78f5129291807dfd1f9083ce6b3be74e408534.data rename to test/repo-example/blocks/CIQC2/CIQC2GMPEBY4FY6LK3KAVQ6KPD2RFEURQB672H4QQPHGWO7HJZAIKNA.data diff --git a/test/repo-example/blocks/12202d19/12202d19d02a8747aaafe5db0ad7ecee53c63fbf4119bb6acaaf8cba3de8ccf3770a.data b/test/repo-example/blocks/CIQC2/CIQC2GOQFKDUPKVP4XNQVV7M5ZJ4MP57IEM3W2WKV6GLUPPIZTZXOCQ.data similarity index 100% rename from test/repo-example/blocks/12202d19/12202d19d02a8747aaafe5db0ad7ecee53c63fbf4119bb6acaaf8cba3de8ccf3770a.data rename to test/repo-example/blocks/CIQC2/CIQC2GOQFKDUPKVP4XNQVV7M5ZJ4MP57IEM3W2WKV6GLUPPIZTZXOCQ.data diff --git a/test/repo-example/blocks/12202d9b/12202d9bdeee51833a1816d90a2d46406e4714193ab9e47690dbbe42260fe16de8d9.data b/test/repo-example/blocks/CIQC3/CIQC3G665ZIYGOQYC3MQULKGIBXEOFAZHK46I5UQ3O7EEJQP4FW6RWI.data similarity index 100% rename from test/repo-example/blocks/12202d9b/12202d9bdeee51833a1816d90a2d46406e4714193ab9e47690dbbe42260fe16de8d9.data rename to test/repo-example/blocks/CIQC3/CIQC3G665ZIYGOQYC3MQULKGIBXEOFAZHK46I5UQ3O7EEJQP4FW6RWI.data diff --git a/test/repo-example/blocks/1220204e/1220204e693ccb04abd065623b9f2182eb3b9e398e8db8f5d4d9be789213c2b6a9aa.data b/test/repo-example/blocks/CIQCA/CIQCATTJHTFQJK6QMVRDXHZBQLVTXHRZR2G3R5OU3G7HREQTYK3KTKQ.data similarity index 100% rename from test/repo-example/blocks/1220204e/1220204e693ccb04abd065623b9f2182eb3b9e398e8db8f5d4d9be789213c2b6a9aa.data rename to test/repo-example/blocks/CIQCA/CIQCATTJHTFQJK6QMVRDXHZBQLVTXHRZR2G3R5OU3G7HREQTYK3KTKQ.data diff --git a/test/repo-example/blocks/122023e3/122023e3cf165402916213caef9870f26b75881cf86c1e7c7204fa35b55917021aa7.data b/test/repo-example/blocks/CIQCH/CIQCHY6PCZKAFELCCPFO7GDQ6JVXLCA47BWB47DSAT5DLNKZC4BBVJY.data similarity index 100% rename from test/repo-example/blocks/122023e3/122023e3cf165402916213caef9870f26b75881cf86c1e7c7204fa35b55917021aa7.data rename to test/repo-example/blocks/CIQCH/CIQCHY6PCZKAFELCCPFO7GDQ6JVXLCA47BWB47DSAT5DLNKZC4BBVJY.data diff --git a/test/repo-example/blocks/12202418/12202418e7fe47e72ca241ee347bb0afcf78845a2f2e1a9b8aa0a70403446013f817.data b/test/repo-example/blocks/CIQCI/CIQCIGHH7ZD6OLFCIHXDI65QV7HXRBC2F4XBVG4KUCTQIA2EMAJ7QFY.data similarity index 100% rename from test/repo-example/blocks/12202418/12202418e7fe47e72ca241ee347bb0afcf78845a2f2e1a9b8aa0a70403446013f817.data rename to test/repo-example/blocks/CIQCI/CIQCIGHH7ZD6OLFCIHXDI65QV7HXRBC2F4XBVG4KUCTQIA2EMAJ7QFY.data diff --git a/test/repo-example/blocks/12202562/12202562b552a79e9ff1184ee0c7fa50f39fe3810564c6a261b20cd04205372941a6.data b/test/repo-example/blocks/CIQCK/CIQCKYVVKKTZ5H7RDBHOBR72KDZZ7Y4BAVSMNITBWIGNAQQFG4UUDJQ.data similarity index 100% rename from test/repo-example/blocks/12202562/12202562b552a79e9ff1184ee0c7fa50f39fe3810564c6a261b20cd04205372941a6.data rename to test/repo-example/blocks/CIQCK/CIQCKYVVKKTZ5H7RDBHOBR72KDZZ7Y4BAVSMNITBWIGNAQQFG4UUDJQ.data diff --git a/test/repo-example/blocks/1220259c/1220259cae55bae8fa6c5b8945839ac89a8a1fa03369f8f5c204913a2913905fad04.data b/test/repo-example/blocks/CIQCL/CIQCLHFOKW5OR6TMLOEULA42ZCNIUH5AGNU7R5OCASITUKITSBP22BA.data similarity index 100% rename from test/repo-example/blocks/1220259c/1220259cae55bae8fa6c5b8945839ac89a8a1fa03369f8f5c204913a2913905fad04.data rename to test/repo-example/blocks/CIQCL/CIQCLHFOKW5OR6TMLOEULA42ZCNIUH5AGNU7R5OCASITUKITSBP22BA.data diff --git a/test/repo-example/blocks/122028d0/122028d0abf61304b10a47837f9f33d87304d79d20d50b8b00a127b8a4fd18e9b237.data b/test/repo-example/blocks/CIQCR/CIQCRUFL6YJQJMIKI6BX7HZT3BZQJV45EDKQXCYAUET3RJH5DDU3ENY.data similarity index 100% rename from test/repo-example/blocks/122028d0/122028d0abf61304b10a47837f9f33d87304d79d20d50b8b00a127b8a4fd18e9b237.data rename to test/repo-example/blocks/CIQCR/CIQCRUFL6YJQJMIKI6BX7HZT3BZQJV45EDKQXCYAUET3RJH5DDU3ENY.data diff --git a/test/repo-example/blocks/12202a59/12202a59b6b978159fe9c8457654d80cba54bcc9f5bd27eac8cf15f7a6ad271174ed.data b/test/repo-example/blocks/CIQCU/CIQCUWNWXF4BLH7JZBCXMVGYBS5FJPGJ6W6SP2WIZ4K7PJVNE4IXJ3I.data similarity index 100% rename from test/repo-example/blocks/12202a59/12202a59b6b978159fe9c8457654d80cba54bcc9f5bd27eac8cf15f7a6ad271174ed.data rename to test/repo-example/blocks/CIQCU/CIQCUWNWXF4BLH7JZBCXMVGYBS5FJPGJ6W6SP2WIZ4K7PJVNE4IXJ3I.data diff --git a/test/repo-example/blocks/12202cb7/12202cb76f3bebf2a211cea2c40a935710ce084e5293cec018d315be10447b6b6b71.data b/test/repo-example/blocks/CIQCZ/CIQCZN3PHPV7FIQRZ2RMICUTK4IM4CCOKKJ45QAY2MK34ECEPNVWW4I.data similarity index 100% rename from test/repo-example/blocks/12202cb7/12202cb76f3bebf2a211cea2c40a935710ce084e5293cec018d315be10447b6b6b71.data rename to test/repo-example/blocks/CIQCZ/CIQCZN3PHPV7FIQRZ2RMICUTK4IM4CCOKKJ45QAY2MK34ECEPNVWW4I.data diff --git a/test/repo-example/blocks/12203ff8/12203ff87b7c43c3fa04f0be34df5ae62f82914b235e731c897c33fd6f3c5f11406d.data b/test/repo-example/blocks/CIQD7/CIQD76D3PRB4H6QE6C7DJX224YXYFEKLENPHGHEJPQZ723Z4L4IUA3I.data similarity index 100% rename from test/repo-example/blocks/12203ff8/12203ff87b7c43c3fa04f0be34df5ae62f82914b235e731c897c33fd6f3c5f11406d.data rename to test/repo-example/blocks/CIQD7/CIQD76D3PRB4H6QE6C7DJX224YXYFEKLENPHGHEJPQZ723Z4L4IUA3I.data diff --git a/test/repo-example/blocks/122031d6/122031d6da265092f1b03fec969243fdcf095c1d219356cdf538ffce705a52d5738d.data b/test/repo-example/blocks/CIQDD/CIQDDVW2EZIJF4NQH7WJNESD7XHQSXA5EGJVNTPVHD7444C2KLKXHDI.data similarity index 100% rename from test/repo-example/blocks/122031d6/122031d6da265092f1b03fec969243fdcf095c1d219356cdf538ffce705a52d5738d.data rename to test/repo-example/blocks/CIQDD/CIQDDVW2EZIJF4NQH7WJNESD7XHQSXA5EGJVNTPVHD7444C2KLKXHDI.data diff --git a/test/repo-example/blocks/122031e7/122031e7a41c15d03feb8cd793c3348ea3b310512d7767a9abfbd7a928a85e977173.data b/test/repo-example/blocks/CIQDD/CIQDDZ5EDQK5AP7LRTLZHQZUR2R3GECRFV3WPKNL7PL2SKFIL2LXC4Y.data similarity index 100% rename from test/repo-example/blocks/122031e7/122031e7a41c15d03feb8cd793c3348ea3b310512d7767a9abfbd7a928a85e977173.data rename to test/repo-example/blocks/CIQDD/CIQDDZ5EDQK5AP7LRTLZHQZUR2R3GECRFV3WPKNL7PL2SKFIL2LXC4Y.data diff --git a/test/repo-example/blocks/12203232/12203232f27e1ad4cfa27eabe7300c92b75037a5eeba0a40bc776472c88d2abcc760.data b/test/repo-example/blocks/CIQDE/CIQDEMXSPYNNJT5CP2V6OMAMSK3VAN5F525AUQF4O5SHFSENFK6MOYA.data similarity index 100% rename from test/repo-example/blocks/12203232/12203232f27e1ad4cfa27eabe7300c92b75037a5eeba0a40bc776472c88d2abcc760.data rename to test/repo-example/blocks/CIQDE/CIQDEMXSPYNNJT5CP2V6OMAMSK3VAN5F525AUQF4O5SHFSENFK6MOYA.data diff --git a/test/repo-example/blocks/1220350a/1220350acf62bdbd344db9a98e4e7cf76710b329ede24582feef6db74baafe4f500d.data b/test/repo-example/blocks/CIQDK/CIQDKCWPMK632NCNXGUY4TT465TRBMZJ5XRELAX655W3OS5K7ZHVADI.data similarity index 100% rename from test/repo-example/blocks/1220350a/1220350acf62bdbd344db9a98e4e7cf76710b329ede24582feef6db74baafe4f500d.data rename to test/repo-example/blocks/CIQDK/CIQDKCWPMK632NCNXGUY4TT465TRBMZJ5XRELAX655W3OS5K7ZHVADI.data diff --git a/test/repo-example/blocks/12203628/12203628a4a19525dd84bbbffe132ec0b0d3be3528fbbcc814feb7f2fbf71ca06162.data b/test/repo-example/blocks/CIQDM/CIQDMKFEUGKSLXMEXO774EZOYCYNHPRVFD53ZSAU7237F67XDSQGCYQ.data similarity index 100% rename from test/repo-example/blocks/12203628/12203628a4a19525dd84bbbffe132ec0b0d3be3528fbbcc814feb7f2fbf71ca06162.data rename to test/repo-example/blocks/CIQDM/CIQDMKFEUGKSLXMEXO774EZOYCYNHPRVFD53ZSAU7237F67XDSQGCYQ.data diff --git a/test/repo-example/blocks/122036be/122036be591a509e38c1df109430bc6d6a63615735efc1ab3b317aee5e1d9888ac75.data b/test/repo-example/blocks/CIQDN/CIQDNPSZDJIJ4OGB34IJIMF4NVVGGYKXGXX4DKZ3GF5O4XQ5TCEKY5I.data similarity index 100% rename from test/repo-example/blocks/122036be/122036be591a509e38c1df109430bc6d6a63615735efc1ab3b317aee5e1d9888ac75.data rename to test/repo-example/blocks/CIQDN/CIQDNPSZDJIJ4OGB34IJIMF4NVVGGYKXGXX4DKZ3GF5O4XQ5TCEKY5I.data diff --git a/test/repo-example/blocks/12203706/12203706326544f96d4bafa6ab0bd2a1efd89ba605b2ee188db80354e372f42637fd.data b/test/repo-example/blocks/CIQDO/CIQDOBRSMVCPS3KLV6TKWC6SUHX5RG5GAWZO4GENXABVJY3S6QTDP7I.data similarity index 100% rename from test/repo-example/blocks/12203706/12203706326544f96d4bafa6ab0bd2a1efd89ba605b2ee188db80354e372f42637fd.data rename to test/repo-example/blocks/CIQDO/CIQDOBRSMVCPS3KLV6TKWC6SUHX5RG5GAWZO4GENXABVJY3S6QTDP7I.data diff --git a/test/repo-example/blocks/12203b5c/12203b5c1250bc4f33b8a1ecb29363eba288017410ee5300cdb50615b0e2693edac6.data b/test/repo-example/blocks/CIQDW/CIQDWXASKC6E6M5YUHWLFE3D5ORIQALUCDXFGAGNWUDBLMHCNE7NVRQ.data similarity index 100% rename from test/repo-example/blocks/12203b5c/12203b5c1250bc4f33b8a1ecb29363eba288017410ee5300cdb50615b0e2693edac6.data rename to test/repo-example/blocks/CIQDW/CIQDWXASKC6E6M5YUHWLFE3D5ORIQALUCDXFGAGNWUDBLMHCNE7NVRQ.data diff --git a/test/repo-example/blocks/12204f7b/12204f7b3de7c738dd8c5eaeba868fa71f6e1cd8d9dae9eb43d7e562597f0b38a8dc.data b/test/repo-example/blocks/CIQE6/CIQE66Z547DTRXMML2XLVBUPU4PW4HGY3HNOT22D27SWEWL7BM4KRXA.data similarity index 100% rename from test/repo-example/blocks/12204f7b/12204f7b3de7c738dd8c5eaeba868fa71f6e1cd8d9dae9eb43d7e562597f0b38a8dc.data rename to test/repo-example/blocks/CIQE6/CIQE66Z547DTRXMML2XLVBUPU4PW4HGY3HNOT22D27SWEWL7BM4KRXA.data diff --git a/test/repo-example/blocks/122040f1/122040f13ab05f941ce841b8b28b7578a8bfa228153c370f81b4bc62ba0d66640555.data b/test/repo-example/blocks/CIQEB/CIQEB4J2WBPZIHHIIG4LFC3VPCUL7IRICU6DOD4BWS6GFOQNMZSAKVI.data similarity index 100% rename from test/repo-example/blocks/122040f1/122040f13ab05f941ce841b8b28b7578a8bfa228153c370f81b4bc62ba0d66640555.data rename to test/repo-example/blocks/CIQEB/CIQEB4J2WBPZIHHIIG4LFC3VPCUL7IRICU6DOD4BWS6GFOQNMZSAKVI.data diff --git a/test/repo-example/blocks/1220414f/1220414ff3a99054999668b5934af1c42e578399a9485c4d19e8b2b65ba5dd723aab.data b/test/repo-example/blocks/CIQEC/CIQECT7TVGIFJGMWNC2ZGSXRYQXFPA4ZVFEFYTIZ5CZLMW5F3VZDVKY.data similarity index 100% rename from test/repo-example/blocks/1220414f/1220414ff3a99054999668b5934af1c42e578399a9485c4d19e8b2b65ba5dd723aab.data rename to test/repo-example/blocks/CIQEC/CIQECT7TVGIFJGMWNC2ZGSXRYQXFPA4ZVFEFYTIZ5CZLMW5F3VZDVKY.data diff --git a/test/repo-example/blocks/122044fb/122044fbd45a9685c6b9d7f332982816e81aacdcfb0b7a742d7ce8d70fd3d62972b9.data b/test/repo-example/blocks/CIQEJ/CIQEJ66ULKLILRVZ27ZTFGBIC3UBVLG47MFXU5BNPTUNOD6T2YUXFOI.data similarity index 100% rename from test/repo-example/blocks/122044fb/122044fbd45a9685c6b9d7f332982816e81aacdcfb0b7a742d7ce8d70fd3d62972b9.data rename to test/repo-example/blocks/CIQEJ/CIQEJ66ULKLILRVZ27ZTFGBIC3UBVLG47MFXU5BNPTUNOD6T2YUXFOI.data diff --git a/test/repo-example/blocks/12204849/12204849953dbcc44598f39816866c6d96355f33eaf1004d61ab1880fdeb3edde2f0.data b/test/repo-example/blocks/CIQEQ/CIQEQSMVHW6MIRMY6OMBNBTMNWLDKXZT5LYQATLBVMMIB7PLH3O6F4A.data similarity index 100% rename from test/repo-example/blocks/12204849/12204849953dbcc44598f39816866c6d96355f33eaf1004d61ab1880fdeb3edde2f0.data rename to test/repo-example/blocks/CIQEQ/CIQEQSMVHW6MIRMY6OMBNBTMNWLDKXZT5LYQATLBVMMIB7PLH3O6F4A.data diff --git a/test/repo-example/blocks/12204a5a/12204a5a95586f52e25811cf214677160e64383755a8c5163ba3c053c3b65777ed16.data b/test/repo-example/blocks/CIQEU/CIQEUWUVLBXVFYSYCHHSCRTXCYHGIOBXKWUMKFR3UPAFHQ5WK5362FQ.data similarity index 100% rename from test/repo-example/blocks/12204a5a/12204a5a95586f52e25811cf214677160e64383755a8c5163ba3c053c3b65777ed16.data rename to test/repo-example/blocks/CIQEU/CIQEUWUVLBXVFYSYCHHSCRTXCYHGIOBXKWUMKFR3UPAFHQ5WK5362FQ.data diff --git a/test/repo-example/blocks/122051b6/122051b6a7a8ac28d2eeddbf4339f78c0220c3023aeabc9a4e58927f30fddce8b505.data b/test/repo-example/blocks/CIQFD/CIQFDNVHVCWCRUXO3W7UGOPXRQBCBQYCHLVLZGSOLCJH6MH53TULKBI.data similarity index 100% rename from test/repo-example/blocks/122051b6/122051b6a7a8ac28d2eeddbf4339f78c0220c3023aeabc9a4e58927f30fddce8b505.data rename to test/repo-example/blocks/CIQFD/CIQFDNVHVCWCRUXO3W7UGOPXRQBCBQYCHLVLZGSOLCJH6MH53TULKBI.data diff --git a/test/repo-example/blocks/12205200/12205200cc6b6f79e1588480d9f9016ccadfda3d8bc7d2f8cdf302aa51dae8dae9da.data b/test/repo-example/blocks/CIQFE/CIQFEAGMNNXXTYKYQSANT6IBNTFN7WR5RPD5F6GN6MBKUUO25DNOTWQ.data similarity index 100% rename from test/repo-example/blocks/12205200/12205200cc6b6f79e1588480d9f9016ccadfda3d8bc7d2f8cdf302aa51dae8dae9da.data rename to test/repo-example/blocks/CIQFE/CIQFEAGMNNXXTYKYQSANT6IBNTFN7WR5RPD5F6GN6MBKUUO25DNOTWQ.data diff --git a/test/repo-example/blocks/122052c6/122052c63c7775396b3f82c639977a7223c2d96a9f70b5fd8b1d513f8c5b69dcaed4.data b/test/repo-example/blocks/CIQFF/CIQFFRR4O52TS2Z7QLDDTF32OIR4FWLKT5YLL7MLDVIT7DC3NHOK5VA.data similarity index 100% rename from test/repo-example/blocks/122052c6/122052c63c7775396b3f82c639977a7223c2d96a9f70b5fd8b1d513f8c5b69dcaed4.data rename to test/repo-example/blocks/CIQFF/CIQFFRR4O52TS2Z7QLDDTF32OIR4FWLKT5YLL7MLDVIT7DC3NHOK5VA.data diff --git a/test/repo-example/blocks/12205411/1220541179c3350d3b5f395966bb71808346f34c68796c10f3129c1511d0172fc9ed.data b/test/repo-example/blocks/CIQFI/CIQFIELZYM2Q2O27HFMWNO3RQCBUN42MNB4WYEHTCKOBKEOQC4X4T3I.data similarity index 100% rename from test/repo-example/blocks/12205411/1220541179c3350d3b5f395966bb71808346f34c68796c10f3129c1511d0172fc9ed.data rename to test/repo-example/blocks/CIQFI/CIQFIELZYM2Q2O27HFMWNO3RQCBUN42MNB4WYEHTCKOBKEOQC4X4T3I.data diff --git a/test/repo-example/blocks/12205994/122059948439065f29619ef41280cbb932be52c56d99c5966b65e0111239f098bbef.data b/test/repo-example/blocks/CIQFT/CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y.data similarity index 100% rename from test/repo-example/blocks/12205994/122059948439065f29619ef41280cbb932be52c56d99c5966b65e0111239f098bbef.data rename to test/repo-example/blocks/CIQFT/CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y.data diff --git a/test/repo-example/blocks/1220614d/1220614dcf483bf2fb19c82b22af9c8d47a35a6d4fdcf85e33db4df6987edee07063.data b/test/repo-example/blocks/CIQGC/CIQGCTOPJA57F6YZZAVSFL44RVD2GWTNJ7OPQXRT3NG7NGD633QHAYY.data similarity index 100% rename from test/repo-example/blocks/1220614d/1220614dcf483bf2fb19c82b22af9c8d47a35a6d4fdcf85e33db4df6987edee07063.data rename to test/repo-example/blocks/CIQGC/CIQGCTOPJA57F6YZZAVSFL44RVD2GWTNJ7OPQXRT3NG7NGD633QHAYY.data diff --git a/test/repo-example/blocks/122062ce/122062ce1f2c91a13a97b596e873b5a3346a644605d7614dca53cd3a59f7385a8abb.data b/test/repo-example/blocks/CIQGF/CIQGFTQ7FSI2COUXWWLOQ45VUM2GUZCGAXLWCTOKKPGTUWPXHBNIVOY.data similarity index 100% rename from test/repo-example/blocks/122062ce/122062ce1f2c91a13a97b596e873b5a3346a644605d7614dca53cd3a59f7385a8abb.data rename to test/repo-example/blocks/CIQGF/CIQGFTQ7FSI2COUXWWLOQ45VUM2GUZCGAXLWCTOKKPGTUWPXHBNIVOY.data diff --git a/test/repo-example/blocks/122065f3/122065f339db69b1605e388c95e9e1dc8451ada862755098a84689328d2a546c2963.data b/test/repo-example/blocks/CIQGL/CIQGL4ZZ3NU3CYC6HCGJL2PB3SCFDLNIMJ2VBGFII2ETFDJKKRWCSYY.data similarity index 100% rename from test/repo-example/blocks/122065f3/122065f339db69b1605e388c95e9e1dc8451ada862755098a84689328d2a546c2963.data rename to test/repo-example/blocks/CIQGL/CIQGL4ZZ3NU3CYC6HCGJL2PB3SCFDLNIMJ2VBGFII2ETFDJKKRWCSYY.data diff --git a/test/repo-example/blocks/122066df/122066df09f34f09cdb6c7c9f62dd5c8fa1895895ecfafc48898434b52285426ffc6.data b/test/repo-example/blocks/CIQGN/CIQGNXYJ6NHQTTNWY7E7MLOVZD5BRFMJL3H27REITBBUWURIKQTP7RQ.data similarity index 100% rename from test/repo-example/blocks/122066df/122066df09f34f09cdb6c7c9f62dd5c8fa1895895ecfafc48898434b52285426ffc6.data rename to test/repo-example/blocks/CIQGN/CIQGNXYJ6NHQTTNWY7E7MLOVZD5BRFMJL3H27REITBBUWURIKQTP7RQ.data diff --git a/test/repo-example/blocks/12206781/122067817186b8ff365c758f387e3ae7f28fa9367ee167c312e6d65a2e02e81ab815.data b/test/repo-example/blocks/CIQGP/CIQGPALRQ24P6NS4OWHTQ7R247ZI7KJWP3QWPQYS43LFULQC5ANLQFI.data similarity index 100% rename from test/repo-example/blocks/12206781/122067817186b8ff365c758f387e3ae7f28fa9367ee167c312e6d65a2e02e81ab815.data rename to test/repo-example/blocks/CIQGP/CIQGPALRQ24P6NS4OWHTQ7R247ZI7KJWP3QWPQYS43LFULQC5ANLQFI.data diff --git a/test/repo-example/blocks/12206990/1220699077ec4b2a853f2c007964cc174ee772e6e9b20cc73d2039b6a950226f60e3.data b/test/repo-example/blocks/CIQGT/CIQGTEDX5RFSVBJ7FQAHSZGMC5HOO4XG5GZAZRZ5EA43NKKQEJXWBYY.data similarity index 100% rename from test/repo-example/blocks/12206990/1220699077ec4b2a853f2c007964cc174ee772e6e9b20cc73d2039b6a950226f60e3.data rename to test/repo-example/blocks/CIQGT/CIQGTEDX5RFSVBJ7FQAHSZGMC5HOO4XG5GZAZRZ5EA43NKKQEJXWBYY.data diff --git a/test/repo-example/blocks/12206bbb/12206bbba0768a844ab1194e5876dfb6ce4399b2ea87a4b718c9850db60faeb50105.data b/test/repo-example/blocks/CIQGX/CIQGXO5AO2FIISVRDFHFQ5W7W3HEHGNS5KD2JNYYZGCQ3NQPV22QCBI.data similarity index 100% rename from test/repo-example/blocks/12206bbb/12206bbba0768a844ab1194e5876dfb6ce4399b2ea87a4b718c9850db60faeb50105.data rename to test/repo-example/blocks/CIQGX/CIQGXO5AO2FIISVRDFHFQ5W7W3HEHGNS5KD2JNYYZGCQ3NQPV22QCBI.data diff --git a/test/repo-example/blocks/12207f93/12207f93b47af638189785c73d414d653468aedcdccff814e89ac755a5502e7041e5.data b/test/repo-example/blocks/CIQH7/CIQH7E5UPL3DQGEXQXDT2QKNMU2GRLW43TH7QFHITLDVLJKQFZYEDZI.data similarity index 100% rename from test/repo-example/blocks/12207f93/12207f93b47af638189785c73d414d653468aedcdccff814e89ac755a5502e7041e5.data rename to test/repo-example/blocks/CIQH7/CIQH7E5UPL3DQGEXQXDT2QKNMU2GRLW43TH7QFHITLDVLJKQFZYEDZI.data diff --git a/test/repo-example/blocks/12207fb8/12207fb898b5d7be46d85feb75d894e16cfa9a7ae5533f8e997cdab2ebadd7506340.data b/test/repo-example/blocks/CIQH7/CIQH7OEYWXL34RWYL7VXLWEU4FWPVGT24VJT7DUZPTNLF25N25IGGQA.data similarity index 100% rename from test/repo-example/blocks/12207fb8/12207fb898b5d7be46d85feb75d894e16cfa9a7ae5533f8e997cdab2ebadd7506340.data rename to test/repo-example/blocks/CIQH7/CIQH7OEYWXL34RWYL7VXLWEU4FWPVGT24VJT7DUZPTNLF25N25IGGQA.data diff --git a/test/repo-example/blocks/12207028/122070286b9afa6620a66f715c7020d68af3d10e1a497971629c07606bfdb812303d.data b/test/repo-example/blocks/CIQHA/CIQHAKDLTL5GMIFGN5YVY4BA22FPHUIODJEXS4LCTQDWA275XAJDAPI.data similarity index 100% rename from test/repo-example/blocks/12207028/122070286b9afa6620a66f715c7020d68af3d10e1a497971629c07606bfdb812303d.data rename to test/repo-example/blocks/CIQHA/CIQHAKDLTL5GMIFGN5YVY4BA22FPHUIODJEXS4LCTQDWA275XAJDAPI.data diff --git a/test/repo-example/blocks/1220709b/1220709b2dcc5f6a90ad64d6fe3a5d202a72b057d1c7f2e682d0776a5363e2cca974.data b/test/repo-example/blocks/CIQHB/CIQHBGZNZRPWVEFNMTLP4OS5EAVHFMCX2HD7FZUC2B3WUU3D4LGKS5A.data similarity index 100% rename from test/repo-example/blocks/1220709b/1220709b2dcc5f6a90ad64d6fe3a5d202a72b057d1c7f2e682d0776a5363e2cca974.data rename to test/repo-example/blocks/CIQHB/CIQHBGZNZRPWVEFNMTLP4OS5EAVHFMCX2HD7FZUC2B3WUU3D4LGKS5A.data diff --git a/test/repo-example/blocks/12207248/12207248d65fb34acf915665eaeb29e2075fc63e678b0281c5b5dc9c36b199e6c051.data b/test/repo-example/blocks/CIQHE/CIQHESGWL6ZUVT4RKZS6V2ZJ4IDV7RR6M6FQFAOFWXOJYNVRTHTMAUI.data similarity index 100% rename from test/repo-example/blocks/12207248/12207248d65fb34acf915665eaeb29e2075fc63e678b0281c5b5dc9c36b199e6c051.data rename to test/repo-example/blocks/CIQHE/CIQHESGWL6ZUVT4RKZS6V2ZJ4IDV7RR6M6FQFAOFWXOJYNVRTHTMAUI.data diff --git a/test/repo-example/blocks/12207369/12207369ddf6b9bb22adc2b957eb3ca7f6a6f3477c85b665f21db50834091f902033.data b/test/repo-example/blocks/CIQHG/CIQHG2O56243WIVNYK4VP2Z4U73KN42HPSC3MZPSDW2QQNAJD6ICAMY.data similarity index 100% rename from test/repo-example/blocks/12207369/12207369ddf6b9bb22adc2b957eb3ca7f6a6f3477c85b665f21db50834091f902033.data rename to test/repo-example/blocks/CIQHG/CIQHG2O56243WIVNYK4VP2Z4U73KN42HPSC3MZPSDW2QQNAJD6ICAMY.data diff --git a/test/repo-example/blocks/12207395/12207395ead6f5161ddf1dc041c8640cb41a635756afb5a66fe49e8675b80a1b49ef.data b/test/repo-example/blocks/CIQHH/CIQHHFPK232RMHO7DXAEDSDEBS2BUY2XK2X3LJTP4SPIM5NYBINUT3Y.data similarity index 100% rename from test/repo-example/blocks/12207395/12207395ead6f5161ddf1dc041c8640cb41a635756afb5a66fe49e8675b80a1b49ef.data rename to test/repo-example/blocks/CIQHH/CIQHHFPK232RMHO7DXAEDSDEBS2BUY2XK2X3LJTP4SPIM5NYBINUT3Y.data diff --git a/test/repo-example/blocks/1220742e/1220742ede16a7d165f4b000e5f4094933b61d3d0b6ffb3cb6ec5b3eed32c0f2a38f.data b/test/repo-example/blocks/CIQHI/CIQHILW6C2T5CZPUWAAOL5AJJEZ3MHJ5BNX7WPFW5RNT53JSYDZKHDY.data similarity index 100% rename from test/repo-example/blocks/1220742e/1220742ede16a7d165f4b000e5f4094933b61d3d0b6ffb3cb6ec5b3eed32c0f2a38f.data rename to test/repo-example/blocks/CIQHI/CIQHILW6C2T5CZPUWAAOL5AJJEZ3MHJ5BNX7WPFW5RNT53JSYDZKHDY.data diff --git a/test/repo-example/blocks/122074d4/122074d42168a1c62b4796e77a9591d8f1fab4685f6674eabecc7773913363b45301.data b/test/repo-example/blocks/CIQHJ/CIQHJVBBNCQ4MK2HS3TXVFMR3DY7VNDIL5THJ2V6ZR3XHEJTMO2FGAI.data similarity index 100% rename from test/repo-example/blocks/122074d4/122074d42168a1c62b4796e77a9591d8f1fab4685f6674eabecc7773913363b45301.data rename to test/repo-example/blocks/CIQHJ/CIQHJVBBNCQ4MK2HS3TXVFMR3DY7VNDIL5THJ2V6ZR3XHEJTMO2FGAI.data diff --git a/test/repo-example/blocks/1220751e/1220751e71f050b51a74159c51829b210f326e73bc4f0a712e1f636625a8515426d8.data b/test/repo-example/blocks/CIQHK/CIQHKHTR6BILKGTUCWOFDAU3EEHTE3TTXRHQU4JOD5RWMJNIKFKCNWA.data similarity index 100% rename from test/repo-example/blocks/1220751e/1220751e71f050b51a74159c51829b210f326e73bc4f0a712e1f636625a8515426d8.data rename to test/repo-example/blocks/CIQHK/CIQHKHTR6BILKGTUCWOFDAU3EEHTE3TTXRHQU4JOD5RWMJNIKFKCNWA.data diff --git a/test/repo-example/blocks/1220783d/1220783d550428fcd841a9579a08e0d10619a4238b6acdc73c8cf6932120e1f6e2df.data b/test/repo-example/blocks/CIQHQ/CIQHQPKVAQUPZWCBVFLZUCHA2EDBTJBDRNVM3RZ4RT3JGIJA4H3OFXY.data similarity index 100% rename from test/repo-example/blocks/1220783d/1220783d550428fcd841a9579a08e0d10619a4238b6acdc73c8cf6932120e1f6e2df.data rename to test/repo-example/blocks/CIQHQ/CIQHQPKVAQUPZWCBVFLZUCHA2EDBTJBDRNVM3RZ4RT3JGIJA4H3OFXY.data diff --git a/test/repo-example/blocks/1220797b/1220797bea239eddf7c11a47143062975b00b162ab286a019a0db2da5abfb67e516e.data b/test/repo-example/blocks/CIQHS/CIQHS67KEOPN356BDJDRIMDCS5NQBMLCVMUGUAM2BWZNUWV7WZ7FC3Q.data similarity index 100% rename from test/repo-example/blocks/1220797b/1220797bea239eddf7c11a47143062975b00b162ab286a019a0db2da5abfb67e516e.data rename to test/repo-example/blocks/CIQHS/CIQHS67KEOPN356BDJDRIMDCS5NQBMLCVMUGUAM2BWZNUWV7WZ7FC3Q.data diff --git a/test/repo-example/blocks/12207a19/12207a19de7e4b4d9e5db771aeca6fff1cc9f6518a7375f1671c61b9219f689c3851.data b/test/repo-example/blocks/CIQHU/CIQHUGO6PZFU3HS5W5Y25STP74OMT5SRRJZXL4LHDRQ3SIM7NCODQUI.data similarity index 100% rename from test/repo-example/blocks/12207a19/12207a19de7e4b4d9e5db771aeca6fff1cc9f6518a7375f1671c61b9219f689c3851.data rename to test/repo-example/blocks/CIQHU/CIQHUGO6PZFU3HS5W5Y25STP74OMT5SRRJZXL4LHDRQ3SIM7NCODQUI.data diff --git a/test/repo-example/blocks/12207b9b/12207b9bf084ba1f973ba3994c3ac9cf556b894633bec0acf87dfd9cf6f957bc0855.data b/test/repo-example/blocks/CIQHX/CIQHXG7QQS5B7FZ3UOMUYOWJZ5KWXCKGGO7MBLHYPX6ZZ5XZK66AQVI.data similarity index 100% rename from test/repo-example/blocks/12207b9b/12207b9bf084ba1f973ba3994c3ac9cf556b894633bec0acf87dfd9cf6f957bc0855.data rename to test/repo-example/blocks/CIQHX/CIQHXG7QQS5B7FZ3UOMUYOWJZ5KWXCKGGO7MBLHYPX6ZZ5XZK66AQVI.data diff --git a/test/repo-example/blocks/12207cef/12207cefb4d9616bb6c0e915da503fb59c077861f3a5b7a9deeca5eff1fcccd3ccec.data b/test/repo-example/blocks/CIQHZ/CIQHZ35U3FQWXNWA5EK5UUB7WWOAO6DB6OS3PKO65SS674P4ZTJ4Z3A.data similarity index 100% rename from test/repo-example/blocks/12207cef/12207cefb4d9616bb6c0e915da503fb59c077861f3a5b7a9deeca5eff1fcccd3ccec.data rename to test/repo-example/blocks/CIQHZ/CIQHZ35U3FQWXNWA5EK5UUB7WWOAO6DB6OS3PKO65SS674P4ZTJ4Z3A.data diff --git a/test/repo-example/blocks/12208ef4/12208ef4750c671599c429b6d952b4a8c5d2d761827c119bd382efdcb0773ec435.data b/test/repo-example/blocks/CIQI5/CIQI55DVBRTRLGOEFG3NSUVUVDC5FV3BQJ6BDG6TQLX5ZMDXH3CDKLY.data similarity index 100% rename from test/repo-example/blocks/12208ef4/12208ef4750c671599c429b6d952b4a8c5d2d761827c119bd382efdcb0773ec435.data rename to test/repo-example/blocks/CIQI5/CIQI55DVBRTRLGOEFG3NSUVUVDC5FV3BQJ6BDG6TQLX5ZMDXH3CDKLY.data diff --git a/test/repo-example/blocks/1220829c/1220829cebf24c4e827d15fde2e687828b9d97a9e1c73190722a0b1ca5ce3e0f85b1.data b/test/repo-example/blocks/CIQIF/CIQIFHHL6JGE5AT5CX66FZUHQKFZ3F5J4HDTDEDSFIFRZJOOHYHYLMI.data similarity index 100% rename from test/repo-example/blocks/1220829c/1220829cebf24c4e827d15fde2e687828b9d97a9e1c73190722a0b1ca5ce3e0f85b1.data rename to test/repo-example/blocks/CIQIF/CIQIFHHL6JGE5AT5CX66FZUHQKFZ3F5J4HDTDEDSFIFRZJOOHYHYLMI.data diff --git a/test/repo-example/blocks/1220853a/1220853a6e9a61a6943c06d8c59d590cdae42b700677790812ce68f1eb2389e6ea23.data b/test/repo-example/blocks/CIQIK/CIQIKOTOTJQ2NFB4A3MMLHKZBTNOIK3QAZ3XSCASZZUPD2ZDRHTOUIY.data similarity index 100% rename from test/repo-example/blocks/1220853a/1220853a6e9a61a6943c06d8c59d590cdae42b700677790812ce68f1eb2389e6ea23.data rename to test/repo-example/blocks/CIQIK/CIQIKOTOTJQ2NFB4A3MMLHKZBTNOIK3QAZ3XSCASZZUPD2ZDRHTOUIY.data diff --git a/test/repo-example/blocks/12208940/12208940fef8c586f7df131d3fdabd1903c36abadcbb55b8f352446e8d67c6c7ee87.data b/test/repo-example/blocks/CIQIS/CIQISQH67DCYN567CMOT7WV5DEB4G2V23S5VLOHTKJCG5DLHY3D65BY.data similarity index 100% rename from test/repo-example/blocks/12208940/12208940fef8c586f7df131d3fdabd1903c36abadcbb55b8f352446e8d67c6c7ee87.data rename to test/repo-example/blocks/CIQIS/CIQISQH67DCYN567CMOT7WV5DEB4G2V23S5VLOHTKJCG5DLHY3D65BY.data diff --git a/test/repo-example/blocks/12208b87/12208b872ca4ee517608331696dd6b3e5cf3497a7845ee8f94456ccf4d1d2f6602b5.data b/test/repo-example/blocks/CIQIX/CIQIXBZMUTXFC5QIGMLJNXLLHZOPGSL2PBC65D4UIVWM6TI5F5TAFNI.data similarity index 100% rename from test/repo-example/blocks/12208b87/12208b872ca4ee517608331696dd6b3e5cf3497a7845ee8f94456ccf4d1d2f6602b5.data rename to test/repo-example/blocks/CIQIX/CIQIXBZMUTXFC5QIGMLJNXLLHZOPGSL2PBC65D4UIVWM6TI5F5TAFNI.data diff --git a/test/repo-example/blocks/12209d4c/12209d4c462f8c4f3c547314b12aa5c2af186e72a9bc4cb354c77280a75a5f37228e.data b/test/repo-example/blocks/CIQJ2/CIQJ2TCGF6GE6PCUOMKLCKVFYKXRQ3TSVG6EZM2UY5ZIBJ22L43SFDQ.data similarity index 100% rename from test/repo-example/blocks/12209d4c/12209d4c462f8c4f3c547314b12aa5c2af186e72a9bc4cb354c77280a75a5f37228e.data rename to test/repo-example/blocks/CIQJ2/CIQJ2TCGF6GE6PCUOMKLCKVFYKXRQ3TSVG6EZM2UY5ZIBJ22L43SFDQ.data diff --git a/test/repo-example/blocks/12209e8e/12209e8eabd807dd930cb0606fb18010e98b2469efc12f55077b1d64e485c8e81255.data b/test/repo-example/blocks/CIQJ5/CIQJ5DVL3AD53EYMWBQG7MMACDUYWJDJ57AS6VIHPMOWJZEFZDUBEVI.data similarity index 100% rename from test/repo-example/blocks/12209e8e/12209e8eabd807dd930cb0606fb18010e98b2469efc12f55077b1d64e485c8e81255.data rename to test/repo-example/blocks/CIQJ5/CIQJ5DVL3AD53EYMWBQG7MMACDUYWJDJ57AS6VIHPMOWJZEFZDUBEVI.data diff --git a/test/repo-example/blocks/122090c0/122090c07a7795c1193510a696d1fdfc0f1e4947cff8e422610996e609dbcb976598.data b/test/repo-example/blocks/CIQJB/CIQJBQD2O6K4CGJVCCTJNUP57QHR4SKHZ74OIITBBGLOMCO3ZOLWLGA.data similarity index 100% rename from test/repo-example/blocks/122090c0/122090c07a7795c1193510a696d1fdfc0f1e4947cff8e422610996e609dbcb976598.data rename to test/repo-example/blocks/CIQJB/CIQJBQD2O6K4CGJVCCTJNUP57QHR4SKHZ74OIITBBGLOMCO3ZOLWLGA.data diff --git a/test/repo-example/blocks/1220929a/1220929a303c39da8a0b67c09697462f687a00c638bcb580feae06452e0c1f20b4.data b/test/repo-example/blocks/CIQJF/CIQJFGRQHQ45VCQLM7AJNF2GF5UHUAGGHC6LLAH6VYDEKLQMD4QLILY.data similarity index 100% rename from test/repo-example/blocks/1220929a/1220929a303c39da8a0b67c09697462f687a00c638bcb580feae06452e0c1f20b4.data rename to test/repo-example/blocks/CIQJF/CIQJFGRQHQ45VCQLM7AJNF2GF5UHUAGGHC6LLAH6VYDEKLQMD4QLILY.data diff --git a/test/repo-example/blocks/1220933b/1220933b41d37fd4508cdff45930dff56baef91c7dc345e73d049ab570abe10dfbb9.data b/test/repo-example/blocks/CIQJG/CIQJGO2B2N75IUEM372FSMG76VV256I4PXBULZZ5ASNLK4FL4EG7XOI.data similarity index 100% rename from test/repo-example/blocks/1220933b/1220933b41d37fd4508cdff45930dff56baef91c7dc345e73d049ab570abe10dfbb9.data rename to test/repo-example/blocks/CIQJG/CIQJGO2B2N75IUEM372FSMG76VV256I4PXBULZZ5ASNLK4FL4EG7XOI.data diff --git a/test/repo-example/blocks/1220941f/1220941f8cb777c014ea4f229b48038414fea4bfe9523e9c06d15b4d731d30d6d624.data b/test/repo-example/blocks/CIQJI/CIQJIH4MW534AFHKJ4RJWSADQQKP5JF75FJD5HAG2FNU24Y5GDLNMJA.data similarity index 100% rename from test/repo-example/blocks/1220941f/1220941f8cb777c014ea4f229b48038414fea4bfe9523e9c06d15b4d731d30d6d624.data rename to test/repo-example/blocks/CIQJI/CIQJIH4MW534AFHKJ4RJWSADQQKP5JF75FJD5HAG2FNU24Y5GDLNMJA.data diff --git a/test/repo-example/blocks/122095a8/122095a86a3b176d88282a3c56b77118e0986ff4784eb0c864e21c306dc6a632d5b9.data b/test/repo-example/blocks/CIQJL/CIQJLKDKHMLW3CBIFI6FNN3RDDQJQ37UPBHLBSDE4IODA3OGUYZNLOI.data similarity index 100% rename from test/repo-example/blocks/122095a8/122095a86a3b176d88282a3c56b77118e0986ff4784eb0c864e21c306dc6a632d5b9.data rename to test/repo-example/blocks/CIQJL/CIQJLKDKHMLW3CBIFI6FNN3RDDQJQ37UPBHLBSDE4IODA3OGUYZNLOI.data diff --git a/test/repo-example/blocks/1220984a/1220984a7e1501c42bbe85bf56f1451c0ea7c81e4038d31434c1eb1f94d33cac1ffd.data b/test/repo-example/blocks/CIQJQ/CIQJQST6CUA4IK56QW7VN4KFDQHKPSA6IA4NGFBUYHVR7FGTHSWB77I.data similarity index 100% rename from test/repo-example/blocks/1220984a/1220984a7e1501c42bbe85bf56f1451c0ea7c81e4038d31434c1eb1f94d33cac1ffd.data rename to test/repo-example/blocks/CIQJQ/CIQJQST6CUA4IK56QW7VN4KFDQHKPSA6IA4NGFBUYHVR7FGTHSWB77I.data diff --git a/test/repo-example/blocks/12209b92/12209b92240f2b17e6f47bf84a518009a1c2ac48afe25032d5f5678398d33b20ea34.data b/test/repo-example/blocks/CIQJX/CIQJXEREB4VRPZXUPP4EUUMABGQ4FLCIV7RFAMWV6VTYHGGTHMQOUNA.data similarity index 100% rename from test/repo-example/blocks/12209b92/12209b92240f2b17e6f47bf84a518009a1c2ac48afe25032d5f5678398d33b20ea34.data rename to test/repo-example/blocks/CIQJX/CIQJXEREB4VRPZXUPP4EUUMABGQ4FLCIV7RFAMWV6VTYHGGTHMQOUNA.data diff --git a/test/repo-example/blocks/12209c37/12209c376abcbaf28dbb16e8a44f24043a4d67324293d32caac64fd374cf73f3194a.data b/test/repo-example/blocks/CIQJY/CIQJYN3KXS5PFDN3C3UKITZEAQ5E2ZZSIKJ5GLFKYZH5G5GPOPZRSSQ.data similarity index 100% rename from test/repo-example/blocks/12209c37/12209c376abcbaf28dbb16e8a44f24043a4d67324293d32caac64fd374cf73f3194a.data rename to test/repo-example/blocks/CIQJY/CIQJYN3KXS5PFDN3C3UKITZEAQ5E2ZZSIKJ5GLFKYZH5G5GPOPZRSSQ.data diff --git a/test/repo-example/blocks/1220aebc/1220aebc7f9014417e6f5e2f2d9330882b8995f1ee71e3cd1f18e003781983ae8169.data b/test/repo-example/blocks/CIQK5/CIQK5PD7SAKEC7TPLYXS3EZQRAVYTFPR5ZY6HTI7DDQAG6AZQOXIC2I.data similarity index 100% rename from test/repo-example/blocks/1220aebc/1220aebc7f9014417e6f5e2f2d9330882b8995f1ee71e3cd1f18e003781983ae8169.data rename to test/repo-example/blocks/CIQK5/CIQK5PD7SAKEC7TPLYXS3EZQRAVYTFPR5ZY6HTI7DDQAG6AZQOXIC2I.data diff --git a/test/repo-example/blocks/1220aee7/1220aee755c2de4672ffe9ad07eb6eaff7f3eb62c5c119e287be963fedf07c193cbf.data b/test/repo-example/blocks/CIQK5/CIQK5Z2VYLPEM4X75GWQP23OV737H23CYXARTYUHX2LD73PQPQMTZPY.data similarity index 100% rename from test/repo-example/blocks/1220aee7/1220aee755c2de4672ffe9ad07eb6eaff7f3eb62c5c119e287be963fedf07c193cbf.data rename to test/repo-example/blocks/CIQK5/CIQK5Z2VYLPEM4X75GWQP23OV737H23CYXARTYUHX2LD73PQPQMTZPY.data diff --git a/test/repo-example/blocks/1220a52c/1220a52c3602030cb912edfe4de97002fdadf9d45666c3be122a2efb5db93c1d5fa6.data b/test/repo-example/blocks/CIQKK/CIQKKLBWAIBQZOIS5X7E32LQAL6236OUKZTMHPQSFIXPWXNZHQOV7JQ.data similarity index 100% rename from test/repo-example/blocks/1220a52c/1220a52c3602030cb912edfe4de97002fdadf9d45666c3be122a2efb5db93c1d5fa6.data rename to test/repo-example/blocks/CIQKK/CIQKKLBWAIBQZOIS5X7E32LQAL6236OUKZTMHPQSFIXPWXNZHQOV7JQ.data diff --git a/test/repo-example/blocks/1220a89a/1220a89aa29de372b8989fe4ce0843acd69557211293a56bb0a1c7218bbc2050a5d1.data b/test/repo-example/blocks/CIQKR/CIQKRGVCTXRXFOEYT7SM4CCDVTLJKVZBCKJ2K25QUHDSDC54EBIKLUI.data similarity index 100% rename from test/repo-example/blocks/1220a89a/1220a89aa29de372b8989fe4ce0843acd69557211293a56bb0a1c7218bbc2050a5d1.data rename to test/repo-example/blocks/CIQKR/CIQKRGVCTXRXFOEYT7SM4CCDVTLJKVZBCKJ2K25QUHDSDC54EBIKLUI.data diff --git a/test/repo-example/blocks/1220bfcc/1220bfccda787baba32b59c78450ac3d20b633360b43992c77289f9ed46d843561e6.data b/test/repo-example/blocks/CIQL7/CIQL7TG2PB52XIZLLHDYIUFMHUQLMMZWBNBZSLDXFCPZ5VDNQQ2WDZQ.data similarity index 100% rename from test/repo-example/blocks/1220bfcc/1220bfccda787baba32b59c78450ac3d20b633360b43992c77289f9ed46d843561e6.data rename to test/repo-example/blocks/CIQL7/CIQL7TG2PB52XIZLLHDYIUFMHUQLMMZWBNBZSLDXFCPZ5VDNQQ2WDZQ.data diff --git a/test/repo-example/blocks/1220b106/1220b106e54902a7d812437110f5c1b8dd9dbc3758d306c6adc0fe7a04d5c8fd3d3c.data b/test/repo-example/blocks/CIQLC/CIQLCBXFJEBKPWASINYRB5OBXDOZ3PBXLDJQNRVNYD7HUBGVZD6T2PA.data similarity index 100% rename from test/repo-example/blocks/1220b106/1220b106e54902a7d812437110f5c1b8dd9dbc3758d306c6adc0fe7a04d5c8fd3d3c.data rename to test/repo-example/blocks/CIQLC/CIQLCBXFJEBKPWASINYRB5OBXDOZ3PBXLDJQNRVNYD7HUBGVZD6T2PA.data diff --git a/test/repo-example/blocks/1220b2c2/1220b2c2d9773a8316428106953b22061a79b6a3f36b7abed609c13a6b06e13c716c.data b/test/repo-example/blocks/CIQLF/CIQLFQWZO45IGFSCQEDJKOZCAYNHTNVD6NVXVPWWBHATU2YG4E6HC3A.data similarity index 100% rename from test/repo-example/blocks/1220b2c2/1220b2c2d9773a8316428106953b22061a79b6a3f36b7abed609c13a6b06e13c716c.data rename to test/repo-example/blocks/CIQLF/CIQLFQWZO45IGFSCQEDJKOZCAYNHTNVD6NVXVPWWBHATU2YG4E6HC3A.data diff --git a/test/repo-example/blocks/1220b42c/1220b42c0cb8915e874e6909edcd67aaa5dd0fba77fe8ee9262b8ffa6c02ab94d033.data b/test/repo-example/blocks/CIQLI/CIQLILAMXCIV5B2ONEE63TLHVKS52D52O77I52JGFOH7U3ACVOKNAMY.data similarity index 100% rename from test/repo-example/blocks/1220b42c/1220b42c0cb8915e874e6909edcd67aaa5dd0fba77fe8ee9262b8ffa6c02ab94d033.data rename to test/repo-example/blocks/CIQLI/CIQLILAMXCIV5B2ONEE63TLHVKS52D52O77I52JGFOH7U3ACVOKNAMY.data diff --git a/test/repo-example/blocks/1220b617/1220b617096833a3ef63aa1e896c1f96ac861c38f6838eb8a24474da9f343cfef127.data b/test/repo-example/blocks/CIQLM/CIQLMFYJNAZ2H33DVIPIS3A7S2WIMHBY62BY5OFCIR2NVHZUHT7PCJY.data similarity index 100% rename from test/repo-example/blocks/1220b617/1220b617096833a3ef63aa1e896c1f96ac861c38f6838eb8a24474da9f343cfef127.data rename to test/repo-example/blocks/CIQLM/CIQLMFYJNAZ2H33DVIPIS3A7S2WIMHBY62BY5OFCIR2NVHZUHT7PCJY.data diff --git a/test/repo-example/blocks/1220cefe/1220cefeb7568c689275e79f9480d743d04ebbc98d140afc21126c08a3053ef24a8f.data b/test/repo-example/blocks/CIQM5/CIQM57VXK2GGRETV46PZJAGXIPIE5O6JRUKAV7BBCJWARIYFH3ZEVDY.data similarity index 100% rename from test/repo-example/blocks/1220cefe/1220cefeb7568c689275e79f9480d743d04ebbc98d140afc21126c08a3053ef24a8f.data rename to test/repo-example/blocks/CIQM5/CIQM57VXK2GGRETV46PZJAGXIPIE5O6JRUKAV7BBCJWARIYFH3ZEVDY.data diff --git a/test/repo-example/blocks/1220c0fc/1220c0fc6b49543d7bf04e83d2a5a7cbe72a83e80f9c7bca1abcaa42298a57a33ff5.data b/test/repo-example/blocks/CIQMB/CIQMB7DLJFKD267QJ2B5FJNHZPTSVA7IB6OHXSQ2XSVEEKMKK6RT75I.data similarity index 100% rename from test/repo-example/blocks/1220c0fc/1220c0fc6b49543d7bf04e83d2a5a7cbe72a83e80f9c7bca1abcaa42298a57a33ff5.data rename to test/repo-example/blocks/CIQMB/CIQMB7DLJFKD267QJ2B5FJNHZPTSVA7IB6OHXSQ2XSVEEKMKK6RT75I.data diff --git a/test/repo-example/blocks/1220c597/1220c59777258deac9d4b8e49e84daf2d449cd81ab2e8b7d2a89e2f5a02d114588a5.data b/test/repo-example/blocks/CIQML/CIQMLF3XEWG6VSOUXDSJ5BG26LKETTMBVMXIW7JKRHRPLIBNCFCYRJI.data similarity index 100% rename from test/repo-example/blocks/1220c597/1220c59777258deac9d4b8e49e84daf2d449cd81ab2e8b7d2a89e2f5a02d114588a5.data rename to test/repo-example/blocks/CIQML/CIQMLF3XEWG6VSOUXDSJ5BG26LKETTMBVMXIW7JKRHRPLIBNCFCYRJI.data diff --git a/test/repo-example/blocks/1220c8af/1220c8af6ecba1e2daa242da154307928e671ed7744c5a1474f67722a3ccb0be2469.data b/test/repo-example/blocks/CIQMR/CIQMRL3OZOQ6FWVCILNBKQYHSKHGOHWXORGFUFDU6Z3SFI6MWC7CI2I.data similarity index 100% rename from test/repo-example/blocks/1220c8af/1220c8af6ecba1e2daa242da154307928e671ed7744c5a1474f67722a3ccb0be2469.data rename to test/repo-example/blocks/CIQMR/CIQMRL3OZOQ6FWVCILNBKQYHSKHGOHWXORGFUFDU6Z3SFI6MWC7CI2I.data diff --git a/test/repo-example/blocks/1220cadf/1220cadf66920934b4e5db9c7fcd8b387fcaf9842671daf799fa4e9ae994cfe3a9d9.data b/test/repo-example/blocks/CIQMV/CIQMVX3GSIETJNHF3OOH7TMLHB74V6MEEZY5V54Z7JHJV2MUZ7R2TWI.data similarity index 100% rename from test/repo-example/blocks/1220cadf/1220cadf66920934b4e5db9c7fcd8b387fcaf9842671daf799fa4e9ae994cfe3a9d9.data rename to test/repo-example/blocks/CIQMV/CIQMVX3GSIETJNHF3OOH7TMLHB74V6MEEZY5V54Z7JHJV2MUZ7R2TWI.data diff --git a/test/repo-example/blocks/1220cc1a/1220cc1a6b60658b47f7f67dff1a135eedd6ba4736a126b2cc04a38e7e3e1a243407.data b/test/repo-example/blocks/CIQMY/CIQMYGTLMBSYWR7X6Z676GQTL3W5NOSHG2QSNMWMASRY47R6DISDIBY.data similarity index 100% rename from test/repo-example/blocks/1220cc1a/1220cc1a6b60658b47f7f67dff1a135eedd6ba4736a126b2cc04a38e7e3e1a243407.data rename to test/repo-example/blocks/CIQMY/CIQMYGTLMBSYWR7X6Z676GQTL3W5NOSHG2QSNMWMASRY47R6DISDIBY.data diff --git a/test/repo-example/blocks/1220dfb8/1220dfb83d070291965675a9d7e9e7321b92e0aeb724606b42b5df689e6b547ccc21.data b/test/repo-example/blocks/CIQN7/CIQN7OB5A4BJDFSWOWU5P2PHGINZFYFOW4SGA22CWXPWRHTLKR6MYII.data similarity index 100% rename from test/repo-example/blocks/1220dfb8/1220dfb83d070291965675a9d7e9e7321b92e0aeb724606b42b5df689e6b547ccc21.data rename to test/repo-example/blocks/CIQN7/CIQN7OB5A4BJDFSWOWU5P2PHGINZFYFOW4SGA22CWXPWRHTLKR6MYII.data diff --git a/test/repo-example/blocks/1220d068/1220d0687dbeeeeca8508ee5dfe8a0d236e05c3f0afe90e43548acf028302527df49.data b/test/repo-example/blocks/CIQNA/CIQNA2D5X3XOZKCQR3S572FA2I3OAXB7BL7JBZBVJCWPAKBQEUT56SI.data similarity index 100% rename from test/repo-example/blocks/1220d068/1220d0687dbeeeeca8508ee5dfe8a0d236e05c3f0afe90e43548acf028302527df49.data rename to test/repo-example/blocks/CIQNA/CIQNA2D5X3XOZKCQR3S572FA2I3OAXB7BL7JBZBVJCWPAKBQEUT56SI.data diff --git a/test/repo-example/blocks/1220d313/1220d313d073066579f96f3f1a5fcc2775e98d319a203f238a389408140add211e41.data b/test/repo-example/blocks/CIQNG/CIQNGE6QOMDGK6PZN47RUX6ME526TDJRTIQD6I4KHCKAQFAK3UQR4QI.data similarity index 100% rename from test/repo-example/blocks/1220d313/1220d313d073066579f96f3f1a5fcc2775e98d319a203f238a389408140add211e41.data rename to test/repo-example/blocks/CIQNG/CIQNGE6QOMDGK6PZN47RUX6ME526TDJRTIQD6I4KHCKAQFAK3UQR4QI.data diff --git a/test/repo-example/blocks/1220e213/1220e213c7180c2051bec353ab32c9a67cf9a32f33c4b560cf34d11bf50c4b6f9b1e.data b/test/repo-example/blocks/CIQOE/CIQOEE6HDAGCAUN6YNJ2WMWJUZ6PTIZPGPCLKYGPGTIRX5IMJNXZWHQ.data similarity index 100% rename from test/repo-example/blocks/1220e213/1220e213c7180c2051bec353ab32c9a67cf9a32f33c4b560cf34d11bf50c4b6f9b1e.data rename to test/repo-example/blocks/CIQOE/CIQOEE6HDAGCAUN6YNJ2WMWJUZ6PTIZPGPCLKYGPGTIRX5IMJNXZWHQ.data diff --git a/test/repo-example/blocks/1220e3b0/1220e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.data b/test/repo-example/blocks/CIQOH/CIQOHMGEIKMPYHAUTL57JSEZN64SIJ5OIHSGJG4TJSSJLGI3PBJLQVI.data similarity index 100% rename from test/repo-example/blocks/1220e3b0/1220e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.data rename to test/repo-example/blocks/CIQOH/CIQOHMGEIKMPYHAUTL57JSEZN64SIJ5OIHSGJG4TJSSJLGI3PBJLQVI.data diff --git a/test/repo-example/blocks/1220e482/1220e482600f1178538696dbe42801a86cb455fd3c20e0c172235e4c86e0a76c670f.data b/test/repo-example/blocks/CIQOJ/CIQOJATAB4IXQU4GS3N6IKABVBWLIVP5HQQOBQLSENPEZBXAU5WGODY.data similarity index 100% rename from test/repo-example/blocks/1220e482/1220e482600f1178538696dbe42801a86cb455fd3c20e0c172235e4c86e0a76c670f.data rename to test/repo-example/blocks/CIQOJ/CIQOJATAB4IXQU4GS3N6IKABVBWLIVP5HQQOBQLSENPEZBXAU5WGODY.data diff --git a/test/repo-example/blocks/1220e57d/1220e57d74c345d324769d7d2267994a7dc8018ecfc65af4d6625931ededfecef583.data b/test/repo-example/blocks/CIQOK/CIQOK7LUYNC5GJDWTV6SEZ4ZJJ64QAMOZ7DFV5GWMJMTD3PN73HPLAY.data similarity index 100% rename from test/repo-example/blocks/1220e57d/1220e57d74c345d324769d7d2267994a7dc8018ecfc65af4d6625931ededfecef583.data rename to test/repo-example/blocks/CIQOK/CIQOK7LUYNC5GJDWTV6SEZ4ZJJ64QAMOZ7DFV5GWMJMTD3PN73HPLAY.data diff --git a/test/repo-example/blocks/1220e605/1220e605408ac3f78113ac9a7fd486441317afc9f967abe2aa05e7c641f7bbe98a37.data b/test/repo-example/blocks/CIQOM/CIQOMBKARLB7PAITVSNH7VEGIQJRPL6J7FT2XYVKAXT4MQPXXPUYUNY.data similarity index 100% rename from test/repo-example/blocks/1220e605/1220e605408ac3f78113ac9a7fd486441317afc9f967abe2aa05e7c641f7bbe98a37.data rename to test/repo-example/blocks/CIQOM/CIQOMBKARLB7PAITVSNH7VEGIQJRPL6J7FT2XYVKAXT4MQPXXPUYUNY.data diff --git a/test/repo-example/blocks/1220e6a0/1220e6a045864ff8569e43e4866c0af807def07b0db2f018808620eb30b980e94011.data b/test/repo-example/blocks/CIQON/CIQONICFQZH7QVU6IPSIM3AK7AD554D3BWZPAGEAQYQOWMFZQDUUAEI.data similarity index 100% rename from test/repo-example/blocks/1220e6a0/1220e6a045864ff8569e43e4866c0af807def07b0db2f018808620eb30b980e94011.data rename to test/repo-example/blocks/CIQON/CIQONICFQZH7QVU6IPSIM3AK7AD554D3BWZPAGEAQYQOWMFZQDUUAEI.data diff --git a/test/repo-example/blocks/1220e93a/1220e93a39630237c5d8d370d0ad31c31e6a6b6ced9791fc2c9fe282fe25491c73d7.data b/test/repo-example/blocks/CIQOS/CIQOSORZMMBDPROY2NYNBLJRYMPGU23M5WLZD7BMT7RIF7RFJEOHHVY.data similarity index 100% rename from test/repo-example/blocks/1220e93a/1220e93a39630237c5d8d370d0ad31c31e6a6b6ced9791fc2c9fe282fe25491c73d7.data rename to test/repo-example/blocks/CIQOS/CIQOSORZMMBDPROY2NYNBLJRYMPGU23M5WLZD7BMT7RIF7RFJEOHHVY.data diff --git a/test/repo-example/blocks/1220ec5b/1220ec5b533a3218991f4377b8b8c2538b95dd29d31eac6433af0fb6fcd83dd80778.data b/test/repo-example/blocks/CIQOY/CIQOYW2THIZBRGI7IN33ROGCKOFZLXJJ2MPKYZBTV4H3N7GYHXMAO6A.data similarity index 100% rename from test/repo-example/blocks/1220ec5b/1220ec5b533a3218991f4377b8b8c2538b95dd29d31eac6433af0fb6fcd83dd80778.data rename to test/repo-example/blocks/CIQOY/CIQOYW2THIZBRGI7IN33ROGCKOFZLXJJ2MPKYZBTV4H3N7GYHXMAO6A.data diff --git a/test/repo-example/blocks/1220ff70/1220ff700c84c919c855797d1b071dcf9e0ca68f385a3b93e3145e289bc555646f66.data b/test/repo-example/blocks/CIQP6/CIQP64AMQTERTSCVPF6RWBY5Z6PAZJUPHBNDXE7DCRPCRG6FKVSG6ZQ.data similarity index 100% rename from test/repo-example/blocks/1220ff70/1220ff700c84c919c855797d1b071dcf9e0ca68f385a3b93e3145e289bc555646f66.data rename to test/repo-example/blocks/CIQP6/CIQP64AMQTERTSCVPF6RWBY5Z6PAZJUPHBNDXE7DCRPCRG6FKVSG6ZQ.data diff --git a/test/repo-example/blocks/1220ff8e/1220ff8e0d20fe2a0b371dd8d7a45aac2e018919f2919da4be2e479c614960c4577f.data b/test/repo-example/blocks/CIQP7/CIQP7DQNED7CUCZXDXMNPJC2VQXADCIZ6KIZ3JF6FZDZYYKJMDCFO7Y.data similarity index 100% rename from test/repo-example/blocks/1220ff8e/1220ff8e0d20fe2a0b371dd8d7a45aac2e018919f2919da4be2e479c614960c4577f.data rename to test/repo-example/blocks/CIQP7/CIQP7DQNED7CUCZXDXMNPJC2VQXADCIZ6KIZ3JF6FZDZYYKJMDCFO7Y.data diff --git a/test/repo-example/blocks/1220f145/1220f145e82f37fcd08875e13c15894d210860ff30faa3f540365dd01f7ab4b00dfb.data b/test/repo-example/blocks/CIQPC/CIQPCRPIF437ZUEIOXQTYFMJJUQQQYH7GD5KH5KAGZO5AH32WSYA36Y.data similarity index 100% rename from test/repo-example/blocks/1220f145/1220f145e82f37fcd08875e13c15894d210860ff30faa3f540365dd01f7ab4b00dfb.data rename to test/repo-example/blocks/CIQPC/CIQPCRPIF437ZUEIOXQTYFMJJUQQQYH7GD5KH5KAGZO5AH32WSYA36Y.data diff --git a/test/repo-example/blocks/1220f7a0/1220f7a06300a26e3a858261f3d9a05aa65f57b3ad326df0643a6d9d08a01c0903d7.data b/test/repo-example/blocks/CIQPP/CIQPPIDDACRG4OUFQJQ7HWNALKTF6V5TVUZG34DEHJWZ2CFADQEQHVY.data similarity index 100% rename from test/repo-example/blocks/1220f7a0/1220f7a06300a26e3a858261f3d9a05aa65f57b3ad326df0643a6d9d08a01c0903d7.data rename to test/repo-example/blocks/CIQPP/CIQPPIDDACRG4OUFQJQ7HWNALKTF6V5TVUZG34DEHJWZ2CFADQEQHVY.data diff --git a/test/repo-example/blocks/1220f8a0/1220f8a01860aa759f3e2f32aee26c0d0b696cc966dac959429127134d1d2592a7de.data b/test/repo-example/blocks/CIQPR/CIQPRIAYMCVHLHZ6F4ZK5YTMBUFWS3GJM3NMSWKCSETRGTI5EWJKPXQ.data similarity index 100% rename from test/repo-example/blocks/1220f8a0/1220f8a01860aa759f3e2f32aee26c0d0b696cc966dac959429127134d1d2592a7de.data rename to test/repo-example/blocks/CIQPR/CIQPRIAYMCVHLHZ6F4ZK5YTMBUFWS3GJM3NMSWKCSETRGTI5EWJKPXQ.data diff --git a/test/repo-example/datastore/CURRENT b/test/repo-example/datastore/CURRENT index 6ba31a31..5b540107 100644 --- a/test/repo-example/datastore/CURRENT +++ b/test/repo-example/datastore/CURRENT @@ -1 +1 @@ -MANIFEST-000009 +MANIFEST-000011 diff --git a/test/repo-example/datastore/LOG b/test/repo-example/datastore/LOG index f5ffd612..fb2ef830 100644 --- a/test/repo-example/datastore/LOG +++ b/test/repo-example/datastore/LOG @@ -1,7 +1,5 @@ -=============== Apr 22, 2016 (WEST) =============== -03:16:42.272495 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed -03:16:42.272857 db@open opening -03:16:42.275673 db@janitor F·4 G·0 -03:16:42.275700 db@open done T·2.831108ms -03:16:42.596938 db@close closing -03:16:42.597082 db@close done T·139.194µs +=============== Aug 19, 2016 (CEST) =============== +15:48:10.633634 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +15:48:10.634191 db@open opening +15:48:10.639318 db@janitor F·4 G·0 +15:48:10.639379 db@open done T·5.16729ms diff --git a/test/repo-example/datastore/LOG.old b/test/repo-example/datastore/LOG.old index 863b68fd..f5ffd612 100644 --- a/test/repo-example/datastore/LOG.old +++ b/test/repo-example/datastore/LOG.old @@ -1,10 +1,7 @@ -=============== Dec 10, 2015 (PST) =============== -07:50:02.056578 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed -07:50:02.057231 db@open opening -07:50:02.057312 journal@recovery F·1 -07:50:02.057514 journal@recovery recovering @3 -07:50:02.058921 mem@flush created L0@5 N·4 S·1KiB "/ip..\xf6\xe4\xa9,v5":"/pk..\xf6\xe4\xa9,v6" -07:50:02.059983 db@janitor F·4 G·0 -07:50:02.060001 db@open done T·2.755926ms -07:50:02.073183 db@close closing -07:50:02.073285 db@close done T·97.522µs +=============== Apr 22, 2016 (WEST) =============== +03:16:42.272495 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +03:16:42.272857 db@open opening +03:16:42.275673 db@janitor F·4 G·0 +03:16:42.275700 db@open done T·2.831108ms +03:16:42.596938 db@close closing +03:16:42.597082 db@close done T·139.194µs diff --git a/test/repo-example/datastore/MANIFEST-000009 b/test/repo-example/datastore/MANIFEST-000009 deleted file mode 100644 index a69b18a4..00000000 Binary files a/test/repo-example/datastore/MANIFEST-000009 and /dev/null differ diff --git a/test/repo-example/datastore/MANIFEST-000011 b/test/repo-example/datastore/MANIFEST-000011 new file mode 100644 index 00000000..7af87ca8 Binary files /dev/null and b/test/repo-example/datastore/MANIFEST-000011 differ diff --git a/test/repo-example/version b/test/repo-example/version index 00750edc..b8626c4c 100644 --- a/test/repo-example/version +++ b/test/repo-example/version @@ -1 +1 @@ -3 +4 diff --git a/test/test-exporter.js b/test/test-exporter.js index 012d141b..baa7f174 100644 --- a/test/test-exporter.js +++ b/test/test-exporter.js @@ -1,168 +1,186 @@ /* eslint-env mocha */ 'use strict' -const unixFSEngine = require('./../src') -const exporter = unixFSEngine.exporter const expect = require('chai').expect const BlockService = require('ipfs-block-service') const DAGService = require('ipfs-merkle-dag').DAGService const UnixFS = require('ipfs-unixfs') -const concat = require('concat-stream') const fs = require('fs') const path = require('path') const bs58 = require('bs58') +const pull = require('pull-stream') +const zip = require('pull-zip') + +const unixFSEngine = require('./../src') +const exporter = unixFSEngine.exporter -let ds +module.exports = (repo) => { + describe('exporter', () => { + let ds -module.exports = function (repo) { - describe('exporter', function () { const bigFile = fs.readFileSync(path.join(__dirname, '/test-data/1.2MiB.txt')) - before((done) => { + before(() => { const bs = new BlockService(repo) - expect(bs).to.exist ds = new DAGService(bs) - expect(ds).to.exist - done() }) it('ensure hash inputs are sanitized', (done) => { const hash = 'QmQmZQxSKQppbsWfVzBvg59Cn3DKtsNVQ94bjAxg2h3Lb8' - const bs = new BlockService(repo) - const ds = new DAGService(bs) const mhBuf = new Buffer(bs58.decode(hash)) - ds.get(hash, (err, fetchedNode) => { - const unmarsh = UnixFS.unmarshal(fetchedNode.data) - expect(err).to.not.exist - const testExport = exporter(mhBuf, ds) - testExport.on('error', (err) => { + + pull( + ds.getStream(hash), + pull.map((node) => UnixFS.unmarshal(node.data)), + pull.collect((err, nodes) => { expect(err).to.not.exist + + const unmarsh = nodes[0] + + pull( + exporter(mhBuf, ds), + pull.collect(onFiles) + ) + + function onFiles (err, files) { + expect(err).to.not.exist + expect(files).to.have.length(1) + expect(files[0]).to.have.property('path', hash) + + fileEql(files[0], unmarsh.data, done) + } }) - testExport.pipe(concat(onFiles)) - - function onFiles (files) { - expect(files).to.be.length(1) - expect(files[0].path).to.equal(hash) - files[0].content.pipe(concat((bldata) => { - expect(bldata).to.deep.equal(unmarsh.data) - done() - })) - } - }) + ) }) it('export a file with no links', (done) => { const hash = 'QmQmZQxSKQppbsWfVzBvg59Cn3DKtsNVQ94bjAxg2h3Lb8' - const bs = new BlockService(repo) - const ds = new DAGService(bs) - ds.get(hash, (err, fetchedNode) => { - const unmarsh = UnixFS.unmarshal(fetchedNode.data) - expect(err).to.not.exist - const testExport = exporter(hash, ds) - testExport.on('error', (err) => { + + pull( + zip( + pull( + ds.getStream(hash), + pull.map((node) => UnixFS.unmarshal(node.data)) + ), + exporter(hash, ds) + ), + pull.collect((err, values) => { expect(err).to.not.exist + const unmarsh = values[0][0] + const file = values[0][1] + + fileEql(file, unmarsh.data, done) }) - testExport.pipe(concat(onFiles)) - - function onFiles (files) { - expect(files).to.be.length(1) - files[0].content.pipe(concat((bldata) => { - expect(bldata).to.deep.equal(unmarsh.data) - done() - })) - } - }) + ) }) it('export a small file with links', (done) => { const hash = 'QmW7BDxEbGqxxSYVtn3peNPQgdDXbWkoQ6J1EFYAEuQV3Q' - const bs = new BlockService(repo) - const ds = new DAGService(bs) - const testExport = exporter(hash, ds) - testExport.on('error', (err) => { - expect(err).to.not.exist - }) - testExport.on('data', (file) => { - file.content.pipe(concat((bldata) => { - expect(bldata).to.deep.equal(bigFile) - done() - })) - }) + pull( + exporter(hash, ds), + pull.collect((err, files) => { + expect(err).to.not.exist + + fileEql(files[0], bigFile, done) + }) + ) }) it('export a large file > 5mb', (done) => { const hash = 'QmRQgufjp9vLE8XK2LGKZSsPCFCF6e4iynCQtNB5X2HBKE' - const bs = new BlockService(repo) - const ds = new DAGService(bs) - const testExport = exporter(hash, ds) - testExport.on('error', (err) => { - expect(err).to.not.exist - }) - testExport.on('data', (file) => { - expect(file.path).to.equal('QmRQgufjp9vLE8XK2LGKZSsPCFCF6e4iynCQtNB5X2HBKE') - file.content.pipe(concat((bldata) => { - expect(bldata).to.exist - done() - })) - }) + pull( + exporter(hash, ds), + pull.collect((err, files) => { + expect(err).to.not.exist + + expect(files[0]).to.have.property('path', 'QmRQgufjp9vLE8XK2LGKZSsPCFCF6e4iynCQtNB5X2HBKE') + fileEql(files[0], null, done) + }) + ) }) it('export a directory', (done) => { const hash = 'QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN' - const bs = new BlockService(repo) - const ds = new DAGService(bs) - const testExport = exporter(hash, ds) - testExport.on('error', (err) => { - expect(err).to.not.exist - }) - testExport.pipe(concat((files) => { - expect(files[0].path).to.equal('QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN') - expect(files[0].content).to.not.exist - - expect(files[1].path).to.equal('QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN/200Bytes.txt') - expect(files[1].content).to.exist - - expect(files[2].path).to.equal('QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN/dir-another') - expect(files[2].content).to.not.exist - expect(files[3].path).to.equal('QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN/level-1') - expect(files[3].content).to.not.exist - - expect(files[4].path).to.equal('QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN/level-1/200Bytes.txt') - expect(files[4].content).to.exist - - expect(files[5].path).to.equal('QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN/level-1/level-2') - expect(files[5].content).to.not.exist + pull( + exporter(hash, ds), + pull.collect((err, files) => { + expect(err).to.not.exist - done() - })) + expect( + files.map((file) => file.path) + ).to.be.eql([ + 'QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN', + 'QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN/200Bytes.txt', + 'QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN/dir-another', + 'QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN/level-1', + 'QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN/level-1/200Bytes.txt', + 'QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN/level-1/level-2' + ]) + + pull( + pull.values(files), + pull.map((file) => Boolean(file.content)), + pull.collect((err, contents) => { + expect(err).to.not.exist + expect(contents).to.be.eql([ + false, + true, + false, + false, + true, + false + ]) + done() + }) + ) + }) + ) }) - it('returns a null stream for dir', (done) => { + it('returns an empty stream for dir', (done) => { const hash = 'QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn' - const bs = new BlockService(repo) - const ds = new DAGService(bs) - const testExport = exporter(hash, ds) - - testExport.on('error', (err) => { - expect(err).to.not.exist - }) - testExport.on('data', (file) => { - expect(file.content).to.not.exist - done() - }) + pull( + exporter(hash, ds), + pull.collect((err, files) => { + expect(err).to.not.exist + expect(files[0].content).to.not.exist + done() + }) + ) }) it('fails on non existent hash', (done) => { // This hash doesn't exist in the repo const hash = 'QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKj3' - const bs = new BlockService(repo) - const ds = new DAGService(bs) - const testExport = exporter(hash, ds) - testExport.on('error', (err) => { - expect(err).to.exist - done() - }) + + pull( + exporter(hash, ds), + pull.collect((err, files) => { + expect(err).to.exist + done() + }) + ) }) }) } + +function fileEql (f1, f2, done) { + pull( + f1.content, + pull.collect((err, data) => { + if (err) return done(err) + + try { + if (f2) { + expect(Buffer.concat(data)).to.be.eql(f2) + } else { + expect(data).to.exist + } + } catch (err) { + return done(err) + } + done() + }) + ) +} diff --git a/test/test-fixed-size-chunker.js b/test/test-fixed-size-chunker.js index 516b3cb0..08a1a250 100644 --- a/test/test-fixed-size-chunker.js +++ b/test/test-fixed-size-chunker.js @@ -1,58 +1,97 @@ /* eslint-env mocha */ 'use strict' -const FixedSizeChunker = require('./../src/chunker-fixed-size') +const chunker = require('./../src/chunker-fixed-size') const fs = require('fs') const expect = require('chai').expect -const stringToStream = require('string-to-stream') -const through = require('through2') const path = require('path') +const pull = require('pull-stream') -const fileStream = () => stringToStream( - fs.readFileSync( - path.join(__dirname, '/test-data/1MiB.txt') - ).toString('hex') +const rawFile = fs.readFileSync( + path.join(__dirname, '/test-data/1MiB.txt') ) -describe('chunker: fixed size', function () { - it('256 Bytes chunks', function (done) { - fileStream() - .pipe(FixedSizeChunker(256)) - .pipe(through((chunk, enc, cb) => { - expect(chunk.length).to.equal(256) - cb() - }, () => { +describe('chunker: fixed size', () => { + it('chunks non flat buffers', (done) => { + const b1 = new Buffer(2 * 256) + const b2 = new Buffer(1 * 256) + const b3 = new Buffer(5 * 256) + + b1.fill('a') + b2.fill('b') + b3.fill('c') + + pull( + pull.values([b1, b2, b3]), + chunker(256), + pull.collect((err, chunks) => { + expect(err).to.not.exists + expect(chunks).to.have.length(8) + chunks.forEach((chunk) => { + expect(chunk).to.have.length(256) + }) + done() + }) + ) + }) + + it('256 Bytes chunks', (done) => { + pull( + pull.infinite(() => Buffer([1])), + pull.take(256 * 12), + chunker(256), + pull.collect((err, chunks) => { + expect(err).to.not.exists + expect(chunks).to.have.length(12) + chunks.forEach((chunk) => { + expect(chunk).to.have.length(256) + }) done() - })) + }) + ) }) - it('256 KiB chunks', function (done) { + it('256 KiB chunks', (done) => { const KiB256 = 262144 - fileStream() - .pipe(FixedSizeChunker(KiB256)) - .pipe(through((chunk, enc, cb) => { - expect(chunk.length).to.equal(KiB256) - cb() - }, () => { + pull( + pull.values(rawFile), + chunker(KiB256), + pull.collect((err, chunks) => { + expect(err).to.not.exists + + expect(chunks).to.have.length(4) + chunks.forEach((chunk) => { + expect(chunk).to.have.length(KiB256) + }) done() - })) + }) + ) }) - it('256 KiB chunks of non scalar filesize', function (done) { - let counter = 0 + it('256 KiB chunks of non scalar filesize', (done) => { const KiB256 = 262144 - fileStream() - .pipe(FixedSizeChunker(KiB256)) - .pipe(through((chunk, enc, cb) => { - if (chunk.length < KiB256) { - expect(counter).to.be.below(2) - counter += 1 - return cb() - } - expect(chunk.length).to.equal(KiB256) - cb() - }, () => { + let file = Buffer.concat([rawFile, new Buffer('hello')]) + + pull( + pull.values(file), + chunker(KiB256), + pull.collect((err, chunks) => { + expect(err).to.not.exists + + expect(chunks).to.have.length(5) + let counter = 0 + + chunks.forEach((chunk) => { + if (chunk.length < KiB256) { + counter++ + } else { + expect(chunk).to.have.length(KiB256) + } + }) + + expect(counter).to.be.eql(1) done() - })) + }) + ) }) }) diff --git a/test/test-importer.js b/test/test-importer.js index fea1c465..d2eaa0e0 100644 --- a/test/test-importer.js +++ b/test/test-importer.js @@ -1,19 +1,26 @@ /* eslint-env mocha */ 'use strict' -const Importer = require('./../src').importer +const importer = require('./../src').importer const expect = require('chai').expect const BlockService = require('ipfs-block-service') const DAGService = require('ipfs-merkle-dag').DAGService -const bs58 = require('bs58') const fs = require('fs') const path = require('path') -const streamifier = require('streamifier') +const pull = require('pull-stream') +const mh = require('multihashes') -let ds +function stringifyMh (files) { + return files.map((file) => { + file.multihash = mh.toB58String(file.multihash) + return file + }) +} module.exports = function (repo) { describe('importer', function () { + let ds + const bigFile = fs.readFileSync(path.join(__dirname, '/test-data/1.2MiB.txt')) const smallFile = fs.readFileSync(path.join(__dirname, '/test-data/200Bytes.txt')) @@ -21,119 +28,143 @@ module.exports = function (repo) { // const dirBig = path.join(__dirname, '/test-data/dir-big') // const dirNested = path.join(__dirname, '/test-data/dir-nested') - before((done) => { + before(() => { const bs = new BlockService(repo) - expect(bs).to.exist ds = new DAGService(bs) - expect(ds).to.exist - done() }) it('bad input', (done) => { - const r = 'banana' - const i = new Importer(ds) - i.on('error', (err) => { - expect(err).to.exist - done() - }) - i.write({path: '200Bytes.txt', content: r}) - i.end() + pull( + pull.values([{ + path: '200Bytes.txt', + content: 'banana' + }]), + importer(ds), + pull.onEnd((err) => { + expect(err).to.exist + done() + }) + ) }) it('small file (smaller than a chunk)', (done) => { - const buffered = smallFile - const r = streamifier.createReadStream(buffered) - const i = new Importer(ds) - i.on('data', (obj) => { - expect(obj.path).to.equal('200Bytes.txt') - expect(bs58.encode(obj.multihash)).to.equal('QmQmZQxSKQppbsWfVzBvg59Cn3DKtsNVQ94bjAxg2h3Lb8') - expect(obj.size).to.equal(211) - }) - i.write({path: '200Bytes.txt', content: r}) - i.end() - i.on('end', () => { - done() - }) + pull( + pull.values([{ + path: '200Bytes.txt', + content: pull.values([smallFile]) + }]), + importer(ds), + pull.collect((err, files) => { + expect(err).to.not.exist + expect(stringifyMh(files)).to.be.eql([{ + path: '200Bytes.txt', + multihash: 'QmQmZQxSKQppbsWfVzBvg59Cn3DKtsNVQ94bjAxg2h3Lb8', + size: 211 + }]) + done() + }) + ) }) it('small file as buffer (smaller than a chunk)', (done) => { - const buffered = smallFile - const i = new Importer(ds) - i.on('data', (obj) => { - expect(obj.path).to.equal('200Bytes.txt') - expect(bs58.encode(obj.multihash)).to.equal('QmQmZQxSKQppbsWfVzBvg59Cn3DKtsNVQ94bjAxg2h3Lb8') - expect(obj.size).to.equal(211) - }) - i.write({path: '200Bytes.txt', content: buffered}) - i.end() - i.on('end', () => { - done() - }) + pull( + pull.values([{ + path: '200Bytes.txt', + content: smallFile + }]), + importer(ds), + pull.collect((err, files) => { + expect(err).to.not.exist + expect(stringifyMh(files)).to.be.eql([{ + path: '200Bytes.txt', + multihash: 'QmQmZQxSKQppbsWfVzBvg59Cn3DKtsNVQ94bjAxg2h3Lb8', + size: 211 + }]) + done() + }) + ) }) it('small file (smaller than a chunk) inside a dir', (done) => { - const buffered = smallFile - const r = streamifier.createReadStream(buffered) - const i = new Importer(ds) - i.on('data', (file) => { - if (file.path === 'foo/bar/200Bytes.txt') { - expect(bs58.encode(file.multihash).toString()) - .to.equal('QmQmZQxSKQppbsWfVzBvg59Cn3DKtsNVQ94bjAxg2h3Lb8') - } - if (file.path === 'foo/bar') { - expect(bs58.encode(file.multihash).toString()) - .to.equal('Qmf5BQbTUyUAvd6Ewct83GYGnE1F6btiC3acLhR8MDxgkD') - } - if (file.path === 'foo') { - expect(bs58.encode(file.multihash).toString()) - .to.equal('QmQrb6KKWGo8w7zKfx2JksptY6wN7B2ysSBdKZr4xMU36d') - } - }) - i.on('error', (err) => { - expect(err).to.not.exist - }) - i.on('end', () => { - done() - }) - i.write({path: 'foo/bar/200Bytes.txt', content: r}) - i.end() + pull( + pull.values([{ + path: 'foo/bar/200Bytes.txt', + content: pull.values([smallFile]) + }]), + importer(ds), + pull.collect((err, files) => { + expect(err).to.not.exist + expect(files.length).to.equal(3) + stringifyMh(files).forEach((file) => { + if (file.path === 'foo/bar/200Bytes.txt') { + expect(file).to.be.eql({ + path: 'foo/bar/200Bytes.txt', + multihash: 'QmQmZQxSKQppbsWfVzBvg59Cn3DKtsNVQ94bjAxg2h3Lb8', + size: 211 + }) + } + if (file.path === 'foo') { + expect(file).to.be.eql({ + path: 'foo', + multihash: 'QmQrb6KKWGo8w7zKfx2JksptY6wN7B2ysSBdKZr4xMU36d', + size: 320 + }) + } + if (file.path === 'foo/bar') { + expect(file).to.be.eql({ + path: 'foo/bar', + multihash: 'Qmf5BQbTUyUAvd6Ewct83GYGnE1F6btiC3acLhR8MDxgkD', + size: 270 + }) + } + }) + done() + }) + ) }) it('file bigger than a single chunk', (done) => { - const buffered = bigFile - const r = streamifier.createReadStream(buffered) - const i = new Importer(ds) - i.on('data', (file) => { - expect(file.path).to.equal('1.2MiB.txt') - expect(bs58.encode(file.multihash)).to.equal('QmW7BDxEbGqxxSYVtn3peNPQgdDXbWkoQ6J1EFYAEuQV3Q') - expect(file.size).to.equal(1258318) - }) - i.on('end', () => { - done() - }) - i.write({path: '1.2MiB.txt', content: r}) - i.end() + pull( + pull.values([{ + path: '1.2MiB.txt', + content: pull.values([bigFile]) + }]), + importer(ds), + pull.collect((err, files) => { + expect(err).to.not.exist + expect(stringifyMh(files)).to.be.eql([{ + path: '1.2MiB.txt', + multihash: 'QmW7BDxEbGqxxSYVtn3peNPQgdDXbWkoQ6J1EFYAEuQV3Q', + size: 1258318 + }]) + done() + }) + ) }) it('file bigger than a single chunk inside a dir', (done) => { - const buffered = bigFile - const r = streamifier.createReadStream(buffered) - const i = new Importer(ds) - i.on('data', (file) => { - if (file.path === 'foo-big/1.2Mib.txt') { - expect(bs58.encode(file.multihash)).to.equal('QmW7BDxEbGqxxSYVtn3peNPQgdDXbWkoQ6J1EFYAEuQV3Q') - expect(file.size).to.equal(1258318) - } - if (file.path === 'foo-big') { - expect(bs58.encode(file.multihash)).to.equal('QmaFgyFJUP4fxFySJCddg2Pj6rpwSywopWk87VEVv52RSj') - expect(file.size).to.equal(1258376) - } - }) - i.on('end', () => { - done() - }) - i.write({path: 'foo-big/1.2MiB.txt', content: r}) - i.end() + pull( + pull.values([{ + path: 'foo-big/1.2MiB.txt', + content: pull.values([bigFile]) + }]), + importer(ds), + pull.collect((err, files) => { + expect(err).to.not.exist + + expect(stringifyMh(files)).to.be.eql([{ + path: 'foo-big/1.2MiB.txt', + multihash: 'QmW7BDxEbGqxxSYVtn3peNPQgdDXbWkoQ6J1EFYAEuQV3Q', + size: 1258318 + }, { + path: 'foo-big', + multihash: 'QmaFgyFJUP4fxFySJCddg2Pj6rpwSywopWk87VEVv52RSj', + size: 1258376 + }]) + + done() + }) + ) }) it.skip('file (that chunk number exceeds max links)', (done) => { @@ -141,90 +172,101 @@ module.exports = function (repo) { }) it('empty directory', (done) => { - const i = new Importer(ds) - i.on('data', (file) => { - expect(file.path).to.equal('empty-dir') - expect(bs58.encode(file.multihash)).to.equal('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') - expect(file.size).to.equal(4) - }) - i.on('error', (err) => { - expect(err).to.not.exist - }) - i.on('end', () => { - done() - }) - i.write({path: 'empty-dir'}) - i.end() + pull( + pull.values([{ + path: 'empty-dir' + }]), + importer(ds), + pull.collect((err, files) => { + expect(err).to.not.exist + + expect(stringifyMh(files)).to.be.eql([{ + path: 'empty-dir', + multihash: 'QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn', + size: 4 + }]) + + done() + }) + ) }) it('directory with files', (done) => { - const r1 = streamifier.createReadStream(smallFile) - const r2 = streamifier.createReadStream(bigFile) - - const i = new Importer(ds) - i.on('data', (file) => { - if (file.path === 'pim/200Bytes.txt') { - expect(bs58.encode(file.multihash).toString()) - .to.equal('QmQmZQxSKQppbsWfVzBvg59Cn3DKtsNVQ94bjAxg2h3Lb8') - } - if (file.path === 'pim/1.2MiB.txt') { - expect(bs58.encode(file.multihash).toString()) - .to.equal('QmW7BDxEbGqxxSYVtn3peNPQgdDXbWkoQ6J1EFYAEuQV3Q') - } - if (file.path === 'pim') { - expect(bs58.encode(file.multihash).toString()) - .to.equal('QmY8a78tx6Tk6naDgWCgTsd9EqGrUJRrH7dDyQhjyrmH2i') - } - }) - i.on('error', (err) => { - expect(err).to.not.exist - }) - i.on('end', () => { - done() - }) - i.write({path: 'pim/200Bytes.txt', content: r1}) - i.write({path: 'pim/1.2MiB.txt', content: r2}) - i.end() + pull( + pull.values([{ + path: 'pim/200Bytes.txt', + content: pull.values([smallFile]) + }, { + path: 'pim/1.2MiB.txt', + content: pull.values([bigFile]) + }]), + importer(ds), + pull.collect((err, files) => { + expect(err).to.not.exist + + expect(stringifyMh(files)).be.eql([{ + path: 'pim/200Bytes.txt', + multihash: 'QmQmZQxSKQppbsWfVzBvg59Cn3DKtsNVQ94bjAxg2h3Lb8', + size: 211 + }, { + path: 'pim/1.2MiB.txt', + multihash: 'QmW7BDxEbGqxxSYVtn3peNPQgdDXbWkoQ6J1EFYAEuQV3Q', + size: 1258318 + }, { + path: 'pim', + multihash: 'QmY8a78tx6Tk6naDgWCgTsd9EqGrUJRrH7dDyQhjyrmH2i', + size: 1258642 + }]) + + done() + }) + ) }) it('nested directory (2 levels deep)', (done) => { - const r1 = streamifier.createReadStream(smallFile) - const r2 = streamifier.createReadStream(bigFile) - const r3 = streamifier.createReadStream(bigFile) - - const i = new Importer(ds) - i.on('data', (file) => { - if (file.path === 'pam/pum/200Bytes.txt') { - expect(bs58.encode(file.multihash).toString()) - .to.equal('QmQmZQxSKQppbsWfVzBvg59Cn3DKtsNVQ94bjAxg2h3Lb8') - } - if (file.path === 'pam/pum/1.2MiB.txt') { - expect(bs58.encode(file.multihash).toString()) - .to.equal('QmW7BDxEbGqxxSYVtn3peNPQgdDXbWkoQ6J1EFYAEuQV3Q') - } - if (file.path === 'pam/1.2MiB.txt') { - expect(bs58.encode(file.multihash).toString()) - .to.equal('QmW7BDxEbGqxxSYVtn3peNPQgdDXbWkoQ6J1EFYAEuQV3Q') - } - if (file.path === 'pam/pum') { - expect(bs58.encode(file.multihash).toString()) - .to.equal('QmY8a78tx6Tk6naDgWCgTsd9EqGrUJRrH7dDyQhjyrmH2i') - } - if (file.path === 'pam') { - expect(bs58.encode(file.multihash).toString()) - .to.equal('QmRgdtzNx1H1BPJqShdhvWZ2D4DA2HUgZJ3XLtoXei27Av') - } - }) - i.on('error', (err) => { - expect(err).to.not.exist - }) - i.on('end', () => { - done() - }) - i.write({path: 'pam/pum/200Bytes.txt', content: r1}) - i.write({path: 'pam/pum/1.2MiB.txt', content: r2}) - i.write({path: 'pam/1.2MiB.txt', content: r3}) - i.end() + pull( + pull.values([{ + path: 'pam/pum/200Bytes.txt', + content: pull.values([smallFile]) + }, { + path: 'pam/pum/1.2MiB.txt', + content: pull.values([bigFile]) + }, { + path: 'pam/1.2MiB.txt', + content: pull.values([bigFile]) + }]), + importer(ds), + pull.collect((err, files) => { + expect(err).to.not.exist + + // need to sort as due to parallel storage the order + // can vary + stringifyMh(files).forEach((file) => { + if (file.path === 'pam/pum/200Bytes.txt') { + expect(file.multihash).to.be.eql('QmQmZQxSKQppbsWfVzBvg59Cn3DKtsNVQ94bjAxg2h3Lb8') + expect(file.size).to.be.eql(211) + } + if (file.path === 'pam/pum/1.2MiB.txt') { + expect(file.multihash).to.be.eql('QmW7BDxEbGqxxSYVtn3peNPQgdDXbWkoQ6J1EFYAEuQV3Q') + expect(file.size).to.be.eql(1258318) + } + if (file.path === 'pam/pum') { + expect(file.multihash).to.be.eql('QmY8a78tx6Tk6naDgWCgTsd9EqGrUJRrH7dDyQhjyrmH2i') + expect(file.size).to.be.eql(1258642) + } + if (file.path === 'pam/1.2MiB.txt') { + expect(file.multihash).to.be.eql('QmW7BDxEbGqxxSYVtn3peNPQgdDXbWkoQ6J1EFYAEuQV3Q') + expect(file.size).to.be.eql(1258318) + } + if (file.path === 'pam') { + expect(file.multihash).to.be.eql('QmRgdtzNx1H1BPJqShdhvWZ2D4DA2HUgZJ3XLtoXei27Av') + expect(file.size).to.be.eql(2517065) + } + }) + + done() + }) + ) }) }) }