Skip to content

Commit 103e999

Browse files
achingbrainachingbrain
committed
feat: consolidate ipfs.add input normalisation
Allows input normalisation function to be shared between ipfs and the http client.
1 parent 4065292 commit 103e999

File tree

3 files changed

+336
-1
lines changed

3 files changed

+336
-1
lines changed

‎package.json

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
"license": "MIT",
2828
"dependencies": {
2929
"buffer": "^5.2.1",
30+
"err-code": "^2.0.0",
3031
"is-buffer": "^2.0.3",
3132
"is-electron": "^2.2.0",
3233
"is-pull-stream": "0.0.0",
@@ -36,9 +37,10 @@
3637
},
3738
"devDependencies": {
3839
"aegir": "^20.0.0",
40+
"async-iterator-all": "^1.0.0",
3941
"chai": "^4.2.0",
4042
"dirty-chai": "^2.0.1",
41-
"electron": "^5.0.7",
43+
"electron": "^6.0.6",
4244
"electron-mocha": "^8.0.3",
4345
"pull-stream": "^3.6.13"
4446
},

‎src/files/normalise-input.js

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
'use strict'
2+
3+
const errCode = require('err-code')
4+
const { Buffer } = require('buffer')
5+
6+
/*
7+
* Transform one of:
8+
*
9+
* ```
10+
* Buffer|ArrayBuffer|TypedArray
11+
* Blob|File
12+
* { path, content: Blob }
13+
* { path, content: String }
14+
* { path, content: Iterable<Number> }
15+
* { path, content: Iterable<Buffer> }
16+
* { path, content: Iterable<Iterable<Number>> }
17+
* { path, content: AsyncIterable<Iterable<Number>> }
18+
* String
19+
* Iterable<Number>
20+
* Iterable<Buffer>
21+
* Iterable<Blob>
22+
* Iterable<{ path, content: Buffer }>
23+
* Iterable<{ path, content: Blob }>
24+
* Iterable<{ path, content: Iterable<Number> }>
25+
* Iterable<{ path, content: AsyncIterable<Buffer> }>
26+
* AsyncIterable<Buffer>
27+
* AsyncIterable<{ path, content: Buffer }>
28+
* AsyncIterable<{ path, content: Blob }>
29+
* AsyncIterable<{ path, content: Iterable<Buffer> }>
30+
* AsyncIterable<{ path, content: AsyncIterable<Buffer> }>
31+
* ```
32+
* Into:
33+
*
34+
* ```
35+
* AsyncIterable<{ path, content: AsyncIterable<Buffer> }>
36+
* ```
37+
*
38+
* @param input Object
39+
* @return AsyncInterable<{ path, content: AsyncIterable<Buffer> }>
40+
*/
41+
module.exports = function normaliseInput (input) {
42+
// must give us something
43+
if (input === null || input === undefined) {
44+
throw errCode(new Error(`Unexpected input: ${input}`, 'ERR_UNEXPECTED_INPUT'))
45+
}
46+
47+
// { path, content: ? }
48+
if (isFileObject(input)) {
49+
return (async function * () { // eslint-disable-line require-await
50+
yield toFileObject(input)
51+
})()
52+
}
53+
54+
// String
55+
if (typeof input === 'string' || input instanceof String) {
56+
return (async function * () { // eslint-disable-line require-await
57+
yield toFileObject(input)
58+
})()
59+
}
60+
61+
// Buffer|ArrayBuffer|TypedArray
62+
// Blob|File
63+
if (isBytes(input) || isBloby(input)) {
64+
return (async function * () { // eslint-disable-line require-await
65+
yield toFileObject(input)
66+
})()
67+
}
68+
69+
// Iterable<?>
70+
if (input[Symbol.iterator]) {
71+
// Iterable<Number>
72+
if (!isNaN(input[0])) {
73+
return (async function * () { // eslint-disable-line require-await
74+
yield toFileObject([input])
75+
})()
76+
}
77+
78+
// Iterable<Buffer>
79+
// Iterable<Blob>
80+
return (async function * () { // eslint-disable-line require-await
81+
for (const chunk of input) {
82+
yield toFileObject(chunk)
83+
}
84+
})()
85+
}
86+
87+
// AsyncIterable<?>
88+
if (input[Symbol.asyncIterator]) {
89+
return (async function * () { // eslint-disable-line require-await
90+
for await (const chunk of input) {
91+
yield toFileObject(chunk)
92+
}
93+
})()
94+
}
95+
96+
throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT')
97+
}
98+
99+
function toFileObject (input) {
100+
return {
101+
path: input.path || '',
102+
content: toAsyncIterable(input.content || input)
103+
}
104+
}
105+
106+
function toAsyncIterable (input) {
107+
// Buffer|ArrayBuffer|TypedArray|array of bytes
108+
if (isBytes(input)) {
109+
return (async function * () { // eslint-disable-line require-await
110+
yield toBuffer(input)
111+
})()
112+
}
113+
114+
if (typeof input === 'string' || input instanceof String) {
115+
return (async function * () { // eslint-disable-line require-await
116+
yield toBuffer(input)
117+
})()
118+
}
119+
120+
// Blob|File
121+
if (isBloby(input)) {
122+
return blobToAsyncGenerator(input)
123+
}
124+
125+
// Iterator<?>
126+
if (input[Symbol.iterator]) {
127+
if (!isNaN(input[0])) {
128+
return (async function * () { // eslint-disable-line require-await
129+
yield toBuffer(input)
130+
})()
131+
}
132+
133+
return (async function * () { // eslint-disable-line require-await
134+
for (const chunk of input) {
135+
yield toBuffer(chunk)
136+
}
137+
})()
138+
}
139+
140+
// AsyncIterable<?>
141+
if (input[Symbol.asyncIterator]) {
142+
return (async function * () {
143+
for await (const chunk of input) {
144+
yield toBuffer(chunk)
145+
}
146+
})()
147+
}
148+
149+
throw errCode(new Error(`Unexpected input: ${input}`, 'ERR_UNEXPECTED_INPUT'))
150+
}
151+
152+
function toBuffer (chunk) {
153+
if (isBytes(chunk)) {
154+
return chunk
155+
}
156+
157+
if (typeof chunk === 'string' || chunk instanceof String) {
158+
return Buffer.from(chunk)
159+
}
160+
161+
if (Array.isArray(chunk)) {
162+
return Buffer.from(chunk)
163+
}
164+
165+
throw new Error('Unexpected input: ' + typeof chunk)
166+
}
167+
168+
function isBytes (obj) {
169+
return Buffer.isBuffer(obj) || ArrayBuffer.isView(obj) || obj instanceof ArrayBuffer
170+
}
171+
172+
function isBloby (obj) {
173+
return typeof Blob !== 'undefined' && obj instanceof global.Blob
174+
}
175+
176+
// An object with a path or content property
177+
function isFileObject (obj) {
178+
return typeof obj === 'object' && (obj.path || obj.content)
179+
}
180+
181+
async function * blobToAsyncGenerator (blob) {
182+
const reader = blob.stream().getReader()
183+
184+
while (true) {
185+
const result = await reader.read()
186+
187+
if (result.done) {
188+
return
189+
}
190+
191+
yield result.value
192+
}
193+
}

‎test/files/normalise-input.spec.js

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
'use strict'
2+
3+
/* eslint-env mocha */
4+
const chai = require('chai')
5+
const dirtyChai = require('dirty-chai')
6+
const normalise = require('../../src/files/normalise-input')
7+
const { supportsFileReader } = require('../../src/supports')
8+
const { Buffer } = require('buffer')
9+
const all = require('async-iterator-all')
10+
11+
chai.use(dirtyChai)
12+
const expect = chai.expect
13+
14+
const STRING = 'hello world'
15+
const BUFFER = Buffer.from(STRING)
16+
const ARRAY = Array.from(BUFFER)
17+
let BLOB
18+
19+
if (supportsFileReader) {
20+
BLOB = new global.Blob([
21+
STRING
22+
], {
23+
type: 'application/octet-stream'
24+
})
25+
}
26+
27+
async function verifyNormalisation (input) {
28+
expect(input.length).to.equal(1)
29+
30+
if (!input[0].content[Symbol.asyncIterator] && !input[0].content[Symbol.iterator]) {
31+
chai.assert.fail(`Content should have been an iterable or an async iterable`)
32+
}
33+
34+
expect(await all(input[0].content)).to.deep.equal([BUFFER])
35+
expect(input[0].path).to.equal('')
36+
}
37+
38+
async function testContent (input) {
39+
const result = await all(normalise(input))
40+
41+
await verifyNormalisation(result)
42+
}
43+
44+
function iterableOf (thing) {
45+
return [thing]
46+
}
47+
48+
function asyncIterableOf (thing) {
49+
return (async function * () { // eslint-disable-line require-await
50+
yield thing
51+
}())
52+
}
53+
54+
describe('normalise-input', function () {
55+
function testInputType (content, name) {
56+
it(name, async function () {
57+
await testContent(content)
58+
})
59+
60+
it(`Iterable<${name}>`, async function () {
61+
await testContent(iterableOf(content))
62+
})
63+
64+
it(`AsyncIterable<${name}>`, async function () {
65+
await testContent(asyncIterableOf(content))
66+
})
67+
68+
if (name !== 'Blob') {
69+
it(`AsyncIterable<Iterable<${name}>>`, async function () {
70+
await testContent(asyncIterableOf(iterableOf(content)))
71+
})
72+
73+
it(`AsyncIterable<AsyncIterable<${name}>>`, async function () {
74+
await testContent(asyncIterableOf(asyncIterableOf(content)))
75+
})
76+
}
77+
78+
it(`{ path: '', content: ${name} }`, async function () {
79+
await testContent({ path: '', content })
80+
})
81+
82+
if (name !== 'Blob') {
83+
it(`{ path: '', content: Iterable<${name}> }`, async function () {
84+
await testContent({ path: '', content: iterableOf(content) })
85+
})
86+
87+
it(`{ path: '', content: AsyncIterable<${name}> }`, async function () {
88+
await testContent({ path: '', content: asyncIterableOf(content) })
89+
})
90+
}
91+
92+
it(`Iterable<{ path: '', content: ${name} }`, async function () {
93+
await testContent(iterableOf({ path: '', content }))
94+
})
95+
96+
if (name !== 'Blob') {
97+
it(`Iterable<{ path: '', content: Iterable<${name}> }>`, async function () {
98+
await testContent(iterableOf({ path: '', content: iterableOf(content) }))
99+
})
100+
101+
it(`Iterable<{ path: '', content: AsyncIterable<${name}> }>`, async function () {
102+
await testContent(iterableOf({ path: '', content: asyncIterableOf(content) }))
103+
})
104+
}
105+
106+
it(`AsyncIterable<{ path: '', content: ${name} }`, async function () {
107+
await testContent(asyncIterableOf({ path: '', content }))
108+
})
109+
110+
if (name !== 'Blob') {
111+
it(`AsyncIterable<{ path: '', content: Iterable<${name}> }>`, async function () {
112+
await testContent(asyncIterableOf({ path: '', content: iterableOf(content) }))
113+
})
114+
115+
it(`AsyncIterable<{ path: '', content: AsyncIterable<${name}> }>`, async function () {
116+
await testContent(asyncIterableOf({ path: '', content: asyncIterableOf(content) }))
117+
})
118+
}
119+
}
120+
121+
describe('String', () => {
122+
testInputType(STRING, 'String')
123+
})
124+
125+
describe('Buffer', () => {
126+
testInputType(BUFFER, 'Buffer')
127+
})
128+
129+
describe('Blob', () => {
130+
if (!supportsFileReader) {
131+
return
132+
}
133+
134+
testInputType(BLOB, 'Blob')
135+
})
136+
137+
describe('Iterable<Number>', () => {
138+
testInputType(ARRAY, 'Iterable<Number>')
139+
})
140+
})

0 commit comments

Comments
 (0)