known differences, buffer.Blob support, private stream, more test

jimmywarting · jimmywarting · commit 943808bbdcec · 2021-05-07T03:09:47.000+02:00
diff --git a/README.md b/README.md
@@ -23,49 +23,78 @@ npm install fetch-blob
     - internal buffers was replaced with Uint8Arrays
     - CommonJS was replaced with ESM
     - The node stream returned by calling `blob.stream()` was replaced with a simple generator function that yields Uint8Array (Breaking change)
+      (Read "Differences from other blobs" for more info.)
 
-  The reasoning behind `Blob.prototype.stream()` is that node readable stream
-  isn't spec compatible with whatwg stream and we didn't want to import a hole whatwg stream polyfill for node
-  or browserify hole node-stream for browsers and picking any flavor over the other. So we decided to opted out
+    All of this changes have made it dependency free of any core node modules, so it would be possible to just import it using http-import from a CDN without any bundling
+
+</details>
+
+<details>
+  <summary>Differences from other Blobs</summary>
+
+  - Unlike NodeJS `buffer.Blob` (Added in: v15.7.0) and browser native Blob this polyfilled version can't be sent via PostMessage
+  - This blob version is more arbitrary, it can be constructed with blob parts that isn't a instance of itself
+  it has to look and behave as a blob to be accepted as a blob part.
+    - The benefit of this is that you can create other types of blobs that don't contain any internal data that has to be read in other ways, such as the `BlobDataItem` created in `from.js` that wraps a file path into a blob-like item and read lazily (nodejs plans to [implement this][fs-blobs] as well)
+  - The `blob.stream()` is the most noticeable differences. It returns a AsyncGeneratorFunction that yields Uint8Arrays
+
+  The reasoning behind `Blob.prototype.stream()` is that NodeJS readable stream
+  isn't spec compatible with whatwg streams and we didn't want to import the hole whatwg stream polyfill for node
+  or browserify NodeJS streams for the browsers and picking any flavor over the other. So we decided to opted out
   of any stream and just implement the bear minium of what both streams have in common which is the asyncIterator
-  that both yields Uint8Array. It would be redundant to convert anything to whatwg streams and than convert it back to
+  that both yields Uint8Array. this is the most isomorphic way with the use of `for-await-of` loops.
+  It would be redundant to convert anything to whatwg streams and than convert it back to
   node streams since you work inside of Node.
   It will probably stay like this until nodejs get native support for whatwg<sup>[1][https://github.com/nodejs/whatwg-stream]</sup> streams and whatwg stream add the node
   equivalent for `Readable.from(iterable)`<sup>[2](https://github.com/whatwg/streams/issues/1018)</sup>
 
-  But for now if you really want/need a Node Stream then you can do so using this transformation
+  But for now if you really need a Node Stream then you can do so using this transformation
   ```js
     import {Readable} from 'stream'
     const stream = Readable.from(blob.stream())
   ```
-  But if you don't need it to be a stream then you can just use the asyncIterator part of it that both whatwg stream and node stream have in common
+  But if you don't need it to be a stream then you can just use the asyncIterator part of it that is isomorphic.
   ```js
     for await (const chunk of blob.stream()) {
       console.log(chunk) // uInt8Array
     }
   ```
-
-  All of this changes have made it dependency free of any core node modules, so it would be possible to just import it using http-import from a CDN without any bundling
-
+  If you need to make some feature detection to fix this different behavior
+  ```js
+  if (Blob.prototype.stream?.constructor?.name === 'AsyncGeneratorFunction') {
+    // not spec compatible, monkey patch it...
+    // (Alternative you could extend the Blob and use super.stream())
+    let orig = Blob.prototype.stream
+    Blob.prototype.stream = function () {
+      const iterator = orig.call(this)
+      return new ReadableStream({
+        async pull (ctrl) {
+          const next = await iterator.next()
+          return next.done ? ctrl.close() : ctrl.enqueue(next.value)
+        }
+      })
+    }
+  }
+  ```
+  Possible feature whatwg version: `ReadableStream.from(iterator)`
+  It's also possible to delete this method and instead use `.slice()` and `.arrayBuffer()` since it has both a public and private stream method
 </details>
 
 ## Usage
 
 ```js
 // Ways to import
-// (note that it's dependency free ESM package so regular http-import from CDN works too)
-import Blob from 'fetch-blob';
-import {Blob} from 'fetch-blob';
-const {Blob} = await import('fetch-blob');
+// (PS it's dependency free ESM package so regular http-import from CDN works too)
+import Blob from 'fetch-blob'
+import {Blob} from 'fetch-blob'
+const {Blob} = await import('fetch-blob')
 
-const blob = new Blob(['hello, world']);
 
 // Ways to read the blob:
+const blob = new Blob(['hello, world'])
 
 await blob.text()
-
 await blob.arrayBuffer()
-
 for await (let chunk of  blob.stream()) { ... }
 
 // turn the async iterator into a node stream
@@ -85,14 +114,14 @@ npm install fetch-blob domexception
 ```js
 // The default export is sync and use fs.stat to retrieve size & last modified
 import blobFromSync from 'fetch-blob/from.js'
-import {Blob, blobFrom, blobFromSync} 'fetch-blob/from.js'
+import {Blob, blobFrom, blobFromSync} from 'fetch-blob/from.js'
 
-const fsBlob1 = blobFromSync('./2-GiB-file.bin');
-const fsBlob2 = await blobFrom('./2-GiB-file.bin');
+const fsBlob1 = blobFromSync('./2-GiB-file.bin')
+const fsBlob2 = await blobFrom('./2-GiB-file.bin')
 
 // Not a 4 GiB memory snapshot, just holds 3 references
 // points to where data is located on the disk
-const blob = new Blob([fsBlob1, fsBlob2, 'memory']);
+const blob = new Blob([fsBlob1, fsBlob2, 'memory'])
 console.log(blob.size) // 4 GiB
 ```
 
@@ -106,3 +135,4 @@ See the [MDN documentation](https://developer.mozilla.org/en-US/docs/Web/API/Blo
 [codecov-url]: https://codecov.io/gh/node-fetch/fetch-blob
 [install-size-image]: https://flat.badgen.net/packagephobia/install/fetch-blob
 [install-size-url]: https://packagephobia.now.sh/result?p=fetch-blob
+[fs-blobs]: https://github.com/nodejs/node/issues/37340
diff --git a/index.js b/index.js
@@ -1,13 +1,43 @@
 // 64 KiB (same size chrome slice theirs blob into Uint8array's)
 const POOL_SIZE = 65536;
 
+/** @param {(Blob | Uint8Array)[]} parts */
+async function * toIterator (parts, clone = true) {
+	for (let part of parts) {
+		if ('stream' in part) {
+			yield * part.stream();
+		} else if (ArrayBuffer.isView(part)) {
+			if (clone) {
+				let position = part.byteOffset;
+				let end = part.byteOffset + part.byteLength;
+				while (position !== end) {
+					const size = Math.min(end - position, POOL_SIZE);
+					const chunk = part.buffer.slice(position, position + size);
+					yield new Uint8Array(chunk);
+					position += chunk.byteLength;
+				}
+			} else {
+				yield part;
+			}
+		} else {
+			// For blobs that have arrayBuffer but no stream method (nodes buffer.Blob)
+			let position = 0;
+			while (position !== part.size) {
+				const chunk = part.slice(position, Math.min(part.size, position + POOL_SIZE));
+				const buffer = await chunk.arrayBuffer();
+				position += buffer.byteLength;
+				yield new Uint8Array(buffer);
+			}
+		}
+	}
+}
+
 export default class Blob {
 
 	/** @type {Array.<(Blob|Uint8Array)>} */
 	#parts = [];
 	#type = '';
 	#size = 0;
-	#avoidClone = false
 
 	/**
 	 * The Blob() constructor returns a new Blob object. The content
@@ -66,12 +96,11 @@ export default class Blob {
 	 * @return {Promise<string>}
 	 */
 	async text() {
-		this.#avoidClone = true
 		// More optimized than using this.arrayBuffer()
 		// that requires twice as much ram
 		const decoder = new TextDecoder();
 		let str = '';
-		for await (let part of this.stream()) {
+		for await (let part of toIterator(this.#parts, false)) {
 			str += decoder.decode(part, { stream: true });
 		}
 		// Remaining
@@ -87,10 +116,9 @@ export default class Blob {
 	 * @return {Promise<ArrayBuffer>}
 	 */
 	async arrayBuffer() {
-		this.#avoidClone = true
 		const data = new Uint8Array(this.size);
 		let offset = 0;
-		for await (const chunk of this.stream()) {
+		for await (const chunk of toIterator(this.#parts, false)) {
 			data.set(chunk, offset);
 			offset += chunk.length;
 		}
@@ -100,30 +128,12 @@ export default class Blob {
 
 	/**
 	 * The Blob stream() implements partial support of the whatwg stream
-	 * by being only async iterable.
+	 * by only being async iterable.
 	 *
 	 * @returns {AsyncGenerator<Uint8Array>}
 	 */
 	async * stream() {
-		for (let part of this.#parts) {
-			if ('stream' in part) {
-				yield * part.stream();
-			} else {
-				if (this.#avoidClone) {
-					yield part
-				} else {
-					let position = part.byteOffset;
-					let end = part.byteOffset + part.byteLength;
-					while (position !== end) {
-						const size = Math.min(end - position, POOL_SIZE);
-						const chunk = part.buffer.slice(position, position + size);
-						yield new Uint8Array(chunk);
-						position += chunk.byteLength;
-					}
-				}
-			}
-		}
-		this.#avoidClone = false
+		yield * toIterator(this.#parts, true);
 	}
 
 	/**
@@ -187,9 +197,11 @@ export default class Blob {
 		return (
 			object &&
 			typeof object === 'object' &&
-			typeof object.stream === 'function' &&
-			object.stream.length === 0 &&
 			typeof object.constructor === 'function' &&
+			(
+				typeof object.stream === 'function' ||
+				typeof object.arrayBuffer === 'function'
+			) &&
 			/^(Blob|File)$/.test(object[Symbol.toStringTag])
 		);
 	}
diff --git a/test.js b/test.js
@@ -2,8 +2,9 @@ import fs from 'fs';
 import test from 'ava';
 import {Response} from 'node-fetch';
 import {Readable} from 'stream';
+import buffer from 'buffer';
 import Blob from './index.js';
-import blobFrom from './from.js';
+import sync, {blobFromSync, blobFrom} from './from.js';
 
 const license = fs.readFileSync('./LICENSE', 'utf-8');
 
@@ -26,11 +27,12 @@ test('Blob ctor parts', async t => {
 		new Uint8Array([101]).buffer,
 		Buffer.from('f'),
 		new Blob(['g']),
-		{}
+		{},
+		new URLSearchParams('foo')
 	];
 
 	const blob = new Blob(parts);
-	t.is(await blob.text(), 'abcdefg[object Object]');
+	t.is(await blob.text(), 'abcdefg[object Object]foo=');
 });
 
 test('Blob size', t => {
@@ -149,13 +151,13 @@ test('Blob works with node-fetch Response.text()', async t => {
 });
 
 test('blob part backed up by filesystem', async t => {
-	const blob = blobFrom('./LICENSE');
+	const blob = blobFromSync('./LICENSE');
 	t.is(await blob.slice(0, 3).text(), license.slice(0, 3));
 	t.is(await blob.slice(4, 11).text(), license.slice(4, 11));
 });
 
 test('Reading after modified should fail', async t => {
-	const blob = blobFrom('./LICENSE');
+	const blob = blobFromSync('./LICENSE');
 	await new Promise(resolve => {
 		setTimeout(resolve, 100);
 	});
@@ -168,13 +170,19 @@ test('Reading after modified should fail', async t => {
 });
 
 test('Reading from the stream created by blobFrom', async t => {
-	const blob = blobFrom('./LICENSE');
+	const blob = blobFromSync('./LICENSE');
+	const actual = await blob.text();
+	t.is(actual, license);
+});
+
+test('create a blob from path asynchronous', async t => {
+	const blob = await blobFrom('./LICENSE');
 	const actual = await blob.text();
 	t.is(actual, license);
 });
 
 test('Reading empty blobs', async t => {
-	const blob = blobFrom('./LICENSE').slice(0, 0);
+	const blob = blobFromSync('./LICENSE').slice(0, 0);
 	const actual = await blob.text();
 	t.is(actual, '');
 });
@@ -196,7 +204,7 @@ test('Instanceof check returns false for nullish values', t => {
 });
 
 /** @see https://github.com/w3c/FileAPI/issues/43 - important to keep boundary value */
-test('Dose not lowercase the blob type', t => {
+test('Dose not lowercase the blob values', t => {
 	const type = 'multipart/form-data; boundary=----WebKitFormBoundaryTKqdrVt01qOBltBd';
 	t.is(new Blob([], {type}).type, type);
 });
@@ -234,3 +242,15 @@ test('Can use named import - as well as default', async t => {
 	const {Blob, default: def} = await import('./index.js');
 	t.is(Blob, def);
 });
+
+test('default from.js exports blobFromSync', t => {
+	t.is(blobFromSync, sync);
+});
+
+if (buffer.Blob) {
+	test('Can wrap buffer.Blob to a fetch-blob', async t => {
+		const blob1 = new buffer.Blob(['blob part']);
+		const blob2 = new Blob([blob1]);
+		t.is(await blob2.text(), 'blob part');
+	});
+}
