[New] parse: add opt-in splitUnquoted option for shell field-splitting of unquoted expansions (#1)

Author: ljharb

README.md

@@ -67,6 +67,20 @@ output
 [ 'beep', '--boop=/home/robot' ]
 ```
 
+## parse with unquoted variable splitting
+
+```js
+var parse = require('shell-quote/parse');
+var xs = parse('a $T', { T: 'c d' }, { splitUnquoted: true });
+console.dir(xs);
+```
+
+output
+
+```
+[ 'a', 'c', 'd' ]
+```
+
 ## parsing shell operators
 
 ```js
@@ -137,6 +151,13 @@ Return an array of arguments from the quoted string `cmd`.
 Interpolate embedded bash-style `$VARNAME` and `${VARNAME}` variables with
 the `env` object which like bash will replace undefined variables with `""`.
 
+By default an expanded variable is a single token even when unquoted.
+Pass `{ splitUnquoted: true }` to split an unquoted expansion into multiple tokens the way a shell performs field splitting,
+using the default `IFS` (space, tab, newline).
+Pass a string to use its characters as the `IFS` instead
+(for example `{ splitUnquoted: ':' }`).
+A quoted expansion (`"$VAR"`) is never split.
+
 Only simple `$VARNAME` and `${VARNAME}` interpolation is supported.
 Bash parameter expansion beyond a plain variable name is not evaluated:
 forms such as array subscripts (`${arr[i]}`), length (`${#arr[@]}`),
@@ -147,12 +168,13 @@ are not interpreted.
 Whitespace inside `${...}` throws a `Bad substitution` error.
 
 `env` is usually an object but it can also be a function to perform lookups.
-When `env(key)` returns a string, its result will be output just like `env[key]`
-would. When `env(key)` returns an object, it will be inserted into the result
+When `env(key)` returns a string, its result will be output just like `env[key]` would.
+When `env(key)` returns an object, it will be inserted into the result
 array like the operator objects.
 
-When a bash operator is encountered, the element in the array with be an object
-with an `"op"` key set to the operator string. For example:
+When a bash operator is encountered,
+the element in the array with be an object with an `"op"` key set to the operator string.
+For example:
 
 ```
 'beep || boop > /byte'
@@ -180,8 +202,6 @@ MIT
 [npm-version-svg]: https://versionbadg.es/ljharb/shell-quote.svg
 [deps-svg]: https://david-dm.org/ljharb/shell-quote.svg
 [deps-url]: https://david-dm.org/ljharb/shell-quote
-[dev-deps-svg]: https://david-dm.org/ljharb/shell-quote/dev-status.svg
-[dev-deps-url]: https://david-dm.org/ljharb/shell-quote#info=devDependencies
 [npm-badge-png]: https://nodei.co/npm/shell-quote.png?downloads=true&stars=true
 [license-image]: https://img.shields.io/npm/l/shell-quote.svg
 [license-url]: LICENSE

parse.d.ts

@@ -22,6 +22,8 @@ declare namespace parse {
 	export interface ParseOptions {
 		/** Custom escape character. Defaults to `\\`. */
 		escape?: string;
+		/** Field-splits an unquoted variable expansion, the way a shell does using `IFS`; quoted expansions are never split. `true` uses the default IFS (space, tab, newline); a string uses its characters as the IFS. An empty string, `false`, or omitting the option disables splitting. Defaults to false. */
+		splitUnquoted?: boolean | string;
 	}
 
 	export type Env =

parse.js

@@ -83,14 +83,15 @@ function getVar(env, pre, key) {
 /**
  * @param {string} string
  * @param {Env} [env]
- * @param {{ escape?: string }} [opts]
+ * @param {{ escape?: string, splitUnquoted?: boolean | string }} [opts]
  * @returns {ParseEntry[]}
  */
 function parseInternal(string, env, opts) {
 	if (!opts) {
 		opts = {};
 	}
 	var BS = opts.escape || '\\';
+	var ifs = opts.splitUnquoted === true ? ' \t\n' : (typeof opts.splitUnquoted === 'string' ? opts.splitUnquoted : '');
 	var BAREWORD = '(\\' + BS + '[\'"' + META + ']|[^\\s\'"' + META + '])+';
 
 	var chunker = new RegExp([
@@ -133,6 +134,11 @@ function parseInternal(string, env, opts) {
 		var quote = false;
 		var esc = false;
 		var out = '';
+		/** @type {string[]} */
+		var words = [];
+		var sawQuote = false;
+		/** @type {number | null} */
+		var pendingNw = null;
 		var isGlob = false;
 		/** @type {number} */
 		var i;
@@ -184,8 +190,30 @@ function parseInternal(string, env, opts) {
 			return getVar(/** @type {NonNullable<typeof env>} */ (env), '', varname);
 		}
 
+		function flushRun() {
+			if (pendingNw === null) {
+				return;
+			}
+			if (pendingNw === 0) {
+				if (out !== '') {
+					words[words.length] = out;
+					out = '';
+				}
+			} else {
+				words[words.length] = out;
+				out = '';
+				for (var fe = 1; fe < pendingNw; fe += 1) {
+					words[words.length] = '';
+				}
+			}
+			pendingNw = null;
+		}
+
 		for (i = 0; i < s.length; i++) {
 			var c = s.charAt(i);
+			if (ifs && c !== DS) {
+				flushRun();
+			}
 			isGlob = isGlob || (!quote && (c === '*' || c === '?'));
 			if (esc) {
 				out += c;
@@ -212,6 +240,7 @@ function parseInternal(string, env, opts) {
 				}
 			} else if (c === DQ || c === SQ) {
 				quote = c;
+				sawQuote = true;
 			} else if (controlRE.test(c)) {
 				return /** @type {ControlOperator} */ ({ op: s });
 			} else if (hash.test(c)) {
@@ -224,7 +253,22 @@ function parseInternal(string, env, opts) {
 			} else if (c === BS) {
 				esc = true;
 			} else if (c === DS) {
-				out += parseEnvVar();
+				var value = parseEnvVar();
+				if (!ifs) {
+					out += value;
+				} else {
+					for (var vi = 0; vi < value.length; vi += 1) {
+						var vc = value.charAt(vi);
+						if (ifs.indexOf(vc) < 0) {
+							flushRun();
+							out += vc;
+						} else if (pendingNw === null) {
+							pendingNw = vc === ' ' || vc === '\t' || vc === '\n' ? 0 : 1;
+						} else if (vc !== ' ' && vc !== '\t' && vc !== '\n') {
+							pendingNw += 1;
+						}
+					}
+				}
 			} else {
 				out += c;
 			}
@@ -234,6 +278,20 @@ function parseInternal(string, env, opts) {
 			return /** @type {GlobPattern} */ ({ op: 'glob', pattern: out });
 		}
 
+		if (ifs) {
+			if (pendingNw !== null && pendingNw > 0) {
+				words[words.length] = out;
+				out = '';
+				for (var te = 1; te < pendingNw; te += 1) {
+					words[words.length] = '';
+				}
+			}
+			if (out !== '' || (sawQuote && words.length === 0)) {
+				words[words.length] = out;
+			}
+			return words;
+		}
+
 		return out;
 	}).reduce(function (prev, arg) { // finalize parsed arguments
 		if (typeof arg === 'undefined') {

test/parse.js

@@ -90,6 +90,47 @@ test('nested parameter expansion', function (t) {
 	t.end();
 });
 
+test('splitUnquoted: field-splits unquoted variable expansion (#1)', function (t) {
+	var env = { T: 'c d', E: '', S: '  c   d  ', W: '   ' };
+	var opts = { splitUnquoted: true };
+
+	t.same(parse('test a b $T', env, opts), ['test', 'a', 'b', 'c', 'd'], 'unquoted expansion splits into separate tokens');
+	t.same(parse('a$T', env, opts), ['ac', 'd'], 'the first field joins the preceding text');
+	t.same(parse('$T x', env, opts), ['c', 'd', 'x'], 'the last field is a token of its own');
+	t.same(parse('x${T}y', env, opts), ['xc', 'dy'], 'fields join text on both sides');
+	t.same(parse('$S', env, opts), ['c', 'd'], 'leading, trailing, and repeated whitespace collapses');
+	t.same(parse('a$S', env, opts), ['a', 'c', 'd'], 'leading whitespace closes the preceding field');
+	t.same(parse('$W', env, opts), [], 'an all-whitespace expansion produces no tokens');
+	t.same(parse('a$W b', env, opts), ['a', 'b'], 'an all-whitespace expansion just separates fields');
+	t.same(parse('$E', env, opts), [], 'an empty unquoted expansion produces no token');
+	t.same(parse('"$T"', env, opts), ['c d'], 'a quoted expansion is never split');
+	t.same(parse('-x "" -y', env, opts), ['-x', '', '-y'], 'a quoted empty string is still preserved');
+	t.same(parse('a $F b', function () { return 'c d'; }, opts), ['a', 'c', 'd', 'b'], 'the env-function path splits too');
+
+	t.same(parse('test a b $T', env), ['test', 'a', 'b', 'c d'], 'without the option, unquoted expansion is not split');
+
+	t.end();
+});
+
+test('splitUnquoted: a string value is a custom IFS (#1)', function (t) {
+	function o(ifs) { return { splitUnquoted: ifs }; }
+
+	t.same(parse('${V}', { V: 'a:b' }, o(':')), ['a', 'b'], 'a non-whitespace IFS char splits fields');
+	t.same(parse('${V}', { V: 'a::b' }, o(':')), ['a', '', 'b'], 'adjacent non-whitespace delimiters yield an empty field');
+	t.same(parse('${V}', { V: ':a:' }, o(':')), ['', 'a'], 'a leading delimiter yields a leading empty; a trailing one does not');
+	t.same(parse('${V}', { V: 'a::' }, o(':')), ['a', ''], 'a trailing double delimiter yields one empty field');
+	t.same(parse('${V}', { V: ':' }, o(':')), [''], 'a lone delimiter yields a single empty field');
+	t.same(parse('${V}', { V: '::' }, o(':')), ['', ''], 'two delimiters yield two empty fields');
+	t.same(parse('${V}${W}', { V: 'a:', W: ':b' }, o(':')), ['a', '', 'b'], 'delimiters spanning an expansion boundary merge into one run');
+	t.same(parse('a${V}${W}z', { V: ':x:', W: ':y:' }, o(':')), ['a', 'x', '', 'y', 'z'], 'fields join literal text on both sides across expansions');
+	t.same(parse('${V}', { V: 'a : b' }, o(' :')), ['a', 'b'], 'whitespace around a non-whitespace delimiter is absorbed');
+	t.same(parse('${V}', { V: 'a b:c' }, o(' :')), ['a', 'b', 'c'], 'mixed IFS: whitespace and non-whitespace each delimit');
+	t.same(parse('${V}', { V: 'a,b' }, o(',')), ['a', 'b'], 'any character can be the IFS');
+	t.same(parse('${V}', { V: 'a b' }, o('')), ['a b'], 'an empty IFS string disables splitting');
+
+	t.end();
+});
+
 test('parse stays linear in token count (GHSA-395f-4hp3-45gv)', function (t) {
 	// the old concat-in-reduce finalizer was O(n^2): this many tokens took
 	// ~minutes, so under the unfixed code this test hangs rather than passes