Tweaks to #709

src/processTargets/modifiers/ItemStage/ItemStage.ts

@@ -104,21 +104,22 @@ function getItemInfosForIterationScope(
   target: Target
 ) {
   const { range, boundary } = getIterationScope(context, target);
-  return rangeToItemInfos(target.editor, range, boundary);
+  return getItemsInRange(target.editor, range, boundary);
 }
 
-function rangeToItemInfos(
+function getItemsInRange(
   editor: TextEditor,
-  collectionRange: Range,
-  collectionBoundary?: [Range, Range]
+  interior: Range,
+  boundary?: [Range, Range]
 ): ItemInfo[] {
-  const tokens = tokenizeRange(editor, collectionRange, collectionBoundary);
+  const tokens = tokenizeRange(editor, interior, boundary);
   const itemInfos: ItemInfo[] = [];
 
   tokens.forEach((token, i) => {
     if (token.type === "separator" || token.type === "boundary") {
       return;
     }
+
     const leadingDelimiterRange = (() => {
       if (tokens[i - 2]?.type === "item") {
         return new Range(tokens[i - 2].range.end, token.range.start);
@@ -128,6 +129,7 @@ function rangeToItemInfos(
       }
       return undefined;
     })();
+
     const trailingDelimiterRange = (() => {
       if (tokens[i + 2]?.type === "item") {
         return new Range(token.range.end, tokens[i + 2].range.start);
@@ -137,24 +139,26 @@ function rangeToItemInfos(
       }
       return undefined;
     })();
+
     // Leading boundary is excluded and leading separator is included
-    const leadingMatchStart =
+    const domainStart =
       tokens[i - 1]?.type === "boundary"
         ? tokens[i - 1].range.end
         : tokens[i - 1]?.type === "separator"
         ? tokens[i - 1].range.start
         : token.range.start;
+
     // Trailing boundary and separator is excluded
-    const trailingMatchEnd =
+    const domainEnd =
       tokens[i + 1]?.type === "boundary" || tokens[i + 1]?.type === "separator"
         ? tokens[i + 1].range.start
         : token.range.end;
-    const matchRange = new Range(leadingMatchStart, trailingMatchEnd);
+
     itemInfos.push({
       contentRange: token.range,
       leadingDelimiterRange,
       trailingDelimiterRange,
-      domain: matchRange,
+      domain: new Range(domainStart, domainEnd),
     });
   });
 

src/processTargets/modifiers/ItemStage/tokenizeRange.ts

@@ -3,21 +3,30 @@ import { Range, TextEditor } from "vscode";
 /**
  * Takes the range for a collection and returns a list of tokens within that collection
  * @param editor The editor containing the range
- * @param collectionRange The range to look for tokens within
- * @param collectionBoundary Optional boundaries for collections. [], {}
+ * @param interior The range to look for tokens within
+ * @param boundary Optional boundaries for collections. [], {}
  * @returns List of tokens
  */
 export function tokenizeRange(
   editor: TextEditor,
-  collectionRange: Range,
-  collectionBoundary?: [Range, Range]
+  interior: Range,
+  boundary?: [Range, Range]
 ): Token[] {
   const { document } = editor;
-  const text = document.getText(collectionRange);
-  const lexemes = text.split(/([,(){}<>[\]"'`]|\\"|\\'|\\`)/g).filter(Boolean);
+  const text = document.getText(interior);
+  /**
+   * The interior range tokenized into delimited regions, including the delimiters themselves.  For example:
+   * `"foo(hello), bar, whatever"` =>
+   * `["foo", "(", "hello", ")", ",", " bar", ",", " whatever"]`
+   */
+  const lexemes = text
+    // NB: Both the delimiters and the text between them are included because we
+    // use a capture group in this split regex
+    .split(/([,(){}<>[\]"'`]|\\"|\\'|\\`)/g)
+    .filter((lexeme) => lexeme.length > 0);
   const joinedLexemes = joinLexemesBySkippingMatchingPairs(lexemes);
   const tokens: Token[] = [];
-  let offset = document.offsetAt(collectionRange.start);
+  let offset = document.offsetAt(interior.start);
 
   joinedLexemes.forEach((lexeme) => {
     // Whitespace found. Just skip
@@ -27,7 +36,7 @@ export function tokenizeRange(
     }
 
     // Separator delimiter found.
-    if (lexeme === delimiter) {
+    if (lexeme === separator) {
       tokens.push({
         type: "separator",
         range: new Range(
@@ -52,11 +61,11 @@ export function tokenizeRange(
     offset += lexeme.length;
   });
 
-  if (collectionBoundary != null) {
+  if (boundary != null) {
     return [
-      { type: "boundary", range: collectionBoundary[0] },
+      { type: "boundary", range: boundary[0] },
       ...tokens,
-      { type: "boundary", range: collectionBoundary[1] },
+      { type: "boundary", range: boundary[1] },
     ];
   }
 
@@ -70,48 +79,58 @@ export function tokenizeRange(
  */
 export function joinLexemesBySkippingMatchingPairs(lexemes: string[]) {
   const result: string[] = [];
-  let delimiterCount = 0;
+  /**
+   * The number of left delimiters minus right delimiters we've seen.  If the
+   * balance is 0, we're at the top level of the collection, so separators are
+   * relevant.  Otherwise we ignore separators because they're nested
+   */
+  let delimiterBalance = 0;
+  /** The most recent opening delimiter we've seen */
   let openingDelimiter: string | null = null;
+  /** The closing delimiter we're currently looking for */
   let closingDelimiter: string | null = null;
+  /**
+   * The index in {@link lexemes} of the first lexeme in the current token we're
+   * merging.
+   */
   let startIndex: number = -1;
 
   lexemes.forEach((lexeme, index) => {
-    // We are waiting for a closing delimiter
-    if (delimiterCount > 0) {
-      // Closing delimiter found
-      if (closingDelimiter === lexeme) {
-        --delimiterCount;
-      }
-      // Additional opening delimiter found
-      else if (openingDelimiter === lexeme) {
-        ++delimiterCount;
+    if (delimiterBalance > 0) {
+      // We are waiting for a closing delimiter
+
+      if (lexeme === closingDelimiter) {
+        // Closing delimiter found
+        --delimiterBalance;
+      } else if (lexeme === openingDelimiter) {
+        // Additional opening delimiter found
+        ++delimiterBalance;
       }
+
+      return;
     }
 
-    // Starting delimiter found
-    else if (delimiters[lexeme] != null) {
+    if (leftToRightMap[lexeme] != null) {
+      // Starting delimiter found
       openingDelimiter = lexeme;
-      closingDelimiter = delimiters[lexeme];
-      delimiterCount = 1;
-      // This is the first lexeme to be joined
-      if (startIndex < 0) {
-        startIndex = index;
-      }
+      closingDelimiter = leftToRightMap[lexeme];
+      delimiterBalance = 1;
     }
 
-    // This is the first lexeme to be joined
-    else if (startIndex < 0) {
+    if (startIndex < 0) {
+      // This is the first lexeme to be joined
       startIndex = index;
     }
 
-    const isDelimiter = lexeme === delimiter && delimiterCount === 0;
+    const isSeparator = lexeme === separator && delimiterBalance === 0;
 
-    // This is the last lexeme to be joined
-    if (isDelimiter || index === lexemes.length - 1) {
-      const endIndex = isDelimiter ? index : index + 1;
+    if (isSeparator || index === lexemes.length - 1) {
+      // This is the last lexeme to be joined
+      const endIndex = isSeparator ? index : index + 1;
       result.push(lexemes.slice(startIndex, endIndex).join(""));
       startIndex = -1;
-      if (isDelimiter) {
+      if (isSeparator) {
+        // Add the separator itself
         result.push(lexeme);
       }
     }
@@ -120,11 +139,11 @@ export function joinLexemesBySkippingMatchingPairs(lexemes: string[]) {
   return result;
 }
 
-const delimiter = ",";
+const separator = ",";
 
 // Mapping between opening and closing delimiters
 /* eslint-disable @typescript-eslint/naming-convention */
-const delimiters: { [key: string]: string } = {
+const leftToRightMap: { [key: string]: string } = {
   "(": ")",
   "{": "}",
   "<": ">",

src/test/suite/fixtures/recorded/itemTextual/clearItem10.yml

@@ -0,0 +1,26 @@
+languageId: typescript
+command:
+  spokenForm: clear item
+  version: 2
+  targets:
+    - type: primitive
+      modifiers:
+        - type: containingScope
+          scopeType: {type: collectionItem}
+  usePrePhraseSnapshot: false
+  action: {name: clearAndSetSelection}
+initialState:
+  documentContents: foo(hello, world)
+  selections:
+    - anchor: {line: 0, character: 10}
+      active: {line: 0, character: 10}
+  marks: {}
+finalState:
+  documentContents: foo(hello, )
+  selections:
+    - anchor: {line: 0, character: 11}
+      active: {line: 0, character: 11}
+  thatMark:
+    - anchor: {line: 0, character: 11}
+      active: {line: 0, character: 11}
+fullTargets: [{type: primitive, mark: {type: cursor}, modifiers: [{type: containingScope, scopeType: {type: collectionItem}}]}]

src/test/suite/fixtures/recorded/itemTextual/clearItem11.yml

@@ -0,0 +1,26 @@
+languageId: typescript
+command:
+  spokenForm: clear item
+  version: 2
+  targets:
+    - type: primitive
+      modifiers:
+        - type: containingScope
+          scopeType: {type: collectionItem}
+  usePrePhraseSnapshot: false
+  action: {name: clearAndSetSelection}
+initialState:
+  documentContents: foo(hello, world)
+  selections:
+    - anchor: {line: 0, character: 7}
+      active: {line: 0, character: 13}
+  marks: {}
+finalState:
+  documentContents: foo()
+  selections:
+    - anchor: {line: 0, character: 4}
+      active: {line: 0, character: 4}
+  thatMark:
+    - anchor: {line: 0, character: 11}
+      active: {line: 0, character: 11}
+fullTargets: [{type: primitive, mark: {type: cursor}, modifiers: [{type: containingScope, scopeType: {type: collectionItem}}]}]

src/test/suite/fixtures/recorded/itemTextual/clearItem9.yml

@@ -0,0 +1,26 @@
+languageId: typescript
+command:
+  spokenForm: clear item
+  version: 2
+  targets:
+    - type: primitive
+      modifiers:
+        - type: containingScope
+          scopeType: {type: collectionItem}
+  usePrePhraseSnapshot: true
+  action: {name: clearAndSetSelection}
+initialState:
+  documentContents: foo(hello, world)
+  selections:
+    - anchor: {line: 0, character: 9}
+      active: {line: 0, character: 9}
+  marks: {}
+finalState:
+  documentContents: foo(, world)
+  selections:
+    - anchor: {line: 0, character: 4}
+      active: {line: 0, character: 4}
+  thatMark:
+    - anchor: {line: 0, character: 4}
+      active: {line: 0, character: 4}
+fullTargets: [{type: primitive, mark: {type: cursor}, modifiers: [{type: containingScope, scopeType: {type: collectionItem}}]}]

src/test/suite/fixtures/recorded/itemTextual/clearItemDrip.yml

@@ -0,0 +1,30 @@
+languageId: typescript
+command:
+  spokenForm: clear item drip
+  version: 2
+  targets:
+    - type: primitive
+      modifiers:
+        - type: containingScope
+          scopeType: {type: collectionItem}
+      mark: {type: decoratedSymbol, symbolColor: default, character: ','}
+  usePrePhraseSnapshot: true
+  action: {name: clearAndSetSelection}
+initialState:
+  documentContents: foo(hello, world)
+  selections:
+    - anchor: {line: 0, character: 13}
+      active: {line: 0, character: 13}
+  marks:
+    default.,:
+      start: {line: 0, character: 9}
+      end: {line: 0, character: 10}
+finalState:
+  documentContents: foo()
+  selections:
+    - anchor: {line: 0, character: 4}
+      active: {line: 0, character: 4}
+  thatMark:
+    - anchor: {line: 0, character: 11}
+      active: {line: 0, character: 11}
+fullTargets: [{type: primitive, mark: {type: decoratedSymbol, symbolColor: default, character: ','}, modifiers: [{type: containingScope, scopeType: {type: collectionItem}}]}]