Token removal range based upon merging of tokens or not

src/processTargets/targetUtil/insertionRemovalBehaviors/TokenInsertionRemovalBehavior.ts

@@ -1,6 +1,7 @@
-import { Range } from "vscode";
+import { Range, TextDocument } from "vscode";
+import { tokenize } from "../../../core/tokenizer";
 import type { Target } from "../../../typings/target.types";
-import { isAtEndOfLine, isAtStartOfLine } from "../../../util/rangeUtils";
+import { expandToFullLine } from "../../../util/rangeUtils";
 import { PlainTarget } from "../../targets";
 import { getDelimitedSequenceRemovalRange } from "./DelimitedSequenceInsertionRemovalBehavior";
 
@@ -66,24 +67,48 @@ export function getTokenRemovalRange(
   target: Target,
   contentRange?: Range
 ): Range {
-  const { start, end } = contentRange ?? target.contentRange;
+  const { document } = target.editor;
+  const actualContentRange = contentRange ?? target.contentRange;
+  const removalRange = getDelimitedSequenceRemovalRange(target, contentRange);
 
-  const leadingDelimiterTarget = getTokenLeadingDelimiterTarget(target);
-  const trailingDelimiterTarget = getTokenTrailingDelimiterTarget(target);
+  if (!actualContentRange.isEqual(removalRange)) {
+    const fullRange = expandToFullLine(target.editor, actualContentRange);
+    const fullText = document.getText(fullRange);
+    const fullTextOffset = document.offsetAt(fullRange.start);
 
-  // If there is a token directly to the left or right of us with no
-  // separating white space, then we might join two tokens if we try to clean
-  // up whitespace space. In this case we just remove the content range
-  // without attempting to clean up white space.
-  //
-  // In the future, we might get more sophisticated and to clean up white space if we can detect that it won't cause two tokens be merged
-  if (
-    (leadingDelimiterTarget == null && !isAtStartOfLine(start)) ||
-    (trailingDelimiterTarget == null && !isAtEndOfLine(target.editor, end))
-  ) {
-    return contentRange ?? target.contentRange;
+    const numTokensContentRangeRemoved = calculateNumberOfTokensAfterRemoval(
+      document,
+      fullText,
+      fullTextOffset,
+      actualContentRange
+    );
+
+    const numTokensRemovalRangeRemoved = calculateNumberOfTokensAfterRemoval(
+      document,
+      fullText,
+      fullTextOffset,
+      removalRange
+    );
+
+    // Using removal range has not merged any tokens. Removal range is ok to use.
+    if (numTokensContentRangeRemoved === numTokensRemovalRangeRemoved) {
+      return removalRange;
+    }
   }
 
-  // Otherwise, behave like a whitespace delimited sequence
-  return getDelimitedSequenceRemovalRange(target, contentRange);
+  // No removal range available or it would merge tokens.
+  return actualContentRange;
+}
+
+function calculateNumberOfTokensAfterRemoval(
+  document: TextDocument,
+  fullText: string,
+  fullTextOffset: number,
+  removalRange: Range
+): number {
+  const startIndex = document.offsetAt(removalRange.start) - fullTextOffset;
+  const endIndex = document.offsetAt(removalRange.end) - fullTextOffset;
+  const modifiedText = fullText.slice(0, startIndex) + fullText.slice(endIndex);
+  const tokens = tokenize(modifiedText, document.languageId, (m) => m);
+  return tokens.length;
 }

src/test/suite/fixtures/recorded/actions/chuckArgMadeAndAirAndJustSoon.yml

@@ -30,15 +30,15 @@ initialState:
       start: {line: 0, character: 23}
       end: {line: 0, character: 29}
 finalState:
-  documentContents: "function myFunk(value: ) { };"
+  documentContents: function myFunk(value:) { };
   selections:
     - anchor: {line: 0, character: 0}
       active: {line: 0, character: 0}
   thatMark:
     - type: UntypedTarget
       contentRange:
-        start: {line: 0, character: 23}
-        end: {line: 0, character: 23}
+        start: {line: 0, character: 22}
+        end: {line: 0, character: 22}
       isReversed: false
       hasExplicitRange: false
 fullTargets: [{type: list, elements: [{type: primitive, mark: {type: decoratedSymbol, symbolColor: default, character: m}, selectionType: token, position: contents, insideOutsideType: outside, modifier: {type: containingScope, scopeType: argumentOrParameter, includeSiblings: false}, isImplicit: false}, {type: primitive, mark: {type: decoratedSymbol, symbolColor: default, character: a}, selectionType: token, position: contents, insideOutsideType: outside, modifier: {type: containingScope, scopeType: argumentOrParameter, includeSiblings: false}, isImplicit: false}, {type: primitive, mark: {type: decoratedSymbol, symbolColor: default, character: s}, selectionType: token, position: contents, insideOutsideType: outside, modifier: {type: toRawSelection}, isImplicit: false}]}]

src/test/suite/fixtures/recorded/languages/css/chuckValue.yml

@@ -13,8 +13,8 @@ initialState:
       active: {line: 0, character: 16}
   marks: {}
 finalState:
-  documentContents: "@namespace ;"
+  documentContents: "@namespace;"
   selections:
-    - anchor: {line: 0, character: 11}
-      active: {line: 0, character: 11}
+    - anchor: {line: 0, character: 10}
+      active: {line: 0, character: 10}
 fullTargets: [{type: primitive, mark: {type: cursor}, selectionType: token, position: contents, insideOutsideType: outside, modifier: {type: containingScope, scopeType: value, includeSiblings: false}, isImplicit: false}]

src/test/suite/fixtures/recorded/languages/css/chuckValue2.yml

@@ -13,7 +13,7 @@ initialState:
       active: {line: 0, character: 6}
   marks: {}
 finalState:
-  documentContents: "@namespace ;"
+  documentContents: "@namespace;"
   selections:
     - anchor: {line: 0, character: 6}
       active: {line: 0, character: 6}

src/test/suite/fixtures/recorded/languages/css/chuckValue3.yml

@@ -18,7 +18,7 @@ initialState:
 finalState:
   documentContents: |
     a { 
-      margin-top: ; 
+      margin-top:; 
     }
   selections:
     - anchor: {line: 1, character: 7}

src/test/suite/fixtures/recorded/languages/rust/chuckCallNear.yml

@@ -22,7 +22,7 @@ initialState:
       end: {line: 0, character: 20}
 finalState:
   documentContents: |
-    let user = ;
+    let user =;
   selections:
     - anchor: {line: 1, character: 0}
       active: {line: 1, character: 0}

src/test/suite/fixtures/recorded/languages/rust/chuckFunkNameFine.yml

@@ -21,7 +21,7 @@ initialState:
       end: {line: 0, character: 2}
 finalState:
   documentContents: |-
-    fn () {
+    fn() {
         println!("Hello, world!");
     }
   selections:

src/test/suite/fixtures/recorded/languages/rust/chuckListOne.yml

@@ -17,8 +17,8 @@ initialState:
       start: {line: 0, character: 9}
       end: {line: 0, character: 10}
 finalState:
-  documentContents: let x = ;
+  documentContents: let x =;
   selections:
-    - anchor: {line: 0, character: 9}
-      active: {line: 0, character: 9}
+    - anchor: {line: 0, character: 8}
+      active: {line: 0, character: 8}
 fullTargets: [{type: primitive, mark: {type: decoratedSymbol, symbolColor: default, character: '1'}, selectionType: token, position: contents, insideOutsideType: outside, modifier: {type: containingScope, scopeType: list, includeSiblings: false}, isImplicit: false}]

src/test/suite/fixtures/recorded/languages/rust/chuckName.yml

@@ -13,7 +13,7 @@ initialState:
       active: {line: 0, character: 5}
   marks: {}
 finalState:
-  documentContents: "let mut : [i32; 3] = [0; 3];"
+  documentContents: "let mut: [i32; 3] = [0; 3];"
   selections:
     - anchor: {line: 0, character: 5}
       active: {line: 0, character: 5}

src/test/suite/fixtures/recorded/languages/rust/chuckName2.yml

@@ -20,7 +20,7 @@ initialState:
   marks: {}
 finalState:
   documentContents: |
-    fn () -> Result<(), Error> {
+    fn() -> Result<(), Error> {
 
     }
   selections:

src/test/suite/fixtures/recorded/languages/scss/chuckValue.yml

@@ -13,8 +13,8 @@ initialState:
       active: {line: 0, character: 16}
   marks: {}
 finalState:
-  documentContents: "@namespace ;"
+  documentContents: "@namespace;"
   selections:
-    - anchor: {line: 0, character: 11}
-      active: {line: 0, character: 11}
+    - anchor: {line: 0, character: 10}
+      active: {line: 0, character: 10}
 fullTargets: [{type: primitive, mark: {type: cursor}, selectionType: token, position: contents, insideOutsideType: outside, modifier: {type: containingScope, scopeType: value, includeSiblings: false}, isImplicit: false}]

src/test/suite/fixtures/recorded/languages/scss/chuckValue2.yml

@@ -13,7 +13,7 @@ initialState:
       active: {line: 0, character: 6}
   marks: {}
 finalState:
-  documentContents: "@namespace ;"
+  documentContents: "@namespace;"
   selections:
     - anchor: {line: 0, character: 6}
       active: {line: 0, character: 6}

src/test/suite/fixtures/recorded/languages/scss/chuckValue3.yml

@@ -18,7 +18,7 @@ initialState:
 finalState:
   documentContents: |
     a { 
-      margin-top: ; 
+      margin-top:; 
     }
   selections:
     - anchor: {line: 1, character: 7}

src/test/suite/fixtures/recorded/positions/chuckAir.yml

@@ -24,7 +24,7 @@ finalState:
     "hello "
     " hello"
     " hello "
-    return .b
+    return.b
   selections:
     - anchor: {line: 4, character: 0}
       active: {line: 4, character: 0}

src/test/suite/fixtures/recorded/selectionTypes/chuckSmallPaintFine.yml

@@ -20,7 +20,7 @@ initialState:
       start: {line: 0, character: 1}
       end: {line: 0, character: 4}
 finalState:
-  documentContents: "[ bar]"
+  documentContents: "[bar]"
   selections:
     - anchor: {line: 0, character: 1}
       active: {line: 0, character: 1}

src/test/suite/fixtures/recorded/selectionTypes/chuckSmallPaintSoap2.yml

@@ -20,8 +20,8 @@ initialState:
       start: {line: 0, character: 4}
       end: {line: 0, character: 8}
 finalState:
-  documentContents: f.o( text)
+  documentContents: f.o(text)
   selections:
-    - anchor: {line: 0, character: 10}
-      active: {line: 0, character: 10}
+    - anchor: {line: 0, character: 9}
+      active: {line: 0, character: 9}
 fullTargets: [{type: primitive, mark: {type: decoratedSymbol, symbolColor: default, character: s}, modifiers: [{type: containingScope, scopeType: {type: boundedNonWhitespaceSequence}}]}]