Proper handling of surrounding pair delimiter prefixes

Author: AndreasArvidsson

data/fixtures/recorded/surroundingPair/parseTree/python/changeInside.yml

@@ -0,0 +1,22 @@
+languageId: python
+command:
+  version: 7
+  spokenForm: change inside
+  action:
+    name: clearAndSetSelection
+    target:
+      type: primitive
+      modifiers:
+        - {type: interiorOnly}
+  usePrePhraseSnapshot: true
+initialState:
+  documentContents: r'command server'
+  selections:
+    - anchor: {line: 0, character: 2}
+      active: {line: 0, character: 2}
+  marks: {}
+finalState:
+  documentContents: r''
+  selections:
+    - anchor: {line: 0, character: 2}
+      active: {line: 0, character: 2}

data/fixtures/recorded/surroundingPair/parseTree/python/changePair2.yml renamed to data/fixtures/recorded/surroundingPair/parseTree/python/changePair3.yml

@@ -11,10 +11,10 @@ command:
           scopeType: {type: surroundingPair, delimiter: any}
   usePrePhraseSnapshot: true
 initialState:
-  documentContents: "\" r\""
+  documentContents: r'command server'
   selections:
-    - anchor: {line: 0, character: 0}
-      active: {line: 0, character: 0}
+    - anchor: {line: 0, character: 2}
+      active: {line: 0, character: 2}
   marks: {}
 finalState:
   documentContents: ""

packages/cursorless-engine/src/processTargets/modifiers/scopeHandlers/SurroundingPairScopeHandler/delimiterMaps.ts

@@ -16,15 +16,9 @@ interface Options {
   isSingleLine?: boolean;
 
   /**
-   * This field can be used to force us to treat the side of the delimiter as
-   * unknown. We usually infer this from the fact that the opening and closing
-   * delimiters are the same, but in some cases they are different, but the side
-   * is actually still unknown. In particular, this is the case for Python
-   * string prefixes, where if we see the prefix it doesn't necessarily mean
-   * that it's an opening delimiter. For example, in  `" r"`, note that the `r`
-   * is just part of the string, not a prefix of the opening delimiter.
+   * The prefixes that can be used before the left side of the delimiter, eg "r"
    */
-  isUnknownSide?: boolean;
+  prefixes?: string[];
 }
 
 type DelimiterMap = Record<
@@ -54,8 +48,6 @@ const delimiterToText: DelimiterMap = Object.freeze({
 
 // https://docs.python.org/3/reference/lexical_analysis.html#string-and-bytes-literals
 const pythonPrefixes = [
-  // Base case without a prefix
-  "",
   // string prefixes
   "r",
   "u",
@@ -102,26 +94,10 @@ const delimiterToTextOverrides: Record<string, Partial<DelimiterMap>> = {
   },
 
   python: {
-    singleQuotes: [
-      pythonPrefixes.map((prefix) => `${prefix}'`),
-      "'",
-      { isSingleLine: true, isUnknownSide: true },
-    ],
-    doubleQuotes: [
-      pythonPrefixes.map((prefix) => `${prefix}"`),
-      '"',
-      { isSingleLine: true, isUnknownSide: true },
-    ],
-    tripleSingleQuotes: [
-      pythonPrefixes.map((prefix) => `${prefix}'''`),
-      "'''",
-      { isUnknownSide: true },
-    ],
-    tripleDoubleQuotes: [
-      pythonPrefixes.map((prefix) => `${prefix}"""`),
-      '"""',
-      { isUnknownSide: true },
-    ],
+    singleQuotes: ["'", "'", { isSingleLine: true, prefixes: pythonPrefixes }],
+    doubleQuotes: ['"', '"', { isSingleLine: true, prefixes: pythonPrefixes }],
+    tripleSingleQuotes: ["'''", "'''", { prefixes: pythonPrefixes }],
+    tripleDoubleQuotes: ['"""', '"""', { prefixes: pythonPrefixes }],
   },
 
   ruby: {

packages/cursorless-engine/src/processTargets/modifiers/scopeHandlers/SurroundingPairScopeHandler/getDelimiterOccurrences.ts

@@ -27,12 +27,8 @@ export function getDelimiterOccurrences(
   const textFragments =
     languageDefinition?.getCaptures(document, "textFragment") ?? [];
 
-  const delimiterTextToDelimiterInfoMap = Object.fromEntries(
-    individualDelimiters.map((individualDelimiter) => [
-      individualDelimiter.text,
-      individualDelimiter,
-    ]),
-  );
+  const delimiterTextToDelimiterInfoMap =
+    getDelimiterTextToDelimiterInfoMap(individualDelimiters);
 
   const text = document.getText();
 
@@ -59,3 +55,25 @@ export function getDelimiterOccurrences(
     };
   });
 }
+
+function getDelimiterTextToDelimiterInfoMap(
+  individualDelimiters: IndividualDelimiter[],
+): Record<string, IndividualDelimiter> {
+  return Object.fromEntries(
+    individualDelimiters.flatMap((individualDelimiter) => {
+      const results = [[individualDelimiter.text, individualDelimiter]];
+      if (individualDelimiter.prefixes.length > 0) {
+        for (const prefix of individualDelimiter.prefixes) {
+          const prefixText = prefix + individualDelimiter.text;
+          const prefixDelimiter: IndividualDelimiter = {
+            ...individualDelimiter,
+            text: prefixText,
+            side: "left",
+          };
+          results.push([prefixText, prefixDelimiter]);
+        }
+      }
+      return results;
+    }),
+  );
+}

packages/cursorless-engine/src/processTargets/modifiers/scopeHandlers/SurroundingPairScopeHandler/getDelimiterRegex.ts

@@ -12,11 +12,23 @@ export function getDelimiterRegex(individualDelimiters: IndividualDelimiter[]) {
   // Create a regex which is a disjunction of all possible left / right
   // delimiter texts
   const individualDelimiterDisjunct = uniq(
-    individualDelimiters.map(({ text }) => text),
-  )
-    .map(escapeRegExp)
-    .join("|");
+    individualDelimiters.flatMap((delimiter) => {
+      const text = escapeRegExp(delimiter.text);
+      const result = [text];
+      for (const prefix of delimiter.prefixes) {
+        // If the prefix is only alpha character, we need to make sure that there is no preceding alpha characters.
+        if (alphaRegex.test(prefix)) {
+          result.push(`(?<!\\w)${prefix}${text}`);
+        } else {
+          result.push(`${escapeRegExp(prefix)}${text}`);
+        }
+      }
+      return result;
+    }),
+  ).join("|");
 
   // Then make sure that we don't allow preceding `\`
   return new RegExp(`(?<!\\\\)(${individualDelimiterDisjunct})`, "gu");
 }
+
+const alphaRegex = /^\w+$/;

packages/cursorless-engine/src/processTargets/modifiers/scopeHandlers/SurroundingPairScopeHandler/getIndividualDelimiters.ts

@@ -21,7 +21,7 @@ import type { IndividualDelimiter } from "./types";
 export function getIndividualDelimiters(
   delimiter: SurroundingPairName,
   languageId: string,
-) {
+): IndividualDelimiter[] {
   const delimiters = complexDelimiterMap[
     delimiter as ComplexSurroundingPairName
   ] ?? [delimiter];
@@ -36,7 +36,7 @@ function getSimpleIndividualDelimiters(
   return delimiters.flatMap((delimiterName) => {
     const [leftDelimiter, rightDelimiter, options] =
       delimiterToText[delimiterName];
-    const { isSingleLine = false, isUnknownSide = false } = options ?? {};
+    const { isSingleLine = false, prefixes = [] } = options ?? {};
 
     // Allow for the fact that a delimiter might have multiple ways to indicate
     // its opening / closing
@@ -54,9 +54,6 @@ function getSimpleIndividualDelimiters(
       const isRight = rightDelimiters.includes(text);
 
       const side = (() => {
-        if (isUnknownSide) {
-          return "unknown";
-        }
         if (isLeft && !isRight) {
           return "left";
         }
@@ -73,6 +70,7 @@ function getSimpleIndividualDelimiters(
         side,
         delimiterName,
         isSingleLine,
+        prefixes,
       };
     });
   });

packages/cursorless-engine/src/processTargets/modifiers/scopeHandlers/SurroundingPairScopeHandler/types.ts

@@ -28,6 +28,11 @@ export interface IndividualDelimiter {
    */
   isSingleLine: boolean;
 
+  /**
+   * The prefixes that can be used before the left side of the delimiter, eg "r"
+   */
+  prefixes: string[];
+
   /**
    * The text that can be used to represent this side of the delimiter, eg "("
    */