Use parse tree

Author: AndreasArvidsson

cursorless-talon/src/spoken_forms.py

@@ -160,6 +160,7 @@ def handle_new_values(csv_name: str, values: list[SpokenFormEntry]):
                 "private.switchStatementSubject",
                 "textFragment",
                 "disqualifyDelimiter",
+                "pairDelimiter",
             ],
             default_list_name="scope_type",
         ),

data/fixtures/scopes/python/pairDelimiter.scope

@@ -0,0 +1,72 @@
+"server"
+'server'
+"""server"""
+'''server'''
+r" r"
+r' r'
+r""" r"""
+r''' r'''
+---
+
+[#1 Content] =
+[#1 Domain] = 4:0-4:2
+  >--<
+4| r" r"
+
+[#1 Removal] = 4:0-4:3
+  >---<
+4| r" r"
+
+[#1 Trailing delimiter] = 4:2-4:3
+    >-<
+4| r" r"
+
+[#1 Insertion delimiter] = " "
+
+
+[#2 Content] =
+[#2 Domain] = 5:0-5:2
+  >--<
+5| r' r'
+
+[#2 Removal] = 5:0-5:3
+  >---<
+5| r' r'
+
+[#2 Trailing delimiter] = 5:2-5:3
+    >-<
+5| r' r'
+
+[#2 Insertion delimiter] = " "
+
+
+[#3 Content] =
+[#3 Domain] = 6:0-6:4
+  >----<
+6| r""" r"""
+
+[#3 Removal] = 6:0-6:5
+  >-----<
+6| r""" r"""
+
+[#3 Trailing delimiter] = 6:4-6:5
+      >-<
+6| r""" r"""
+
+[#3 Insertion delimiter] = " "
+
+
+[#4 Content] =
+[#4 Domain] = 7:0-7:4
+  >----<
+7| r''' r'''
+
+[#4 Removal] = 7:0-7:5
+  >-----<
+7| r''' r'''
+
+[#4 Trailing delimiter] = 7:4-7:5
+      >-<
+7| r''' r'''
+
+[#4 Insertion delimiter] = " "

packages/common/src/scopeSupportFacets/python.ts

@@ -14,6 +14,7 @@ export const pythonScopeSupport: LanguageScopeSupportFacetMap = {
   namedFunction: supported,
   anonymousFunction: supported,
   disqualifyDelimiter: supported,
+  pairDelimiter: supported,
 
   "argument.actual": supported,
   "argument.actual.iteration": supported,

packages/common/src/scopeSupportFacets/scopeSupportFacetInfos.ts

@@ -298,6 +298,11 @@ export const scopeSupportFacetInfos: Record<
       "Used to disqualify a token from being treated as a surrounding pair delimiter. This will usually be operators containing `>` or `<`, eg `<`, `<=`, `->`, etc",
     scopeType: "disqualifyDelimiter",
   },
+  pairDelimiter: {
+    description:
+      "A pair delimiter, eg parentheses, brackets, braces, quotes, etc",
+    scopeType: "pairDelimiter",
+  },
 
   "branch.if": {
     description: "An if/elif/else branch",

packages/common/src/scopeSupportFacets/scopeSupportFacets.types.ts

@@ -80,6 +80,7 @@ export const scopeSupportFacets = [
   "textFragment.string.multiLine",
 
   "disqualifyDelimiter",
+  "pairDelimiter",
 
   "branch.if",
   "branch.if.iteration",

packages/common/src/types/command/PartialTargetDescriptor.types.ts

@@ -205,6 +205,7 @@ export const simpleScopeTypeTypes = [
   // Private scope types
   "textFragment",
   "disqualifyDelimiter",
+  "pairDelimiter",
 ] as const;
 
 export function isSimpleScopeType(

packages/cursorless-engine/src/languages/TreeSitterQuery/queryPredicateOperators.ts

@@ -62,6 +62,21 @@ class HasMultipleChildrenOfType extends QueryPredicateOperator<HasMultipleChildr
   }
 }
 
+/**
+ * A predicate operator that returns true if the nodes text matched the regular expression
+ */
+class Match extends QueryPredicateOperator<Match> {
+  name = "match?" as const;
+  schema = z.tuple([q.node, q.string]);
+
+  run(nodeInfo: MutableQueryCapture, pattern: string) {
+    const { document, range } = nodeInfo;
+    const regex = new RegExp(pattern, "ds");
+    const text = document.getText(range);
+    return regex.test(text);
+  }
+}
+
 class ChildRange extends QueryPredicateOperator<ChildRange> {
   name = "child-range!" as const;
   schema = z.union([
@@ -277,4 +292,5 @@ export const queryPredicateOperators = [
   new InsertionDelimiter(),
   new SingleOrMultilineDelimiter(),
   new HasMultipleChildrenOfType(),
+  new Match(),
 ];

packages/cursorless-engine/src/processTargets/modifiers/scopeHandlers/SurroundingPairScopeHandler/delimiterMaps.ts

@@ -14,12 +14,6 @@ interface Options {
    * salient example is strings.
    */
   isSingleLine?: boolean;
-
-  /**
-   * The prefixes that can be used before the left side of the delimiter, eg "r".
-   * Note that the empty string is always considered an acceptable prefix
-   */
-  prefixes?: string[];
 }
 
 type DelimiterMap = Record<
@@ -47,36 +41,6 @@ const delimiterToText: DelimiterMap = Object.freeze({
   squareBrackets: ["[", "]"],
 });
 
-// https://docs.python.org/3/reference/lexical_analysis.html#string-and-bytes-literals
-const pythonPrefixes = [
-  // string prefixes
-  "r",
-  "u",
-  "R",
-  "U",
-  "f",
-  "F",
-  "fr",
-  "Fr",
-  "fR",
-  "FR",
-  "rf",
-  "rF",
-  "Rf",
-  "RF",
-  // byte prefixes
-  "b",
-  "B",
-  "br",
-  "Br",
-  "bR",
-  "BR",
-  "rb",
-  "rB",
-  "Rb",
-  "RB",
-];
-
 // FIXME: Probably remove these as part of
 // https://github.com/cursorless-dev/cursorless/issues/1812#issuecomment-1691493746
 const delimiterToTextOverrides: Record<string, Partial<DelimiterMap>> = {
@@ -95,10 +59,8 @@ const delimiterToTextOverrides: Record<string, Partial<DelimiterMap>> = {
   },
 
   python: {
-    singleQuotes: ["'", "'", { isSingleLine: true, prefixes: pythonPrefixes }],
-    doubleQuotes: ['"', '"', { isSingleLine: true, prefixes: pythonPrefixes }],
-    tripleSingleQuotes: ["'''", "'''", { prefixes: pythonPrefixes }],
-    tripleDoubleQuotes: ['"""', '"""', { prefixes: pythonPrefixes }],
+    tripleSingleQuotes: ["'''", "'''"],
+    tripleDoubleQuotes: ['"""', '"""'],
   },
 
   ruby: {

packages/cursorless-engine/src/processTargets/modifiers/scopeHandlers/SurroundingPairScopeHandler/getDelimiterOccurrences.ts

@@ -31,13 +31,19 @@ export function getDelimiterOccurrences(
   const disqualifyDelimiters = new OneWayRangeFinder(
     getSortedCaptures(languageDefinition, document, "disqualifyDelimiter"),
   );
-  // We need a tree for text fragments since they can be nested
+  const pairDelimiters = new OneWayRangeFinder(
+    getSortedCaptures(languageDefinition, document, "pairDelimiter"),
+  );
   const textFragments = new OneWayNestedRangeFinder(
     getSortedCaptures(languageDefinition, document, "textFragment"),
   );
 
-  const delimiterTextToDelimiterInfoMap =
-    getDelimiterTextToDelimiterInfoMap(individualDelimiters);
+  const delimiterTextToDelimiterInfoMap = Object.fromEntries(
+    individualDelimiters.map((individualDelimiter) => [
+      individualDelimiter.text,
+      individualDelimiter,
+    ]),
+  );
 
   const regexMatches = matchAllIterator(
     document.getText(),
@@ -48,28 +54,37 @@ export function getDelimiterOccurrences(
 
   for (const match of regexMatches) {
     const text = match[0];
-    const range = new Range(
+    const matchRange = new Range(
       document.positionAt(match.index!),
       document.positionAt(match.index! + text.length),
     );
 
-    const delimiter = disqualifyDelimiters.getContaining(range);
-    const isDisqualified = delimiter != null && !delimiter.hasError();
+    const disqualifiedDelimiter = ifNoErrors(
+      disqualifyDelimiters.getContaining(matchRange),
+    );
 
-    if (!isDisqualified) {
-      const textFragmentRange =
-        textFragments.getSmallestContaining(range)?.range;
-      results.push({
-        delimiterInfo: delimiterTextToDelimiterInfoMap[text],
-        textFragmentRange,
-        range,
-      });
+    if (disqualifiedDelimiter != null) {
+      continue;
     }
+
+    results.push({
+      delimiterInfo: delimiterTextToDelimiterInfoMap[text],
+      textFragmentRange: ifNoErrors(
+        textFragments.getSmallestContaining(matchRange),
+      )?.range,
+      range:
+        ifNoErrors(pairDelimiters.getContaining(matchRange))?.range ??
+        matchRange,
+    });
   }
 
   return results;
 }
 
+function ifNoErrors(capture?: QueryCapture): QueryCapture | undefined {
+  return capture != null && !capture.hasError() ? capture : undefined;
+}
+
 function getSortedCaptures(
   languageDefinition: LanguageDefinition | undefined,
   document: TextDocument,
@@ -79,23 +94,3 @@ function getSortedCaptures(
   items.sort((a, b) => a.range.start.compareTo(b.range.start));
   return items;
 }
-
-function getDelimiterTextToDelimiterInfoMap(
-  individualDelimiters: IndividualDelimiter[],
-): Record<string, IndividualDelimiter> {
-  return Object.fromEntries(
-    individualDelimiters.flatMap((individualDelimiter) => {
-      const results = [[individualDelimiter.text, individualDelimiter]];
-      for (const prefix of individualDelimiter.prefixes) {
-        const prefixText = prefix + individualDelimiter.text;
-        const prefixDelimiter: IndividualDelimiter = {
-          ...individualDelimiter,
-          text: prefixText,
-          side: "left",
-        };
-        results.push([prefixText, prefixDelimiter]);
-      }
-      return results;
-    }),
-  );
-}

packages/cursorless-engine/src/processTargets/modifiers/scopeHandlers/SurroundingPairScopeHandler/getDelimiterRegex.ts

@@ -12,23 +12,11 @@ export function getDelimiterRegex(individualDelimiters: IndividualDelimiter[]) {
   // Create a regex which is a disjunction of all possible left / right
   // delimiter texts
   const individualDelimiterDisjunct = uniq(
-    individualDelimiters.flatMap((delimiter) => {
-      const text = escapeRegExp(delimiter.text);
-      const result = [text];
-      for (const prefix of delimiter.prefixes) {
-        // If the prefix is only alpha character, we need to make sure that there is no preceding alpha characters.
-        if (alphaRegex.test(prefix)) {
-          result.push(`(?<!\\w)${prefix}${text}`);
-        } else {
-          result.push(`${escapeRegExp(prefix)}${text}`);
-        }
-      }
-      return result;
-    }),
-  ).join("|");
+    individualDelimiters.map(({ text }) => text),
+  )
+    .map(escapeRegExp)
+    .join("|");
 
   // Then make sure that we don't allow preceding `\`
   return new RegExp(`(?<!\\\\)(${individualDelimiterDisjunct})`, "gu");
 }
-
-const alphaRegex = /^\w+$/;