Talon language support (#1555)

Resolves #817

**Condition**
- [x] context match
- [x] every context match

**Command**
- [x] command
- [x] every command

**Statement**
- [x] command line
- [x] command block
- [x] settings block
- [x] script action call
- [x] script key action call
- [x] script sleep action call
- [x] script assignment
- [x] key binding
- [x] tag binding
- [x] gamepad binding
- [x] parrot binding
- [x] face binding
- [x] every statement in file
- [x] every statement in command block
- [x] every statement in settings block

**Name / Collection key**
- [x] context match left
- [x] command left
- [x] settings block left
- [x] setting left
- [x] tag() left
- [x] key() left
- [x] gamepad() left
- [x] face() left
- [x] parrot() left
- [x] every name/key in context block
- [x] every name/key in file

**Value** 
- [x] context match right
- [x] command right
- [x] settings block right
- [x] setting right
- [x] tag() right
- [x] key() right
- [x] gamepad() right
- [x] face() right
- [x] parrot() right
- [x] every value in context block
- [x] every value in settings block
- [x] every value in file

**Interior**
- [x] command right
- [ ] settings block right
- [x] every command interior in file

**Function call**
- [x] action
- [x] key action
- [x] sleep action
- [x] every call in command block

**Function callee**
- [x] action
- [x] key action
- [x] sleep action
- [x] every callee in command block

**Argument or parameter**
- [x] action
- [x] key action
- [x] sleep action
- [x] every argument in action
 
**Update Tree sitter patterns**
- [x]  Exclude leading/trailing whitespace from `implicit_string`
Waiting for release of
wenkokke/tree-sitter-talon#23
- [x] Removed trailing parentheses `key(` and `sleep(` actions
~~Waiting for release of
wenkokke/tree-sitter-talon#26
    Instead solved with a pattern predicate

## Checklist

- [x] I have added
[tests](https://www.cursorless.org/docs/contributing/test-case-recorder/)
- [ ] Look into failures with new tree-sitter-talon. Do they occur on
shipped version?
  - [ ] "every value bat" in

          aaa: "bbb"
          ccc: "ddd"

  - [x] wenkokke/tree-sitter-talon#29
  - [x] `"take key"` targeting `settings()`
  - [x] `"take key"` targeting `tag()`
  - [x] Rollback whitespace removal
- [x] If the above are all fine on shipped tree-sitter-talon, let's just
file follow-up to upgrade tree-sitter-talon later
- [ ] I have updated the
[docs](https://github.com/cursorless-dev/cursorless/tree/main/docs) and
[cheatsheet](https://github.com/cursorless-dev/cursorless/tree/main/cursorless-talon/src/cheatsheet)
- [x] I have not broken the cheatsheet

---------

Co-authored-by: Pokey Rule <[email protected]>
Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com>

Author: 3 people

cursorless-talon/src/modifiers/scopes.py

@@ -64,6 +64,8 @@
     "paragraph": "namedParagraph",
     "subparagraph": "subParagraph",
     "environment": "environment",
+    # Talon
+    "command": "command",
     # Text-based scope types
     "char": "character",
     "word": "word",

packages/common/src/extensionDependencies.ts

@@ -3,4 +3,5 @@ export const extensionDependencies = [
   "ms-toolsai.jupyter",
   "scalameta.metals",
   "ms-python.python",
+  "mrob95.vscode-talonscript",
 ];

packages/common/src/types/command/PartialTargetDescriptor.types.ts

@@ -134,6 +134,7 @@ export type SimpleScopeTypeType =
   | "xmlElement"
   | "xmlEndTag"
   | "xmlStartTag"
+  | "notebookCell"
   // Latex scope types
   | "part"
   | "chapter"
@@ -154,7 +155,8 @@ export type SimpleScopeTypeType =
   | "nonWhitespaceSequence"
   | "boundedNonWhitespaceSequence"
   | "url"
-  | "notebookCell";
+  // Talon
+  | "command";
 
 export interface SimpleScopeType {
   type: SimpleScopeTypeType;

packages/cursorless-engine/src/generateSpokenForm/defaultSpokenForms/modifiers.ts

@@ -84,6 +84,8 @@ export const scopeSpokenForms = {
   namedParagraph: "paragraph",
   subParagraph: "subparagraph",
   environment: "environment",
+  // Talon
+  command: "command",
   // Text-based scope types
   character: "char",
   word: "word",

packages/cursorless-engine/src/languages/TreeSitterQuery/QueryCapture.ts

@@ -1,4 +1,4 @@
-import { Range } from "@cursorless/common";
+import { Range, TextDocument } from "@cursorless/common";
 import { Point } from "web-tree-sitter";
 
 /**
@@ -35,6 +35,9 @@ export interface QueryCapture {
    * captures with the same name and domain into a single scope with multiple
    * content ranges. */
   readonly allowMultiple: boolean;
+
+  /** The insertion delimiter to use if any */
+  readonly insertionDelimiter: string | undefined;
 }
 
 /**
@@ -57,8 +60,10 @@ export interface MutableQueryCapture extends QueryCapture {
    */
   readonly node: Omit<SimpleSyntaxNode, "startPosition" | "endPosition">;
 
+  readonly document: TextDocument;
   range: Range;
   allowMultiple: boolean;
+  insertionDelimiter: string | undefined;
 }
 
 /**

packages/cursorless-engine/src/languages/TreeSitterQuery/TreeSitterQuery.ts

@@ -78,7 +78,9 @@ export class TreeSitterQuery {
           captures: captures.map(({ name, node }) => ({
             name,
             node,
+            document,
             range: getNodeRange(node),
+            insertionDelimiter: undefined,
             allowMultiple: false,
           })),
         }),
@@ -112,6 +114,9 @@ export class TreeSitterQuery {
               .map(({ range }) => range)
               .reduce((accumulator, range) => range.union(accumulator)),
             allowMultiple: captures.some((capture) => capture.allowMultiple),
+            insertionDelimiter: captures.find(
+              (capture) => capture.insertionDelimiter != null,
+            )?.insertionDelimiter,
           };
         });
 

packages/cursorless-engine/src/languages/TreeSitterQuery/checkCaptureStartEnd.test.ts

@@ -5,7 +5,7 @@ import assert = require("assert");
 
 interface TestCase {
   name: string;
-  captures: Omit<QueryCapture, "allowMultiple">[];
+  captures: Omit<QueryCapture, "allowMultiple" | "insertionDelimiter">[];
   isValid: boolean;
   expectedErrorMessageIds: string[];
 }
@@ -192,6 +192,7 @@ suite("checkCaptureStartEnd", () => {
         testCase.captures.map((capture) => ({
           ...capture,
           allowMultiple: false,
+          insertionDelimiter: undefined,
         })),
         messages,
       );

packages/cursorless-engine/src/languages/TreeSitterQuery/queryPredicateOperators.ts

@@ -1,3 +1,4 @@
+import { Range } from "@cursorless/common";
 import z from "zod";
 import { makeRangeFromPositions } from "../../util/nodeSelectors";
 import { MutableQueryCapture } from "./QueryCapture";
@@ -18,6 +19,17 @@ class NotType extends QueryPredicateOperator<NotType> {
   }
 }
 
+/**
+ * A predicate operator that returns true if the nodes range is not empty.
+ */
+class NotEmpty extends QueryPredicateOperator<NotEmpty> {
+  name = "not-empty?" as const;
+  schema = z.tuple([q.node]);
+  run({ range }: MutableQueryCapture) {
+    return !range.isEmpty;
+  }
+}
+
 /**
  * A predicate operator that returns true if the node's parent is not of the
  * given type. For example, `(not-parent-type? @foo string)` will reject the
@@ -81,6 +93,47 @@ class ChildRange extends QueryPredicateOperator<ChildRange> {
   }
 }
 
+/**
+ * A predicate operator that modifies the range of the match to shrink to regex
+ * match.  For example, `(#shrink-to-match! @foo "\\S+")` will modify the range
+ * of the `@foo` capture to exclude whitespace.
+ *
+ * If convenient, you can use a special capture group called `keep` to indicate
+ * the part of the match that should be kept.  For example,
+ *
+ * ```
+ * (#shrink-to-match! @foo "^\s+(?<keep>.*)$")
+ * ```
+ *
+ * will modify the range of the `@foo` capture to skip any leading whitespace.
+ */
+class ShrinkToMatch extends QueryPredicateOperator<ShrinkToMatch> {
+  name = "shrink-to-match!" as const;
+  schema = z.tuple([q.node, q.string]);
+
+  run(nodeInfo: MutableQueryCapture, pattern: string) {
+    const { document, range } = nodeInfo;
+    const text = document.getText(range);
+    const match = text.match(new RegExp(pattern, "ds"));
+
+    if (match?.index == null) {
+      throw Error(`No match for pattern '${pattern}'`);
+    }
+
+    const [startOffset, endOffset] =
+      match.indices?.groups?.keep ?? match.indices![0];
+
+    const baseOffset = document.offsetAt(range.start);
+
+    nodeInfo.range = new Range(
+      document.positionAt(baseOffset + startOffset),
+      document.positionAt(baseOffset + endOffset),
+    );
+
+    return true;
+  }
+}
+
 class AllowMultiple extends QueryPredicateOperator<AllowMultiple> {
   name = "allow-multiple!" as const;
   schema = z.tuple([q.node]);
@@ -92,10 +145,29 @@ class AllowMultiple extends QueryPredicateOperator<AllowMultiple> {
   }
 }
 
+/**
+ * A predicate operator that sets the insertion delimiter of the match. For
+ * example, `(#insertion-delimiter! @foo ", ")` will set the insertion delimiter
+ * of the `@foo` capture to `", "`.
+ */
+class InsertionDelimiter extends QueryPredicateOperator<InsertionDelimiter> {
+  name = "insertion-delimiter!" as const;
+  schema = z.tuple([q.node, q.string]);
+
+  run(nodeInfo: MutableQueryCapture, insertionDelimiter: string) {
+    nodeInfo.insertionDelimiter = insertionDelimiter;
+
+    return true;
+  }
+}
+
 export const queryPredicateOperators = [
   new NotType(),
+  new NotEmpty(),
   new NotParentType(),
   new IsNthChild(),
   new ChildRange(),
+  new ShrinkToMatch(),
   new AllowMultiple(),
+  new InsertionDelimiter(),
 ];

packages/cursorless-engine/src/languages/TreeSitterQuery/rewriteStartOfEndOf.test.ts

@@ -1,6 +1,6 @@
-import { Range } from "@cursorless/common";
-import { MutableQueryCapture } from "./QueryCapture";
+import { Range, TextDocument } from "@cursorless/common";
 import { SyntaxNode } from "web-tree-sitter";
+import { MutableQueryCapture } from "./QueryCapture";
 import { rewriteStartOfEndOf } from "./rewriteStartOfEndOf";
 import assert = require("assert");
 
@@ -50,24 +50,21 @@ const testCases: TestCase[] = [
   },
 ];
 
+function fillOutCapture(capture: NameRange): MutableQueryCapture {
+  return {
+    ...capture,
+    allowMultiple: false,
+    insertionDelimiter: undefined,
+    document: null as unknown as TextDocument,
+    node: null as unknown as SyntaxNode,
+  };
+}
+
 suite("rewriteStartOfEndOf", () => {
   for (const testCase of testCases) {
     test(testCase.name, () => {
-      const actual = rewriteStartOfEndOf(
-        testCase.captures.map((capture) => ({
-          ...capture,
-          allowMultiple: false,
-          node: null as unknown as SyntaxNode,
-        })),
-      );
-      assert.deepStrictEqual(
-        actual,
-        testCase.expected.map((capture) => ({
-          ...capture,
-          allowMultiple: false,
-          node: null as unknown as SyntaxNode,
-        })),
-      );
+      const actual = rewriteStartOfEndOf(testCase.captures.map(fillOutCapture));
+      assert.deepStrictEqual(actual, testCase.expected.map(fillOutCapture));
     });
   }
 });

packages/cursorless-engine/src/processTargets/modifiers/scopeHandlers/TreeSitterScopeHandler/TreeSitterScopeHandler.ts

@@ -1,4 +1,4 @@
-import { SimpleScopeType, TextEditor } from "@cursorless/common";
+import { Range, SimpleScopeType, TextEditor } from "@cursorless/common";
 import { TreeSitterQuery } from "../../../../languages/TreeSitterQuery";
 import { QueryMatch } from "../../../../languages/TreeSitterQuery/QueryCapture";
 import ScopeTypeTarget from "../../../targets/ScopeTypeTarget";
@@ -48,25 +48,19 @@ export class TreeSitterScopeHandler extends BaseTreeSitterScopeHandler {
       return undefined;
     }
 
-    const { range: contentRange, allowMultiple } = capture;
+    const { range: contentRange, allowMultiple, insertionDelimiter } = capture;
 
     const domain =
       getRelatedRange(match, scopeTypeType, "domain", true) ?? contentRange;
 
     const removalRange = getRelatedRange(match, scopeTypeType, "removal", true);
 
-    const leadingDelimiterRange = getRelatedRange(
-      match,
-      scopeTypeType,
-      "leading",
-      true,
+    const leadingDelimiterRange = dropEmptyRange(
+      getRelatedRange(match, scopeTypeType, "leading", true),
     );
 
-    const trailingDelimiterRange = getRelatedRange(
-      match,
-      scopeTypeType,
-      "trailing",
-      true,
+    const trailingDelimiterRange = dropEmptyRange(
+      getRelatedRange(match, scopeTypeType, "trailing", true),
     );
 
     const interiorRange = getRelatedRange(
@@ -90,9 +84,13 @@ export class TreeSitterScopeHandler extends BaseTreeSitterScopeHandler {
           leadingDelimiterRange,
           trailingDelimiterRange,
           interiorRange,
-          // FIXME: Add delimiter text
+          delimiter: insertionDelimiter,
         }),
       ],
     };
   }
 }
+
+function dropEmptyRange(range?: Range) {
+  return range != null && !range.isEmpty ? range : undefined;
+}