Break sentence scope on [.!?] (#1747)

sbd would sometimes not break lines ending with [.!?] when
`new_line_boundaries: false` is used

## Checklist

- [x] I have added
[tests](https://www.cursorless.org/docs/contributing/test-case-recorder/)
- [/] I have updated the
[docs](https://github.com/cursorless-dev/cursorless/tree/main/docs) and
[cheatsheet](https://github.com/cursorless-dev/cursorless/tree/main/cursorless-talon/src/cheatsheet)
- [/] I have not broken the cheatsheet

---------

Co-authored-by: Pokey Rule <[email protected]>

Author: AndreasArvidsson

packages/cursorless-engine/src/processTargets/modifiers/scopeHandlers/SentenceScopeHandler/SentenceSegmenter.ts

@@ -1,10 +1,19 @@
 import * as sbd from "sbd";
 import { MatchedText, matchRegex, testRegex } from "../../../../util/regex";
 
-// A sentence starts with a letter
-const leadingOffsetRegex = /\p{L}/u;
-// A line with no letters is invalid and breaks sentences
-const invalidLineRegex = /(\n[^\p{L}]*\n)/gu;
+// A sentence starts with a letter with adjacent leading symbols. Whitespace excluded.
+const leadingOffsetRegex = /\S*\p{L}/u;
+/**
+ * This regex is used to split the text that comes back from sbd. Anything
+ * matching this regex will be discarded from the returned sentence, and split
+ * the sentence in two if it occurs in the middle of a sentence. The regex
+ * matches either of the following:
+ * 1. An entire line containing no letters.
+ * 2. If a line ends in `[.!?]`, possibly followed by whitespace, we split on
+ *    that. This is a workaround for #1753. FIXME: Remove this second term once
+ *    #1753 is fixed.
+ */
+const skipPartRegex = /(\r?\n[^\p{L}]*\r?\n)|(?<=[.!?])(\s*\r?\n)/gu;
 
 const options: sbd.Options = {
   ["newline_boundaries"]: false,
@@ -17,10 +26,10 @@ export default class SentenceSegmenter {
     let index = 0;
 
     for (const sentence of sentences) {
-      const parts = sentence.split(invalidLineRegex);
+      const parts = sentence.split(skipPartRegex).filter((p) => p != null);
 
       for (const part of parts) {
-        if (!isInvalidLine(part)) {
+        if (!skipPart(part)) {
           const segment = createSegment(part, index);
           if (segment != null) {
             yield segment;
@@ -56,6 +65,6 @@ function createSegment(
   };
 }
 
-function isInvalidLine(text: string): boolean {
-  return testRegex(invalidLineRegex, text);
+function skipPart(text: string): boolean {
+  return testRegex(skipPartRegex, text);
 }

packages/cursorless-engine/src/test/fixtures/sentenceSegmeter.fixture.ts

@@ -26,10 +26,14 @@ export const sentenceSegmenterFixture: Fixture[] = [
   },
   {
     input: "1Foo",
-    expectedOutput: ["Foo"],
+    expectedOutput: ["1Foo"],
   },
   {
     input: "_foo",
+    expectedOutput: ["_foo"],
+  },
+  {
+    input: "* foo",
     expectedOutput: ["foo"],
   },
   {
@@ -40,6 +44,18 @@ export const sentenceSegmenterFixture: Fixture[] = [
     input: "* Foo \nbar. *",
     expectedOutput: ["Foo \nbar."],
   },
+  {
+    input: "Foo bar. Baz bongo.",
+    expectedOutput: ["Foo bar.", "Baz bongo."],
+  },
+  {
+    input: "Foo bar. \nBaz bongo.",
+    expectedOutput: ["Foo bar.", "Baz bongo."],
+  },
+  {
+    input: "Foo Bar. \nBaz bongo.",
+    expectedOutput: ["Foo Bar.", "Baz bongo."],
+  },
   {
     input: "Foo \n*\nbar",
     expectedOutput: ["Foo", "bar"],

packages/cursorless-vscode-e2e/src/suite/fixtures/recorded/scopes/sentence/changeEverySentence.yml

@@ -0,0 +1,29 @@
+languageId: markdown
+command:
+  version: 6
+  spokenForm: change every sentence
+  action:
+    name: clearAndSetSelection
+    target:
+      type: primitive
+      modifiers:
+        - type: everyScope
+          scopeType: {type: sentence}
+  usePrePhraseSnapshot: true
+initialState:
+  documentContents: |-
+    1. **Custom subtitles** - User customizable subtitles for Talon. 
+        - [Subtitles and notifications](./core/on_phrase/subtitles_and_notifications)
+  selections:
+    - anchor: {line: 0, character: 0}
+      active: {line: 0, character: 0}
+  marks: {}
+finalState:
+  documentContents: |-
+    1.  
+        - 
+  selections:
+    - anchor: {line: 0, character: 3}
+      active: {line: 0, character: 3}
+    - anchor: {line: 1, character: 6}
+      active: {line: 1, character: 6}