fix: [createTokenizer] moveOnBreak logic at end-of-file

Signed-off-by: Lexus Drumgold <unicornware@flexdevelopment.llc>

Author: unicornware

__fixtures__/constructs/cli.mts

@@ -4,6 +4,7 @@
  */
 
 import codes from '#enums/codes'
+import eof from '#fixtures/constructs/eof'
 import longFlag from '#fixtures/constructs/flag-long'
 import operand from '#fixtures/constructs/operand'
 import type { ConstructRecord } from '@flex-development/vfile-tokenizer'
@@ -13,6 +14,6 @@ import type { ConstructRecord } from '@flex-development/vfile-tokenizer'
  *
  * @const {ConstructRecord} cli
  */
-const cli: ConstructRecord = { [codes.hyphen]: longFlag, null: operand }
+const cli: ConstructRecord = { [codes.hyphen]: longFlag, null: [operand, eof] }
 
 export default cli

__fixtures__/constructs/eof.mts

@@ -5,6 +5,7 @@
 
 import codes from '#enums/codes'
 import tt from '#fixtures/tt'
+import eos from '#utils/eof'
 import type {
   Code,
   Construct,
@@ -21,26 +22,12 @@ import { ok as assert } from 'devlop'
  */
 const eof: Construct = {
   name: tt.eof,
-  test: testEOF,
+  test: eos,
   tokenize: tokenizeEOF
 }
 
 export default eof
 
-/**
- * Check if the current character `code` can start the end of file construct.
- *
- * @this {TokenizeContext}
- *
- * @param {Code} code
- *  The current character code
- * @return {boolean}
- *  `true` if `code` can start construct
- */
-function testEOF(this: TokenizeContext, code: Code): boolean {
-  return code === codes.eof
-}
-
 /**
  * Tokenize the end of file.
  *

src/__snapshots__/tokenize.integration.snap

@@ -179,6 +179,46 @@ exports[`integration:tokenize > should work with constructs (1) 1`] = `
       "value": "",
     },
   ],
+  [
+    "enter",
+    {
+      "end": {
+        "_bufferIndex": -1,
+        "_index": 2,
+        "column": 14,
+        "line": 1,
+        "offset": 13,
+      },
+      "start": {
+        "_bufferIndex": -1,
+        "_index": 1,
+        "column": 14,
+        "line": 1,
+        "offset": 13,
+      },
+      "type": "eof",
+    },
+  ],
+  [
+    "exit",
+    {
+      "end": {
+        "_bufferIndex": -1,
+        "_index": 2,
+        "column": 14,
+        "line": 1,
+        "offset": 13,
+      },
+      "start": {
+        "_bufferIndex": -1,
+        "_index": 1,
+        "column": 14,
+        "line": 1,
+        "offset": 13,
+      },
+      "type": "eof",
+    },
+  ],
 ]
 `;
 
@@ -316,6 +356,46 @@ exports[`integration:tokenize > should work with constructs (2) 1`] = `
       "value": "3",
     },
   ],
+  [
+    "enter",
+    {
+      "end": {
+        "_bufferIndex": -1,
+        "_index": 3,
+        "column": 18,
+        "line": 1,
+        "offset": 17,
+      },
+      "start": {
+        "_bufferIndex": -1,
+        "_index": 2,
+        "column": 18,
+        "line": 1,
+        "offset": 17,
+      },
+      "type": "eof",
+    },
+  ],
+  [
+    "exit",
+    {
+      "end": {
+        "_bufferIndex": -1,
+        "_index": 3,
+        "column": 18,
+        "line": 1,
+        "offset": 17,
+      },
+      "start": {
+        "_bufferIndex": -1,
+        "_index": 2,
+        "column": 18,
+        "line": 1,
+        "offset": 17,
+      },
+      "type": "eof",
+    },
+  ],
 ]
 `;
 
@@ -546,6 +626,46 @@ exports[`integration:tokenize > should work with constructs (4) 1`] = `
       "value": "@flex-development/vfile-tokenizer",
     },
   ],
+  [
+    "enter",
+    {
+      "end": {
+        "_bufferIndex": -1,
+        "_index": 3,
+        "column": 43,
+        "line": 1,
+        "offset": 42,
+      },
+      "start": {
+        "_bufferIndex": -1,
+        "_index": 2,
+        "column": 43,
+        "line": 1,
+        "offset": 42,
+      },
+      "type": "eof",
+    },
+  ],
+  [
+    "exit",
+    {
+      "end": {
+        "_bufferIndex": -1,
+        "_index": 3,
+        "column": 43,
+        "line": 1,
+        "offset": 42,
+      },
+      "start": {
+        "_bufferIndex": -1,
+        "_index": 2,
+        "column": 43,
+        "line": 1,
+        "offset": 42,
+      },
+      "type": "eof",
+    },
+  ],
 ]
 `;
 

src/__tests__/tokenize.integration.spec.mts

@@ -22,8 +22,8 @@ import type {
   EventType,
   FileLike,
   List,
+  Options,
   Token,
-  TokenizeOptions,
   Value
 } from '@flex-development/vfile-tokenizer'
 import pkg from '@flex-development/vfile-tokenizer/package.json'
@@ -54,7 +54,7 @@ describe('integration:tokenize', () => {
 
   it.each<[
     value: FileLike | List<FileLike | Value> | Value | null | undefined,
-    options?: Partial<TokenizeOptions> | null | undefined
+    options?: Partial<Options> | null | undefined
   ]>([
     [['hello', 'world']],
     [read('__fixtures__/markdown/code-fenced.md'), { tabSize: 2 }]

src/create-tokenizer.mts

@@ -400,7 +400,10 @@ function createTokenizer(this: void, options: Options): TokenizeContext {
           if (context.code === codes.break && peek() !== codes.break) {
             context.previous = context.code
             context.code = code
-            if (options.moveOnBreak) move(context.previous)
+
+            if (options.moveOnBreak && context.code !== codes.eof) {
+              move(context.previous)
+            }
           }
 
           /**
@@ -769,8 +772,7 @@ function createTokenizer(this: void, options: Options): TokenizeContext {
       if (!Array.isArray(chunk)) { // not in buffer chunk.
         assert(place._bufferIndex < 0, 'expected negative `_bufferIndex`')
         code = chunk
-      } else { // in buffer chunk.
-        assert(place._bufferIndex >= 0, 'expected non-negative `_bufferIndex`')
+      } else { // in or at end of buffer chunk.
         code = chunk[place._bufferIndex]
       }
     }

src/enums/codes.mts

@@ -90,7 +90,7 @@ const codes = {
   digit7: 55, // 7
   digit8: 56, // 8
   digit9: 57, // 9
-  colon: 58, // =
+  colon: 58, // :
   semicolon: 59, // ;
   leftAngleBracket: 60, // <
   lt: 60, // <

src/interfaces/__tests__/options-tokenize.spec-d.mts

@@ -11,7 +11,6 @@ export type { default as Options } from '#interfaces/options'
 export type {
   default as PreprocessOptions
 } from '#interfaces/options-preprocess'
-export type { default as TokenizeOptions } from '#interfaces/options-tokenize'
 export type { default as Place } from '#interfaces/place'
 export type { default as Position } from '#interfaces/position'
 export type { default as Preprocess } from '#interfaces/preprocess'

src/interfaces/index.mts

@@ -10,8 +10,8 @@ import type {
   Event,
   FileLike,
   List,
+  Options,
   TokenizeContext,
-  TokenizeOptions,
   Value
 } from '@flex-development/vfile-tokenizer'
 
@@ -21,34 +21,37 @@ import type {
  * @see {@linkcode Event}
  * @see {@linkcode FileLike}
  * @see {@linkcode List}
- * @see {@linkcode TokenizeOptions}
+ * @see {@linkcode Options}
+ * @see {@linkcode TokenizeContext}
  * @see {@linkcode Value}
  *
  * @this {void}
  *
  * @param {FileLike | List<FileLike | Value> | Value | null | undefined} value
  *  The file, value, or list of files and/or values to tokenize
- * @param {TokenizeOptions} options
- *  Configuration options
+ * @param {Options | TokenizeContext} options
+ *  Configuration options or the tokenizer to use
  * @return {Event[]}
  *  List of events
  */
 function tokenize(
   this: void,
   value: FileLike | List<FileLike | Value> | Value | null | undefined,
-  options: TokenizeOptions
+  options: Options | TokenizeContext
 ): Event[] {
   /**
    * Tokenize context.
    *
-   * @const {TokenizeContext} context
+   * @var {TokenizeContext} context
    */
-  const context: TokenizeContext = options.tokenizer ?? createTokenizer(options)
+  let context: TokenizeContext = options as TokenizeContext
+
+  // create tokenizer.
+  if (!('write' in options)) context = createTokenizer(options)
 
   // write chunks to stream.
   if (value !== null && value !== undefined) {
-    for (const chunk of toList(value)) context.write(chunk)
-    context.write(codes.eof)
+    for (const chunk of [...toList(value), codes.eof]) context.write(chunk)
   }
 
   return [...context.events]