feat(linter/plugins): add tokens property to Program (#16020)

Add a `tokens` property to `Program`. Like ESLint, `program.tokens` contains only syntax tokens, and does not include comments.

Implemented as a getter, which lazily tokenizes source text on first access, using the same mechanism as other tokens APIs.

Note: If user has an AST to access the `tokens` getter on, `sourceText` must have been initialized, so the getter doesn't need to check that.

Author: overlookmotel

apps/oxlint/src-js/generated/deserialize.js

@@ -1,6 +1,8 @@
 // Auto-generated code, DO NOT EDIT DIRECTLY!
 // To edit this generated file you have to edit `tasks/ast_tools/src/generators/raw_transfer.rs`.
 
+import { tokens, initTokens } from '../plugins/tokens.js';
+
 let uint8,
   uint32,
   float64,
@@ -62,6 +64,10 @@ function deserializeProgram(pos) {
         Object.defineProperty(this, 'comments', { value: comments });
         return comments;
       },
+      get tokens() {
+        tokens === null && initTokens();
+        return tokens;
+      },
       start: 0,
       end,
       range: [0, end],

apps/oxlint/src-js/generated/types.d.ts

@@ -2,15 +2,17 @@
 // To edit this generated file you have to edit `tasks/ast_tools/src/generators/typescript.rs`.
 
 import { Span } from '../plugins/location.ts';
+import { Token } from '../plugins/tokens.ts';
 import { Comment } from '../plugins/types.ts';
-export { Span, Comment };
+export { Span, Comment, Token };
 
 export interface Program extends Span {
   type: 'Program';
   body: Array<Directive | Statement>;
   sourceType: ModuleKind;
   hashbang: Hashbang | null;
   comments: Comment[];
+  tokens: Token[];
   parent: null;
 }
 

apps/oxlint/src-js/plugins/tokens.ts

@@ -135,7 +135,7 @@ export interface TemplateToken extends BaseToken {
 
 // Tokens for the current file parsed by TS-ESLint.
 // Created lazily only when needed.
-let tokens: Token[] | null = null;
+export let tokens: Token[] | null = null;
 let comments: CommentToken[] | null = null;
 let tokensWithComments: Token[] | null = null;
 
@@ -146,8 +146,10 @@ let tsEslintParse: typeof import('@typescript-eslint/typescript-estree').parse |
 
 /**
  * Initialize TS-ESLint tokens for current file.
+ *
+ * Caller must ensure `sourceText` is initialized before calling this function.
  */
-function initTokens() {
+export function initTokens() {
   debugAssertIsNonNull(sourceText);
 
   // Lazy-load TS-ESLint.

apps/oxlint/test/fixtures/tokens/.oxlintrc.json

@@ -0,0 +1,9 @@
+{
+  "jsPlugins": ["./plugin.ts"],
+  "categories": {
+    "correctness": "off"
+  },
+  "rules": {
+    "tokens-plugin/tokens": "error"
+  }
+}

apps/oxlint/test/fixtures/tokens/files/index.js

@@ -0,0 +1,8 @@
+// Leading comment
+
+let x = /* inline comment */ 1;
+
+// Another comment
+let y = 2;
+
+// Trailing comment

apps/oxlint/test/fixtures/tokens/output.snap.md

@@ -0,0 +1,36 @@
+# Exit code
+1
+
+# stdout
+```
+  x tokens-plugin(tokens): Tokens:
+  | Keyword           loc=3:0-3:3     range=20-23 "let"
+  | Identifier        loc=3:4-3:5     range=24-25 "x"
+  | Punctuator        loc=3:6-3:7     range=26-27 "="
+  | Numeric           loc=3:29-3:30   range=49-50 "1"
+  | Punctuator        loc=3:30-3:31   range=50-51 ";"
+  | Keyword           loc=6:0-6:3     range=72-75 "let"
+  | Identifier        loc=6:4-6:5     range=76-77 "y"
+  | Punctuator        loc=6:6-6:7     range=78-79 "="
+  | Numeric           loc=6:8-6:9     range=80-81 "2"
+  | Punctuator        loc=6:9-6:10    range=81-82 ";"
+   ,-[files/index.js:1:1]
+ 1 | ,-> // Leading comment
+ 2 | |   
+ 3 | |   let x = /* inline comment */ 1;
+ 4 | |   
+ 5 | |   // Another comment
+ 6 | |   let y = 2;
+ 7 | |   
+ 8 | `-> // Trailing comment
+   `----
+
+Found 0 warnings and 1 error.
+Finished in Xms on 1 file using X threads.
+```
+
+# stderr
+```
+WARNING: JS plugins are experimental and not subject to semver.
+Breaking changes are possible while JS plugins support is under development.
+```

apps/oxlint/test/fixtures/tokens/plugin.ts

@@ -0,0 +1,33 @@
+import type { Plugin, Rule } from '../../../dist/index.js';
+
+const rule: Rule = {
+  create(context) {
+    const { sourceCode } = context,
+      { ast } = sourceCode;
+
+    // Note: Comments should not appear in `ast.tokens`
+    context.report({
+      message:
+        `Tokens:\n` +
+        ast.tokens
+          .map(
+            ({ type, loc, range, value }) =>
+              `${type.padEnd(17)} ` +
+              `loc=${loc.start.line}:${loc.start.column}-${loc.end.line}:${loc.end.column} `.padEnd(16) +
+              `range=${range[0]}-${range[1]} `.padEnd(10) +
+              `"${value}"`,
+          )
+          .join('\n'),
+      node: { range: [0, sourceCode.text.length] },
+    });
+
+    return {};
+  },
+};
+
+const plugin: Plugin = {
+  meta: { name: 'tokens-plugin' },
+  rules: { tokens: rule },
+};
+
+export default plugin;

crates/oxc_ast/src/serialize/mod.rs

@@ -142,6 +142,12 @@ impl Program<'_> {
             return comments;
         },
         /* END_IF */
+        /* IF LINTER */
+        get tokens() {
+            if (tokens === null) initTokens();
+            return tokens;
+        },
+        /* END_IF */
         start,
         end,
         ...(RANGE && { range: [start, end] }),

tasks/ast_tools/src/generators/raw_transfer.rs

@@ -134,6 +134,10 @@ fn generate_deserializers(
 
     #[rustfmt::skip]
     let mut code = format!("
+        /* IF LINTER */
+        import {{ tokens, initTokens }} from '../plugins/tokens.js';
+        /* END_IF */
+
         let uint8, uint32, float64, sourceText, sourceIsAscii, sourceByteLen;
 
         let astId = 0;

tasks/ast_tools/src/generators/typescript.rs

@@ -479,11 +479,11 @@ fn amend_oxlint_types(code: &str) -> String {
 
     let mut code = SPAN_REGEX.replace(code, SpanReplacer).into_owned();
 
-    // Add `comments` field to `Program`
+    // Add `comments` and `tokens` fields to `Program`
     #[expect(clippy::items_after_statements)]
     const HASHBANG_FIELD: &str = "hashbang: Hashbang | null;";
     let index = code.find(HASHBANG_FIELD).unwrap();
-    code.insert_str(index + HASHBANG_FIELD.len(), "comments: Comment[];");
+    code.insert_str(index + HASHBANG_FIELD.len(), "comments: Comment[]; tokens: Token[];");
 
     // Make `parent` fields non-optional
     #[expect(clippy::disallowed_methods)]
@@ -492,8 +492,9 @@ fn amend_oxlint_types(code: &str) -> String {
     #[rustfmt::skip]
     code.insert_str(0, "
         import { Span } from '../plugins/location.ts';
+        import { Token } from '../plugins/tokens.ts';
         import { Comment } from '../plugins/types.ts';
-        export { Span, Comment };
+        export { Span, Comment, Token };
 
     ");