feat(ohm-js): Add support for XID_Start, XID_Continue, White_Space

pdubroy · pdubroy · commit 448cc86cccf0 · 2025-08-02T19:22:42.000+02:00
Fixes #180, 9 years later
diff --git a/doc/syntax-reference.md b/doc/syntax-reference.md
@@ -176,6 +176,8 @@ as well as multiline (`/* */`) comments like:
 
 <code id="applySyntactic">applySyntactic&lt;<i>ruleName</i>&gt;</code>: Allows the syntactic rule _ruleName_ to be applied in a lexical context, which is otherwise not allowed. Spaces are skipped _before_ and _after_ the rule application. _New in Ohm v16.1.0._
 
+<code>unicodeChar&lt;<i>categoryOrProp</i>&gt;</code>: matches a single Unicode code point from a given category, or with a given binary property. _categoryOrProp_ is a terminal that is either (a) a valid Unicode General_Category value (e.g. `"Zl"`), or (b) one of the following binary property names: `"XID_Start"`, `"XID_Continue"`, `"White_Space"`.
+
 ## Grammar Syntax
 
 ### Grammar Inheritance
diff --git a/packages/ohm-js/src/UnicodeCategories.js b/packages/ohm-js/src/UnicodeCategories.js
diff --git a/packages/ohm-js/src/pexprs-main.js b/packages/ohm-js/src/pexprs-main.js
@@ -1,4 +1,4 @@
-import {UnicodeCategories} from './UnicodeCategories.js';
+import {UnicodeBinaryProperties, UnicodeCategories} from './unicode.js';
 import * as common from './common.js';
 
 // --------------------------------------------------------------------
@@ -180,9 +180,17 @@ export class Apply extends PExpr {
 // Unicode character
 
 export class UnicodeChar extends PExpr {
-  constructor(category) {
+  constructor(categoryOrProp) {
     super();
-    this.category = category;
-    this.pattern = UnicodeCategories[category];
+    this.categoryOrProp = categoryOrProp;
+    if (categoryOrProp in UnicodeCategories) {
+      this.pattern = UnicodeCategories[categoryOrProp];
+    } else if (categoryOrProp in UnicodeBinaryProperties) {
+      this.pattern = UnicodeBinaryProperties[categoryOrProp];
+    } else {
+      throw new Error(
+        `Invalid Unicode category or property name: ${JSON.stringify(categoryOrProp)}`
+      );
+    }
   }
 }
diff --git a/packages/ohm-js/src/pexprs-outputRecipe.js b/packages/ohm-js/src/pexprs-outputRecipe.js
@@ -92,5 +92,5 @@ pexprs.Apply.prototype.outputRecipe = function(formals, grammarInterval) {
 };
 
 pexprs.UnicodeChar.prototype.outputRecipe = function(formals, grammarInterval) {
-  return ['unicodeChar', getMetaInfo(this, grammarInterval), this.category];
+  return ['unicodeChar', getMetaInfo(this, grammarInterval), this.categoryOrProp];
 };
diff --git a/packages/ohm-js/src/pexprs-toDisplayString.js b/packages/ohm-js/src/pexprs-toDisplayString.js
@@ -38,5 +38,5 @@ pexprs.Apply.prototype.toDisplayString = function() {
 };
 
 pexprs.UnicodeChar.prototype.toDisplayString = function() {
-  return 'Unicode [' + this.category + '] character';
+  return 'Unicode [' + this.categoryOrProp + '] character';
 };
diff --git a/packages/ohm-js/src/pexprs-toFailure.js b/packages/ohm-js/src/pexprs-toFailure.js
@@ -45,7 +45,7 @@ pexprs.Apply.prototype.toFailure = function(grammar) {
 };
 
 pexprs.UnicodeChar.prototype.toFailure = function(grammar) {
-  return new Failure(this, 'a Unicode [' + this.category + '] character', 'description');
+  return new Failure(this, 'a Unicode [' + this.categoryOrProp + '] character', 'description');
 };
 
 pexprs.Alt.prototype.toFailure = function(grammar) {
diff --git a/packages/ohm-js/src/pexprs-toString.js b/packages/ohm-js/src/pexprs-toString.js
@@ -72,5 +72,5 @@ pexprs.Apply.prototype.toString = function() {
 };
 
 pexprs.UnicodeChar.prototype.toString = function() {
-  return '\\p{' + this.category + '}';
+  return '\\p{' + this.categoryOrProp + '}';
 };
diff --git a/packages/ohm-js/src/unicode.js b/packages/ohm-js/src/unicode.js
@@ -0,0 +1,54 @@
+// The full list of categories from:
+// https://www.unicode.org/Public/UCD/latest/ucd/extracted/DerivedGeneralCategory.txt.
+
+const toRegExp = val => new RegExp(String.raw`\p{${val}}`, 'u');
+
+/*
+  grep -v '^#' DerivedGeneralCategory.txt \
+    | cut -d';' -f2 \
+    | awk 'NF{print $1}' \
+    | sort -u \
+    | awk '{printf "\x27%s\x27,\n",$1}'
+ */
+
+export const UnicodeCategories = Object.fromEntries(
+  [
+    'Cc',
+    'Cf',
+    'Cn',
+    'Co',
+    'Cs',
+    'Ll',
+    'Lm',
+    'Lo',
+    'Lt',
+    'Lu',
+    'Mc',
+    'Me',
+    'Mn',
+    'Nd',
+    'Nl',
+    'No',
+    'Pc',
+    'Pd',
+    'Pe',
+    'Pf',
+    'Pi',
+    'Po',
+    'Ps',
+    'Sc',
+    'Sk',
+    'Sm',
+    'So',
+    'Zl',
+    'Zp',
+    'Zs'
+  ].map(cat => [cat, toRegExp(cat)])
+);
+UnicodeCategories['Ltmo'] = /\p{Lt}|\p{Lm}|\p{Lo}/u;
+
+// We only support a few of these for now, but could add more later.
+// See https://www.unicode.org/Public/UCD/latest/ucd/PropertyAliases.txt
+export const UnicodeBinaryProperties = Object.fromEntries(
+  ['XID_Start', 'XID_Continue', 'White_Space'].map(prop => [prop, toRegExp(prop)])
+);
diff --git a/packages/wasm/src/index.js b/packages/wasm/src/index.js
@@ -915,7 +915,7 @@ export class Compiler {
         case pexprs.Terminal:
           return ir.terminal(exp.obj);
         case pexprs.UnicodeChar:
-          return ir.unicodeChar(exp.category);
+          return ir.unicodeChar(exp.categoryOrProp);
         default:
           throw new Error(`not handled: ${exp.constructor.name}`);
       }
@@ -1691,13 +1691,13 @@ export class Compiler {
     const {asm} = this;
 
     // TODO: Add support for more categories, by calling out to the host.
-    assert(['Ll', 'Lu', 'Ltmo'].includes(exp.category));
+    assert(['Ll', 'Lu', 'Ltmo'].includes(exp.categoryOrProp));
 
     const makeLabels = () =>
       asciiChars.map(c => {
         const isLowercase = 'a' <= c && c <= 'z';
         const isUppercase = 'A' <= c && c <= 'Z';
-        if ((exp.category === 'Lu' && isUppercase) || (exp.category === 'Ll' && isLowercase)) {
+        if ((exp.categoryOrProp === 'Lu' && isUppercase) || (exp.categoryOrProp === 'Ll' && isLowercase)) {
           return w.labelidx(asm.depthOf('fastSuccess'));
         }
         return w.labelidx(asm.depthOf('failure'));
@@ -1723,7 +1723,7 @@ export class Compiler {
 
                   // Push the arg: a bitmap indicating the categories.
                   // prettier-ignore
-                  switch (exp.category) {
+                  switch (exp.categoryOrProp) {
                     case 'Lu': asm.i32Const(1 << 1); break;
                     case 'Ll': asm.i32Const(1 << 2); break;
                     case 'Ltmo': asm.i32Const((1 << 3) | (1 << 4) | (1 << 5)); break;
diff --git a/packages/wasm/src/ir.ts b/packages/wasm/src/ir.ts
@@ -165,12 +165,12 @@ export const terminal = (value: string, caseInsensitive = false): Terminal => ({
 
 export interface UnicodeChar {
   type: 'UnicodeChar';
-  category: string;
+  categoryOrProp: string;
 }
 
-export const unicodeChar = (category: string): UnicodeChar => ({
+export const unicodeChar = (categoryOrProp: string): UnicodeChar => ({
   type: 'UnicodeChar',
-  category
+  categoryOrProp
 });
 
 // Types that are specific to the IR
@@ -351,7 +351,7 @@ export function toString(exp: Expr): string {
     case 'Terminal':
       return JSON.stringify(exp.value);
     case 'UnicodeChar':
-      return `$unicodeChar<${JSON.stringify(exp.category)}>`;
+      return `$unicodeChar<${JSON.stringify(exp.categoryOrProp)}>`;
     case 'Dispatch':
       return `$dispatch`; // TODO: Improve this.
     case 'Lex':
