Merge pull request #9560 from keymanapp/feat/developer/9515-drop-u1234-epic-ldml

Author: srl295

developer/src/kmc-kmn/src/compiler/compiler.ts

@@ -421,6 +421,17 @@ export class KmnCompiler implements UnicodeSetParser {
     return Module.kmcmp_testSentry();
   }
 
+  /** convert `\u{1234}` to `\u1234` etc */
+  public static fixNewPattern(pattern: string) : string {
+    pattern = pattern.replaceAll(/\\u\{([0-9a-fA-F]{6})\}/g, `\\U00$1`);
+    pattern = pattern.replaceAll(/\\u\{([0-9a-fA-F]{5})\}/g, `\\U000$1`);
+    pattern = pattern.replaceAll(/\\u\{([0-9a-fA-F]{4})\}/g, `\\u$1`);
+    pattern = pattern.replaceAll(/\\u\{([0-9a-fA-F]{3})\}/g, `\\u0$1`);
+    pattern = pattern.replaceAll(/\\u\{([0-9a-fA-F]{2})\}/g, `\\u00$1`);
+    pattern = pattern.replaceAll(/\\u\{([0-9a-fA-F]{1})\}/g, `\\u000$1`);
+    return pattern;
+  }
+
   /**
    *
    * @param pattern UnicodeSet pattern such as `[a-z]`
@@ -435,6 +446,8 @@ export class KmnCompiler implements UnicodeSetParser {
 
     // TODO-LDML: Catch OOM
     const buf = this.wasmExports.malloc(rangeCount * 2 * Module.HEAPU32.BYTES_PER_ELEMENT);
+    // fix \u1234 pattern format
+    pattern = KmnCompiler.fixNewPattern(pattern);
     /** If <= 0: return code. If positive: range count */
     const rc = Module.kmcmp_parseUnicodeSet(pattern, buf, rangeCount * 2);
     if (rc >= 0) {
@@ -459,6 +472,8 @@ export class KmnCompiler implements UnicodeSetParser {
       /* c8 ignore next 2 */
       return null;
     }
+    // fix \u1234 pattern format
+    pattern = KmnCompiler.fixNewPattern(pattern);
     // call with rangeCount = 0 to invoke in 'preflight' mode.
     const rc = Module.kmcmp_parseUnicodeSet(pattern, 0, 0);
     if (rc >= 0) {

developer/src/kmc-kmn/test/test-wasm-uset.ts

@@ -6,6 +6,22 @@ import { CompilerMessages } from '../src/compiler/messages.js';
 import { compilerErrorFormatCode } from '@keymanapp/common-types';
 
 describe('Compiler UnicodeSet function', function() {
+  it('should fixup "short" \\u{} escapes', function () {
+    assert.equal(KmnCompiler.fixNewPattern(`\\u{A}`), `\\u000A`); // "
+    assert.equal(KmnCompiler.fixNewPattern(`\\u{22}`), `\\u0022`); // "
+    assert.equal(KmnCompiler.fixNewPattern(`\\u{ead}`), `\\u0ead`); // "
+  });
+  it('should fixup \\u1234 format escapes', function() {
+    assert.equal(KmnCompiler.fixNewPattern(`\\u{1234}`), `\\u1234`);
+    assert.equal(KmnCompiler.fixNewPattern(`\\u1234`), `\\u1234`);
+    assert.equal(KmnCompiler.fixNewPattern(`[\\u{1234}-\\u{5678}]`), `[\\u1234-\\u5678]`);
+    assert.equal(KmnCompiler.fixNewPattern(`something else`), `something else`);
+  });
+  it('should fixup supplemental \\u format escapes', function() {
+    assert.equal(KmnCompiler.fixNewPattern(`\\u{1F640}`), `\\U0001F640`);
+    assert.equal(KmnCompiler.fixNewPattern(`\\u{10FFFD}`),`\\U0010FFFD`);
+  });
+
   it('should start', async function() {
     const compiler = new KmnCompiler();
     const callbacks = new TestCompilerCallbacks();
@@ -51,6 +67,33 @@ describe('Compiler UnicodeSet function', function() {
     assert.deepEqual(callbacks.messages, []);
     assert.equal(len2, set.length);
   });
+  it('should compile an even more complex uset', async function() {
+    const compiler = new KmnCompiler();
+    const callbacks = new TestCompilerCallbacks();
+    assert(await compiler.init(callbacks));
+    assert(compiler.verifyInitialized());
+
+    const pat = "[\\u{10FFFD}\\u{2019}\\u{22}\\u{a}\\u{ead}\\u{1F640}]";
+    const set = compiler.parseUnicodeSet(pat, 23);
+
+    assert.equal(set.length, 6);
+    // verify we're all single chars
+    for (let i = 0; i<set.length; i++) {
+      assert.equal(set.ranges[i][0], set.ranges[i][1], `Range ${i} should be a single char`);
+    }
+    // check the single char value
+    assert.equal(set.ranges[0][0], 0x000A);
+    assert.equal(set.ranges[1][0], 0x0022);
+    assert.equal(set.ranges[2][0], 0x0EAD);
+    assert.equal(set.ranges[3][0], 0x2019);
+    assert.equal(set.ranges[4][0], 0x1F640);
+    assert.equal(set.ranges[5][0], 0x10FFFD);
+    assert.deepEqual(callbacks.messages, []);
+    callbacks.clear();
+    const len2 = compiler.sizeUnicodeSet(pat);
+    assert.deepEqual(callbacks.messages, []);
+    assert.equal(len2, set.length);
+  });
   it('should fail in various ways', async function() {
     const compiler = new KmnCompiler();
     const callbacks = new TestCompilerCallbacks();

developer/src/kmc-ldml/src/compiler/messages.ts

@@ -90,9 +90,6 @@ export class CompilerMessages {
 
   static Error_InvalidHardware = (o:{form: string}) => m(this.ERROR_InvalidHardware,
     `layers has invalid value form=${o.form}`);
-  /**
-   * Note: may not hit this due to XML validation.
-   */
   static ERROR_InvalidHardware = SevError | 0x0013;
 
   static Error_InvalidModifier = (o:{layer: string, modifier: string}) => m(this.ERROR_InvalidModifier,

developer/src/kmc-ldml/test/test-layr.ts

@@ -3,7 +3,7 @@ import { assert } from 'chai';
 import { LayrCompiler } from '../src/compiler/layr.js';
 import { CompilerMessages } from '../src/compiler/messages.js';
 import { compilerTestCallbacks, loadSectionFixture, testCompilationCases } from './helpers/index.js';
-import { KMXPlus, CommonTypesMessages } from '@keymanapp/common-types';
+import { KMXPlus } from '@keymanapp/common-types';
 import { constants } from '@keymanapp/ldml-keyboard-constants';
 
 import Layr = KMXPlus.Layr;
@@ -106,11 +106,9 @@ describe('layr', function () {
     },
     {
       subpath: 'sections/layr/invalid-invalid-form.xml',
-      errors: [CommonTypesMessages.Error_SchemaValidationError({
-        instancePath: '/keyboard/layers/0/form',
-        keyword: 'enum',
-        message: 'must be equal to one of the allowed values',
-        params: `allowedValues="touch,us,iso,jis,abnt2"`}),],
+      errors: [CompilerMessages.Error_InvalidHardware({
+        form: 'holographic',
+      }),],
     },
     {
       // missing layer element

resources/standards-data/ldml-keyboards/techpreview/3.0/fr-t-k0-azerty.xml

@@ -43,7 +43,7 @@
 		<!-- Note: displays is only used for keycap presentation -->
 		<!-- this example is not required for this keyboard as we use the spacing
 			modifiers -->
-		<display to="\u0300" display="${grave}" /> <!-- display combining grave as modifier letter grave ˋ -->
+		<display to="\u{0300}" display="${grave}" /> <!-- display combining grave as modifier letter grave ˋ -->
 		<!-- Note: We discussed why the existing displayMap was used for display.
 			rather than adding something closer to the key layout. 1. This way we can
 			tell the renderer what to do. Could be double diacritics, spacing issues,
@@ -89,19 +89,19 @@
 		<!-- test key -->
 		<key id="a" flicks="a" to="a" longPress="à â á ä ã å ā" />
 		<flicks id="a">
-			<flick directions="nw" to="\u1234" />
-			<flick directions="nw se" to="\uFFFF" />
-			<flick directions="e" to="\uFFF0" />
+			<flick directions="nw" to="\u{1234}" />
+			<flick directions="nw se" to="\u{FFFF}" />
+			<flick directions="e" to="\u{FFF0}" />
 		</flicks>
 
 		<!-- test key -->
 		<key id="A" flicks="b" to="A" longPress="À Á Ä Ã Å Ā" />
 
 		<!-- test flick -->
 		<flicks id="b">
-			<flick directions="nw" to="\u4567" />
-			<flick directions="nw se" to="\uFFFF" />
-			<flick directions="e" to="\uFFF0" />
+			<flick directions="nw" to="\u{4567}" />
+			<flick directions="nw se" to="\u{FFFF}" />
+			<flick directions="e" to="\u{FFF0}" />
 		</flicks>
 
 		<!-- TODO: all additional maps, hardware and touch -->
@@ -203,9 +203,9 @@
 		<transformGroup>
 			<!-- this is a reorder group -->
 			<!-- nod-Lana partial example -->
-			<reorder from="\u1A60" order="127" />
-			<reorder from="\u1A6B" order="42" />
-			<reorder from="[\u1A75-\u1A79]" order="55" />
+			<reorder from="\u{1A60}" order="127" />
+			<reorder from="\u{1A6B}" order="42" />
+			<reorder from="[\u{1A75}-\u{1A79}]" order="55" />
 			<!-- ... partial example ... -->
 		</transformGroup>
 	</transforms>

resources/standards-data/ldml-keyboards/techpreview/cldr_info.json

@@ -1,5 +1,5 @@
 {
-  "sha": "44903d0867e42df37f17cf21d28938044eb1edc0",
-  "description": "release-44-m1-75-g44903d0867",
-  "date": "Thu, 17 Aug 2023 19:14:19 +0000"
+  "sha": "61b74a36de8329daed152005133a699ae7f2012b",
+  "description": "release-44-alpha2-5-g61b74a36de",
+  "date": "Thu, 14 Sep 2023 07:38:47 +0000"
 }

resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboard.dtd

@@ -10,7 +10,7 @@ The CLDR Keyboard Subcommittee is currently developing major changes to the CLDR
 Please view the subcommittee page for the most recent information.
 <https://cldr.unicode.org/index/keyboard-workgroup> -->
 
-<!ELEMENT keyboard ( import*, locales?, version?, info?, names, settings?, vkeys?, displays?, keys?, layers*, variables?, transforms*, special* ) >
+<!ELEMENT keyboard ( import*, locales?, version?, info?, names, settings?, vkeys?, displays?, keys?, forms?, layers*, variables?, transforms*, special* ) >
     <!--@TECHPREVIEW-->
 <!ATTLIST keyboard locale CDATA #REQUIRED >
     <!--@MATCH:validity/bcp47-wellformed-->
@@ -161,9 +161,24 @@ Please view the subcommittee page for the most recent information.
     <!--@VALUE-->
     <!--@ALLOWS_UESC-->
 
+<!ELEMENT forms ( import*, form*, special* ) >
+    <!--@TECHPREVIEW-->
+
+<!ELEMENT form ( scanCodes+, special* ) >
+    <!--@TECHPREVIEW-->
+<!ATTLIST form id NMTOKEN #IMPLIED >
+    <!--@MATCH:any-->
+
+<!ELEMENT scanCodes EMPTY >
+    <!--@TECHPREVIEW-->
+<!ATTLIST scanCodes codes NMTOKENS #REQUIRED >
+    <!--@MATCH:regex/[0-9a-fA-F]{2}( [0-9a-fA-F]{2})*-->
+    <!--@VALUE-->
+
 <!ELEMENT layers ( import*, layer*, special* ) >
     <!--@TECHPREVIEW-->
-<!ATTLIST layers form (touch | us | iso | jis | abnt2) #REQUIRED >
+<!ATTLIST layers form NMTOKEN #REQUIRED >
+    <!--@MATCH:any-->
 <!ATTLIST layers minDeviceWidth CDATA #IMPLIED >
     <!--@MATCH:range/1~999-->