Merge pull request #432 from messageformat/mf2-updates

MF2 spec updates: BiDi isolation, u:options

Author: XM5jDcsHTyGJtQqlCi

mf2/fluent/src/fluent.test.ts

@@ -331,7 +331,7 @@ for (const [title, { locale = 'en', src, tests }] of Object.entries(
 )) {
   describe(title, () => {
     const data = fluentToResourceData(src).data;
-    const res = fluentToResource(data, locale);
+    const res = fluentToResource(data, locale, { bidiIsolation: 'none' });
 
     test('validate', () => {
       for (const [id, group] of res) {
@@ -444,6 +444,7 @@ describe('formatToParts', () => {
         {
           type: 'number',
           source: '$num',
+          dir: 'ltr',
           locale: 'en',
           parts: [{ type: 'integer', value: '42' }]
         }
@@ -455,7 +456,9 @@ describe('formatToParts', () => {
       const foo = res.get('foo')?.get('')?.formatToParts(undefined, onError);
       expect(foo).toEqual([
         { type: 'literal', value: 'Foo ' },
-        { type: 'fallback', source: '$num' }
+        { type: 'bidiIsolation', value: '\u2068' },
+        { type: 'fallback', source: '$num' },
+        { type: 'bidiIsolation', value: '\u2069' }
       ]);
       expect(onError).toHaveBeenCalledTimes(1);
     });
@@ -469,7 +472,9 @@ describe('formatToParts', () => {
           source: '|foo|',
           parts: [
             { type: 'literal', value: 'Foo ' },
-            { type: 'fallback', source: '$num' }
+            { type: 'bidiIsolation', value: '\u2068' },
+            { type: 'fallback', source: '$num' },
+            { type: 'bidiIsolation', value: '\u2069' }
           ]
         }
       ]);
@@ -572,6 +577,7 @@ describe('formatToParts', () => {
         {
           type: 'number',
           source: '$num',
+          dir: 'ltr',
           locale: 'en',
           parts: [{ type: 'integer', value: '42' }]
         }

mf2/fluent/src/functions.ts

@@ -1,7 +1,9 @@
+import type { MessagePart } from 'messageformat';
 import type {
   MessageFunctionContext,
   MessageValue
 } from 'messageformat/functions';
+import { getLocaleDir } from 'messageformat/functions/utils';
 import type { FluentMessageResource } from './index.js';
 import { valueToMessageRef } from './message-to-fluent.js';
 
@@ -20,27 +22,41 @@ import { valueToMessageRef } from './message-to-fluent.js';
  */
 export function getFluentFunctions(res: FluentMessageResource) {
   function message(
-    { locales, onError, source }: MessageFunctionContext,
+    ctx: MessageFunctionContext,
     options: Record<string, unknown>,
     input?: unknown
   ) {
+    const { onError, source } = ctx;
+    const locale = ctx.locales[0];
+    const dir = ctx.dir ?? getLocaleDir(locale);
     const { msgId, msgAttr } = valueToMessageRef(input ? String(input) : '');
     const mf = res.get(msgId)?.get(msgAttr ?? '');
     if (!mf) throw new Error(`Message not available: ${msgId}`);
 
     let str: string | undefined;
     return {
       type: 'fluent-message' as const,
-      locale: locales[0],
       source,
+      dir,
+      locale,
       selectKey(keys) {
         str ??= mf.format(options, onError);
         return keys.has(str) ? str : null;
       },
-      toParts() {
+      toParts(): [
+        {
+          type: 'fluent-message';
+          source: string;
+          dir?: 'ltr' | 'rtl';
+          parts: MessagePart[];
+        }
+      ] {
         const parts = mf.formatToParts(options, onError);
-        const res = { type: 'fluent-message' as const, source, parts };
-        return [res] as [typeof res];
+        const res =
+          dir === 'ltr' || dir === 'rtl'
+            ? { type: 'fluent-message' as const, source, dir, locale, parts }
+            : { type: 'fluent-message' as const, source, locale, parts };
+        return [res];
       },
       toString: () => (str ??= mf.format(options, onError)),
       valueOf: () => (str ??= mf.format(options, onError))

mf2/icu-messageformat-1/src/functions.ts

@@ -5,6 +5,7 @@ import {
   type MessageValue,
   datetime
 } from 'messageformat/functions';
+import { getLocaleDir } from 'messageformat/functions/utils';
 
 function getParam(options: Record<string, unknown>) {
   if (options.param) {
@@ -107,10 +108,11 @@ function duration(
 }
 
 function number(
-  { localeMatcher, locales, source }: MessageFunctionContext,
+  ctx: MessageFunctionContext,
   options: Record<string, unknown>,
   input?: unknown
 ): MessageNumber {
+  const { locales, source } = ctx;
   const origNum = typeof input === 'bigint' ? input : Number(input);
   let num = origNum;
   const offset = Number(options.pluralOffset);
@@ -120,7 +122,7 @@ function number(
   }
 
   const opt: Intl.NumberFormatOptions & Intl.PluralRulesOptions = {
-    localeMatcher
+    localeMatcher: ctx.localeMatcher
   };
   switch (getParam(options)) {
     case 'integer':
@@ -138,12 +140,17 @@ function number(
   if (options.type === 'ordinal') opt.type = 'ordinal';
 
   let locale: string | undefined;
+  let dir = ctx.dir;
   let nf: Intl.NumberFormat | undefined;
   let cat: Intl.LDMLPluralRule | undefined;
   let str: string | undefined;
   return {
     type: 'number',
     source,
+    get dir() {
+      dir ??= getLocaleDir(this.locale);
+      return dir;
+    },
     get locale() {
       return (locale ??= Intl.NumberFormat.supportedLocalesOf(locales, opt)[0]);
     },
@@ -161,7 +168,10 @@ function number(
       nf ??= new Intl.NumberFormat(locales, opt);
       const parts = nf.formatToParts(num);
       locale ??= nf.resolvedOptions().locale;
-      return [{ type: 'number', source, locale, parts }];
+      dir ??= getLocaleDir(locale);
+      return dir === 'ltr' || dir === 'rtl'
+        ? [{ type: 'number', source, dir, locale, parts }]
+        : [{ type: 'number', source, locale, parts }];
     },
     toString() {
       nf ??= new Intl.NumberFormat(locales, opt);

mf2/icu-messageformat-1/src/mf1.test.ts

@@ -376,7 +376,7 @@ for (const [title, cases] of Object.entries(testCases)) {
 
           test(strParam.join(', '), () => {
             const data = mf1ToMessageData(parse(src));
-            const mf = mf1ToMessage(data, locale);
+            const mf = mf1ToMessage(data, locale, { bidiIsolation: 'none' });
             const req = validate(data, type => {
               throw new Error(`Validation failed: ${type}`);
             });

mf2/messageformat/src/cst/names.ts

@@ -54,9 +54,9 @@ export function parseNameValue(
   if (!isNameStartCode(cc)) return null;
   cc = src.charCodeAt(++pos);
   while (isNameCharCode(cc)) cc = src.charCodeAt(++pos);
-  const name = src.substring(nameStart, pos);
+  const value = src.substring(nameStart, pos).normalize();
   if (bidiChars.has(cc)) pos += 1;
-  return { value: name, end: pos };
+  return { value, end: pos };
 }
 
 export function isValidUnquotedLiteral(str: string): boolean {

mf2/messageformat/src/cst/parse-cst.ts

@@ -142,10 +142,13 @@ function parseVariant(ctx: ParseContext, start: number): CST.Variant {
 
     if (pos > start && !ws.hasWS) ctx.onError('missing-syntax', pos, "' '");
 
-    const key =
-      ch === '*'
-        ? ({ type: '*', start: pos, end: pos + 1 } satisfies CST.CatchallKey)
-        : parseLiteral(ctx, pos, true);
+    let key: CST.CatchallKey | CST.Literal;
+    if (ch === '*') {
+      key = { type: '*', start: pos, end: pos + 1 };
+    } else {
+      key = parseLiteral(ctx, pos, true);
+      key.value = key.value.normalize();
+    }
     if (key.end === pos) break; // error; reported in pattern.errors
     keys.push(key);
     pos = key.end;

mf2/messageformat/src/data-model/format-markup.ts

@@ -1,6 +1,7 @@
+import { MessageResolutionError } from '../errors.js';
 import type { Context } from '../format-context.js';
 import type { MessageMarkupPart } from '../formatted-parts.js';
-import { resolveValue } from './resolve-value.js';
+import { getValueSource, resolveValue } from './resolve-value.js';
 import type { Markup } from './types.js';
 
 export function formatMarkup(
@@ -13,12 +14,18 @@ export function formatMarkup(
   if (options?.size) {
     part.options = {};
     for (const [name, value] of options) {
-      let rv = resolveValue(ctx, value);
-      if (typeof rv === 'object' && typeof rv?.valueOf === 'function') {
-        const vv = rv.valueOf();
-        if (vv !== rv) rv = vv;
+      if (name === 'u:dir' || name === 'u:locale') {
+        const msg = `The option ${name} is not valid for markup`;
+        const optSource = getValueSource(value);
+        ctx.onError(new MessageResolutionError('bad-option', msg, optSource));
+      } else {
+        let rv = resolveValue(ctx, value);
+        if (typeof rv === 'object' && typeof rv?.valueOf === 'function') {
+          rv = rv.valueOf();
+        }
+        if (name === 'u:id') part.id = String(rv);
+        else part.options[name] = rv;
       }
-      part.options[name] = rv;
     }
   }
   return part;

mf2/messageformat/src/data-model/function-context.ts

@@ -1,18 +1,62 @@
+import { MessageResolutionError } from '../errors.js';
 import type { Context } from '../format-context.js';
+import { getValueSource, resolveValue } from './resolve-value.js';
+import { Options } from './types.js';
 
 export class MessageFunctionContext {
   #ctx: Context;
+  #locales: Intl.Locale[];
+  readonly dir: 'ltr' | 'rtl' | 'auto' | undefined;
+  readonly id: string | undefined;
   readonly source: string;
-  constructor(ctx: Context, source: string) {
+  constructor(ctx: Context, source: string, options?: Options) {
     this.#ctx = ctx;
+
+    this.#locales = ctx.locales;
+    const localeOpt = options?.get('u:locale');
+    if (localeOpt) {
+      let rl = resolveValue(ctx, localeOpt);
+      try {
+        if (typeof rl === 'object' && typeof rl?.valueOf === 'function') {
+          rl = rl.valueOf();
+        }
+        this.#locales = Array.isArray(rl)
+          ? rl.map(lc => new Intl.Locale(lc))
+          : [new Intl.Locale(String(rl))];
+      } catch {
+        const msg = 'Unsupported value for u:locale option';
+        const optSource = getValueSource(localeOpt);
+        ctx.onError(new MessageResolutionError('bad-option', msg, optSource));
+      }
+    }
+
+    this.dir = undefined;
+    const dirOpt = options?.get('u:dir');
+    if (dirOpt) {
+      const dir = String(resolveValue(ctx, dirOpt));
+      if (dir === 'ltr' || dir === 'rtl' || dir === 'auto') {
+        this.dir = dir;
+      } else {
+        const msg = 'Unsupported value for u:dir option';
+        const optSource = getValueSource(dirOpt);
+        ctx.onError(new MessageResolutionError('bad-option', msg, optSource));
+      }
+    }
+
+    const idOpt = options?.get('u:id');
+    this.id = idOpt ? String(resolveValue(ctx, idOpt)) : undefined;
+
     this.source = source;
   }
+
   get localeMatcher() {
     return this.#ctx.localeMatcher;
   }
+
   get locales() {
-    return this.#ctx.locales.slice();
+    return this.#locales.map(String);
   }
+
   get onError() {
     return this.#ctx.onError;
   }

mf2/messageformat/src/data-model/function-ref.test.ts

@@ -6,9 +6,10 @@ import {
 
 test('Custom function', () => {
   const functions = {
-    custom: ({ source, locales: [locale] }, _opt, input) => ({
+    custom: ({ dir, source, locales: [locale] }, _opt, input) => ({
       type: 'custom',
       source,
+      dir: dir ?? 'auto',
       locale,
       toParts: () => [
         { type: 'custom', source, locale, value: `part:${input}` }
@@ -17,17 +18,19 @@ test('Custom function', () => {
     })
   } satisfies MessageFunctions;
   const mf = new MessageFormat('en', '{$var :custom}', { functions });
-  expect(mf.format({ var: 42 })).toEqual('str:42');
+  expect(mf.format({ var: 42 })).toEqual('\u2068str:42\u2069');
   expect(mf.formatToParts({ var: 42 })).toEqual([
-    { type: 'custom', source: '$var', locale: 'en', value: 'part:42' }
+    { type: 'bidiIsolation', value: '\u2068' },
+    { type: 'custom', source: '$var', locale: 'en', value: 'part:42' },
+    { type: 'bidiIsolation', value: '\u2069' }
   ]);
 });
 
 describe('inputs with options', () => {
   test('local variable with :number expression', () => {
     const mf = new MessageFormat(
       'en',
-      `.local $val = {12345678 :number useGrouping=false}
+      `.local $val = {12345678 :number useGrouping=never}
       {{{$val :number minimumFractionDigits=2}}}`
     );
     //const val = new MessageNumber(null, BigInt(12345678), { options: { useGrouping: false } });
@@ -74,20 +77,28 @@ describe('inputs with options', () => {
 });
 
 describe('Type casts based on runtime', () => {
+  const date = '2000-01-01T15:00:00';
+
   test('boolean function option with literal value', () => {
-    const mfTrue = new MessageFormat('en', '{$var :number useGrouping=true}');
-    expect(mfTrue.format({ var: 1234 })).toBe('1,234');
-    const mfFalse = new MessageFormat('en', '{$var :number useGrouping=false}');
-    expect(mfFalse.format({ var: 1234 })).toBe('1234');
+    const mfTrue = new MessageFormat(
+      'en',
+      '{$date :datetime timeStyle=short hour12=true}'
+    );
+    expect(mfTrue.format({ date })).toMatch(/3:00/);
+    const mfFalse = new MessageFormat(
+      'en',
+      '{$date :datetime timeStyle=short hour12=false}'
+    );
+    expect(mfFalse.format({ date })).toMatch(/15:00/);
   });
 
   test('boolean function option with variable value', () => {
     const mf = new MessageFormat(
       'en',
-      '{$var :number useGrouping=$useGrouping}'
+      '{$date :datetime timeStyle=short hour12=$hour12}'
     );
-    expect(mf.format({ var: 1234, useGrouping: 'false' })).toBe('1234');
-    expect(mf.format({ var: 1234, useGrouping: false })).toBe('1234');
+    expect(mf.format({ date, hour12: 'false' })).toMatch(/15:00/);
+    expect(mf.format({ date, hour12: false })).toMatch(/15:00/);
   });
 });
 
@@ -98,9 +109,11 @@ describe('Function return is not a MessageValue', () => {
     } satisfies MessageFunctions;
     const mf = new MessageFormat('en', '{:fail}', { functions });
     const onError = jest.fn();
-    expect(mf.format(undefined, onError)).toEqual('{:fail}');
+    expect(mf.format(undefined, onError)).toEqual('\u2068{:fail}\u2069');
     expect(mf.formatToParts(undefined, onError)).toEqual([
-      { type: 'fallback', source: ':fail' }
+      { type: 'bidiIsolation', value: '\u2068' },
+      { type: 'fallback', source: ':fail' },
+      { type: 'bidiIsolation', value: '\u2069' }
     ]);
     expect(onError).toHaveBeenCalledTimes(2);
   });
@@ -111,9 +124,11 @@ describe('Function return is not a MessageValue', () => {
     } satisfies MessageFunctions;
     const mf = new MessageFormat('en', '{42 :fail}', { functions });
     const onError = jest.fn();
-    expect(mf.format(undefined, onError)).toEqual('{|42|}');
+    expect(mf.format(undefined, onError)).toEqual('\u2068{|42|}\u2069');
     expect(mf.formatToParts(undefined, onError)).toEqual([
-      { type: 'fallback', source: '|42|' }
+      { type: 'bidiIsolation', value: '\u2068' },
+      { type: 'fallback', source: '|42|' },
+      { type: 'bidiIsolation', value: '\u2069' }
     ]);
     expect(onError).toHaveBeenCalledTimes(2);
   });