Skip to content

Commit c293b00

Browse files
XM5jDcsHTyGJtQqlCiXM5jDcsHTyGJtQqlCi
committed
feat(mf2)!: Match on variables instead of expressions (unicode-org/message-format-wg#877)
Also adds declaration support to XLIFF mapping
1 parent 79ac455 commit c293b00

23 files changed

+539
-325
lines changed

‎packages/mf2-fluent/src/fluent-to-message.ts‎

Lines changed: 38 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,9 @@ import deepEqual from 'fast-deep-equal';
33
import {
44
Expression,
55
FunctionAnnotation,
6+
InputDeclaration,
67
Literal,
8+
LocalDeclaration,
79
PatternMessage,
810
SelectMessage,
911
VariableRef,
@@ -49,16 +51,21 @@ function findSelectArgs(pattern: Fluent.Pattern): SelectArg[] {
4951
return args;
5052
}
5153

52-
function asSelectExpression(
54+
function asSelectorDeclaration(
5355
{ selector, defaultName, keys }: SelectArg,
56+
index: number,
5457
detectNumberSelection: boolean = true
55-
): Expression {
58+
): InputDeclaration | LocalDeclaration {
5659
switch (selector.type) {
5760
case 'StringLiteral':
5861
return {
59-
type: 'expression',
60-
arg: asValue(selector),
61-
annotation: { type: 'function', name: 'string' }
62+
type: 'local',
63+
name: `_${index}`,
64+
value: {
65+
type: 'expression',
66+
arg: asValue(selector),
67+
annotation: { type: 'function', name: 'string' }
68+
}
6269
};
6370
case 'VariableReference': {
6471
let name = detectNumberSelection ? 'number' : 'string';
@@ -74,15 +81,28 @@ function asSelectExpression(
7481
}
7582
}
7683
return {
77-
type: 'expression',
78-
arg: asValue(selector),
79-
annotation: { type: 'function', name }
84+
type: 'input',
85+
name: selector.id.name,
86+
value: {
87+
type: 'expression',
88+
arg: asValue(selector),
89+
annotation: { type: 'function', name }
90+
}
8091
};
8192
}
8293
}
83-
return asExpression(selector);
94+
const exp = asExpression(selector);
95+
return exp.arg?.type === 'variable'
96+
? {
97+
type: 'input',
98+
name: exp.arg.name,
99+
value: exp as Expression<VariableRef>
100+
}
101+
: { type: 'local', name: `_${index}`, value: exp };
84102
}
85103

104+
function asValue(exp: Fluent.VariableReference): VariableRef;
105+
function asValue(exp: Fluent.InlineExpression): Literal | VariableRef;
86106
function asValue(exp: Fluent.InlineExpression): Literal | VariableRef {
87107
switch (exp.type) {
88108
case 'NumberLiteral':
@@ -248,7 +268,7 @@ export function fluentToMessage(
248268
keys: key.map((k, i) =>
249269
k === CATCHALL
250270
? { type: '*', value: args[i].defaultName }
251-
: { type: 'literal', quoted: false, value: String(k) }
271+
: { type: 'literal', value: String(k) }
252272
),
253273
value: []
254274
}));
@@ -299,10 +319,16 @@ export function fluentToMessage(
299319
}
300320
addParts(ast, []);
301321

322+
const declarations = args.map((arg, index) =>
323+
asSelectorDeclaration(arg, index, detectNumberSelection)
324+
);
302325
return {
303326
type: 'select',
304-
declarations: [],
305-
selectors: args.map(arg => asSelectExpression(arg, detectNumberSelection)),
327+
declarations,
328+
selectors: declarations.map(decl => ({
329+
type: 'variable',
330+
name: decl.name
331+
})),
306332
variants
307333
};
308334
}

‎packages/mf2-fluent/src/fluent.test.ts‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -715,24 +715,24 @@ describe('fluentToResourceData', () => {
715715
const msg = data.get('multi')?.get('') as SelectMessage;
716716
expect(msg.variants.map(v => v.keys)).toMatchObject([
717717
[
718-
{ type: 'literal', quoted: false, value: '0' },
719-
{ type: 'literal', quoted: false, value: 'feminine' }
718+
{ type: 'literal', value: '0' },
719+
{ type: 'literal', value: 'feminine' }
720720
],
721721
[
722-
{ type: 'literal', quoted: false, value: '0' },
723-
{ type: 'literal', quoted: false, value: 'masculine' }
722+
{ type: 'literal', value: '0' },
723+
{ type: 'literal', value: 'masculine' }
724724
],
725725
[
726-
{ type: 'literal', quoted: false, value: '0' },
726+
{ type: 'literal', value: '0' },
727727
{ type: '*', value: 'neuter' }
728728
],
729729
[
730730
{ type: '*', value: 'other' },
731-
{ type: 'literal', quoted: false, value: 'feminine' }
731+
{ type: 'literal', value: 'feminine' }
732732
],
733733
[
734734
{ type: '*', value: 'other' },
735-
{ type: 'literal', quoted: false, value: 'masculine' }
735+
{ type: 'literal', value: 'masculine' }
736736
],
737737
[
738738
{ type: '*', value: 'other' },
@@ -783,9 +783,7 @@ describe('messagetoFluent', () => {
783783
}
784784
}
785785
],
786-
selectors: [
787-
{ type: 'expression', arg: { type: 'variable', name: 'local' } }
788-
],
786+
selectors: [{ type: 'variable', name: 'local' }],
789787
variants: [
790788
{
791789
keys: [{ type: '*' }],

‎packages/mf2-fluent/src/message-to-fluent.ts‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ export function messageToFluent(
7777
}));
7878
const k0 = variants[0].keys;
7979
while (k0.length > 0) {
80-
const sel = expressionToFluent(ctx, msg.selectors[k0.length - 1]);
80+
const sel = variableRefToFluent(ctx, msg.selectors[k0.length - 1]);
8181
let baseKeys: (Literal | CatchallKey)[] = [];
8282
let exp: Fluent.SelectExpression | undefined;
8383
for (let i = 0; i < variants.length; ++i) {
@@ -273,7 +273,12 @@ function variableRefToFluent(
273273
{ name }: VariableRef
274274
): Fluent.InlineExpression {
275275
const local = ctx.declarations.find(decl => decl.name === name);
276-
return local?.value
277-
? expressionToFluent(ctx, local.value)
278-
: new Fluent.VariableReference(new Fluent.Identifier(name));
276+
if (local?.value) {
277+
const idx = ctx.declarations.indexOf(local);
278+
return expressionToFluent(
279+
{ ...ctx, declarations: ctx.declarations.slice(0, idx) },
280+
local.value
281+
);
282+
}
283+
return new Fluent.VariableReference(new Fluent.Identifier(name));
279284
}

‎packages/mf2-icu-mf1/src/mf1-to-message-data.ts‎

Lines changed: 36 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@ import type * as AST from '@messageformat/parser';
22
import type {
33
Expression,
44
FunctionAnnotation,
5+
InputDeclaration,
56
Message,
67
Options,
7-
VariableRef,
88
Variant
99
} from 'messageformat';
1010

@@ -54,8 +54,7 @@ function findSelectArgs(tokens: AST.Token[]): SelectArg[] {
5454

5555
function tokenToPart(
5656
token: AST.Token,
57-
pluralArg: string | null,
58-
pluralOffset: number | null
57+
pluralArg: string | null
5958
): string | Expression {
6059
switch (token.type) {
6160
case 'content':
@@ -84,58 +83,48 @@ function tokenToPart(
8483
annotation
8584
};
8685
}
87-
case 'octothorpe': {
88-
if (!pluralArg) return '#';
89-
const annotation: FunctionAnnotation = {
90-
type: 'function',
91-
name: 'number'
92-
};
93-
if (pluralOffset) {
94-
annotation.options = new Map([
95-
['pluralOffset', { type: 'literal', value: String(pluralOffset) }]
96-
]);
97-
}
98-
return {
99-
type: 'expression',
100-
arg: { type: 'variable', name: pluralArg },
101-
annotation
102-
};
103-
}
86+
case 'octothorpe':
87+
return pluralArg
88+
? { type: 'expression', arg: { type: 'variable', name: pluralArg } }
89+
: '#';
10490
/* istanbul ignore next - never happens */
10591
default:
10692
throw new Error(`Unsupported token type: ${token.type}`);
10793
}
10894
}
10995

110-
function argToExpression({
96+
function argToInputDeclaration({
11197
arg: selName,
11298
pluralOffset,
11399
type
114-
}: SelectArg): Expression {
115-
const arg: VariableRef = { type: 'variable', name: selName };
100+
}: SelectArg): InputDeclaration {
101+
let annotation: FunctionAnnotation;
116102
if (type === 'select') {
117-
return {
118-
type: 'expression',
119-
arg,
120-
annotation: { type: 'function', name: 'string' }
121-
};
122-
}
103+
annotation = { type: 'function', name: 'string' };
104+
} else {
105+
const options: Options = new Map();
106+
if (pluralOffset) {
107+
options.set('pluralOffset', {
108+
type: 'literal',
109+
value: String(pluralOffset)
110+
});
111+
}
112+
if (type === 'selectordinal') {
113+
options.set('type', { type: 'literal', value: 'ordinal' });
114+
}
123115

124-
const options: Options = new Map();
125-
if (pluralOffset) {
126-
options.set('pluralOffset', {
127-
type: 'literal',
128-
value: String(pluralOffset)
129-
});
130-
}
131-
if (type === 'selectordinal') {
132-
options.set('type', { type: 'literal', value: 'ordinal' });
116+
annotation = { type: 'function', name: 'number' };
117+
if (options.size) annotation.options = options;
133118
}
134-
135-
const annotation: FunctionAnnotation = { type: 'function', name: 'number' };
136-
if (options.size) annotation.options = options;
137-
138-
return { type: 'expression', arg, annotation };
119+
return {
120+
type: 'input',
121+
name: selName,
122+
value: {
123+
type: 'expression',
124+
arg: { type: 'variable', name: selName },
125+
annotation
126+
}
127+
};
139128
}
140129

141130
/**
@@ -157,7 +146,7 @@ export function mf1ToMessageData(ast: AST.Token[]): Message {
157146
return {
158147
type: 'message',
159148
declarations: [],
160-
pattern: ast.map(token => tokenToPart(token, null, null))
149+
pattern: ast.map(token => tokenToPart(token, null))
161150
};
162151
}
163152

@@ -221,7 +210,7 @@ export function mf1ToMessageData(ast: AST.Token[]): Message {
221210
})
222211
) {
223212
const i = vp.length - 1;
224-
const part = tokenToPart(token, pluralArg, pluralOffset);
213+
const part = tokenToPart(token, pluralArg);
225214
if (typeof vp[i] === 'string' && typeof part === 'string') {
226215
vp[i] += part;
227216
} else {
@@ -236,8 +225,8 @@ export function mf1ToMessageData(ast: AST.Token[]): Message {
236225

237226
return {
238227
type: 'select',
239-
declarations: [],
240-
selectors: args.map(argToExpression),
228+
declarations: args.map(argToInputDeclaration),
229+
selectors: args.map(arg => ({ type: 'variable', name: arg.arg })),
241230
variants
242231
};
243232
}

‎packages/mf2-messageformat/src/cst/parse-cst.ts‎

Lines changed: 17 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import { parseDeclarations } from './declarations.js';
33
import { parseExpression } from './expression.js';
44
import type * as CST from './types.js';
55
import { whitespaces } from './util.js';
6-
import { parseLiteral, parseText } from './values.js';
6+
import { parseLiteral, parseText, parseVariable } from './values.js';
77

88
export class ParseContext {
99
readonly errors: MessageSyntaxError[] = [];
@@ -91,29 +91,30 @@ function parseSelectMessage(
9191
): CST.SelectMessage {
9292
let pos = start + 6; // '.match'
9393
const match: CST.Syntax<'.match'> = { start, end: pos, value: '.match' };
94-
pos += whitespaces(ctx.source, pos);
94+
let ws = whitespaces(ctx.source, pos);
95+
if (ws === 0) ctx.onError('missing-syntax', pos, "' '");
96+
pos += ws;
9597

96-
const selectors: CST.Expression[] = [];
97-
while (ctx.source[pos] === '{') {
98-
const sel = parseExpression(ctx, pos);
99-
const body = sel.markup ?? sel.annotation;
100-
if (body && body.type !== 'function') {
101-
ctx.onError('parse-error', body.start, body.end);
102-
}
98+
const selectors: CST.VariableRef[] = [];
99+
while (ctx.source[pos] === '$') {
100+
const sel = parseVariable(ctx, pos);
103101
selectors.push(sel);
104102
pos = sel.end;
105-
pos += whitespaces(ctx.source, pos);
106-
}
107-
if (selectors.length === 0) {
108-
ctx.onError('empty-token', pos, pos + 1);
103+
ws = whitespaces(ctx.source, pos);
104+
if (ws === 0) ctx.onError('missing-syntax', pos, "' '");
105+
pos += ws;
109106
}
107+
if (selectors.length === 0) ctx.onError('empty-token', pos, pos + 1);
110108

111109
const variants: CST.Variant[] = [];
112-
pos += whitespaces(ctx.source, pos);
113110
while (pos < ctx.source.length) {
114111
const variant = parseVariant(ctx, pos);
115-
variants.push(variant);
116-
pos = variant.end;
112+
if (variant.end > pos) {
113+
variants.push(variant);
114+
pos = variant.end;
115+
} else {
116+
pos += 1;
117+
}
117118
pos += whitespaces(ctx.source, pos);
118119
}
119120

‎packages/mf2-messageformat/src/cst/stringify-cst.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ export function stringifyCST(cst: CST.Message): string {
3838

3939
if (cst.type === 'select') {
4040
str += cst.match.value;
41-
for (const sel of cst.selectors) str += ' ' + stringifyExpression(sel);
41+
for (const sel of cst.selectors) str += ' ' + stringifyValue(sel);
4242
for (const { keys, value } of cst.variants) {
4343
str += '\n';
4444
for (const key of keys) str += stringifyValue(key) + ' ';

‎packages/mf2-messageformat/src/cst/types.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ export interface SelectMessage {
2424
type: 'select';
2525
declarations: Declaration[];
2626
match: Syntax<'.match'>;
27-
selectors: Expression[];
27+
selectors: VariableRef[];
2828
variants: Variant[];
2929
errors: MessageSyntaxError[];
3030
}

‎packages/mf2-messageformat/src/data-model/from-cst.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ export function messageFromCST(msg: CST.Message): Model.Message {
2424
return {
2525
type: 'select',
2626
declarations,
27-
selectors: msg.selectors.map(sel => asExpression(sel, false)),
27+
selectors: msg.selectors.map(sel => asValue(sel)),
2828
variants: msg.variants.map(variant => ({
2929
keys: variant.keys.map(key =>
3030
key.type === '*' ? { type: '*', [cst]: key } : asValue(key)

0 commit comments

Comments
 (0)