Skip to content

Commit 49d471b

Browse files
authored
Merge pull request #8954 from keymanapp/feature-kmc-kmw
epic: kmc-kmw KeymanWeb compiler in Typescript 🗜
2 parents c53ec3c + c92f28e commit 49d471b

File tree

277 files changed

+38060
-1622
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

277 files changed

+38060
-1622
lines changed

common/include/kmx_file.h

Lines changed: 40 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,24 @@
77

88
#include <km_types.h>
99

10+
/*
11+
When we read .kmx files, they have no alignment guarantees, so we need to tell
12+
the compiler to generate unaligned-safe code for accesses to COMP_ structure
13+
members. Note we are assuming that COMP_KEYBOARD is aligned because it is
14+
always the start of the file, so will be at the start of any buffer which will
15+
automatically be aligned correctly.
16+
*/
17+
#ifdef __EMSCRIPTEN__
18+
typedef KMX_DWORD __attribute__((aligned(1))) KMX_DWORD_unaligned;
19+
typedef KMX_BOOL __attribute__((aligned(1))) KMX_BOOL_unaligned;
20+
typedef KMX_WORD __attribute__((aligned(1))) KMX_WORD_unaligned;
21+
#else
22+
// TODO: consider other platforms
23+
#define KMX_DWORD_unaligned KMX_DWORD
24+
#define KMX_BOOL_unaligned KMX_BOOL
25+
#define KMX_WORD_unaligned KMX_WORD
26+
#endif
27+
1028
#ifdef KMN_KBP
1129
// TODO: move this to a common namespace keyman::common::kmx_file or similar in the future
1230
namespace km {
@@ -153,7 +171,13 @@ namespace kmx {
153171

154172
#define TSS__KEYMAN_150_MAX 43
155173

156-
#define TSS__MAX 43
174+
/* Keyman 17.0 system stores */
175+
176+
#define TSS_DISPLAYMAP 44
177+
178+
#define TSS__KEYMAN_170_MAX 44
179+
180+
#define TSS__MAX 44
157181

158182
/* wm_keyman_control_internal message control codes */
159183

@@ -297,27 +321,27 @@ namespace kmx {
297321
#define K_NOTMODIFIERFLAG 0xFF00 // I4548
298322

299323
struct COMP_STORE {
300-
KMX_DWORD dwSystemID;
301-
KMX_DWORD dpName;
302-
KMX_DWORD dpString;
324+
KMX_DWORD_unaligned dwSystemID;
325+
KMX_DWORD_unaligned dpName;
326+
KMX_DWORD_unaligned dpString;
303327
};
304328

305329
struct COMP_KEY {
306-
KMX_WORD Key;
307-
KMX_WORD _reserved;
308-
KMX_DWORD Line;
309-
KMX_DWORD ShiftFlags;
310-
KMX_DWORD dpOutput;
311-
KMX_DWORD dpContext;
330+
KMX_WORD_unaligned Key;
331+
KMX_WORD_unaligned _reserved;
332+
KMX_DWORD_unaligned Line;
333+
KMX_DWORD_unaligned ShiftFlags;
334+
KMX_DWORD_unaligned dpOutput;
335+
KMX_DWORD_unaligned dpContext;
312336
};
313337

314338
struct COMP_GROUP {
315-
KMX_DWORD dpName;
316-
KMX_DWORD dpKeyArray; // [LPKEY] address of first item in key array
317-
KMX_DWORD dpMatch;
318-
KMX_DWORD dpNoMatch;
319-
KMX_DWORD cxKeyArray; // in array entries
320-
KMX_BOOL fUsingKeys; // group(xx) [using keys] <-- specified or not
339+
KMX_DWORD_unaligned dpName;
340+
KMX_DWORD_unaligned dpKeyArray; // [LPKEY] address of first item in key array
341+
KMX_DWORD_unaligned dpMatch;
342+
KMX_DWORD_unaligned dpNoMatch;
343+
KMX_DWORD_unaligned cxKeyArray; // in array entries
344+
KMX_BOOL_unaligned fUsingKeys; // group(xx) [using keys] <-- specified or not
321345
};
322346

323347
struct COMP_KEYBOARD {
Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
# displaymap.schema.json
2+
3+
This mapping file provides data for remapping the touch layout and visual
4+
keyboard key caps. The primary purpose of this file is to provide a pathway for
5+
consistent display of diacritics and other unattached marks which may be
6+
displayed on the keyboard by use of a special font with formatted glyphs in the
7+
Private Use Area, which will have consistent display across all platforms and
8+
not rely on platform-specific or font-specific rendering behaviors.
9+
10+
This file can be generated by `kmc analyze osk-char-use` command, or hand
11+
crafted. The compiler uses only the `str` and `pua` values in the file, although
12+
there may be additional data in the file provided for reference purposes. The
13+
file should have the following structure, in this example, mapping the Unicode
14+
values `U+17BB U+17C7` (ុះ) to the Private Use Area code `U+F19F`:
15+
16+
```json
17+
{
18+
"map": [
19+
{
20+
"pua": "F19F",
21+
"str": "ុះ",
22+
"unicode": "17BB 17C7",
23+
"usages": [
24+
"khmer_angkor.kvks",
25+
"khmer_angkor.keyman-touch-layout"
26+
]
27+
},
28+
...
29+
]
30+
}
31+
```
32+
33+
The file can be passed without modification to the [ttkbdfont.py script][2]
34+
(unsupported) to generate a Kbd font. The font may need some manual editing as
35+
insertion of dotted circle (`U+25CC`) as a base may not always be possible
36+
automatically, and combined marks may not render as a cluster in some scenarios.
37+
The open source tool [FontForge][3] is suitable for making these kinds of minor
38+
adjustments to the generated font.
39+
40+
## Standard conventions for use of displayMaps
41+
42+
In the Keyman keyboards repository, the PUA range used should start at `U+F100`.
43+
44+
`&displayMap` JSON files should be named `Kbd<script>.json` where `<script>` is
45+
a [4 letter script subtag][4], for example `KbdKhmr.json`, and the corresponding
46+
font is named `KbdKhmr.ttf` (with face name `KbdKhmr`). These fonts are intended
47+
only for use in visual keyboards and touch layouts and generally will not be
48+
suitable for any other use. An optional suffix to a font may be included, if
49+
appropriate, for example, `KbdArab-Nastaliq.ttf` (face name `KbdArab-Nastaliq`).
50+
51+
## Fields in the `map` objects
52+
53+
* `pua`: must be a single hex value, by convention in the PUA range starting at
54+
`U+F100`. Do not include the `U+` prefix.
55+
* `str`: contains the codepoints that will be mapped to the PUA code;
56+
if `U+25CC` (◌, dotted circle) is specified in the key cap, it will be
57+
stripped before matching in the `&displayMap`, so `U+25CC` should not be
58+
included in `str`.
59+
* `unicode`: must be the unicode codepoints contained in the `str` field, as
60+
hexadecimal values, without initial `U+` prefix. Used only for reference when
61+
reading the file.
62+
* `usages`: data provided by `kmc analyze osk-char use` on which files this
63+
specific string is found in. This may be an array of strings, or optionally
64+
may be an array of objects with `filename` and `count` fields.
65+
66+
See [`&displaymap` system store][1] online documentation for more information on
67+
how this format is used.
68+
69+
## 2023-06-17 1.0
70+
71+
* Initial version 1.0, describes osk usage data file format generated by kmc
72+
analyze osk-char-use. Schema is kept reasonably loose for future expansion.
73+
74+
[1]: https://help.keyman.com/developer/language/reference/displaymap
75+
[2]: https://github.com/silnrsi/palaso-python/blob/master/scripts/font/ttkbdfont.py
76+
[3]: https://fontforge.org
77+
[4]: https://help.keyman.com/developer/current-version/reference/bcp-47#toc-the-script-subtag
Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
{
2+
"$ref": "#/definitions/displayMap",
3+
4+
"definitions": {
5+
"displayMap": {
6+
"type": "object",
7+
"properties": {
8+
"map": {
9+
"type": "array",
10+
"items": {
11+
"$ref": "#/definitions/map"
12+
}
13+
}
14+
}
15+
},
16+
"map": {
17+
"type": "object",
18+
"properties": {
19+
"pua": { "type": "string" },
20+
"str": { "type": "string" },
21+
"unicode": { "type": "string" },
22+
"usages": { "anyOf": [
23+
{ "type": "array", "items": { "$ref": "#/definitions/usage" } },
24+
{ "type": "array", "items": { "type": "string" } }
25+
] }
26+
}
27+
},
28+
"usage": {
29+
"type": "object",
30+
"properties": {
31+
"filename": { "type": "string" },
32+
"count": { "type": "number" }
33+
}
34+
}
35+
}
36+
}

common/schemas/kpj-9.0/README.md

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# kpj-9.0.schema.json
2+
3+
**Note:** `KeymanDeveloperProject.Options.Version` is currently implicitly
4+
always '1.0'. It will be required for version 2.0 and later of the format.
5+
6+
This is a schema for for supporting legacy versions of .kpj, from Keyman
7+
Developer 9.0 and earlier, is now available. Note that this schema is not fully
8+
validating on the legacy fields, but just sufficient to pass a valid
9+
file.
10+
11+
You should be using kpj.schema.json for most purposes
Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
{
2+
"title": "kpj-9.0.xsd",
3+
"$schema": "http://json-schema.org/schema#",
4+
"type": "object",
5+
"properties": {
6+
"KeymanDeveloperProject": {
7+
"properties": {
8+
"Options": {
9+
"$ref": "#/definitions/Options"
10+
},
11+
"Files": {
12+
"$ref": "#/definitions/Files"
13+
},
14+
"templatepath": { "type":"string", "$comment": "Keyman Developer 9, no longer used in modern .kpj files" },
15+
"stringspath": { "type":"string", "$comment": "Keyman Developer 9, no longer used in modern .kpj files" },
16+
"state": { "type": "string", "$comment": "Keyman Developer 9, no longer used in modern .kpj files" },
17+
"Modules": { "oneOf": [{"type": "object"}, {"type": "string"}], "$comment": "Keyman Developer 9, no longer used in modern .kpj files" },
18+
"MRU": { "type": "object", "$comment": "Keyman Developer 9, no longer used in modern .kpj files" }
19+
},
20+
"required": [
21+
],
22+
"additionalProperties": false,
23+
"type": "object"
24+
}
25+
},
26+
"required": [
27+
"KeymanDeveloperProject"
28+
],
29+
"additionalProperties": false,
30+
"definitions": {
31+
"Options": {
32+
"type": "object",
33+
"properties": {
34+
"BuildPath": {
35+
"type": "string"
36+
},
37+
"SourcePath": {
38+
"type": "string"
39+
},
40+
"CompilerWarningsAsErrors": {
41+
"type": "string",
42+
"pattern": "^(True|False)$"
43+
},
44+
"WarnDeprecatedCode": {
45+
"type": "string",
46+
"pattern": "^(True|False)$"
47+
},
48+
"CheckFilenameConventions": {
49+
"type": "string",
50+
"pattern": "^(True|False)$"
51+
},
52+
"ProjectType": {
53+
"type": "string",
54+
"pattern": "^(keyboard|lexicalmodel)$"
55+
},
56+
"Version": {
57+
"type": "string",
58+
"pattern": "^(1\\.0|2\\.0)$"
59+
}
60+
},
61+
"required": [
62+
],
63+
"additionalProperties": false
64+
},
65+
"Files": {
66+
"type": "object",
67+
"properties": {
68+
"File": {
69+
"type": "array",
70+
"items": {
71+
"$ref": "#/definitions/File"
72+
}
73+
}
74+
},
75+
"additionalProperties": false,
76+
"required": [
77+
"File"
78+
]
79+
},
80+
"File": {
81+
"type": "object",
82+
"properties": {
83+
"ID": {
84+
"type": "string"
85+
},
86+
"Filename": {
87+
"type": "string"
88+
},
89+
"Filepath": {
90+
"type": "string"
91+
},
92+
"FileVersion": {
93+
"type": "string"
94+
},
95+
"FileType": {
96+
"type": "string"
97+
},
98+
"Details": {
99+
"$ref": "#/definitions/FileDetails"
100+
},
101+
"ParentFileID": {
102+
"type": "string"
103+
},
104+
"FullPath": { "type":"string", "$comment": "Keyman Developer 9, no longer used in modern .kpj files" },
105+
"IDEState": { "type":"object", "$comment": "Keyman Developer 9, no longer used in modern .kpj files" }
106+
},
107+
"required": [
108+
"Filename"
109+
],
110+
"additionalProperties": false
111+
},
112+
"FileDetails": {
113+
"type": "object",
114+
"properties": {
115+
"Name": {
116+
"type": "string"
117+
},
118+
"Copyright": {
119+
"type": "string"
120+
},
121+
"Message": {
122+
"type": "string"
123+
},
124+
"Version": {
125+
"type": "string"
126+
},
127+
"Debug": { "type":"string", "$comment": "Keyman Developer 9, no longer used in modern .kpj files" }
128+
},
129+
"required": [
130+
],
131+
"additionalProperties": false
132+
}
133+
}
134+
}

common/schemas/kpj/README.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@
33
**Note:** `KeymanDeveloperProject.Options.Version` is currently implicitly
44
always '1.0'. It will be required for version 2.0 and later of the format.
55

6+
**Note:** An additional schema file, kpj-9.0.schema.json, for supporting legacy
7+
versions of .kpj, from Keyman Developer 9.0 and earlier, is now available.
8+
69
## 2023-02-27 2.0
710
* Version 2.0 makes 'Files' optional (internally, Files/File will be ignored,
811
deleted on load and populated from folder structure). Adds Options/SourcePath,

common/web/input-processor/tests/cases/languageProcessor.js

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import { Mock } from '@keymanapp/keyboard-processor';
1111
import { LexicalModelCompiler } from '@keymanapp/kmc-model';
1212
import { fileURLToPath } from 'url';
1313
import path from 'path';
14+
import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers';
1415

1516
// Required initialization setup.
1617
global.keyman = {}; // So that keyboard-based checks against the global `keyman` succeed.
@@ -22,9 +23,11 @@ String.kmwEnableSupplementaryPlane(false);
2223
// Test the KeyboardProcessor interface.
2324
describe('LanguageProcessor', function() {
2425
let worker;
26+
const callbacks = new TestCompilerCallbacks();
2527

2628
beforeEach(function() {
2729
worker = LMWorker.constructInstance();
30+
callbacks.clear();
2831
});
2932

3033
afterEach(function() {
@@ -54,7 +57,7 @@ describe('LanguageProcessor', function() {
5457
});
5558

5659
describe('.predict', function() {
57-
let compiler = new LexicalModelCompiler();
60+
let compiler = new LexicalModelCompiler(callbacks);
5861
const MODEL_ID = 'example.qaa.trivial';
5962

6063
// ES-module mode leaves out `__dirname`, so we rebuild it using other components.

common/web/keyman-version/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
"license": "MIT",
2121
"type": "module",
2222
"devDependencies": {
23-
"@types/node": "^18.7.13",
23+
"@types/node": "^20.4.1",
2424
"typescript": "^4.9.5"
2525
}
2626
}

common/web/types/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
src/schemas/

0 commit comments

Comments
 (0)