Merge pull request #9607 from keymanapp/feat/developer/kmc-package-windows-installer

feat(developer): Windows package installer compiler 🎺

Author: mcdurdin

common/web/types/src/package/kps-file.ts

@@ -160,8 +160,14 @@ export interface KpsFileStartMenuItems {
 }
 
 export interface KpsFileStrings {
-  //TODO: validate this structure
-  string: string[] | string;
+  string: KpsFileString[] | KpsFileString;
+}
+
+export interface KpsFileString {
+  $: {
+    name: string;
+    value: string;
+  }
 }
 
 export interface KpsFileLanguageExamples {

core/tests/unit/ldml/keyboards/meson.build

@@ -71,7 +71,7 @@ endforeach
 
 foreach kbd : tests_with_testdata
   configure_file(
-    command: kmc_cmd + ['build-test-data', '@INPUT@', '--out-file', '@OUTPUT@'],
+    command: kmc_cmd + ['build', 'ldml-test-data', '@INPUT@', '--out-file', '@OUTPUT@'],
     input: kbd + '-test.xml',
     output: kbd + '-test.json',
   )
@@ -90,7 +90,7 @@ foreach kbd : tests_from_cldr
     output: kbd + '.kmx',
   )
   configure_file(
-    command: kmc_cmd + ['build-test-data', '@INPUT@', '--out-file', '@OUTPUT@'],
+    command: kmc_cmd + ['build', 'ldml-test-data', '@INPUT@', '--out-file', '@OUTPUT@'],
     input: join_paths(ldml_testdata, kbd + '-test.xml'),
     output: kbd + '-test.json',
   )

developer/src/kmc-package/src/compiler/kmp-compiler.ts

@@ -22,6 +22,11 @@ export class KmpCompiler {
   }
 
   public transformKpsToKmpObject(kpsFilename: string): KmpJsonFile.KmpJsonFile {
+    const kps = this.loadKpsFile(kpsFilename);
+    return this.transformKpsFileToKmpObject(kpsFilename, kps);
+  }
+
+  public loadKpsFile(kpsFilename: string): KpsFile.KpsFile {
     // Load the KPS data from XML as JS structured data.
     const buffer = this.callbacks.loadFile(kpsFilename);
     if(!buffer) {
@@ -41,7 +46,11 @@ export class KmpCompiler {
         return a;
     })();
 
-    let kps: KpsFile.KpsFile = kpsPackage.package;
+    const kps: KpsFile.KpsFile = kpsPackage.package;
+    return kps;
+  }
+
+  public transformKpsFileToKmpObject(kpsFilename: string, kps: KpsFile.KpsFile): KmpJsonFile.KmpJsonFile {
 
     //
     // To convert to kmp.json, we need to:

developer/src/kmc-package/src/compiler/windows-package-installer-compiler.ts

@@ -0,0 +1,145 @@
+/**
+ * Create a .exe installer that bundles one or more .kmp files, together with
+ * setup.exe, keymandesktop.msi, and generates and includes a setup.inf also.
+ *
+ * This module is effectively deprecated, but is present to keep parity with the
+ * legacy kmcomp compiler. Thus, it is included as part of the package compiler,
+ * and will be removed in a future version.
+ *
+ * This tool assumes that the installer .msi is the same version as the
+ * compiler, unlike the legacy compiler which read this metadata from the .msi.
+ */
+
+import JSZip from 'jszip';
+import { CompilerCallbacks, KeymanFileTypes, KmpJsonFile, KpsFile } from "@keymanapp/common-types";
+import KEYMAN_VERSION from "@keymanapp/keyman-version";
+import { KmpCompiler } from "./kmp-compiler.js";
+import { CompilerMessages } from "./messages.js";
+
+const SETUP_INF_FILENAME = 'setup.inf';
+const PRODUCT_NAME = 'Keyman';
+
+export interface WindowsPackageInstallerSources {
+  msiFilename: string;
+  setupExeFilename: string;
+  licenseFilename: string; // MIT license
+  titleImageFilename?: string;
+
+  appName?: string;
+  startDisabled: boolean;
+  startWithConfiguration: boolean;
+};
+
+export class WindowsPackageInstallerCompiler {
+  private kmpCompiler: KmpCompiler;
+
+  constructor(private callbacks: CompilerCallbacks) {
+    this.kmpCompiler = new KmpCompiler(this.callbacks);
+  }
+
+  public async compile(kpsFilename: string, sources: WindowsPackageInstallerSources): Promise<Uint8Array> {
+    const kps = this.kmpCompiler.loadKpsFile(kpsFilename);
+
+    // Check existence of required files
+    for(const filename of [sources.licenseFilename, sources.msiFilename, sources.setupExeFilename]) {
+      if(!this.callbacks.fs.existsSync(filename)) {
+        this.callbacks.reportMessage(CompilerMessages.Error_FileDoesNotExist({filename}));
+        return null;
+      }
+    }
+
+    // Check existence of optional files
+    for(const filename of [sources.titleImageFilename]) {
+      if(filename && !this.callbacks.fs.existsSync(filename)) {
+        this.callbacks.reportMessage(CompilerMessages.Error_FileDoesNotExist({filename}));
+        return null;
+      }
+    }
+
+    // Note: we never use the MSIFileName field from the .kps any more
+    // Nor do we use the MSIOptions field.
+
+    // Build the zip
+    const zipBuffer = await this.buildZip(kps, kpsFilename, sources);
+    if(!zipBuffer) {
+      // Error messages already reported by buildZip
+      return null;
+    }
+
+    // Build the sfx
+    const sfxBuffer = this.buildSfx(zipBuffer, sources);
+    return sfxBuffer;
+  }
+
+  private async buildZip(kps: KpsFile.KpsFile, kpsFilename: string, sources: WindowsPackageInstallerSources): Promise<Uint8Array> {
+    const kmpJson: KmpJsonFile.KmpJsonFile = this.kmpCompiler.transformKpsFileToKmpObject(kpsFilename, kps);
+    if(!kmpJson.info?.name?.description) {
+      this.callbacks.reportMessage(CompilerMessages.Error_PackageNameCannotBeBlank());
+      return null;
+    }
+
+    const kmpFilename = this.callbacks.path.basename(kpsFilename, KeymanFileTypes.Source.Package) + KeymanFileTypes.Binary.Package;
+    const setupInfBuffer = this.buildSetupInf(sources, kmpJson, kmpFilename, kps);
+    const kmpBuffer = await this.kmpCompiler.buildKmpFile(kpsFilename, kmpJson);
+
+    // Note that this does not technically generate a "valid" sfx according to
+    // the zip spec, because the offsets in the .zip are relative to the start
+    // of the zip, rather than to the start of the sfx. However, as Keyman's
+    // installer chops out the zip from the sfx and loads it in a new stream, it
+    // works as expected.
+    const zip = JSZip();
+    zip.file(SETUP_INF_FILENAME, setupInfBuffer);
+    zip.file(kmpFilename, kmpBuffer);
+    zip.file(this.callbacks.path.basename(sources.msiFilename), this.callbacks.loadFile(sources.msiFilename));
+    zip.file(this.callbacks.path.basename(sources.licenseFilename), this.callbacks.loadFile(sources.licenseFilename));
+    if(sources.titleImageFilename) {
+      zip.file(this.callbacks.path.basename(sources.titleImageFilename), this.callbacks.loadFile(sources.titleImageFilename));
+    }
+
+    return zip.generateAsync({type: 'uint8array', compression:'DEFLATE'});
+  }
+
+  private buildSetupInf(sources: WindowsPackageInstallerSources, kmpJson: KmpJsonFile.KmpJsonFile, kmpFilename: string, kps: KpsFile.KpsFile) {
+    let setupInf = `[Setup]
+Version=${KEYMAN_VERSION.VERSION}
+MSIFileName=${this.callbacks.path.basename(sources.msiFilename)}
+MSIOptions=
+AppName=${sources.appName ?? PRODUCT_NAME}
+License=${this.callbacks.path.basename(sources.licenseFilename)}
+`;
+    if (sources.titleImageFilename) {
+      setupInf += `TitleImage=${this.callbacks.path.basename(sources.titleImageFilename)}\n`;
+    }
+    if (kmpJson.options.graphicFile) {
+      setupInf += `BitmapFileName=${this.callbacks.path.basename(kmpJson.options.graphicFile)}\n`;
+    }
+    if (sources.startDisabled) {
+      setupInf += `StartDisabled=True\n`;
+    }
+    if (sources.startWithConfiguration) {
+      setupInf += `StartWithConfiguration=True\n`;
+    }
+
+    setupInf += `\n[Packages]\n`;
+    setupInf += kmpFilename + '\n';
+    // TODO: multiple packages?
+    const strings = !kps.strings?.string ? [] : (Array.isArray(kps.strings.string) ? kps.strings.string : [kps.strings.string]);
+    if (strings.length) {
+      setupInf += `\n[Strings]\n`;
+      for (const str of strings) {
+        setupInf += `${str.$?.name}=${str.$?.value}\n`;
+      }
+    }
+
+    const setupInfBuffer = new TextEncoder().encode(setupInf);
+    return setupInfBuffer;
+  }
+
+  private buildSfx(zipBuffer: Uint8Array, sources: WindowsPackageInstallerSources): Uint8Array {
+    const setupRedistBuffer = this.callbacks.loadFile(sources.setupExeFilename);
+    const buffer = new Uint8Array(setupRedistBuffer.length + zipBuffer.length);
+    buffer.set(setupRedistBuffer, 0);
+    buffer.set(zipBuffer, setupRedistBuffer.length);
+    return buffer;
+  }
+}

developer/src/kmc-package/src/main.ts

@@ -1,2 +1,3 @@
 export { KmpCompiler } from "./compiler/kmp-compiler.js";
-export { PackageValidation } from "./compiler/package-validation.js";
+export { PackageValidation } from "./compiler/package-validation.js";
+export { WindowsPackageInstallerSources, WindowsPackageInstallerCompiler } from "./compiler/windows-package-installer-compiler.js";

developer/src/kmc-package/test/fixtures/windows-installer/keymandesktop.txt

@@ -0,0 +1,3 @@
+This placeholder file represents keymandesktop.msi, the Windows Installer package for Keyman for Windows.
+
+(Not including the real file to reduce file size in repo, as it is not necessary in order to complete the unit test)

developer/src/kmc-package/test/fixtures/windows-installer/license.txt

@@ -0,0 +1,3 @@
+This placeholder file represents license.txt.
+
+(Not including the real file to reduce file size in repo, as it is not necessary in order to complete the unit test)

developer/src/kmc-package/test/fixtures/windows-installer/setup.txt

@@ -0,0 +1,3 @@
+This placeholder file represents setup-redist.exe, the sfx loader.
+
+(Not including the real file to reduce file size in repo, as it is not necessary in order to complete the unit test)

developer/src/kmc-package/test/test-windows-package-installer-compiler.ts

@@ -0,0 +1,79 @@
+import 'mocha';
+import * as fs from 'fs';
+import * as path from 'path';
+import { assert } from 'chai';
+import JSZip from 'jszip';
+import KEYMAN_VERSION from "@keymanapp/keyman-version";
+import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers';
+import { makePathToFixture } from './helpers/index.js';
+import { WindowsPackageInstallerCompiler, WindowsPackageInstallerSources } from '../src/compiler/windows-package-installer-compiler.js';
+
+describe('WindowsPackageInstallerCompiler', function () {
+  it(`should build an SFX archive`, async function () {
+    this.timeout(10000); // this test can take a little while to run
+
+    const callbacks = new TestCompilerCallbacks();
+    let compiler = new WindowsPackageInstallerCompiler(callbacks);
+
+    const kpsPath = makePathToFixture('khmer_angkor', 'source', 'khmer_angkor.kps');
+    const sources: WindowsPackageInstallerSources = {
+      licenseFilename: makePathToFixture('windows-installer', 'license.txt'),
+      msiFilename: makePathToFixture('windows-installer', 'keymandesktop.txt'),
+      setupExeFilename: makePathToFixture('windows-installer', 'setup.txt'),
+      startDisabled: false,
+      startWithConfiguration: true,
+      appName: 'Testing',
+    };
+
+    const sfxBuffer = await compiler.compile(kpsPath, sources);
+
+    // This returns a buffer with a SFX loader and a zip suffix. For the sake of repository size
+    // we actually provide a stub SFX loader and a stub MSI file, which is enough to verify that
+    // the compiler is generating what it thinks is a valid file.
+
+    const zip = JSZip();
+
+    // Check that file.kmp contains just 3 files - setup.inf, keymandesktop.msi, and khmer_angkor.kmp,
+    // and that they match exactly what we expect
+    const setupExeSize = fs.statSync(sources.setupExeFilename).size;
+    const setupExe = sfxBuffer.slice(0, setupExeSize);
+    const zipBuffer = sfxBuffer.slice(setupExeSize);
+
+    // Verify setup.exe sfx loader
+    const setupExeFixture = fs.readFileSync(sources.setupExeFilename);
+    assert.deepEqual(setupExe, setupExeFixture);
+
+    // Load the zip from the buffer
+    const zipFile = await zip.loadAsync(zipBuffer, {checkCRC32: true});
+
+    // Verify setup.inf; note that BitmapFileName splash.gif comes from the .kmp
+    const setupInfFixture = `[Setup]
+Version=${KEYMAN_VERSION.VERSION}
+MSIFileName=${path.basename(sources.msiFilename)}
+MSIOptions=
+AppName=${sources.appName}
+License=${path.basename(sources.licenseFilename)}
+BitmapFileName=splash.gif
+StartWithConfiguration=True
+
+[Packages]
+khmer_angkor.kmp
+`;
+
+    const setupInf = new TextDecoder().decode(await zipFile.file('setup.inf').async('uint8array'));
+    assert.equal(setupInf.trim(), setupInfFixture.trim());
+
+    const verifyFile = async (filename: string) => {
+      const fixture = fs.readFileSync(filename);
+      const file = await zipFile.file(path.basename(filename)).async('uint8array');
+      assert.deepEqual(file, fixture, `File in zip '${filename}' did not match fixture`);
+    };
+
+    await verifyFile(sources.msiFilename);
+    await verifyFile(sources.licenseFilename);
+
+    // We only test for existence of the file in the zip for now
+    const kmp = await zipFile.file('khmer_angkor.kmp').async('uint8array');
+    assert.isNotEmpty(kmp);
+  });
+});

developer/src/kmc/README.md

@@ -38,17 +38,29 @@ To see more command line options by using the `--help` option:
 
     kmc --help
 
-kmlmc Usage
------------
+---
 
-To compile a lexical model from its `.model.ts` source, use `kmlmc`:
+To compile a lexical model from its `.model.ts` source, use `kmc`:
 
-    kmlmc my-lexical-model.model.ts --outFile my-lexical-model.js
+    kmc build my-lexical-model.model.ts --outFile my-lexical-model.model.js
 
 To see more command line options by using the `--help` option:
 
-    kmlmc --help
-    kmlmp --help
+    kmc --help
+
+---
+
+kmc can now build package installers for Windows. Example usage (Bash on
+Windows, using 'node .' instead of 'kmc' to run the local build):
+
+```
+node . build windows-package-installer \
+  $KEYMAN_ROOT/developer/src/kmc-package/test/fixtures/khmer_angkor/source/khmer_angkor.kps \
+  --msi /c/Program\ Files\ \(x86\)/Common\ Files/Keyman/Cached\ Installer\ Files/keymandesktop.msi \
+  --exe $KEYMAN_ROOT/windows/bin/desktop/setup-redist.exe \
+  --license $KEYMAN_ROOT/LICENSE.md \
+  --out-file ./khmer.exe
+```
 
 How to build from source
 ------------------------