i18n: use translation table for source strings; shrink .exe

The QLJS_TRANSLATABLE macro creates a translatable_message object
containing a const char* and a uint16_t. The uint16_t is an index into a
translation table, and the const char* is a pointer to the untranslated
C string. This has some problems:

* Each translatable_message consumes 10 bytes. These bytes add up
  quickly in the diagnostics table.
* Each translatable_message requires a relocation entry (assuming
  position-independent code generation), so it actually consumes more
  than 10 bytes.
* In the future, I want to compress the translation string table [1].
  This compression would not affect untranslated strings.

Change the QLJS_TRANSLATABLE macro to not store a const char*. Instead,
put the untranslated string into the translation table.

This commit reduces the size of the CLI by a dozen or two kilobytes:

 file                                         |  before |   after | change
----------------------------------------------+---------+---------+--------
 2.4.0-1_amd64.deb::quick-lint-js             |  787568 |  771184 |    -2%
 manual/windows-arm.zip::quick-lint-js.exe    |  863744 |  861696 |     0%
 manual/windows-arm64.zip::quick-lint-js.exe  | 1017856 | 1012224 |    -1%
 manual/windows-x86.zip::quick-lint-js.exe    | 1165838 | 1163790 |     0%
 manual/windows.zip::quick-lint-js.exe        | 1185280 | 1177088 |    -1%
 npm/2.4.0.tgz::darwin-arm64/quick-lint-js    |  655376 |  654528 |     0%
 npm/2.4.0.tgz::darwin-x64/quick-lint-js      | 1244384 | 1221152 |    -2%
 npm/2.4.0.tgz::linux-arm/quick-lint-js       |  496944 |  492848 |    -1%
 npm/2.4.0.tgz::linux-arm64/quick-lint-js     |  771000 |  750520 |    -3%
 npm/2.4.0.tgz::linux-x64/quick-lint-js       |  831528 |  823336 |    -1%
 npm/2.4.0.tgz::win32-arm64/quick-lint-js.exe | 1017856 | 1012224 |    -1%
 npm/2.4.0.tgz::win32-ia32/quick-lint-js.exe  | 1165838 | 1163790 |     0%
 npm/2.4.0.tgz::win32-x64/quick-lint-js.exe   | 1185280 | 1177088 |    -1%
 vscode/2.4.0.vsix::darwin-arm64.node         |  507144 |  539320 |     6%
 vscode/2.4.0.vsix::darwin-x64.node           |  769008 |  805024 |     5%
 vscode/2.4.0.vsix::linux-arm.node            |  436068 |  431972 |    -1%
 vscode/2.4.0.vsix::linux-arm64.node          |  685840 |  669456 |    -2%
 vscode/2.4.0.vsix::linux-x64.node            |  747760 |  735472 |    -2%
 vscode/2.4.0.vsix::win32-arm.node            |  514560 |  551936 |     7%
 vscode/2.4.0.vsix::win32-arm64.node          |  642560 |  677376 |     5%
 vscode/2.4.0.vsix::win32-ia32.node           |  921102 |  919054 |     0%
 vscode/2.4.0.vsix::win32-x64.node            |  928768 |  923136 |    -1%
 website/website.zip::vscode.wasm             |  290283 |  328057 |    13%

The size of quick-lint-js.wasm and the Visual Studio Code .node files
increased because the translation table is now included in the binary.
Previously the translation table was dead-stripped because localization
is disabled for WASM and for VS Code. We want the localization table in
the .wasm file [2] and VS Code extension [3] anyway, so the size
increase is tolerable.

[1] #687
[2] #433
[3] #529

Author: strager

benchmark/benchmark-translation.cpp

@@ -3,6 +3,7 @@
 
 #include <benchmark/benchmark.h>
 #include <quick-lint-js/assert.h>
+#include <quick-lint-js/string-view.h>
 #include <quick-lint-js/translation.h>
 
 namespace quick_lint_js {
@@ -12,56 +13,73 @@ void benchmark_translate_from_source_code(benchmark::State &state) {
   messages.use_messages_from_source_code();
   for (auto _ : state) {
     ::benchmark::DoNotOptimize(messages.translate(
-        "variable assigned before its declaration"_translatable));
+        QLJS_TRANSLATABLE("variable assigned before its declaration")));
     ::benchmark::DoNotOptimize(
-        messages.translate("variable declared here"_translatable));
-    ::benchmark::DoNotOptimize(messages.translate(
-        "~~~ invalid string, do not use outside benchmark ~~~"_translatable));
+        messages.translate(QLJS_TRANSLATABLE("variable declared here")));
+    ::benchmark::DoNotOptimize(messages.translate(QLJS_TRANSLATABLE(
+        "~~~ invalid string, do not use outside benchmark ~~~")));
   }
 }
 BENCHMARK(benchmark_translate_from_source_code);
 
+struct translatable_message_with_original {
+  translatable_message translatable;
+  const char *original;
+};
+
 void benchmark_translate_from_translation_hit(benchmark::State &state) {
-  static constexpr translatable_message messages_to_translate[] = {
-      "variable assigned before its declaration"_translatable,
-      "variable declared here"_translatable,
-  };
+  static constexpr translatable_message_with_original messages_to_translate[] =
+      {
+          {QLJS_TRANSLATABLE("variable assigned before its declaration"),
+           "variable assigned before its declaration"},
+          {QLJS_TRANSLATABLE("variable declared here"),
+           "variable declared here"},
+      };
 
   translatable_messages messages;
   bool have_translation = messages.use_messages_from_locale("en@loud");
   QLJS_ALWAYS_ASSERT(have_translation);
-  for (const translatable_message &message : messages_to_translate) {
+  for (const translatable_message_with_original &message :
+       messages_to_translate) {
     // Messages should be translated.
-    QLJS_ALWAYS_ASSERT(
-        std::strcmp(messages.translate(message), message.c_str()) != 0);
+    QLJS_ALWAYS_ASSERT(std::strcmp(messages.translate(message.translatable),
+                                   message.original) != 0);
   }
 
   for (auto _ : state) {
-    for (const translatable_message &message : messages_to_translate) {
-      ::benchmark::DoNotOptimize(messages.translate(message));
+    for (const translatable_message_with_original &message :
+         messages_to_translate) {
+      ::benchmark::DoNotOptimize(messages.translate(message.translatable));
     }
   }
 }
 BENCHMARK(benchmark_translate_from_translation_hit);
 
 void benchmark_translate_from_translation_miss(benchmark::State &state) {
-  static constexpr translatable_message messages_to_translate[] = {
-      "~~~ invalid string, do not use outside benchmark ~~~"_translatable,
-      "another invalid string, do not use outside benchmark"_translatable,
-  };
+  static constexpr translatable_message_with_original messages_to_translate[] =
+      {
+          {QLJS_TRANSLATABLE(
+               "~~~ invalid string, do not use outside benchmark ~~~"),
+           "~~~ invalid string, do not use outside benchmark ~~~"},
+          {QLJS_TRANSLATABLE(
+               "another invalid string, do not use outside benchmark"),
+           "another invalid string, do not use outside benchmark"},
+      };
 
   translatable_messages messages;
   bool have_translation = messages.use_messages_from_locale("en@loud");
   QLJS_ALWAYS_ASSERT(have_translation);
-  for (const translatable_message &message : messages_to_translate) {
+  for (const translatable_message_with_original &message :
+       messages_to_translate) {
     // Messages should not be translated.
-    QLJS_ALWAYS_ASSERT(
-        std::strcmp(messages.translate(message), message.c_str()) == 0);
+    QLJS_ALWAYS_ASSERT(std::strcmp(messages.translate(message.translatable),
+                                   message.original) == 0);
   }
 
   for (auto _ : state) {
-    for (const translatable_message &message : messages_to_translate) {
-      ::benchmark::DoNotOptimize(messages.translate(message));
+    for (const translatable_message_with_original &message :
+         messages_to_translate) {
+      ::benchmark::DoNotOptimize(messages.translate(message.translatable));
     }
   }
 }
@@ -87,7 +105,9 @@ BENCHMARK_CAPTURE(benchmark_load_translations, posix, "POSIX");
 
 void benchmark_load_translations_and_find_hit(benchmark::State &state) {
   static constexpr translatable_message message_to_translate =
-      "variable assigned before its declaration"_translatable;
+      QLJS_TRANSLATABLE("variable assigned before its declaration");
+  static constexpr const char *untranslated_message =
+      "variable assigned before its declaration";
 
   const char *locale = "en@loud";
   {
@@ -96,7 +116,7 @@ void benchmark_load_translations_and_find_hit(benchmark::State &state) {
     bool have_translation = messages.use_messages_from_locale(locale);
     QLJS_ALWAYS_ASSERT(have_translation);
     QLJS_ALWAYS_ASSERT(std::strcmp(messages.translate(message_to_translate),
-                                   message_to_translate.c_str()) != 0);
+                                   untranslated_message) != 0);
   }
 
   for (auto _ : state) {
@@ -110,7 +130,9 @@ BENCHMARK(benchmark_load_translations_and_find_hit);
 
 void benchmark_load_translations_and_find_miss(benchmark::State &state) {
   static constexpr translatable_message message_to_translate =
-      "~~~ invalid string, do not use outside benchmark ~~~"_translatable;
+      QLJS_TRANSLATABLE("~~~ invalid string, do not use outside benchmark ~~~");
+  static constexpr const char *untranslated_message =
+      "~~~ invalid string, do not use outside benchmark ~~~";
 
   const char *locale = "en@loud";
   {
@@ -119,7 +141,7 @@ void benchmark_load_translations_and_find_miss(benchmark::State &state) {
     bool have_translation = messages.use_messages_from_locale(locale);
     QLJS_ALWAYS_ASSERT(have_translation);
     QLJS_ALWAYS_ASSERT(std::strcmp(messages.translate(message_to_translate),
-                                   message_to_translate.c_str()) == 0);
+                                   untranslated_message) == 0);
   }
 
   for (auto _ : state) {

po/de.po

@@ -20,7 +20,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: [email protected]\n"
-"POT-Creation-Date: 2022-04-15 21:50-0700\n"
+"POT-Creation-Date: 2022-04-16 00:28-0700\n"
 "PO-Revision-Date: 2021-11-18 03:48+0100\n"
 "Last-Translator: Nico Sonack <[email protected]>\n"
 "Language-Team: German\n"
@@ -30,6 +30,22 @@ msgstr ""
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=2; plural=(n != 1);\n"
 
+#: benchmark/benchmark-translation.cpp src/quick-lint-js/error.h
+msgid "variable assigned before its declaration"
+msgstr "Zuweisung an Variable vor Deklaration"
+
+#: benchmark/benchmark-translation.cpp src/quick-lint-js/error.h
+msgid "variable declared here"
+msgstr "Variablendeklaration ist hier"
+
+#: benchmark/benchmark-translation.cpp
+msgid "~~~ invalid string, do not use outside benchmark ~~~"
+msgstr ""
+
+#: benchmark/benchmark-translation.cpp
+msgid "another invalid string, do not use outside benchmark"
+msgstr ""
+
 #: src/diagnostic-formatter.cpp
 msgid "'do-while' loop"
 msgstr ""
@@ -82,14 +98,6 @@ msgstr ""
 msgid "children end here"
 msgstr ""
 
-#: src/quick-lint-js/error.h
-msgid "variable assigned before its declaration"
-msgstr "Zuweisung an Variable vor Deklaration"
-
-#: src/quick-lint-js/error.h
-msgid "variable declared here"
-msgstr "Variablendeklaration ist hier"
-
 #: src/quick-lint-js/error.h
 msgid "'=' changes variables; to compare, use '===' instead"
 msgstr ""
@@ -1068,5 +1076,46 @@ msgstr ""
 msgid "continue can only be used inside of a loop"
 msgstr "continue ist nur innerhalb von Schleifen gültig"
 
+#: test/test-diagnostic-formatter.cpp
+#: test/test-vim-qflist-json-error-reporter.cpp
+msgid "something happened"
+msgstr ""
+
+#: test/test-diagnostic-formatter.cpp
+msgid "see here"
+msgstr ""
+
+#: test/test-diagnostic-formatter.cpp
+msgid "this {0} looks fishy"
+msgstr ""
+
+#: test/test-diagnostic-formatter.cpp
+msgid "this {1} looks fishy"
+msgstr ""
+
+#: test/test-diagnostic-formatter.cpp
+msgid "free {1} and {0} {1} {2}"
+msgstr ""
+
+#: test/test-diagnostic-formatter.cpp
+msgid "what is this '{1}' nonsense?"
+msgstr ""
+
+#: test/test-diagnostic-formatter.cpp
+msgid "a {{0} b }} c"
+msgstr ""
+
+#: test/test-diagnostic-formatter.cpp
+msgid "expected {1:headlinese}"
+msgstr ""
+
+#: test/test-diagnostic-formatter.cpp
+msgid "expected {1:singular}"
+msgstr ""
+
+#: test/test-vim-qflist-json-error-reporter.cpp
+msgid "here"
+msgstr ""
+
 #~ msgid "invalid binding in let statement"
 #~ msgstr "Ungültiges Binding in let-Statement"

po/[email protected]

@@ -20,7 +20,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: [email protected]\n"
-"POT-Creation-Date: 2022-04-15 21:50-0700\n"
+"POT-Creation-Date: 2022-04-16 00:28-0700\n"
 "PO-Revision-Date: 2021-11-17 17:05+0100\n"
 "Last-Translator: Matthew \"strager\" Glazar <[email protected]>\n"
 "Language-Team: English\n"
@@ -30,6 +30,22 @@ msgstr ""
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=2; plural=(n != 1);\n"
 
+#: benchmark/benchmark-translation.cpp src/quick-lint-js/error.h
+msgid "variable assigned before its declaration"
+msgstr "VARIABLE USED BEFORE DECLARATION: {0}"
+
+#: benchmark/benchmark-translation.cpp src/quick-lint-js/error.h
+msgid "variable declared here"
+msgstr "VARIABLE DECLARED HERE"
+
+#: benchmark/benchmark-translation.cpp
+msgid "~~~ invalid string, do not use outside benchmark ~~~"
+msgstr ""
+
+#: benchmark/benchmark-translation.cpp
+msgid "another invalid string, do not use outside benchmark"
+msgstr ""
+
 #: src/diagnostic-formatter.cpp
 msgid "'do-while' loop"
 msgstr ""
@@ -82,14 +98,6 @@ msgstr ""
 msgid "children end here"
 msgstr ""
 
-#: src/quick-lint-js/error.h
-msgid "variable assigned before its declaration"
-msgstr "VARIABLE USED BEFORE DECLARATION: {0}"
-
-#: src/quick-lint-js/error.h
-msgid "variable declared here"
-msgstr "VARIABLE DECLARED HERE"
-
 #: src/quick-lint-js/error.h
 msgid "'=' changes variables; to compare, use '===' instead"
 msgstr ""
@@ -1047,5 +1055,46 @@ msgstr "BREAK CAN ONLY BE USED INSIDE OF A LOOP OR SWITCH"
 msgid "continue can only be used inside of a loop"
 msgstr "CONTINUE CAN ONLY BE USED INSIDE OF A LOOP"
 
+#: test/test-diagnostic-formatter.cpp
+#: test/test-vim-qflist-json-error-reporter.cpp
+msgid "something happened"
+msgstr ""
+
+#: test/test-diagnostic-formatter.cpp
+msgid "see here"
+msgstr ""
+
+#: test/test-diagnostic-formatter.cpp
+msgid "this {0} looks fishy"
+msgstr ""
+
+#: test/test-diagnostic-formatter.cpp
+msgid "this {1} looks fishy"
+msgstr ""
+
+#: test/test-diagnostic-formatter.cpp
+msgid "free {1} and {0} {1} {2}"
+msgstr ""
+
+#: test/test-diagnostic-formatter.cpp
+msgid "what is this '{1}' nonsense?"
+msgstr ""
+
+#: test/test-diagnostic-formatter.cpp
+msgid "a {{0} b }} c"
+msgstr ""
+
+#: test/test-diagnostic-formatter.cpp
+msgid "expected {1:headlinese}"
+msgstr ""
+
+#: test/test-diagnostic-formatter.cpp
+msgid "expected {1:singular}"
+msgstr ""
+
+#: test/test-vim-qflist-json-error-reporter.cpp
+msgid "here"
+msgstr ""
+
 #~ msgid "invalid binding in let statement"
 #~ msgstr "INVALID BINDING IN LET STATEMENT"