Always use utf-8 in language.ini

api.c

@@ -53,6 +53,17 @@ static void expand_string_table (string_table_t* string_table) {
       string_table->capacity * sizeof(string_entry_t));
 }
 
+static char* from_utf8 (char* utf8) {
+  int utf16len = MultiByteToWideChar(CP_UTF8, 0, utf8, -1, NULL, 0);
+  wchar_t* utf16 = calloc(utf16len, sizeof(wchar_t));
+  MultiByteToWideChar(CP_UTF8, 0, utf8, -1, utf16, utf16len);
+  int local_len = WideCharToMultiByte(CP_UTF8, 0, utf16, -1, NULL, 0, NULL, NULL);
+  char* local = calloc(local_len, sizeof(char));
+  WideCharToMultiByte(CP_ACP, 0, utf16, -1, local, local_len, NULL, NULL);
+  free(utf16);
+  return local;
+}
+
 static char* unescape (char* input) {
   int l = strlen(input);
   int j = 0;
@@ -100,17 +111,31 @@ HANDLE aoc_ini_load_strings (char* filename) {
 
   string_table.entries = calloc(string_table.capacity, sizeof(string_entry_t));
 
+  char is_unicode = FALSE;
   char* read_ptr = strtok(content, "\n");
   while (read_ptr != NULL) {
+    if (read_ptr[0] == '[') {
+      if (strncmp(read_ptr, "[unicode]", strlen("[unicode]")) == 0) {
+        is_unicode = TRUE;
+      }
+    }
     if (read_ptr[0] != ';') {
       if (sscanf(read_ptr, "%d=%4096[^\n\r]", &id, cur_string) == 2) {
         dbg_print("found string %d: '%s'\n", id, cur_string);
         string_table.entries[string_table.size].id = id;
-        char* unescaped = unescape(cur_string);
+        char* local_string = cur_string;
+        if (is_unicode) {
+          local_string = from_utf8(cur_string);
+        }
+        char* unescaped = unescape(local_string);
+        if (is_unicode) {
+          free(local_string);
+        }
         string_table.entries[string_table.size].value = unescaped;
         string_table.entries[string_table.size].size = strlen(unescaped);
 
         string_table.size++;
+
         if (string_table.size >= string_table.capacity) {
           expand_string_table(&string_table);
         }

getstrings.c

@@ -1,6 +1,7 @@
 #define WIN32_MEAN_AND_LEAN
 #include <windows.h>
 #include <stdio.h>
+#include <wchar.h>
 
 /**
  * getstrings: tool to dump AoC language strings from dll files
@@ -11,41 +12,77 @@
  *   getstrings -rc /path/to/language.dll
  */
 
-char* escape (char* str, char quoted) {
+wchar_t* escape (wchar_t* str) {
   int i = 0;
   int j = 0;
-  char escaped[8195];
-  if (quoted) {
-    escaped[j++] = '"';
+  wchar_t escaped[8193];
+  while (str[i] != L'\0' && j < 8193) {
+    switch (str[i]) {
+      case L'\r':
+        escaped[j++] = L'\\';
+        escaped[j++] = L'r';
+        break;
+      case L'\n':
+        escaped[j++] = L'\\';
+        escaped[j++] = L'n';
+        break;
+      default:
+        escaped[j++] = str[i];
+        break;
+    }
+    i++;
   }
-  while (str[i] != '\0' && j < 8193) {
+  escaped[j++] = L'\0';
+
+  wchar_t* heap = calloc(j, sizeof(wchar_t));
+  wmemcpy(heap, escaped, j);
+  return heap;
+}
+
+const char* hexchars = "0123456789ABCDEF";
+
+char* encode_utf16 (wchar_t* str) {
+  int i = 0;
+  int j = 0;
+  char escaped[32776];
+  escaped[j++] = 'L';
+  escaped[j++] = '"';
+  while (str[i] != L'\0' && j < 32773) {
     switch (str[i]) {
-      case '\r':
+      case L'\r':
         escaped[j++] = '\\';
         escaped[j++] = 'r';
         break;
-      case '\n':
+      case L'\n':
         escaped[j++] = '\\';
         escaped[j++] = 'n';
         break;
-      case '"':
-        if (quoted) {
+      case L'"':
+        escaped[j++] = '\\';
+        escaped[j++] = '"';
+        break;
+        // fall through
+      default:
+        if (str[i] > 127) {
           escaped[j++] = '\\';
+          escaped[j++] = 'x';
+          escaped[j++] = hexchars[(str[i] & 0xF000) >> 12];
+          escaped[j++] = hexchars[(str[i] & 0x0F00) >> 8];
+          escaped[j++] = hexchars[(str[i] & 0x00F0) >> 4];
+          escaped[j++] = hexchars[str[i] & 0x000F];
+        } else {
           escaped[j++] = str[i];
-          break;
         }
-        // fall through
-      default:
-        escaped[j++] = str[i];
         break;
     }
     i++;
   }
-  if (quoted) {
-    escaped[j++] = '"';
-  }
+  escaped[j++] = '"';
   escaped[j++] = '\0';
-  return strdup(escaped);
+
+  char* heap = calloc(j, sizeof(char));
+  memcpy(heap, escaped, j);
+  return heap;
 }
 
 int main (int argc, char** argv) {
@@ -69,22 +106,26 @@ int main (int argc, char** argv) {
     return 1;
   }
 
-  char string[8192];
+  wchar_t string16[8192];
+  char escaped8[16384];
   if (as_string_table) {
     printf("STRINGTABLE {\n");
   }
 
   for (int i = 0; i < 0xFFFF; i++) {
-    int len = LoadStringA(library, i, string, 8192);
+    int len = LoadStringW(library, i, string16, 8192);
     if (len) {
-      string[len] = '\0';
-      char* escaped = escape(string, as_string_table);
+      string16[len] = L'\0';
       if (as_string_table) {
-        printf("  %d, %s\n", i, escaped);
+        char* encoded = encode_utf16(string16);
+        printf("  %d, %s\n", i, encoded);
+        free(encoded);
       } else {
-        printf("%d=%s\n", i, escaped);
+        wchar_t* escaped16 = escape(string16);
+        WideCharToMultiByte(CP_UTF8, 0, escaped16, -1, escaped8, sizeof(escaped8), NULL, NULL);
+        printf("%d=%s\n", i, escaped8);
+        free(escaped16);
       }
-      free(escaped);
     }
   }
 

main.c

@@ -17,13 +17,16 @@ static char file_exists (char* filename) {
 static void init () {
   aoc_ini_init();
 
+  char ini_path[MAX_PATH];
+  memset(ini_path, 0, sizeof(ini_path));
   if (*up_mod_game_dir[0] != '\0') {
-    char ini_path[MAX_PATH];
     // up_mod_game_dir contains trailing '\'
     sprintf(ini_path, "%s%s", *up_mod_game_dir, "language.ini");
-    if (file_exists(ini_path)) {
-      aoc_ini_load_strings(ini_path);
-    }
+  } else {
+    sprintf(ini_path, "%s", "language.ini");
+  }
+  if (file_exists(ini_path)) {
+    aoc_ini_load_strings(ini_path);
   }
 }
 

strings.rc

@@ -1,31 +1,31 @@
 STRINGTABLE {
-  8022, "Research Loom (+15 villager hit points; +1/+2P armor)"
-  8374, "Research Heated Shot (+125% tower, +4 castle attack vs. ships)"
-  8377, "Research Shipwright (ships cost 20% less wood; build 35% faster)."
-  8415, "Research Parthian Tactics (+1/+2P Cavalry Archer armor, increased attack vs. pikemen)"
-  8438, "Research Shinkichon (+1 range Mangonels, Onagers)"
-  9475, "Cannot load that old recorded game."
-  9764, "Turbo Random Map"
-  10890, "Full Random"
-  10902, "Blind Random"
-  20152, "<b>Goths<b> \nInfantry civilization \n\n� Infantry cost -25% starting in Feudal Age\n� Infantry +1 attack vs. buildings \n� Villagers +5 attack vs. wild boar; hunters carry +15 meat \n� +10 population in Imperial Age \n\n<b>Unique Unit:<b> Huskarl (infantry) \n\n<b>Unique Techs:<b> Anarchy (create Huskarls at Barracks); Perfusion (Barracks work 50% faster) \n<b>Team Bonus:<b> Barracks work 20% faster"
-  20154, "<b>Japanese<b> \nInfantry civilization \n\n� Fishing Ships 2X hit points; +2P armor; work rate +5% Dark, +10% Feudal, +15% Castle, +20% Imperial Age \n� Mill, Lumber/Mining Camps cost -50% \n� Infantry attack 25% faster starting in Feudal Age \n\n<b>Unique Unit:<b> Samurai (infantry) \n\n<b>Unique Technology:<b> Kataparuto (Trebuchets fire, pack faster) \n\n<b>Team Bonus:<b> galleys +50% line of sight"
-  20155, "<b>Chinese<b> \nArcher civilization \n\n� Start with +3 villagers but -50 wood, -200 food \n� Technologies cost -10% Feudal, \n -15% Castle, -20% Imperial Age \n� Town Centers support 10 population \n� Demolition ships +50% hit points \n\n<b>Unique Unit:<b> Chu Ko Nu (archer) \n\n<b>Unique Technology:<b> Rocketry (+2P Chu Ko Nu attack, +4P scorpions) \n\n<b>Team Bonus:<b> Farms +45 food"
-  20156, "<b>Byzantines<b> \nDefensive civilization \n� Buildings +10% HPs Dark, +20% Feudal, \n +30% Castle, +40% Imperial Age \n� Camels, skirmishers, Pikemen, Halberdiers cost -25% \n� Fire ships +20% attack \n� Advance to Imperial Age costs -33% \n� Town Watch free \n\n<b>Unique Unit:<b> Cataphract (cavalry) \n\n<b>Unique Tech:<b> Logistica (Cataphracts cause trample damage) \n\n<b>Team Bonus:<b> Monks +50% heal speed"
-  20163, "<b>Spanish<b> \nGunpowder and Monk civilization \n� Builders work 30% faster \n� Blacksmith upgrades don't cost gold \n� Cannon Galleons benefit from Ballistics (fire faster, more accurately) \n\n <b>Unique Units:<b> Conquistador (mounted hand cannoneer), Missionary (mounted Monk) \n� Hand Cannoneers and Bombard Cannons fire 15% faster \n\n <b>Unique Tech:<b> Supremacy (villagers better in combat) \n\n<b>Team Bonus:<b> Trade units generate +33% gold"
-  20164, "<b>Aztecs<b> \nInfantry and Monk civilization \n\n� Villagers carry +5 \n� Military units created 15% faster \n� +5 Monk hit points for each Monastery technology \n� Loom free\n\n <b>Unique Unit:<b> Jaguar Warrior (infantry) \n\n <b>Unique Technology:<b> Garland Wars (+4 infantry attack) \n\n<b>Team Bonus:<b> Relics generate +33% gold"
-  20167, "<b>Koreans<b> \nTower and naval civilization \n\n� Villagers +3 line of sight \n� Stone miners work 20% faster \n� Tower upgrades free (Bombard Tower requires Chemistry)\n� Tower range +1 Castle, +2 Imperial Age \n\n <b>Unique Units:<b> War Wagon (cavalry archer), Turtle Ship (war ship)\n\n <b>Unique Technology:<b> Shinkichon \n(+1 range Mangonels, Onagers) \n <b>Team Bonus:<b> Mangonels, Onagers +1 range"
-  26094, "Build <b> Battering Ram<b> (<cost>) \nSlow, lumbering siege weapon used to reduce enemy towns to ruins. Attack bonus vs. buildings; resistant to archer attack. Infantry and foot archers can garrison inside for protection. Garrisoned infantry and pikemen increase speed and attack vs. buildings. <i> Upgrades: attack (University); to Capped Ram 300F (Siege Workshop); more resistant to Monks (Monastery).<i> \n<hp> <attack> <armor> <piercearmor> <range> <garrison>"
-  26154, "Build <b> Guard Tower<b> (<cost>) \nStronger than Watch Tower, with greater fighting ability. Units can garrison inside for protection and additional attack strength. <i> Upgrades: line of sight (Town Center); attack, range (Blacksmith); hit points, armor, ship attack; to Keep 500F, 350W (University).<i> \n<hp> <attack> <armor> <piercearmor> <range> <garrison>"
-  26178, "Build <b> Watch Tower<b> (<cost>) \nBasic stone tower. Automatically attacks enemy units and buildings within range. Units can garrison inside for protection and additional attack strength. <i> Upgrades: line of sight (Town Center); attack, range (Blacksmith); hit points, armor, ship attack; to Guard Tower 100F, 250W (University).<i> \n<hp> <attack> <armor> <piercearmor> <range> <garrison>"
-  26203, "Build <b> Stone Wall<b> (<cost>) \nStronger than Palisade Wall but more expensive. Slows down your enemies so you have a chance to fend them off. <i> Upgrades: line of sight (Town Center); hit points, armor, to Fortified Wall 200F, 100W (University).<i> \n<hp> <attack> <armor> <piercearmor> <range>"
-  26289, "Build <b> Capped Ram<b> (<cost>) \nStronger than Battering Ram. Attack bonus vs. buildings; resistant to archer attack. Infantry and foot archers can garrison inside for protection. Garrisoned infantry and pikemen increase speed and attack vs. buildings. <i> Upgrades: attack (University); to Siege Ram 1000F (Siege Workshop); more resistant to Monks (Monastery).<i> \n<hp> <attack> <armor> <piercearmor> <range> <garrison>"
-  28022, "Research <b>Loom<b> (<cost>) \nMakes your villagers harder to kill by providing +15 hit points and +1 normal/+2 pierce armor."
-  28374, "Research <b>Heated Shot<b> (<cost>) \n Towers cause 125% more damage to ships; Castles cause 25% more damage to ships."
-  28377, "Research <b>Shipwright<b> (<cost>) \nShips cost 20% less wood; build 35% faster."
-  28415, "Research <b>Parthian Tactics<b> (<cost>) \n Cavalry Archers have +1 normal/+2 pierce armor; Cavalry Archers have +4 attack, Mangudai +2 attack vs. pikemen."
-  28438, "Research <b>Shinkichon<b> (<cost>) \n Mangonels, Onagers, and Siege Onagers have +1 range."
-  30133, "Full Random - Chooses one of the following maps: Arabia, Archipelago, Baltic, Coastal, Continental, Ghost Lake, Highland, Islands, Mediterranean, Migration, Mongolia, Oasis, Rivers, Scandinavia, Team Islands, or Yucatan."
-  30147, "Blind Random - Chooses one of the following maps: Arabia, Archipelago, Baltic, Black Forest, Coastal, Continental, Crater Lake, Ghost Lake, Gold Rush, Highland, Islands, Mediterranean, Migration, Mongolia, Oasis, Rivers, Salt Marsh, Scandinavia, Team Islands, or Yucatan."
-  30177, "Turbo Random Map - Buildings create units faster, villagers gather faster, build faster, and carry more."
+  8022, L"Research Loom (+15 villager hit points; +1/+2P armor)"
+  8374, L"Research Heated Shot (+125% tower, +4 castle attack vs. ships)"
+  8377, L"Research Shipwright (ships cost 20% less wood; build 35% faster)."
+  8415, L"Research Parthian Tactics (+1/+2P Cavalry Archer armor, increased attack vs. pikemen)"
+  8438, L"Research Shinkichon (+1 range Mangonels, Onagers)"
+  9475, L"Cannot load that old recorded game."
+  9764, L"Turbo Random Map"
+  10890, L"Full Random"
+  10902, L"Blind Random"
+  20152, L"<b>Goths<b> \nInfantry civilization \n\n\x00B7 Infantry cost -25% starting in Feudal Age\n\x00B7 Infantry +1 attack vs. buildings \n\x00B7 Villagers +5 attack vs. wild boar; hunters carry +15 meat \n\x00B7 +10 population in Imperial Age \n\n<b>Unique Unit:<b> Huskarl (infantry) \n\n<b>Unique Techs:<b> Anarchy (create Huskarls at Barracks); Perfusion (Barracks work 50% faster) \n<b>Team Bonus:<b> Barracks work 20% faster"
+  20154, L"<b>Japanese<b> \nInfantry civilization \n\n\x00B7 Fishing Ships 2X hit points; +2P armor; work rate +5% Dark, +10% Feudal, +15% Castle, +20% Imperial Age \n\x00B7 Mill, Lumber/Mining Camps cost -50% \n\x00B7 Infantry attack 25% faster starting in Feudal Age \n\n<b>Unique Unit:<b> Samurai (infantry) \n\n<b>Unique Technology:<b> Kataparuto (Trebuchets fire, pack faster) \n\n<b>Team Bonus:<b> galleys +50% line of sight"
+  20155, L"<b>Chinese<b> \nArcher civilization \n\n\x00B7 Start with +3 villagers but -50 wood, -200 food \n\x00B7 Technologies cost -10% Feudal, \n -15% Castle, -20% Imperial Age \n\x00B7 Town Centers support 10 population \n\x00B7 Demolition ships +50% hit points \n\n<b>Unique Unit:<b> Chu Ko Nu (archer) \n\n<b>Unique Technology:<b> Rocketry (+2P Chu Ko Nu attack, +4P scorpions) \n\n<b>Team Bonus:<b> Farms +45 food"
+  20156, L"<b>Byzantines<b> \nDefensive civilization \n\x00B7 Buildings +10% HPs Dark, +20% Feudal, \n +30% Castle, +40% Imperial Age \n\x00B7 Camels, skirmishers, Pikemen, Halberdiers cost -25% \n\x00B7 Fire ships +20% attack \n\x00B7 Advance to Imperial Age costs -33% \n\x00B7 Town Watch free \n\n<b>Unique Unit:<b> Cataphract (cavalry) \n\n<b>Unique Tech:<b> Logistica (Cataphracts cause trample damage) \n\n<b>Team Bonus:<b> Monks +50% heal speed"
+  20163, L"<b>Spanish<b> \nGunpowder and Monk civilization \n\x00B7 Builders work 30% faster \n\x00B7 Blacksmith upgrades don't cost gold \n\x00B7 Cannon Galleons benefit from Ballistics (fire faster, more accurately) \n\n <b>Unique Units:<b> Conquistador (mounted hand cannoneer), Missionary (mounted Monk) \n\x00B7 Hand Cannoneers and Bombard Cannons fire 15% faster \n\n <b>Unique Tech:<b> Supremacy (villagers better in combat) \n\n<b>Team Bonus:<b> Trade units generate +33% gold"
+  20164, L"<b>Aztecs<b> \nInfantry and Monk civilization \n\n\x00B7 Villagers carry +5 \n\x00B7 Military units created 15% faster \n\x00B7 +5 Monk hit points for each Monastery technology \n\x00B7 Loom free\n\n <b>Unique Unit:<b> Jaguar Warrior (infantry) \n\n <b>Unique Technology:<b> Garland Wars (+4 infantry attack) \n\n<b>Team Bonus:<b> Relics generate +33% gold"
+  20167, L"<b>Koreans<b> \nTower and naval civilization \n\n\x00B7 Villagers +3 line of sight \n\x00B7 Stone miners work 20% faster \n\x00B7 Tower upgrades free (Bombard Tower requires Chemistry)\n\x00B7 Tower range +1 Castle, +2 Imperial Age \n\n <b>Unique Units:<b> War Wagon (cavalry archer), Turtle Ship (war ship)\n\n <b>Unique Technology:<b> Shinkichon \n(+1 range Mangonels, Onagers) \n <b>Team Bonus:<b> Mangonels, Onagers +1 range"
+  26094, L"Build <b> Battering Ram<b> (<cost>) \nSlow, lumbering siege weapon used to reduce enemy towns to ruins. Attack bonus vs. buildings; resistant to archer attack. Infantry and foot archers can garrison inside for protection. Garrisoned infantry and pikemen increase speed and attack vs. buildings. <i> Upgrades: attack (University); to Capped Ram 300F (Siege Workshop); more resistant to Monks (Monastery).<i> \n<hp> <attack> <armor> <piercearmor> <range> <garrison>"
+  26154, L"Build <b> Guard Tower<b> (<cost>) \nStronger than Watch Tower, with greater fighting ability. Units can garrison inside for protection and additional attack strength. <i> Upgrades: line of sight (Town Center); attack, range (Blacksmith); hit points, armor, ship attack; to Keep 500F, 350W (University).<i> \n<hp> <attack> <armor> <piercearmor> <range> <garrison>"
+  26178, L"Build <b> Watch Tower<b> (<cost>) \nBasic stone tower. Automatically attacks enemy units and buildings within range. Units can garrison inside for protection and additional attack strength. <i> Upgrades: line of sight (Town Center); attack, range (Blacksmith); hit points, armor, ship attack; to Guard Tower 100F, 250W (University).<i> \n<hp> <attack> <armor> <piercearmor> <range> <garrison>"
+  26203, L"Build <b> Stone Wall<b> (<cost>) \nStronger than Palisade Wall but more expensive. Slows down your enemies so you have a chance to fend them off. <i> Upgrades: line of sight (Town Center); hit points, armor, to Fortified Wall 200F, 100W (University).<i> \n<hp> <attack> <armor> <piercearmor> <range>"
+  26289, L"Build <b> Capped Ram<b> (<cost>) \nStronger than Battering Ram. Attack bonus vs. buildings; resistant to archer attack. Infantry and foot archers can garrison inside for protection. Garrisoned infantry and pikemen increase speed and attack vs. buildings. <i> Upgrades: attack (University); to Siege Ram 1000F (Siege Workshop); more resistant to Monks (Monastery).<i> \n<hp> <attack> <armor> <piercearmor> <range> <garrison>"
+  28022, L"Research <b>Loom<b> (<cost>) \nMakes your villagers harder to kill by providing +15 hit points and +1 normal/+2 pierce armor."
+  28374, L"Research <b>Heated Shot<b> (<cost>) \n Towers cause 125% more damage to ships; Castles cause 25% more damage to ships."
+  28377, L"Research <b>Shipwright<b> (<cost>) \nShips cost 20% less wood; build 35% faster."
+  28415, L"Research <b>Parthian Tactics<b> (<cost>) \n Cavalry Archers have +1 normal/+2 pierce armor; Cavalry Archers have +4 attack, Mangudai +2 attack vs. pikemen."
+  28438, L"Research <b>Shinkichon<b> (<cost>) \n Mangonels, Onagers, and Siege Onagers have +1 range."
+  30133, L"Full Random - Chooses one of the following maps: Arabia, Archipelago, Baltic, Coastal, Continental, Ghost Lake, Highland, Islands, Mediterranean, Migration, Mongolia, Oasis, Rivers, Scandinavia, Team Islands, or Yucatan."
+  30147, L"Blind Random - Chooses one of the following maps: Arabia, Archipelago, Baltic, Black Forest, Coastal, Continental, Crater Lake, Ghost Lake, Gold Rush, Highland, Islands, Mediterranean, Migration, Mongolia, Oasis, Rivers, Salt Marsh, Scandinavia, Team Islands, or Yucatan."
+  30177, L"Turbo Random Map - Buildings create units faster, villagers gather faster, build faster, and carry more."
 }