Optimize Dictionary Import Memory Use

package.json

@@ -118,9 +118,9 @@
         "dexie-export-import": "^4.1.4",
         "hangul-js": "^0.2.6",
         "kanji-processor": "^1.0.2",
+        "linkedom": "^0.18.10",
         "parse5": "^7.2.1",
-        "yomitan-handlebars": "git+https://github.com/yomidevs/yomitan-handlebars.git#12aff5e3550954d7d3a98a5917ff7d579f3cce25",
-        "linkedom": "^0.18.10"
+        "yomitan-handlebars": "git+https://github.com/yomidevs/yomitan-handlebars.git#12aff5e3550954d7d3a98a5917ff7d579f3cce25"
     },
     "lint-staged": {
         "*.md": "prettier --write"

test/data/dictionaries/invalid-dictionary10/index.json

@@ -0,0 +1,7 @@
+{
+    "title": "Invalid Dictionary 10",
+    "format": 3,
+    "revision": "test",
+    "sequenced": true,
+    "description": "String entries in term bank instead of arrays"
+}

test/data/dictionaries/invalid-dictionary10/term_bank_1.json

@@ -0,0 +1 @@
+["hello", "world"]

test/data/dictionaries/invalid-dictionary11/index.json

@@ -0,0 +1,7 @@
+{
+    "title": "Invalid Dictionary 11",
+    "format": 3,
+    "revision": "test",
+    "sequenced": true,
+    "description": "Null entries in term bank"
+}

test/data/dictionaries/invalid-dictionary11/term_bank_1.json

@@ -0,0 +1 @@
+[null, null]

test/data/dictionaries/invalid-dictionary12/index.json

@@ -0,0 +1,7 @@
+{
+    "title": "Invalid Dictionary 12",
+    "format": 3,
+    "revision": "test",
+    "sequenced": true,
+    "description": "Mixed valid array entry followed by invalid string entry in term bank"
+}

test/data/dictionaries/invalid-dictionary12/term_bank_1.json

@@ -0,0 +1 @@
+[["打", "だ", "", "", 1, "", 1, ""], "invalid"]

test/data/dictionaries/invalid-dictionary7/index.json

@@ -0,0 +1,7 @@
+{
+    "title": "Invalid Dictionary 7",
+    "format": 3,
+    "revision": "test",
+    "sequenced": true,
+    "description": "Non-array entries in term bank (numbers)"
+}

test/data/dictionaries/invalid-dictionary7/term_bank_1.json

@@ -0,0 +1 @@
+[1, 2, 3]

test/data/dictionaries/invalid-dictionary8/index.json

@@ -0,0 +1,7 @@
+{
+    "title": "Invalid Dictionary 8",
+    "format": 3,
+    "revision": "test",
+    "sequenced": true,
+    "description": "Boolean entries in term bank instead of arrays"
+}

test/data/dictionaries/invalid-dictionary8/term_bank_1.json

@@ -0,0 +1 @@
+[true, false]

test/data/dictionaries/invalid-dictionary9/index.json

@@ -0,0 +1,7 @@
+{
+    "title": "Invalid Dictionary 9",
+    "format": 3,
+    "revision": "test",
+    "sequenced": true,
+    "description": "Object entries in term bank instead of arrays"
+}

test/data/dictionaries/invalid-dictionary9/term_bank_1.json

@@ -0,0 +1 @@
+[{"key": "val"}]

test/data/json.json

@@ -22,6 +22,18 @@
         {"path": "test/data/dictionaries/invalid-dictionary5/index.json", "ignore": true},
         {"path": "test/data/dictionaries/invalid-dictionary6/term_meta_bank_1.json", "ignore": true},
         {"path": "test/data/dictionaries/invalid-dictionary6/index.json", "ignore": true},
+        {"path": "test/data/dictionaries/invalid-dictionary7/term_bank_1.json", "ignore": true},
+        {"path": "test/data/dictionaries/invalid-dictionary7/index.json", "ignore": true},
+        {"path": "test/data/dictionaries/invalid-dictionary8/term_bank_1.json", "ignore": true},
+        {"path": "test/data/dictionaries/invalid-dictionary8/index.json", "ignore": true},
+        {"path": "test/data/dictionaries/invalid-dictionary9/term_bank_1.json", "ignore": true},
+        {"path": "test/data/dictionaries/invalid-dictionary9/index.json", "ignore": true},
+        {"path": "test/data/dictionaries/invalid-dictionary10/term_bank_1.json", "ignore": true},
+        {"path": "test/data/dictionaries/invalid-dictionary10/index.json", "ignore": true},
+        {"path": "test/data/dictionaries/invalid-dictionary11/term_bank_1.json", "ignore": true},
+        {"path": "test/data/dictionaries/invalid-dictionary11/index.json", "ignore": true},
+        {"path": "test/data/dictionaries/invalid-dictionary12/term_bank_1.json", "ignore": true},
+        {"path": "test/data/dictionaries/invalid-dictionary12/index.json", "ignore": true},
         {"path": "test/jsconfig.json", "ignore": true},
         {"path": "test/data/vitest.write.config.json", "ignore": true},
         {"path": "test/data/vitest.options.config.json", "ignore": true},

test/database.test.js

@@ -16,6 +16,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
+import {BlobWriter, TextReader, ZipWriter} from '@zip.js/zip.js';
 import {IDBFactory, IDBKeyRange} from 'fake-indexeddb';
 import {readFileSync} from 'node:fs';
 import {fileURLToPath} from 'node:url';
@@ -43,6 +44,22 @@ async function createTestDictionaryArchiveData(dictionary, dictionaryName) {
     return await createDictionaryArchiveData(dictionaryDirectory, dictionaryName);
 }
 
+/**
+ * Creates a dictionary zip archive with raw file contents, bypassing JSON parse/re-serialize.
+ * This allows testing with intentionally malformed JSON that parseJson would reject.
+ * @param {Record<string, string>} files Map of filename to raw string content
+ * @returns {Promise<ArrayBuffer>}
+ */
+async function createRawDictionaryArchiveData(files) {
+    const zipFileWriter = new BlobWriter();
+    const zipWriter = new ZipWriter(zipFileWriter, {level: 0});
+    for (const [fileName, content] of Object.entries(files)) {
+        await zipWriter.add(fileName, new TextReader(content));
+    }
+    const blob = await zipWriter.close();
+    return await blob.arrayBuffer();
+}
+
 /**
  * @param {import('vitest').ExpectStatic} expect
  * @param {import('dictionary-importer').OnProgressCallback} [onProgress]
@@ -158,6 +175,12 @@ describe('Database', () => {
             {name: 'invalid-dictionary4'},
             {name: 'invalid-dictionary5'},
             {name: 'invalid-dictionary6'},
+            {name: 'invalid-dictionary7'},
+            {name: 'invalid-dictionary8'},
+            {name: 'invalid-dictionary9'},
+            {name: 'invalid-dictionary10'},
+            {name: 'invalid-dictionary11'},
+            {name: 'invalid-dictionary12'},
         ];
         describe.each(invalidDictionaries)('Invalid dictionary: $name', ({name}) => {
             test('Has invalid data', async ({expect}) => {
@@ -173,6 +196,41 @@ describe('Database', () => {
             });
         });
     });
+    describe('Invalid raw dictionaries', () => {
+        const indexJson = JSON.stringify({title: 'Raw Test', format: 3, revision: 'test', sequenced: true});
+        const validEntry = '["打","だ","n","n",1,["definition"],1,""]';
+        const rawInvalidDictionaries = [
+            {name: 'missing comma between entries', termBank: `[${validEntry}${validEntry}]`},
+            {name: 'leading comma', termBank: `[,${validEntry}]`},
+            {name: 'double comma', termBank: `[${validEntry},,${validEntry}]`},
+            {name: 'trailing garbage after array', termBank: `[${validEntry}]garbage`},
+            {name: 'leading garbage before array', termBank: `garbage[${validEntry}]`},
+            {name: 'concatenated arrays', termBank: `[${validEntry}][${validEntry}]`},
+            {name: 'empty file', termBank: ''},
+            {name: 'whitespace only', termBank: '   '},
+            {name: 'just a number', termBank: '123'},
+            {name: 'just a string', termBank: '"hello"'},
+            {name: 'just null', termBank: 'null'},
+            {name: 'unclosed array', termBank: `[${validEntry}`},
+            {name: 'unclosed entry', termBank: '[["a","b"'},
+        ];
+        describe.each(rawInvalidDictionaries)('Raw invalid: $name', ({termBank}) => {
+            test('Has invalid data', async ({expect}) => {
+                const dictionaryDatabase = new DictionaryDatabase();
+                await dictionaryDatabase.prepare();
+
+                const testDictionarySource = await createRawDictionaryArchiveData({
+                    'index.json': indexJson,
+                    'term_bank_1.json': termBank,
+                });
+
+                /** @type {import('dictionary-importer').ImportDetails} */
+                const importDetails = {prefixWildcardsSupported: false, yomitanVersion: '0.0.0.0'};
+                await expect.soft(createDictionaryImporter(expect).importDictionary(dictionaryDatabase, testDictionarySource, importDetails)).rejects.toThrow('Dictionary has invalid data');
+                await dictionaryDatabase.close();
+            });
+        });
+    });
     describe('Database valid usage', () => {
         const testDataFilePath = join(dirname, 'data/database-test-cases.json');
         /** @type {import('test/database').DatabaseTestData} */