Update to zip.js for tests (#698)

* Use zip.js

* Set compression level to 0

* Update comments

4 files changed, 141 insertions, 69 deletions

diff --git a/dev/dictionary-archive-util.js b/dev/dictionary-archive-util.js
new file mode 100644
index 00000000..ead5790b
--- /dev/null
+++ b/dev/dictionary-archive-util.js
@@ -0,0 +1,110 @@
+/*
+ * Copyright (C) 2024  Yomitan Authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import {BlobWriter, TextReader, TextWriter, Uint8ArrayReader, ZipReader, ZipWriter} from '@zip.js/zip.js';
+import {readFileSync, readdirSync} from 'fs';
+import {join} from 'path';
+import {parseJson} from './json.js';
+
+/**
+ * Creates a zip archive from the given dictionary directory.
+ * @param {string} dictionaryDirectory
+ * @param {string} [dictionaryName]
+ * @returns {Promise<ArrayBuffer>}
+ */
+export async function createDictionaryArchiveData(dictionaryDirectory, dictionaryName) {
+    const fileNames = readdirSync(dictionaryDirectory);
+    const zipFileWriter = new BlobWriter();
+    // Level 0 compression used since decompression in the node environment is not supported.
+    // See dev/lib/zip.js for more details.
+    const zipWriter = new ZipWriter(zipFileWriter, {
+        level: 0
+    });
+    for (const fileName of fileNames) {
+        if (/\.json$/.test(fileName)) {
+            const content = readFileSync(join(dictionaryDirectory, fileName), {encoding: 'utf8'});
+            /** @type {import('dictionary-data').Index} */
+            const json = parseJson(content);
+            if (fileName === 'index.json' && typeof dictionaryName === 'string') {
+                json.title = dictionaryName;
+            }
+            await zipWriter.add(fileName, new TextReader(JSON.stringify(json, null, 0)));
+        } else {
+            const content = readFileSync(join(dictionaryDirectory, fileName), {encoding: null});
+            await zipWriter.add(fileName, new Blob([content]).stream());
+        }
+    }
+    const blob = await zipWriter.close();
+    return await blob.arrayBuffer();
+}
+
+/**
+ * @param {import('@zip.js/zip.js').Entry} entry
+ * @returns {Promise<string>}
+ */
+export async function readArchiveEntryDataString(entry) {
+    if (typeof entry.getData === 'undefined') { throw new Error('Cannot get index data'); }
+    return await entry.getData(new TextWriter());
+}
+
+/**
+ * @template [T=unknown]
+ * @param {import('@zip.js/zip.js').Entry} entry
+ * @returns {Promise<T>}
+ */
+export async function readArchiveEntryDataJson(entry) {
+    const indexContent = await readArchiveEntryDataString(entry);
+    return parseJson(indexContent);
+}
+
+/**
+ * @param {ArrayBuffer} data
+ * @returns {Promise<import('@zip.js/zip.js').Entry[]>}
+ */
+export async function getDictionaryArchiveEntries(data) {
+    const zipFileReader = new Uint8ArrayReader(new Uint8Array(data));
+    const zipReader = new ZipReader(zipFileReader);
+    return await zipReader.getEntries();
+}
+
+/**
+ * @template T
+ * @param {import('@zip.js/zip.js').Entry[]} entries
+ * @param {string} fileName
+ * @returns {Promise<T>}
+ */
+export async function getDictionaryArchiveJson(entries, fileName) {
+    const entry = entries.find((item) => item.filename === fileName);
+    if (typeof entry === 'undefined') { throw new Error(`File not found: ${fileName}`); }
+    return await readArchiveEntryDataJson(entry);
+}
+
+/**
+ * @returns {string}
+ */
+export function getIndexFileName() {
+    return 'index.json';
+}
+
+/**
+ * @param {ArrayBuffer} data
+ * @returns {Promise<import('dictionary-data').Index>}
+ */
+export async function getDictionaryArchiveIndex(data) {
+    const entries = await getDictionaryArchiveEntries(data);
+    return await getDictionaryArchiveJson(entries, getIndexFileName());
+}
diff --git a/dev/dictionary-validate.js b/dev/dictionary-validate.js
index 18bba99e..b770f311 100644
--- a/dev/dictionary-validate.js
+++ b/dev/dictionary-validate.js
@@ -17,10 +17,10 @@
  */
 
 import fs from 'fs';
-import JSZip from 'jszip';
 import path from 'path';
 import {performance} from 'perf_hooks';
 import {fileURLToPath} from 'url';
+import {getDictionaryArchiveEntries, getDictionaryArchiveJson, getIndexFileName, readArchiveEntryDataJson} from './dictionary-archive-util.js';
 import {parseJson} from './json.js';
 import {createJsonSchema} from './schema-validate.js';
 import {toError} from './to-error.js';
@@ -39,30 +39,31 @@ function readSchema(relativeFileName) {
 
 /**
  * @param {import('dev/schema-validate').ValidateMode} mode
- * @param {import('jszip')} zip
+ * @param {import('@zip.js/zip.js').Entry[]} entries
  * @param {import('dev/dictionary-validate').SchemasDetails} schemasDetails
  */
-async function validateDictionaryBanks(mode, zip, schemasDetails) {
-    for (const [fileName, file] of Object.entries(zip.files)) {
+async function validateDictionaryBanks(mode, entries, schemasDetails) {
+    for (const entry of entries) {
+        const {filename} = entry;
         for (const [fileNameFormat, schema] of schemasDetails) {
-            if (!fileNameFormat.test(fileName)) { continue; }
+            if (!fileNameFormat.test(filename)) { continue; }
 
             let jsonSchema;
             try {
                 jsonSchema = createJsonSchema(mode, schema);
             } catch (e) {
                 const e2 = toError(e);
-                e2.message += `\n(in file ${fileName})}`;
+                e2.message += `\n(in file ${filename})`;
                 throw e2;
             }
 
-            const data = parseJson(await file.async('string'));
+            const data = await readArchiveEntryDataJson(entry);
 
             try {
                 jsonSchema.validate(data);
             } catch (e) {
                 const e2 = toError(e);
-                e2.message += `\n(in file ${fileName})}`;
+                e2.message += `\n(in file ${filename})`;
                 throw e2;
             }
             break;
@@ -73,18 +74,13 @@ async function validateDictionaryBanks(mode, zip, schemasDetails) {
 /**
  * Validates a dictionary from its zip archive.
  * @param {import('dev/schema-validate').ValidateMode} mode
- * @param {import('jszip')} archive
+ * @param {ArrayBuffer} archiveData
  * @param {import('dev/dictionary-validate').Schemas} schemas
  */
-export async function validateDictionary(mode, archive, schemas) {
-    const indexFileName = 'index.json';
-    const indexFile = archive.files[indexFileName];
-    if (!indexFile) {
-        throw new Error('No dictionary index found in archive');
-    }
-
-    /** @type {import('dictionary-data').Index} */
-    const index = parseJson(await indexFile.async('string'));
+export async function validateDictionary(mode, archiveData, schemas) {
+    const entries = await getDictionaryArchiveEntries(archiveData);
+    const indexFileName = getIndexFileName();
+    const index = await getDictionaryArchiveJson(entries, indexFileName);
     const version = index.format || index.version;
 
     try {
@@ -92,7 +88,7 @@ export async function validateDictionary(mode, archive, schemas) {
         jsonSchema.validate(index);
     } catch (e) {
         const e2 = toError(e);
-        e2.message += `\n(in file ${indexFileName})}`;
+        e2.message += `\n(in file ${indexFileName})`;
         throw e2;
     }
 
@@ -105,7 +101,7 @@ export async function validateDictionary(mode, archive, schemas) {
         [/^tag_bank_(\d+)\.json$/, schemas.tagBankV3]
     ];
 
-    await validateDictionaryBanks(mode, archive, schemasDetails);
+    await validateDictionaryBanks(mode, entries, schemasDetails);
 }
 
 /**
@@ -138,8 +134,7 @@ export async function testDictionaryFiles(mode, dictionaryFileNames) {
         try {
             console.log(`Validating ${dictionaryFileName}...`);
             const source = fs.readFileSync(dictionaryFileName);
-            const archive = await JSZip.loadAsync(source);
-            await validateDictionary(mode, archive, schemas);
+            await validateDictionary(mode, source.buffer, schemas);
             const end = performance.now();
             console.log(`No issues detected (${((end - start) / 1000).toFixed(2)}s)`);
         } catch (e) {
diff --git a/dev/lib/zip.js b/dev/lib/zip.js
index 62d0784c..ee603c25 100644
--- a/dev/lib/zip.js
+++ b/dev/lib/zip.js
@@ -15,4 +15,18 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
+/**
+ * This script is importing a file within the '@zip.js/zip.js' dependency rather than
+ * simply importing '@zip.js/zip.js'.
+ *
+ * This is done in order to only import the subset of functionality that the extension needs.
+ *
+ * In this case, this subset only includes the components to support decompression using web workers.
+ *
+ * Therefore, if this file or the built library file is imported in a development, testing, or
+ * benchmark script, it will not be able to properly decompress the data of compressed zip files.
+ *
+ * As a workaround, testing zip data can be generated using {level: 0} compression.
+ */
+
 export * from '@zip.js/zip.js/lib/zip.js';
diff --git a/dev/util.js b/dev/util.js
index 5a4c5788..67d8d9d8 100644
--- a/dev/util.js
+++ b/dev/util.js
@@ -17,9 +17,7 @@
  */
 
 import fs from 'fs';
-import JSZip from 'jszip';
 import path from 'path';
-import {parseJson} from './json.js';
 
 /**
  * @param {string} baseDirectory
@@ -47,48 +45,3 @@ export function getAllFiles(baseDirectory, predicate = null) {
     }
     return results;
 }
-
-/**
- * Creates a zip archive from the given dictionary directory.
- * @param {string} dictionaryDirectory
- * @param {string} [dictionaryName]
- * @returns {import('jszip')}
- */
-export function createDictionaryArchive(dictionaryDirectory, dictionaryName) {
-    const fileNames = fs.readdirSync(dictionaryDirectory);
-
-    // Const zipFileWriter = new BlobWriter();
-    // const zipWriter = new ZipWriter(zipFileWriter);
-    const archive = new JSZip();
-
-    for (const fileName of fileNames) {
-        if (/\.json$/.test(fileName)) {
-            const content = fs.readFileSync(path.join(dictionaryDirectory, fileName), {encoding: 'utf8'});
-            const json = parseJson(content);
-            if (fileName === 'index.json' && typeof dictionaryName === 'string') {
-                /** @type {import('dictionary-data').Index} */ (json).title = dictionaryName;
-            }
-            archive.file(fileName, JSON.stringify(json, null, 0));
-
-            // Await zipWriter.add(fileName, new TextReader(JSON.stringify(json, null, 0)));
-        } else {
-            const content = fs.readFileSync(path.join(dictionaryDirectory, fileName), {encoding: null});
-            archive.file(fileName, content);
-
-            // console.log('adding');
-            // const r = new TextReader(content);
-            // console.log(r.readUint8Array(0, 10));
-            // console.log('reader done');
-            // await zipWriter.add(fileName, r);
-            // console.log('??');
-        }
-    }
-    // Await zipWriter.close();
-
-    // Retrieves the Blob object containing the zip content into `zipFileBlob`. It
-    // is also returned by zipWriter.close() for more convenience.
-    // const zipFileBlob = await zipFileWriter.getData();
-    return archive;
-
-    // Return zipFileBlob;
-}