diff options
Diffstat (limited to 'dev')
| -rw-r--r-- | dev/dictionary-archive-util.js | 110 | ||||
| -rw-r--r-- | dev/dictionary-validate.js | 39 | ||||
| -rw-r--r-- | dev/lib/zip.js | 14 | ||||
| -rw-r--r-- | dev/util.js | 47 |
4 files changed, 141 insertions, 69 deletions
diff --git a/dev/dictionary-archive-util.js b/dev/dictionary-archive-util.js new file mode 100644 index 00000000..ead5790b --- /dev/null +++ b/dev/dictionary-archive-util.js @@ -0,0 +1,110 @@ +/* + * Copyright (C) 2024 Yomitan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +import {BlobWriter, TextReader, TextWriter, Uint8ArrayReader, ZipReader, ZipWriter} from '@zip.js/zip.js'; +import {readFileSync, readdirSync} from 'fs'; +import {join} from 'path'; +import {parseJson} from './json.js'; + +/** + * Creates a zip archive from the given dictionary directory. + * @param {string} dictionaryDirectory + * @param {string} [dictionaryName] + * @returns {Promise<ArrayBuffer>} + */ +export async function createDictionaryArchiveData(dictionaryDirectory, dictionaryName) { + const fileNames = readdirSync(dictionaryDirectory); + const zipFileWriter = new BlobWriter(); + // Level 0 compression used since decompression in the node environment is not supported. + // See dev/lib/zip.js for more details. + const zipWriter = new ZipWriter(zipFileWriter, { + level: 0 + }); + for (const fileName of fileNames) { + if (/\.json$/.test(fileName)) { + const content = readFileSync(join(dictionaryDirectory, fileName), {encoding: 'utf8'}); + /** @type {import('dictionary-data').Index} */ + const json = parseJson(content); + if (fileName === 'index.json' && typeof dictionaryName === 'string') { + json.title = dictionaryName; + } + await zipWriter.add(fileName, new TextReader(JSON.stringify(json, null, 0))); + } else { + const content = readFileSync(join(dictionaryDirectory, fileName), {encoding: null}); + await zipWriter.add(fileName, new Blob([content]).stream()); + } + } + const blob = await zipWriter.close(); + return await blob.arrayBuffer(); +} + +/** + * @param {import('@zip.js/zip.js').Entry} entry + * @returns {Promise<string>} + */ +export async function readArchiveEntryDataString(entry) { + if (typeof entry.getData === 'undefined') { throw new Error('Cannot get index data'); } + return await entry.getData(new TextWriter()); +} + +/** + * @template [T=unknown] + * @param {import('@zip.js/zip.js').Entry} entry + * @returns {Promise<T>} + */ +export async function readArchiveEntryDataJson(entry) { + const indexContent = await readArchiveEntryDataString(entry); + return parseJson(indexContent); +} + +/** + * @param {ArrayBuffer} data + * @returns {Promise<import('@zip.js/zip.js').Entry[]>} + */ +export async function getDictionaryArchiveEntries(data) { + const zipFileReader = new Uint8ArrayReader(new Uint8Array(data)); + const zipReader = new ZipReader(zipFileReader); + return await zipReader.getEntries(); +} + +/** + * @template T + * @param {import('@zip.js/zip.js').Entry[]} entries + * @param {string} fileName + * @returns {Promise<T>} + */ +export async function getDictionaryArchiveJson(entries, fileName) { + const entry = entries.find((item) => item.filename === fileName); + if (typeof entry === 'undefined') { throw new Error(`File not found: ${fileName}`); } + return await readArchiveEntryDataJson(entry); +} + +/** + * @returns {string} + */ +export function getIndexFileName() { + return 'index.json'; +} + +/** + * @param {ArrayBuffer} data + * @returns {Promise<import('dictionary-data').Index>} + */ +export async function getDictionaryArchiveIndex(data) { + const entries = await getDictionaryArchiveEntries(data); + return await getDictionaryArchiveJson(entries, getIndexFileName()); +} diff --git a/dev/dictionary-validate.js b/dev/dictionary-validate.js index 18bba99e..b770f311 100644 --- a/dev/dictionary-validate.js +++ b/dev/dictionary-validate.js @@ -17,10 +17,10 @@ */ import fs from 'fs'; -import JSZip from 'jszip'; import path from 'path'; import {performance} from 'perf_hooks'; import {fileURLToPath} from 'url'; +import {getDictionaryArchiveEntries, getDictionaryArchiveJson, getIndexFileName, readArchiveEntryDataJson} from './dictionary-archive-util.js'; import {parseJson} from './json.js'; import {createJsonSchema} from './schema-validate.js'; import {toError} from './to-error.js'; @@ -39,30 +39,31 @@ function readSchema(relativeFileName) { /** * @param {import('dev/schema-validate').ValidateMode} mode - * @param {import('jszip')} zip + * @param {import('@zip.js/zip.js').Entry[]} entries * @param {import('dev/dictionary-validate').SchemasDetails} schemasDetails */ -async function validateDictionaryBanks(mode, zip, schemasDetails) { - for (const [fileName, file] of Object.entries(zip.files)) { +async function validateDictionaryBanks(mode, entries, schemasDetails) { + for (const entry of entries) { + const {filename} = entry; for (const [fileNameFormat, schema] of schemasDetails) { - if (!fileNameFormat.test(fileName)) { continue; } + if (!fileNameFormat.test(filename)) { continue; } let jsonSchema; try { jsonSchema = createJsonSchema(mode, schema); } catch (e) { const e2 = toError(e); - e2.message += `\n(in file ${fileName})}`; + e2.message += `\n(in file ${filename})`; throw e2; } - const data = parseJson(await file.async('string')); + const data = await readArchiveEntryDataJson(entry); try { jsonSchema.validate(data); } catch (e) { const e2 = toError(e); - e2.message += `\n(in file ${fileName})}`; + e2.message += `\n(in file ${filename})`; throw e2; } break; @@ -73,18 +74,13 @@ async function validateDictionaryBanks(mode, zip, schemasDetails) { /** * Validates a dictionary from its zip archive. * @param {import('dev/schema-validate').ValidateMode} mode - * @param {import('jszip')} archive + * @param {ArrayBuffer} archiveData * @param {import('dev/dictionary-validate').Schemas} schemas */ -export async function validateDictionary(mode, archive, schemas) { - const indexFileName = 'index.json'; - const indexFile = archive.files[indexFileName]; - if (!indexFile) { - throw new Error('No dictionary index found in archive'); - } - - /** @type {import('dictionary-data').Index} */ - const index = parseJson(await indexFile.async('string')); +export async function validateDictionary(mode, archiveData, schemas) { + const entries = await getDictionaryArchiveEntries(archiveData); + const indexFileName = getIndexFileName(); + const index = await getDictionaryArchiveJson(entries, indexFileName); const version = index.format || index.version; try { @@ -92,7 +88,7 @@ export async function validateDictionary(mode, archive, schemas) { jsonSchema.validate(index); } catch (e) { const e2 = toError(e); - e2.message += `\n(in file ${indexFileName})}`; + e2.message += `\n(in file ${indexFileName})`; throw e2; } @@ -105,7 +101,7 @@ export async function validateDictionary(mode, archive, schemas) { [/^tag_bank_(\d+)\.json$/, schemas.tagBankV3] ]; - await validateDictionaryBanks(mode, archive, schemasDetails); + await validateDictionaryBanks(mode, entries, schemasDetails); } /** @@ -138,8 +134,7 @@ export async function testDictionaryFiles(mode, dictionaryFileNames) { try { console.log(`Validating ${dictionaryFileName}...`); const source = fs.readFileSync(dictionaryFileName); - const archive = await JSZip.loadAsync(source); - await validateDictionary(mode, archive, schemas); + await validateDictionary(mode, source.buffer, schemas); const end = performance.now(); console.log(`No issues detected (${((end - start) / 1000).toFixed(2)}s)`); } catch (e) { diff --git a/dev/lib/zip.js b/dev/lib/zip.js index 62d0784c..ee603c25 100644 --- a/dev/lib/zip.js +++ b/dev/lib/zip.js @@ -15,4 +15,18 @@ * along with this program. If not, see <https://www.gnu.org/licenses/>. */ +/** + * This script is importing a file within the '@zip.js/zip.js' dependency rather than + * simply importing '@zip.js/zip.js'. + * + * This is done in order to only import the subset of functionality that the extension needs. + * + * In this case, this subset only includes the components to support decompression using web workers. + * + * Therefore, if this file or the built library file is imported in a development, testing, or + * benchmark script, it will not be able to properly decompress the data of compressed zip files. + * + * As a workaround, testing zip data can be generated using {level: 0} compression. + */ + export * from '@zip.js/zip.js/lib/zip.js'; diff --git a/dev/util.js b/dev/util.js index 5a4c5788..67d8d9d8 100644 --- a/dev/util.js +++ b/dev/util.js @@ -17,9 +17,7 @@ */ import fs from 'fs'; -import JSZip from 'jszip'; import path from 'path'; -import {parseJson} from './json.js'; /** * @param {string} baseDirectory @@ -47,48 +45,3 @@ export function getAllFiles(baseDirectory, predicate = null) { } return results; } - -/** - * Creates a zip archive from the given dictionary directory. - * @param {string} dictionaryDirectory - * @param {string} [dictionaryName] - * @returns {import('jszip')} - */ -export function createDictionaryArchive(dictionaryDirectory, dictionaryName) { - const fileNames = fs.readdirSync(dictionaryDirectory); - - // Const zipFileWriter = new BlobWriter(); - // const zipWriter = new ZipWriter(zipFileWriter); - const archive = new JSZip(); - - for (const fileName of fileNames) { - if (/\.json$/.test(fileName)) { - const content = fs.readFileSync(path.join(dictionaryDirectory, fileName), {encoding: 'utf8'}); - const json = parseJson(content); - if (fileName === 'index.json' && typeof dictionaryName === 'string') { - /** @type {import('dictionary-data').Index} */ (json).title = dictionaryName; - } - archive.file(fileName, JSON.stringify(json, null, 0)); - - // Await zipWriter.add(fileName, new TextReader(JSON.stringify(json, null, 0))); - } else { - const content = fs.readFileSync(path.join(dictionaryDirectory, fileName), {encoding: null}); - archive.file(fileName, content); - - // console.log('adding'); - // const r = new TextReader(content); - // console.log(r.readUint8Array(0, 10)); - // console.log('reader done'); - // await zipWriter.add(fileName, r); - // console.log('??'); - } - } - // Await zipWriter.close(); - - // Retrieves the Blob object containing the zip content into `zipFileBlob`. It - // is also returned by zipWriter.close() for more convenience. - // const zipFileBlob = await zipFileWriter.getData(); - return archive; - - // Return zipFileBlob; -} |