summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorCashew <52880648+Scrub1492@users.noreply.github.com>2024-01-20 10:43:27 +0700
committerGitHub <noreply@github.com>2024-01-20 03:43:27 +0000
commit0011afe435f565dc5669f030da64e2c6e10ce05e (patch)
treeb7cc9598d6a958cbfdd15fff32294ab85dda550e
parentbe267e1994f8fda2ef68562c5c3a5edc9442a8a8 (diff)
improve dictionary validation (#520)
* improve dictionary validation * fix * improve performance * add undefined check * rename types * rename types * rename
-rw-r--r--dev/dictionary-validate.js63
-rw-r--r--ext/js/dictionary/dictionary-importer.js24
-rw-r--r--types/dev/dictionary-validate.d.ts5
-rw-r--r--types/ext/dictionary-importer.d.ts4
4 files changed, 53 insertions, 43 deletions
diff --git a/dev/dictionary-validate.js b/dev/dictionary-validate.js
index 8953dbf8..18bba99e 100644
--- a/dev/dictionary-validate.js
+++ b/dev/dictionary-validate.js
@@ -40,35 +40,33 @@ function readSchema(relativeFileName) {
/**
* @param {import('dev/schema-validate').ValidateMode} mode
* @param {import('jszip')} zip
- * @param {string} fileNameFormat
- * @param {import('dev/dictionary-validate').Schema} schema
+ * @param {import('dev/dictionary-validate').SchemasDetails} schemasDetails
*/
-async function validateDictionaryBanks(mode, zip, fileNameFormat, schema) {
- let jsonSchema;
- try {
- jsonSchema = createJsonSchema(mode, schema);
- } catch (e) {
- const e2 = toError(e);
- e2.message += `\n(in file ${fileNameFormat})}`;
- throw e2;
- }
- let index = 1;
- while (true) {
- const fileName = fileNameFormat.replace(/\?/, `${index}`);
+async function validateDictionaryBanks(mode, zip, schemasDetails) {
+ for (const [fileName, file] of Object.entries(zip.files)) {
+ for (const [fileNameFormat, schema] of schemasDetails) {
+ if (!fileNameFormat.test(fileName)) { continue; }
- const file = zip.files[fileName];
- if (!file) { break; }
+ let jsonSchema;
+ try {
+ jsonSchema = createJsonSchema(mode, schema);
+ } catch (e) {
+ const e2 = toError(e);
+ e2.message += `\n(in file ${fileName})}`;
+ throw e2;
+ }
- const data = parseJson(await file.async('string'));
- try {
- jsonSchema.validate(data);
- } catch (e) {
- const e2 = toError(e);
- e2.message += `\n(in file ${fileName})}`;
- throw e2;
- }
+ const data = parseJson(await file.async('string'));
- ++index;
+ try {
+ jsonSchema.validate(data);
+ } catch (e) {
+ const e2 = toError(e);
+ e2.message += `\n(in file ${fileName})}`;
+ throw e2;
+ }
+ break;
+ }
}
}
@@ -98,11 +96,16 @@ export async function validateDictionary(mode, archive, schemas) {
throw e2;
}
- await validateDictionaryBanks(mode, archive, 'term_bank_?.json', version === 1 ? schemas.termBankV1 : schemas.termBankV3);
- await validateDictionaryBanks(mode, archive, 'term_meta_bank_?.json', schemas.termMetaBankV3);
- await validateDictionaryBanks(mode, archive, 'kanji_bank_?.json', version === 1 ? schemas.kanjiBankV1 : schemas.kanjiBankV3);
- await validateDictionaryBanks(mode, archive, 'kanji_meta_bank_?.json', schemas.kanjiMetaBankV3);
- await validateDictionaryBanks(mode, archive, 'tag_bank_?.json', schemas.tagBankV3);
+ /** @type {import('dev/dictionary-validate').SchemasDetails} */
+ const schemasDetails = [
+ [/^term_bank_(\d+)\.json$/, version === 1 ? schemas.termBankV1 : schemas.termBankV3],
+ [/^term_meta_bank_(\d+)\.json$/, schemas.termMetaBankV3],
+ [/^kanji_bank_(\d+)\.json$/, version === 1 ? schemas.kanjiBankV1 : schemas.kanjiBankV3],
+ [/^kanji_meta_bank_(\d+)\.json$/, schemas.kanjiMetaBankV3],
+ [/^tag_bank_(\d+)\.json$/, schemas.tagBankV3]
+ ];
+
+ await validateDictionaryBanks(mode, archive, schemasDetails);
}
/**
diff --git a/ext/js/dictionary/dictionary-importer.js b/ext/js/dictionary/dictionary-importer.js
index a9726aaf..067c8a3a 100644
--- a/ext/js/dictionary/dictionary-importer.js
+++ b/ext/js/dictionary/dictionary-importer.js
@@ -115,13 +115,13 @@ export class DictionaryImporter {
// Files
/** @type {import('dictionary-importer').QueryDetails} */
- const queryDetails = new Map([
+ const queryDetails = [
['termFiles', /^term_bank_(\d+)\.json$/],
['termMetaFiles', /^term_meta_bank_(\d+)\.json$/],
['kanjiFiles', /^kanji_bank_(\d+)\.json$/],
['kanjiMetaFiles', /^kanji_meta_bank_(\d+)\.json$/],
['tagFiles', /^tag_bank_(\d+)\.json$/]
- ]);
+ ];
const {termFiles, termMetaFiles, kanjiFiles, kanjiMetaFiles, tagFiles} = Object.fromEntries(this._getArchiveFiles(fileMap, queryDetails));
// Load data
@@ -692,16 +692,18 @@ export class DictionaryImporter {
_getArchiveFiles(fileMap, queryDetails) {
/** @type {import('dictionary-importer').QueryResult} */
const results = new Map();
- for (const [name, value] of fileMap.entries()) {
- for (const [fileType, fileNameFormat] of queryDetails.entries()) {
- let entries = results.get(fileType);
- if (typeof entries === 'undefined') {
- entries = [];
- results.set(fileType, entries);
- }
- if (fileNameFormat.test(name)) {
- entries.push(value);
+ for (const [fileType] of queryDetails) {
+ results.set(fileType, []);
+ }
+
+ for (const [fileName, fileEntry] of fileMap.entries()) {
+ for (const [fileType, fileNameFormat] of queryDetails) {
+ if (!fileNameFormat.test(fileName)) { continue; }
+ const entries = results.get(fileType);
+
+ if (typeof entries !== 'undefined') {
+ entries.push(fileEntry);
break;
}
}
diff --git a/types/dev/dictionary-validate.d.ts b/types/dev/dictionary-validate.d.ts
index d3f8a8be..02d01b78 100644
--- a/types/dev/dictionary-validate.d.ts
+++ b/types/dev/dictionary-validate.d.ts
@@ -29,3 +29,8 @@ export type Schemas = {
termBankV3: Schema;
termMetaBankV3: Schema;
};
+
+/**
+ * An array of tuples of a regular expression for file types inside a dictionary and its corresponding schema.
+ */
+export type SchemasDetails = [fileNameFormat: RegExp, schema: unknown][];
diff --git a/types/ext/dictionary-importer.d.ts b/types/ext/dictionary-importer.d.ts
index 828ca86a..17bd1bb4 100644
--- a/types/ext/dictionary-importer.d.ts
+++ b/types/ext/dictionary-importer.d.ts
@@ -113,9 +113,9 @@ export type ImportRequirementContext = {
export type ArchiveFileMap = Map<string, ZipJS.Entry>;
/**
- * A map of file types inside a dictionary and its corresponding regular expressions.
+ * An array of tuples of a file type inside a dictionary and its corresponding regular expression.
*/
-export type QueryDetails = Map<string, RegExp>;
+export type QueryDetails = [fileType: string, fileNameFormat: RegExp][];
/**
* A map of file types inside a dictionary and its matching entries.