summaryrefslogtreecommitdiff
path: root/ext/js/language/dictionary-importer.js
diff options
context:
space:
mode:
authortoasted-nutbread <toasted-nutbread@users.noreply.github.com>2023-11-27 12:48:14 -0500
committertoasted-nutbread <toasted-nutbread@users.noreply.github.com>2023-11-27 12:48:14 -0500
commit4da4827bcbcdd1ef163f635d9b29416ff272b0bb (patch)
treea8a0f1a8befdb78a554e1be91f2c6059ca3ad5f9 /ext/js/language/dictionary-importer.js
parentfd6bba8a2a869eaf2b2c1fa49001f933fce3c618 (diff)
Add JSDoc type annotations to project (rebased)
Diffstat (limited to 'ext/js/language/dictionary-importer.js')
-rw-r--r--ext/js/language/dictionary-importer.js338
1 files changed, 272 insertions, 66 deletions
diff --git a/ext/js/language/dictionary-importer.js b/ext/js/language/dictionary-importer.js
index 791d1a77..aa6d7ae6 100644
--- a/ext/js/language/dictionary-importer.js
+++ b/ext/js/language/dictionary-importer.js
@@ -20,13 +20,27 @@ import * as ajvSchemas from '../../lib/validate-schemas.js';
import {BlobWriter, TextWriter, Uint8ArrayReader, ZipReader, configure} from '../../lib/zip.js';
import {stringReverse} from '../core.js';
import {MediaUtil} from '../media/media-util.js';
+
export class DictionaryImporter {
+ /**
+ * @param {import('dictionary-importer-media-loader').GenericMediaLoader} mediaLoader
+ * @param {import('dictionary-importer').OnProgressCallback} [onProgress]
+ */
constructor(mediaLoader, onProgress) {
+ /** @type {import('dictionary-importer-media-loader').GenericMediaLoader} */
this._mediaLoader = mediaLoader;
+ /** @type {import('dictionary-importer').OnProgressCallback} */
this._onProgress = typeof onProgress === 'function' ? onProgress : () => {};
- this._progressData = null;
+ /** @type {import('dictionary-importer').ProgressData} */
+ this._progressData = this._createProgressData();
}
+ /**
+ * @param {DictionaryDatabase} dictionaryDatabase
+ * @param {ArrayBuffer} archiveContent
+ * @param {import('dictionary-importer').ImportDetails} details
+ * @returns {Promise<import('dictionary-importer').ImportResult>}
+ */
async importDictionary(dictionaryDatabase, archiveContent, details) {
if (!dictionaryDatabase) {
throw new Error('Invalid database');
@@ -69,9 +83,9 @@ export class DictionaryImporter {
}
const dictionaryTitle = index.title;
- const version = index.format || index.version;
+ const version = typeof index.format === 'number' ? index.format : index.version;
- if (!dictionaryTitle || !index.revision) {
+ if (typeof version !== 'number' || !dictionaryTitle || !index.revision) {
throw new Error('Unrecognized dictionary format');
}
@@ -80,13 +94,6 @@ export class DictionaryImporter {
throw new Error('Dictionary is already imported');
}
- // Data format converters
- const convertTermBankEntry = (version === 1 ? this._convertTermBankEntryV1.bind(this) : this._convertTermBankEntryV3.bind(this));
- const convertTermMetaBankEntry = this._convertTermMetaBankEntry.bind(this);
- const convertKanjiBankEntry = (version === 1 ? this._convertKanjiBankEntryV1.bind(this) : this._convertKanjiBankEntryV3.bind(this));
- const convertKanjiMetaBankEntry = this._convertKanjiMetaBankEntry.bind(this);
- const convertTagBankEntry = this._convertTagBankEntry.bind(this);
-
// Load schemas
this._progressNextStep(0);
const dataBankSchemas = this._getDataBankSchemas(version);
@@ -100,11 +107,19 @@ export class DictionaryImporter {
// Load data
this._progressNextStep(termFiles.length + termMetaFiles.length + kanjiFiles.length + kanjiMetaFiles.length + tagFiles.length);
- const termList = await this._readFileSequence(termFiles, convertTermBankEntry, dataBankSchemas[0], dictionaryTitle);
- const termMetaList = await this._readFileSequence(termMetaFiles, convertTermMetaBankEntry, dataBankSchemas[1], dictionaryTitle);
- const kanjiList = await this._readFileSequence(kanjiFiles, convertKanjiBankEntry, dataBankSchemas[2], dictionaryTitle);
- const kanjiMetaList = await this._readFileSequence(kanjiMetaFiles, convertKanjiMetaBankEntry, dataBankSchemas[3], dictionaryTitle);
- const tagList = await this._readFileSequence(tagFiles, convertTagBankEntry, dataBankSchemas[4], dictionaryTitle);
+ const termList = await (
+ version === 1 ?
+ this._readFileSequence(termFiles, this._convertTermBankEntryV1.bind(this), dataBankSchemas[0], dictionaryTitle) :
+ this._readFileSequence(termFiles, this._convertTermBankEntryV3.bind(this), dataBankSchemas[0], dictionaryTitle)
+ );
+ const termMetaList = await this._readFileSequence(termMetaFiles, this._convertTermMetaBankEntry.bind(this), dataBankSchemas[1], dictionaryTitle);
+ const kanjiList = await (
+ version === 1 ?
+ this._readFileSequence(kanjiFiles, this._convertKanjiBankEntryV1.bind(this), dataBankSchemas[2], dictionaryTitle) :
+ this._readFileSequence(kanjiFiles, this._convertKanjiBankEntryV3.bind(this), dataBankSchemas[2], dictionaryTitle)
+ );
+ const kanjiMetaList = await this._readFileSequence(kanjiMetaFiles, this._convertKanjiMetaBankEntry.bind(this), dataBankSchemas[3], dictionaryTitle);
+ const tagList = await this._readFileSequence(tagFiles, this._convertTagBankEntry.bind(this), dataBankSchemas[4], dictionaryTitle);
this._addOldIndexTags(index, tagList, dictionaryTitle);
// Prefix wildcard support
@@ -119,6 +134,7 @@ export class DictionaryImporter {
// Extended data support
this._progressNextStep(termList.length);
const formatProgressInterval = 1000;
+ /** @type {import('dictionary-importer').ImportRequirement[]} */
const requirements = [];
for (let i = 0, ii = termList.length; i < ii; ++i) {
const entry = termList[i];
@@ -142,6 +158,7 @@ export class DictionaryImporter {
// Add dictionary descriptor
this._progressNextStep(termList.length + termMetaList.length + kanjiList.length + kanjiMetaList.length + tagList.length + media.length);
+ /** @type {import('dictionary-importer').SummaryCounts} */
const counts = {
terms: {total: termList.length},
termMeta: this._getMetaCounts(termMetaList),
@@ -154,9 +171,15 @@ export class DictionaryImporter {
dictionaryDatabase.bulkAdd('dictionaries', [summary], 0, 1);
// Add data
+ /** @type {Error[]} */
const errors = [];
const maxTransactionLength = 1000;
+ /**
+ * @template {import('dictionary-database').ObjectStoreName} T
+ * @param {T} objectStoreName
+ * @param {import('dictionary-database').ObjectStoreData<T>[]} entries
+ */
const bulkAdd = async (objectStoreName, entries) => {
const ii = entries.length;
for (let i = 0; i < ii; i += maxTransactionLength) {
@@ -165,7 +188,7 @@ export class DictionaryImporter {
try {
await dictionaryDatabase.bulkAdd(objectStoreName, entries, i, count);
} catch (e) {
- errors.push(e);
+ errors.push(e instanceof Error ? e : new Error(`${e}`));
}
this._progressData.index += count;
@@ -185,16 +208,27 @@ export class DictionaryImporter {
return {result: summary, errors};
}
- _progressReset() {
- this._progressData = {
+ /**
+ * @returns {import('dictionary-importer').ProgressData}
+ */
+ _createProgressData() {
+ return {
stepIndex: 0,
stepCount: 6,
index: 0,
count: 0
};
+ }
+
+ /** */
+ _progressReset() {
+ this._progressData = this._createProgressData();
this._progress();
}
+ /**
+ * @param {number} count
+ */
_progressNextStep(count) {
++this._progressData.stepIndex;
this._progressData.index = 0;
@@ -202,17 +236,31 @@ export class DictionaryImporter {
this._progress();
}
+ /** */
_progress() {
this._onProgress(this._progressData);
}
+ /**
+ * @param {string} dictionaryTitle
+ * @param {number} version
+ * @param {import('dictionary-data').Index} index
+ * @param {{prefixWildcardsSupported: boolean, counts: import('dictionary-importer').SummaryCounts}} details
+ * @returns {import('dictionary-importer').Summary}
+ */
_createSummary(dictionaryTitle, version, index, details) {
+ const indexSequenced = index.sequenced;
+ const {prefixWildcardsSupported, counts} = details;
+
+ /** @type {import('dictionary-importer').Summary} */
const summary = {
title: dictionaryTitle,
revision: index.revision,
- sequenced: index.sequenced,
+ sequenced: typeof indexSequenced === 'boolean' && indexSequenced,
version,
- importDate: Date.now()
+ importDate: Date.now(),
+ prefixWildcardsSupported,
+ counts
};
const {author, url, description, attribution, frequencyMode} = index;
@@ -222,11 +270,14 @@ export class DictionaryImporter {
if (typeof attribution === 'string') { summary.attribution = attribution; }
if (typeof frequencyMode === 'string') { summary.frequencyMode = frequencyMode; }
- Object.assign(summary, details);
-
return summary;
}
+ /**
+ *
+ * @param schema
+ * @param fileName
+ */
_formatAjvSchemaError(schema, fileName) {
const e2 = new Error(`Dictionary has invalid data in '${fileName}'`);
e2.data = schema.errors;
@@ -234,6 +285,10 @@ export class DictionaryImporter {
return e2;
}
+ /**
+ *
+ * @param version
+ */
_getDataBankSchemas(version) {
const termBank = (
version === 1 ?
@@ -252,6 +307,13 @@ export class DictionaryImporter {
return [termBank, termMetaBank, kanjiBank, kanjiMetaBank, tagBank];
}
+ /**
+ * @param {import('dictionary-data').TermGlossaryText|import('dictionary-data').TermGlossaryImage|import('dictionary-data').TermGlossaryStructuredContent} data
+ * @param {import('dictionary-database').DatabaseTermEntry} entry
+ * @param {import('dictionary-importer').ImportRequirement[]} requirements
+ * @returns {import('dictionary-data').TermGlossary}
+ * @throws {Error}
+ */
_formatDictionaryTermGlossaryObject(data, entry, requirements) {
switch (data.type) {
case 'text':
@@ -261,16 +323,32 @@ export class DictionaryImporter {
case 'structured-content':
return this._formatStructuredContent(data, entry, requirements);
default:
- throw new Error(`Unhandled data type: ${data.type}`);
+ throw new Error(`Unhandled data type: ${/** @type {import('core').SerializableObject} */ (data).type}`);
}
}
+ /**
+ * @param {import('dictionary-data').TermGlossaryImage} data
+ * @param {import('dictionary-database').DatabaseTermEntry} entry
+ * @param {import('dictionary-importer').ImportRequirement[]} requirements
+ * @returns {import('dictionary-data').TermGlossaryImage}
+ */
_formatDictionaryTermGlossaryImage(data, entry, requirements) {
- const target = {};
- requirements.push({type: 'image', target, args: [data, entry]});
+ /** @type {import('dictionary-data').TermGlossaryImage} */
+ const target = {
+ type: 'image',
+ path: '' // Will be populated during requirement resolution
+ };
+ requirements.push({type: 'image', target, source: data, entry});
return target;
}
+ /**
+ * @param {import('dictionary-data').TermGlossaryStructuredContent} data
+ * @param {import('dictionary-database').DatabaseTermEntry} entry
+ * @param {import('dictionary-importer').ImportRequirement[]} requirements
+ * @returns {import('dictionary-data').TermGlossaryStructuredContent}
+ */
_formatStructuredContent(data, entry, requirements) {
const content = this._prepareStructuredContent(data.content, entry, requirements);
return {
@@ -279,6 +357,12 @@ export class DictionaryImporter {
};
}
+ /**
+ * @param {import('structured-content').Content} content
+ * @param {import('dictionary-database').DatabaseTermEntry} entry
+ * @param {import('dictionary-importer').ImportRequirement[]} requirements
+ * @returns {import('structured-content').Content}
+ */
_prepareStructuredContent(content, entry, requirements) {
if (typeof content === 'string' || !(typeof content === 'object' && content !== null)) {
return content;
@@ -301,12 +385,27 @@ export class DictionaryImporter {
return content;
}
+ /**
+ * @param {import('structured-content').ImageElement} content
+ * @param {import('dictionary-database').DatabaseTermEntry} entry
+ * @param {import('dictionary-importer').ImportRequirement[]} requirements
+ * @returns {import('structured-content').ImageElement}
+ */
_prepareStructuredContentImage(content, entry, requirements) {
- const target = {};
- requirements.push({type: 'structured-content-image', target, args: [content, entry]});
+ /** @type {import('structured-content').ImageElement} */
+ const target = {
+ tag: 'img',
+ path: '' // Will be populated during requirement resolution
+ };
+ requirements.push({type: 'structured-content-image', target, source: content, entry});
return target;
}
+ /**
+ *
+ * @param requirements
+ * @param zipEntriesObject
+ */
async _resolveAsyncRequirements(requirements, zipEntriesObject) {
const media = new Map();
const context = {zipEntriesObject, media};
@@ -320,37 +419,65 @@ export class DictionaryImporter {
};
}
+ /**
+ * @param {import('dictionary-importer').ImportRequirementContext} context
+ * @param {import('dictionary-importer').ImportRequirement} requirement
+ */
async _resolveAsyncRequirement(context, requirement) {
- const {type, target, args} = requirement;
- let result;
- switch (type) {
+ switch (requirement.type) {
case 'image':
- result = await this._resolveDictionaryTermGlossaryImage(context, ...args);
+ await this._resolveDictionaryTermGlossaryImage(
+ context,
+ requirement.target,
+ requirement.source,
+ requirement.entry
+ );
break;
case 'structured-content-image':
- result = await this._resolveStructuredContentImage(context, ...args);
+ await this._resolveStructuredContentImage(
+ context,
+ requirement.target,
+ requirement.source,
+ requirement.entry
+ );
break;
default:
return;
}
- Object.assign(target, result);
++this._progressData.index;
this._progress();
}
- async _resolveDictionaryTermGlossaryImage(context, data, entry) {
- return await this._createImageData(context, data, entry, {type: 'image'});
- }
-
- async _resolveStructuredContentImage(context, content, entry) {
- const {verticalAlign, sizeUnits} = content;
- const result = await this._createImageData(context, content, entry, {tag: 'img'});
- if (typeof verticalAlign === 'string') { result.verticalAlign = verticalAlign; }
- if (typeof sizeUnits === 'string') { result.sizeUnits = sizeUnits; }
- return result;
- }
-
- async _createImageData(context, data, entry, attributes) {
+ /**
+ * @param {import('dictionary-importer').ImportRequirementContext} context
+ * @param {import('dictionary-data').TermGlossaryImage} target
+ * @param {import('dictionary-data').TermGlossaryImage} source
+ * @param {import('dictionary-database').DatabaseTermEntry} entry
+ */
+ async _resolveDictionaryTermGlossaryImage(context, target, source, entry) {
+ await this._createImageData(context, target, source, entry);
+ }
+
+ /**
+ * @param {import('dictionary-importer').ImportRequirementContext} context
+ * @param {import('structured-content').ImageElement} target
+ * @param {import('structured-content').ImageElement} source
+ * @param {import('dictionary-database').DatabaseTermEntry} entry
+ */
+ async _resolveStructuredContentImage(context, target, source, entry) {
+ const {verticalAlign, sizeUnits} = source;
+ await this._createImageData(context, target, source, entry);
+ if (typeof verticalAlign === 'string') { target.verticalAlign = verticalAlign; }
+ if (typeof sizeUnits === 'string') { target.sizeUnits = sizeUnits; }
+ }
+
+ /**
+ * @param {import('dictionary-importer').ImportRequirementContext} context
+ * @param {import('structured-content').ImageElementBase} target
+ * @param {import('structured-content').ImageElementBase} source
+ * @param {import('dictionary-database').DatabaseTermEntry} entry
+ */
+ async _createImageData(context, target, source, entry) {
const {
path,
width: preferredWidth,
@@ -363,26 +490,37 @@ export class DictionaryImporter {
background,
collapsed,
collapsible
- } = data;
+ } = source;
const {width, height} = await this._getImageMedia(context, path, entry);
- const newData = Object.assign({}, attributes, {path, width, height});
- if (typeof preferredWidth === 'number') { newData.preferredWidth = preferredWidth; }
- if (typeof preferredHeight === 'number') { newData.preferredHeight = preferredHeight; }
- if (typeof title === 'string') { newData.title = title; }
- if (typeof description === 'string') { newData.description = description; }
- if (typeof pixelated === 'boolean') { newData.pixelated = pixelated; }
- if (typeof imageRendering === 'string') { newData.imageRendering = imageRendering; }
- if (typeof appearance === 'string') { newData.appearance = appearance; }
- if (typeof background === 'boolean') { newData.background = background; }
- if (typeof collapsed === 'boolean') { newData.collapsed = collapsed; }
- if (typeof collapsible === 'boolean') { newData.collapsible = collapsible; }
- return newData;
- }
-
+ target.path = path;
+ target.width = width;
+ target.height = height;
+ if (typeof preferredWidth === 'number') { target.preferredWidth = preferredWidth; }
+ if (typeof preferredHeight === 'number') { target.preferredHeight = preferredHeight; }
+ if (typeof title === 'string') { target.title = title; }
+ if (typeof description === 'string') { target.description = description; }
+ if (typeof pixelated === 'boolean') { target.pixelated = pixelated; }
+ if (typeof imageRendering === 'string') { target.imageRendering = imageRendering; }
+ if (typeof appearance === 'string') { target.appearance = appearance; }
+ if (typeof background === 'boolean') { target.background = background; }
+ if (typeof collapsed === 'boolean') { target.collapsed = collapsed; }
+ if (typeof collapsible === 'boolean') { target.collapsible = collapsible; }
+ }
+
+ /**
+ * @param {import('dictionary-importer').ImportRequirementContext} context
+ * @param {string} path
+ * @param {import('dictionary-database').DatabaseTermEntry} entry
+ * @returns {Promise<import('dictionary-database').MediaDataArrayBufferContent>}
+ */
async _getImageMedia(context, path, entry) {
const {media} = context;
const {dictionary} = entry;
+ /**
+ * @param {string} message
+ * @returns {Error}
+ */
const createError = (message) => {
const {expression, reading} = entry;
const readingSource = reading.length > 0 ? ` (${reading})`: '';
@@ -437,6 +575,10 @@ export class DictionaryImporter {
return mediaData;
}
+ /**
+ * @param {string} url
+ * @returns {Promise<unknown>}
+ */
async _fetchJsonAsset(url) {
const response = await fetch(url, {
method: 'GET',
@@ -452,6 +594,11 @@ export class DictionaryImporter {
return await response.json();
}
+ /**
+ * @param {import('dictionary-data').TermV1} entry
+ * @param {string} dictionary
+ * @returns {import('dictionary-database').DatabaseTermEntry}
+ */
_convertTermBankEntryV1(entry, dictionary) {
let [expression, reading, definitionTags, rules, score, ...glossary] = entry;
expression = this._normalizeTermOrReading(expression);
@@ -459,6 +606,11 @@ export class DictionaryImporter {
return {expression, reading, definitionTags, rules, score, glossary, dictionary};
}
+ /**
+ * @param {import('dictionary-data').TermV3} entry
+ * @param {string} dictionary
+ * @returns {import('dictionary-database').DatabaseTermEntry}
+ */
_convertTermBankEntryV3(entry, dictionary) {
let [expression, reading, definitionTags, rules, score, glossary, sequence, termTags] = entry;
expression = this._normalizeTermOrReading(expression);
@@ -466,40 +618,75 @@ export class DictionaryImporter {
return {expression, reading, definitionTags, rules, score, glossary, sequence, termTags, dictionary};
}
+ /**
+ * @param {import('dictionary-data').TermMeta} entry
+ * @param {string} dictionary
+ * @returns {import('dictionary-database').DatabaseTermMeta}
+ */
_convertTermMetaBankEntry(entry, dictionary) {
const [expression, mode, data] = entry;
- return {expression, mode, data, dictionary};
+ return /** @type {import('dictionary-database').DatabaseTermMeta} */ ({expression, mode, data, dictionary});
}
+ /**
+ * @param {import('dictionary-data').KanjiV1} entry
+ * @param {string} dictionary
+ * @returns {import('dictionary-database').DatabaseKanjiEntry}
+ */
_convertKanjiBankEntryV1(entry, dictionary) {
const [character, onyomi, kunyomi, tags, ...meanings] = entry;
return {character, onyomi, kunyomi, tags, meanings, dictionary};
}
+ /**
+ * @param {import('dictionary-data').KanjiV3} entry
+ * @param {string} dictionary
+ * @returns {import('dictionary-database').DatabaseKanjiEntry}
+ */
_convertKanjiBankEntryV3(entry, dictionary) {
const [character, onyomi, kunyomi, tags, meanings, stats] = entry;
return {character, onyomi, kunyomi, tags, meanings, stats, dictionary};
}
+ /**
+ * @param {import('dictionary-data').KanjiMeta} entry
+ * @param {string} dictionary
+ * @returns {import('dictionary-database').DatabaseKanjiMeta}
+ */
_convertKanjiMetaBankEntry(entry, dictionary) {
const [character, mode, data] = entry;
return {character, mode, data, dictionary};
}
+ /**
+ * @param {import('dictionary-data').Tag} entry
+ * @param {string} dictionary
+ * @returns {import('dictionary-database').Tag}
+ */
_convertTagBankEntry(entry, dictionary) {
const [name, category, order, notes, score] = entry;
return {name, category, order, notes, score, dictionary};
}
+ /**
+ * @param {import('dictionary-data').Index} index
+ * @param {import('dictionary-database').Tag[]} results
+ * @param {string} dictionary
+ */
_addOldIndexTags(index, results, dictionary) {
const {tagMeta} = index;
if (typeof tagMeta !== 'object' || tagMeta === null) { return; }
- for (const name of Object.keys(tagMeta)) {
- const {category, order, notes, score} = tagMeta[name];
+ for (const [name, value] of Object.entries(tagMeta)) {
+ const {category, order, notes, score} = value;
results.push({name, category, order, notes, score, dictionary});
}
}
+ /**
+ *
+ * @param zipEntriesObject
+ * @param fileNameFormat
+ */
_getArchiveFiles(zipEntriesObject, fileNameFormat) {
const indexPosition = fileNameFormat.indexOf('?');
const prefix = fileNameFormat.substring(0, indexPosition);
@@ -513,6 +700,13 @@ export class DictionaryImporter {
return results;
}
+ /**
+ *
+ * @param files
+ * @param convertEntry
+ * @param schemaName
+ * @param dictionaryTitle
+ */
async _readFileSequence(files, convertEntry, schemaName, dictionaryTitle) {
const progressData = this._progressData;
let startIndex = 0;
@@ -534,20 +728,28 @@ export class DictionaryImporter {
progressData.index = startIndex + 1;
this._progress();
- for (const entry of entries) {
- results.push(convertEntry(entry, dictionaryTitle));
+ if (Array.isArray(entries)) {
+ for (const entry of entries) {
+ results.push(convertEntry(/** @type {TEntry} */ (entry), dictionaryTitle));
+ }
}
}
return results;
}
+ /**
+ * @param {import('dictionary-database').DatabaseTermMeta[]|import('dictionary-database').DatabaseKanjiMeta[]} metaList
+ * @returns {import('dictionary-importer').SummaryMetaCount}
+ */
_getMetaCounts(metaList) {
+ /** @type {Map<string, number>} */
const countsMap = new Map();
for (const {mode} of metaList) {
let count = countsMap.get(mode);
count = typeof count !== 'undefined' ? count + 1 : 1;
countsMap.set(mode, count);
}
+ /** @type {import('dictionary-importer').SummaryMetaCount} */
const counts = {total: metaList.length};
for (const [key, value] of countsMap.entries()) {
if (Object.prototype.hasOwnProperty.call(counts, key)) { continue; }
@@ -556,6 +758,10 @@ export class DictionaryImporter {
return counts;
}
+ /**
+ * @param {string} text
+ * @returns {string}
+ */
_normalizeTermOrReading(text) {
// Note: this function should not perform String.normalize on the text,
// as it will characters in an undesirable way.