aboutsummaryrefslogtreecommitdiff
path: root/ext/js
diff options
context:
space:
mode:
authortoasted-nutbread <toasted-nutbread@users.noreply.github.com>2021-07-31 19:13:41 -0400
committerGitHub <noreply@github.com>2021-07-31 19:13:41 -0400
commitcd3f47a3595878c2ff7506589ac24ff91ed75eea (patch)
treeb44dfcfdeecabb0ad800b6a5d70f83d0973f5c74 /ext/js
parent01c5c5c04bd9234c64d6f57ee0ea65b0f478a1b7 (diff)
Dictionary import progress improvements (#1868)
* Update loop vars * Update loop * Improve progress reporting during the import process
Diffstat (limited to 'ext/js')
-rw-r--r--ext/js/language/dictionary-importer.js85
-rw-r--r--ext/js/pages/settings/dictionary-import-controller.js47
2 files changed, 106 insertions, 26 deletions
diff --git a/ext/js/language/dictionary-importer.js b/ext/js/language/dictionary-importer.js
index f3e86654..89417ca6 100644
--- a/ext/js/language/dictionary-importer.js
+++ b/ext/js/language/dictionary-importer.js
@@ -24,7 +24,8 @@
class DictionaryImporter {
constructor(mediaLoader, onProgress) {
this._mediaLoader = mediaLoader;
- this._onProgress = onProgress;
+ this._onProgress = typeof onProgress === 'function' ? onProgress : () => {};
+ this._progressData = null;
}
async importDictionary(dictionaryDatabase, archiveContent, details) {
@@ -35,7 +36,7 @@ class DictionaryImporter {
throw new Error('Database is not ready');
}
- const hasOnProgress = (typeof this._onProgress === 'function');
+ this._progressReset();
// Read archive
const archive = await JSZip.loadAsync(archiveContent);
@@ -72,6 +73,7 @@ class DictionaryImporter {
const convertTagBankEntry = this._convertTagBankEntry.bind(this);
// Load schemas
+ this._progressNextStep(0);
const dataBankSchemaPaths = this._getDataBankSchemaPaths(version);
const dataBankSchemas = await Promise.all(dataBankSchemaPaths.map((path) => this._getSchema(path)));
@@ -83,6 +85,7 @@ class DictionaryImporter {
const tagFiles = this._getArchiveFiles(archive, 'tag_bank_?.json');
// Load data
+ this._progressNextStep(termFiles.length + termMetaFiles.length + kanjiFiles.length + kanjiMetaFiles.length + tagFiles.length);
const termList = await this._readFileSequence(termFiles, convertTermBankEntry, dataBankSchemas[0], dictionaryTitle);
const termMetaList = await this._readFileSequence(termMetaFiles, convertTermMetaBankEntry, dataBankSchemas[1], dictionaryTitle);
const kanjiList = await this._readFileSequence(kanjiFiles, convertKanjiBankEntry, dataBankSchemas[2], dictionaryTitle);
@@ -100,17 +103,26 @@ class DictionaryImporter {
}
// Extended data support
+ this._progressNextStep(termList.length);
+ const formatProgressInterval = 1000;
const requirements = [];
- for (const entry of termList) {
+ for (let i = 0, ii = termList.length; i < ii; ++i) {
+ const entry = termList[i];
const glossaryList = entry.glossary;
- for (let i = 0, ii = glossaryList.length; i < ii; ++i) {
- const glossary = glossaryList[i];
+ for (let j = 0, jj = glossaryList.length; j < jj; ++j) {
+ const glossary = glossaryList[j];
if (typeof glossary !== 'object' || glossary === null) { continue; }
- glossaryList[i] = this._formatDictionaryTermGlossaryObject(glossary, entry, requirements);
+ glossaryList[j] = this._formatDictionaryTermGlossaryObject(glossary, entry, requirements);
+ }
+ if ((i % formatProgressInterval) === 0) {
+ this._progressData.index = i;
+ this._progress();
}
}
+ this._progress();
// Async requirements
+ this._progressNextStep(requirements.length);
const {media} = await this._resolveAsyncRequirements(requirements, archive);
// Add dictionary
@@ -119,15 +131,8 @@ class DictionaryImporter {
dictionaryDatabase.bulkAdd('dictionaries', [summary], 0, 1);
// Add data
+ this._progressNextStep(termList.length + termMetaList.length + kanjiList.length + kanjiMetaList.length + tagList.length);
const errors = [];
- const total = (
- termList.length +
- termMetaList.length +
- kanjiList.length +
- kanjiMetaList.length +
- tagList.length
- );
- let loadedCount = 0;
const maxTransactionLength = 1000;
const bulkAdd = async (objectStoreName, entries) => {
@@ -141,10 +146,8 @@ class DictionaryImporter {
errors.push(e);
}
- loadedCount += count;
- if (hasOnProgress) {
- this._onProgress(total, loadedCount);
- }
+ this._progressData.index += count;
+ this._progress();
}
};
@@ -155,9 +158,32 @@ class DictionaryImporter {
await bulkAdd('tagMeta', tagList);
await bulkAdd('media', media);
+ this._progress();
+
return {result: summary, errors};
}
+ _progressReset() {
+ this._progressData = {
+ stepIndex: 0,
+ stepCount: 6,
+ index: 0,
+ count: 0
+ };
+ this._progress();
+ }
+
+ _progressNextStep(count) {
+ ++this._progressData.stepIndex;
+ this._progressData.index = 0;
+ this._progressData.count = count;
+ this._progress();
+ }
+
+ _progress() {
+ this._onProgress(this._progressData);
+ }
+
_createSummary(dictionaryTitle, version, index, details) {
const summary = {
title: dictionaryTitle,
@@ -328,6 +354,8 @@ class DictionaryImporter {
return;
}
Object.assign(target, result);
+ ++this._progressData.index;
+ this._progress();
}
async _resolveDictionaryTermGlossaryImage(context, data, entry) {
@@ -500,10 +528,31 @@ class DictionaryImporter {
}
async _readFileSequence(files, convertEntry, schema, dictionaryTitle) {
+ const progressData = this._progressData;
+ let count = 0;
+ let startIndex = 0;
+ if (typeof this._onProgress === 'function') {
+ schema.progressInterval = 1000;
+ schema.progress = (s) => {
+ const index = s.getValueStackLength() > 1 ? s.getValueStackItem(1).path : 0;
+ progressData.index = startIndex + (index / count);
+ this._progress();
+ };
+ }
+
const results = [];
for (const file of files) {
const entries = JSON.parse(await file.async('string'));
+
+ count = Array.isArray(entries) ? Math.max(entries.length, 1) : 1;
+ startIndex = progressData.index;
+ this._progress();
+
this._validateJsonSchema(entries, schema, file.name);
+
+ progressData.index = startIndex + 1;
+ this._progress();
+
for (const entry of entries) {
results.push(convertEntry(entry, dictionaryTitle));
}
diff --git a/ext/js/pages/settings/dictionary-import-controller.js b/ext/js/pages/settings/dictionary-import-controller.js
index b18eeb6b..addba1fa 100644
--- a/ext/js/pages/settings/dictionary-import-controller.js
+++ b/ext/js/pages/settings/dictionary-import-controller.js
@@ -140,13 +140,28 @@ class DictionaryImportController {
prefixWildcardsSupported: optionsFull.global.database.prefixWildcardsSupported
};
- const onProgress = (total, current) => {
- const percent = (current / total * 100.0);
+ let statusPrefix = '';
+ let stepIndex = -2;
+ const onProgress = (data) => {
+ const {stepIndex: stepIndex2, index, count} = data;
+ if (stepIndex !== stepIndex2) {
+ stepIndex = stepIndex2;
+ const labelText = `${statusPrefix} - Step ${stepIndex2 + 1} of ${data.stepCount}: ${this._getImportLabel(stepIndex2)}...`;
+ for (const label of infoLabels) { label.textContent = labelText; }
+ }
+
+ const percent = count > 0 ? (index / count * 100.0) : 0.0;
const cssString = `${percent}%`;
- const statusString = `${percent.toFixed(0)}%`;
+ const statusString = `${Math.floor(percent).toFixed(0)}%`;
for (const progressBar of progressBars) { progressBar.style.width = cssString; }
for (const label of statusLabels) { label.textContent = statusString; }
- this._triggerStorageChanged();
+
+ switch (stepIndex2) {
+ case -2: // Initialize
+ case 5: // Data import
+ this._triggerStorageChanged();
+ break;
+ }
};
const fileCount = files.length;
@@ -156,10 +171,13 @@ class DictionaryImportController {
importInfo.textContent = `(${i + 1} of ${fileCount})`;
}
- onProgress(1, 0);
-
- const labelText = `Importing dictionary${fileCount > 1 ? ` (${i + 1} of ${fileCount})` : ''}...`;
- for (const label of infoLabels) { label.textContent = labelText; }
+ statusPrefix = `Importing dictionary${fileCount > 1 ? ` (${i + 1} of ${fileCount})` : ''}`;
+ onProgress({
+ stepIndex: -1,
+ stepCount: 6,
+ index: 0,
+ count: 0
+ });
if (statusFooter !== null) { statusFooter.setTaskActive(progressSelector, true); }
await this._importDictionary(files[i], importDetails, onProgress);
@@ -180,6 +198,19 @@ class DictionaryImportController {
}
}
+ _getImportLabel(stepIndex) {
+ switch (stepIndex) {
+ case -1:
+ case 0: return 'Loading dictionary';
+ case 1: return 'Loading schemas';
+ case 2: return 'Validating data';
+ case 3: return 'Processing data';
+ case 4: return 'Post-processing data';
+ case 5: return 'Importing data';
+ default: return '';
+ }
+ }
+
async _importDictionary(file, importDetails, onProgress) {
const dictionaryImporter = new DictionaryImporterThreaded(onProgress);
const archiveContent = await this._readFile(file);