From 8c4a50f68c9543c14cfe76abd2f6f42135b0e13d Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sat, 31 Jul 2021 14:46:09 -0400 Subject: DictionaryImporterThreaded (#1865) * Create new classes for importing dictionaries from a separate thread * Use threaded importer * Update worker tests --- ext/js/language/dictionary-importer-threaded.js | 85 ++++++++++++++++++++++ ext/js/language/dictionary-importer-worker-main.js | 42 +++++++++++ .../dictionary-importer-worker-media-loader.js | 65 +++++++++++++++++ ext/js/language/dictionary-importer-worker.js | 83 +++++++++++++++++++++ .../pages/settings/dictionary-import-controller.js | 36 +++------ 5 files changed, 286 insertions(+), 25 deletions(-) create mode 100644 ext/js/language/dictionary-importer-threaded.js create mode 100644 ext/js/language/dictionary-importer-worker-main.js create mode 100644 ext/js/language/dictionary-importer-worker-media-loader.js create mode 100644 ext/js/language/dictionary-importer-worker.js (limited to 'ext/js') diff --git a/ext/js/language/dictionary-importer-threaded.js b/ext/js/language/dictionary-importer-threaded.js new file mode 100644 index 00000000..a251906b --- /dev/null +++ b/ext/js/language/dictionary-importer-threaded.js @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2021 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* global + * DictionaryImporterMediaLoader + */ + +class DictionaryImporterThreaded { + importDictionary(archiveContent, details, onProgress) { + return new Promise((resolve, reject) => { + const dictionaryImporterMediaLoader = new DictionaryImporterMediaLoader(); + const worker = new Worker('/js/language/dictionary-importer-worker-main.js', {}); + const onMessage = (e) => { + const {action, params} = e.data; + switch (action) { + case 'complete': + worker.removeEventListener('message', onMessage); + worker.terminate(); + this._onComplete(params, resolve, reject); + break; + case 'progress': + this._onProgress(params, onProgress); + break; + case 'getImageResolution': + this._onGetImageResolution(params, worker, dictionaryImporterMediaLoader); + break; + } + }; + worker.addEventListener('message', onMessage); + worker.postMessage({ + action: 'import', + params: {details, archiveContent} + }, [archiveContent]); + }); + } + + // Private + + _onComplete(params, resolve, reject) { + const {error} = params; + if (typeof error !== 'undefined') { + reject(deserializeError(error)); + } else { + resolve(this._formatResult(params.result)); + } + } + + _formatResult(data) { + const {result, errors} = data; + const errors2 = errors.map((error) => deserializeError(error)); + return {result, errors: errors2}; + } + + _onProgress(params, onProgress) { + if (typeof onProgress !== 'function') { return; } + const {args} = params; + onProgress(...args); + } + + async _onGetImageResolution(params, worker, dictionaryImporterMediaLoader) { + const {id, mediaType, content} = params; + let response; + try { + const result = await dictionaryImporterMediaLoader.getImageResolution(mediaType, content); + response = {id, result}; + } catch (e) { + response = {id, error: serializeError(e)}; + } + worker.postMessage({action: 'getImageResolution.response', params: response}); + } +} diff --git a/ext/js/language/dictionary-importer-worker-main.js b/ext/js/language/dictionary-importer-worker-main.js new file mode 100644 index 00000000..100bb4fb --- /dev/null +++ b/ext/js/language/dictionary-importer-worker-main.js @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2021 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* global + * DictionaryImporterWorker + */ + +self.importScripts( + '/lib/jszip.min.js', + '/js/core.js', + '/js/data/database.js', + '/js/data/json-schema.js', + '/js/general/cache-map.js', + '/js/language/dictionary-database.js', + '/js/language/dictionary-importer.js', + '/js/language/dictionary-importer-worker.js', + '/js/language/dictionary-importer-worker-media-loader.js', + '/js/media/media-util.js' +); + +(() => { + try { + const dictionaryImporterWorker = new DictionaryImporterWorker(); + dictionaryImporterWorker.prepare(); + } catch (e) { + log.error(e); + } +})(); diff --git a/ext/js/language/dictionary-importer-worker-media-loader.js b/ext/js/language/dictionary-importer-worker-media-loader.js new file mode 100644 index 00000000..5d5d3593 --- /dev/null +++ b/ext/js/language/dictionary-importer-worker-media-loader.js @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2021 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/** + * Class used for loading and validating media from a worker thread + * during the dictionary import process. + */ +class DictionaryImporterWorkerMediaLoader { + /** + * Creates a new instance of the media loader. + */ + constructor() { + this._requests = new Map(); + } + + /** + * Handles a response message posted to the worker thread. + * @param params Details of the response. + */ + handleMessage(params) { + const {id} = params; + const request = this._requests.get(id); + if (typeof request === 'undefined') { return; } + this._requests.delete(id); + const {error} = params; + if (typeof error !== 'undefined') { + request.reject(deserializeError(error)); + } else { + request.resolve(params.result); + } + } + + /** + * Attempts to load an image using a base64 encoded content and a media type + * and returns its resolution. + * @param mediaType The media type for the image content. + * @param content The binary content for the image, encoded in base64. + * @returns A Promise which resolves with {width, height} on success, + * otherwise an error is thrown. + */ + getImageResolution(mediaType, content) { + return new Promise((resolve, reject) => { + const id = generateId(16); + this._requests.set(id, {resolve, reject}); + self.postMessage({ + action: 'getImageResolution', + params: {id, mediaType, content} + }); + }); + } +} diff --git a/ext/js/language/dictionary-importer-worker.js b/ext/js/language/dictionary-importer-worker.js new file mode 100644 index 00000000..f44a10f9 --- /dev/null +++ b/ext/js/language/dictionary-importer-worker.js @@ -0,0 +1,83 @@ +/* + * Copyright (C) 2021 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* global + * DictionaryDatabase + * DictionaryImporter + * DictionaryImporterWorkerMediaLoader + */ + +class DictionaryImporterWorker { + constructor() { + this._mediaLoader = new DictionaryImporterWorkerMediaLoader(); + } + + prepare() { + self.addEventListener('message', this._onMessage.bind(this), false); + } + + // Private + + _onMessage(e) { + const {action, params} = e.data; + switch (action) { + case 'import': + this._onImport(params); + break; + case 'getImageResolution.response': + this._mediaLoader.handleMessage(params); + break; + } + } + + async _onImport({details, archiveContent}) { + const onProgress = (...args) => { + self.postMessage({ + action: 'progress', + params: {args} + }); + }; + let response; + try { + const result = await this._importDictionary(archiveContent, details, onProgress); + response = {result}; + } catch (e) { + response = {error: serializeError(e)}; + } + self.postMessage({action: 'complete', params: response}); + } + + async _importDictionary(archiveContent, importDetails, onProgress) { + const dictionaryDatabase = await this._getPreparedDictionaryDatabase(); + try { + const dictionaryImporter = new DictionaryImporter(this._mediaLoader); + const {result, errors} = await dictionaryImporter.importDictionary(dictionaryDatabase, archiveContent, importDetails, onProgress); + return { + result, + errors: errors.map((error) => serializeError(error)) + }; + } finally { + dictionaryDatabase.close(); + } + } + + async _getPreparedDictionaryDatabase() { + const dictionaryDatabase = new DictionaryDatabase(); + await dictionaryDatabase.prepare(); + return dictionaryDatabase; + } +} diff --git a/ext/js/pages/settings/dictionary-import-controller.js b/ext/js/pages/settings/dictionary-import-controller.js index 128e18cb..5e51a48a 100644 --- a/ext/js/pages/settings/dictionary-import-controller.js +++ b/ext/js/pages/settings/dictionary-import-controller.js @@ -17,9 +17,7 @@ /* global * DictionaryController - * DictionaryDatabase - * DictionaryImporter - * DictionaryImporterMediaLoader + * DictionaryImporterThreaded */ class DictionaryImportController { @@ -183,22 +181,16 @@ class DictionaryImportController { } async _importDictionary(file, importDetails, onProgress) { - const dictionaryDatabase = await this._getPreparedDictionaryDatabase(); - try { - const dictionaryImporterMediaLoader = new DictionaryImporterMediaLoader(); - const dictionaryImporter = new DictionaryImporter(dictionaryImporterMediaLoader); - const archiveContent = await this._readFile(file); - const {result, errors} = await dictionaryImporter.importDictionary(dictionaryDatabase, archiveContent, importDetails, onProgress); - yomichan.api.triggerDatabaseUpdated('dictionary', 'import'); - const errors2 = await this._addDictionarySettings(result.sequenced, result.title); - - if (errors.length > 0) { - const allErrors = [...errors, ...errors2]; - allErrors.push(new Error(`Dictionary may not have been imported properly: ${allErrors.length} error${allErrors.length === 1 ? '' : 's'} reported.`)); - this._showErrors(allErrors); - } - } finally { - dictionaryDatabase.close(); + const dictionaryImporter = new DictionaryImporterThreaded(); + const archiveContent = await this._readFile(file); + const {result, errors} = await dictionaryImporter.importDictionary(archiveContent, importDetails, onProgress); + yomichan.api.triggerDatabaseUpdated('dictionary', 'import'); + const errors2 = await this._addDictionarySettings(result.sequenced, result.title); + + if (errors.length > 0) { + const allErrors = [...errors, ...errors2]; + allErrors.push(new Error(`Dictionary may not have been imported properly: ${allErrors.length} error${allErrors.length === 1 ? '' : 's'} reported.`)); + this._showErrors(allErrors); } } @@ -311,12 +303,6 @@ class DictionaryImportController { } } - async _getPreparedDictionaryDatabase() { - const dictionaryDatabase = new DictionaryDatabase(); - await dictionaryDatabase.prepare(); - return dictionaryDatabase; - } - async _modifyGlobalSettings(targets) { const results = await this._settingsController.modifyGlobalSettings(targets); const errors = []; -- cgit v1.2.3