Merge pull request #325 from jbukl/offscreen-merge23.11.18.0

Move DB into offscreen (on Chrome) to avoid lookup delays caused by service worker startup lag

3 files changed, 307 insertions, 111 deletions

diff --git a/ext/js/background/backend.js b/ext/js/background/backend.js
index e4186d02..bf4841f8 100644
--- a/ext/js/background/backend.js
+++ b/ext/js/background/backend.js
@@ -35,6 +35,7 @@ import {Translator} from '../language/translator.js';
 import {AudioDownloader} from '../media/audio-downloader.js';
 import {MediaUtil} from '../media/media-util.js';
 import {yomitan} from '../yomitan.js';
+import {OffscreenProxy, DictionaryDatabaseProxy, TranslatorProxy, ClipboardReaderProxy} from './offscreen-proxy.js';
 import {ProfileConditionsUtil} from './profile-conditions-util.js';
 import {RequestBuilder} from './request-builder.js';
 import {ScriptManager} from './script-manager.js';
@@ -50,15 +51,15 @@ export class Backend {
     constructor() {
         this._japaneseUtil = new JapaneseUtil(wanakana);
         this._environment = new Environment();
-        this._dictionaryDatabase = new DictionaryDatabase();
-        this._translator = new Translator({
-            japaneseUtil: this._japaneseUtil,
-            database: this._dictionaryDatabase
-        });
         this._anki = new AnkiConnect();
         this._mecab = new Mecab();
 
         if (!chrome.offscreen) {
+            this._dictionaryDatabase = new DictionaryDatabase();
+            this._translator = new Translator({
+                japaneseUtil: this._japaneseUtil,
+                database: this._dictionaryDatabase
+            });
             this._clipboardReader = new ClipboardReader({
                 // eslint-disable-next-line no-undef
                 document: (typeof document === 'object' && document !== null ? document : null),
@@ -66,10 +67,10 @@ export class Backend {
                 richContentPasteTargetSelector: '#clipboard-rich-content-paste-target'
             });
         } else {
-            this._clipboardReader = {
-                getText: this._getTextOffscreen.bind(this),
-                getImage: this._getImageOffscreen.bind(this)
-            };
+            this._offscreen = new OffscreenProxy();
+            this._dictionaryDatabase = new DictionaryDatabaseProxy(this._offscreen);
+            this._translator = new TranslatorProxy(this._offscreen);
+            this._clipboardReader = new ClipboardReaderProxy(this._offscreen);
         }
 
         this._clipboardMonitor = new ClipboardMonitor({
@@ -106,8 +107,6 @@ export class Backend {
         this._permissions = null;
         this._permissionsUtil = new PermissionsUtil();
 
-        this._creatingOffscreen = null;
-
         this._messageHandlers = new Map([
             ['requestBackendReadySignal',    {async: false, contentScript: true,  handler: this._onApiRequestBackendReadySignal.bind(this)}],
             ['optionsGet',                   {async: false, contentScript: true,  handler: this._onApiOptionsGet.bind(this)}],
@@ -230,7 +229,7 @@ export class Backend {
             await this._requestBuilder.prepare();
             await this._environment.prepare();
             if (chrome.offscreen) {
-                await this._setupOffscreenDocument();
+                await this._offscreen.prepare();
             }
             this._clipboardReader.browser = this._environment.getInfo().browser;
 
@@ -754,6 +753,49 @@ export class Backend {
         }
     }
 
+    _onApiOpenCrossFramePort({targetTabId, targetFrameId}, sender) {
+        const sourceTabId = (sender && sender.tab ? sender.tab.id : null);
+        if (typeof sourceTabId !== 'number') {
+            throw new Error('Port does not have an associated tab ID');
+        }
+        const sourceFrameId = sender.frameId;
+        if (typeof sourceFrameId !== 'number') {
+            throw new Error('Port does not have an associated frame ID');
+        }
+
+        const sourceDetails = {
+            name: 'cross-frame-communication-port',
+            otherTabId: targetTabId,
+            otherFrameId: targetFrameId
+        };
+        const targetDetails = {
+            name: 'cross-frame-communication-port',
+            otherTabId: sourceTabId,
+            otherFrameId: sourceFrameId
+        };
+        let sourcePort = chrome.tabs.connect(sourceTabId, {frameId: sourceFrameId, name: JSON.stringify(sourceDetails)});
+        let targetPort = chrome.tabs.connect(targetTabId, {frameId: targetFrameId, name: JSON.stringify(targetDetails)});
+
+        const cleanup = () => {
+            this._checkLastError(chrome.runtime.lastError);
+            if (targetPort !== null) {
+                targetPort.disconnect();
+                targetPort = null;
+            }
+            if (sourcePort !== null) {
+                sourcePort.disconnect();
+                sourcePort = null;
+            }
+        };
+
+        sourcePort.onMessage.addListener((message) => { targetPort.postMessage(message); });
+        targetPort.onMessage.addListener((message) => { sourcePort.postMessage(message); });
+        sourcePort.onDisconnect.addListener(cleanup);
+        targetPort.onDisconnect.addListener(cleanup);
+
+        return {targetTabId, targetFrameId};
+    }
+
     // Command handlers
 
     async _onCommandOpenSearchPage(params) {
@@ -1624,20 +1666,6 @@ export class Backend {
         return await (json ? response.json() : response.text());
     }
 
-    _sendMessagePromise(...args) {
-        return new Promise((resolve, reject) => {
-            const callback = (response) => {
-                try {
-                    resolve(this._getMessageResponseResult(response));
-                } catch (error) {
-                    reject(error);
-                }
-            };
-
-            chrome.runtime.sendMessage(...args, callback);
-        });
-    }
-
     _sendMessageIgnoreResponse(...args) {
         const callback = () => this._checkLastError(chrome.runtime.lastError);
         chrome.runtime.sendMessage(...args, callback);
@@ -2239,87 +2267,4 @@ export class Backend {
         }
         return results;
     }
-
-    async _getTextOffscreen(useRichText) {
-        return this._sendMessagePromise({action: 'clipboardGetTextOffscreen', params: {useRichText}});
-    }
-
-    async _getImageOffscreen() {
-        return this._sendMessagePromise({action: 'clipboardGetImageOffscreen'});
-    }
-
-    _onApiOpenCrossFramePort({targetTabId, targetFrameId}, sender) {
-        const sourceTabId = (sender && sender.tab ? sender.tab.id : null);
-        if (typeof sourceTabId !== 'number') {
-            throw new Error('Port does not have an associated tab ID');
-        }
-        const sourceFrameId = sender.frameId;
-        if (typeof sourceFrameId !== 'number') {
-            throw new Error('Port does not have an associated frame ID');
-        }
-
-        const sourceDetails = {
-            name: 'cross-frame-communication-port',
-            otherTabId: targetTabId,
-            otherFrameId: targetFrameId
-        };
-        const targetDetails = {
-            name: 'cross-frame-communication-port',
-            otherTabId: sourceTabId,
-            otherFrameId: sourceFrameId
-        };
-        let sourcePort = chrome.tabs.connect(sourceTabId, {frameId: sourceFrameId, name: JSON.stringify(sourceDetails)});
-        let targetPort = chrome.tabs.connect(targetTabId, {frameId: targetFrameId, name: JSON.stringify(targetDetails)});
-
-        const cleanup = () => {
-            this._checkLastError(chrome.runtime.lastError);
-            if (targetPort !== null) {
-                targetPort.disconnect();
-                targetPort = null;
-            }
-            if (sourcePort !== null) {
-                sourcePort.disconnect();
-                sourcePort = null;
-            }
-        };
-
-        sourcePort.onMessage.addListener((message) => { targetPort.postMessage(message); });
-        targetPort.onMessage.addListener((message) => { sourcePort.postMessage(message); });
-        sourcePort.onDisconnect.addListener(cleanup);
-        targetPort.onDisconnect.addListener(cleanup);
-
-        return {targetTabId, targetFrameId};
-    }
-
-    // https://developer.chrome.com/docs/extensions/reference/offscreen/
-    async _setupOffscreenDocument() {
-        if (await this._hasOffscreenDocument()) {
-            return;
-        }
-        if (this._creatingOffscreen) {
-            await this._creatingOffscreen;
-            return;
-        }
-
-        this._creatingOffscreen = chrome.offscreen.createDocument({
-            url: 'offscreen.html',
-            reasons: ['CLIPBOARD'],
-            justification: 'Access to the clipboard'
-        });
-        await this._creatingOffscreen;
-        this._creatingOffscreen = null;
-    }
-    async _hasOffscreenDocument() {
-        const offscreenUrl = chrome.runtime.getURL('offscreen.html');
-        if (!chrome.runtime.getContexts) { // chrome version <116
-            const matchedClients = await clients.matchAll();
-            return await matchedClients.some((client) => client.url === offscreenUrl);
-        }
-
-        const contexts = await chrome.runtime.getContexts({
-            contextTypes: ['OFFSCREEN_DOCUMENT'],
-            documentUrls: [offscreenUrl]
-        });
-        return !!contexts.length;
-    }
 }
diff --git a/ext/js/background/offscreen-proxy.js b/ext/js/background/offscreen-proxy.js
new file mode 100644
index 00000000..c01f523d
--- /dev/null
+++ b/ext/js/background/offscreen-proxy.js
@@ -0,0 +1,172 @@
+/*
+ * Copyright (C) 2023  Yomitan Authors
+ * Copyright (C) 2016-2022  Yomichan Authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import {deserializeError, isObject} from '../core.js';
+import {ArrayBufferUtil} from '../data/sandbox/array-buffer-util.js';
+
+export class OffscreenProxy {
+    constructor() {
+        this._creatingOffscreen = null;
+    }
+
+    // https://developer.chrome.com/docs/extensions/reference/offscreen/
+    async prepare() {
+        if (await this._hasOffscreenDocument()) {
+            return;
+        }
+        if (this._creatingOffscreen) {
+            await this._creatingOffscreen;
+            return;
+        }
+
+        this._creatingOffscreen = chrome.offscreen.createDocument({
+            url: 'offscreen.html',
+            reasons: ['CLIPBOARD'],
+            justification: 'Access to the clipboard'
+        });
+        await this._creatingOffscreen;
+        this._creatingOffscreen = null;
+    }
+
+    async _hasOffscreenDocument() {
+        const offscreenUrl = chrome.runtime.getURL('offscreen.html');
+        if (!chrome.runtime.getContexts) { // chrome version <116
+            const matchedClients = await clients.matchAll();
+            return await matchedClients.some((client) => client.url === offscreenUrl);
+        }
+
+        const contexts = await chrome.runtime.getContexts({
+            contextTypes: ['OFFSCREEN_DOCUMENT'],
+            documentUrls: [offscreenUrl]
+        });
+        return !!contexts.length;
+    }
+
+    sendMessagePromise(...args) {
+        return new Promise((resolve, reject) => {
+            const callback = (response) => {
+                try {
+                    resolve(this._getMessageResponseResult(response));
+                } catch (error) {
+                    reject(error);
+                }
+            };
+
+            chrome.runtime.sendMessage(...args, callback);
+        });
+    }
+
+    _getMessageResponseResult(response) {
+        let error = chrome.runtime.lastError;
+        if (error) {
+            throw new Error(error.message);
+        }
+        if (!isObject(response)) {
+            throw new Error('Offscreen document did not respond');
+        }
+        error = response.error;
+        if (error) {
+            throw deserializeError(error);
+        }
+        return response.result;
+    }
+}
+
+export class DictionaryDatabaseProxy {
+    constructor(offscreen) {
+        this._offscreen = offscreen;
+    }
+
+    prepare() {
+        return this._offscreen.sendMessagePromise({action: 'databasePrepareOffscreen'});
+    }
+
+    getDictionaryInfo() {
+        return this._offscreen.sendMessagePromise({action: 'getDictionaryInfoOffscreen'});
+    }
+
+    purge() {
+        return this._offscreen.sendMessagePromise({action: 'databasePurgeOffscreen'});
+    }
+
+    async getMedia(targets) {
+        const serializedMedia = await this._offscreen.sendMessagePromise({action: 'databaseGetMediaOffscreen', params: {targets}});
+        const media = serializedMedia.map((m) => ({...m, content: ArrayBufferUtil.base64ToArrayBuffer(m.content)}));
+        return media;
+    }
+}
+
+export class TranslatorProxy {
+    constructor(offscreen) {
+        this._offscreen = offscreen;
+    }
+
+    prepare(deinflectionReasons) {
+        return this._offscreen.sendMessagePromise({action: 'translatorPrepareOffscreen', params: {deinflectionReasons}});
+    }
+
+    async findKanji(text, findKanjiOptions) {
+        const enabledDictionaryMapList = [...findKanjiOptions.enabledDictionaryMap];
+        const modifiedKanjiOptions = {
+            ...findKanjiOptions,
+            enabledDictionaryMap: enabledDictionaryMapList
+        };
+        return this._offscreen.sendMessagePromise({action: 'findKanjiOffscreen', params: {text, findKanjiOptions: modifiedKanjiOptions}});
+    }
+
+    async findTerms(mode, text, findTermsOptions) {
+        const {enabledDictionaryMap, excludeDictionaryDefinitions, textReplacements} = findTermsOptions;
+        const enabledDictionaryMapList = [...enabledDictionaryMap];
+        const excludeDictionaryDefinitionsList = excludeDictionaryDefinitions ? [...excludeDictionaryDefinitions] : null;
+        const textReplacementsSerialized = textReplacements.map((group) => {
+            if (!group) {
+                return group;
+            }
+            return group.map((opt) => ({...opt, pattern: opt.pattern.toString()}));
+        });
+        const modifiedFindTermsOptions = {
+            ...findTermsOptions,
+            enabledDictionaryMap: enabledDictionaryMapList,
+            excludeDictionaryDefinitions: excludeDictionaryDefinitionsList,
+            textReplacements: textReplacementsSerialized
+        };
+        return this._offscreen.sendMessagePromise({action: 'findTermsOffscreen', params: {mode, text, findTermsOptions: modifiedFindTermsOptions}});
+    }
+
+    async getTermFrequencies(termReadingList, dictionaries) {
+        return this._offscreen.sendMessagePromise({action: 'getTermFrequenciesOffscreen', params: {termReadingList, dictionaries}});
+    }
+
+    clearDatabaseCaches() {
+        return this._offscreen.sendMessagePromise({action: 'clearDatabaseCachesOffscreen'});
+    }
+}
+
+export class ClipboardReaderProxy {
+    constructor(offscreen) {
+        this._offscreen = offscreen;
+    }
+
+    async getText(useRichText) {
+        return this._offscreen.sendMessagePromise({action: 'clipboardGetTextOffscreen', params: {useRichText}});
+    }
+
+    async getImage() {
+        return this._offscreen.sendMessagePromise({action: 'clipboardGetImageOffscreen'});
+    }
+}
diff --git a/ext/js/background/offscreen.js b/ext/js/background/offscreen.js
index 84ad6141..27cee8c4 100644
--- a/ext/js/background/offscreen.js
+++ b/ext/js/background/offscreen.js
@@ -16,8 +16,13 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
+import * as wanakana from '../../lib/wanakana.js';
 import {ClipboardReader} from '../comm/clipboard-reader.js';
 import {invokeMessageHandler} from '../core.js';
+import {ArrayBufferUtil} from '../data/sandbox/array-buffer-util.js';
+import {DictionaryDatabase} from '../language/dictionary-database.js';
+import {JapaneseUtil} from '../language/sandbox/japanese-util.js';
+import {Translator} from '../language/translator.js';
 import {yomitan} from '../yomitan.js';
 
 /**
@@ -29,6 +34,12 @@ export class Offscreen {
      * Creates a new instance.
      */
     constructor() {
+        this._japaneseUtil = new JapaneseUtil(wanakana);
+        this._dictionaryDatabase = new DictionaryDatabase();
+        this._translator = new Translator({
+            japaneseUtil: this._japaneseUtil,
+            database: this._dictionaryDatabase
+        });
         this._clipboardReader = new ClipboardReader({
             // eslint-disable-next-line no-undef
             document: (typeof document === 'object' && document !== null ? document : null),
@@ -37,12 +48,24 @@ export class Offscreen {
         });
 
         this._messageHandlers = new Map([
-            ['clipboardGetTextOffscreen',                 {async: true,  contentScript: true,  handler: this._getTextHandler.bind(this)}],
-            ['clipboardGetImageOffscreen',                 {async: true,  contentScript: true,  handler: this._getImageHandler.bind(this)}]
+            ['clipboardGetTextOffscreen',    {async: true,  contentScript: true,  handler: this._getTextHandler.bind(this)}],
+            ['clipboardGetImageOffscreen',   {async: true,  contentScript: true,  handler: this._getImageHandler.bind(this)}],
+            ['databasePrepareOffscreen',     {async: true,  contentScript: true,  handler: this._prepareDatabaseHandler.bind(this)}],
+            ['getDictionaryInfoOffscreen',   {async: true,  contentScript: true,  handler: this._getDictionaryInfoHandler.bind(this)}],
+            ['databasePurgeOffscreen',       {async: true,  contentScript: true,  handler: this._purgeDatabaseHandler.bind(this)}],
+            ['databaseGetMediaOffscreen',    {async: true,  contentScript: true,  handler: this._getMediaHandler.bind(this)}],
+            ['translatorPrepareOffscreen',   {async: false,  contentScript: true,  handler: this._prepareTranslatorHandler.bind(this)}],
+            ['findKanjiOffscreen',           {async: true,  contentScript: true,  handler: this._findKanjiHandler.bind(this)}],
+            ['findTermsOffscreen',           {async: true,  contentScript: true,  handler: this._findTermsHandler.bind(this)}],
+            ['getTermFrequenciesOffscreen',  {async: true,  contentScript: true,  handler: this._getTermFrequenciesHandler.bind(this)}],
+            ['clearDatabaseCachesOffscreen', {async: false,  contentScript: true,  handler: this._clearDatabaseCachesHandler.bind(this)}]
+
         ]);
 
         const onMessage = this._onMessage.bind(this);
         chrome.runtime.onMessage.addListener(onMessage);
+
+        this._prepareDatabasePromise = null;
     }
 
     _getTextHandler({useRichText}) {
@@ -53,6 +76,62 @@ export class Offscreen {
         return this._clipboardReader.getImage();
     }
 
+    _prepareDatabaseHandler() {
+        if (this._prepareDatabasePromise !== null) {
+            return this._prepareDatabasePromise;
+        }
+        this._prepareDatabasePromise = this._dictionaryDatabase.prepare();
+        return this._prepareDatabasePromise;
+    }
+
+    _getDictionaryInfoHandler() {
+        return this._dictionaryDatabase.getDictionaryInfo();
+    }
+
+    _purgeDatabaseHandler() {
+        return this._dictionaryDatabase.purge();
+    }
+
+    async _getMediaHandler({targets}) {
+        const media = await this._dictionaryDatabase.getMedia(targets);
+        const serializedMedia = media.map((m) => ({...m, content: ArrayBufferUtil.arrayBufferToBase64(m.content)}));
+        return serializedMedia;
+    }
+
+    _prepareTranslatorHandler({deinflectionReasons}) {
+        return this._translator.prepare(deinflectionReasons);
+    }
+
+    _findKanjiHandler({text, findKanjiOptions}) {
+        findKanjiOptions.enabledDictionaryMap = new Map(findKanjiOptions.enabledDictionaryMap);
+        return this._translator.findKanji(text, findKanjiOptions);
+    }
+
+    _findTermsHandler({mode, text, findTermsOptions}) {
+        findTermsOptions.enabledDictionaryMap = new Map(findTermsOptions.enabledDictionaryMap);
+        if (findTermsOptions.excludeDictionaryDefinitions) {
+            findTermsOptions.excludeDictionaryDefinitions = new Set(findTermsOptions.excludeDictionaryDefinitions);
+        }
+        findTermsOptions.textReplacements = findTermsOptions.textReplacements.map((group) => {
+            if (!group) {
+                return group;
+            }
+            return group.map((opt) => {
+                const [, pattern, flags] = opt.pattern.match(/\/(.*?)\/([a-z]*)?$/i); // https://stackoverflow.com/a/33642463
+                return {...opt, pattern: new RegExp(pattern, flags ?? '')};
+            });
+        });
+        return this._translator.findTerms(mode, text, findTermsOptions);
+    }
+
+    _getTermFrequenciesHandler({termReadingList, dictionaries}) {
+        return this._translator.getTermFrequencies(termReadingList, dictionaries);
+    }
+
+    _clearDatabaseCachesHandler() {
+        return this._translator.clearDatabaseCaches();
+    }
+
     _onMessage({action, params}, sender, callback) {
         const messageHandler = this._messageHandlers.get(action);
         if (typeof messageHandler === 'undefined') { return false; }