Text scanner improvements (#1056)

* Only ignore nodes on non-web pages

* Fix issue where options might not be assigned on nested frontends

* Refactor default TextScanner options

* Add option to enable search only on click

* Simplify restore state assignment

* Update options context passing

* Fix empty title

* Use TextScanner to scan content inside of Display

* Rename ignoreNodes to excludeSelector(s)

* Fix options update incorrectly triggering a re-search

* Fix copy throwing an error on the search page

* Replace _onSearchQueryUpdated with _search

* Use include selector instead of exclude selector

6 files changed, 214 insertions, 168 deletions

diff --git a/ext/bg/js/query-parser.js b/ext/bg/js/query-parser.js
index 16af77b2..d3065188 100644
--- a/ext/bg/js/query-parser.js
+++ b/ext/bg/js/query-parser.js
@@ -34,8 +34,6 @@ class QueryParser extends EventDispatcher {
         this._queryParserModeSelect = document.querySelector('#query-parser-mode-select');
         this._textScanner = new TextScanner({
             node: this._queryParser,
-            ignoreElements: () => [],
-            ignorePoint: null,
             getOptionsContext,
             documentUtil,
             searchTerms: true,
diff --git a/ext/bg/js/search.js b/ext/bg/js/search.js
index 498f4ade..476370bf 100644
--- a/ext/bg/js/search.js
+++ b/ext/bg/js/search.js
@@ -62,7 +62,7 @@ class DisplaySearch extends Display {
     async prepare() {
         await super.prepare();
         await this.updateOptions();
-        yomichan.on('optionsUpdated', () => this.updateOptions());
+        yomichan.on('optionsUpdated', this._onOptionsUpdated.bind(this));
 
         this.on('contentUpdating', this._onContentUpdating.bind(this));
         this.on('modeChange', this._onModeChange.bind(this));
@@ -126,15 +126,6 @@ class DisplaySearch extends Display {
         }
     }
 
-    async updateOptions() {
-        await super.updateOptions();
-        if (!this._isPrepared) { return; }
-        const query = this._queryInput.value;
-        if (query) {
-            this._onSearchQueryUpdated(query, false);
-        }
-    }
-
     postProcessQuery(query) {
         if (this._wanakanaEnabled) {
             try {
@@ -148,6 +139,14 @@ class DisplaySearch extends Display {
 
     // Private
 
+    async _onOptionsUpdated() {
+        await this.updateOptions();
+        const query = this._queryInput.value;
+        if (query) {
+            this._search(false);
+        }
+    }
+
     _onContentUpdating({type, content, source}) {
         let animate = false;
         let valid = false;
@@ -183,12 +182,12 @@ class DisplaySearch extends Display {
         e.preventDefault();
         e.stopImmediatePropagation();
         this.blurElement(e.currentTarget);
-        this._search();
+        this._search(true);
     }
 
     _onSearch(e) {
         e.preventDefault();
-        this._search();
+        this._search(true);
     }
 
     _onCopy() {
@@ -197,27 +196,8 @@ class DisplaySearch extends Display {
     }
 
     _onExternalSearchUpdate({text, animate=true}) {
-        this._onSearchQueryUpdated(text, animate);
-    }
-
-    _onSearchQueryUpdated(query, animate) {
-        const details = {
-            focus: false,
-            history: false,
-            params: {
-                query
-            },
-            state: {
-                focusEntry: 0,
-                sentence: {text: query, offset: 0},
-                url: window.location.href
-            },
-            content: {
-                definitions: null,
-                animate
-            }
-        };
-        this.setContent(details);
+        this._queryInput.value = text;
+        this._search(animate);
     }
 
     _onWanakanaEnableChange(e) {
@@ -362,9 +342,25 @@ class DisplaySearch extends Display {
         });
     }
 
-    _search() {
+    _search(animate) {
         const query = this._queryInput.value;
-        this._onSearchQueryUpdated(query, true);
+        const details = {
+            focus: false,
+            history: false,
+            params: {
+                query
+            },
+            state: {
+                focusEntry: 0,
+                sentence: {text: query, offset: 0},
+                url: window.location.href
+            },
+            content: {
+                definitions: null,
+                animate
+            }
+        };
+        this.setContent(details);
     }
 
     _updateSearchHeight() {
diff --git a/ext/fg/js/frontend.js b/ext/fg/js/frontend.js
index 6ae3b06d..49c8a91c 100644
--- a/ext/fg/js/frontend.js
+++ b/ext/fg/js/frontend.js
@@ -322,11 +322,13 @@ class Frontend {
         });
         this._updateTextScannerEnabled();
 
-        const ignoreNodes = ['.scan-disable', '.scan-disable *'];
-        if (!this._options.scanning.enableOnPopupExpressions) {
-            ignoreNodes.push('.source-text', '.source-text *');
+        if (this._pageType !== 'web') {
+            const excludeSelectors = ['.scan-disable', '.scan-disable *'];
+            if (!scanningOptions.enableOnPopupExpressions) {
+                excludeSelectors.push('.source-text', '.source-text *');
+            }
+            this._textScanner.excludeSelector = excludeSelectors.join(',');
         }
-        this._textScanner.ignoreNodes = ignoreNodes.join(',');
 
         this._updateContentScale();
 
@@ -527,7 +529,7 @@ class Frontend {
     }
 
     _updateTextScannerEnabled() {
-        const enabled = (this._options.general.enable && !this._disabledOverride);
+        const enabled = (this._options !== null && this._options.general.enable && !this._disabledOverride);
         this._textScanner.setEnabled(enabled);
     }
 
diff --git a/ext/mixed/js/display.js b/ext/mixed/js/display.js
index e051893e..acab9345 100644
--- a/ext/mixed/js/display.js
+++ b/ext/mixed/js/display.js
@@ -27,6 +27,7 @@
  * PopupFactory
  * QueryParser
  * TemplateRendererProxy
+ * TextScanner
  * WindowScroll
  * api
  * dynamicLoader
@@ -48,7 +49,6 @@ class Display extends EventDispatcher {
         });
         this._styleNode = null;
         this._eventListeners = new EventListenerCollection();
-        this._clickScanPrevent = false;
         this._setContentToken = null;
         this._autoPlayAudioTimer = null;
         this._autoPlayAudioDelay = 400;
@@ -104,6 +104,7 @@ class Display extends EventDispatcher {
         this._frameEndpoint = (pageType === 'popup' ? new FrameEndpoint() : null);
         this._browser = null;
         this._copyTextarea = null;
+        this._definitionTextScanner = null;
 
         this.registerActions([
             ['close',             () => { this.onEscape(); }],
@@ -311,6 +312,7 @@ class Display extends EventDispatcher {
         });
 
         this._updateNestedFrontend(options);
+        this._updateDefinitionTextScanner(options);
     }
 
     autoPlayAudio() {
@@ -348,6 +350,7 @@ class Display extends EventDispatcher {
         const url = `${location.protocol}//${location.host}${location.pathname}?${urlSearchParams.toString()}`;
 
         if (history && this._historyHasChanged) {
+            this._updateHistoryState();
             this._history.pushState(state, content, url);
         } else {
             this._history.clear();
@@ -648,24 +651,18 @@ class Display extends EventDispatcher {
             e.preventDefault();
             if (!this._historyHasState()) { return; }
 
-            const link = e.target;
-            const {state} = this._history;
-
-            state.focusEntry = this._getClosestDefinitionIndex(link);
-            state.scrollX = this._windowScroll.x;
-            state.scrollY = this._windowScroll.y;
-            this._historyStateUpdate(state);
-
-            const query = link.textContent;
-            const definitions = await api.kanjiFind(query, this.getOptionsContext());
+            const {state: {sentence}} = this._history;
+            const optionsContext = this.getOptionsContext();
+            const query = e.currentTarget.textContent;
+            const definitions = await api.kanjiFind(query, optionsContext);
             const details = {
                 focus: false,
                 history: true,
                 params: this._createSearchParams('kanji', query, false),
                 state: {
                     focusEntry: 0,
-                    sentence: state.sentence,
-                    optionsContext: state.optionsContext
+                    sentence,
+                    optionsContext
                 },
                 content: {
                     definitions
@@ -677,88 +674,6 @@ class Display extends EventDispatcher {
         }
     }
 
-    _onGlossaryMouseDown(e) {
-        if (DocumentUtil.isMouseButtonPressed(e, 'primary')) {
-            this._clickScanPrevent = false;
-        }
-    }
-
-    _onGlossaryMouseMove() {
-        this._clickScanPrevent = true;
-    }
-
-    _onGlossaryMouseUp(e) {
-        if (!this._clickScanPrevent && DocumentUtil.isMouseButtonPressed(e, 'primary')) {
-            try {
-                this._onTermLookup(e);
-            } catch (error) {
-                this.onError(error);
-            }
-        }
-    }
-
-    async _onTermLookup(e) {
-        if (!this._historyHasState()) { return; }
-
-        const termLookupResults = await this._termLookup(e);
-        if (!termLookupResults || !this._historyHasState()) { return; }
-
-        const {state} = this._history;
-        const {textSource, definitions} = termLookupResults;
-
-        const scannedElement = e.target;
-        const sentenceExtent = this._options.anki.sentenceExt;
-        const layoutAwareScan = this._options.scanning.layoutAwareScan;
-        const sentence = this._documentUtil.extractSentence(textSource, sentenceExtent, layoutAwareScan);
-
-        state.focusEntry = this._getClosestDefinitionIndex(scannedElement);
-        state.scrollX = this._windowScroll.x;
-        state.scrollY = this._windowScroll.y;
-        this._historyStateUpdate(state);
-
-        const query = textSource.text();
-        const details = {
-            focus: false,
-            history: true,
-            params: this._createSearchParams('terms', query, false),
-            state: {
-                focusEntry: 0,
-                sentence,
-                optionsContext: state.optionsContext
-            },
-            content: {
-                definitions
-            }
-        };
-        this.setContent(details);
-    }
-
-    async _termLookup(e) {
-        e.preventDefault();
-
-        const {length: scanLength, deepDomScan: deepScan, layoutAwareScan} = this._options.scanning;
-        const textSource = this._documentUtil.getRangeFromPoint(e.clientX, e.clientY, deepScan);
-        if (textSource === null) {
-            return false;
-        }
-
-        let definitions, length;
-        try {
-            textSource.setEndOffset(scanLength, layoutAwareScan);
-
-            ({definitions, length} = await api.termsFind(textSource.text(), {}, this.getOptionsContext()));
-            if (definitions.length === 0) {
-                return false;
-            }
-
-            textSource.setEndOffset(length, layoutAwareScan);
-        } finally {
-            textSource.cleanup();
-        }
-
-        return {textSource, definitions};
-    }
-
     _onAudioPlay(e) {
         e.preventDefault();
         const link = e.currentTarget;
@@ -942,7 +857,7 @@ class Display extends EventDispatcher {
         if (this._setContentToken !== token) { return true; }
 
         if (changeHistory) {
-            this._historyStateUpdate(state, content);
+            this._replaceHistoryStateNoNavigate(state, content);
         }
 
         eventArgs.source = source;
@@ -1054,17 +969,17 @@ class Display extends EventDispatcher {
     }
 
     _setTitleText(text) {
-        let title = '';
+        let title = this._defaultTitle;
         if (text.length > 0) {
             // Chrome limits title to 1024 characters
             const ellipsis = '...';
             const separator = ' - ';
-            const maxLength = this._titleMaxLength - this._defaultTitle.length - separator.length;
+            const maxLength = this._titleMaxLength - title.length - separator.length;
             if (text.length > maxLength) {
                 text = `${text.substring(0, Math.max(0, maxLength - ellipsis.length))}${ellipsis}`;
             }
 
-            title = `${text}${separator}${this._defaultTitle}`;
+            title = `${text}${separator}${title}`;
         }
         document.title = title;
     }
@@ -1384,12 +1299,20 @@ class Display extends EventDispatcher {
         return isObject(this._history.state);
     }
 
-    _historyStateUpdate(state, content) {
+    _updateHistoryState() {
+        const {state, content} = this._history;
+        if (!isObject(state)) { return; }
+
+        state.focusEntry = this._index;
+        state.scrollX = this._windowScroll.x;
+        state.scrollY = this._windowScroll.y;
+        this._replaceHistoryStateNoNavigate(state, content);
+    }
+
+    _replaceHistoryStateNoNavigate(state, content) {
         const historyChangeIgnorePre = this._historyChangeIgnore;
         try {
             this._historyChangeIgnore = true;
-            if (typeof state === 'undefined') { state = this._history.state; }
-            if (typeof content === 'undefined') { content = this._history.content; }
             this._history.replaceState(state, content);
         } finally {
             this._historyChangeIgnore = historyChangeIgnorePre;
@@ -1702,7 +1625,7 @@ class Display extends EventDispatcher {
     }
 
     _copyHostSelection() {
-        if (window.getSelection().toString()) { return false; }
+        if (this._ownerFrameId === null || window.getSelection().toString()) { return false; }
         this._copyHostSelectionInner();
         return true;
     }
@@ -1766,10 +1689,89 @@ class Display extends EventDispatcher {
         this._addMultipleEventListeners(entry, '.action-play-audio', 'click', this._onAudioPlay.bind(this));
         this._addMultipleEventListeners(entry, '.kanji-link', 'click', this._onKanjiLookup.bind(this));
         this._addMultipleEventListeners(entry, '.debug-log-link', 'click', this._onDebugLogClick.bind(this));
-        if (this._options !== null && this._options.scanning.enablePopupSearch) {
-            this._addMultipleEventListeners(entry, '.term-glossary-item,.tag', 'mouseup', this._onGlossaryMouseUp.bind(this));
-            this._addMultipleEventListeners(entry, '.term-glossary-item,.tag', 'mousedown', this._onGlossaryMouseDown.bind(this));
-            this._addMultipleEventListeners(entry, '.term-glossary-item,.tag', 'mousemove', this._onGlossaryMouseMove.bind(this));
+    }
+
+    _updateDefinitionTextScanner(options) {
+        if (!options.scanning.enablePopupSearch) {
+            if (this._definitionTextScanner !== null) {
+                this._definitionTextScanner.setEnabled(false);
+            }
+            return;
         }
+
+        if (this._definitionTextScanner === null) {
+            this._definitionTextScanner = new TextScanner({
+                node: window,
+                getOptionsContext: this.getOptionsContext.bind(this),
+                documentUtil: this._documentUtil,
+                searchTerms: true,
+                searchKanji: false,
+                searchOnClick: true,
+                searchOnClickOnly: true
+            });
+            this._definitionTextScanner.prepare();
+            this._definitionTextScanner.on('searched', this._onDefinitionTextScannerSearched.bind(this));
+        }
+
+        const scanningOptions = options.scanning;
+        this._definitionTextScanner.setOptions({
+            inputs: [{
+                include: 'mouse0',
+                exclude: '',
+                types: {mouse: true, pen: false, touch: false},
+                options: {
+                    searchTerms: true,
+                    searchKanji: true,
+                    scanOnTouchMove: false,
+                    scanOnPenHover: false,
+                    scanOnPenPress: false,
+                    scanOnPenRelease: false,
+                    preventTouchScrolling: false
+                }
+            }],
+            deepContentScan: scanningOptions.deepDomScan,
+            selectText: false,
+            delay: scanningOptions.delay,
+            touchInputEnabled: false,
+            pointerEventsEnabled: false,
+            scanLength: scanningOptions.length,
+            sentenceExtent: options.anki.sentenceExt,
+            layoutAwareScan: scanningOptions.layoutAwareScan,
+            preventMiddleMouse: false
+        });
+
+        const includeSelector = '.term-glossary-item,.term-glossary-item *,.tag,.tag *';
+        this._definitionTextScanner.includeSelector = includeSelector;
+
+        this._definitionTextScanner.setEnabled(true);
+    }
+
+    _onDefinitionTextScannerSearched({type, definitions, sentence, textSource, optionsContext, error}) {
+        if (error !== null && !yomichan.isExtensionUnloaded) {
+            yomichan.logError(error);
+        }
+
+        if (type === null) { return; }
+
+        const query = textSource.text();
+        const details = {
+            focus: false,
+            history: true,
+            params: {
+                type,
+                query,
+                wildcards: 'off'
+            },
+            state: {
+                focusEntry: 0,
+                sentence,
+                optionsContext
+            },
+            content: {
+                definitions
+            }
+        };
+        this._definitionTextScanner.clearSelection(true);
+        this.setContent(details);
     }
 }
diff --git a/ext/mixed/js/document-util.js b/ext/mixed/js/document-util.js
index da27a75d..611ff98c 100644
--- a/ext/mixed/js/document-util.js
+++ b/ext/mixed/js/document-util.js
@@ -261,6 +261,18 @@ class DocumentUtil {
         return false;
     }
 
+    static everyNodeMatchesSelector(nodes, selector) {
+        const ELEMENT_NODE = Node.ELEMENT_NODE;
+        for (let node of nodes) {
+            while (true) {
+                if (node === null) { return false; }
+                if (node.nodeType === ELEMENT_NODE && node.matches(selector)) { break; }
+                node = node.parentNode;
+            }
+        }
+        return true;
+    }
+
     static getModifierKeys(os) {
         switch (os) {
             case 'win':
diff --git a/ext/mixed/js/text-scanner.js b/ext/mixed/js/text-scanner.js
index 66d37c93..f0903370 100644
--- a/ext/mixed/js/text-scanner.js
+++ b/ext/mixed/js/text-scanner.js
@@ -21,19 +21,31 @@
  */
 
 class TextScanner extends EventDispatcher {
-    constructor({node, ignoreElements, ignorePoint, documentUtil, getOptionsContext, searchTerms=false, searchKanji=false, searchOnClick=false}) {
+    constructor({
+        node,
+        documentUtil,
+        getOptionsContext,
+        ignoreElements=null,
+        ignorePoint=null,
+        searchTerms=false,
+        searchKanji=false,
+        searchOnClick=false,
+        searchOnClickOnly=false
+    }) {
         super();
         this._node = node;
-        this._ignoreElements = ignoreElements;
-        this._ignorePoint = ignorePoint;
         this._documentUtil = documentUtil;
         this._getOptionsContext = getOptionsContext;
+        this._ignoreElements = ignoreElements;
+        this._ignorePoint = ignorePoint;
         this._searchTerms = searchTerms;
         this._searchKanji = searchKanji;
         this._searchOnClick = searchOnClick;
+        this._searchOnClickOnly = searchOnClickOnly;
 
         this._isPrepared = false;
-        this._ignoreNodes = null;
+        this._includeSelector = null;
+        this._excludeSelector = null;
 
         this._inputInfoCurrent = null;
         this._scanTimerPromise = null;
@@ -76,12 +88,20 @@ class TextScanner extends EventDispatcher {
         this._canClearSelection = value;
     }
 
-    get ignoreNodes() {
-        return this._ignoreNodes;
+    get includeSelector() {
+        return this._includeSelector;
+    }
+
+    set includeSelector(value) {
+        this._includeSelector = value;
     }
 
-    set ignoreNodes(value) {
-        this._ignoreNodes = value;
+    get excludeSelector() {
+        return this._excludeSelector;
+    }
+
+    set excludeSelector(value) {
+        this._excludeSelector = value;
     }
 
     prepare() {
@@ -178,15 +198,8 @@ class TextScanner extends EventDispatcher {
 
         clonedTextSource.setEndOffset(length, layoutAwareScan);
 
-        if (this._ignoreNodes !== null) {
-            length = clonedTextSource.text().length;
-            while (
-                length > 0 &&
-                DocumentUtil.anyNodeMatchesSelector(clonedTextSource.getNodesInRange(), this._ignoreNodes)
-            ) {
-                --length;
-                clonedTextSource.setEndOffset(length, layoutAwareScan);
-            }
+        if (this._excludeSelector !== null) {
+            this._constrainTextSource(clonedTextSource, this._includeSelector, this._excludeSelector, layoutAwareScan);
         }
 
         return clonedTextSource.text();
@@ -287,7 +300,7 @@ class TextScanner extends EventDispatcher {
     }
 
     _onMouseOver(e) {
-        if (this._ignoreElements().includes(e.target)) {
+        if (this._ignoreElements !== null && this._ignoreElements().includes(e.target)) {
             this._scanTimerClear();
         }
     }
@@ -613,7 +626,9 @@ class TextScanner extends EventDispatcher {
 
     _hookEvents() {
         let eventListenerInfos;
-        if (this._arePointerEventsSupported()) {
+        if (this._searchOnClickOnly) {
+            eventListenerInfos = this._getMouseClickOnlyEventListeners();
+        } else if (this._arePointerEventsSupported()) {
             eventListenerInfos = this._getPointerEventListeners();
         } else {
             eventListenerInfos = this._getMouseEventListeners();
@@ -652,6 +667,11 @@ class TextScanner extends EventDispatcher {
         ];
     }
 
+    _getMouseClickOnlyEventListeners() {
+        return [
+            [this._node, 'click', this._onClick.bind(this)]
+        ];
+    }
     _getTouchEventListeners() {
         return [
             [this._node, 'auxclick', this._onAuxClick.bind(this)],
@@ -873,4 +893,20 @@ class TextScanner extends EventDispatcher {
         const cachedPointerType = this._pointerIdTypeMap.get(e.pointerId);
         return (typeof cachedPointerType !== 'undefined' ? cachedPointerType : e.pointerType);
     }
+
+    _constrainTextSource(textSource, includeSelector, excludeSelector, layoutAwareScan) {
+        let length = textSource.text().length;
+        while (length > 0) {
+            const nodes = textSource.getNodesInRange();
+            if (
+                (includeSelector !== null && !DocumentUtil.everyNodeMatchesSelector(nodes, includeSelector)) ||
+                (excludeSelector !== null && DocumentUtil.anyNodeMatchesSelector(nodes, excludeSelector))
+            ) {
+                --length;
+                textSource.setEndOffset(length, layoutAwareScan);
+            } else {
+                break;
+            }
+        }
+    }
 }