Merge branch 'master' into testing

7 files changed, 737 insertions, 44 deletions

diff --git a/ext/mixed/js/api.js b/ext/mixed/js/api.js
index 0ab07039..feec94df 100644
--- a/ext/mixed/js/api.js
+++ b/ext/mixed/js/api.js
@@ -53,12 +53,12 @@ function apiKanjiFind(text, optionsContext) {
     return _apiInvoke('kanjiFind', {text, optionsContext});
 }
 
-function apiDefinitionAdd(definition, mode, context, optionsContext) {
-    return _apiInvoke('definitionAdd', {definition, mode, context, optionsContext});
+function apiDefinitionAdd(definition, mode, context, details, optionsContext) {
+    return _apiInvoke('definitionAdd', {definition, mode, context, details, optionsContext});
 }
 
-function apiDefinitionsAddable(definitions, modes, optionsContext) {
-    return _apiInvoke('definitionsAddable', {definitions, modes, optionsContext});
+function apiDefinitionsAddable(definitions, modes, context, optionsContext) {
+    return _apiInvoke('definitionsAddable', {definitions, modes, context, optionsContext});
 }
 
 function apiNoteView(noteId) {
diff --git a/ext/mixed/js/core.js b/ext/mixed/js/core.js
index 0d50e915..db7fc69b 100644
--- a/ext/mixed/js/core.js
+++ b/ext/mixed/js/core.js
@@ -132,6 +132,30 @@ function parseUrl(url) {
     return {baseUrl, queryParams};
 }
 
+function areSetsEqual(set1, set2) {
+    if (set1.size !== set2.size) {
+        return false;
+    }
+
+    for (const value of set1) {
+        if (!set2.has(value)) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+function getSetIntersection(set1, set2) {
+    const result = [];
+    for (const value of set1) {
+        if (set2.has(value)) {
+            result.push(value);
+        }
+    }
+    return result;
+}
+
 
 /*
  * Async utilities
@@ -254,11 +278,16 @@ const yomichan = (() => {
         constructor() {
             super();
 
-            this._isBackendPreparedResolve = null;
-            this._isBackendPreparedPromise = new Promise((resolve) => (this._isBackendPreparedResolve = resolve));
+            this._isBackendPreparedPromise = this.getTemporaryListenerResult(
+                chrome.runtime.onMessage,
+                ({action}, {resolve}) => {
+                    if (action === 'backendPrepared') {
+                        resolve();
+                    }
+                }
+            );
 
             this._messageHandlers = new Map([
-                ['backendPrepared', this._onBackendPrepared.bind(this)],
                 ['getUrl', this._onMessageGetUrl.bind(this)],
                 ['optionsUpdated', this._onMessageOptionsUpdated.bind(this)],
                 ['zoomChanged', this._onMessageZoomChanged.bind(this)]
@@ -288,6 +317,42 @@ const yomichan = (() => {
             this.trigger('orphaned', {error});
         }
 
+        getTemporaryListenerResult(eventHandler, userCallback, timeout=null) {
+            if (!(
+                typeof eventHandler.addListener === 'function' &&
+                typeof eventHandler.removeListener === 'function'
+            )) {
+                throw new Error('Event handler type not supported');
+            }
+
+            return new Promise((resolve, reject) => {
+                const runtimeMessageCallback = ({action, params}, sender, sendResponse) => {
+                    let timeoutId = null;
+                    if (timeout !== null) {
+                        timeoutId = window.setTimeout(() => {
+                            timeoutId = null;
+                            eventHandler.removeListener(runtimeMessageCallback);
+                            reject(new Error(`Listener timed out in ${timeout} ms`));
+                        }, timeout);
+                    }
+
+                    const cleanupResolve = (value) => {
+                        if (timeoutId !== null) {
+                            window.clearTimeout(timeoutId);
+                            timeoutId = null;
+                        }
+                        eventHandler.removeListener(runtimeMessageCallback);
+                        sendResponse();
+                        resolve(value);
+                    };
+
+                    userCallback({action, params}, {resolve: cleanupResolve, sender});
+                };
+
+                eventHandler.addListener(runtimeMessageCallback);
+            });
+        }
+
         // Private
 
         _onMessage({action, params}, sender, callback) {
@@ -299,10 +364,6 @@ const yomichan = (() => {
             return false;
         }
 
-        _onBackendPrepared() {
-            this._isBackendPreparedResolve();
-        }
-
         _onMessageGetUrl() {
             return {url: window.location.href};
         }
diff --git a/ext/mixed/js/display-generator.js b/ext/mixed/js/display-generator.js
index 49afc44b..f1122e3d 100644
--- a/ext/mixed/js/display-generator.js
+++ b/ext/mixed/js/display-generator.js
@@ -19,11 +19,13 @@
 /* global
  * TemplateHandler
  * apiGetDisplayTemplatesHtml
+ * jp
  */
 
 class DisplayGenerator {
     constructor() {
         this._templateHandler = null;
+        this._termPitchAccentStaticTemplateIsSetup = false;
     }
 
     async prepare() {
@@ -36,17 +38,33 @@ class DisplayGenerator {
 
         const expressionsContainer = node.querySelector('.term-expression-list');
         const reasonsContainer = node.querySelector('.term-reasons');
+        const pitchesContainer = node.querySelector('.term-pitch-accent-group-list');
         const frequenciesContainer = node.querySelector('.frequencies');
         const definitionsContainer = node.querySelector('.term-definition-list');
         const debugInfoContainer = node.querySelector('.debug-info');
+        const bodyContainer = node.querySelector('.term-entry-body');
+
+        const pitches = DisplayGenerator._getPitchInfos(details);
+        const pitchCount = pitches.reduce((i, v) => i + v[1].length, 0);
 
         const expressionMulti = Array.isArray(details.expressions);
         const definitionMulti = Array.isArray(details.definitions);
+        const expressionCount = expressionMulti ? details.expressions.length : 1;
+        const definitionCount = definitionMulti ? details.definitions.length : 1;
+        const uniqueExpressionCount = Array.isArray(details.expression) ? new Set(details.expression).size : 1;
 
         node.dataset.expressionMulti = `${expressionMulti}`;
         node.dataset.definitionMulti = `${definitionMulti}`;
-        node.dataset.expressionCount = `${expressionMulti ? details.expressions.length : 1}`;
-        node.dataset.definitionCount = `${definitionMulti ? details.definitions.length : 1}`;
+        node.dataset.expressionCount = `${expressionCount}`;
+        node.dataset.definitionCount = `${definitionCount}`;
+        node.dataset.uniqueExpressionCount = `${uniqueExpressionCount}`;
+        node.dataset.pitchAccentDictionaryCount = `${pitches.length}`;
+        node.dataset.pitchAccentCount = `${pitchCount}`;
+
+        bodyContainer.dataset.sectionCount = `${
+            (definitionCount > 0 ? 1 : 0) +
+            (pitches.length > 0 ? 1 : 0)
+        }`;
 
         const termTags = details.termTags;
         let expressions = details.expressions;
@@ -55,6 +73,7 @@ class DisplayGenerator {
         DisplayGenerator._appendMultiple(expressionsContainer, this.createTermExpression.bind(this), expressions, [[details, termTags]]);
         DisplayGenerator._appendMultiple(reasonsContainer, this.createTermReason.bind(this), details.reasons);
         DisplayGenerator._appendMultiple(frequenciesContainer, this.createFrequencyTag.bind(this), details.frequencies);
+        DisplayGenerator._appendMultiple(pitchesContainer, this.createPitches.bind(this), pitches);
         DisplayGenerator._appendMultiple(definitionsContainer, this.createTermDefinitionItem.bind(this), details.definitions, [details]);
 
         if (debugInfoContainer !== null) {
@@ -261,6 +280,133 @@ class DisplayGenerator {
         return node;
     }
 
+    createPitches(details) {
+        if (!this._termPitchAccentStaticTemplateIsSetup) {
+            this._termPitchAccentStaticTemplateIsSetup = true;
+            const t = this._templateHandler.instantiate('term-pitch-accent-static');
+            document.head.appendChild(t);
+        }
+
+        const [dictionary, dictionaryPitches] = details;
+
+        const node = this._templateHandler.instantiate('term-pitch-accent-group');
+        node.dataset.dictionary = dictionary;
+        node.dataset.pitchesMulti = 'true';
+        node.dataset.pitchesCount = `${dictionaryPitches.length}`;
+
+        const tag = this.createTag({notes: '', name: dictionary, category: 'pitch-accent-dictionary'});
+        node.querySelector('.term-pitch-accent-group-tag-list').appendChild(tag);
+
+        const n = node.querySelector('.term-pitch-accent-list');
+        DisplayGenerator._appendMultiple(n, this.createPitch.bind(this), dictionaryPitches);
+
+        return node;
+    }
+
+    createPitch(details) {
+        const {reading, position, tags, exclusiveExpressions, exclusiveReadings} = details;
+        const morae = jp.getKanaMorae(reading);
+
+        const node = this._templateHandler.instantiate('term-pitch-accent');
+
+        node.dataset.pitchAccentPosition = `${position}`;
+        node.dataset.tagCount = `${tags.length}`;
+
+        let n = node.querySelector('.term-pitch-accent-position');
+        n.textContent = `${position}`;
+
+        n = node.querySelector('.term-pitch-accent-tag-list');
+        DisplayGenerator._appendMultiple(n, this.createTag.bind(this), tags);
+
+        n = node.querySelector('.term-pitch-accent-disambiguation-list');
+        this.createPitchAccentDisambiguations(n, exclusiveExpressions, exclusiveReadings);
+
+        n = node.querySelector('.term-pitch-accent-characters');
+        for (let i = 0, ii = morae.length; i < ii; ++i) {
+            const mora = morae[i];
+            const highPitch = jp.isMoraPitchHigh(i, position);
+            const highPitchNext = jp.isMoraPitchHigh(i + 1, position);
+
+            const n1 = this._templateHandler.instantiate('term-pitch-accent-character');
+            const n2 = n1.querySelector('.term-pitch-accent-character-inner');
+
+            n1.dataset.position = `${i}`;
+            n1.dataset.pitch = highPitch ? 'high' : 'low';
+            n1.dataset.pitchNext = highPitchNext ? 'high' : 'low';
+            n2.textContent = mora;
+
+            n.appendChild(n1);
+        }
+
+        if (morae.length > 0) {
+            this.populatePitchGraph(node.querySelector('.term-pitch-accent-graph'), position, morae);
+        }
+
+        return node;
+    }
+
+    createPitchAccentDisambiguations(container, exclusiveExpressions, exclusiveReadings) {
+        const templateName = 'term-pitch-accent-disambiguation';
+        for (const exclusiveExpression of exclusiveExpressions) {
+            const node = this._templateHandler.instantiate(templateName);
+            node.dataset.type = 'expression';
+            node.textContent = exclusiveExpression;
+            container.appendChild(node);
+        }
+
+        for (const exclusiveReading of exclusiveReadings) {
+            const node = this._templateHandler.instantiate(templateName);
+            node.dataset.type = 'reading';
+            node.textContent = exclusiveReading;
+            container.appendChild(node);
+        }
+
+        container.dataset.multi = 'true';
+        container.dataset.count = `${exclusiveExpressions.length + exclusiveReadings.length}`;
+        container.dataset.expressionCount = `${exclusiveExpressions.length}`;
+        container.dataset.readingCount = `${exclusiveReadings.length}`;
+    }
+
+    populatePitchGraph(svg, position, morae) {
+        const svgns = svg.getAttribute('xmlns');
+        const ii = morae.length;
+        svg.setAttribute('viewBox', `0 0 ${50 * (ii + 1)} 100`);
+
+        const pathPoints = [];
+        for (let i = 0; i < ii; ++i) {
+            const highPitch = jp.isMoraPitchHigh(i, position);
+            const highPitchNext = jp.isMoraPitchHigh(i + 1, position);
+            const graphic = (highPitch && !highPitchNext ? '#term-pitch-accent-graph-dot-downstep' : '#term-pitch-accent-graph-dot');
+            const x = `${i * 50 + 25}`;
+            const y = highPitch ? '25' : '75';
+            const use = document.createElementNS(svgns, 'use');
+            use.setAttribute('href', graphic);
+            use.setAttribute('x', x);
+            use.setAttribute('y', y);
+            svg.appendChild(use);
+            pathPoints.push(`${x} ${y}`);
+        }
+
+        let path = svg.querySelector('.term-pitch-accent-graph-line');
+        path.setAttribute('d', `M${pathPoints.join(' L')}`);
+
+        pathPoints.splice(0, ii - 1);
+        {
+            const highPitch = jp.isMoraPitchHigh(ii, position);
+            const x = `${ii * 50 + 25}`;
+            const y = highPitch ? '25' : '75';
+            const use = document.createElementNS(svgns, 'use');
+            use.setAttribute('href', '#term-pitch-accent-graph-triangle');
+            use.setAttribute('x', x);
+            use.setAttribute('y', y);
+            svg.appendChild(use);
+            pathPoints.push(`${x} ${y}`);
+        }
+
+        path = svg.querySelector('.term-pitch-accent-graph-line-tail');
+        path.setAttribute('d', `M${pathPoints.join(' L')}`);
+    }
+
     createFrequencyTag(details) {
         const node = this._templateHandler.instantiate('tag-frequency');
 
@@ -283,7 +429,7 @@ class DisplayGenerator {
     _appendKanjiLinks(container, text) {
         let part = '';
         for (const c of text) {
-            if (DisplayGenerator._isCharacterKanji(c)) {
+            if (jp.isCodePointKanji(c.codePointAt(0))) {
                 if (part.length > 0) {
                     container.appendChild(document.createTextNode(part));
                     part = '';
@@ -300,30 +446,28 @@ class DisplayGenerator {
         }
     }
 
-    static _isCharacterKanji(c) {
-        const code = c.codePointAt(0);
-        return (
-            code >= 0x4e00 && code < 0x9fb0 ||
-            code >= 0x3400 && code < 0x4dc0
-        );
-    }
-
-    static _appendMultiple(container, createItem, detailsArray, fallback=[]) {
+    static _appendMultiple(container, createItem, detailsIterable, fallback=[]) {
         if (container === null) { return 0; }
 
-        const isArray = Array.isArray(detailsArray);
-        if (!isArray) { detailsArray = fallback; }
-
-        container.dataset.multi = `${isArray}`;
-        container.dataset.count = `${detailsArray.length}`;
+        const multi = (
+            detailsIterable !== null &&
+            typeof detailsIterable === 'object' &&
+            typeof detailsIterable[Symbol.iterator] !== 'undefined'
+        );
+        if (!multi) { detailsIterable = fallback; }
 
-        for (const details of detailsArray) {
+        let count = 0;
+        for (const details of detailsIterable) {
             const item = createItem(details);
             if (item === null) { continue; }
             container.appendChild(item);
+            ++count;
         }
 
-        return detailsArray.length;
+        container.dataset.multi = `${multi}`;
+        container.dataset.count = `${count}`;
+
+        return count;
     }
 
     static _appendFurigana(container, segments, addText) {
@@ -349,4 +493,79 @@ class DisplayGenerator {
             container.appendChild(document.createTextNode(parts[i]));
         }
     }
+
+    static _getPitchInfos(definition) {
+        const results = new Map();
+
+        const allExpressions = new Set();
+        const allReadings = new Set();
+        const expressions = definition.expressions;
+        const sources = Array.isArray(expressions) ? expressions : [definition];
+        for (const {pitches: expressionPitches, expression} of sources) {
+            allExpressions.add(expression);
+            for (const {reading, pitches, dictionary} of expressionPitches) {
+                allReadings.add(reading);
+                let dictionaryResults = results.get(dictionary);
+                if (typeof dictionaryResults === 'undefined') {
+                    dictionaryResults = [];
+                    results.set(dictionary, dictionaryResults);
+                }
+
+                for (const {position, tags} of pitches) {
+                    let pitchInfo = DisplayGenerator._findExistingPitchInfo(reading, position, tags, dictionaryResults);
+                    if (pitchInfo === null) {
+                        pitchInfo = {expressions: new Set(), reading, position, tags};
+                        dictionaryResults.push(pitchInfo);
+                    }
+                    pitchInfo.expressions.add(expression);
+                }
+            }
+        }
+
+        for (const dictionaryResults of results.values()) {
+            for (const result of dictionaryResults) {
+                const exclusiveExpressions = [];
+                const exclusiveReadings = [];
+                const resultExpressions = result.expressions;
+                if (!areSetsEqual(resultExpressions, allExpressions)) {
+                    exclusiveExpressions.push(...getSetIntersection(resultExpressions, allExpressions));
+                }
+                if (allReadings.size > 1) {
+                    exclusiveReadings.push(result.reading);
+                }
+                result.exclusiveExpressions = exclusiveExpressions;
+                result.exclusiveReadings = exclusiveReadings;
+            }
+        }
+
+        return [...results.entries()];
+    }
+
+    static _findExistingPitchInfo(reading, position, tags, pitchInfoList) {
+        for (const pitchInfo of pitchInfoList) {
+            if (
+                pitchInfo.reading === reading &&
+                pitchInfo.position === position &&
+                DisplayGenerator._areTagListsEqual(pitchInfo.tags, tags)
+            ) {
+                return pitchInfo;
+            }
+        }
+        return null;
+    }
+
+    static _areTagListsEqual(tagList1, tagList2) {
+        const ii = tagList1.length;
+        if (tagList2.length !== ii) { return false; }
+
+        for (let i = 0; i < ii; ++i) {
+            const tag1 = tagList1[i];
+            const tag2 = tagList2[i];
+            if (tag1.name !== tag2.name || tag1.dictionary !== tag2.dictionary) {
+                return false;
+            }
+        }
+
+        return true;
+    }
 }
diff --git a/ext/mixed/js/display.js b/ext/mixed/js/display.js
index 515e28a7..2f456c3e 100644
--- a/ext/mixed/js/display.js
+++ b/ext/mixed/js/display.js
@@ -40,6 +40,7 @@ class Display {
         this.spinner = spinner;
         this.container = container;
         this.definitions = [];
+        this.optionsContext = null;
         this.options = null;
         this.context = null;
         this.index = 0;
@@ -165,12 +166,11 @@ class Display {
         this.setInteractive(true);
     }
 
-    async prepare(options=null) {
+    async prepare() {
         await yomichan.prepare();
-        const displayGeneratorPromise = this.displayGenerator.prepare();
-        const updateOptionsPromise = this.updateOptions(options);
-        await Promise.all([displayGeneratorPromise, updateOptionsPromise]);
-        yomichan.on('optionsUpdated', () => this.updateOptions(null));
+        await this.displayGenerator.prepare();
+        await this.updateOptions();
+        yomichan.on('optionsUpdated', () => this.updateOptions());
     }
 
     onError(_error) {
@@ -369,11 +369,11 @@ class Display {
     }
 
     getOptionsContext() {
-        throw new Error('Override me');
+        return this.optionsContext;
     }
 
-    async updateOptions(options) {
-        this.options = options ? options : await apiOptionsGet(this.getOptionsContext());
+    async updateOptions() {
+        this.options = await apiOptionsGet(this.getOptionsContext());
         this.updateDocumentOptions(this.options);
         this.updateTheme(this.options.general.popupTheme);
         this.setCustomCss(this.options.general.customPopupCss);
@@ -385,6 +385,9 @@ class Display {
         data.audioEnabled = `${options.audio.enabled}`;
         data.compactGlossaries = `${options.general.compactGlossaries}`;
         data.enableSearchTags = `${options.scanning.enableSearchTags}`;
+        data.showPitchAccentDownstepNotation = `${options.general.showPitchAccentDownstepNotation}`;
+        data.showPitchAccentPositionNotation = `${options.general.showPitchAccentPositionNotation}`;
+        data.showPitchAccentGraph = `${options.general.showPitchAccentGraph}`;
         data.debug = `${options.general.debugInfo}`;
     }
 
@@ -749,15 +752,16 @@ class Display {
         try {
             this.setSpinnerVisible(true);
 
-            const context = {};
+            const details = {};
             if (this.noteUsesScreenshot(mode)) {
                 const screenshot = await this.getScreenshot();
                 if (screenshot) {
-                    context.screenshot = screenshot;
+                    details.screenshot = screenshot;
                 }
             }
 
-            const noteId = await apiDefinitionAdd(definition, mode, context, this.getOptionsContext());
+            const context = await this._getNoteContext();
+            const noteId = await apiDefinitionAdd(definition, mode, context, details, this.getOptionsContext());
             if (noteId) {
                 const index = this.definitions.indexOf(definition);
                 const adderButton = this.adderButtonFind(index, mode);
@@ -905,12 +909,17 @@ class Display {
 
     async getDefinitionsAddable(definitions, modes) {
         try {
-            return await apiDefinitionsAddable(definitions, modes, this.getOptionsContext());
+            const context = await this._getNoteContext();
+            return await apiDefinitionsAddable(definitions, modes, context, this.getOptionsContext());
         } catch (e) {
             return [];
         }
     }
 
+    async getDocumentTitle() {
+        return document.title;
+    }
+
     static indexOf(nodeList, node) {
         for (let i = 0, ii = nodeList.length; i < ii; ++i) {
             if (nodeList[i] === node) {
@@ -931,6 +940,15 @@ class Display {
         return (typeof key === 'string' ? (key.length === 1 ? key.toUpperCase() : key) : '');
     }
 
+    async _getNoteContext() {
+        const documentTitle = await this.getDocumentTitle();
+        return {
+            document: {
+                title: documentTitle
+            }
+        };
+    }
+
     async _getAudioUri(definition, source) {
         const optionsContext = this.getOptionsContext();
         return await apiAudioGetUri(definition, source, optionsContext);
diff --git a/ext/mixed/js/japanese.js b/ext/mixed/js/japanese.js
new file mode 100644
index 00000000..e6b9a8a0
--- /dev/null
+++ b/ext/mixed/js/japanese.js
@@ -0,0 +1,148 @@
+/*
+ * Copyright (C) 2020  Alex Yatskov <alex@foosoft.net>
+ * Author: Alex Yatskov <alex@foosoft.net>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+const jp = (() => {
+    const HIRAGANA_RANGE = [0x3040, 0x309f];
+    const KATAKANA_RANGE = [0x30a0, 0x30ff];
+    const KANA_RANGES = [HIRAGANA_RANGE, KATAKANA_RANGE];
+
+    const CJK_UNIFIED_IDEOGRAPHS_RANGE = [0x4e00, 0x9fff];
+    const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A_RANGE = [0x3400, 0x4dbf];
+    const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B_RANGE = [0x20000, 0x2a6df];
+    const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_C_RANGE = [0x2a700, 0x2b73f];
+    const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_D_RANGE = [0x2b740, 0x2b81f];
+    const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_E_RANGE = [0x2b820, 0x2ceaf];
+    const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_F_RANGE = [0x2ceb0, 0x2ebef];
+    const CJK_COMPATIBILITY_IDEOGRAPHS_SUPPLEMENT_RANGE = [0x2f800, 0x2fa1f];
+    const CJK_UNIFIED_IDEOGRAPHS_RANGES = [
+        CJK_UNIFIED_IDEOGRAPHS_RANGE,
+        CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A_RANGE,
+        CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B_RANGE,
+        CJK_UNIFIED_IDEOGRAPHS_EXTENSION_C_RANGE,
+        CJK_UNIFIED_IDEOGRAPHS_EXTENSION_D_RANGE,
+        CJK_UNIFIED_IDEOGRAPHS_EXTENSION_E_RANGE,
+        CJK_UNIFIED_IDEOGRAPHS_EXTENSION_F_RANGE,
+        CJK_COMPATIBILITY_IDEOGRAPHS_SUPPLEMENT_RANGE
+    ];
+
+    // Japanese character ranges, roughly ordered in order of expected frequency
+    const JAPANESE_RANGES = [
+        HIRAGANA_RANGE,
+        KATAKANA_RANGE,
+
+        ...CJK_UNIFIED_IDEOGRAPHS_RANGES,
+
+        [0xff66, 0xff9f], // Halfwidth katakana
+
+        [0x30fb, 0x30fc], // Katakana punctuation
+        [0xff61, 0xff65], // Kana punctuation
+        [0x3000, 0x303f], // CJK punctuation
+
+        [0xff10, 0xff19], // Fullwidth numbers
+        [0xff21, 0xff3a], // Fullwidth upper case Latin letters
+        [0xff41, 0xff5a], // Fullwidth lower case Latin letters
+
+        [0xff01, 0xff0f], // Fullwidth punctuation 1
+        [0xff1a, 0xff1f], // Fullwidth punctuation 2
+        [0xff3b, 0xff3f], // Fullwidth punctuation 3
+        [0xff5b, 0xff60], // Fullwidth punctuation 4
+        [0xffe0, 0xffee]  // Currency markers
+    ];
+
+    const SMALL_KANA_SET = new Set(Array.from('ぁぃぅぇぉゃゅょゎァィゥェォャュョヮ'));
+
+
+    // Character code testing functions
+
+    function isCodePointKanji(codePoint) {
+        return isCodePointInRanges(codePoint, CJK_UNIFIED_IDEOGRAPHS_RANGES);
+    }
+
+    function isCodePointKana(codePoint) {
+        return isCodePointInRanges(codePoint, KANA_RANGES);
+    }
+
+    function isCodePointJapanese(codePoint) {
+        return isCodePointInRanges(codePoint, JAPANESE_RANGES);
+    }
+
+    function isCodePointInRanges(codePoint, ranges) {
+        for (const [min, max] of ranges) {
+            if (codePoint >= min && codePoint <= max) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+
+    // String testing functions
+
+    function isStringEntirelyKana(str) {
+        if (str.length === 0) { return false; }
+        for (const c of str) {
+            if (!isCodePointKana(c.codePointAt(0))) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    function isStringPartiallyJapanese(str) {
+        if (str.length === 0) { return false; }
+        for (const c of str) {
+            if (isCodePointJapanese(c.codePointAt(0))) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+
+    // Mora functions
+
+    function isMoraPitchHigh(moraIndex, pitchAccentPosition) {
+        return pitchAccentPosition === 0 ? (moraIndex > 0) : (moraIndex < pitchAccentPosition);
+    }
+
+    function getKanaMorae(text) {
+        const morae = [];
+        let i;
+        for (const c of text) {
+            if (SMALL_KANA_SET.has(c) && (i = morae.length) > 0) {
+                morae[i - 1] += c;
+            } else {
+                morae.push(c);
+            }
+        }
+        return morae;
+    }
+
+
+    // Exports
+
+    return {
+        isCodePointKanji,
+        isCodePointKana,
+        isCodePointJapanese,
+        isStringEntirelyKana,
+        isStringPartiallyJapanese,
+        isMoraPitchHigh,
+        getKanaMorae
+    };
+})();
diff --git a/ext/mixed/js/object-property-accessor.js b/ext/mixed/js/object-property-accessor.js
new file mode 100644
index 00000000..108afc0d
--- /dev/null
+++ b/ext/mixed/js/object-property-accessor.js
@@ -0,0 +1,244 @@
+/*
+ * Copyright (C) 2016-2020  Alex Yatskov <alex@foosoft.net>
+ * Author: Alex Yatskov <alex@foosoft.net>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * Class used to get and set generic properties of an object by using path strings.
+ */
+class ObjectPropertyAccessor {
+    constructor(target, setter=null) {
+        this._target = target;
+        this._setter = (typeof setter === 'function' ? setter : null);
+    }
+
+    getProperty(pathArray, pathLength) {
+        let target = this._target;
+        const ii = typeof pathLength === 'number' ? Math.min(pathArray.length, pathLength) : pathArray.length;
+        for (let i = 0; i < ii; ++i) {
+            const key = pathArray[i];
+            if (!ObjectPropertyAccessor.hasProperty(target, key)) {
+                throw new Error(`Invalid path: ${ObjectPropertyAccessor.getPathString(pathArray.slice(0, i + 1))}`);
+            }
+            target = target[key];
+        }
+        return target;
+    }
+
+    setProperty(pathArray, value) {
+        if (pathArray.length === 0) {
+            throw new Error('Invalid path');
+        }
+
+        const target = this.getProperty(pathArray, pathArray.length - 1);
+        const key = pathArray[pathArray.length - 1];
+        if (!ObjectPropertyAccessor.isValidPropertyType(target, key)) {
+            throw new Error(`Invalid path: ${ObjectPropertyAccessor.getPathString(pathArray)}`);
+        }
+
+        if (this._setter !== null) {
+            this._setter(target, key, value, pathArray);
+        } else {
+            target[key] = value;
+        }
+    }
+
+    static getPathString(pathArray) {
+        const regexShort = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
+        let pathString = '';
+        let first = true;
+        for (let part of pathArray) {
+            switch (typeof part) {
+                case 'number':
+                    if (Math.floor(part) !== part || part < 0) {
+                        throw new Error('Invalid index');
+                    }
+                    part = `[${part}]`;
+                    break;
+                case 'string':
+                    if (!regexShort.test(part)) {
+                        const escapedPart = part.replace(/["\\]/g, '\\$&');
+                        part = `["${escapedPart}"]`;
+                    } else {
+                        if (!first) {
+                            part = `.${part}`;
+                        }
+                    }
+                    break;
+                default:
+                    throw new Error(`Invalid type: ${typeof part}`);
+            }
+            pathString += part;
+            first = false;
+        }
+        return pathString;
+    }
+
+    static getPathArray(pathString) {
+        const pathArray = [];
+        let state = 'empty';
+        let quote = 0;
+        let value = '';
+        let escaped = false;
+        for (const c of pathString) {
+            const v = c.codePointAt(0);
+            switch (state) {
+                case 'empty': // Empty
+                case 'id-start': // Expecting identifier start
+                    if (v === 0x5b) { // '['
+                        if (state === 'id-start') {
+                            throw new Error(`Unexpected character: ${c}`);
+                        }
+                        state = 'open-bracket';
+                    } else if (
+                        (v >= 0x41 && v <= 0x5a) || // ['A', 'Z']
+                        (v >= 0x61 && v <= 0x7a) || // ['a', 'z']
+                        v === 0x5f // '_'
+                    ) {
+                        state = 'id';
+                        value += c;
+                    } else {
+                        throw new Error(`Unexpected character: ${c}`);
+                    }
+                    break;
+                case 'id': // Identifier
+                    if (
+                        (v >= 0x41 && v <= 0x5a) || // ['A', 'Z']
+                        (v >= 0x61 && v <= 0x7a) || // ['a', 'z']
+                        (v >= 0x30 && v <= 0x39) || // ['0', '9']
+                        v === 0x5f // '_'
+                    ) {
+                        value += c;
+                    } else if (v === 0x5b) { // '['
+                        pathArray.push(value);
+                        value = '';
+                        state = 'open-bracket';
+                    } else if (v === 0x2e) { // '.'
+                        pathArray.push(value);
+                        value = '';
+                        state = 'id-start';
+                    } else {
+                        throw new Error(`Unexpected character: ${c}`);
+                    }
+                    break;
+                case 'open-bracket': // Open bracket
+                    if (v === 0x22 || v === 0x27) { // '"' or '\''
+                        quote = v;
+                        state = 'string';
+                    } else if (v >= 0x30 && v <= 0x39) { // ['0', '9']
+                        state = 'number';
+                        value += c;
+                    } else {
+                        throw new Error(`Unexpected character: ${c}`);
+                    }
+                    break;
+                case 'string': // Quoted string
+                    if (escaped) {
+                        value += c;
+                        escaped = false;
+                    } else if (v === 0x5c) { // '\\'
+                        escaped = true;
+                    } else if (v !== quote) {
+                        value += c;
+                    } else {
+                        state = 'close-bracket';
+                    }
+                    break;
+                case 'number': // Number
+                    if (v >= 0x30 && v <= 0x39) { // ['0', '9']
+                        value += c;
+                    } else if (v === 0x5d) { // ']'
+                        pathArray.push(Number.parseInt(value, 10));
+                        value = '';
+                        state = 'next';
+                    } else {
+                        throw new Error(`Unexpected character: ${c}`);
+                    }
+                    break;
+                case 'close-bracket': // Expecting closing bracket after quoted string
+                    if (v === 0x5d) { // ']'
+                        pathArray.push(value);
+                        value = '';
+                        state = 'next';
+                    } else {
+                        throw new Error(`Unexpected character: ${c}`);
+                    }
+                    break;
+                case 'next': // Expecting . or [
+                    if (v === 0x5b) { // '['
+                        state = 'open-bracket';
+                    } else if (v === 0x2e) { // '.'
+                        state = 'id-start';
+                    } else {
+                        throw new Error(`Unexpected character: ${c}`);
+                    }
+                    break;
+            }
+        }
+        switch (state) {
+            case 'empty':
+            case 'next':
+                break;
+            case 'id':
+                pathArray.push(value);
+                value = '';
+                break;
+            default:
+                throw new Error('Path not terminated correctly');
+        }
+        return pathArray;
+    }
+
+    static hasProperty(object, property) {
+        switch (typeof property) {
+            case 'string':
+                return (
+                    typeof object === 'object' &&
+                    object !== null &&
+                    !Array.isArray(object) &&
+                    Object.prototype.hasOwnProperty.call(object, property)
+                );
+            case 'number':
+                return (
+                    Array.isArray(object) &&
+                    property >= 0 &&
+                    property < object.length &&
+                    property === Math.floor(property)
+                );
+            default:
+                return false;
+        }
+    }
+
+    static isValidPropertyType(object, property) {
+        switch (typeof property) {
+            case 'string':
+                return (
+                    typeof object === 'object' &&
+                    object !== null &&
+                    !Array.isArray(object)
+                );
+            case 'number':
+                return (
+                    Array.isArray(object) &&
+                    property >= 0 &&
+                    property === Math.floor(property)
+                );
+            default:
+                return false;
+        }
+    }
+}
diff --git a/ext/mixed/js/text-scanner.js b/ext/mixed/js/text-scanner.js
index a08e09fb..b8156c01 100644
--- a/ext/mixed/js/text-scanner.js
+++ b/ext/mixed/js/text-scanner.js
@@ -23,13 +23,15 @@
  */
 
 class TextScanner {
-    constructor(node, ignoreNodes, ignoreElements, ignorePoints) {
+    constructor(node, ignoreElements, ignorePoints) {
         this.node = node;
-        this.ignoreNodes = (Array.isArray(ignoreNodes) && ignoreNodes.length > 0 ? ignoreNodes.join(',') : null);
         this.ignoreElements = ignoreElements;
         this.ignorePoints = ignorePoints;
 
+        this.ignoreNodes = null;
+
         this.scanTimerPromise = null;
+        this.causeCurrent = null;
         this.textSourceCurrent = null;
         this.pendingLookup = false;
         this.options = null;
@@ -298,6 +300,7 @@ class TextScanner {
                 this.pendingLookup = true;
                 const result = await this.onSearchSource(textSource, cause);
                 if (result !== null) {
+                    this.causeCurrent = cause;
                     this.textSourceCurrent = textSource;
                     if (this.options.scanning.selectText) {
                         textSource.select();