rework text processors (#793)24.4.21.0

* rework text processors

* rename text-preprocessors file

* Fix search header left margins on small screens (#839)

* Refocuses search input on backspace (#840)

Fixes #775. Note that this behavior gets overridden if backspace is set
as a shortcut action.

* Change hotkey triggering condition to account for IME usage (#837)

_isKeyCharacterInput only worked when not using an IME, as inside of an
IME when a keydown event is fired, the key is reported as "Process",
which does not have a key.length equal to 1. This resulted in hotkeys
being triggered while typing, which this commit fixes.

---------

Co-authored-by: James Maa <jmaa@berkeley.edu>
Co-authored-by: Kuuuube <61125188+Kuuuube@users.noreply.github.com>
Co-authored-by: Andrew Thomas Sartor <andrew@sartor.net>

6 files changed, 77 insertions, 314 deletions

diff --git a/test/data/anki-note-builder-test-results.json b/test/data/anki-note-builder-test-results.json
index d8d5ab0f..162be7fe 100644
--- a/test/data/anki-note-builder-test-results.json
+++ b/test/data/anki-note-builder-test-results.json
@@ -2548,8 +2548,8 @@
         "audio": "",
         "clipboard-image": "",
         "clipboard-text": "",
-        "cloze-body": "打",
-        "cloze-body-kana": "だ",
+        "cloze-body": "打(う)",
+        "cloze-body-kana": "だ(う)",
         "cloze-prefix": "cloze-prefix",
         "cloze-suffix": "cloze-suffix",
         "conjugation": "",
@@ -2577,8 +2577,8 @@
         "screenshot": "",
         "search-query": "fullQuery",
         "selection-text": "",
-        "sentence": "cloze-prefix打cloze-suffix",
-        "sentence-furigana": "cloze-prefix打cloze-suffix",
+        "sentence": "cloze-prefix打(う)cloze-suffix",
+        "sentence-furigana": "cloze-prefix打(う)cloze-suffix",
         "tags": "n",
         "url": "<a href=\"url:\">url:</a>"
       },
@@ -2586,8 +2586,8 @@
         "audio": "",
         "clipboard-image": "",
         "clipboard-text": "",
-        "cloze-body": "打",
-        "cloze-body-kana": "ダース",
+        "cloze-body": "打(う)",
+        "cloze-body-kana": "ダース(う)",
         "cloze-prefix": "cloze-prefix",
         "cloze-suffix": "cloze-suffix",
         "conjugation": "",
@@ -2615,8 +2615,8 @@
         "screenshot": "",
         "search-query": "fullQuery",
         "selection-text": "",
-        "sentence": "cloze-prefix打cloze-suffix",
-        "sentence-furigana": "cloze-prefix打cloze-suffix",
+        "sentence": "cloze-prefix打(う)cloze-suffix",
+        "sentence-furigana": "cloze-prefix打(う)cloze-suffix",
         "tags": "abbr, n",
         "url": "<a href=\"url:\">url:</a>"
       }
diff --git a/test/data/translator-test-inputs.json b/test/data/translator-test-inputs.json
index c9047716..9e62954e 100644
--- a/test/data/translator-test-inputs.json
+++ b/test/data/translator-test-inputs.json
@@ -191,7 +191,7 @@
                         null,
                         [
                             {
-                                "pattern": "\\(([^)]*)(?:\\)|$)",
+                                "pattern": "\\(([^)]*)(?:\\))",
                                 "flags": "g",
                                 "replacement": ""
                             }
@@ -214,7 +214,7 @@
                         null,
                         [
                             {
-                                "pattern": "\\(([^)]*)(?:\\)|$)",
+                                "pattern": "\\(([^)]*)(?:\\))",
                                 "flags": "g",
                                 "replacement": "$1"
                             }
diff --git a/test/data/translator-test-results-note-data1.json b/test/data/translator-test-results-note-data1.json
index f580ac53..f0f32fa8 100644
--- a/test/data/translator-test-results-note-data1.json
+++ b/test/data/translator-test-results-note-data1.json
@@ -22773,7 +22773,7 @@
           "type": "term",
           "id": 1,
           "source": "打",
-          "rawSource": "打",
+          "rawSource": "打(う)",
           "sourceTerm": "打",
           "inflectionRuleChainCandidates": [
             {
@@ -23087,7 +23087,7 @@
           "type": "term",
           "id": 2,
           "source": "打",
-          "rawSource": "打",
+          "rawSource": "打(う)",
           "sourceTerm": "打",
           "inflectionRuleChainCandidates": [
             {
diff --git a/test/data/translator-test-results.json b/test/data/translator-test-results.json
index da2f8da2..b3574b46 100644
--- a/test/data/translator-test-results.json
+++ b/test/data/translator-test-results.json
@@ -12904,7 +12904,7 @@
         "dictionaryIndex": 0,
         "dictionaryPriority": 0,
         "sourceTermExactMatchCount": 1,
-        "maxOriginalTextLength": 1,
+        "maxOriginalTextLength": 4,
         "headwords": [
           {
             "index": 0,
@@ -12912,7 +12912,7 @@
             "reading": "だ",
             "sources": [
               {
-                "originalText": "打",
+                "originalText": "打(う)",
                 "transformedText": "打",
                 "deinflectedText": "打",
                 "matchType": "exact",
@@ -13072,7 +13072,7 @@
         "dictionaryIndex": 0,
         "dictionaryPriority": 0,
         "sourceTermExactMatchCount": 1,
-        "maxOriginalTextLength": 1,
+        "maxOriginalTextLength": 4,
         "headwords": [
           {
             "index": 0,
@@ -13080,7 +13080,7 @@
             "reading": "ダース",
             "sources": [
               {
-                "originalText": "打",
+                "originalText": "打(う)",
                 "transformedText": "打",
                 "deinflectedText": "打",
                 "matchType": "exact",
diff --git a/test/japanese-util.test.js b/test/japanese-util.test.js
index 5f64a714..bff51f85 100644
--- a/test/japanese-util.test.js
+++ b/test/japanese-util.test.js
@@ -17,7 +17,6 @@
  */
 
 import {describe, expect, test} from 'vitest';
-import {TextSourceMap} from '../ext/js/general/text-source-map.js';
 import * as jpw from '../ext/js/language/ja/japanese-wanakana.js';
 import * as jp from '../ext/js/language/ja/japanese.js';
 
@@ -194,54 +193,46 @@ describe('Japanese utility functions', () => {
     });
 
     describe('convertHalfWidthKanaToFullWidth', () => {
-        /** @type {[string: string, expected: string, expectedSourceMapping?: number[]][]} */
+        /** @type {[string: string, expected: string][]} */
         const data = [
             ['0123456789', '0123456789'],
             ['abcdefghij', 'abcdefghij'],
             ['カタカナ', 'カタカナ'],
             ['ひらがな', 'ひらがな'],
-            ['ｶｷ', 'カキ', [1, 1]],
-            ['ｶﾞｷ', 'ガキ', [2, 1]],
-            ['ﾆﾎﾝ', 'ニホン', [1, 1, 1]],
-            ['ﾆｯﾎﾟﾝ', 'ニッポン', [1, 1, 2, 1]]
+            ['ｶｷ', 'カキ'],
+            ['ｶﾞｷ', 'ガキ'],
+            ['ﾆﾎﾝ', 'ニホン'],
+            ['ﾆｯﾎﾟﾝ', 'ニッポン']
         ];
 
-        for (const [string, expected, expectedSourceMapping] of data) {
-            test(`${string} -> ${expected}${typeof expectedSourceMapping !== 'undefined' ? ', ' + JSON.stringify(expectedSourceMapping) : ''}`, () => {
-                const sourceMap = new TextSourceMap(string);
-                const actual1 = jp.convertHalfWidthKanaToFullWidth(string, null);
-                const actual2 = jp.convertHalfWidthKanaToFullWidth(string, sourceMap);
+        for (const [string, expected] of data) {
+            test(`${string} -> ${expected}`, () => {
+                const actual1 = jp.convertHalfWidthKanaToFullWidth(string);
+                const actual2 = jp.convertHalfWidthKanaToFullWidth(string);
                 expect(actual1).toStrictEqual(expected);
                 expect(actual2).toStrictEqual(expected);
-                if (typeof expectedSourceMapping !== 'undefined') {
-                    expect(sourceMap.equals(new TextSourceMap(string, expectedSourceMapping))).toBe(true);
-                }
             });
         }
     });
 
     describe('convertAlphabeticToKana', () => {
-        /** @type {[string: string, expected: string, expectedSourceMapping?: number[]][]} */
+        /** @type {[string: string, expected: string][]} */
         const data = [
             ['0123456789', '0123456789'],
-            ['abcdefghij', 'あbcでfgひj', [1, 1, 1, 2, 1, 1, 2, 1]],
-            ['ABCDEFGHIJ', 'あbcでfgひj', [1, 1, 1, 2, 1, 1, 2, 1]], // wanakana.toHiragana converts text to lower case
+            ['abcdefghij', 'あbcでfgひj'],
+            ['ABCDEFGHIJ', 'あbcでfgひj'], // wanakana.toHiragana converts text to lower case
             ['カタカナ', 'カタカナ'],
             ['ひらがな', 'ひらがな'],
-            ['chikara', 'ちから', [3, 2, 2]],
-            ['CHIKARA', 'ちから', [3, 2, 2]]
+            ['chikara', 'ちから'],
+            ['CHIKARA', 'ちから']
         ];
 
-        for (const [string, expected, expectedSourceMapping] of data) {
-            test(`${string} -> ${string}${typeof expectedSourceMapping !== 'undefined' ? ', ' + JSON.stringify(expectedSourceMapping) : ''}`, () => {
-                const sourceMap = new TextSourceMap(string);
-                const actual1 = jpw.convertAlphabeticToKana(string, null);
-                const actual2 = jpw.convertAlphabeticToKana(string, sourceMap);
+        for (const [string, expected] of data) {
+            test(`${string} -> ${string}`, () => {
+                const actual1 = jpw.convertAlphabeticToKana(string);
+                const actual2 = jpw.convertAlphabeticToKana(string);
                 expect(actual1).toStrictEqual(expected);
                 expect(actual2).toStrictEqual(expected);
-                if (typeof expectedSourceMapping !== 'undefined') {
-                    expect(sourceMap.equals(new TextSourceMap(string, expectedSourceMapping))).toBe(true);
-                }
             });
         }
     });
@@ -765,59 +756,54 @@ describe('Japanese utility functions', () => {
     });
 
     describe('collapseEmphaticSequences', () => {
-        /** @type {[input: [text: string, fullCollapse: boolean], output: [expected: string, expectedSourceMapping: number[]]][]} */
+        /** @type {[input: [text: string, fullCollapse: boolean], output: string][]} */
         const data = [
-            [['かこい', false], ['かこい', [1, 1, 1]]],
-            [['かこい', true], ['かこい', [1, 1, 1]]],
-            [['かっこい', false], ['かっこい', [1, 1, 1, 1]]],
-            [['かっこい', true], ['かこい', [2, 1, 1]]],
-            [['かっっこい', false], ['かっこい', [1, 2, 1, 1]]],
-            [['かっっこい', true], ['かこい', [3, 1, 1]]],
-            [['かっっっこい', false], ['かっこい', [1, 3, 1, 1]]],
-            [['かっっっこい', true], ['かこい', [4, 1, 1]]],
-
-            [['こい', false], ['こい', [1, 1]]],
-            [['こい', true], ['こい', [1, 1]]],
-            [['っこい', false], ['っこい', [1, 1, 1]]],
-            [['っこい', true], ['こい', [2, 1]]],
-            [['っっこい', false], ['っこい', [2, 1, 1]]],
-            [['っっこい', true], ['こい', [3, 1]]],
-            [['っっっこい', false], ['っこい', [3, 1, 1]]],
-            [['っっっこい', true], ['こい', [4, 1]]],
-
-            [['すごい', false], ['すごい', [1, 1, 1]]],
-            [['すごい', true], ['すごい', [1, 1, 1]]],
-            [['すごーい', false], ['すごーい', [1, 1, 1, 1]]],
-            [['すごーい', true], ['すごい', [1, 2, 1]]],
-            [['すごーーい', false], ['すごーい', [1, 1, 2, 1]]],
-            [['すごーーい', true], ['すごい', [1, 3, 1]]],
-            [['すっごーい', false], ['すっごーい', [1, 1, 1, 1, 1]]],
-            [['すっごーい', true], ['すごい', [2, 2, 1]]],
-            [['すっっごーーい', false], ['すっごーい', [1, 2, 1, 2, 1]]],
-            [['すっっごーーい', true], ['すごい', [3, 3, 1]]],
-
-            [['', false], ['', []]],
-            [['', true], ['', []]],
-            [['っ', false], ['っ', [1]]],
-            [['っ', true], ['', [1]]],
-            [['っっ', false], ['っ', [2]]],
-            [['っっ', true], ['', [2]]],
-            [['っっっ', false], ['っ', [3]]],
-            [['っっっ', true], ['', [3]]]
+            [['かこい', false], 'かこい'],
+            [['かこい', true], 'かこい'],
+            [['かっこい', false], 'かっこい'],
+            [['かっこい', true], 'かこい'],
+            [['かっっこい', false], 'かっこい'],
+            [['かっっこい', true], 'かこい'],
+            [['かっっっこい', false], 'かっこい'],
+            [['かっっっこい', true], 'かこい'],
+
+            [['こい', false], 'こい'],
+            [['こい', true], 'こい'],
+            [['っこい', false], 'っこい'],
+            [['っこい', true], 'こい'],
+            [['っっこい', false], 'っこい'],
+            [['っっこい', true], 'こい'],
+            [['っっっこい', false], 'っこい'],
+            [['っっっこい', true], 'こい'],
+
+            [['すごい', false], 'すごい'],
+            [['すごい', true], 'すごい'],
+            [['すごーい', false], 'すごーい'],
+            [['すごーい', true], 'すごい'],
+            [['すごーーい', false], 'すごーい'],
+            [['すごーーい', true], 'すごい'],
+            [['すっごーい', false], 'すっごーい'],
+            [['すっごーい', true], 'すごい'],
+            [['すっっごーーい', false], 'すっごーい'],
+            [['すっっごーーい', true], 'すごい'],
+
+            [['', false], ''],
+            [['', true], ''],
+            [['っ', false], 'っ'],
+            [['っ', true], ''],
+            [['っっ', false], 'っ'],
+            [['っっ', true], ''],
+            [['っっっ', false], 'っ'],
+            [['っっっ', true], '']
         ];
 
         test.each(data)('%o -> %o', (input, output) => {
             const [text, fullCollapse] = input;
-            const [expected, expectedSourceMapping] = output;
-
-            const sourceMap = new TextSourceMap(text);
-            const actual1 = jp.collapseEmphaticSequences(text, fullCollapse, null);
-            const actual2 = jp.collapseEmphaticSequences(text, fullCollapse, sourceMap);
-            expect(actual1).toStrictEqual(expected);
-            expect(actual2).toStrictEqual(expected);
-            if (typeof expectedSourceMapping !== 'undefined') {
-                expect(sourceMap.equals(new TextSourceMap(text, expectedSourceMapping))).toBe(true);
-            }
+
+            const actual1 = jp.collapseEmphaticSequences(text, fullCollapse);
+            const actual2 = jp.collapseEmphaticSequences(text, fullCollapse);
+            expect(actual1).toStrictEqual(output);
+            expect(actual2).toStrictEqual(output);
         });
     });
 
diff --git a/test/text-source-map.test.js b/test/text-source-map.test.js
deleted file mode 100644
index 09341774..00000000
--- a/test/text-source-map.test.js
+++ /dev/null
@@ -1,223 +0,0 @@
-/*
- * Copyright (C) 2023-2024  Yomitan Authors
- * Copyright (C) 2020-2022  Yomichan Authors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-import {describe, expect, test} from 'vitest';
-import {TextSourceMap} from '../ext/js/general/text-source-map.js';
-
-describe('TextSourceMap', () => {
-    describe('Source', () => {
-        const data = [
-            ['source1'],
-            ['source2'],
-            ['source3']
-        ];
-
-        test.each(data)('source-test-%#', (source) => {
-            const sourceMap = new TextSourceMap(source);
-            expect(source).toStrictEqual(sourceMap.source);
-        });
-    });
-
-    describe('Equals', () => {
-        /** @type {[args1: [source1: string, mapping1: ?(number[])], args2: [source2: string, mapping2: ?(number[])], expectedEquals: boolean][]} */
-        const data = [
-            [['source1', null], ['source1', null], true],
-            [['source2', null], ['source2', null], true],
-            [['source3', null], ['source3', null], true],
-
-            [['source1', [1, 1, 1, 1, 1, 1, 1]], ['source1', null], true],
-            [['source2', [1, 1, 1, 1, 1, 1, 1]], ['source2', null], true],
-            [['source3', [1, 1, 1, 1, 1, 1, 1]], ['source3', null], true],
-
-            [['source1', null], ['source1', [1, 1, 1, 1, 1, 1, 1]], true],
-            [['source2', null], ['source2', [1, 1, 1, 1, 1, 1, 1]], true],
-            [['source3', null], ['source3', [1, 1, 1, 1, 1, 1, 1]], true],
-
-            [['source1', [1, 1, 1, 1, 1, 1, 1]], ['source1', [1, 1, 1, 1, 1, 1, 1]], true],
-            [['source2', [1, 1, 1, 1, 1, 1, 1]], ['source2', [1, 1, 1, 1, 1, 1, 1]], true],
-            [['source3', [1, 1, 1, 1, 1, 1, 1]], ['source3', [1, 1, 1, 1, 1, 1, 1]], true],
-
-            [['source1', [1, 2, 1, 3]], ['source1', [1, 2, 1, 3]], true],
-            [['source2', [1, 2, 1, 3]], ['source2', [1, 2, 1, 3]], true],
-            [['source3', [1, 2, 1, 3]], ['source3', [1, 2, 1, 3]], true],
-
-            [['source1', [1, 3, 1, 2]], ['source1', [1, 2, 1, 3]], false],
-            [['source2', [1, 3, 1, 2]], ['source2', [1, 2, 1, 3]], false],
-            [['source3', [1, 3, 1, 2]], ['source3', [1, 2, 1, 3]], false],
-
-            [['source1', [1, 1, 1, 1, 1, 1, 1]], ['source4', [1, 1, 1, 1, 1, 1, 1]], false],
-            [['source2', [1, 1, 1, 1, 1, 1, 1]], ['source5', [1, 1, 1, 1, 1, 1, 1]], false],
-            [['source3', [1, 1, 1, 1, 1, 1, 1]], ['source6', [1, 1, 1, 1, 1, 1, 1]], false]
-        ];
-
-        test.each(data)('equals-test-%#', ([source1, mapping1], [source2, mapping2], expectedEquals) => {
-            const sourceMap1 = new TextSourceMap(source1, mapping1);
-            const sourceMap2 = new TextSourceMap(source2, mapping2);
-            expect(sourceMap1.equals(sourceMap1)).toBe(true);
-            expect(sourceMap2.equals(sourceMap2)).toBe(true);
-            expect(sourceMap1.equals(sourceMap2)).toStrictEqual(expectedEquals);
-        });
-    });
-
-    describe('GetSourceLength', () => {
-        /** @type {[args: [source: string, mapping: number[]], finalLength: number, expectedValue: number][]} */
-        const data = [
-            [['source', [1, 1, 1, 1, 1, 1]], 1, 1],
-            [['source', [1, 1, 1, 1, 1, 1]], 2, 2],
-            [['source', [1, 1, 1, 1, 1, 1]], 3, 3],
-            [['source', [1, 1, 1, 1, 1, 1]], 4, 4],
-            [['source', [1, 1, 1, 1, 1, 1]], 5, 5],
-            [['source', [1, 1, 1, 1, 1, 1]], 6, 6],
-
-            [['source', [2, 2, 2]], 1, 2],
-            [['source', [2, 2, 2]], 2, 4],
-            [['source', [2, 2, 2]], 3, 6],
-
-            [['source', [3, 3]], 1, 3],
-            [['source', [3, 3]], 2, 6],
-
-            [['source', [6, 6]], 1, 6]
-        ];
-
-        test.each(data)('get-source-length-test-%#', ([source, mapping], finalLength, expectedValue) => {
-            const sourceMap = new TextSourceMap(source, mapping);
-            expect(sourceMap.getSourceLength(finalLength)).toStrictEqual(expectedValue);
-        });
-    });
-
-    describe('CombineInsert', () => {
-        /** @type {[args: [source: string, mapping: ?(number[])], expectedArgs: [expectedSource: string, expectedMapping: ?(number[])], operations: [operation: string, arg1: number, arg2: number][]][]} */
-        const data = [
-            // No operations
-            [
-                ['source', null],
-                ['source', [1, 1, 1, 1, 1, 1]],
-                []
-            ],
-
-            // Combine
-            [
-                ['source', null],
-                ['source', [3, 1, 1, 1]],
-                [
-                    ['combine', 0, 2]
-                ]
-            ],
-            [
-                ['source', null],
-                ['source', [1, 1, 1, 3]],
-                [
-                    ['combine', 3, 2]
-                ]
-            ],
-            [
-                ['source', null],
-                ['source', [3, 3]],
-                [
-                    ['combine', 0, 2],
-                    ['combine', 1, 2]
-                ]
-            ],
-            [
-                ['source', null],
-                ['source', [3, 3]],
-                [
-                    ['combine', 3, 2],
-                    ['combine', 0, 2]
-                ]
-            ],
-
-            // Insert
-            [
-                ['source', null],
-                ['source', [0, 1, 1, 1, 1, 1, 1]],
-                [
-                    ['insert', 0, 0]
-                ]
-            ],
-            [
-                ['source', null],
-                ['source', [1, 1, 1, 1, 1, 1, 0]],
-                [
-                    ['insert', 6, 0]
-                ]
-            ],
-            [
-                ['source', null],
-                ['source', [0, 1, 1, 1, 1, 1, 1, 0]],
-                [
-                    ['insert', 0, 0],
-                    ['insert', 7, 0]
-                ]
-            ],
-            [
-                ['source', null],
-                ['source', [0, 1, 1, 1, 1, 1, 1, 0]],
-                [
-                    ['insert', 6, 0],
-                    ['insert', 0, 0]
-                ]
-            ],
-
-            // Mixed
-            [
-                ['source', null],
-                ['source', [3, 0, 3]],
-                [
-                    ['combine', 0, 2],
-                    ['insert', 1, 0],
-                    ['combine', 2, 2]
-                ]
-            ],
-            [
-                ['source', null],
-                ['source', [3, 0, 3]],
-                [
-                    ['combine', 0, 2],
-                    ['combine', 1, 2],
-                    ['insert', 1, 0]
-                ]
-            ],
-            [
-                ['source', null],
-                ['source', [3, 0, 3]],
-                [
-                    ['insert', 3, 0],
-                    ['combine', 0, 2],
-                    ['combine', 2, 2]
-                ]
-            ]
-        ];
-
-        test.each(data)('combine-insert-test-%#', ([source, mapping], [expectedSource, expectedMapping], operations) => {
-            const sourceMap = new TextSourceMap(source, mapping);
-            const expectedSourceMap = new TextSourceMap(expectedSource, expectedMapping);
-            for (const [operation, ...args] of operations) {
-                switch (operation) {
-                    case 'combine':
-                        sourceMap.combine(...args);
-                        break;
-                    case 'insert':
-                        sourceMap.insert(...args);
-                        break;
-                }
-            }
-            expect(sourceMap.equals(expectedSourceMap)).toBe(true);
-        });
-    });
-});