Add normalization of combining dakuten and handakuten to ja preprocessors (#1136)

* Add normalization of combining dakuten and handakuten to ja preprocessors

* Fix typo

* Remove redundant variable assignment

* Fix first character processed incorrectly when it is a character that gets combined

* Add test for combining dakuten and handakuten

1 files changed, 9 insertions, 0 deletions

diff --git a/ext/js/language/ja/japanese-text-preprocessors.js b/ext/js/language/ja/japanese-text-preprocessors.js
index 2d0d23b3..cdd8ce9a 100644
--- a/ext/js/language/ja/japanese-text-preprocessors.js
+++ b/ext/js/language/ja/japanese-text-preprocessors.js
@@ -24,6 +24,7 @@ import {
     convertHalfWidthKanaToFullWidth,
     convertHiraganaToKatakana as convertHiraganaToKatakanaFunction,
     convertKatakanaToHiragana as convertKatakanaToHiraganaFunction,
+    normalizeCombiningCharacters as normalizeCombiningCharactersFunction,
 } from './japanese.js';
 
 /** @type {import('language').TextProcessor<boolean>} */
@@ -90,3 +91,11 @@ export const collapseEmphaticSequences = {
         return str;
     },
 };
+
+/** @type {import('language').TextProcessor<boolean>} */
+export const normalizeCombiningCharacters = {
+    name: 'Normalize combining characters',
+    description: 'ド → ド (U+30C8 U+3099 → U+30C9)',
+    options: basicTextProcessorOptions,
+    process: (str, setting) => (setting ? normalizeCombiningCharactersFunction(str) : str),
+};