Implement new translation options

2 files changed, 252 insertions, 5 deletions

diff --git a/ext/bg/js/translator.js b/ext/bg/js/translator.js
index bb78d46d..88a5c6e8 100644
--- a/ext/bg/js/translator.js
+++ b/ext/bg/js/translator.js
@@ -240,6 +240,7 @@ class Translator {
 
                 definitions.push({
                     source: deinflection.source,
+                    rawSource: deinflection.rawSource,
                     reasons: deinflection.reasons,
                     score: definition.score,
                     id: definition.id,
@@ -260,7 +261,7 @@ class Translator {
 
         let length = 0;
         for (const definition of definitions) {
-            length = Math.max(length, definition.source.length);
+            length = Math.max(length, definition.rawSource.length);
         }
 
         return [definitions, length];
@@ -274,6 +275,7 @@ class Translator {
 
         return [{
             source: text,
+            rawSource: text,
             term: text,
             rules: 0,
             definitions,
@@ -320,27 +322,71 @@ class Translator {
         return deinflections.filter((e) => e.definitions.length > 0);
     }
 
-    getAllDeinflections(text, _options) {
+    getAllDeinflections(text, options) {
+        const translationOptions = options.translation;
         const textOptionVariantArray = [
-            [false, true] // convert katakana to hiragana
+            Translator.getTextOptionEntryVariants(translationOptions.convertKatakanaToHiragana),
+            Translator.getTextOptionEntryVariants(translationOptions.convertHalfWidthCharacters),
+            Translator.getTextOptionEntryVariants(translationOptions.convertNumericCharacters),
+            Translator.getTextOptionEntryVariants(translationOptions.convertAlphabeticCharacters)
         ];
 
         const deinflections = [];
         const used = new Set();
-        for (const [hiragana] of Translator.getArrayVariants(textOptionVariantArray)) {
+        for (const [hiragana, halfWidth, numeric, alphabetic] of Translator.getArrayVariants(textOptionVariantArray)) {
             let text2 = text;
+            let sourceMapping = null;
+            if (halfWidth) {
+                if (sourceMapping === null) { sourceMapping = Translator.createTextSourceMapping(text2); }
+                text2 = jpConvertHalfWidthKanaToFullWidth(text2, sourceMapping);
+            }
+            if (numeric) { text2 = jpConvertNumericTofullWidth(text2); }
+            if (alphabetic) {
+                if (sourceMapping === null) { sourceMapping = Translator.createTextSourceMapping(text2); }
+                text2 = jpConvertAlphabeticToKana(text2, sourceMapping);
+            }
             if (hiragana) { text2 = jpKatakanaToHiragana(text2); }
 
             for (let i = text2.length; i > 0; --i) {
                 const text2Substring = text2.substring(0, i);
                 if (used.has(text2Substring)) { break; }
                 used.add(text2Substring);
-                deinflections.push(...this.deinflector.deinflect(text2Substring));
+                for (const deinflection of this.deinflector.deinflect(text2Substring)) {
+                    deinflection.rawSource = Translator.getDeinflectionRawSource(text, i, sourceMapping);
+                    deinflections.push(deinflection);
+                }
             }
         }
         return deinflections;
     }
 
+    static getTextOptionEntryVariants(value) {
+        switch (value) {
+            case 'true': return [true];
+            case 'variant': return [false, true];
+            default: return [false];
+        }
+    }
+
+    static getDeinflectionRawSource(source, length, sourceMapping) {
+        if (sourceMapping === null) {
+            return source.substring(0, length);
+        }
+
+        let result = '';
+        let index = 0;
+        for (let i = 0; i < length; ++i) {
+            const c = sourceMapping[i];
+            result += source.substring(index, index + c);
+            index += c;
+        }
+        return result;
+    }
+
+    static createTextSourceMapping(text) {
+        return new Array(text.length).fill(1);
+    }
+
     async findKanji(text, options) {
         const dictionaries = dictEnabledSet(options);
         const titles = Object.keys(dictionaries);
diff --git a/ext/mixed/js/japanese.js b/ext/mixed/js/japanese.js
index 23b2bd36..26349094 100644
--- a/ext/mixed/js/japanese.js
+++ b/ext/mixed/js/japanese.js
@@ -17,6 +17,65 @@
  */
 
 
+const jpHalfWidthCharacterMapping = new Map([
+    ['ｦ', 'ヲヺ-'],
+    ['ｧ', 'ァ--'],
+    ['ｨ', 'ィ--'],
+    ['ｩ', 'ゥ--'],
+    ['ｪ', 'ェ--'],
+    ['ｫ', 'ォ--'],
+    ['ｬ', 'ャ--'],
+    ['ｭ', 'ュ--'],
+    ['ｮ', 'ョ--'],
+    ['ｯ', 'ッ--'],
+    ['ｰ', 'ー--'],
+    ['ｱ', 'ア--'],
+    ['ｲ', 'イ--'],
+    ['ｳ', 'ウヴ-'],
+    ['ｴ', 'エ--'],
+    ['ｵ', 'オ--'],
+    ['ｶ', 'カガ-'],
+    ['ｷ', 'キギ-'],
+    ['ｸ', 'クグ-'],
+    ['ｹ', 'ケゲ-'],
+    ['ｺ', 'コゴ-'],
+    ['ｻ', 'サザ-'],
+    ['ｼ', 'シジ-'],
+    ['ｽ', 'スズ-'],
+    ['ｾ', 'セゼ-'],
+    ['ｿ', 'ソゾ-'],
+    ['ﾀ', 'タダ-'],
+    ['ﾁ', 'チヂ-'],
+    ['ﾂ', 'ツヅ-'],
+    ['ﾃ', 'テデ-'],
+    ['ﾄ', 'トド-'],
+    ['ﾅ', 'ナ--'],
+    ['ﾆ', 'ニ--'],
+    ['ﾇ', 'ヌ--'],
+    ['ﾈ', 'ネ--'],
+    ['ﾉ', 'ノ--'],
+    ['ﾊ', 'ハバパ'],
+    ['ﾋ', 'ヒビピ'],
+    ['ﾌ', 'フブプ'],
+    ['ﾍ', 'ヘベペ'],
+    ['ﾎ', 'ホボポ'],
+    ['ﾏ', 'マ--'],
+    ['ﾐ', 'ミ--'],
+    ['ﾑ', 'ム--'],
+    ['ﾒ', 'メ--'],
+    ['ﾓ', 'モ--'],
+    ['ﾔ', 'ヤ--'],
+    ['ﾕ', 'ユ--'],
+    ['ﾖ', 'ヨ--'],
+    ['ﾗ', 'ラ--'],
+    ['ﾘ', 'リ--'],
+    ['ﾙ', 'ル--'],
+    ['ﾚ', 'レ--'],
+    ['ﾛ', 'ロ--'],
+    ['ﾜ', 'ワ--'],
+    ['ﾝ', 'ン--']
+]);
+
 function jpIsKanji(c) {
     const code = c.charCodeAt(0);
     return code >= 0x4e00 && code < 0x9fb0 || code >= 0x3400 && code < 0x4dc0;
@@ -175,3 +234,145 @@ function jpDistributeFuriganaInflected(expression, reading, source) {
 
     return output;
 }
+
+function jpConvertHalfWidthKanaToFullWidth(text, sourceMapping) {
+    let result = '';
+    const ii = text.length;
+    const hasSourceMapping = Array.isArray(sourceMapping);
+
+    for (let i = 0; i < ii; ++i) {
+        const c = text[i];
+        const mapping = jpHalfWidthCharacterMapping.get(c);
+        if (typeof mapping !== 'string') {
+            result += c;
+            continue;
+        }
+
+        let index = 0;
+        switch (text.charCodeAt(i + 1)) {
+            case 0xff9e: // dakuten
+                index = 1;
+                break;
+            case 0xff9f: // handakuten
+                index = 2;
+                break;
+        }
+
+        let c2 = mapping[index];
+        if (index > 0) {
+            if (c2 === '-') { // invalid
+                index = 0;
+                c2 = mapping[0];
+            } else {
+                ++i;
+            }
+        }
+
+        if (hasSourceMapping && index > 0) {
+            index = result.length;
+            const v = sourceMapping.splice(index + 1, 1)[0];
+            sourceMapping[index] += v;
+        }
+        result += c2;
+    }
+
+    return result;
+}
+
+function jpConvertNumericTofullWidth(text) {
+    let result = '';
+    for (let i = 0, ii = text.length; i < ii; ++i) {
+        let c = text.charCodeAt(i);
+        if (c >= 0x30 && c <= 0x39) { // ['0', '9']
+            c += 0xff10 - 0x30; // 0xff10 = '0' full width
+            result += String.fromCharCode(c);
+        } else {
+            result += text[i];
+        }
+    }
+    return result;
+}
+
+function jpConvertAlphabeticToKana(text, sourceMapping) {
+    let part = '';
+    let result = '';
+    const ii = text.length;
+
+    if (sourceMapping.length === ii) {
+        sourceMapping.length = ii;
+        sourceMapping.fill(1);
+    }
+
+    for (let i = 0; i < ii; ++i) {
+        let c = text.charCodeAt(i);
+        if (c >= 0x41 && c <= 0x5a) { // ['A', 'Z']
+            c -= 0x41;
+        } else if (c >= 0x61 && c <= 0x7a) { // ['a', 'z']
+            c -= 0x61;
+        } else if (c >= 0xff21 && c <= 0xff3a) { // ['A', 'Z'] full width
+            c -= 0xff21;
+        } else if (c >= 0xff41 && c <= 0xff5a) { // ['a', 'z'] full width
+            c -= 0xff41;
+        } else {
+            if (part.length > 0) {
+                result += jpToHiragana(part, sourceMapping, result.length);
+                part = '';
+            }
+            result += text[i];
+            continue;
+        }
+        part += String.fromCharCode(c + 0x61); // + 'a'
+    }
+
+    if (part.length > 0) {
+        result += jpToHiragana(part, sourceMapping, result.length);
+    }
+    return result;
+}
+
+function jpToHiragana(text, sourceMapping, sourceMappingStart) {
+    const result = wanakana.toHiragana(text);
+
+    // Generate source mapping
+    if (Array.isArray(sourceMapping)) {
+        if (typeof sourceMappingStart !== 'number') { sourceMappingStart = 0; }
+        let i = 0;
+        let resultPos = 0;
+        const ii = text.length;
+        while (i < ii) {
+            // Find smallest matching substring
+            let iNext = i + 1;
+            let resultPosNext = result.length;
+            while (iNext < ii) {
+                const t = wanakana.toHiragana(text.substring(0, iNext));
+                if (t === result.substring(0, t.length)) {
+                    resultPosNext = t.length;
+                    break;
+                }
+                ++iNext;
+            }
+
+            // Merge characters
+            const removals = iNext - i - 1;
+            if (removals > 0) {
+                let sum = 0;
+                const vs = sourceMapping.splice(sourceMappingStart + 1, removals);
+                for (const v of vs) { sum += v; }
+                sourceMapping[sourceMappingStart] += sum;
+            }
+            ++sourceMappingStart;
+
+            // Empty elements
+            const additions = resultPosNext - resultPos - 1;
+            for (let j = 0; j < additions; ++j) {
+                sourceMapping.splice(sourceMappingStart, 0, 0);
+                ++sourceMappingStart;
+            }
+
+            i = iNext;
+            resultPos = resultPosNext;
+        }
+    }
+
+    return result;
+}