From f361139d744e58a6c33841cee227d13d1970bb98 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Wed, 6 May 2020 19:37:36 -0400 Subject: Japanese util refactor (#510) * Convert mixed japanese.js to utility class * Copy functions from bg/js/japanese.js into mixed/js/japanese.js * Remove bg/js/japanese.js * Make wanakana dependency optional * Update tests --- ext/mixed/js/japanese.js | 507 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 452 insertions(+), 55 deletions(-) (limited to 'ext/mixed') diff --git a/ext/mixed/js/japanese.js b/ext/mixed/js/japanese.js index ced486dd..801dec84 100644 --- a/ext/mixed/js/japanese.js +++ b/ext/mixed/js/japanese.js @@ -16,6 +16,11 @@ */ const jp = (() => { + const ITERATION_MARK_CODE_POINT = 0x3005; + const HIRAGANA_SMALL_TSU_CODE_POINT = 0x3063; + const KATAKANA_SMALL_TSU_CODE_POINT = 0x30c3; + const KANA_PROLONGED_SOUND_MARK_CODE_POINT = 0x30fc; + const HIRAGANA_RANGE = [0x3040, 0x309f]; const KATAKANA_RANGE = [0x30a0, 0x30ff]; const KANA_RANGES = [HIRAGANA_RANGE, KATAKANA_RANGE]; @@ -65,20 +70,65 @@ const jp = (() => { const SMALL_KANA_SET = new Set(Array.from('ぁぃぅぇぉゃゅょゎァィゥェォャュョヮ')); + const HALFWIDTH_KATAKANA_MAPPING = new Map([ + ['ｦ', 'ヲヺ-'], + ['ｧ', 'ァ--'], + ['ｨ', 'ィ--'], + ['ｩ', 'ゥ--'], + ['ｪ', 'ェ--'], + ['ｫ', 'ォ--'], + ['ｬ', 'ャ--'], + ['ｭ', 'ュ--'], + ['ｮ', 'ョ--'], + ['ｯ', 'ッ--'], + ['ｰ', 'ー--'], + ['ｱ', 'ア--'], + ['ｲ', 'イ--'], + ['ｳ', 'ウヴ-'], + ['ｴ', 'エ--'], + ['ｵ', 'オ--'], + ['ｶ', 'カガ-'], + ['ｷ', 'キギ-'], + ['ｸ', 'クグ-'], + ['ｹ', 'ケゲ-'], + ['ｺ', 'コゴ-'], + ['ｻ', 'サザ-'], + ['ｼ', 'シジ-'], + ['ｽ', 'スズ-'], + ['ｾ', 'セゼ-'], + ['ｿ', 'ソゾ-'], + ['ﾀ', 'タダ-'], + ['ﾁ', 'チヂ-'], + ['ﾂ', 'ツヅ-'], + ['ﾃ', 'テデ-'], + ['ﾄ', 'トド-'], + ['ﾅ', 'ナ--'], + ['ﾆ', 'ニ--'], + ['ﾇ', 'ヌ--'], + ['ﾈ', 'ネ--'], + ['ﾉ', 'ノ--'], + ['ﾊ', 'ハバパ'], + ['ﾋ', 'ヒビピ'], + ['ﾌ', 'フブプ'], + ['ﾍ', 'ヘベペ'], + ['ﾎ', 'ホボポ'], + ['ﾏ', 'マ--'], + ['ﾐ', 'ミ--'], + ['ﾑ', 'ム--'], + ['ﾒ', 'メ--'], + ['ﾓ', 'モ--'], + ['ﾔ', 'ヤ--'], + ['ﾕ', 'ユ--'], + ['ﾖ', 'ヨ--'], + ['ﾗ', 'ラ--'], + ['ﾘ', 'リ--'], + ['ﾙ', 'ル--'], + ['ﾚ', 'レ--'], + ['ﾛ', 'ロ--'], + ['ﾜ', 'ワ--'], + ['ﾝ', 'ン--'] + ]); - // Character code testing functions - - function isCodePointKanji(codePoint) { - return isCodePointInRanges(codePoint, CJK_UNIFIED_IDEOGRAPHS_RANGES); - } - - function isCodePointKana(codePoint) { - return isCodePointInRanges(codePoint, KANA_RANGES); - } - - function isCodePointJapanese(codePoint) { - return isCodePointInRanges(codePoint, JAPANESE_RANGES); - } function isCodePointInRanges(codePoint, ranges) { for (const [min, max] of ranges) { @@ -89,63 +139,410 @@ const jp = (() => { return false; } + function getWanakana() { + try { + if (typeof wanakana !== 'undefined') { + // eslint-disable-next-line no-undef + return wanakana; + } + } catch (e) { + // NOP + } + return null; + } + - // String testing functions + class JapaneseUtil { + constructor(wanakana=null) { + this._wanakana = wanakana; + } - function isStringEntirelyKana(str) { - if (str.length === 0) { return false; } - for (const c of str) { - if (!isCodePointKana(c.codePointAt(0))) { - return false; + // Character code testing functions + + isCodePointKanji(codePoint) { + return isCodePointInRanges(codePoint, CJK_UNIFIED_IDEOGRAPHS_RANGES); + } + + isCodePointKana(codePoint) { + return isCodePointInRanges(codePoint, KANA_RANGES); + } + + isCodePointJapanese(codePoint) { + return isCodePointInRanges(codePoint, JAPANESE_RANGES); + } + + // String testing functions + + isStringEntirelyKana(str) { + if (str.length === 0) { return false; } + for (const c of str) { + if (!isCodePointInRanges(c.codePointAt(0), KANA_RANGES)) { + return false; + } } + return true; } - return true; - } - function isStringPartiallyJapanese(str) { - if (str.length === 0) { return false; } - for (const c of str) { - if (isCodePointJapanese(c.codePointAt(0))) { - return true; + isStringPartiallyJapanese(str) { + if (str.length === 0) { return false; } + for (const c of str) { + if (isCodePointInRanges(c.codePointAt(0), JAPANESE_RANGES)) { + return true; + } } + return false; } - return false; - } + // Mora functions - // Mora functions + isMoraPitchHigh(moraIndex, pitchAccentPosition) { + switch (pitchAccentPosition) { + case 0: return (moraIndex > 0); + case 1: return (moraIndex < 1); + default: return (moraIndex > 0 && moraIndex < pitchAccentPosition); + } + } - function isMoraPitchHigh(moraIndex, pitchAccentPosition) { - switch (pitchAccentPosition) { - case 0: return (moraIndex > 0); - case 1: return (moraIndex < 1); - default: return (moraIndex > 0 && moraIndex < pitchAccentPosition); + getKanaMorae(text) { + const morae = []; + let i; + for (const c of text) { + if (SMALL_KANA_SET.has(c) && (i = morae.length) > 0) { + morae[i - 1] += c; + } else { + morae.push(c); + } + } + return morae; } - } - function getKanaMorae(text) { - const morae = []; - let i; - for (const c of text) { - if (SMALL_KANA_SET.has(c) && (i = morae.length) > 0) { - morae[i - 1] += c; - } else { - morae.push(c); + // Conversion functions + + convertKatakanaToHiragana(text) { + const wanakana = this._getWanakana(); + let result = ''; + for (const c of text) { + if (wanakana.isKatakana(c)) { + result += wanakana.toHiragana(c); + } else { + result += c; + } } + + return result; } - return morae; - } + convertHiraganaToKatakana(text) { + const wanakana = this._getWanakana(); + let result = ''; + for (const c of text) { + if (wanakana.isHiragana(c)) { + result += wanakana.toKatakana(c); + } else { + result += c; + } + } + + return result; + } + + convertToRomaji(text) { + const wanakana = this._getWanakana(); + return wanakana.toRomaji(text); + } + + convertReading(expression, reading, readingMode) { + switch (readingMode) { + case 'hiragana': + return this.convertKatakanaToHiragana(reading); + case 'katakana': + return this.convertHiraganaToKatakana(reading); + case 'romaji': + if (reading) { + return this.convertToRomaji(reading); + } else { + if (this.isStringEntirelyKana(expression)) { + return this.convertToRomaji(expression); + } + } + return reading; + case 'none': + return ''; + default: + return reading; + } + } + + convertNumericToFullWidth(text) { + let result = ''; + for (const char of text) { + let c = char.codePointAt(0); + if (c >= 0x30 && c <= 0x39) { // ['0', '9'] + c += 0xff10 - 0x30; // 0xff10 = '0' full width + result += String.fromCodePoint(c); + } else { + result += char; + } + } + return result; + } + + convertHalfWidthKanaToFullWidth(text, sourceMap=null) { + let result = ''; + + // This function is safe to use charCodeAt instead of codePointAt, since all + // the relevant characters are represented with a single UTF-16 character code. + for (let i = 0, ii = text.length; i < ii; ++i) { + const c = text[i]; + const mapping = HALFWIDTH_KATAKANA_MAPPING.get(c); + if (typeof mapping !== 'string') { + result += c; + continue; + } + + let index = 0; + switch (text.charCodeAt(i + 1)) { + case 0xff9e: // dakuten + index = 1; + break; + case 0xff9f: // handakuten + index = 2; + break; + } + + let c2 = mapping[index]; + if (index > 0) { + if (c2 === '-') { // invalid + index = 0; + c2 = mapping[0]; + } else { + ++i; + } + } + + if (sourceMap !== null && index > 0) { + sourceMap.combine(result.length, 1); + } + result += c2; + } + + return result; + } + + convertAlphabeticToKana(text, sourceMap=null) { + let part = ''; + let result = ''; + + for (const char of text) { + // Note: 0x61 is the character code for 'a' + let c = char.codePointAt(0); + if (c >= 0x41 && c <= 0x5a) { // ['A', 'Z'] + c += (0x61 - 0x41); + } else if (c >= 0x61 && c <= 0x7a) { // ['a', 'z'] + // NOP; c += (0x61 - 0x61); + } else if (c >= 0xff21 && c <= 0xff3a) { // ['A', 'Z'] fullwidth + c += (0x61 - 0xff21); + } else if (c >= 0xff41 && c <= 0xff5a) { // ['a', 'z'] fullwidth + c += (0x61 - 0xff41); + } else if (c === 0x2d || c === 0xff0d) { // '-' or fullwidth dash + c = 0x2d; // '-' + } else { + if (part.length > 0) { + result += this._convertAlphabeticPartToKana(part, sourceMap, result.length); + part = ''; + } + result += char; + continue; + } + part += String.fromCodePoint(c); + } + + if (part.length > 0) { + result += this._convertAlphabeticPartToKana(part, sourceMap, result.length); + } + return result; + } + + // Furigana distribution + + distributeFurigana(expression, reading) { + const fallback = [{furigana: reading, text: expression}]; + if (!reading) { + return fallback; + } + + let isAmbiguous = false; + const segmentize = (reading2, groups) => { + if (groups.length === 0 || isAmbiguous) { + return []; + } + + const group = groups[0]; + if (group.mode === 'kana') { + if (this.convertKatakanaToHiragana(reading2).startsWith(this.convertKatakanaToHiragana(group.text))) { + const readingLeft = reading2.substring(group.text.length); + const segs = segmentize(readingLeft, groups.splice(1)); + if (segs) { + return [{text: group.text, furigana: ''}].concat(segs); + } + } + } else { + let foundSegments = null; + for (let i = reading2.length; i >= group.text.length; --i) { + const readingUsed = reading2.substring(0, i); + const readingLeft = reading2.substring(i); + const segs = segmentize(readingLeft, groups.slice(1)); + if (segs) { + if (foundSegments !== null) { + // more than one way to segmentize the tail, mark as ambiguous + isAmbiguous = true; + return null; + } + foundSegments = [{text: group.text, furigana: readingUsed}].concat(segs); + } + // there is only one way to segmentize the last non-kana group + if (groups.length === 1) { + break; + } + } + return foundSegments; + } + }; + + const groups = []; + let modePrev = null; + for (const c of expression) { + const codePoint = c.codePointAt(0); + const modeCurr = this.isCodePointKanji(codePoint) || codePoint === ITERATION_MARK_CODE_POINT ? 'kanji' : 'kana'; + if (modeCurr === modePrev) { + groups[groups.length - 1].text += c; + } else { + groups.push({mode: modeCurr, text: c}); + modePrev = modeCurr; + } + } + + const segments = segmentize(reading, groups); + if (segments && !isAmbiguous) { + return segments; + } + return fallback; + } + + distributeFuriganaInflected(expression, reading, source) { + const output = []; + + let stemLength = 0; + const shortest = Math.min(source.length, expression.length); + const sourceHiragana = this.convertKatakanaToHiragana(source); + const expressionHiragana = this.convertKatakanaToHiragana(expression); + while (stemLength < shortest && sourceHiragana[stemLength] === expressionHiragana[stemLength]) { + ++stemLength; + } + const offset = source.length - stemLength; + + const stemExpression = source.substring(0, source.length - offset); + const stemReading = reading.substring( + 0, + offset === 0 ? reading.length : reading.length - expression.length + stemLength + ); + for (const segment of this.distributeFurigana(stemExpression, stemReading)) { + output.push(segment); + } + + if (stemLength !== source.length) { + output.push({text: source.substring(stemLength), furigana: ''}); + } + + return output; + } + + // Miscellaneous + + collapseEmphaticSequences(text, fullCollapse, sourceMap=null) { + let result = ''; + let collapseCodePoint = -1; + const hasSourceMap = (sourceMap !== null); + for (const char of text) { + const c = char.codePointAt(0); + if ( + c === HIRAGANA_SMALL_TSU_CODE_POINT || + c === KATAKANA_SMALL_TSU_CODE_POINT || + c === KANA_PROLONGED_SOUND_MARK_CODE_POINT + ) { + if (collapseCodePoint !== c) { + collapseCodePoint = c; + if (!fullCollapse) { + result += char; + continue; + } + } + } else { + collapseCodePoint = -1; + result += char; + continue; + } + + if (hasSourceMap) { + sourceMap.combine(Math.max(0, result.length - 1), 1); + } + } + return result; + } + + // Private + + _getWanakana() { + const wanakana = this._wanakana; + if (wanakana === null) { throw new Error('Functions which use WanaKana are not supported in this context'); } + return wanakana; + } + + _convertAlphabeticPartToKana(text, sourceMap, sourceMapStart) { + const wanakana = this._getWanakana(); + const result = wanakana.toHiragana(text); + + // Generate source mapping + if (sourceMap !== null) { + let i = 0; + let resultPos = 0; + const ii = text.length; + while (i < ii) { + // Find smallest matching substring + let iNext = i + 1; + let resultPosNext = result.length; + while (iNext < ii) { + const t = wanakana.toHiragana(text.substring(0, iNext)); + if (t === result.substring(0, t.length)) { + resultPosNext = t.length; + break; + } + ++iNext; + } + + // Merge characters + const removals = iNext - i - 1; + if (removals > 0) { + sourceMap.combine(sourceMapStart, removals); + } + ++sourceMapStart; + + // Empty elements + const additions = resultPosNext - resultPos - 1; + for (let j = 0; j < additions; ++j) { + sourceMap.insert(sourceMapStart, 0); + ++sourceMapStart; + } + + i = iNext; + resultPos = resultPosNext; + } + } + + return result; + } + } - // Exports - return { - isCodePointKanji, - isCodePointKana, - isCodePointJapanese, - isStringEntirelyKana, - isStringPartiallyJapanese, - isMoraPitchHigh, - getKanaMorae - }; + return new JapaneseUtil(getWanakana()); })(); -- cgit v1.2.3