From 0f15cca2dff995218a52ff7066008da4cd414e3f Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Wed, 11 Mar 2020 20:33:01 -0400 Subject: Convert Japanese utilities to a module-like style --- ext/bg/js/audio-uri-builder.js | 4 +- ext/bg/js/backend.js | 18 +- ext/bg/js/clipboard-monitor.js | 4 +- ext/bg/js/handlebars.js | 9 +- ext/bg/js/japanese.js | 770 +++++++++++++++++++++-------------------- ext/bg/js/translator.js | 24 +- 6 files changed, 421 insertions(+), 408 deletions(-) (limited to 'ext/bg/js') diff --git a/ext/bg/js/audio-uri-builder.js b/ext/bg/js/audio-uri-builder.js index 499c3441..158006bb 100644 --- a/ext/bg/js/audio-uri-builder.js +++ b/ext/bg/js/audio-uri-builder.js @@ -17,7 +17,7 @@ */ /* global - * jpIsStringEntirelyKana + * jp */ class AudioUriBuilder { @@ -66,7 +66,7 @@ class AudioUriBuilder { let kana = definition.reading; let kanji = definition.expression; - if (!kana && jpIsStringEntirelyKana(kanji)) { + if (!kana && jp.isStringEntirelyKana(kanji)) { kana = kanji; kanji = null; } diff --git a/ext/bg/js/backend.js b/ext/bg/js/backend.js index 978c5a4a..b217e64d 100644 --- a/ext/bg/js/backend.js +++ b/ext/bg/js/backend.js @@ -32,9 +32,7 @@ * dictEnabledSet * dictTermsSort * handlebarsRenderDynamic - * jpConvertReading - * jpDistributeFuriganaInflected - * jpKatakanaToHiragana + * jp * optionsLoad * optionsSave * profileConditionsDescriptor @@ -402,13 +400,13 @@ class Backend { dictTermsSort(definitions); const {expression, reading} = definitions[0]; const source = text.substring(0, sourceLength); - for (const {text: text2, furigana} of jpDistributeFuriganaInflected(expression, reading, source)) { - const reading2 = jpConvertReading(text2, furigana, options.parsing.readingMode); + for (const {text: text2, furigana} of jp.distributeFuriganaInflected(expression, reading, source)) { + const reading2 = jp.convertReading(text2, furigana, options.parsing.readingMode); term.push({text: text2, reading: reading2}); } text = text.substring(source.length); } else { - const reading = jpConvertReading(text[0], null, options.parsing.readingMode); + const reading = jp.convertReading(text[0], null, options.parsing.readingMode); term.push({text: text[0], reading}); text = text.substring(1); } @@ -427,16 +425,16 @@ class Backend { for (const {expression, reading, source} of parsedLine) { const term = []; if (expression !== null && reading !== null) { - for (const {text: text2, furigana} of jpDistributeFuriganaInflected( + for (const {text: text2, furigana} of jp.distributeFuriganaInflected( expression, - jpKatakanaToHiragana(reading), + jp.convertKatakanaToHiragana(reading), source )) { - const reading2 = jpConvertReading(text2, furigana, options.parsing.readingMode); + const reading2 = jp.convertReading(text2, furigana, options.parsing.readingMode); term.push({text: text2, reading: reading2}); } } else { - const reading2 = jpConvertReading(source, null, options.parsing.readingMode); + const reading2 = jp.convertReading(source, null, options.parsing.readingMode); term.push({text: source, reading: reading2}); } result.push(term); diff --git a/ext/bg/js/clipboard-monitor.js b/ext/bg/js/clipboard-monitor.js index 9a881f57..c67525fc 100644 --- a/ext/bg/js/clipboard-monitor.js +++ b/ext/bg/js/clipboard-monitor.js @@ -17,7 +17,7 @@ */ /* global - * jpIsStringPartiallyJapanese + * jp */ class ClipboardMonitor extends EventDispatcher { @@ -54,7 +54,7 @@ class ClipboardMonitor extends EventDispatcher { text !== this._previousText ) { this._previousText = text; - if (jpIsStringPartiallyJapanese(text)) { + if (jp.isStringPartiallyJapanese(text)) { this.trigger('change', {text}); } } diff --git a/ext/bg/js/handlebars.js b/ext/bg/js/handlebars.js index e3ce6bd0..5fda5baa 100644 --- a/ext/bg/js/handlebars.js +++ b/ext/bg/js/handlebars.js @@ -18,8 +18,7 @@ /* global * Handlebars - * jpDistributeFurigana - * jpIsCodePointKanji + * jp */ function handlebarsEscape(text) { @@ -33,7 +32,7 @@ function handlebarsDumpObject(options) { function handlebarsFurigana(options) { const definition = options.fn(this); - const segs = jpDistributeFurigana(definition.expression, definition.reading); + const segs = jp.distributeFurigana(definition.expression, definition.reading); let result = ''; for (const seg of segs) { @@ -49,7 +48,7 @@ function handlebarsFurigana(options) { function handlebarsFuriganaPlain(options) { const definition = options.fn(this); - const segs = jpDistributeFurigana(definition.expression, definition.reading); + const segs = jp.distributeFurigana(definition.expression, definition.reading); let result = ''; for (const seg of segs) { @@ -66,7 +65,7 @@ function handlebarsFuriganaPlain(options) { function handlebarsKanjiLinks(options) { let result = ''; for (const c of options.fn(this)) { - if (jpIsCodePointKanji(c.codePointAt(0))) { + if (jp.isCodePointKanji(c.codePointAt(0))) { result += `${c}`; } else { result += c; diff --git a/ext/bg/js/japanese.js b/ext/bg/js/japanese.js index 3b37754d..182d5b98 100644 --- a/ext/bg/js/japanese.js +++ b/ext/bg/js/japanese.js @@ -20,439 +20,461 @@ * wanakana */ -const JP_HALFWIDTH_KATAKANA_MAPPING = new Map([ - ['ｦ', 'ヲヺ-'], - ['ｧ', 'ァ--'], - ['ｨ', 'ィ--'], - ['ｩ', 'ゥ--'], - ['ｪ', 'ェ--'], - ['ｫ', 'ォ--'], - ['ｬ', 'ャ--'], - ['ｭ', 'ュ--'], - ['ｮ', 'ョ--'], - ['ｯ', 'ッ--'], - ['ｰ', 'ー--'], - ['ｱ', 'ア--'], - ['ｲ', 'イ--'], - ['ｳ', 'ウヴ-'], - ['ｴ', 'エ--'], - ['ｵ', 'オ--'], - ['ｶ', 'カガ-'], - ['ｷ', 'キギ-'], - ['ｸ', 'クグ-'], - ['ｹ', 'ケゲ-'], - ['ｺ', 'コゴ-'], - ['ｻ', 'サザ-'], - ['ｼ', 'シジ-'], - ['ｽ', 'スズ-'], - ['ｾ', 'セゼ-'], - ['ｿ', 'ソゾ-'], - ['ﾀ', 'タダ-'], - ['ﾁ', 'チヂ-'], - ['ﾂ', 'ツヅ-'], - ['ﾃ', 'テデ-'], - ['ﾄ', 'トド-'], - ['ﾅ', 'ナ--'], - ['ﾆ', 'ニ--'], - ['ﾇ', 'ヌ--'], - ['ﾈ', 'ネ--'], - ['ﾉ', 'ノ--'], - ['ﾊ', 'ハバパ'], - ['ﾋ', 'ヒビピ'], - ['ﾌ', 'フブプ'], - ['ﾍ', 'ヘベペ'], - ['ﾎ', 'ホボポ'], - ['ﾏ', 'マ--'], - ['ﾐ', 'ミ--'], - ['ﾑ', 'ム--'], - ['ﾒ', 'メ--'], - ['ﾓ', 'モ--'], - ['ﾔ', 'ヤ--'], - ['ﾕ', 'ユ--'], - ['ﾖ', 'ヨ--'], - ['ﾗ', 'ラ--'], - ['ﾘ', 'リ--'], - ['ﾙ', 'ル--'], - ['ﾚ', 'レ--'], - ['ﾛ', 'ロ--'], - ['ﾜ', 'ワ--'], - ['ﾝ', 'ン--'] -]); - -const JP_HIRAGANA_RANGE = [0x3040, 0x309f]; -const JP_KATAKANA_RANGE = [0x30a0, 0x30ff]; -const JP_KANA_RANGES = [JP_HIRAGANA_RANGE, JP_KATAKANA_RANGE]; - -const JP_CJK_COMMON_RANGE = [0x4e00, 0x9fff]; -const JP_CJK_RARE_RANGE = [0x3400, 0x4dbf]; -const JP_CJK_RANGES = [JP_CJK_COMMON_RANGE, JP_CJK_RARE_RANGE]; - -const JP_ITERATION_MARK_CHAR_CODE = 0x3005; - -// Japanese character ranges, roughly ordered in order of expected frequency -const JP_JAPANESE_RANGES = [ - JP_HIRAGANA_RANGE, - JP_KATAKANA_RANGE, - - JP_CJK_COMMON_RANGE, - JP_CJK_RARE_RANGE, - - [0xff66, 0xff9f], // Halfwidth katakana - - [0x30fb, 0x30fc], // Katakana punctuation - [0xff61, 0xff65], // Kana punctuation - [0x3000, 0x303f], // CJK punctuation - - [0xff10, 0xff19], // Fullwidth numbers - [0xff21, 0xff3a], // Fullwidth upper case Latin letters - [0xff41, 0xff5a], // Fullwidth lower case Latin letters - - [0xff01, 0xff0f], // Fullwidth punctuation 1 - [0xff1a, 0xff1f], // Fullwidth punctuation 2 - [0xff3b, 0xff3f], // Fullwidth punctuation 3 - [0xff5b, 0xff60], // Fullwidth punctuation 4 - [0xffe0, 0xffee] // Currency markers -]; - - -// Helper functions - -function _jpIsCodePointInRanges(codePoint, ranges) { - for (const [min, max] of ranges) { - if (codePoint >= min && codePoint <= max) { - return true; - } +const jp = (() => { + const HALFWIDTH_KATAKANA_MAPPING = new Map([ + ['ｦ', 'ヲヺ-'], + ['ｧ', 'ァ--'], + ['ｨ', 'ィ--'], + ['ｩ', 'ゥ--'], + ['ｪ', 'ェ--'], + ['ｫ', 'ォ--'], + ['ｬ', 'ャ--'], + ['ｭ', 'ュ--'], + ['ｮ', 'ョ--'], + ['ｯ', 'ッ--'], + ['ｰ', 'ー--'], + ['ｱ', 'ア--'], + ['ｲ', 'イ--'], + ['ｳ', 'ウヴ-'], + ['ｴ', 'エ--'], + ['ｵ', 'オ--'], + ['ｶ', 'カガ-'], + ['ｷ', 'キギ-'], + ['ｸ', 'クグ-'], + ['ｹ', 'ケゲ-'], + ['ｺ', 'コゴ-'], + ['ｻ', 'サザ-'], + ['ｼ', 'シジ-'], + ['ｽ', 'スズ-'], + ['ｾ', 'セゼ-'], + ['ｿ', 'ソゾ-'], + ['ﾀ', 'タダ-'], + ['ﾁ', 'チヂ-'], + ['ﾂ', 'ツヅ-'], + ['ﾃ', 'テデ-'], + ['ﾄ', 'トド-'], + ['ﾅ', 'ナ--'], + ['ﾆ', 'ニ--'], + ['ﾇ', 'ヌ--'], + ['ﾈ', 'ネ--'], + ['ﾉ', 'ノ--'], + ['ﾊ', 'ハバパ'], + ['ﾋ', 'ヒビピ'], + ['ﾌ', 'フブプ'], + ['ﾍ', 'ヘベペ'], + ['ﾎ', 'ホボポ'], + ['ﾏ', 'マ--'], + ['ﾐ', 'ミ--'], + ['ﾑ', 'ム--'], + ['ﾒ', 'メ--'], + ['ﾓ', 'モ--'], + ['ﾔ', 'ヤ--'], + ['ﾕ', 'ユ--'], + ['ﾖ', 'ヨ--'], + ['ﾗ', 'ラ--'], + ['ﾘ', 'リ--'], + ['ﾙ', 'ル--'], + ['ﾚ', 'レ--'], + ['ﾛ', 'ロ--'], + ['ﾜ', 'ワ--'], + ['ﾝ', 'ン--'] + ]); + + const HIRAGANA_RANGE = [0x3040, 0x309f]; + const KATAKANA_RANGE = [0x30a0, 0x30ff]; + const KANA_RANGES = [HIRAGANA_RANGE, KATAKANA_RANGE]; + + const CJK_COMMON_RANGE = [0x4e00, 0x9fff]; + const CJK_RARE_RANGE = [0x3400, 0x4dbf]; + const CJK_RANGES = [CJK_COMMON_RANGE, CJK_RARE_RANGE]; + + const ITERATION_MARK_CODE_POINT = 0x3005; + + // Japanese character ranges, roughly ordered in order of expected frequency + const JAPANESE_RANGES = [ + HIRAGANA_RANGE, + KATAKANA_RANGE, + + CJK_COMMON_RANGE, + CJK_RARE_RANGE, + + [0xff66, 0xff9f], // Halfwidth katakana + + [0x30fb, 0x30fc], // Katakana punctuation + [0xff61, 0xff65], // Kana punctuation + [0x3000, 0x303f], // CJK punctuation + + [0xff10, 0xff19], // Fullwidth numbers + [0xff21, 0xff3a], // Fullwidth upper case Latin letters + [0xff41, 0xff5a], // Fullwidth lower case Latin letters + + [0xff01, 0xff0f], // Fullwidth punctuation 1 + [0xff1a, 0xff1f], // Fullwidth punctuation 2 + [0xff3b, 0xff3f], // Fullwidth punctuation 3 + [0xff5b, 0xff60], // Fullwidth punctuation 4 + [0xffe0, 0xffee] // Currency markers + ]; + + + // Character code testing functions + + function isCodePointKanji(codePoint) { + return isCodePointInRanges(codePoint, CJK_RANGES); } - return false; -} - - -// Character code testing functions -function jpIsCodePointKanji(codePoint) { - return _jpIsCodePointInRanges(codePoint, JP_CJK_RANGES); -} + function isCodePointKana(codePoint) { + return isCodePointInRanges(codePoint, KANA_RANGES); + } -function jpIsCodePointKana(codePoint) { - return _jpIsCodePointInRanges(codePoint, JP_KANA_RANGES); -} + function isCodePointJapanese(codePoint) { + return isCodePointInRanges(codePoint, JAPANESE_RANGES); + } -function jpIsCodePointJapanese(codePoint) { - return _jpIsCodePointInRanges(codePoint, JP_JAPANESE_RANGES); -} + function isCodePointInRanges(codePoint, ranges) { + for (const [min, max] of ranges) { + if (codePoint >= min && codePoint <= max) { + return true; + } + } + return false; + } -// String testing functions + // String testing functions -function jpIsStringEntirelyKana(str) { - if (str.length === 0) { return false; } - for (const c of str) { - if (!jpIsCodePointKana(c.codePointAt(0))) { - return false; + function isStringEntirelyKana(str) { + if (str.length === 0) { return false; } + for (const c of str) { + if (!isCodePointKana(c.codePointAt(0))) { + return false; + } } + return true; } - return true; -} - -function jpIsStringPartiallyJapanese(str) { - if (str.length === 0) { return false; } - for (const c of str) { - if (jpIsCodePointJapanese(c.codePointAt(0))) { - return true; + + function isStringPartiallyJapanese(str) { + if (str.length === 0) { return false; } + for (const c of str) { + if (isCodePointJapanese(c.codePointAt(0))) { + return true; + } } + return false; } - return false; -} -// Conversion functions + // Conversion functions -function jpKatakanaToHiragana(text) { - let result = ''; - for (const c of text) { - if (wanakana.isKatakana(c)) { - result += wanakana.toHiragana(c); - } else { - result += c; + function convertKatakanaToHiragana(text) { + let result = ''; + for (const c of text) { + if (wanakana.isKatakana(c)) { + result += wanakana.toHiragana(c); + } else { + result += c; + } } - } - return result; -} - -function jpHiraganaToKatakana(text) { - let result = ''; - for (const c of text) { - if (wanakana.isHiragana(c)) { - result += wanakana.toKatakana(c); - } else { - result += c; - } + return result; } - return result; -} - -function jpToRomaji(text) { - return wanakana.toRomaji(text); -} - -function jpConvertReading(expressionFragment, readingFragment, readingMode) { - switch (readingMode) { - case 'hiragana': - return jpKatakanaToHiragana(readingFragment || ''); - case 'katakana': - return jpHiraganaToKatakana(readingFragment || ''); - case 'romaji': - if (readingFragment) { - return jpToRomaji(readingFragment); + function convertHiraganaToKatakana(text) { + let result = ''; + for (const c of text) { + if (wanakana.isHiragana(c)) { + result += wanakana.toKatakana(c); } else { - if (jpIsStringEntirelyKana(expressionFragment)) { - return jpToRomaji(expressionFragment); - } + result += c; } - return readingFragment; - case 'none': - return null; - default: - return readingFragment; + } + + return result; } -} -function jpDistributeFurigana(expression, reading) { - const fallback = [{furigana: reading, text: expression}]; - if (!reading) { - return fallback; + function convertToRomaji(text) { + return wanakana.toRomaji(text); } - let isAmbiguous = false; - const segmentize = (reading2, groups) => { - if (groups.length === 0 || isAmbiguous) { - return []; + function convertReading(expressionFragment, readingFragment, readingMode) { + switch (readingMode) { + case 'hiragana': + return convertKatakanaToHiragana(readingFragment || ''); + case 'katakana': + return convertHiraganaToKatakana(readingFragment || ''); + case 'romaji': + if (readingFragment) { + return convertToRomaji(readingFragment); + } else { + if (isStringEntirelyKana(expressionFragment)) { + return convertToRomaji(expressionFragment); + } + } + return readingFragment; + case 'none': + return null; + default: + return readingFragment; } + } - const group = groups[0]; - if (group.mode === 'kana') { - if (jpKatakanaToHiragana(reading2).startsWith(jpKatakanaToHiragana(group.text))) { - const readingLeft = reading2.substring(group.text.length); - const segs = segmentize(readingLeft, groups.splice(1)); - if (segs) { - return [{text: group.text}].concat(segs); - } + function convertNumericTofullWidth(text) { + let result = ''; + for (const char of text) { + let c = char.codePointAt(0); + if (c >= 0x30 && c <= 0x39) { // ['0', '9'] + c += 0xff10 - 0x30; // 0xff10 = '0' full width + result += String.fromCodePoint(c); + } else { + result += char; } - } else { - let foundSegments = null; - for (let i = reading2.length; i >= group.text.length; --i) { - const readingUsed = reading2.substring(0, i); - const readingLeft = reading2.substring(i); - const segs = segmentize(readingLeft, groups.slice(1)); - if (segs) { - if (foundSegments !== null) { - // more than one way to segmentize the tail, mark as ambiguous - isAmbiguous = true; - return null; - } - foundSegments = [{text: group.text, furigana: readingUsed}].concat(segs); - } - // there is only one way to segmentize the last non-kana group - if (groups.length === 1) { + } + return result; + } + + function convertHalfWidthKanaToFullWidth(text, sourceMapping) { + let result = ''; + const hasSourceMapping = Array.isArray(sourceMapping); + + // This function is safe to use charCodeAt instead of codePointAt, since all + // the relevant characters are represented with a single UTF-16 character code. + for (let i = 0, ii = text.length; i < ii; ++i) { + const c = text[i]; + const mapping = HALFWIDTH_KATAKANA_MAPPING.get(c); + if (typeof mapping !== 'string') { + result += c; + continue; + } + + let index = 0; + switch (text.charCodeAt(i + 1)) { + case 0xff9e: // dakuten + index = 1; + break; + case 0xff9f: // handakuten + index = 2; break; + } + + let c2 = mapping[index]; + if (index > 0) { + if (c2 === '-') { // invalid + index = 0; + c2 = mapping[0]; + } else { + ++i; } } - return foundSegments; - } - }; - const groups = []; - let modePrev = null; - for (const c of expression) { - const codePoint = c.codePointAt(0); - const modeCurr = jpIsCodePointKanji(codePoint) || codePoint === JP_ITERATION_MARK_CHAR_CODE ? 'kanji' : 'kana'; - if (modeCurr === modePrev) { - groups[groups.length - 1].text += c; - } else { - groups.push({mode: modeCurr, text: c}); - modePrev = modeCurr; + if (hasSourceMapping && index > 0) { + index = result.length; + const v = sourceMapping.splice(index + 1, 1)[0]; + sourceMapping[index] += v; + } + result += c2; } - } - - const segments = segmentize(reading, groups); - if (segments && !isAmbiguous) { - return segments; - } - return fallback; -} - -function jpDistributeFuriganaInflected(expression, reading, source) { - const output = []; - - let stemLength = 0; - const shortest = Math.min(source.length, expression.length); - const sourceHiragana = jpKatakanaToHiragana(source); - const expressionHiragana = jpKatakanaToHiragana(expression); - while (stemLength < shortest && sourceHiragana[stemLength] === expressionHiragana[stemLength]) { - ++stemLength; - } - const offset = source.length - stemLength; - - const stemExpression = source.substring(0, source.length - offset); - const stemReading = reading.substring( - 0, - offset === 0 ? reading.length : reading.length - expression.length + stemLength - ); - for (const segment of jpDistributeFurigana(stemExpression, stemReading)) { - output.push(segment); - } - if (stemLength !== source.length) { - output.push({text: source.substring(stemLength)}); + return result; } - return output; -} - -function jpConvertHalfWidthKanaToFullWidth(text, sourceMapping) { - let result = ''; - const hasSourceMapping = Array.isArray(sourceMapping); - - // This function is safe to use charCodeAt instead of codePointAt, since all - // the relevant characters are represented with a single UTF-16 character code. - for (let i = 0, ii = text.length; i < ii; ++i) { - const c = text[i]; - const mapping = JP_HALFWIDTH_KATAKANA_MAPPING.get(c); - if (typeof mapping !== 'string') { - result += c; - continue; - } + function convertAlphabeticToKana(text, sourceMapping) { + let part = ''; + let result = ''; + const ii = text.length; - let index = 0; - switch (text.charCodeAt(i + 1)) { - case 0xff9e: // dakuten - index = 1; - break; - case 0xff9f: // handakuten - index = 2; - break; + if (sourceMapping.length === ii) { + sourceMapping.length = ii; + sourceMapping.fill(1); } - let c2 = mapping[index]; - if (index > 0) { - if (c2 === '-') { // invalid - index = 0; - c2 = mapping[0]; + for (const char of text) { + // Note: 0x61 is the character code for 'a' + let c = char.codePointAt(0); + if (c >= 0x41 && c <= 0x5a) { // ['A', 'Z'] + c += (0x61 - 0x41); + } else if (c >= 0x61 && c <= 0x7a) { // ['a', 'z'] + // NOP; c += (0x61 - 0x61); + } else if (c >= 0xff21 && c <= 0xff3a) { // ['A', 'Z'] fullwidth + c += (0x61 - 0xff21); + } else if (c >= 0xff41 && c <= 0xff5a) { // ['a', 'z'] fullwidth + c += (0x61 - 0xff41); + } else if (c === 0x2d || c === 0xff0d) { // '-' or fullwidth dash + c = 0x2d; // '-' } else { - ++i; + if (part.length > 0) { + result += convertAlphabeticPartToKana(part, sourceMapping, result.length); + part = ''; + } + result += char; + continue; } + part += String.fromCodePoint(c); } - if (hasSourceMapping && index > 0) { - index = result.length; - const v = sourceMapping.splice(index + 1, 1)[0]; - sourceMapping[index] += v; + if (part.length > 0) { + result += convertAlphabeticPartToKana(part, sourceMapping, result.length); } - result += c2; + return result; } - return result; -} - -function jpConvertNumericTofullWidth(text) { - let result = ''; - for (const char of text) { - let c = char.codePointAt(0); - if (c >= 0x30 && c <= 0x39) { // ['0', '9'] - c += 0xff10 - 0x30; // 0xff10 = '0' full width - result += String.fromCodePoint(c); - } else { - result += char; - } - } - return result; -} + function convertAlphabeticPartToKana(text, sourceMapping, sourceMappingStart) { + const result = wanakana.toHiragana(text); + + // Generate source mapping + if (Array.isArray(sourceMapping)) { + if (typeof sourceMappingStart !== 'number') { sourceMappingStart = 0; } + let i = 0; + let resultPos = 0; + const ii = text.length; + while (i < ii) { + // Find smallest matching substring + let iNext = i + 1; + let resultPosNext = result.length; + while (iNext < ii) { + const t = wanakana.toHiragana(text.substring(0, iNext)); + if (t === result.substring(0, t.length)) { + resultPosNext = t.length; + break; + } + ++iNext; + } -function jpConvertAlphabeticToKana(text, sourceMapping) { - let part = ''; - let result = ''; - const ii = text.length; + // Merge characters + const removals = iNext - i - 1; + if (removals > 0) { + let sum = 0; + const vs = sourceMapping.splice(sourceMappingStart + 1, removals); + for (const v of vs) { sum += v; } + sourceMapping[sourceMappingStart] += sum; + } + ++sourceMappingStart; - if (sourceMapping.length === ii) { - sourceMapping.length = ii; - sourceMapping.fill(1); - } + // Empty elements + const additions = resultPosNext - resultPos - 1; + for (let j = 0; j < additions; ++j) { + sourceMapping.splice(sourceMappingStart, 0, 0); + ++sourceMappingStart; + } - for (const char of text) { - // Note: 0x61 is the character code for 'a' - let c = char.codePointAt(0); - if (c >= 0x41 && c <= 0x5a) { // ['A', 'Z'] - c += (0x61 - 0x41); - } else if (c >= 0x61 && c <= 0x7a) { // ['a', 'z'] - // NOP; c += (0x61 - 0x61); - } else if (c >= 0xff21 && c <= 0xff3a) { // ['A', 'Z'] fullwidth - c += (0x61 - 0xff21); - } else if (c >= 0xff41 && c <= 0xff5a) { // ['a', 'z'] fullwidth - c += (0x61 - 0xff41); - } else if (c === 0x2d || c === 0xff0d) { // '-' or fullwidth dash - c = 0x2d; // '-' - } else { - if (part.length > 0) { - result += jpToHiragana(part, sourceMapping, result.length); - part = ''; + i = iNext; + resultPos = resultPosNext; } - result += char; - continue; } - part += String.fromCodePoint(c); - } - if (part.length > 0) { - result += jpToHiragana(part, sourceMapping, result.length); + return result; } - return result; -} -function jpToHiragana(text, sourceMapping, sourceMappingStart) { - const result = wanakana.toHiragana(text); - // Generate source mapping - if (Array.isArray(sourceMapping)) { - if (typeof sourceMappingStart !== 'number') { sourceMappingStart = 0; } - let i = 0; - let resultPos = 0; - const ii = text.length; - while (i < ii) { - // Find smallest matching substring - let iNext = i + 1; - let resultPosNext = result.length; - while (iNext < ii) { - const t = wanakana.toHiragana(text.substring(0, iNext)); - if (t === result.substring(0, t.length)) { - resultPosNext = t.length; - break; - } - ++iNext; - } + // Furigana distribution - // Merge characters - const removals = iNext - i - 1; - if (removals > 0) { - let sum = 0; - const vs = sourceMapping.splice(sourceMappingStart + 1, removals); - for (const v of vs) { sum += v; } - sourceMapping[sourceMappingStart] += sum; + function distributeFurigana(expression, reading) { + const fallback = [{furigana: reading, text: expression}]; + if (!reading) { + return fallback; + } + + let isAmbiguous = false; + const segmentize = (reading2, groups) => { + if (groups.length === 0 || isAmbiguous) { + return []; } - ++sourceMappingStart; - // Empty elements - const additions = resultPosNext - resultPos - 1; - for (let j = 0; j < additions; ++j) { - sourceMapping.splice(sourceMappingStart, 0, 0); - ++sourceMappingStart; + const group = groups[0]; + if (group.mode === 'kana') { + if (convertKatakanaToHiragana(reading2).startsWith(convertKatakanaToHiragana(group.text))) { + const readingLeft = reading2.substring(group.text.length); + const segs = segmentize(readingLeft, groups.splice(1)); + if (segs) { + return [{text: group.text}].concat(segs); + } + } + } else { + let foundSegments = null; + for (let i = reading2.length; i >= group.text.length; --i) { + const readingUsed = reading2.substring(0, i); + const readingLeft = reading2.substring(i); + const segs = segmentize(readingLeft, groups.slice(1)); + if (segs) { + if (foundSegments !== null) { + // more than one way to segmentize the tail, mark as ambiguous + isAmbiguous = true; + return null; + } + foundSegments = [{text: group.text, furigana: readingUsed}].concat(segs); + } + // there is only one way to segmentize the last non-kana group + if (groups.length === 1) { + break; + } + } + return foundSegments; + } + }; + + const groups = []; + let modePrev = null; + for (const c of expression) { + const codePoint = c.codePointAt(0); + const modeCurr = isCodePointKanji(codePoint) || codePoint === ITERATION_MARK_CODE_POINT ? 'kanji' : 'kana'; + if (modeCurr === modePrev) { + groups[groups.length - 1].text += c; + } else { + groups.push({mode: modeCurr, text: c}); + modePrev = modeCurr; } + } + + const segments = segmentize(reading, groups); + if (segments && !isAmbiguous) { + return segments; + } + return fallback; + } + + function distributeFuriganaInflected(expression, reading, source) { + const output = []; + + let stemLength = 0; + const shortest = Math.min(source.length, expression.length); + const sourceHiragana = convertKatakanaToHiragana(source); + const expressionHiragana = convertKatakanaToHiragana(expression); + while (stemLength < shortest && sourceHiragana[stemLength] === expressionHiragana[stemLength]) { + ++stemLength; + } + const offset = source.length - stemLength; + + const stemExpression = source.substring(0, source.length - offset); + const stemReading = reading.substring( + 0, + offset === 0 ? reading.length : reading.length - expression.length + stemLength + ); + for (const segment of distributeFurigana(stemExpression, stemReading)) { + output.push(segment); + } - i = iNext; - resultPos = resultPosNext; + if (stemLength !== source.length) { + output.push({text: source.substring(stemLength)}); } + + return output; } - return result; -} + + // Exports + + return { + isCodePointKanji, + isCodePointKana, + isCodePointJapanese, + isStringEntirelyKana, + isStringPartiallyJapanese, + convertKatakanaToHiragana, + convertHiraganaToKatakana, + convertToRomaji, + convertReading, + convertNumericTofullWidth, + convertHalfWidthKanaToFullWidth, + convertAlphabeticToKana, + distributeFurigana, + distributeFuriganaInflected + }; +})(); diff --git a/ext/bg/js/translator.js b/ext/bg/js/translator.js index 25da9bf0..54d046cf 100644 --- a/ext/bg/js/translator.js +++ b/ext/bg/js/translator.js @@ -29,13 +29,7 @@ * dictTermsMergeBySequence * dictTermsSort * dictTermsUndupe - * jpConvertAlphabeticToKana - * jpConvertHalfWidthKanaToFullWidth - * jpConvertNumericTofullWidth - * jpDistributeFurigana - * jpHiraganaToKatakana - * jpIsCodePointJapanese - * jpKatakanaToHiragana + * jp * requestJson */ @@ -275,7 +269,7 @@ class Translator { const termTags = await this.expandTags(definition.termTags, definition.dictionary); const {expression, reading} = definition; - const furiganaSegments = jpDistributeFurigana(expression, reading); + const furiganaSegments = jp.distributeFurigana(expression, reading); definitions.push({ source: deinflection.source, @@ -376,20 +370,20 @@ class Translator { let sourceMapping = null; if (halfWidth) { if (sourceMapping === null) { sourceMapping = Translator.createTextSourceMapping(text2); } - text2 = jpConvertHalfWidthKanaToFullWidth(text2, sourceMapping); + text2 = jp.convertHalfWidthKanaToFullWidth(text2, sourceMapping); } if (numeric) { - text2 = jpConvertNumericTofullWidth(text2); + text2 = jp.convertNumericTofullWidth(text2); } if (alphabetic) { if (sourceMapping === null) { sourceMapping = Translator.createTextSourceMapping(text2); } - text2 = jpConvertAlphabeticToKana(text2, sourceMapping); + text2 = jp.convertAlphabeticToKana(text2, sourceMapping); } if (katakana) { - text2 = jpHiraganaToKatakana(text2); + text2 = jp.convertHiraganaToKatakana(text2); } if (hiragana) { - text2 = jpKatakanaToHiragana(text2); + text2 = jp.convertKatakanaToHiragana(text2); } for (let i = text2.length; i > 0; --i) { @@ -590,7 +584,7 @@ class Translator { } static createExpression(expression, reading, termTags=null, termFrequency=null) { - const furiganaSegments = jpDistributeFurigana(expression, reading); + const furiganaSegments = jp.distributeFurigana(expression, reading); return { expression, reading, @@ -639,7 +633,7 @@ class Translator { if (!options.scanning.alphanumeric) { let newText = ''; for (const c of text) { - if (!jpIsCodePointJapanese(c.codePointAt(0))) { + if (!jp.isCodePointJapanese(c.codePointAt(0))) { break; } newText += c; -- cgit v1.2.3 From 264820f2087e7dee13e358ba703d3dd863ed7faa Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sat, 14 Mar 2020 16:11:07 -0400 Subject: Add more unicode code point ranges --- ext/bg/js/japanese.js | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) (limited to 'ext/bg/js') diff --git a/ext/bg/js/japanese.js b/ext/bg/js/japanese.js index 182d5b98..4c2df674 100644 --- a/ext/bg/js/japanese.js +++ b/ext/bg/js/japanese.js @@ -84,9 +84,24 @@ const jp = (() => { const KATAKANA_RANGE = [0x30a0, 0x30ff]; const KANA_RANGES = [HIRAGANA_RANGE, KATAKANA_RANGE]; - const CJK_COMMON_RANGE = [0x4e00, 0x9fff]; - const CJK_RARE_RANGE = [0x3400, 0x4dbf]; - const CJK_RANGES = [CJK_COMMON_RANGE, CJK_RARE_RANGE]; + const CJK_UNIFIED_IDEOGRAPHS_RANGE = [0x4e00, 0x9fff]; + const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A_RANGE = [0x3400, 0x4dbf]; + const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B_RANGE = [0x20000, 0x2a6df]; + const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_C_RANGE = [0x2a700, 0x2b73f]; + const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_D_RANGE = [0x2b740, 0x2b81f]; + const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_E_RANGE = [0x2b820, 0x2ceaf]; + const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_F_RANGE = [0x2ceb0, 0x2ebef]; + const CJK_COMPATIBILITY_IDEOGRAPHS_SUPPLEMENT_RANGE = [0x2f800, 0x2fa1f]; + const CJK_UNIFIED_IDEOGRAPHS_RANGES = [ + CJK_UNIFIED_IDEOGRAPHS_RANGE, + CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A_RANGE, + CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B_RANGE, + CJK_UNIFIED_IDEOGRAPHS_EXTENSION_C_RANGE, + CJK_UNIFIED_IDEOGRAPHS_EXTENSION_D_RANGE, + CJK_UNIFIED_IDEOGRAPHS_EXTENSION_E_RANGE, + CJK_UNIFIED_IDEOGRAPHS_EXTENSION_F_RANGE, + CJK_COMPATIBILITY_IDEOGRAPHS_SUPPLEMENT_RANGE + ]; const ITERATION_MARK_CODE_POINT = 0x3005; @@ -95,8 +110,7 @@ const jp = (() => { HIRAGANA_RANGE, KATAKANA_RANGE, - CJK_COMMON_RANGE, - CJK_RARE_RANGE, + ...CJK_UNIFIED_IDEOGRAPHS_RANGES, [0xff66, 0xff9f], // Halfwidth katakana @@ -119,7 +133,7 @@ const jp = (() => { // Character code testing functions function isCodePointKanji(codePoint) { - return isCodePointInRanges(codePoint, CJK_RANGES); + return isCodePointInRanges(codePoint, CJK_UNIFIED_IDEOGRAPHS_RANGES); } function isCodePointKana(codePoint) { -- cgit v1.2.3 From 248a18dd72c687a470246c26d5c74e440058bf55 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sat, 14 Mar 2020 16:38:12 -0400 Subject: Fix case issue --- ext/bg/js/japanese.js | 4 ++-- ext/bg/js/translator.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'ext/bg/js') diff --git a/ext/bg/js/japanese.js b/ext/bg/js/japanese.js index 4c2df674..fa40fc98 100644 --- a/ext/bg/js/japanese.js +++ b/ext/bg/js/japanese.js @@ -231,7 +231,7 @@ const jp = (() => { } } - function convertNumericTofullWidth(text) { + function convertNumericToFullWidth(text) { let result = ''; for (const char of text) { let c = char.codePointAt(0); @@ -485,7 +485,7 @@ const jp = (() => { convertHiraganaToKatakana, convertToRomaji, convertReading, - convertNumericTofullWidth, + convertNumericToFullWidth, convertHalfWidthKanaToFullWidth, convertAlphabeticToKana, distributeFurigana, diff --git a/ext/bg/js/translator.js b/ext/bg/js/translator.js index 54d046cf..6f43f7b0 100644 --- a/ext/bg/js/translator.js +++ b/ext/bg/js/translator.js @@ -373,7 +373,7 @@ class Translator { text2 = jp.convertHalfWidthKanaToFullWidth(text2, sourceMapping); } if (numeric) { - text2 = jp.convertNumericTofullWidth(text2); + text2 = jp.convertNumericToFullWidth(text2); } if (alphabetic) { if (sourceMapping === null) { sourceMapping = Translator.createTextSourceMapping(text2); } -- cgit v1.2.3 From a50b76fd219b873df7bb7e3b6a1b03850c59f239 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sat, 14 Mar 2020 17:10:53 -0400 Subject: Remove unnecessary sourceMapping population in convertAlphabeticToKana --- ext/bg/js/japanese.js | 6 ------ 1 file changed, 6 deletions(-) (limited to 'ext/bg/js') diff --git a/ext/bg/js/japanese.js b/ext/bg/js/japanese.js index fa40fc98..d2a577e6 100644 --- a/ext/bg/js/japanese.js +++ b/ext/bg/js/japanese.js @@ -293,12 +293,6 @@ const jp = (() => { function convertAlphabeticToKana(text, sourceMapping) { let part = ''; let result = ''; - const ii = text.length; - - if (sourceMapping.length === ii) { - sourceMapping.length = ii; - sourceMapping.fill(1); - } for (const char of text) { // Note: 0x61 is the character code for 'a' -- cgit v1.2.3 From 77a2cc60e9a4a89da354cadb1bf060204ee3b951 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sat, 21 Mar 2020 13:18:34 -0400 Subject: Move basic string/character testing functions into a mixed/js/japanese.js --- ext/bg/background.html | 1 + ext/bg/js/japanese.js | 106 +++------------------------------------- ext/bg/search.html | 1 + ext/bg/settings.html | 1 + ext/mixed/js/japanese.js | 124 +++++++++++++++++++++++++++++++++++++++++++++++ test/test-japanese.js | 1 + 6 files changed, 135 insertions(+), 99 deletions(-) create mode 100644 ext/mixed/js/japanese.js (limited to 'ext/bg/js') diff --git a/ext/bg/background.html b/ext/bg/background.html index 44abe8fd..f7cf6e55 100644 --- a/ext/bg/background.html +++ b/ext/bg/background.html @@ -20,6 +20,7 @@ + diff --git a/ext/bg/js/japanese.js b/ext/bg/js/japanese.js index d2a577e6..c5873cf1 100644 --- a/ext/bg/js/japanese.js +++ b/ext/bg/js/japanese.js @@ -17,10 +17,11 @@ */ /* global + * jp * wanakana */ -const jp = (() => { +(() => { const HALFWIDTH_KATAKANA_MAPPING = new Map([ ['ｦ', 'ヲヺ-'], ['ｧ', 'ァ--'], @@ -80,101 +81,13 @@ const jp = (() => { ['ﾝ', 'ン--'] ]); - const HIRAGANA_RANGE = [0x3040, 0x309f]; - const KATAKANA_RANGE = [0x30a0, 0x30ff]; - const KANA_RANGES = [HIRAGANA_RANGE, KATAKANA_RANGE]; - - const CJK_UNIFIED_IDEOGRAPHS_RANGE = [0x4e00, 0x9fff]; - const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A_RANGE = [0x3400, 0x4dbf]; - const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B_RANGE = [0x20000, 0x2a6df]; - const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_C_RANGE = [0x2a700, 0x2b73f]; - const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_D_RANGE = [0x2b740, 0x2b81f]; - const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_E_RANGE = [0x2b820, 0x2ceaf]; - const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_F_RANGE = [0x2ceb0, 0x2ebef]; - const CJK_COMPATIBILITY_IDEOGRAPHS_SUPPLEMENT_RANGE = [0x2f800, 0x2fa1f]; - const CJK_UNIFIED_IDEOGRAPHS_RANGES = [ - CJK_UNIFIED_IDEOGRAPHS_RANGE, - CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A_RANGE, - CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B_RANGE, - CJK_UNIFIED_IDEOGRAPHS_EXTENSION_C_RANGE, - CJK_UNIFIED_IDEOGRAPHS_EXTENSION_D_RANGE, - CJK_UNIFIED_IDEOGRAPHS_EXTENSION_E_RANGE, - CJK_UNIFIED_IDEOGRAPHS_EXTENSION_F_RANGE, - CJK_COMPATIBILITY_IDEOGRAPHS_SUPPLEMENT_RANGE - ]; - const ITERATION_MARK_CODE_POINT = 0x3005; - // Japanese character ranges, roughly ordered in order of expected frequency - const JAPANESE_RANGES = [ - HIRAGANA_RANGE, - KATAKANA_RANGE, - - ...CJK_UNIFIED_IDEOGRAPHS_RANGES, - - [0xff66, 0xff9f], // Halfwidth katakana - - [0x30fb, 0x30fc], // Katakana punctuation - [0xff61, 0xff65], // Kana punctuation - [0x3000, 0x303f], // CJK punctuation - - [0xff10, 0xff19], // Fullwidth numbers - [0xff21, 0xff3a], // Fullwidth upper case Latin letters - [0xff41, 0xff5a], // Fullwidth lower case Latin letters - - [0xff01, 0xff0f], // Fullwidth punctuation 1 - [0xff1a, 0xff1f], // Fullwidth punctuation 2 - [0xff3b, 0xff3f], // Fullwidth punctuation 3 - [0xff5b, 0xff60], // Fullwidth punctuation 4 - [0xffe0, 0xffee] // Currency markers - ]; - - - // Character code testing functions - - function isCodePointKanji(codePoint) { - return isCodePointInRanges(codePoint, CJK_UNIFIED_IDEOGRAPHS_RANGES); - } - - function isCodePointKana(codePoint) { - return isCodePointInRanges(codePoint, KANA_RANGES); - } - - function isCodePointJapanese(codePoint) { - return isCodePointInRanges(codePoint, JAPANESE_RANGES); - } - function isCodePointInRanges(codePoint, ranges) { - for (const [min, max] of ranges) { - if (codePoint >= min && codePoint <= max) { - return true; - } - } - return false; - } + // Existing functions - - // String testing functions - - function isStringEntirelyKana(str) { - if (str.length === 0) { return false; } - for (const c of str) { - if (!isCodePointKana(c.codePointAt(0))) { - return false; - } - } - return true; - } - - function isStringPartiallyJapanese(str) { - if (str.length === 0) { return false; } - for (const c of str) { - if (isCodePointJapanese(c.codePointAt(0))) { - return true; - } - } - return false; - } + const isCodePointKanji = jp.isCodePointKanji; + const isStringEntirelyKana = jp.isStringEntirelyKana; // Conversion functions @@ -469,12 +382,7 @@ const jp = (() => { // Exports - return { - isCodePointKanji, - isCodePointKana, - isCodePointJapanese, - isStringEntirelyKana, - isStringPartiallyJapanese, + Object.assign(jp, { convertKatakanaToHiragana, convertHiraganaToKatakana, convertToRomaji, @@ -484,5 +392,5 @@ const jp = (() => { convertAlphabeticToKana, distributeFurigana, distributeFuriganaInflected - }; + }); })(); diff --git a/ext/bg/search.html b/ext/bg/search.html index f4c1a737..eacc1893 100644 --- a/ext/bg/search.html +++ b/ext/bg/search.html @@ -74,6 +74,7 @@ + diff --git a/ext/bg/settings.html b/ext/bg/settings.html index 0db76d71..cfe20be4 100644 --- a/ext/bg/settings.html +++ b/ext/bg/settings.html @@ -1088,6 +1088,7 @@ + diff --git a/ext/mixed/js/japanese.js b/ext/mixed/js/japanese.js new file mode 100644 index 00000000..61a247b2 --- /dev/null +++ b/ext/mixed/js/japanese.js @@ -0,0 +1,124 @@ +/* + * Copyright (C) 2020 Alex Yatskov + * Author: Alex Yatskov + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +const jp = (() => { + const HIRAGANA_RANGE = [0x3040, 0x309f]; + const KATAKANA_RANGE = [0x30a0, 0x30ff]; + const KANA_RANGES = [HIRAGANA_RANGE, KATAKANA_RANGE]; + + const CJK_UNIFIED_IDEOGRAPHS_RANGE = [0x4e00, 0x9fff]; + const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A_RANGE = [0x3400, 0x4dbf]; + const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B_RANGE = [0x20000, 0x2a6df]; + const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_C_RANGE = [0x2a700, 0x2b73f]; + const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_D_RANGE = [0x2b740, 0x2b81f]; + const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_E_RANGE = [0x2b820, 0x2ceaf]; + const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_F_RANGE = [0x2ceb0, 0x2ebef]; + const CJK_COMPATIBILITY_IDEOGRAPHS_SUPPLEMENT_RANGE = [0x2f800, 0x2fa1f]; + const CJK_UNIFIED_IDEOGRAPHS_RANGES = [ + CJK_UNIFIED_IDEOGRAPHS_RANGE, + CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A_RANGE, + CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B_RANGE, + CJK_UNIFIED_IDEOGRAPHS_EXTENSION_C_RANGE, + CJK_UNIFIED_IDEOGRAPHS_EXTENSION_D_RANGE, + CJK_UNIFIED_IDEOGRAPHS_EXTENSION_E_RANGE, + CJK_UNIFIED_IDEOGRAPHS_EXTENSION_F_RANGE, + CJK_COMPATIBILITY_IDEOGRAPHS_SUPPLEMENT_RANGE + ]; + + // Japanese character ranges, roughly ordered in order of expected frequency + const JAPANESE_RANGES = [ + HIRAGANA_RANGE, + KATAKANA_RANGE, + + ...CJK_UNIFIED_IDEOGRAPHS_RANGES, + + [0xff66, 0xff9f], // Halfwidth katakana + + [0x30fb, 0x30fc], // Katakana punctuation + [0xff61, 0xff65], // Kana punctuation + [0x3000, 0x303f], // CJK punctuation + + [0xff10, 0xff19], // Fullwidth numbers + [0xff21, 0xff3a], // Fullwidth upper case Latin letters + [0xff41, 0xff5a], // Fullwidth lower case Latin letters + + [0xff01, 0xff0f], // Fullwidth punctuation 1 + [0xff1a, 0xff1f], // Fullwidth punctuation 2 + [0xff3b, 0xff3f], // Fullwidth punctuation 3 + [0xff5b, 0xff60], // Fullwidth punctuation 4 + [0xffe0, 0xffee] // Currency markers + ]; + + + // Character code testing functions + + function isCodePointKanji(codePoint) { + return isCodePointInRanges(codePoint, CJK_UNIFIED_IDEOGRAPHS_RANGES); + } + + function isCodePointKana(codePoint) { + return isCodePointInRanges(codePoint, KANA_RANGES); + } + + function isCodePointJapanese(codePoint) { + return isCodePointInRanges(codePoint, JAPANESE_RANGES); + } + + function isCodePointInRanges(codePoint, ranges) { + for (const [min, max] of ranges) { + if (codePoint >= min && codePoint <= max) { + return true; + } + } + return false; + } + + + // String testing functions + + function isStringEntirelyKana(str) { + if (str.length === 0) { return false; } + for (const c of str) { + if (!isCodePointKana(c.codePointAt(0))) { + return false; + } + } + return true; + } + + function isStringPartiallyJapanese(str) { + if (str.length === 0) { return false; } + for (const c of str) { + if (isCodePointJapanese(c.codePointAt(0))) { + return true; + } + } + return false; + } + + + // Exports + + return { + isCodePointKanji, + isCodePointKana, + isCodePointJapanese, + isStringEntirelyKana, + isStringPartiallyJapanese + }; +})(); diff --git a/test/test-japanese.js b/test/test-japanese.js index 78f63c0b..32e4d176 100644 --- a/test/test-japanese.js +++ b/test/test-japanese.js @@ -22,6 +22,7 @@ const {VM} = require('./yomichan-vm'); const vm = new VM(); vm.execute([ 'mixed/lib/wanakana.min.js', + 'mixed/js/japanese.js', 'bg/js/japanese.js' ]); const jp = vm.get('jp'); -- cgit v1.2.3