From fafa746a632b1907d9cca262f689d7bec4e0f940 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sat, 17 Jul 2021 17:10:25 -0400 Subject: Sandbox script folders (#1837) * Move scripts * Update paths * Fix ordering * Simplify eslint rules --- ext/js/language/dictionary-data-util.js | 299 ---------- ext/js/language/japanese-util.js | 716 ------------------------ ext/js/language/sandbox/dictionary-data-util.js | 299 ++++++++++ ext/js/language/sandbox/japanese-util.js | 716 ++++++++++++++++++++++++ 4 files changed, 1015 insertions(+), 1015 deletions(-) delete mode 100644 ext/js/language/dictionary-data-util.js delete mode 100644 ext/js/language/japanese-util.js create mode 100644 ext/js/language/sandbox/dictionary-data-util.js create mode 100644 ext/js/language/sandbox/japanese-util.js (limited to 'ext/js/language') diff --git a/ext/js/language/dictionary-data-util.js b/ext/js/language/dictionary-data-util.js deleted file mode 100644 index 951e10ff..00000000 --- a/ext/js/language/dictionary-data-util.js +++ /dev/null @@ -1,299 +0,0 @@ -/* - * Copyright (C) 2020-2021 Yomichan Authors - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -class DictionaryDataUtil { - static groupTermTags(dictionaryEntry) { - const {headwords} = dictionaryEntry; - const headwordCount = headwords.length; - const uniqueCheck = (headwordCount > 1); - const resultsIndexMap = new Map(); - const results = []; - for (let i = 0; i < headwordCount; ++i) { - const {tags} = headwords[i]; - for (const tag of tags) { - if (uniqueCheck) { - const {name, category, notes, dictionary} = tag; - const key = this._createMapKey([name, category, notes, dictionary]); - const index = resultsIndexMap.get(key); - if (typeof index !== 'undefined') { - const existingItem = results[index]; - existingItem.headwordIndices.push(i); - continue; - } - resultsIndexMap.set(key, results.length); - } - - const item = {tag, headwordIndices: [i]}; - results.push(item); - } - } - return results; - } - - static groupTermFrequencies(dictionaryEntry) { - const {headwords, frequencies} = dictionaryEntry; - - const map1 = new Map(); - for (const {headwordIndex, dictionary, hasReading, frequency} of frequencies) { - const {term, reading} = headwords[headwordIndex]; - - let map2 = map1.get(dictionary); - if (typeof map2 === 'undefined') { - map2 = new Map(); - map1.set(dictionary, map2); - } - - const readingKey = hasReading ? reading : null; - const key = this._createMapKey([term, readingKey]); - let frequencyData = map2.get(key); - if (typeof frequencyData === 'undefined') { - frequencyData = {term, reading: readingKey, values: new Set()}; - map2.set(key, frequencyData); - } - - frequencyData.values.add(frequency); - } - return this._createFrequencyGroupsFromMap(map1); - } - - static groupKanjiFrequencies(frequencies) { - const map1 = new Map(); - for (const {dictionary, character, frequency} of frequencies) { - let map2 = map1.get(dictionary); - if (typeof map2 === 'undefined') { - map2 = new Map(); - map1.set(dictionary, map2); - } - - let frequencyData = map2.get(character); - if (typeof frequencyData === 'undefined') { - frequencyData = {character, values: new Set()}; - map2.set(character, frequencyData); - } - - frequencyData.values.add(frequency); - } - return this._createFrequencyGroupsFromMap(map1); - } - - static getPitchAccentInfos(dictionaryEntry) { - const {headwords, pronunciations} = dictionaryEntry; - - const allTerms = new Set(); - const allReadings = new Set(); - for (const {term, reading} of headwords) { - allTerms.add(term); - allReadings.add(reading); - } - - const pitchAccentInfoMap = new Map(); - for (const {headwordIndex, dictionary, pitches} of pronunciations) { - const {term, reading} = headwords[headwordIndex]; - let dictionaryPitchAccentInfoList = pitchAccentInfoMap.get(dictionary); - if (typeof dictionaryPitchAccentInfoList === 'undefined') { - dictionaryPitchAccentInfoList = []; - pitchAccentInfoMap.set(dictionary, dictionaryPitchAccentInfoList); - } - for (const {position, nasalPositions, devoicePositions, tags} of pitches) { - let pitchAccentInfo = this._findExistingPitchAccentInfo(reading, position, nasalPositions, devoicePositions, tags, dictionaryPitchAccentInfoList); - if (pitchAccentInfo === null) { - pitchAccentInfo = { - terms: new Set(), - reading, - position, - nasalPositions, - devoicePositions, - tags, - exclusiveTerms: [], - exclusiveReadings: [] - }; - dictionaryPitchAccentInfoList.push(pitchAccentInfo); - } - pitchAccentInfo.terms.add(term); - } - } - - const multipleReadings = (allReadings.size > 1); - for (const dictionaryPitchAccentInfoList of pitchAccentInfoMap.values()) { - for (const pitchAccentInfo of dictionaryPitchAccentInfoList) { - const {terms, reading, exclusiveTerms, exclusiveReadings} = pitchAccentInfo; - if (!this._areSetsEqual(terms, allTerms)) { - exclusiveTerms.push(...this._getSetIntersection(terms, allTerms)); - } - if (multipleReadings) { - exclusiveReadings.push(reading); - } - pitchAccentInfo.terms = [...terms]; - } - } - - const results2 = []; - for (const [dictionary, pitches] of pitchAccentInfoMap.entries()) { - results2.push({dictionary, pitches}); - } - return results2; - } - - static getTermFrequency(termTags) { - let totalScore = 0; - for (const {score} of termTags) { - totalScore += score; - } - if (totalScore > 0) { - return 'popular'; - } else if (totalScore < 0) { - return 'rare'; - } else { - return 'normal'; - } - } - - static getDisambiguations(headwords, headwordIndices, allTermsSet, allReadingsSet) { - if (allTermsSet.size <= 1 && allReadingsSet.size <= 1) { return []; } - - const terms = new Set(); - const readings = new Set(); - for (const headwordIndex of headwordIndices) { - const {term, reading} = headwords[headwordIndex]; - terms.add(term); - readings.add(reading); - } - - const disambiguations = []; - const addTerms = !this._areSetsEqual(terms, allTermsSet); - const addReadings = !this._areSetsEqual(readings, allReadingsSet); - if (addTerms) { - disambiguations.push(...this._getSetIntersection(terms, allTermsSet)); - } - if (addReadings) { - if (addTerms) { - for (const term of terms) { - readings.delete(term); - } - } - disambiguations.push(...this._getSetIntersection(readings, allReadingsSet)); - } - return disambiguations; - } - - static isNonNounVerbOrAdjective(wordClasses) { - let isVerbOrAdjective = false; - let isSuruVerb = false; - let isNoun = false; - for (const wordClass of wordClasses) { - switch (wordClass) { - case 'v1': - case 'v5': - case 'vk': - case 'vz': - case 'adj-i': - isVerbOrAdjective = true; - break; - case 'vs': - isVerbOrAdjective = true; - isSuruVerb = true; - break; - case 'n': - isNoun = true; - break; - } - } - return isVerbOrAdjective && !(isSuruVerb && isNoun); - } - - // Private - - static _createFrequencyGroupsFromMap(map) { - const results = []; - for (const [dictionary, map2] of map.entries()) { - const frequencies = []; - for (const frequencyData of map2.values()) { - frequencyData.values = [...frequencyData.values]; - frequencies.push(frequencyData); - } - results.push({dictionary, frequencies}); - } - return results; - } - - static _findExistingPitchAccentInfo(reading, position, nasalPositions, devoicePositions, tags, pitchAccentInfoList) { - for (const pitchInfo of pitchAccentInfoList) { - if ( - pitchInfo.reading === reading && - pitchInfo.position === position && - this._areArraysEqual(pitchInfo.nasalPositions, nasalPositions) && - this._areArraysEqual(pitchInfo.devoicePositions, devoicePositions) && - this._areTagListsEqual(pitchInfo.tags, tags) - ) { - return pitchInfo; - } - } - return null; - } - - static _areArraysEqual(array1, array2) { - const ii = array1.length; - if (ii !== array2.length) { return false; } - for (let i = 0; i < ii; ++i) { - if (array1[i] !== array2[i]) { return false; } - } - return true; - } - - static _areTagListsEqual(tagList1, tagList2) { - const ii = tagList1.length; - if (tagList2.length !== ii) { return false; } - - for (let i = 0; i < ii; ++i) { - const tag1 = tagList1[i]; - const tag2 = tagList2[i]; - if (tag1.name !== tag2.name || tag1.dictionary !== tag2.dictionary) { - return false; - } - } - - return true; - } - - static _areSetsEqual(set1, set2) { - if (set1.size !== set2.size) { - return false; - } - - for (const value of set1) { - if (!set2.has(value)) { - return false; - } - } - - return true; - } - - static _getSetIntersection(set1, set2) { - const result = []; - for (const value of set1) { - if (set2.has(value)) { - result.push(value); - } - } - return result; - } - - static _createMapKey(array) { - return JSON.stringify(array); - } -} diff --git a/ext/js/language/japanese-util.js b/ext/js/language/japanese-util.js deleted file mode 100644 index c7f79751..00000000 --- a/ext/js/language/japanese-util.js +++ /dev/null @@ -1,716 +0,0 @@ -/* - * Copyright (C) 2020-2021 Yomichan Authors - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -const JapaneseUtil = (() => { - const ITERATION_MARK_CODE_POINT = 0x3005; - const HIRAGANA_SMALL_TSU_CODE_POINT = 0x3063; - const KATAKANA_SMALL_TSU_CODE_POINT = 0x30c3; - const KATAKANA_SMALL_KA_CODE_POINT = 0x30f5; - const KATAKANA_SMALL_KE_CODE_POINT = 0x30f6; - const KANA_PROLONGED_SOUND_MARK_CODE_POINT = 0x30fc; - - const HIRAGANA_RANGE = [0x3040, 0x309f]; - const KATAKANA_RANGE = [0x30a0, 0x30ff]; - - const HIRAGANA_CONVERSION_RANGE = [0x3041, 0x3096]; - const KATAKANA_CONVERSION_RANGE = [0x30a1, 0x30f6]; - - const KANA_RANGES = [HIRAGANA_RANGE, KATAKANA_RANGE]; - - const CJK_UNIFIED_IDEOGRAPHS_RANGE = [0x4e00, 0x9fff]; - const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A_RANGE = [0x3400, 0x4dbf]; - const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B_RANGE = [0x20000, 0x2a6df]; - const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_C_RANGE = [0x2a700, 0x2b73f]; - const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_D_RANGE = [0x2b740, 0x2b81f]; - const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_E_RANGE = [0x2b820, 0x2ceaf]; - const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_F_RANGE = [0x2ceb0, 0x2ebef]; - const CJK_COMPATIBILITY_IDEOGRAPHS_SUPPLEMENT_RANGE = [0x2f800, 0x2fa1f]; - const CJK_UNIFIED_IDEOGRAPHS_RANGES = [ - CJK_UNIFIED_IDEOGRAPHS_RANGE, - CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A_RANGE, - CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B_RANGE, - CJK_UNIFIED_IDEOGRAPHS_EXTENSION_C_RANGE, - CJK_UNIFIED_IDEOGRAPHS_EXTENSION_D_RANGE, - CJK_UNIFIED_IDEOGRAPHS_EXTENSION_E_RANGE, - CJK_UNIFIED_IDEOGRAPHS_EXTENSION_F_RANGE, - CJK_COMPATIBILITY_IDEOGRAPHS_SUPPLEMENT_RANGE - ]; - - // Japanese character ranges, roughly ordered in order of expected frequency - const JAPANESE_RANGES = [ - HIRAGANA_RANGE, - KATAKANA_RANGE, - - ...CJK_UNIFIED_IDEOGRAPHS_RANGES, - - [0xff66, 0xff9f], // Halfwidth katakana - - [0x30fb, 0x30fc], // Katakana punctuation - [0xff61, 0xff65], // Kana punctuation - [0x3000, 0x303f], // CJK punctuation - - [0xff10, 0xff19], // Fullwidth numbers - [0xff21, 0xff3a], // Fullwidth upper case Latin letters - [0xff41, 0xff5a], // Fullwidth lower case Latin letters - - [0xff01, 0xff0f], // Fullwidth punctuation 1 - [0xff1a, 0xff1f], // Fullwidth punctuation 2 - [0xff3b, 0xff3f], // Fullwidth punctuation 3 - [0xff5b, 0xff60], // Fullwidth punctuation 4 - [0xffe0, 0xffee] // Currency markers - ]; - - const SMALL_KANA_SET = new Set(Array.from('ぁぃぅぇぉゃゅょゎァィゥェォャュョヮ')); - - const HALFWIDTH_KATAKANA_MAPPING = new Map([ - ['ｦ', 'ヲヺ-'], - ['ｧ', 'ァ--'], - ['ｨ', 'ィ--'], - ['ｩ', 'ゥ--'], - ['ｪ', 'ェ--'], - ['ｫ', 'ォ--'], - ['ｬ', 'ャ--'], - ['ｭ', 'ュ--'], - ['ｮ', 'ョ--'], - ['ｯ', 'ッ--'], - ['ｰ', 'ー--'], - ['ｱ', 'ア--'], - ['ｲ', 'イ--'], - ['ｳ', 'ウヴ-'], - ['ｴ', 'エ--'], - ['ｵ', 'オ--'], - ['ｶ', 'カガ-'], - ['ｷ', 'キギ-'], - ['ｸ', 'クグ-'], - ['ｹ', 'ケゲ-'], - ['ｺ', 'コゴ-'], - ['ｻ', 'サザ-'], - ['ｼ', 'シジ-'], - ['ｽ', 'スズ-'], - ['ｾ', 'セゼ-'], - ['ｿ', 'ソゾ-'], - ['ﾀ', 'タダ-'], - ['ﾁ', 'チヂ-'], - ['ﾂ', 'ツヅ-'], - ['ﾃ', 'テデ-'], - ['ﾄ', 'トド-'], - ['ﾅ', 'ナ--'], - ['ﾆ', 'ニ--'], - ['ﾇ', 'ヌ--'], - ['ﾈ', 'ネ--'], - ['ﾉ', 'ノ--'], - ['ﾊ', 'ハバパ'], - ['ﾋ', 'ヒビピ'], - ['ﾌ', 'フブプ'], - ['ﾍ', 'ヘベペ'], - ['ﾎ', 'ホボポ'], - ['ﾏ', 'マ--'], - ['ﾐ', 'ミ--'], - ['ﾑ', 'ム--'], - ['ﾒ', 'メ--'], - ['ﾓ', 'モ--'], - ['ﾔ', 'ヤ--'], - ['ﾕ', 'ユ--'], - ['ﾖ', 'ヨ--'], - ['ﾗ', 'ラ--'], - ['ﾘ', 'リ--'], - ['ﾙ', 'ル--'], - ['ﾚ', 'レ--'], - ['ﾛ', 'ロ--'], - ['ﾜ', 'ワ--'], - ['ﾝ', 'ン--'] - ]); - - const VOWEL_TO_KANA_MAPPING = new Map([ - ['a', 'ぁあかがさざただなはばぱまゃやらゎわヵァアカガサザタダナハバパマャヤラヮワヵヷ'], - ['i', 'ぃいきぎしじちぢにひびぴみりゐィイキギシジチヂニヒビピミリヰヸ'], - ['u', 'ぅうくぐすずっつづぬふぶぷむゅゆるゥウクグスズッツヅヌフブプムュユルヴ'], - ['e', 'ぇえけげせぜてでねへべぺめれゑヶェエケゲセゼテデネヘベペメレヱヶヹ'], - ['o', 'ぉおこごそぞとどのほぼぽもょよろをォオコゴソゾトドノホボポモョヨロヲヺ'], - ['', 'のノ'] - ]); - - const KANA_TO_VOWEL_MAPPING = (() => { - const map = new Map(); - for (const [vowel, characters] of VOWEL_TO_KANA_MAPPING) { - for (const character of characters) { - map.set(character, vowel); - } - } - return map; - })(); - - const DIACRITIC_MAPPING = (() => { - const kana = 'うゔ-かが-きぎ-くぐ-けげ-こご-さざ-しじ-すず-せぜ-そぞ-ただ-ちぢ-つづ-てで-とど-はばぱひびぴふぶぷへべぺほぼぽワヷ-ヰヸ-ウヴ-ヱヹ-ヲヺ-カガ-キギ-クグ-ケゲ-コゴ-サザ-シジ-スズ-セゼ-ソゾ-タダ-チヂ-ツヅ-テデ-トド-ハバパヒビピフブプヘベペホボポ'; - const map = new Map(); - for (let i = 0, ii = kana.length; i < ii; i += 3) { - const character = kana[i]; - const dakuten = kana[i + 1]; - const handakuten = kana[i + 2]; - map.set(dakuten, {character, type: 'dakuten'}); - if (handakuten !== '-') { - map.set(handakuten, {character, type: 'handakuten'}); - } - } - return map; - })(); - - - function isCodePointInRange(codePoint, [min, max]) { - return (codePoint >= min && codePoint <= max); - } - - function isCodePointInRanges(codePoint, ranges) { - for (const [min, max] of ranges) { - if (codePoint >= min && codePoint <= max) { - return true; - } - } - return false; - } - - function getProlongedHiragana(previousCharacter) { - switch (KANA_TO_VOWEL_MAPPING.get(previousCharacter)) { - case 'a': return 'あ'; - case 'i': return 'い'; - case 'u': return 'う'; - case 'e': return 'え'; - case 'o': return 'う'; - default: return null; - } - } - - - // eslint-disable-next-line no-shadow - class JapaneseUtil { - constructor(wanakana=null) { - this._wanakana = wanakana; - } - - // Character code testing functions - - isCodePointKanji(codePoint) { - return isCodePointInRanges(codePoint, CJK_UNIFIED_IDEOGRAPHS_RANGES); - } - - isCodePointKana(codePoint) { - return isCodePointInRanges(codePoint, KANA_RANGES); - } - - isCodePointJapanese(codePoint) { - return isCodePointInRanges(codePoint, JAPANESE_RANGES); - } - - // String testing functions - - isStringEntirelyKana(str) { - if (str.length === 0) { return false; } - for (const c of str) { - if (!isCodePointInRanges(c.codePointAt(0), KANA_RANGES)) { - return false; - } - } - return true; - } - - isStringPartiallyJapanese(str) { - if (str.length === 0) { return false; } - for (const c of str) { - if (isCodePointInRanges(c.codePointAt(0), JAPANESE_RANGES)) { - return true; - } - } - return false; - } - - // Mora functions - - isMoraPitchHigh(moraIndex, pitchAccentPosition) { - switch (pitchAccentPosition) { - case 0: return (moraIndex > 0); - case 1: return (moraIndex < 1); - default: return (moraIndex > 0 && moraIndex < pitchAccentPosition); - } - } - - getPitchCategory(text, pitchAccentPosition, isVerbOrAdjective) { - if (pitchAccentPosition === 0) { - return 'heiban'; - } - if (isVerbOrAdjective) { - return pitchAccentPosition > 0 ? 'kifuku' : null; - } - if (pitchAccentPosition === 1) { - return 'atamadaka'; - } - if (pitchAccentPosition > 1) { - return pitchAccentPosition >= this.getKanaMoraCount(text) ? 'odaka' : 'nakadaka'; - } - return null; - } - - getKanaMorae(text) { - const morae = []; - let i; - for (const c of text) { - if (SMALL_KANA_SET.has(c) && (i = morae.length) > 0) { - morae[i - 1] += c; - } else { - morae.push(c); - } - } - return morae; - } - - getKanaMoraCount(text) { - let moraCount = 0; - for (const c of text) { - if (!(SMALL_KANA_SET.has(c) && moraCount > 0)) { - ++moraCount; - } - } - return moraCount; - } - - // Conversion functions - - convertToKana(text) { - return this._getWanakana().toKana(text); - } - - convertToKanaSupported() { - return this._wanakana !== null; - } - - convertKatakanaToHiragana(text) { - let result = ''; - const offset = (HIRAGANA_CONVERSION_RANGE[0] - KATAKANA_CONVERSION_RANGE[0]); - for (let char of text) { - const codePoint = char.codePointAt(0); - if (codePoint === KATAKANA_SMALL_KA_CODE_POINT || codePoint === KATAKANA_SMALL_KE_CODE_POINT) { - // No change - } else if (codePoint === KANA_PROLONGED_SOUND_MARK_CODE_POINT) { - if (result.length > 0) { - const char2 = getProlongedHiragana(result[result.length - 1]); - if (char2 !== null) { char = char2; } - } - } else if (isCodePointInRange(codePoint, KATAKANA_CONVERSION_RANGE)) { - char = String.fromCodePoint(codePoint + offset); - } - result += char; - } - return result; - } - - convertHiraganaToKatakana(text) { - let result = ''; - const offset = (KATAKANA_CONVERSION_RANGE[0] - HIRAGANA_CONVERSION_RANGE[0]); - for (let char of text) { - const codePoint = char.codePointAt(0); - if (isCodePointInRange(codePoint, HIRAGANA_CONVERSION_RANGE)) { - char = String.fromCodePoint(codePoint + offset); - } - result += char; - } - return result; - } - - convertToRomaji(text) { - const wanakana = this._getWanakana(); - return wanakana.toRomaji(text); - } - - convertToRomajiSupported() { - return this._wanakana !== null; - } - - convertNumericToFullWidth(text) { - let result = ''; - for (const char of text) { - let c = char.codePointAt(0); - if (c >= 0x30 && c <= 0x39) { // ['0', '9'] - c += 0xff10 - 0x30; // 0xff10 = '0' full width - result += String.fromCodePoint(c); - } else { - result += char; - } - } - return result; - } - - convertHalfWidthKanaToFullWidth(text, sourceMap=null) { - let result = ''; - - // This function is safe to use charCodeAt instead of codePointAt, since all - // the relevant characters are represented with a single UTF-16 character code. - for (let i = 0, ii = text.length; i < ii; ++i) { - const c = text[i]; - const mapping = HALFWIDTH_KATAKANA_MAPPING.get(c); - if (typeof mapping !== 'string') { - result += c; - continue; - } - - let index = 0; - switch (text.charCodeAt(i + 1)) { - case 0xff9e: // dakuten - index = 1; - break; - case 0xff9f: // handakuten - index = 2; - break; - } - - let c2 = mapping[index]; - if (index > 0) { - if (c2 === '-') { // invalid - index = 0; - c2 = mapping[0]; - } else { - ++i; - } - } - - if (sourceMap !== null && index > 0) { - sourceMap.combine(result.length, 1); - } - result += c2; - } - - return result; - } - - convertAlphabeticToKana(text, sourceMap=null) { - let part = ''; - let result = ''; - - for (const char of text) { - // Note: 0x61 is the character code for 'a' - let c = char.codePointAt(0); - if (c >= 0x41 && c <= 0x5a) { // ['A', 'Z'] - c += (0x61 - 0x41); - } else if (c >= 0x61 && c <= 0x7a) { // ['a', 'z'] - // NOP; c += (0x61 - 0x61); - } else if (c >= 0xff21 && c <= 0xff3a) { // ['A', 'Z'] fullwidth - c += (0x61 - 0xff21); - } else if (c >= 0xff41 && c <= 0xff5a) { // ['a', 'z'] fullwidth - c += (0x61 - 0xff41); - } else if (c === 0x2d || c === 0xff0d) { // '-' or fullwidth dash - c = 0x2d; // '-' - } else { - if (part.length > 0) { - result += this._convertAlphabeticPartToKana(part, sourceMap, result.length); - part = ''; - } - result += char; - continue; - } - part += String.fromCodePoint(c); - } - - if (part.length > 0) { - result += this._convertAlphabeticPartToKana(part, sourceMap, result.length); - } - return result; - } - - convertAlphabeticToKanaSupported() { - return this._wanakana !== null; - } - - getKanaDiacriticInfo(character) { - const info = DIACRITIC_MAPPING.get(character); - return typeof info !== 'undefined' ? {character: info.character, type: info.type} : null; - } - - // Furigana distribution - - distributeFurigana(term, reading) { - if (reading === term) { - // Same - return [this._createFuriganaSegment(term, '')]; - } - - const groups = []; - let groupPre = null; - let isKanaPre = null; - for (const c of term) { - const codePoint = c.codePointAt(0); - const isKana = !(this.isCodePointKanji(codePoint) || codePoint === ITERATION_MARK_CODE_POINT); - if (isKana === isKanaPre) { - groupPre.text += c; - } else { - groupPre = {isKana, text: c, textNormalized: null}; - groups.push(groupPre); - isKanaPre = isKana; - } - } - for (const group of groups) { - if (group.isKana) { - group.textNormalized = this.convertKatakanaToHiragana(group.text); - } - } - - const readingNormalized = this.convertKatakanaToHiragana(reading); - const segments = this._segmentizeFurigana(reading, readingNormalized, groups, 0); - if (segments !== null) { - return segments; - } - - // Fallback - return [this._createFuriganaSegment(term, reading)]; - } - - distributeFuriganaInflected(term, reading, source) { - const termNormalized = this.convertKatakanaToHiragana(term); - const readingNormalized = this.convertKatakanaToHiragana(reading); - const sourceNormalized = this.convertKatakanaToHiragana(source); - - let mainText = term; - let stemLength = this._getStemLength(termNormalized, sourceNormalized); - - // Check if source is derived from the reading instead of the term - const readingStemLength = this._getStemLength(readingNormalized, sourceNormalized); - if (readingStemLength > 0 && readingStemLength >= stemLength) { - mainText = reading; - stemLength = readingStemLength; - reading = `${source.substring(0, stemLength)}${reading.substring(stemLength)}`; - } - - const segments = []; - if (stemLength > 0) { - mainText = `${source.substring(0, stemLength)}${mainText.substring(stemLength)}`; - const segments2 = this.distributeFurigana(mainText, reading); - let consumed = 0; - for (const segment of segments2) { - const {text} = segment; - const start = consumed; - consumed += text.length; - if (consumed < stemLength) { - segments.push(segment); - } else if (consumed === stemLength) { - segments.push(segment); - break; - } else { - if (start < stemLength) { - segments.push(this._createFuriganaSegment(mainText.substring(start, stemLength), '')); - } - break; - } - } - } - - if (stemLength < source.length) { - const remainder = source.substring(stemLength); - const segmentCount = segments.length; - if (segmentCount > 0 && segments[segmentCount - 1].reading.length === 0) { - // Append to the last segment if it has an empty reading - segments[segmentCount - 1].text += remainder; - } else { - // Otherwise, create a new segment - segments.push(this._createFuriganaSegment(remainder, '')); - } - } - - return segments; - } - - // Miscellaneous - - collapseEmphaticSequences(text, fullCollapse, sourceMap=null) { - let result = ''; - let collapseCodePoint = -1; - const hasSourceMap = (sourceMap !== null); - for (const char of text) { - const c = char.codePointAt(0); - if ( - c === HIRAGANA_SMALL_TSU_CODE_POINT || - c === KATAKANA_SMALL_TSU_CODE_POINT || - c === KANA_PROLONGED_SOUND_MARK_CODE_POINT - ) { - if (collapseCodePoint !== c) { - collapseCodePoint = c; - if (!fullCollapse) { - result += char; - continue; - } - } - } else { - collapseCodePoint = -1; - result += char; - continue; - } - - if (hasSourceMap) { - sourceMap.combine(Math.max(0, result.length - 1), 1); - } - } - return result; - } - - // Private - - _createFuriganaSegment(text, reading) { - return {text, reading}; - } - - _segmentizeFurigana(reading, readingNormalized, groups, groupsStart) { - const groupCount = groups.length - groupsStart; - if (groupCount <= 0) { - return reading.length === 0 ? [] : null; - } - - const group = groups[groupsStart]; - const {isKana, text} = group; - const textLength = text.length; - if (isKana) { - const {textNormalized} = group; - if (readingNormalized.startsWith(textNormalized)) { - const segments = this._segmentizeFurigana( - reading.substring(textLength), - readingNormalized.substring(textLength), - groups, - groupsStart + 1 - ); - if (segments !== null) { - if (reading.startsWith(text)) { - segments.unshift(this._createFuriganaSegment(text, '')); - } else { - segments.unshift(...this._getFuriganaKanaSegments(text, reading)); - } - return segments; - } - } - return null; - } else { - let result = null; - for (let i = reading.length; i >= textLength; --i) { - const segments = this._segmentizeFurigana( - reading.substring(i), - readingNormalized.substring(i), - groups, - groupsStart + 1 - ); - if (segments !== null) { - if (result !== null) { - // More than one way to segmentize the tail; mark as ambiguous - return null; - } - const segmentReading = reading.substring(0, i); - segments.unshift(this._createFuriganaSegment(text, segmentReading)); - result = segments; - } - // There is only one way to segmentize the last non-kana group - if (groupCount === 1) { - break; - } - } - return result; - } - } - - _getFuriganaKanaSegments(text, reading) { - const textLength = text.length; - const newSegments = []; - let start = 0; - let state = (reading[0] === text[0]); - for (let i = 1; i < textLength; ++i) { - const newState = (reading[i] === text[i]); - if (state === newState) { continue; } - newSegments.push(this._createFuriganaSegment(text.substring(start, i), state ? '' : reading.substring(start, i))); - state = newState; - start = i; - } - newSegments.push(this._createFuriganaSegment(text.substring(start, textLength), state ? '' : reading.substring(start, textLength))); - return newSegments; - } - - _getWanakana() { - const wanakana = this._wanakana; - if (wanakana === null) { throw new Error('Functions which use WanaKana are not supported in this context'); } - return wanakana; - } - - _convertAlphabeticPartToKana(text, sourceMap, sourceMapStart) { - const wanakana = this._getWanakana(); - const result = wanakana.toHiragana(text); - - // Generate source mapping - if (sourceMap !== null) { - let i = 0; - let resultPos = 0; - const ii = text.length; - while (i < ii) { - // Find smallest matching substring - let iNext = i + 1; - let resultPosNext = result.length; - while (iNext < ii) { - const t = wanakana.toHiragana(text.substring(0, iNext)); - if (t === result.substring(0, t.length)) { - resultPosNext = t.length; - break; - } - ++iNext; - } - - // Merge characters - const removals = iNext - i - 1; - if (removals > 0) { - sourceMap.combine(sourceMapStart, removals); - } - ++sourceMapStart; - - // Empty elements - const additions = resultPosNext - resultPos - 1; - for (let j = 0; j < additions; ++j) { - sourceMap.insert(sourceMapStart, 0); - ++sourceMapStart; - } - - i = iNext; - resultPos = resultPosNext; - } - } - - return result; - } - - _getStemLength(text1, text2) { - const minLength = Math.min(text1.length, text2.length); - if (minLength === 0) { return 0; } - - let i = 0; - while (true) { - const char1 = text1.codePointAt(i); - const char2 = text2.codePointAt(i); - if (char1 !== char2) { break; } - const charLength = String.fromCodePoint(char1).length; - i += charLength; - if (i >= minLength) { - if (i > minLength) { - i -= charLength; // Don't consume partial UTF16 surrogate characters - } - break; - } - } - return i; - } - } - - - return JapaneseUtil; -})(); diff --git a/ext/js/language/sandbox/dictionary-data-util.js b/ext/js/language/sandbox/dictionary-data-util.js new file mode 100644 index 00000000..951e10ff --- /dev/null +++ b/ext/js/language/sandbox/dictionary-data-util.js @@ -0,0 +1,299 @@ +/* + * Copyright (C) 2020-2021 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +class DictionaryDataUtil { + static groupTermTags(dictionaryEntry) { + const {headwords} = dictionaryEntry; + const headwordCount = headwords.length; + const uniqueCheck = (headwordCount > 1); + const resultsIndexMap = new Map(); + const results = []; + for (let i = 0; i < headwordCount; ++i) { + const {tags} = headwords[i]; + for (const tag of tags) { + if (uniqueCheck) { + const {name, category, notes, dictionary} = tag; + const key = this._createMapKey([name, category, notes, dictionary]); + const index = resultsIndexMap.get(key); + if (typeof index !== 'undefined') { + const existingItem = results[index]; + existingItem.headwordIndices.push(i); + continue; + } + resultsIndexMap.set(key, results.length); + } + + const item = {tag, headwordIndices: [i]}; + results.push(item); + } + } + return results; + } + + static groupTermFrequencies(dictionaryEntry) { + const {headwords, frequencies} = dictionaryEntry; + + const map1 = new Map(); + for (const {headwordIndex, dictionary, hasReading, frequency} of frequencies) { + const {term, reading} = headwords[headwordIndex]; + + let map2 = map1.get(dictionary); + if (typeof map2 === 'undefined') { + map2 = new Map(); + map1.set(dictionary, map2); + } + + const readingKey = hasReading ? reading : null; + const key = this._createMapKey([term, readingKey]); + let frequencyData = map2.get(key); + if (typeof frequencyData === 'undefined') { + frequencyData = {term, reading: readingKey, values: new Set()}; + map2.set(key, frequencyData); + } + + frequencyData.values.add(frequency); + } + return this._createFrequencyGroupsFromMap(map1); + } + + static groupKanjiFrequencies(frequencies) { + const map1 = new Map(); + for (const {dictionary, character, frequency} of frequencies) { + let map2 = map1.get(dictionary); + if (typeof map2 === 'undefined') { + map2 = new Map(); + map1.set(dictionary, map2); + } + + let frequencyData = map2.get(character); + if (typeof frequencyData === 'undefined') { + frequencyData = {character, values: new Set()}; + map2.set(character, frequencyData); + } + + frequencyData.values.add(frequency); + } + return this._createFrequencyGroupsFromMap(map1); + } + + static getPitchAccentInfos(dictionaryEntry) { + const {headwords, pronunciations} = dictionaryEntry; + + const allTerms = new Set(); + const allReadings = new Set(); + for (const {term, reading} of headwords) { + allTerms.add(term); + allReadings.add(reading); + } + + const pitchAccentInfoMap = new Map(); + for (const {headwordIndex, dictionary, pitches} of pronunciations) { + const {term, reading} = headwords[headwordIndex]; + let dictionaryPitchAccentInfoList = pitchAccentInfoMap.get(dictionary); + if (typeof dictionaryPitchAccentInfoList === 'undefined') { + dictionaryPitchAccentInfoList = []; + pitchAccentInfoMap.set(dictionary, dictionaryPitchAccentInfoList); + } + for (const {position, nasalPositions, devoicePositions, tags} of pitches) { + let pitchAccentInfo = this._findExistingPitchAccentInfo(reading, position, nasalPositions, devoicePositions, tags, dictionaryPitchAccentInfoList); + if (pitchAccentInfo === null) { + pitchAccentInfo = { + terms: new Set(), + reading, + position, + nasalPositions, + devoicePositions, + tags, + exclusiveTerms: [], + exclusiveReadings: [] + }; + dictionaryPitchAccentInfoList.push(pitchAccentInfo); + } + pitchAccentInfo.terms.add(term); + } + } + + const multipleReadings = (allReadings.size > 1); + for (const dictionaryPitchAccentInfoList of pitchAccentInfoMap.values()) { + for (const pitchAccentInfo of dictionaryPitchAccentInfoList) { + const {terms, reading, exclusiveTerms, exclusiveReadings} = pitchAccentInfo; + if (!this._areSetsEqual(terms, allTerms)) { + exclusiveTerms.push(...this._getSetIntersection(terms, allTerms)); + } + if (multipleReadings) { + exclusiveReadings.push(reading); + } + pitchAccentInfo.terms = [...terms]; + } + } + + const results2 = []; + for (const [dictionary, pitches] of pitchAccentInfoMap.entries()) { + results2.push({dictionary, pitches}); + } + return results2; + } + + static getTermFrequency(termTags) { + let totalScore = 0; + for (const {score} of termTags) { + totalScore += score; + } + if (totalScore > 0) { + return 'popular'; + } else if (totalScore < 0) { + return 'rare'; + } else { + return 'normal'; + } + } + + static getDisambiguations(headwords, headwordIndices, allTermsSet, allReadingsSet) { + if (allTermsSet.size <= 1 && allReadingsSet.size <= 1) { return []; } + + const terms = new Set(); + const readings = new Set(); + for (const headwordIndex of headwordIndices) { + const {term, reading} = headwords[headwordIndex]; + terms.add(term); + readings.add(reading); + } + + const disambiguations = []; + const addTerms = !this._areSetsEqual(terms, allTermsSet); + const addReadings = !this._areSetsEqual(readings, allReadingsSet); + if (addTerms) { + disambiguations.push(...this._getSetIntersection(terms, allTermsSet)); + } + if (addReadings) { + if (addTerms) { + for (const term of terms) { + readings.delete(term); + } + } + disambiguations.push(...this._getSetIntersection(readings, allReadingsSet)); + } + return disambiguations; + } + + static isNonNounVerbOrAdjective(wordClasses) { + let isVerbOrAdjective = false; + let isSuruVerb = false; + let isNoun = false; + for (const wordClass of wordClasses) { + switch (wordClass) { + case 'v1': + case 'v5': + case 'vk': + case 'vz': + case 'adj-i': + isVerbOrAdjective = true; + break; + case 'vs': + isVerbOrAdjective = true; + isSuruVerb = true; + break; + case 'n': + isNoun = true; + break; + } + } + return isVerbOrAdjective && !(isSuruVerb && isNoun); + } + + // Private + + static _createFrequencyGroupsFromMap(map) { + const results = []; + for (const [dictionary, map2] of map.entries()) { + const frequencies = []; + for (const frequencyData of map2.values()) { + frequencyData.values = [...frequencyData.values]; + frequencies.push(frequencyData); + } + results.push({dictionary, frequencies}); + } + return results; + } + + static _findExistingPitchAccentInfo(reading, position, nasalPositions, devoicePositions, tags, pitchAccentInfoList) { + for (const pitchInfo of pitchAccentInfoList) { + if ( + pitchInfo.reading === reading && + pitchInfo.position === position && + this._areArraysEqual(pitchInfo.nasalPositions, nasalPositions) && + this._areArraysEqual(pitchInfo.devoicePositions, devoicePositions) && + this._areTagListsEqual(pitchInfo.tags, tags) + ) { + return pitchInfo; + } + } + return null; + } + + static _areArraysEqual(array1, array2) { + const ii = array1.length; + if (ii !== array2.length) { return false; } + for (let i = 0; i < ii; ++i) { + if (array1[i] !== array2[i]) { return false; } + } + return true; + } + + static _areTagListsEqual(tagList1, tagList2) { + const ii = tagList1.length; + if (tagList2.length !== ii) { return false; } + + for (let i = 0; i < ii; ++i) { + const tag1 = tagList1[i]; + const tag2 = tagList2[i]; + if (tag1.name !== tag2.name || tag1.dictionary !== tag2.dictionary) { + return false; + } + } + + return true; + } + + static _areSetsEqual(set1, set2) { + if (set1.size !== set2.size) { + return false; + } + + for (const value of set1) { + if (!set2.has(value)) { + return false; + } + } + + return true; + } + + static _getSetIntersection(set1, set2) { + const result = []; + for (const value of set1) { + if (set2.has(value)) { + result.push(value); + } + } + return result; + } + + static _createMapKey(array) { + return JSON.stringify(array); + } +} diff --git a/ext/js/language/sandbox/japanese-util.js b/ext/js/language/sandbox/japanese-util.js new file mode 100644 index 00000000..c7f79751 --- /dev/null +++ b/ext/js/language/sandbox/japanese-util.js @@ -0,0 +1,716 @@ +/* + * Copyright (C) 2020-2021 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +const JapaneseUtil = (() => { + const ITERATION_MARK_CODE_POINT = 0x3005; + const HIRAGANA_SMALL_TSU_CODE_POINT = 0x3063; + const KATAKANA_SMALL_TSU_CODE_POINT = 0x30c3; + const KATAKANA_SMALL_KA_CODE_POINT = 0x30f5; + const KATAKANA_SMALL_KE_CODE_POINT = 0x30f6; + const KANA_PROLONGED_SOUND_MARK_CODE_POINT = 0x30fc; + + const HIRAGANA_RANGE = [0x3040, 0x309f]; + const KATAKANA_RANGE = [0x30a0, 0x30ff]; + + const HIRAGANA_CONVERSION_RANGE = [0x3041, 0x3096]; + const KATAKANA_CONVERSION_RANGE = [0x30a1, 0x30f6]; + + const KANA_RANGES = [HIRAGANA_RANGE, KATAKANA_RANGE]; + + const CJK_UNIFIED_IDEOGRAPHS_RANGE = [0x4e00, 0x9fff]; + const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A_RANGE = [0x3400, 0x4dbf]; + const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B_RANGE = [0x20000, 0x2a6df]; + const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_C_RANGE = [0x2a700, 0x2b73f]; + const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_D_RANGE = [0x2b740, 0x2b81f]; + const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_E_RANGE = [0x2b820, 0x2ceaf]; + const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_F_RANGE = [0x2ceb0, 0x2ebef]; + const CJK_COMPATIBILITY_IDEOGRAPHS_SUPPLEMENT_RANGE = [0x2f800, 0x2fa1f]; + const CJK_UNIFIED_IDEOGRAPHS_RANGES = [ + CJK_UNIFIED_IDEOGRAPHS_RANGE, + CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A_RANGE, + CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B_RANGE, + CJK_UNIFIED_IDEOGRAPHS_EXTENSION_C_RANGE, + CJK_UNIFIED_IDEOGRAPHS_EXTENSION_D_RANGE, + CJK_UNIFIED_IDEOGRAPHS_EXTENSION_E_RANGE, + CJK_UNIFIED_IDEOGRAPHS_EXTENSION_F_RANGE, + CJK_COMPATIBILITY_IDEOGRAPHS_SUPPLEMENT_RANGE + ]; + + // Japanese character ranges, roughly ordered in order of expected frequency + const JAPANESE_RANGES = [ + HIRAGANA_RANGE, + KATAKANA_RANGE, + + ...CJK_UNIFIED_IDEOGRAPHS_RANGES, + + [0xff66, 0xff9f], // Halfwidth katakana + + [0x30fb, 0x30fc], // Katakana punctuation + [0xff61, 0xff65], // Kana punctuation + [0x3000, 0x303f], // CJK punctuation + + [0xff10, 0xff19], // Fullwidth numbers + [0xff21, 0xff3a], // Fullwidth upper case Latin letters + [0xff41, 0xff5a], // Fullwidth lower case Latin letters + + [0xff01, 0xff0f], // Fullwidth punctuation 1 + [0xff1a, 0xff1f], // Fullwidth punctuation 2 + [0xff3b, 0xff3f], // Fullwidth punctuation 3 + [0xff5b, 0xff60], // Fullwidth punctuation 4 + [0xffe0, 0xffee] // Currency markers + ]; + + const SMALL_KANA_SET = new Set(Array.from('ぁぃぅぇぉゃゅょゎァィゥェォャュョヮ')); + + const HALFWIDTH_KATAKANA_MAPPING = new Map([ + ['ｦ', 'ヲヺ-'], + ['ｧ', 'ァ--'], + ['ｨ', 'ィ--'], + ['ｩ', 'ゥ--'], + ['ｪ', 'ェ--'], + ['ｫ', 'ォ--'], + ['ｬ', 'ャ--'], + ['ｭ', 'ュ--'], + ['ｮ', 'ョ--'], + ['ｯ', 'ッ--'], + ['ｰ', 'ー--'], + ['ｱ', 'ア--'], + ['ｲ', 'イ--'], + ['ｳ', 'ウヴ-'], + ['ｴ', 'エ--'], + ['ｵ', 'オ--'], + ['ｶ', 'カガ-'], + ['ｷ', 'キギ-'], + ['ｸ', 'クグ-'], + ['ｹ', 'ケゲ-'], + ['ｺ', 'コゴ-'], + ['ｻ', 'サザ-'], + ['ｼ', 'シジ-'], + ['ｽ', 'スズ-'], + ['ｾ', 'セゼ-'], + ['ｿ', 'ソゾ-'], + ['ﾀ', 'タダ-'], + ['ﾁ', 'チヂ-'], + ['ﾂ', 'ツヅ-'], + ['ﾃ', 'テデ-'], + ['ﾄ', 'トド-'], + ['ﾅ', 'ナ--'], + ['ﾆ', 'ニ--'], + ['ﾇ', 'ヌ--'], + ['ﾈ', 'ネ--'], + ['ﾉ', 'ノ--'], + ['ﾊ', 'ハバパ'], + ['ﾋ', 'ヒビピ'], + ['ﾌ', 'フブプ'], + ['ﾍ', 'ヘベペ'], + ['ﾎ', 'ホボポ'], + ['ﾏ', 'マ--'], + ['ﾐ', 'ミ--'], + ['ﾑ', 'ム--'], + ['ﾒ', 'メ--'], + ['ﾓ', 'モ--'], + ['ﾔ', 'ヤ--'], + ['ﾕ', 'ユ--'], + ['ﾖ', 'ヨ--'], + ['ﾗ', 'ラ--'], + ['ﾘ', 'リ--'], + ['ﾙ', 'ル--'], + ['ﾚ', 'レ--'], + ['ﾛ', 'ロ--'], + ['ﾜ', 'ワ--'], + ['ﾝ', 'ン--'] + ]); + + const VOWEL_TO_KANA_MAPPING = new Map([ + ['a', 'ぁあかがさざただなはばぱまゃやらゎわヵァアカガサザタダナハバパマャヤラヮワヵヷ'], + ['i', 'ぃいきぎしじちぢにひびぴみりゐィイキギシジチヂニヒビピミリヰヸ'], + ['u', 'ぅうくぐすずっつづぬふぶぷむゅゆるゥウクグスズッツヅヌフブプムュユルヴ'], + ['e', 'ぇえけげせぜてでねへべぺめれゑヶェエケゲセゼテデネヘベペメレヱヶヹ'], + ['o', 'ぉおこごそぞとどのほぼぽもょよろをォオコゴソゾトドノホボポモョヨロヲヺ'], + ['', 'のノ'] + ]); + + const KANA_TO_VOWEL_MAPPING = (() => { + const map = new Map(); + for (const [vowel, characters] of VOWEL_TO_KANA_MAPPING) { + for (const character of characters) { + map.set(character, vowel); + } + } + return map; + })(); + + const DIACRITIC_MAPPING = (() => { + const kana = 'うゔ-かが-きぎ-くぐ-けげ-こご-さざ-しじ-すず-せぜ-そぞ-ただ-ちぢ-つづ-てで-とど-はばぱひびぴふぶぷへべぺほぼぽワヷ-ヰヸ-ウヴ-ヱヹ-ヲヺ-カガ-キギ-クグ-ケゲ-コゴ-サザ-シジ-スズ-セゼ-ソゾ-タダ-チヂ-ツヅ-テデ-トド-ハバパヒビピフブプヘベペホボポ'; + const map = new Map(); + for (let i = 0, ii = kana.length; i < ii; i += 3) { + const character = kana[i]; + const dakuten = kana[i + 1]; + const handakuten = kana[i + 2]; + map.set(dakuten, {character, type: 'dakuten'}); + if (handakuten !== '-') { + map.set(handakuten, {character, type: 'handakuten'}); + } + } + return map; + })(); + + + function isCodePointInRange(codePoint, [min, max]) { + return (codePoint >= min && codePoint <= max); + } + + function isCodePointInRanges(codePoint, ranges) { + for (const [min, max] of ranges) { + if (codePoint >= min && codePoint <= max) { + return true; + } + } + return false; + } + + function getProlongedHiragana(previousCharacter) { + switch (KANA_TO_VOWEL_MAPPING.get(previousCharacter)) { + case 'a': return 'あ'; + case 'i': return 'い'; + case 'u': return 'う'; + case 'e': return 'え'; + case 'o': return 'う'; + default: return null; + } + } + + + // eslint-disable-next-line no-shadow + class JapaneseUtil { + constructor(wanakana=null) { + this._wanakana = wanakana; + } + + // Character code testing functions + + isCodePointKanji(codePoint) { + return isCodePointInRanges(codePoint, CJK_UNIFIED_IDEOGRAPHS_RANGES); + } + + isCodePointKana(codePoint) { + return isCodePointInRanges(codePoint, KANA_RANGES); + } + + isCodePointJapanese(codePoint) { + return isCodePointInRanges(codePoint, JAPANESE_RANGES); + } + + // String testing functions + + isStringEntirelyKana(str) { + if (str.length === 0) { return false; } + for (const c of str) { + if (!isCodePointInRanges(c.codePointAt(0), KANA_RANGES)) { + return false; + } + } + return true; + } + + isStringPartiallyJapanese(str) { + if (str.length === 0) { return false; } + for (const c of str) { + if (isCodePointInRanges(c.codePointAt(0), JAPANESE_RANGES)) { + return true; + } + } + return false; + } + + // Mora functions + + isMoraPitchHigh(moraIndex, pitchAccentPosition) { + switch (pitchAccentPosition) { + case 0: return (moraIndex > 0); + case 1: return (moraIndex < 1); + default: return (moraIndex > 0 && moraIndex < pitchAccentPosition); + } + } + + getPitchCategory(text, pitchAccentPosition, isVerbOrAdjective) { + if (pitchAccentPosition === 0) { + return 'heiban'; + } + if (isVerbOrAdjective) { + return pitchAccentPosition > 0 ? 'kifuku' : null; + } + if (pitchAccentPosition === 1) { + return 'atamadaka'; + } + if (pitchAccentPosition > 1) { + return pitchAccentPosition >= this.getKanaMoraCount(text) ? 'odaka' : 'nakadaka'; + } + return null; + } + + getKanaMorae(text) { + const morae = []; + let i; + for (const c of text) { + if (SMALL_KANA_SET.has(c) && (i = morae.length) > 0) { + morae[i - 1] += c; + } else { + morae.push(c); + } + } + return morae; + } + + getKanaMoraCount(text) { + let moraCount = 0; + for (const c of text) { + if (!(SMALL_KANA_SET.has(c) && moraCount > 0)) { + ++moraCount; + } + } + return moraCount; + } + + // Conversion functions + + convertToKana(text) { + return this._getWanakana().toKana(text); + } + + convertToKanaSupported() { + return this._wanakana !== null; + } + + convertKatakanaToHiragana(text) { + let result = ''; + const offset = (HIRAGANA_CONVERSION_RANGE[0] - KATAKANA_CONVERSION_RANGE[0]); + for (let char of text) { + const codePoint = char.codePointAt(0); + if (codePoint === KATAKANA_SMALL_KA_CODE_POINT || codePoint === KATAKANA_SMALL_KE_CODE_POINT) { + // No change + } else if (codePoint === KANA_PROLONGED_SOUND_MARK_CODE_POINT) { + if (result.length > 0) { + const char2 = getProlongedHiragana(result[result.length - 1]); + if (char2 !== null) { char = char2; } + } + } else if (isCodePointInRange(codePoint, KATAKANA_CONVERSION_RANGE)) { + char = String.fromCodePoint(codePoint + offset); + } + result += char; + } + return result; + } + + convertHiraganaToKatakana(text) { + let result = ''; + const offset = (KATAKANA_CONVERSION_RANGE[0] - HIRAGANA_CONVERSION_RANGE[0]); + for (let char of text) { + const codePoint = char.codePointAt(0); + if (isCodePointInRange(codePoint, HIRAGANA_CONVERSION_RANGE)) { + char = String.fromCodePoint(codePoint + offset); + } + result += char; + } + return result; + } + + convertToRomaji(text) { + const wanakana = this._getWanakana(); + return wanakana.toRomaji(text); + } + + convertToRomajiSupported() { + return this._wanakana !== null; + } + + convertNumericToFullWidth(text) { + let result = ''; + for (const char of text) { + let c = char.codePointAt(0); + if (c >= 0x30 && c <= 0x39) { // ['0', '9'] + c += 0xff10 - 0x30; // 0xff10 = '0' full width + result += String.fromCodePoint(c); + } else { + result += char; + } + } + return result; + } + + convertHalfWidthKanaToFullWidth(text, sourceMap=null) { + let result = ''; + + // This function is safe to use charCodeAt instead of codePointAt, since all + // the relevant characters are represented with a single UTF-16 character code. + for (let i = 0, ii = text.length; i < ii; ++i) { + const c = text[i]; + const mapping = HALFWIDTH_KATAKANA_MAPPING.get(c); + if (typeof mapping !== 'string') { + result += c; + continue; + } + + let index = 0; + switch (text.charCodeAt(i + 1)) { + case 0xff9e: // dakuten + index = 1; + break; + case 0xff9f: // handakuten + index = 2; + break; + } + + let c2 = mapping[index]; + if (index > 0) { + if (c2 === '-') { // invalid + index = 0; + c2 = mapping[0]; + } else { + ++i; + } + } + + if (sourceMap !== null && index > 0) { + sourceMap.combine(result.length, 1); + } + result += c2; + } + + return result; + } + + convertAlphabeticToKana(text, sourceMap=null) { + let part = ''; + let result = ''; + + for (const char of text) { + // Note: 0x61 is the character code for 'a' + let c = char.codePointAt(0); + if (c >= 0x41 && c <= 0x5a) { // ['A', 'Z'] + c += (0x61 - 0x41); + } else if (c >= 0x61 && c <= 0x7a) { // ['a', 'z'] + // NOP; c += (0x61 - 0x61); + } else if (c >= 0xff21 && c <= 0xff3a) { // ['A', 'Z'] fullwidth + c += (0x61 - 0xff21); + } else if (c >= 0xff41 && c <= 0xff5a) { // ['a', 'z'] fullwidth + c += (0x61 - 0xff41); + } else if (c === 0x2d || c === 0xff0d) { // '-' or fullwidth dash + c = 0x2d; // '-' + } else { + if (part.length > 0) { + result += this._convertAlphabeticPartToKana(part, sourceMap, result.length); + part = ''; + } + result += char; + continue; + } + part += String.fromCodePoint(c); + } + + if (part.length > 0) { + result += this._convertAlphabeticPartToKana(part, sourceMap, result.length); + } + return result; + } + + convertAlphabeticToKanaSupported() { + return this._wanakana !== null; + } + + getKanaDiacriticInfo(character) { + const info = DIACRITIC_MAPPING.get(character); + return typeof info !== 'undefined' ? {character: info.character, type: info.type} : null; + } + + // Furigana distribution + + distributeFurigana(term, reading) { + if (reading === term) { + // Same + return [this._createFuriganaSegment(term, '')]; + } + + const groups = []; + let groupPre = null; + let isKanaPre = null; + for (const c of term) { + const codePoint = c.codePointAt(0); + const isKana = !(this.isCodePointKanji(codePoint) || codePoint === ITERATION_MARK_CODE_POINT); + if (isKana === isKanaPre) { + groupPre.text += c; + } else { + groupPre = {isKana, text: c, textNormalized: null}; + groups.push(groupPre); + isKanaPre = isKana; + } + } + for (const group of groups) { + if (group.isKana) { + group.textNormalized = this.convertKatakanaToHiragana(group.text); + } + } + + const readingNormalized = this.convertKatakanaToHiragana(reading); + const segments = this._segmentizeFurigana(reading, readingNormalized, groups, 0); + if (segments !== null) { + return segments; + } + + // Fallback + return [this._createFuriganaSegment(term, reading)]; + } + + distributeFuriganaInflected(term, reading, source) { + const termNormalized = this.convertKatakanaToHiragana(term); + const readingNormalized = this.convertKatakanaToHiragana(reading); + const sourceNormalized = this.convertKatakanaToHiragana(source); + + let mainText = term; + let stemLength = this._getStemLength(termNormalized, sourceNormalized); + + // Check if source is derived from the reading instead of the term + const readingStemLength = this._getStemLength(readingNormalized, sourceNormalized); + if (readingStemLength > 0 && readingStemLength >= stemLength) { + mainText = reading; + stemLength = readingStemLength; + reading = `${source.substring(0, stemLength)}${reading.substring(stemLength)}`; + } + + const segments = []; + if (stemLength > 0) { + mainText = `${source.substring(0, stemLength)}${mainText.substring(stemLength)}`; + const segments2 = this.distributeFurigana(mainText, reading); + let consumed = 0; + for (const segment of segments2) { + const {text} = segment; + const start = consumed; + consumed += text.length; + if (consumed < stemLength) { + segments.push(segment); + } else if (consumed === stemLength) { + segments.push(segment); + break; + } else { + if (start < stemLength) { + segments.push(this._createFuriganaSegment(mainText.substring(start, stemLength), '')); + } + break; + } + } + } + + if (stemLength < source.length) { + const remainder = source.substring(stemLength); + const segmentCount = segments.length; + if (segmentCount > 0 && segments[segmentCount - 1].reading.length === 0) { + // Append to the last segment if it has an empty reading + segments[segmentCount - 1].text += remainder; + } else { + // Otherwise, create a new segment + segments.push(this._createFuriganaSegment(remainder, '')); + } + } + + return segments; + } + + // Miscellaneous + + collapseEmphaticSequences(text, fullCollapse, sourceMap=null) { + let result = ''; + let collapseCodePoint = -1; + const hasSourceMap = (sourceMap !== null); + for (const char of text) { + const c = char.codePointAt(0); + if ( + c === HIRAGANA_SMALL_TSU_CODE_POINT || + c === KATAKANA_SMALL_TSU_CODE_POINT || + c === KANA_PROLONGED_SOUND_MARK_CODE_POINT + ) { + if (collapseCodePoint !== c) { + collapseCodePoint = c; + if (!fullCollapse) { + result += char; + continue; + } + } + } else { + collapseCodePoint = -1; + result += char; + continue; + } + + if (hasSourceMap) { + sourceMap.combine(Math.max(0, result.length - 1), 1); + } + } + return result; + } + + // Private + + _createFuriganaSegment(text, reading) { + return {text, reading}; + } + + _segmentizeFurigana(reading, readingNormalized, groups, groupsStart) { + const groupCount = groups.length - groupsStart; + if (groupCount <= 0) { + return reading.length === 0 ? [] : null; + } + + const group = groups[groupsStart]; + const {isKana, text} = group; + const textLength = text.length; + if (isKana) { + const {textNormalized} = group; + if (readingNormalized.startsWith(textNormalized)) { + const segments = this._segmentizeFurigana( + reading.substring(textLength), + readingNormalized.substring(textLength), + groups, + groupsStart + 1 + ); + if (segments !== null) { + if (reading.startsWith(text)) { + segments.unshift(this._createFuriganaSegment(text, '')); + } else { + segments.unshift(...this._getFuriganaKanaSegments(text, reading)); + } + return segments; + } + } + return null; + } else { + let result = null; + for (let i = reading.length; i >= textLength; --i) { + const segments = this._segmentizeFurigana( + reading.substring(i), + readingNormalized.substring(i), + groups, + groupsStart + 1 + ); + if (segments !== null) { + if (result !== null) { + // More than one way to segmentize the tail; mark as ambiguous + return null; + } + const segmentReading = reading.substring(0, i); + segments.unshift(this._createFuriganaSegment(text, segmentReading)); + result = segments; + } + // There is only one way to segmentize the last non-kana group + if (groupCount === 1) { + break; + } + } + return result; + } + } + + _getFuriganaKanaSegments(text, reading) { + const textLength = text.length; + const newSegments = []; + let start = 0; + let state = (reading[0] === text[0]); + for (let i = 1; i < textLength; ++i) { + const newState = (reading[i] === text[i]); + if (state === newState) { continue; } + newSegments.push(this._createFuriganaSegment(text.substring(start, i), state ? '' : reading.substring(start, i))); + state = newState; + start = i; + } + newSegments.push(this._createFuriganaSegment(text.substring(start, textLength), state ? '' : reading.substring(start, textLength))); + return newSegments; + } + + _getWanakana() { + const wanakana = this._wanakana; + if (wanakana === null) { throw new Error('Functions which use WanaKana are not supported in this context'); } + return wanakana; + } + + _convertAlphabeticPartToKana(text, sourceMap, sourceMapStart) { + const wanakana = this._getWanakana(); + const result = wanakana.toHiragana(text); + + // Generate source mapping + if (sourceMap !== null) { + let i = 0; + let resultPos = 0; + const ii = text.length; + while (i < ii) { + // Find smallest matching substring + let iNext = i + 1; + let resultPosNext = result.length; + while (iNext < ii) { + const t = wanakana.toHiragana(text.substring(0, iNext)); + if (t === result.substring(0, t.length)) { + resultPosNext = t.length; + break; + } + ++iNext; + } + + // Merge characters + const removals = iNext - i - 1; + if (removals > 0) { + sourceMap.combine(sourceMapStart, removals); + } + ++sourceMapStart; + + // Empty elements + const additions = resultPosNext - resultPos - 1; + for (let j = 0; j < additions; ++j) { + sourceMap.insert(sourceMapStart, 0); + ++sourceMapStart; + } + + i = iNext; + resultPos = resultPosNext; + } + } + + return result; + } + + _getStemLength(text1, text2) { + const minLength = Math.min(text1.length, text2.length); + if (minLength === 0) { return 0; } + + let i = 0; + while (true) { + const char1 = text1.codePointAt(i); + const char2 = text2.codePointAt(i); + if (char1 !== char2) { break; } + const charLength = String.fromCodePoint(char1).length; + i += charLength; + if (i >= minLength) { + if (i > minLength) { + i -= charLength; // Don't consume partial UTF16 surrogate characters + } + break; + } + } + return i; + } + } + + + return JapaneseUtil; +})(); -- cgit v1.2.3