summaryrefslogtreecommitdiff
path: root/ext
diff options
context:
space:
mode:
Diffstat (limited to 'ext')
-rw-r--r--ext/bg/background.html1
-rw-r--r--ext/bg/js/audio-uri-builder.js4
-rw-r--r--ext/bg/js/backend.js18
-rw-r--r--ext/bg/js/clipboard-monitor.js4
-rw-r--r--ext/bg/js/handlebars.js9
-rw-r--r--ext/bg/js/japanese.js712
-rw-r--r--ext/bg/js/translator.js24
-rw-r--r--ext/bg/search.html1
-rw-r--r--ext/bg/settings.html1
-rw-r--r--ext/fg/float.html1
-rw-r--r--ext/mixed/js/display-generator.js11
-rw-r--r--ext/mixed/js/japanese.js124
12 files changed, 480 insertions, 430 deletions
diff --git a/ext/bg/background.html b/ext/bg/background.html
index 44abe8fd..f7cf6e55 100644
--- a/ext/bg/background.html
+++ b/ext/bg/background.html
@@ -20,6 +20,7 @@
<script src="/mixed/js/core.js"></script>
<script src="/mixed/js/dom.js"></script>
+ <script src="/mixed/js/japanese.js"></script>
<script src="/bg/js/anki.js"></script>
<script src="/bg/js/anki-note-builder.js"></script>
diff --git a/ext/bg/js/audio-uri-builder.js b/ext/bg/js/audio-uri-builder.js
index 499c3441..158006bb 100644
--- a/ext/bg/js/audio-uri-builder.js
+++ b/ext/bg/js/audio-uri-builder.js
@@ -17,7 +17,7 @@
*/
/* global
- * jpIsStringEntirelyKana
+ * jp
*/
class AudioUriBuilder {
@@ -66,7 +66,7 @@ class AudioUriBuilder {
let kana = definition.reading;
let kanji = definition.expression;
- if (!kana && jpIsStringEntirelyKana(kanji)) {
+ if (!kana && jp.isStringEntirelyKana(kanji)) {
kana = kanji;
kanji = null;
}
diff --git a/ext/bg/js/backend.js b/ext/bg/js/backend.js
index 978c5a4a..b217e64d 100644
--- a/ext/bg/js/backend.js
+++ b/ext/bg/js/backend.js
@@ -32,9 +32,7 @@
* dictEnabledSet
* dictTermsSort
* handlebarsRenderDynamic
- * jpConvertReading
- * jpDistributeFuriganaInflected
- * jpKatakanaToHiragana
+ * jp
* optionsLoad
* optionsSave
* profileConditionsDescriptor
@@ -402,13 +400,13 @@ class Backend {
dictTermsSort(definitions);
const {expression, reading} = definitions[0];
const source = text.substring(0, sourceLength);
- for (const {text: text2, furigana} of jpDistributeFuriganaInflected(expression, reading, source)) {
- const reading2 = jpConvertReading(text2, furigana, options.parsing.readingMode);
+ for (const {text: text2, furigana} of jp.distributeFuriganaInflected(expression, reading, source)) {
+ const reading2 = jp.convertReading(text2, furigana, options.parsing.readingMode);
term.push({text: text2, reading: reading2});
}
text = text.substring(source.length);
} else {
- const reading = jpConvertReading(text[0], null, options.parsing.readingMode);
+ const reading = jp.convertReading(text[0], null, options.parsing.readingMode);
term.push({text: text[0], reading});
text = text.substring(1);
}
@@ -427,16 +425,16 @@ class Backend {
for (const {expression, reading, source} of parsedLine) {
const term = [];
if (expression !== null && reading !== null) {
- for (const {text: text2, furigana} of jpDistributeFuriganaInflected(
+ for (const {text: text2, furigana} of jp.distributeFuriganaInflected(
expression,
- jpKatakanaToHiragana(reading),
+ jp.convertKatakanaToHiragana(reading),
source
)) {
- const reading2 = jpConvertReading(text2, furigana, options.parsing.readingMode);
+ const reading2 = jp.convertReading(text2, furigana, options.parsing.readingMode);
term.push({text: text2, reading: reading2});
}
} else {
- const reading2 = jpConvertReading(source, null, options.parsing.readingMode);
+ const reading2 = jp.convertReading(source, null, options.parsing.readingMode);
term.push({text: source, reading: reading2});
}
result.push(term);
diff --git a/ext/bg/js/clipboard-monitor.js b/ext/bg/js/clipboard-monitor.js
index 9a881f57..c67525fc 100644
--- a/ext/bg/js/clipboard-monitor.js
+++ b/ext/bg/js/clipboard-monitor.js
@@ -17,7 +17,7 @@
*/
/* global
- * jpIsStringPartiallyJapanese
+ * jp
*/
class ClipboardMonitor extends EventDispatcher {
@@ -54,7 +54,7 @@ class ClipboardMonitor extends EventDispatcher {
text !== this._previousText
) {
this._previousText = text;
- if (jpIsStringPartiallyJapanese(text)) {
+ if (jp.isStringPartiallyJapanese(text)) {
this.trigger('change', {text});
}
}
diff --git a/ext/bg/js/handlebars.js b/ext/bg/js/handlebars.js
index e3ce6bd0..5fda5baa 100644
--- a/ext/bg/js/handlebars.js
+++ b/ext/bg/js/handlebars.js
@@ -18,8 +18,7 @@
/* global
* Handlebars
- * jpDistributeFurigana
- * jpIsCodePointKanji
+ * jp
*/
function handlebarsEscape(text) {
@@ -33,7 +32,7 @@ function handlebarsDumpObject(options) {
function handlebarsFurigana(options) {
const definition = options.fn(this);
- const segs = jpDistributeFurigana(definition.expression, definition.reading);
+ const segs = jp.distributeFurigana(definition.expression, definition.reading);
let result = '';
for (const seg of segs) {
@@ -49,7 +48,7 @@ function handlebarsFurigana(options) {
function handlebarsFuriganaPlain(options) {
const definition = options.fn(this);
- const segs = jpDistributeFurigana(definition.expression, definition.reading);
+ const segs = jp.distributeFurigana(definition.expression, definition.reading);
let result = '';
for (const seg of segs) {
@@ -66,7 +65,7 @@ function handlebarsFuriganaPlain(options) {
function handlebarsKanjiLinks(options) {
let result = '';
for (const c of options.fn(this)) {
- if (jpIsCodePointKanji(c.codePointAt(0))) {
+ if (jp.isCodePointKanji(c.codePointAt(0))) {
result += `<a href="#" class="kanji-link">${c}</a>`;
} else {
result += c;
diff --git a/ext/bg/js/japanese.js b/ext/bg/js/japanese.js
index 3b37754d..c5873cf1 100644
--- a/ext/bg/js/japanese.js
+++ b/ext/bg/js/japanese.js
@@ -17,442 +17,380 @@
*/
/* global
+ * jp
* wanakana
*/
-const JP_HALFWIDTH_KATAKANA_MAPPING = new Map([
- ['ｦ', 'ヲヺ-'],
- ['ｧ', 'ァ--'],
- ['ｨ', 'ィ--'],
- ['ｩ', 'ゥ--'],
- ['ｪ', 'ェ--'],
- ['ｫ', 'ォ--'],
- ['ｬ', 'ャ--'],
- ['ｭ', 'ュ--'],
- ['ｮ', 'ョ--'],
- ['ｯ', 'ッ--'],
- ['ｰ', 'ー--'],
- ['ｱ', 'ア--'],
- ['ｲ', 'イ--'],
- ['ｳ', 'ウヴ-'],
- ['ｴ', 'エ--'],
- ['ｵ', 'オ--'],
- ['ｶ', 'カガ-'],
- ['ｷ', 'キギ-'],
- ['ｸ', 'クグ-'],
- ['ｹ', 'ケゲ-'],
- ['ｺ', 'コゴ-'],
- ['ｻ', 'サザ-'],
- ['ｼ', 'シジ-'],
- ['ｽ', 'スズ-'],
- ['ｾ', 'セゼ-'],
- ['ｿ', 'ソゾ-'],
- ['ﾀ', 'タダ-'],
- ['ﾁ', 'チヂ-'],
- ['ﾂ', 'ツヅ-'],
- ['ﾃ', 'テデ-'],
- ['ﾄ', 'トド-'],
- ['ﾅ', 'ナ--'],
- ['ﾆ', 'ニ--'],
- ['ﾇ', 'ヌ--'],
- ['ﾈ', 'ネ--'],
- ['ﾉ', 'ノ--'],
- ['ﾊ', 'ハバパ'],
- ['ﾋ', 'ヒビピ'],
- ['ﾌ', 'フブプ'],
- ['ﾍ', 'ヘベペ'],
- ['ﾎ', 'ホボポ'],
- ['ﾏ', 'マ--'],
- ['ﾐ', 'ミ--'],
- ['ﾑ', 'ム--'],
- ['ﾒ', 'メ--'],
- ['ﾓ', 'モ--'],
- ['ﾔ', 'ヤ--'],
- ['ﾕ', 'ユ--'],
- ['ﾖ', 'ヨ--'],
- ['ﾗ', 'ラ--'],
- ['ﾘ', 'リ--'],
- ['ﾙ', 'ル--'],
- ['ﾚ', 'レ--'],
- ['ﾛ', 'ロ--'],
- ['ﾜ', 'ワ--'],
- ['ﾝ', 'ン--']
-]);
-
-const JP_HIRAGANA_RANGE = [0x3040, 0x309f];
-const JP_KATAKANA_RANGE = [0x30a0, 0x30ff];
-const JP_KANA_RANGES = [JP_HIRAGANA_RANGE, JP_KATAKANA_RANGE];
-
-const JP_CJK_COMMON_RANGE = [0x4e00, 0x9fff];
-const JP_CJK_RARE_RANGE = [0x3400, 0x4dbf];
-const JP_CJK_RANGES = [JP_CJK_COMMON_RANGE, JP_CJK_RARE_RANGE];
-
-const JP_ITERATION_MARK_CHAR_CODE = 0x3005;
-
-// Japanese character ranges, roughly ordered in order of expected frequency
-const JP_JAPANESE_RANGES = [
- JP_HIRAGANA_RANGE,
- JP_KATAKANA_RANGE,
-
- JP_CJK_COMMON_RANGE,
- JP_CJK_RARE_RANGE,
-
- [0xff66, 0xff9f], // Halfwidth katakana
-
- [0x30fb, 0x30fc], // Katakana punctuation
- [0xff61, 0xff65], // Kana punctuation
- [0x3000, 0x303f], // CJK punctuation
-
- [0xff10, 0xff19], // Fullwidth numbers
- [0xff21, 0xff3a], // Fullwidth upper case Latin letters
- [0xff41, 0xff5a], // Fullwidth lower case Latin letters
-
- [0xff01, 0xff0f], // Fullwidth punctuation 1
- [0xff1a, 0xff1f], // Fullwidth punctuation 2
- [0xff3b, 0xff3f], // Fullwidth punctuation 3
- [0xff5b, 0xff60], // Fullwidth punctuation 4
- [0xffe0, 0xffee] // Currency markers
-];
-
-
-// Helper functions
-
-function _jpIsCodePointInRanges(codePoint, ranges) {
- for (const [min, max] of ranges) {
- if (codePoint >= min && codePoint <= max) {
- return true;
+(() => {
+ const HALFWIDTH_KATAKANA_MAPPING = new Map([
+ ['ｦ', 'ヲヺ-'],
+ ['ｧ', 'ァ--'],
+ ['ｨ', 'ィ--'],
+ ['ｩ', 'ゥ--'],
+ ['ｪ', 'ェ--'],
+ ['ｫ', 'ォ--'],
+ ['ｬ', 'ャ--'],
+ ['ｭ', 'ュ--'],
+ ['ｮ', 'ョ--'],
+ ['ｯ', 'ッ--'],
+ ['ｰ', 'ー--'],
+ ['ｱ', 'ア--'],
+ ['ｲ', 'イ--'],
+ ['ｳ', 'ウヴ-'],
+ ['ｴ', 'エ--'],
+ ['ｵ', 'オ--'],
+ ['ｶ', 'カガ-'],
+ ['ｷ', 'キギ-'],
+ ['ｸ', 'クグ-'],
+ ['ｹ', 'ケゲ-'],
+ ['ｺ', 'コゴ-'],
+ ['ｻ', 'サザ-'],
+ ['ｼ', 'シジ-'],
+ ['ｽ', 'スズ-'],
+ ['ｾ', 'セゼ-'],
+ ['ｿ', 'ソゾ-'],
+ ['ﾀ', 'タダ-'],
+ ['ﾁ', 'チヂ-'],
+ ['ﾂ', 'ツヅ-'],
+ ['ﾃ', 'テデ-'],
+ ['ﾄ', 'トド-'],
+ ['ﾅ', 'ナ--'],
+ ['ﾆ', 'ニ--'],
+ ['ﾇ', 'ヌ--'],
+ ['ﾈ', 'ネ--'],
+ ['ﾉ', 'ノ--'],
+ ['ﾊ', 'ハバパ'],
+ ['ﾋ', 'ヒビピ'],
+ ['ﾌ', 'フブプ'],
+ ['ﾍ', 'ヘベペ'],
+ ['ﾎ', 'ホボポ'],
+ ['ﾏ', 'マ--'],
+ ['ﾐ', 'ミ--'],
+ ['ﾑ', 'ム--'],
+ ['ﾒ', 'メ--'],
+ ['ﾓ', 'モ--'],
+ ['ﾔ', 'ヤ--'],
+ ['ﾕ', 'ユ--'],
+ ['ﾖ', 'ヨ--'],
+ ['ﾗ', 'ラ--'],
+ ['ﾘ', 'リ--'],
+ ['ﾙ', 'ル--'],
+ ['ﾚ', 'レ--'],
+ ['ﾛ', 'ロ--'],
+ ['ﾜ', 'ワ--'],
+ ['ﾝ', 'ン--']
+ ]);
+
+ const ITERATION_MARK_CODE_POINT = 0x3005;
+
+
+ // Existing functions
+
+ const isCodePointKanji = jp.isCodePointKanji;
+ const isStringEntirelyKana = jp.isStringEntirelyKana;
+
+
+ // Conversion functions
+
+ function convertKatakanaToHiragana(text) {
+ let result = '';
+ for (const c of text) {
+ if (wanakana.isKatakana(c)) {
+ result += wanakana.toHiragana(c);
+ } else {
+ result += c;
+ }
}
- }
- return false;
-}
-
-
-// Character code testing functions
-function jpIsCodePointKanji(codePoint) {
- return _jpIsCodePointInRanges(codePoint, JP_CJK_RANGES);
-}
-
-function jpIsCodePointKana(codePoint) {
- return _jpIsCodePointInRanges(codePoint, JP_KANA_RANGES);
-}
+ return result;
+ }
-function jpIsCodePointJapanese(codePoint) {
- return _jpIsCodePointInRanges(codePoint, JP_JAPANESE_RANGES);
-}
+ function convertHiraganaToKatakana(text) {
+ let result = '';
+ for (const c of text) {
+ if (wanakana.isHiragana(c)) {
+ result += wanakana.toKatakana(c);
+ } else {
+ result += c;
+ }
+ }
+ return result;
+ }
-// String testing functions
+ function convertToRomaji(text) {
+ return wanakana.toRomaji(text);
+ }
-function jpIsStringEntirelyKana(str) {
- if (str.length === 0) { return false; }
- for (const c of str) {
- if (!jpIsCodePointKana(c.codePointAt(0))) {
- return false;
+ function convertReading(expressionFragment, readingFragment, readingMode) {
+ switch (readingMode) {
+ case 'hiragana':
+ return convertKatakanaToHiragana(readingFragment || '');
+ case 'katakana':
+ return convertHiraganaToKatakana(readingFragment || '');
+ case 'romaji':
+ if (readingFragment) {
+ return convertToRomaji(readingFragment);
+ } else {
+ if (isStringEntirelyKana(expressionFragment)) {
+ return convertToRomaji(expressionFragment);
+ }
+ }
+ return readingFragment;
+ case 'none':
+ return null;
+ default:
+ return readingFragment;
}
}
- return true;
-}
-
-function jpIsStringPartiallyJapanese(str) {
- if (str.length === 0) { return false; }
- for (const c of str) {
- if (jpIsCodePointJapanese(c.codePointAt(0))) {
- return true;
+
+ function convertNumericToFullWidth(text) {
+ let result = '';
+ for (const char of text) {
+ let c = char.codePointAt(0);
+ if (c >= 0x30 && c <= 0x39) { // ['0', '9']
+ c += 0xff10 - 0x30; // 0xff10 = '0' full width
+ result += String.fromCodePoint(c);
+ } else {
+ result += char;
+ }
}
+ return result;
}
- return false;
-}
+ function convertHalfWidthKanaToFullWidth(text, sourceMapping) {
+ let result = '';
+ const hasSourceMapping = Array.isArray(sourceMapping);
+
+ // This function is safe to use charCodeAt instead of codePointAt, since all
+ // the relevant characters are represented with a single UTF-16 character code.
+ for (let i = 0, ii = text.length; i < ii; ++i) {
+ const c = text[i];
+ const mapping = HALFWIDTH_KATAKANA_MAPPING.get(c);
+ if (typeof mapping !== 'string') {
+ result += c;
+ continue;
+ }
-// Conversion functions
-
-function jpKatakanaToHiragana(text) {
- let result = '';
- for (const c of text) {
- if (wanakana.isKatakana(c)) {
- result += wanakana.toHiragana(c);
- } else {
- result += c;
- }
- }
+ let index = 0;
+ switch (text.charCodeAt(i + 1)) {
+ case 0xff9e: // dakuten
+ index = 1;
+ break;
+ case 0xff9f: // handakuten
+ index = 2;
+ break;
+ }
- return result;
-}
+ let c2 = mapping[index];
+ if (index > 0) {
+ if (c2 === '-') { // invalid
+ index = 0;
+ c2 = mapping[0];
+ } else {
+ ++i;
+ }
+ }
-function jpHiraganaToKatakana(text) {
- let result = '';
- for (const c of text) {
- if (wanakana.isHiragana(c)) {
- result += wanakana.toKatakana(c);
- } else {
- result += c;
+ if (hasSourceMapping && index > 0) {
+ index = result.length;
+ const v = sourceMapping.splice(index + 1, 1)[0];
+ sourceMapping[index] += v;
+ }
+ result += c2;
}
+
+ return result;
}
- return result;
-}
-
-function jpToRomaji(text) {
- return wanakana.toRomaji(text);
-}
-
-function jpConvertReading(expressionFragment, readingFragment, readingMode) {
- switch (readingMode) {
- case 'hiragana':
- return jpKatakanaToHiragana(readingFragment || '');
- case 'katakana':
- return jpHiraganaToKatakana(readingFragment || '');
- case 'romaji':
- if (readingFragment) {
- return jpToRomaji(readingFragment);
+ function convertAlphabeticToKana(text, sourceMapping) {
+ let part = '';
+ let result = '';
+
+ for (const char of text) {
+ // Note: 0x61 is the character code for 'a'
+ let c = char.codePointAt(0);
+ if (c >= 0x41 && c <= 0x5a) { // ['A', 'Z']
+ c += (0x61 - 0x41);
+ } else if (c >= 0x61 && c <= 0x7a) { // ['a', 'z']
+ // NOP; c += (0x61 - 0x61);
+ } else if (c >= 0xff21 && c <= 0xff3a) { // ['A', 'Z'] fullwidth
+ c += (0x61 - 0xff21);
+ } else if (c >= 0xff41 && c <= 0xff5a) { // ['a', 'z'] fullwidth
+ c += (0x61 - 0xff41);
+ } else if (c === 0x2d || c === 0xff0d) { // '-' or fullwidth dash
+ c = 0x2d; // '-'
} else {
- if (jpIsStringEntirelyKana(expressionFragment)) {
- return jpToRomaji(expressionFragment);
+ if (part.length > 0) {
+ result += convertAlphabeticPartToKana(part, sourceMapping, result.length);
+ part = '';
}
+ result += char;
+ continue;
}
- return readingFragment;
- case 'none':
- return null;
- default:
- return readingFragment;
- }
-}
-
-function jpDistributeFurigana(expression, reading) {
- const fallback = [{furigana: reading, text: expression}];
- if (!reading) {
- return fallback;
- }
+ part += String.fromCodePoint(c);
+ }
- let isAmbiguous = false;
- const segmentize = (reading2, groups) => {
- if (groups.length === 0 || isAmbiguous) {
- return [];
+ if (part.length > 0) {
+ result += convertAlphabeticPartToKana(part, sourceMapping, result.length);
}
+ return result;
+ }
- const group = groups[0];
- if (group.mode === 'kana') {
- if (jpKatakanaToHiragana(reading2).startsWith(jpKatakanaToHiragana(group.text))) {
- const readingLeft = reading2.substring(group.text.length);
- const segs = segmentize(readingLeft, groups.splice(1));
- if (segs) {
- return [{text: group.text}].concat(segs);
- }
- }
- } else {
- let foundSegments = null;
- for (let i = reading2.length; i >= group.text.length; --i) {
- const readingUsed = reading2.substring(0, i);
- const readingLeft = reading2.substring(i);
- const segs = segmentize(readingLeft, groups.slice(1));
- if (segs) {
- if (foundSegments !== null) {
- // more than one way to segmentize the tail, mark as ambiguous
- isAmbiguous = true;
- return null;
+ function convertAlphabeticPartToKana(text, sourceMapping, sourceMappingStart) {
+ const result = wanakana.toHiragana(text);
+
+ // Generate source mapping
+ if (Array.isArray(sourceMapping)) {
+ if (typeof sourceMappingStart !== 'number') { sourceMappingStart = 0; }
+ let i = 0;
+ let resultPos = 0;
+ const ii = text.length;
+ while (i < ii) {
+ // Find smallest matching substring
+ let iNext = i + 1;
+ let resultPosNext = result.length;
+ while (iNext < ii) {
+ const t = wanakana.toHiragana(text.substring(0, iNext));
+ if (t === result.substring(0, t.length)) {
+ resultPosNext = t.length;
+ break;
}
- foundSegments = [{text: group.text, furigana: readingUsed}].concat(segs);
+ ++iNext;
}
- // there is only one way to segmentize the last non-kana group
- if (groups.length === 1) {
- break;
+
+ // Merge characters
+ const removals = iNext - i - 1;
+ if (removals > 0) {
+ let sum = 0;
+ const vs = sourceMapping.splice(sourceMappingStart + 1, removals);
+ for (const v of vs) { sum += v; }
+ sourceMapping[sourceMappingStart] += sum;
}
+ ++sourceMappingStart;
+
+ // Empty elements
+ const additions = resultPosNext - resultPos - 1;
+ for (let j = 0; j < additions; ++j) {
+ sourceMapping.splice(sourceMappingStart, 0, 0);
+ ++sourceMappingStart;
+ }
+
+ i = iNext;
+ resultPos = resultPosNext;
}
- return foundSegments;
- }
- };
-
- const groups = [];
- let modePrev = null;
- for (const c of expression) {
- const codePoint = c.codePointAt(0);
- const modeCurr = jpIsCodePointKanji(codePoint) || codePoint === JP_ITERATION_MARK_CHAR_CODE ? 'kanji' : 'kana';
- if (modeCurr === modePrev) {
- groups[groups.length - 1].text += c;
- } else {
- groups.push({mode: modeCurr, text: c});
- modePrev = modeCurr;
}
- }
- const segments = segmentize(reading, groups);
- if (segments && !isAmbiguous) {
- return segments;
- }
- return fallback;
-}
-
-function jpDistributeFuriganaInflected(expression, reading, source) {
- const output = [];
-
- let stemLength = 0;
- const shortest = Math.min(source.length, expression.length);
- const sourceHiragana = jpKatakanaToHiragana(source);
- const expressionHiragana = jpKatakanaToHiragana(expression);
- while (stemLength < shortest && sourceHiragana[stemLength] === expressionHiragana[stemLength]) {
- ++stemLength;
- }
- const offset = source.length - stemLength;
-
- const stemExpression = source.substring(0, source.length - offset);
- const stemReading = reading.substring(
- 0,
- offset === 0 ? reading.length : reading.length - expression.length + stemLength
- );
- for (const segment of jpDistributeFurigana(stemExpression, stemReading)) {
- output.push(segment);
+ return result;
}
- if (stemLength !== source.length) {
- output.push({text: source.substring(stemLength)});
- }
- return output;
-}
-
-function jpConvertHalfWidthKanaToFullWidth(text, sourceMapping) {
- let result = '';
- const hasSourceMapping = Array.isArray(sourceMapping);
-
- // This function is safe to use charCodeAt instead of codePointAt, since all
- // the relevant characters are represented with a single UTF-16 character code.
- for (let i = 0, ii = text.length; i < ii; ++i) {
- const c = text[i];
- const mapping = JP_HALFWIDTH_KATAKANA_MAPPING.get(c);
- if (typeof mapping !== 'string') {
- result += c;
- continue;
- }
+ // Furigana distribution
- let index = 0;
- switch (text.charCodeAt(i + 1)) {
- case 0xff9e: // dakuten
- index = 1;
- break;
- case 0xff9f: // handakuten
- index = 2;
- break;
+ function distributeFurigana(expression, reading) {
+ const fallback = [{furigana: reading, text: expression}];
+ if (!reading) {
+ return fallback;
}
- let c2 = mapping[index];
- if (index > 0) {
- if (c2 === '-') { // invalid
- index = 0;
- c2 = mapping[0];
- } else {
- ++i;
+ let isAmbiguous = false;
+ const segmentize = (reading2, groups) => {
+ if (groups.length === 0 || isAmbiguous) {
+ return [];
}
- }
- if (hasSourceMapping && index > 0) {
- index = result.length;
- const v = sourceMapping.splice(index + 1, 1)[0];
- sourceMapping[index] += v;
+ const group = groups[0];
+ if (group.mode === 'kana') {
+ if (convertKatakanaToHiragana(reading2).startsWith(convertKatakanaToHiragana(group.text))) {
+ const readingLeft = reading2.substring(group.text.length);
+ const segs = segmentize(readingLeft, groups.splice(1));
+ if (segs) {
+ return [{text: group.text}].concat(segs);
+ }
+ }
+ } else {
+ let foundSegments = null;
+ for (let i = reading2.length; i >= group.text.length; --i) {
+ const readingUsed = reading2.substring(0, i);
+ const readingLeft = reading2.substring(i);
+ const segs = segmentize(readingLeft, groups.slice(1));
+ if (segs) {
+ if (foundSegments !== null) {
+ // more than one way to segmentize the tail, mark as ambiguous
+ isAmbiguous = true;
+ return null;
+ }
+ foundSegments = [{text: group.text, furigana: readingUsed}].concat(segs);
+ }
+ // there is only one way to segmentize the last non-kana group
+ if (groups.length === 1) {
+ break;
+ }
+ }
+ return foundSegments;
+ }
+ };
+
+ const groups = [];
+ let modePrev = null;
+ for (const c of expression) {
+ const codePoint = c.codePointAt(0);
+ const modeCurr = isCodePointKanji(codePoint) || codePoint === ITERATION_MARK_CODE_POINT ? 'kanji' : 'kana';
+ if (modeCurr === modePrev) {
+ groups[groups.length - 1].text += c;
+ } else {
+ groups.push({mode: modeCurr, text: c});
+ modePrev = modeCurr;
+ }
}
- result += c2;
- }
- return result;
-}
-
-function jpConvertNumericTofullWidth(text) {
- let result = '';
- for (const char of text) {
- let c = char.codePointAt(0);
- if (c >= 0x30 && c <= 0x39) { // ['0', '9']
- c += 0xff10 - 0x30; // 0xff10 = '0' full width
- result += String.fromCodePoint(c);
- } else {
- result += char;
+ const segments = segmentize(reading, groups);
+ if (segments && !isAmbiguous) {
+ return segments;
}
+ return fallback;
}
- return result;
-}
-function jpConvertAlphabeticToKana(text, sourceMapping) {
- let part = '';
- let result = '';
- const ii = text.length;
+ function distributeFuriganaInflected(expression, reading, source) {
+ const output = [];
- if (sourceMapping.length === ii) {
- sourceMapping.length = ii;
- sourceMapping.fill(1);
- }
+ let stemLength = 0;
+ const shortest = Math.min(source.length, expression.length);
+ const sourceHiragana = convertKatakanaToHiragana(source);
+ const expressionHiragana = convertKatakanaToHiragana(expression);
+ while (stemLength < shortest && sourceHiragana[stemLength] === expressionHiragana[stemLength]) {
+ ++stemLength;
+ }
+ const offset = source.length - stemLength;
+
+ const stemExpression = source.substring(0, source.length - offset);
+ const stemReading = reading.substring(
+ 0,
+ offset === 0 ? reading.length : reading.length - expression.length + stemLength
+ );
+ for (const segment of distributeFurigana(stemExpression, stemReading)) {
+ output.push(segment);
+ }
- for (const char of text) {
- // Note: 0x61 is the character code for 'a'
- let c = char.codePointAt(0);
- if (c >= 0x41 && c <= 0x5a) { // ['A', 'Z']
- c += (0x61 - 0x41);
- } else if (c >= 0x61 && c <= 0x7a) { // ['a', 'z']
- // NOP; c += (0x61 - 0x61);
- } else if (c >= 0xff21 && c <= 0xff3a) { // ['A', 'Z'] fullwidth
- c += (0x61 - 0xff21);
- } else if (c >= 0xff41 && c <= 0xff5a) { // ['a', 'z'] fullwidth
- c += (0x61 - 0xff41);
- } else if (c === 0x2d || c === 0xff0d) { // '-' or fullwidth dash
- c = 0x2d; // '-'
- } else {
- if (part.length > 0) {
- result += jpToHiragana(part, sourceMapping, result.length);
- part = '';
- }
- result += char;
- continue;
+ if (stemLength !== source.length) {
+ output.push({text: source.substring(stemLength)});
}
- part += String.fromCodePoint(c);
- }
- if (part.length > 0) {
- result += jpToHiragana(part, sourceMapping, result.length);
+ return output;
}
- return result;
-}
-
-function jpToHiragana(text, sourceMapping, sourceMappingStart) {
- const result = wanakana.toHiragana(text);
-
- // Generate source mapping
- if (Array.isArray(sourceMapping)) {
- if (typeof sourceMappingStart !== 'number') { sourceMappingStart = 0; }
- let i = 0;
- let resultPos = 0;
- const ii = text.length;
- while (i < ii) {
- // Find smallest matching substring
- let iNext = i + 1;
- let resultPosNext = result.length;
- while (iNext < ii) {
- const t = wanakana.toHiragana(text.substring(0, iNext));
- if (t === result.substring(0, t.length)) {
- resultPosNext = t.length;
- break;
- }
- ++iNext;
- }
- // Merge characters
- const removals = iNext - i - 1;
- if (removals > 0) {
- let sum = 0;
- const vs = sourceMapping.splice(sourceMappingStart + 1, removals);
- for (const v of vs) { sum += v; }
- sourceMapping[sourceMappingStart] += sum;
- }
- ++sourceMappingStart;
-
- // Empty elements
- const additions = resultPosNext - resultPos - 1;
- for (let j = 0; j < additions; ++j) {
- sourceMapping.splice(sourceMappingStart, 0, 0);
- ++sourceMappingStart;
- }
-
- i = iNext;
- resultPos = resultPosNext;
- }
- }
- return result;
-}
+ // Exports
+
+ Object.assign(jp, {
+ convertKatakanaToHiragana,
+ convertHiraganaToKatakana,
+ convertToRomaji,
+ convertReading,
+ convertNumericToFullWidth,
+ convertHalfWidthKanaToFullWidth,
+ convertAlphabeticToKana,
+ distributeFurigana,
+ distributeFuriganaInflected
+ });
+})();
diff --git a/ext/bg/js/translator.js b/ext/bg/js/translator.js
index 25da9bf0..6f43f7b0 100644
--- a/ext/bg/js/translator.js
+++ b/ext/bg/js/translator.js
@@ -29,13 +29,7 @@
* dictTermsMergeBySequence
* dictTermsSort
* dictTermsUndupe
- * jpConvertAlphabeticToKana
- * jpConvertHalfWidthKanaToFullWidth
- * jpConvertNumericTofullWidth
- * jpDistributeFurigana
- * jpHiraganaToKatakana
- * jpIsCodePointJapanese
- * jpKatakanaToHiragana
+ * jp
* requestJson
*/
@@ -275,7 +269,7 @@ class Translator {
const termTags = await this.expandTags(definition.termTags, definition.dictionary);
const {expression, reading} = definition;
- const furiganaSegments = jpDistributeFurigana(expression, reading);
+ const furiganaSegments = jp.distributeFurigana(expression, reading);
definitions.push({
source: deinflection.source,
@@ -376,20 +370,20 @@ class Translator {
let sourceMapping = null;
if (halfWidth) {
if (sourceMapping === null) { sourceMapping = Translator.createTextSourceMapping(text2); }
- text2 = jpConvertHalfWidthKanaToFullWidth(text2, sourceMapping);
+ text2 = jp.convertHalfWidthKanaToFullWidth(text2, sourceMapping);
}
if (numeric) {
- text2 = jpConvertNumericTofullWidth(text2);
+ text2 = jp.convertNumericToFullWidth(text2);
}
if (alphabetic) {
if (sourceMapping === null) { sourceMapping = Translator.createTextSourceMapping(text2); }
- text2 = jpConvertAlphabeticToKana(text2, sourceMapping);
+ text2 = jp.convertAlphabeticToKana(text2, sourceMapping);
}
if (katakana) {
- text2 = jpHiraganaToKatakana(text2);
+ text2 = jp.convertHiraganaToKatakana(text2);
}
if (hiragana) {
- text2 = jpKatakanaToHiragana(text2);
+ text2 = jp.convertKatakanaToHiragana(text2);
}
for (let i = text2.length; i > 0; --i) {
@@ -590,7 +584,7 @@ class Translator {
}
static createExpression(expression, reading, termTags=null, termFrequency=null) {
- const furiganaSegments = jpDistributeFurigana(expression, reading);
+ const furiganaSegments = jp.distributeFurigana(expression, reading);
return {
expression,
reading,
@@ -639,7 +633,7 @@ class Translator {
if (!options.scanning.alphanumeric) {
let newText = '';
for (const c of text) {
- if (!jpIsCodePointJapanese(c.codePointAt(0))) {
+ if (!jp.isCodePointJapanese(c.codePointAt(0))) {
break;
}
newText += c;
diff --git a/ext/bg/search.html b/ext/bg/search.html
index f4c1a737..eacc1893 100644
--- a/ext/bg/search.html
+++ b/ext/bg/search.html
@@ -74,6 +74,7 @@
<script src="/mixed/js/core.js"></script>
<script src="/mixed/js/dom.js"></script>
<script src="/mixed/js/api.js"></script>
+ <script src="/mixed/js/japanese.js"></script>
<script src="/bg/js/dictionary.js"></script>
<script src="/bg/js/handlebars.js"></script>
diff --git a/ext/bg/settings.html b/ext/bg/settings.html
index 0db76d71..cfe20be4 100644
--- a/ext/bg/settings.html
+++ b/ext/bg/settings.html
@@ -1088,6 +1088,7 @@
<script src="/mixed/js/core.js"></script>
<script src="/mixed/js/dom.js"></script>
<script src="/mixed/js/api.js"></script>
+ <script src="/mixed/js/japanese.js"></script>
<script src="/bg/js/anki.js"></script>
<script src="/bg/js/anki-note-builder.js"></script>
diff --git a/ext/fg/float.html b/ext/fg/float.html
index 7bbed565..3ccf68eb 100644
--- a/ext/fg/float.html
+++ b/ext/fg/float.html
@@ -43,6 +43,7 @@
<script src="/mixed/js/core.js"></script>
<script src="/mixed/js/dom.js"></script>
<script src="/mixed/js/api.js"></script>
+ <script src="/mixed/js/japanese.js"></script>
<script src="/fg/js/document.js"></script>
<script src="/fg/js/source.js"></script>
diff --git a/ext/mixed/js/display-generator.js b/ext/mixed/js/display-generator.js
index 49afc44b..41f7315a 100644
--- a/ext/mixed/js/display-generator.js
+++ b/ext/mixed/js/display-generator.js
@@ -19,6 +19,7 @@
/* global
* TemplateHandler
* apiGetDisplayTemplatesHtml
+ * jp
*/
class DisplayGenerator {
@@ -283,7 +284,7 @@ class DisplayGenerator {
_appendKanjiLinks(container, text) {
let part = '';
for (const c of text) {
- if (DisplayGenerator._isCharacterKanji(c)) {
+ if (jp.isCodePointKanji(c.codePointAt(0))) {
if (part.length > 0) {
container.appendChild(document.createTextNode(part));
part = '';
@@ -300,14 +301,6 @@ class DisplayGenerator {
}
}
- static _isCharacterKanji(c) {
- const code = c.codePointAt(0);
- return (
- code >= 0x4e00 && code < 0x9fb0 ||
- code >= 0x3400 && code < 0x4dc0
- );
- }
-
static _appendMultiple(container, createItem, detailsArray, fallback=[]) {
if (container === null) { return 0; }
diff --git a/ext/mixed/js/japanese.js b/ext/mixed/js/japanese.js
new file mode 100644
index 00000000..61a247b2
--- /dev/null
+++ b/ext/mixed/js/japanese.js
@@ -0,0 +1,124 @@
+/*
+ * Copyright (C) 2020 Alex Yatskov <alex@foosoft.net>
+ * Author: Alex Yatskov <alex@foosoft.net>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+const jp = (() => {
+ const HIRAGANA_RANGE = [0x3040, 0x309f];
+ const KATAKANA_RANGE = [0x30a0, 0x30ff];
+ const KANA_RANGES = [HIRAGANA_RANGE, KATAKANA_RANGE];
+
+ const CJK_UNIFIED_IDEOGRAPHS_RANGE = [0x4e00, 0x9fff];
+ const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A_RANGE = [0x3400, 0x4dbf];
+ const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B_RANGE = [0x20000, 0x2a6df];
+ const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_C_RANGE = [0x2a700, 0x2b73f];
+ const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_D_RANGE = [0x2b740, 0x2b81f];
+ const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_E_RANGE = [0x2b820, 0x2ceaf];
+ const CJK_UNIFIED_IDEOGRAPHS_EXTENSION_F_RANGE = [0x2ceb0, 0x2ebef];
+ const CJK_COMPATIBILITY_IDEOGRAPHS_SUPPLEMENT_RANGE = [0x2f800, 0x2fa1f];
+ const CJK_UNIFIED_IDEOGRAPHS_RANGES = [
+ CJK_UNIFIED_IDEOGRAPHS_RANGE,
+ CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A_RANGE,
+ CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B_RANGE,
+ CJK_UNIFIED_IDEOGRAPHS_EXTENSION_C_RANGE,
+ CJK_UNIFIED_IDEOGRAPHS_EXTENSION_D_RANGE,
+ CJK_UNIFIED_IDEOGRAPHS_EXTENSION_E_RANGE,
+ CJK_UNIFIED_IDEOGRAPHS_EXTENSION_F_RANGE,
+ CJK_COMPATIBILITY_IDEOGRAPHS_SUPPLEMENT_RANGE
+ ];
+
+ // Japanese character ranges, roughly ordered in order of expected frequency
+ const JAPANESE_RANGES = [
+ HIRAGANA_RANGE,
+ KATAKANA_RANGE,
+
+ ...CJK_UNIFIED_IDEOGRAPHS_RANGES,
+
+ [0xff66, 0xff9f], // Halfwidth katakana
+
+ [0x30fb, 0x30fc], // Katakana punctuation
+ [0xff61, 0xff65], // Kana punctuation
+ [0x3000, 0x303f], // CJK punctuation
+
+ [0xff10, 0xff19], // Fullwidth numbers
+ [0xff21, 0xff3a], // Fullwidth upper case Latin letters
+ [0xff41, 0xff5a], // Fullwidth lower case Latin letters
+
+ [0xff01, 0xff0f], // Fullwidth punctuation 1
+ [0xff1a, 0xff1f], // Fullwidth punctuation 2
+ [0xff3b, 0xff3f], // Fullwidth punctuation 3
+ [0xff5b, 0xff60], // Fullwidth punctuation 4
+ [0xffe0, 0xffee] // Currency markers
+ ];
+
+
+ // Character code testing functions
+
+ function isCodePointKanji(codePoint) {
+ return isCodePointInRanges(codePoint, CJK_UNIFIED_IDEOGRAPHS_RANGES);
+ }
+
+ function isCodePointKana(codePoint) {
+ return isCodePointInRanges(codePoint, KANA_RANGES);
+ }
+
+ function isCodePointJapanese(codePoint) {
+ return isCodePointInRanges(codePoint, JAPANESE_RANGES);
+ }
+
+ function isCodePointInRanges(codePoint, ranges) {
+ for (const [min, max] of ranges) {
+ if (codePoint >= min && codePoint <= max) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+
+ // String testing functions
+
+ function isStringEntirelyKana(str) {
+ if (str.length === 0) { return false; }
+ for (const c of str) {
+ if (!isCodePointKana(c.codePointAt(0))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ function isStringPartiallyJapanese(str) {
+ if (str.length === 0) { return false; }
+ for (const c of str) {
+ if (isCodePointJapanese(c.codePointAt(0))) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+
+ // Exports
+
+ return {
+ isCodePointKanji,
+ isCodePointKana,
+ isCodePointJapanese,
+ isStringEntirelyKana,
+ isStringPartiallyJapanese
+ };
+})();