add language select, abstract text transformations (#584)

* Copy functions from JapaneseUtil

* Remove JapaneseUtil

* Update usages of JapaneseUtil functions

* part1

* frotend done?

* fix tests

* offscreen and type complications

* add tests

* start fixing tests

* keep fixing tests

* fix tests

* Copy functions from JapaneseUtil

* Remove JapaneseUtil

* Update usages of JapaneseUtil functions

* delete pt

* renames

* add tests

* kebab-case filenames

* lint

* minor fixes

* merge

* fixes

* fix part of comments

* fix more comments

* delete unused types

* comment

* comment

* do backend

* other files

* move fetch utils to own file

* remove extra line

* add extra line

* remove unnecessary export

* simplify folder structure

* remove redundant async

* fix param type in api

* fix language index

* undo changes to cssStyleApplier

* undo changes to utilities.js

* undo changes to utilities.js

* simplify language util

* lint

* undo phantom changes to anki integration

* require textTransformations options

* explicit locale in localeCompare

* punctuate notes

* prefer early exit

* rename LanguageOptionsObjectMap

* rename to textPreprocessor

* tuple with names instead of boolean array

* safe data setting

* optional chaining

* simplify LanguageOptions

* encapsulate languages

* delete language util

* nullable language in text preprocessors controller

* rename transform to process

* remove settings

* make translation advanced again

* remove unused getTextTransformations api call

* comments

* change language types

* RIP flags

* comments

* fix tests

* lint

* Text preprocessor type changes (#10)

* Add types

* Update types

* Simplify type check

* Refactor typing and structuring of language definitions

* lint

* update translator benchmark

* undo markdown changes

* undo markdown changes

* undo markdown changes

* more merge

* simplify language controller

---------

Co-authored-by: toasted-nutbread <toasted-nutbread@users.noreply.github.com>
Co-authored-by: Darius Jahandarie <djahandarie@gmail.com>

1 files changed, 77 insertions, 0 deletions

diff --git a/ext/js/language/ja/language-japanese.js b/ext/js/language/ja/language-japanese.js
new file mode 100644
index 00000000..ced34bcd
--- /dev/null
+++ b/ext/js/language/ja/language-japanese.js
@@ -0,0 +1,77 @@
+/*
+ * Copyright (C) 2024  Yomitan Authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import {basicTextPreprocessorOptions} from '../text-preprocessors.js';
+import {convertAlphabeticToKana} from './japanese-wanakana.js';
+import {collapseEmphaticSequences, convertHalfWidthKanaToFullWidth, convertHiraganaToKatakana, convertKatakanaToHiragana, convertNumericToFullWidth} from './japanese.js';
+
+/** @type {import('language-japanese').JapaneseLanguageDescriptor} */
+export const descriptor = {
+    name: 'Japanese',
+    iso: 'ja',
+    exampleText: '読め',
+    textPreprocessors: {
+        convertHalfWidthCharacters: {
+            name: 'Convert half width characters to full width',
+            description: 'ﾖﾐﾁｬﾝ → ヨミチャン',
+            options: basicTextPreprocessorOptions,
+            /** @type {import('language').TextPreprocessorFunction<boolean>} */
+            process: (str, setting, sourceMap) => (setting ? convertHalfWidthKanaToFullWidth(str, sourceMap) : str)
+        },
+        convertNumericCharacters: {
+            name: 'Convert numeric characters to full width',
+            description: '1234 → １２３４',
+            options: basicTextPreprocessorOptions,
+            /** @type {import('language').TextPreprocessorFunction<boolean>} */
+            process: (str, setting) => (setting ? convertNumericToFullWidth(str) : str)
+        },
+        convertAlphabeticCharacters: {
+            name: 'Convert alphabetic characters to hiragana',
+            description: 'yomichan → よみちゃん',
+            options: basicTextPreprocessorOptions,
+            /** @type {import('language').TextPreprocessorFunction<boolean>} */
+            process: (str, setting, sourceMap) => (setting ? convertAlphabeticToKana(str, sourceMap) : str)
+        },
+        convertHiraganaToKatakana: {
+            name: 'Convert hiragana to katakana',
+            description: 'よみちゃん → ヨミチャン',
+            options: basicTextPreprocessorOptions,
+            /** @type {import('language').TextPreprocessorFunction<boolean>} */
+            process: (str, setting) => (setting ? convertHiraganaToKatakana(str) : str)
+        },
+        convertKatakanaToHiragana: {
+            name: 'Convert katakana to hiragana',
+            description: 'ヨミチャン → よみちゃん',
+            options: basicTextPreprocessorOptions,
+            /** @type {import('language').TextPreprocessorFunction<boolean>} */
+            process: (str, setting) => (setting ? convertKatakanaToHiragana(str) : str)
+        },
+        collapseEmphaticSequences: {
+            name: 'Collapse emphatic character sequences',
+            description: 'すっっごーーい → すっごーい / すごい',
+            options: [[false, false], [true, false], [true, true]],
+            /** @type {import('language').TextPreprocessorFunction<[collapseEmphatic: boolean, collapseEmphaticFull: boolean]>} */
+            process: (str, setting, sourceMap) => {
+                const [collapseEmphatic, collapseEmphaticFull] = setting;
+                if (collapseEmphatic) {
+                    str = collapseEmphaticSequences(str, collapseEmphaticFull, sourceMap);
+                }
+                return str;
+            }
+        }
+    }
+};