add language select, abstract text transformations (#584)

* Copy functions from JapaneseUtil

* Remove JapaneseUtil

* Update usages of JapaneseUtil functions

* part1

* frotend done?

* fix tests

* offscreen and type complications

* add tests

* start fixing tests

* keep fixing tests

* fix tests

* Copy functions from JapaneseUtil

* Remove JapaneseUtil

* Update usages of JapaneseUtil functions

* delete pt

* renames

* add tests

* kebab-case filenames

* lint

* minor fixes

* merge

* fixes

* fix part of comments

* fix more comments

* delete unused types

* comment

* comment

* do backend

* other files

* move fetch utils to own file

* remove extra line

* add extra line

* remove unnecessary export

* simplify folder structure

* remove redundant async

* fix param type in api

* fix language index

* undo changes to cssStyleApplier

* undo changes to utilities.js

* undo changes to utilities.js

* simplify language util

* lint

* undo phantom changes to anki integration

* require textTransformations options

* explicit locale in localeCompare

* punctuate notes

* prefer early exit

* rename LanguageOptionsObjectMap

* rename to textPreprocessor

* tuple with names instead of boolean array

* safe data setting

* optional chaining

* simplify LanguageOptions

* encapsulate languages

* delete language util

* nullable language in text preprocessors controller

* rename transform to process

* remove settings

* make translation advanced again

* remove unused getTextTransformations api call

* comments

* change language types

* RIP flags

* comments

* fix tests

* lint

* Text preprocessor type changes (#10)

* Add types

* Update types

* Simplify type check

* Refactor typing and structuring of language definitions

* lint

* update translator benchmark

* undo markdown changes

* undo markdown changes

* undo markdown changes

* more merge

* simplify language controller

---------

Co-authored-by: toasted-nutbread <toasted-nutbread@users.noreply.github.com>
Co-authored-by: Darius Jahandarie <djahandarie@gmail.com>

8 files changed, 127 insertions, 42 deletions

diff --git a/types/ext/api.d.ts b/types/ext/api.d.ts
index 1f4fc0a9..85f4c146 100644
--- a/types/ext/api.d.ts
+++ b/types/ext/api.d.ts
@@ -26,6 +26,7 @@ import type * as DictionaryDatabase from './dictionary-database';
 import type * as DictionaryImporter from './dictionary-importer';
 import type * as Environment from './environment';
 import type * as Extension from './extension';
+import type * as Language from './language';
 import type * as Log from './log';
 import type * as Settings from './settings';
 import type * as SettingsModifications from './settings-modifications';
@@ -380,6 +381,10 @@ type ApiSurface = {
         params: void;
         return: boolean;
     };
+    getLanguageSummaries: {
+        params: void;
+        return: Language.LanguageSummary[];
+    };
 };
 
 type ApiExtraArgs = [sender: chrome.runtime.MessageSender];
diff --git a/types/ext/language-english.d.ts b/types/ext/language-english.d.ts
new file mode 100644
index 00000000..ed501d57
--- /dev/null
+++ b/types/ext/language-english.d.ts
@@ -0,0 +1,25 @@
+/*
+ * Copyright (C) 2024  Yomitan Authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import type {LanguageDescriptor, TextPreprocessor} from './language';
+
+export type EnglishTextPreprocessorDescriptor = {
+    capitalizeFirstLetter: TextPreprocessor<boolean>;
+    decapitalize: TextPreprocessor<boolean>;
+};
+
+export type EnglishLanguageDescriptor = LanguageDescriptor<EnglishTextPreprocessorDescriptor>;
diff --git a/types/ext/language-japanese.d.ts b/types/ext/language-japanese.d.ts
new file mode 100644
index 00000000..1a627ed1
--- /dev/null
+++ b/types/ext/language-japanese.d.ts
@@ -0,0 +1,29 @@
+/*
+ * Copyright (C) 2024  Yomitan Authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import type {LanguageDescriptor, TextPreprocessor} from './language';
+
+export type JapaneseTextPreprocessorDescriptor = {
+    convertHalfWidthCharacters: TextPreprocessor<boolean>;
+    convertNumericCharacters: TextPreprocessor<boolean>;
+    convertAlphabeticCharacters: TextPreprocessor<boolean>;
+    convertHiraganaToKatakana: TextPreprocessor<boolean>;
+    convertKatakanaToHiragana: TextPreprocessor<boolean>;
+    collapseEmphaticSequences: TextPreprocessor<[collapseEmphatic: boolean, collapseEmphaticFull: boolean]>;
+};
+
+export type JapaneseLanguageDescriptor = LanguageDescriptor<JapaneseTextPreprocessorDescriptor>;
diff --git a/types/ext/language.d.ts b/types/ext/language.d.ts
new file mode 100644
index 00000000..247c7795
--- /dev/null
+++ b/types/ext/language.d.ts
@@ -0,0 +1,57 @@
+/*
+ * Copyright (C) 2024  Yomitan Authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import type {TextSourceMap} from '../../ext/js/general/text-source-map.js';
+import type {SafeAny} from './core';
+
+export type TextPreprocessorOptions<T = unknown> = T[];
+
+export type TextPreprocessorFunction<T = unknown> = (str: string, setting: T, sourceMap: TextSourceMap) => string;
+
+export type TextPreprocessor<T = unknown> = {
+    name: string;
+    description: string;
+    options: TextPreprocessorOptions<T>;
+    process: TextPreprocessorFunction<T>;
+};
+
+export type LanguageAndPreprocessors = {
+    iso: string;
+    textPreprocessors: TextPreprocessorWithId<unknown>[];
+};
+
+export type TextPreprocessorWithId<T = unknown> = {
+    id: string;
+    textPreprocessor: TextPreprocessor<T>;
+};
+
+export type LanguageSummary = {
+    name: string;
+    iso: string;
+    exampleText: string;
+};
+
+export type LanguageDescriptor<TTextPreprocessorDescriptor extends TextPreprocessorDescriptor> = {
+    name: string;
+    iso: string;
+    exampleText: string;
+    textPreprocessors: TTextPreprocessorDescriptor;
+};
+
+export type TextPreprocessorDescriptor = {
+    [key: string]: TextPreprocessor<SafeAny>;
+};
diff --git a/types/ext/settings.d.ts b/types/ext/settings.d.ts
index a900dbe6..45466c3d 100644
--- a/types/ext/settings.d.ts
+++ b/types/ext/settings.d.ts
@@ -101,6 +101,7 @@ export type ProfileOptions = {
 
 export type GeneralOptions = {
     enable: boolean;
+    language: string;
     resultOutputMode: ResultOutputMode;
     debugInfo: boolean;
     maxResults: number;
diff --git a/types/ext/translation-internal.d.ts b/types/ext/translation-internal.d.ts
index 82704c54..7006221e 100644
--- a/types/ext/translation-internal.d.ts
+++ b/types/ext/translation-internal.d.ts
@@ -18,6 +18,7 @@
 import type * as DictionaryDatabase from './dictionary-database';
 import type * as Dictionary from './dictionary';
 import type * as Translation from './translation';
+import type * as Language from './language';
 
 export type TextDeinflectionOptions = [
     textReplacements: Translation.FindTermsTextReplacement[] | null,
@@ -47,3 +48,5 @@ export type DatabaseDeinflection = {
     inflectionRuleChainCandidates: Dictionary.InflectionRuleChainCandidate[];
     databaseEntries: DictionaryDatabase.TermEntry[];
 };
+
+export type PreprocessorOptionsSpace = Map<string, Language.TextPreprocessorOptions<unknown>>;
diff --git a/types/ext/translation.d.ts b/types/ext/translation.d.ts
index c9a61be0..2e4d1a66 100644
--- a/types/ext/translation.d.ts
+++ b/types/ext/translation.d.ts
@@ -81,30 +81,6 @@ export type FindTermsOptions = {
      */
     removeNonJapaneseCharacters: boolean;
     /**
-     * Whether or not half-width characters should be converted to full-width characters.
-     */
-    convertHalfWidthCharacters: FindTermsVariantMode;
-    /**
-     * Whether or not ASCII numeric characters should be converted to full-width numeric characters.
-     */
-    convertNumericCharacters: FindTermsVariantMode;
-    /**
-     * Whether or not alphabetic characters should be converted to kana.
-     */
-    convertAlphabeticCharacters: FindTermsVariantMode;
-    /**
-     * Whether or not hiragana characters should be converted to katakana.
-     */
-    convertHiraganaToKatakana: FindTermsVariantMode;
-    /**
-     * Whether or not katakana characters should be converted to hiragana.
-     */
-    convertKatakanaToHiragana: FindTermsVariantMode;
-    /**
-     * How emphatic character sequences should be collapsed.
-     */
-    collapseEmphaticSequences: FindTermsEmphaticSequencesMode;
-    /**
      * An iterable sequence of text replacements to be applied during the term lookup process.
      */
     textReplacements: FindTermsTextReplacements;
@@ -121,6 +97,10 @@ export type FindTermsOptions = {
      * Whether every substring should be searched for, or only whole words.
      */
     searchResolution: SearchResolution;
+    /**
+     * ISO-639 code of the language.
+     */
+    language: string;
 };
 
 /**
@@ -134,16 +114,6 @@ export type FindTermsMatchType = Dictionary.TermSourceMatchType;
 export type FindTermsSortOrder = 'ascending' | 'descending';
 
 /**
- * Mode describing how to handle variations.
- */
-export type FindTermsVariantMode = 'false' | 'true' | 'variant';
-
-/**
- * Mode describing how to handle emphatic sequence variations.
- */
-export type FindTermsEmphaticSequencesMode = 'false' | 'true' | 'full';
-
-/**
  * Information about how text should be replaced when looking up terms.
  */
 export type FindTermsTextReplacement = {
diff --git a/types/test/translator.d.ts b/types/test/translator.d.ts
index e3199225..efd5cc3f 100644
--- a/types/test/translator.d.ts
+++ b/types/test/translator.d.ts
@@ -15,8 +15,8 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-import type {FindTermsMatchType, FindTermsSortOrder, FindTermsVariantMode, FindTermsEmphaticSequencesMode, FindKanjiDictionary, FindTermDictionary} from '../ext/translation';
-import type {SearchResolution} from 'settings';
+import type {FindTermsMatchType, FindTermsSortOrder, FindKanjiDictionary, FindTermDictionary} from '../ext/translation';
+import type {SearchResolution} from '../ext/settings';
 import type {FindTermsMode} from 'translator';
 import type {DictionaryEntry} from 'dictionary';
 import type {NoteData} from 'anki-templates';
@@ -44,16 +44,11 @@ export type FindTermsOptionsPreset = {
     sortFrequencyDictionary?: string | null;
     sortFrequencyDictionaryOrder?: FindTermsSortOrder;
     removeNonJapaneseCharacters?: boolean;
-    convertHalfWidthCharacters?: FindTermsVariantMode;
-    convertNumericCharacters?: FindTermsVariantMode;
-    convertAlphabeticCharacters?: FindTermsVariantMode;
-    convertHiraganaToKatakana?: FindTermsVariantMode;
-    convertKatakanaToHiragana?: FindTermsVariantMode;
-    collapseEmphaticSequences?: FindTermsEmphaticSequencesMode;
     textReplacements?: (FindTermsTextReplacement[] | null)[];
     enabledDictionaryMap?: [key: string, value: FindTermDictionary][];
     excludeDictionaryDefinitions?: string[] | null;
     searchResolution?: SearchResolution;
+    language?: string;
 };
 
 export type OptionsType = OptionsPreset['type'];