add language select, abstract text transformations (#584)

* Copy functions from JapaneseUtil

* Remove JapaneseUtil

* Update usages of JapaneseUtil functions

* part1

* frotend done?

* fix tests

* offscreen and type complications

* add tests

* start fixing tests

* keep fixing tests

* fix tests

* Copy functions from JapaneseUtil

* Remove JapaneseUtil

* Update usages of JapaneseUtil functions

* delete pt

* renames

* add tests

* kebab-case filenames

* lint

* minor fixes

* merge

* fixes

* fix part of comments

* fix more comments

* delete unused types

* comment

* comment

* do backend

* other files

* move fetch utils to own file

* remove extra line

* add extra line

* remove unnecessary export

* simplify folder structure

* remove redundant async

* fix param type in api

* fix language index

* undo changes to cssStyleApplier

* undo changes to utilities.js

* undo changes to utilities.js

* simplify language util

* lint

* undo phantom changes to anki integration

* require textTransformations options

* explicit locale in localeCompare

* punctuate notes

* prefer early exit

* rename LanguageOptionsObjectMap

* rename to textPreprocessor

* tuple with names instead of boolean array

* safe data setting

* optional chaining

* simplify LanguageOptions

* encapsulate languages

* delete language util

* nullable language in text preprocessors controller

* rename transform to process

* remove settings

* make translation advanced again

* remove unused getTextTransformations api call

* comments

* change language types

* RIP flags

* comments

* fix tests

* lint

* Text preprocessor type changes (#10)

* Add types

* Update types

* Simplify type check

* Refactor typing and structuring of language definitions

* lint

* update translator benchmark

* undo markdown changes

* undo markdown changes

* undo markdown changes

* more merge

* simplify language controller

---------

Co-authored-by: toasted-nutbread <toasted-nutbread@users.noreply.github.com>
Co-authored-by: Darius Jahandarie <djahandarie@gmail.com>

1 files changed, 38 insertions, 22 deletions

diff --git a/test/data/translator-test-inputs.json b/test/data/translator-test-inputs.json
index 03ec938a..c9047716 100644
--- a/test/data/translator-test-inputs.json
+++ b/test/data/translator-test-inputs.json
@@ -21,12 +21,7 @@
             "sortFrequencyDictionary": null,
             "sortFrequencyDictionaryOrder": "descending",
             "removeNonJapaneseCharacters": true,
-            "convertHalfWidthCharacters": "false",
-            "convertNumericCharacters": "false",
-            "convertAlphabeticCharacters": "false",
-            "convertHiraganaToKatakana": "false",
-            "convertKatakanaToHiragana": "false",
-            "collapseEmphaticSequences": "false",
+            "language": "ja",
             "textReplacements": [
                 null
             ],
@@ -361,7 +356,7 @@
             "options": "default"
         },
         {
-            "name": "Test text transformations - convertNumericCharacters",
+            "name": "Test text preprocessors - convertNumericCharacters",
             "func": "findTerms",
             "mode": "split",
             "text": "39",
@@ -369,13 +364,12 @@
                 "default",
                 {
                     "type": "terms",
-                    "convertNumericCharacters": "true",
                     "removeNonJapaneseCharacters": false
                 }
             ]
         },
         {
-            "name": "Test text transformations - convertAlphabeticCharacters",
+            "name": "Test text preprocessors - convertAlphabeticCharacters",
             "func": "findTerms",
             "mode": "split",
             "text": "utsu",
@@ -383,61 +377,83 @@
                 "default",
                 {
                     "type": "terms",
-                    "convertAlphabeticCharacters": "true",
                     "removeNonJapaneseCharacters": false
                 }
             ]
         },
         {
-            "name": "Test text transformations - convertKatakanaToHiragana",
+            "name": "Test text preprocessors - convertKatakanaToHiragana",
             "func": "findTerms",
             "mode": "split",
             "text": "ウツ",
             "options": [
                 "default",
                 {
-                    "type": "terms",
-                    "convertKatakanaToHiragana": "true"
+                    "type": "terms"
                 }
             ]
         },
         {
-            "name": "Test text transformations - convertHiraganaToKatakana",
+            "name": "Test text preprocessors - convertHiraganaToKatakana",
             "func": "findTerms",
             "mode": "split",
             "text": "てきすと",
             "options": [
                 "default",
                 {
-                    "type": "terms",
-                    "convertHiraganaToKatakana": "true"
+                    "type": "terms"
                 }
             ]
         },
         {
-            "name": "Test text transformations - convertHalfWidthCharacters, convertKatakanaToHiragana",
+            "name": "Test text preprocessors - convertHalfWidthCharacters, convertKatakanaToHiragana",
             "func": "findTerms",
             "mode": "split",
             "text": "ｳﾂ",
             "options": [
                 "default",
                 {
-                    "type": "terms",
-                    "convertHalfWidthCharacters": "true",
-                    "convertKatakanaToHiragana": "true"
+                    "type": "terms"
                 }
             ]
         },
         {
-            "name": "Test text transformations - collapseEmphaticSequences",
+            "name": "Test text preprocessors - collapseEmphaticSequences",
             "func": "findTerms",
             "mode": "split",
             "text": "すっっごーーい",
             "options": [
                 "default",
                 {
+                    "type": "terms"
+                }
+            ]
+        },
+        {
+            "name": "Test text preprocessors - capitalizeFirstLetter",
+            "func": "findTerms",
+            "mode": "split",
+            "text": "english",
+            "options": [
+                "default",
+                {
                     "type": "terms",
-                    "collapseEmphaticSequences": "full"
+                    "language": "en",
+                    "removeNonJapaneseCharacters": false
+                }
+            ]
+        },
+        {
+            "name": "Test text preprocessors - decapitalize",
+            "func": "findTerms",
+            "mode": "split",
+            "text": "LANGUAGE",
+            "options": [
+                "default",
+                {
+                    "type": "terms",
+                    "language": "en",
+                    "removeNonJapaneseCharacters": false
                 }
             ]
         }