add translator text transformation tests (#565)

* Copy functions from JapaneseUtil

* Remove JapaneseUtil

* Update usages of JapaneseUtil functions

* add tests

* Copy functions from JapaneseUtil

* Remove JapaneseUtil

* Update usages of JapaneseUtil functions

---------

Co-authored-by: toasted-nutbread <toasted-nutbread@users.noreply.github.com>
Co-authored-by: Darius Jahandarie <djahandarie@gmail.com>

1 files changed, 309 insertions, 0 deletions

diff --git a/test/data/anki-note-builder-test-results.json b/test/data/anki-note-builder-test-results.json
index 7129d748..b93b466b 100644
--- a/test/data/anki-note-builder-test-results.json
+++ b/test/data/anki-note-builder-test-results.json
@@ -2903,5 +2903,314 @@
         "url": "<a href=\"url:\">url:</a>"
       }
     ]
+  },
+  {
+    "name": "Test text transformations - convertNumericCharacters",
+    "results": [
+      {
+        "audio": "",
+        "clipboard-image": "",
+        "clipboard-text": "",
+        "cloze-body": "39",
+        "cloze-prefix": "cloze-prefix",
+        "cloze-suffix": "cloze-suffix",
+        "conjugation": "",
+        "dictionary": "Test Dictionary 2",
+        "document-title": "title",
+        "expression": "３９",
+        "frequencies": "",
+        "furigana": "<ruby>３９<rt>さんきゅう</rt></ruby>",
+        "furigana-plain": "３９[さんきゅう]",
+        "glossary": "<div style=\"text-align: left;\"><i>(Test Dictionary 2)</i> sankyuu definition</div>",
+        "glossary-brief": "<div style=\"text-align: left;\">sankyuu definition</div>",
+        "glossary-no-dictionary": "<div style=\"text-align: left;\">sankyuu definition</div>",
+        "part-of-speech": "Unknown",
+        "pitch-accents": "No pitch accent data",
+        "pitch-accent-graphs": "No pitch accent data",
+        "pitch-accent-positions": "No pitch accent data",
+        "phonetic-transcriptions": "",
+        "reading": "さんきゅう",
+        "screenshot": "",
+        "search-query": "fullQuery",
+        "selection-text": "",
+        "sentence": "cloze-prefix39cloze-suffix",
+        "sentence-furigana": "cloze-prefix39cloze-suffix",
+        "tags": "",
+        "url": "<a href=\"url:\">url:</a>"
+      }
+    ]
+  },
+  {
+    "name": "Test text transformations - convertAlphabeticCharacters",
+    "results": [
+      {
+        "audio": "",
+        "clipboard-image": "",
+        "clipboard-text": "",
+        "cloze-body": "utsu",
+        "cloze-prefix": "cloze-prefix",
+        "cloze-suffix": "cloze-suffix",
+        "conjugation": "",
+        "dictionary": "Test Dictionary 2",
+        "document-title": "title",
+        "expression": "打つ",
+        "frequencies": "<ul style=\"text-align: left;\"><li>Test Dictionary 2: 2</li><li>Test Dictionary 2: 6</li><li>Test Dictionary 2: 10</li><li>Test Dictionary 2: sixteen</li><li>Test Dictionary 2: twenty-two (22)</li><li>Test Dictionary 2: 28</li></ul>",
+        "furigana": "<ruby>打<rt>う</rt></ruby>つ",
+        "furigana-plain": "打[う]つ",
+        "glossary": "<div style=\"text-align: left;\"><i>(vt, Test Dictionary 2)</i> <ul><li>utsu definition 1</li><li>utsu definition 2</li></ul></div>",
+        "glossary-brief": "<div style=\"text-align: left;\"><ul><li>utsu definition 1</li><li>utsu definition 2</li></ul></div>",
+        "glossary-no-dictionary": "<div style=\"text-align: left;\"><i>(vt)</i> <ul><li>utsu definition 1</li><li>utsu definition 2</li></ul></div>",
+        "part-of-speech": "Godan verb",
+        "pitch-accents": "No pitch accent data",
+        "pitch-accent-graphs": "No pitch accent data",
+        "pitch-accent-positions": "No pitch accent data",
+        "phonetic-transcriptions": "",
+        "reading": "うつ",
+        "screenshot": "",
+        "search-query": "fullQuery",
+        "selection-text": "",
+        "sentence": "cloze-prefixutsucloze-suffix",
+        "sentence-furigana": "cloze-prefixutsucloze-suffix",
+        "tags": "vt",
+        "url": "<a href=\"url:\">url:</a>"
+      },
+      {
+        "audio": "",
+        "clipboard-image": "",
+        "clipboard-text": "",
+        "cloze-body": "utsu",
+        "cloze-prefix": "cloze-prefix",
+        "cloze-suffix": "cloze-suffix",
+        "conjugation": "",
+        "dictionary": "Test Dictionary 2",
+        "document-title": "title",
+        "expression": "打つ",
+        "frequencies": "<ul style=\"text-align: left;\"><li>Test Dictionary 2: 2</li><li>Test Dictionary 2: 6</li><li>Test Dictionary 2: 10</li><li>Test Dictionary 2: sixteen</li><li>Test Dictionary 2: twenty-two (22)</li><li>Test Dictionary 2: 28</li></ul>",
+        "furigana": "<ruby>打<rt>う</rt></ruby>つ",
+        "furigana-plain": "打[う]つ",
+        "glossary": "<div style=\"text-align: left;\"><i>(vt, Test Dictionary 2)</i> <ul><li>utsu definition 3</li><li>utsu definition 4</li></ul></div>",
+        "glossary-brief": "<div style=\"text-align: left;\"><ul><li>utsu definition 3</li><li>utsu definition 4</li></ul></div>",
+        "glossary-no-dictionary": "<div style=\"text-align: left;\"><i>(vt)</i> <ul><li>utsu definition 3</li><li>utsu definition 4</li></ul></div>",
+        "part-of-speech": "Godan verb",
+        "pitch-accents": "No pitch accent data",
+        "pitch-accent-graphs": "No pitch accent data",
+        "pitch-accent-positions": "No pitch accent data",
+        "phonetic-transcriptions": "",
+        "reading": "うつ",
+        "screenshot": "",
+        "search-query": "fullQuery",
+        "selection-text": "",
+        "sentence": "cloze-prefixutsucloze-suffix",
+        "sentence-furigana": "cloze-prefixutsucloze-suffix",
+        "tags": "vt",
+        "url": "<a href=\"url:\">url:</a>"
+      }
+    ]
+  },
+  {
+    "name": "Test text transformations - convertKatakanaToHiragana",
+    "results": [
+      {
+        "audio": "",
+        "clipboard-image": "",
+        "clipboard-text": "",
+        "cloze-body": "ウツ",
+        "cloze-prefix": "cloze-prefix",
+        "cloze-suffix": "cloze-suffix",
+        "conjugation": "",
+        "dictionary": "Test Dictionary 2",
+        "document-title": "title",
+        "expression": "打つ",
+        "frequencies": "<ul style=\"text-align: left;\"><li>Test Dictionary 2: 2</li><li>Test Dictionary 2: 6</li><li>Test Dictionary 2: 10</li><li>Test Dictionary 2: sixteen</li><li>Test Dictionary 2: twenty-two (22)</li><li>Test Dictionary 2: 28</li></ul>",
+        "furigana": "<ruby>打<rt>う</rt></ruby>つ",
+        "furigana-plain": "打[う]つ",
+        "glossary": "<div style=\"text-align: left;\"><i>(vt, Test Dictionary 2)</i> <ul><li>utsu definition 1</li><li>utsu definition 2</li></ul></div>",
+        "glossary-brief": "<div style=\"text-align: left;\"><ul><li>utsu definition 1</li><li>utsu definition 2</li></ul></div>",
+        "glossary-no-dictionary": "<div style=\"text-align: left;\"><i>(vt)</i> <ul><li>utsu definition 1</li><li>utsu definition 2</li></ul></div>",
+        "part-of-speech": "Godan verb",
+        "pitch-accents": "No pitch accent data",
+        "pitch-accent-graphs": "No pitch accent data",
+        "pitch-accent-positions": "No pitch accent data",
+        "phonetic-transcriptions": "",
+        "reading": "うつ",
+        "screenshot": "",
+        "search-query": "fullQuery",
+        "selection-text": "",
+        "sentence": "cloze-prefixウツcloze-suffix",
+        "sentence-furigana": "cloze-prefixウツcloze-suffix",
+        "tags": "vt",
+        "url": "<a href=\"url:\">url:</a>"
+      },
+      {
+        "audio": "",
+        "clipboard-image": "",
+        "clipboard-text": "",
+        "cloze-body": "ウツ",
+        "cloze-prefix": "cloze-prefix",
+        "cloze-suffix": "cloze-suffix",
+        "conjugation": "",
+        "dictionary": "Test Dictionary 2",
+        "document-title": "title",
+        "expression": "打つ",
+        "frequencies": "<ul style=\"text-align: left;\"><li>Test Dictionary 2: 2</li><li>Test Dictionary 2: 6</li><li>Test Dictionary 2: 10</li><li>Test Dictionary 2: sixteen</li><li>Test Dictionary 2: twenty-two (22)</li><li>Test Dictionary 2: 28</li></ul>",
+        "furigana": "<ruby>打<rt>う</rt></ruby>つ",
+        "furigana-plain": "打[う]つ",
+        "glossary": "<div style=\"text-align: left;\"><i>(vt, Test Dictionary 2)</i> <ul><li>utsu definition 3</li><li>utsu definition 4</li></ul></div>",
+        "glossary-brief": "<div style=\"text-align: left;\"><ul><li>utsu definition 3</li><li>utsu definition 4</li></ul></div>",
+        "glossary-no-dictionary": "<div style=\"text-align: left;\"><i>(vt)</i> <ul><li>utsu definition 3</li><li>utsu definition 4</li></ul></div>",
+        "part-of-speech": "Godan verb",
+        "pitch-accents": "No pitch accent data",
+        "pitch-accent-graphs": "No pitch accent data",
+        "pitch-accent-positions": "No pitch accent data",
+        "phonetic-transcriptions": "",
+        "reading": "うつ",
+        "screenshot": "",
+        "search-query": "fullQuery",
+        "selection-text": "",
+        "sentence": "cloze-prefixウツcloze-suffix",
+        "sentence-furigana": "cloze-prefixウツcloze-suffix",
+        "tags": "vt",
+        "url": "<a href=\"url:\">url:</a>"
+      }
+    ]
+  },
+  {
+    "name": "Test text transformations - convertHiraganaToKatakana",
+    "results": [
+      {
+        "audio": "",
+        "clipboard-image": "",
+        "clipboard-text": "",
+        "cloze-body": "てきすと",
+        "cloze-prefix": "cloze-prefix",
+        "cloze-suffix": "cloze-suffix",
+        "conjugation": "",
+        "dictionary": "Test Dictionary 2",
+        "document-title": "title",
+        "expression": "テキスト",
+        "frequencies": "",
+        "furigana": "テキスト",
+        "furigana-plain": "テキスト",
+        "glossary": "<div style=\"text-align: left;\"><i>(n, Test Dictionary 2)</i> <ul><li>text definition 1</li><li>text definition 2</li></ul></div>",
+        "glossary-brief": "<div style=\"text-align: left;\"><ul><li>text definition 1</li><li>text definition 2</li></ul></div>",
+        "glossary-no-dictionary": "<div style=\"text-align: left;\"><i>(n)</i> <ul><li>text definition 1</li><li>text definition 2</li></ul></div>",
+        "part-of-speech": "Noun",
+        "pitch-accents": "No pitch accent data",
+        "pitch-accent-graphs": "No pitch accent data",
+        "pitch-accent-positions": "No pitch accent data",
+        "phonetic-transcriptions": "",
+        "reading": "テキスト",
+        "screenshot": "",
+        "search-query": "fullQuery",
+        "selection-text": "",
+        "sentence": "cloze-prefixてきすとcloze-suffix",
+        "sentence-furigana": "cloze-prefixてきすとcloze-suffix",
+        "tags": "n",
+        "url": "<a href=\"url:\">url:</a>"
+      }
+    ]
+  },
+  {
+    "name": "Test text transformations - convertHalfWidthCharacters, convertKatakanaToHiragana",
+    "results": [
+      {
+        "audio": "",
+        "clipboard-image": "",
+        "clipboard-text": "",
+        "cloze-body": "ｳﾂ",
+        "cloze-prefix": "cloze-prefix",
+        "cloze-suffix": "cloze-suffix",
+        "conjugation": "",
+        "dictionary": "Test Dictionary 2",
+        "document-title": "title",
+        "expression": "打つ",
+        "frequencies": "<ul style=\"text-align: left;\"><li>Test Dictionary 2: 2</li><li>Test Dictionary 2: 6</li><li>Test Dictionary 2: 10</li><li>Test Dictionary 2: sixteen</li><li>Test Dictionary 2: twenty-two (22)</li><li>Test Dictionary 2: 28</li></ul>",
+        "furigana": "<ruby>打<rt>う</rt></ruby>つ",
+        "furigana-plain": "打[う]つ",
+        "glossary": "<div style=\"text-align: left;\"><i>(vt, Test Dictionary 2)</i> <ul><li>utsu definition 1</li><li>utsu definition 2</li></ul></div>",
+        "glossary-brief": "<div style=\"text-align: left;\"><ul><li>utsu definition 1</li><li>utsu definition 2</li></ul></div>",
+        "glossary-no-dictionary": "<div style=\"text-align: left;\"><i>(vt)</i> <ul><li>utsu definition 1</li><li>utsu definition 2</li></ul></div>",
+        "part-of-speech": "Godan verb",
+        "pitch-accents": "No pitch accent data",
+        "pitch-accent-graphs": "No pitch accent data",
+        "pitch-accent-positions": "No pitch accent data",
+        "phonetic-transcriptions": "",
+        "reading": "うつ",
+        "screenshot": "",
+        "search-query": "fullQuery",
+        "selection-text": "",
+        "sentence": "cloze-prefixｳﾂcloze-suffix",
+        "sentence-furigana": "cloze-prefixｳﾂcloze-suffix",
+        "tags": "vt",
+        "url": "<a href=\"url:\">url:</a>"
+      },
+      {
+        "audio": "",
+        "clipboard-image": "",
+        "clipboard-text": "",
+        "cloze-body": "ｳﾂ",
+        "cloze-prefix": "cloze-prefix",
+        "cloze-suffix": "cloze-suffix",
+        "conjugation": "",
+        "dictionary": "Test Dictionary 2",
+        "document-title": "title",
+        "expression": "打つ",
+        "frequencies": "<ul style=\"text-align: left;\"><li>Test Dictionary 2: 2</li><li>Test Dictionary 2: 6</li><li>Test Dictionary 2: 10</li><li>Test Dictionary 2: sixteen</li><li>Test Dictionary 2: twenty-two (22)</li><li>Test Dictionary 2: 28</li></ul>",
+        "furigana": "<ruby>打<rt>う</rt></ruby>つ",
+        "furigana-plain": "打[う]つ",
+        "glossary": "<div style=\"text-align: left;\"><i>(vt, Test Dictionary 2)</i> <ul><li>utsu definition 3</li><li>utsu definition 4</li></ul></div>",
+        "glossary-brief": "<div style=\"text-align: left;\"><ul><li>utsu definition 3</li><li>utsu definition 4</li></ul></div>",
+        "glossary-no-dictionary": "<div style=\"text-align: left;\"><i>(vt)</i> <ul><li>utsu definition 3</li><li>utsu definition 4</li></ul></div>",
+        "part-of-speech": "Godan verb",
+        "pitch-accents": "No pitch accent data",
+        "pitch-accent-graphs": "No pitch accent data",
+        "pitch-accent-positions": "No pitch accent data",
+        "phonetic-transcriptions": "",
+        "reading": "うつ",
+        "screenshot": "",
+        "search-query": "fullQuery",
+        "selection-text": "",
+        "sentence": "cloze-prefixｳﾂcloze-suffix",
+        "sentence-furigana": "cloze-prefixｳﾂcloze-suffix",
+        "tags": "vt",
+        "url": "<a href=\"url:\">url:</a>"
+      }
+    ]
+  },
+  {
+    "name": "Test text transformations - collapseEmphaticSequences",
+    "results": [
+      {
+        "audio": "",
+        "clipboard-image": "",
+        "clipboard-text": "",
+        "cloze-body": "すっっごーーい",
+        "cloze-prefix": "cloze-prefix",
+        "cloze-suffix": "cloze-suffix",
+        "conjugation": "",
+        "dictionary": "Test Dictionary 2",
+        "document-title": "title",
+        "expression": "凄い",
+        "frequencies": "",
+        "furigana": "<ruby>凄<rt>すご</rt></ruby>い",
+        "furigana-plain": "凄[すご]い",
+        "glossary": "<div style=\"text-align: left;\"><i>(adj-i, Test Dictionary 2)</i> sugoi definition</div>",
+        "glossary-brief": "<div style=\"text-align: left;\">sugoi definition</div>",
+        "glossary-no-dictionary": "<div style=\"text-align: left;\"><i>(adj-i)</i> sugoi definition</div>",
+        "part-of-speech": "I-adjective",
+        "pitch-accents": "No pitch accent data",
+        "pitch-accent-graphs": "No pitch accent data",
+        "pitch-accent-positions": "No pitch accent data",
+        "phonetic-transcriptions": "",
+        "reading": "すごい",
+        "screenshot": "",
+        "search-query": "fullQuery",
+        "selection-text": "",
+        "sentence": "cloze-prefixすっっごーーいcloze-suffix",
+        "sentence-furigana": "cloze-prefixすっっごーーいcloze-suffix",
+        "tags": "adj-i",
+        "url": "<a href=\"url:\">url:</a>"
+      }
+    ]
   }
 ]
\ No newline at end of file