expand deinflection format (#745)

* abstract deinflections

* undo redundant changes

* remove cast

* switch to js

* MultiLanguageTransformer

* comments

* comments

* fix test

* suffixInflection

* fix bench

* substring instead of replace

* without heuristic

* suffixMap

* add other language deinflections

* wip

* catch cycles

* fix tests

* uninflect to deinflect

* use less regex

* add suru masu stem deinflection

1 files changed, 8 insertions, 3 deletions

diff --git a/ext/js/language/language-descriptors.js b/ext/js/language/language-descriptors.js
index 6675f78e..b947649d 100644
--- a/ext/js/language/language-descriptors.js
+++ b/ext/js/language/language-descriptors.js
@@ -17,10 +17,13 @@
 
 import {removeArabicScriptDiacritics} from './ar/arabic-text-preprocessors.js';
 import {eszettPreprocessor} from './de/german-text-preprocessors.js';
+import {englishTransforms} from './en/english-transforms.js';
 import {collapseEmphaticSequences, convertAlphabeticCharacters, convertHalfWidthCharacters, convertHiraganaToKatakana, convertNumericCharacters} from './ja/japanese-text-preprocessors.js';
+import {japaneseTransforms} from './ja/japanese-transforms.js';
 import {isStringPartiallyJapanese} from './ja/japanese.js';
 import {removeLatinDiacritics} from './la/latin-text-preprocessors.js';
 import {removeRussianDiacritics, yoToE} from './ru/russian-text-preprocessors.js';
+import {albanianTransforms} from './sq/albanian-transforms.js';
 import {capitalizeFirstLetter, decapitalize} from './text-preprocessors.js';
 
 const capitalizationPreprocessors = {
@@ -58,7 +61,7 @@ const languageDescriptors = [
         name: 'English',
         exampleText: 'read',
         textPreprocessors: capitalizationPreprocessors,
-        languageTransformsFile: '/js/language/en/english-transforms.json'
+        languageTransforms: englishTransforms
     },
     {
         iso: 'es',
@@ -109,6 +112,7 @@ const languageDescriptors = [
         name: 'Latin',
         exampleText: 'legere',
         textPreprocessors: {
+            ...capitalizationPreprocessors,
             removeLatinDiacritics
         }
     },
@@ -124,7 +128,7 @@ const languageDescriptors = [
             convertHiraganaToKatakana,
             collapseEmphaticSequences
         },
-        languageTransformsFile: '/js/language/ja/japanese-transforms.json'
+        languageTransforms: japaneseTransforms
     },
     {
         iso: 'km',
@@ -170,7 +174,8 @@ const languageDescriptors = [
         iso: 'sq',
         name: 'Albanian',
         exampleText: 'ndihmojme',
-        textPreprocessors: capitalizationPreprocessors
+        textPreprocessors: capitalizationPreprocessors,
+        languageTransforms: albanianTransforms
     },
     {
         iso: 'sv',