expand deinflection format (#745)

* abstract deinflections

* undo redundant changes

* remove cast

* switch to js

* MultiLanguageTransformer

* comments

* comments

* fix test

* suffixInflection

* fix bench

* substring instead of replace

* without heuristic

* suffixMap

* add other language deinflections

* wip

* catch cycles

* fix tests

* uninflect to deinflect

* use less regex

* add suru masu stem deinflection

1 files changed, 24 insertions, 29 deletions

diff --git a/test/language-transformer-cycles.test.js b/test/language-transformer-cycles.test.js
index 751f65a6..ab14eaa4 100644
--- a/test/language-transformer-cycles.test.js
+++ b/test/language-transformer-cycles.test.js
@@ -15,13 +15,9 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-import {readFileSync} from 'fs';
-import {join, dirname as pathDirname} from 'path';
-import {fileURLToPath} from 'url';
 import {describe, test} from 'vitest';
-import {parseJson} from '../dev/json.js';
 import {LanguageTransformer} from '../ext/js/language/language-transformer.js';
-import {getLanguageSummaries} from '../ext/js/language/languages.js';
+import {getAllLanguageTransformDescriptors} from '../ext/js/language/languages.js';
 
 class DeinflectionNode {
     /**
@@ -80,12 +76,12 @@ class DeinflectionNode {
 class RuleNode {
     /**
      * @param {string} groupName
-     * @param {import('language-transformer').Rule} rule
+     * @param {import('language-transformer').SuffixRule} rule
      */
     constructor(groupName, rule) {
         /** @type {string} */
         this.groupName = groupName;
-        /** @type {import('language-transformer').Rule} */
+        /** @type {import('language-transformer').SuffixRule} */
         this.rule = rule;
     }
 }
@@ -104,39 +100,36 @@ function arraysAreEqual(rules1, rules2) {
     return true;
 }
 
-const dirname = pathDirname(fileURLToPath(import.meta.url));
-const descriptors = [];
-const languageSummaries = getLanguageSummaries();
-for (const {languageTransformsFile} of languageSummaries) {
-    if (!languageTransformsFile) { continue; }
-    /** @type {import('language-transformer').LanguageTransformDescriptor} */
-    const descriptor = parseJson(readFileSync(join(dirname, `../ext/${languageTransformsFile}`), {encoding: 'utf8'}));
-    descriptors.push(descriptor);
-}
+const languagesWithTransforms = getAllLanguageTransformDescriptors();
 
-describe.each(descriptors)('Cycles Test $language', (descriptor) => {
+describe.each(languagesWithTransforms)('Cycles Test $iso', ({languageTransforms}) => {
     test('Check for cycles', ({expect}) => {
         const languageTransformer = new LanguageTransformer();
-        languageTransformer.addDescriptor(descriptor);
+        languageTransformer.addDescriptor(languageTransforms);
 
         /** @type {RuleNode[]} */
         const ruleNodes = [];
-        for (const [groupName, reasonInfo] of Object.entries(descriptor.transforms)) {
+        for (const [groupName, reasonInfo] of Object.entries(languageTransforms.transforms)) {
             for (const rule of reasonInfo.rules) {
-                ruleNodes.push(new RuleNode(groupName, rule));
+                if (rule.type === 'suffix') {
+                    ruleNodes.push(new RuleNode(groupName, /** @type {import('language-transformer').SuffixRule}*/ (rule)));
+                }
             }
         }
 
         /** @type {DeinflectionNode[]} */
         const deinflectionNodes = [];
-        for (const ruleNode of ruleNodes) {
-            deinflectionNodes.push(new DeinflectionNode(`?${ruleNode.rule.suffixIn}`, [], null, null));
+        for (const {rule: {isInflected}} of ruleNodes) {
+            const suffixIn = isInflected.source.substring(0, isInflected.source.length - 1);
+            deinflectionNodes.push(new DeinflectionNode(`?${suffixIn}`, [], null, null));
         }
+
         for (let i = 0; i < deinflectionNodes.length; ++i) {
             const deinflectionNode = deinflectionNodes[i];
             const {text, ruleNames} = deinflectionNode;
             for (const ruleNode of ruleNodes) {
-                const {suffixIn, suffixOut, conditionsIn, conditionsOut} = ruleNode.rule;
+                const {isInflected, deinflected: suffixOut, conditionsIn, conditionsOut} = ruleNode.rule;
+                const suffixIn = isInflected.source.substring(0, isInflected.source.length - 1);
                 if (
                     !LanguageTransformer.conditionsMatch(
                         languageTransformer.getConditionFlagsFromConditionTypes(ruleNames),
@@ -158,12 +151,14 @@ describe.each(descriptors)('Cycles Test $language', (descriptor) => {
                 // Cycle check
                 if (deinflectionNode.historyIncludes(newDeinflectionNode)) {
                     const stack = [];
-                    for (const item of newDeinflectionNode.getHistory()) {
-                        stack.push(
-                            item.ruleNode === null ?
-                                `${item.text} (start)` :
-                                `${item.text} (${item.ruleNode.groupName}, ${item.ruleNode.rule.conditionsIn.join(',')}=>${item.ruleNode.rule.conditionsOut.join(',')}, ${item.ruleNode.rule.suffixIn}=>${item.ruleNode.rule.suffixOut})`
-                        );
+                    for (const {text: itemText, ruleNode: itemNode} of newDeinflectionNode.getHistory()) {
+                        if (itemNode !== null) {
+                            const itemSuffixIn = itemNode.rule.isInflected.source.substring(0, itemNode.rule.isInflected.source.length - 1);
+                            const itemSuffixOut = itemNode.rule.deinflected;
+                            stack.push(`${itemText} (${itemNode.groupName}, ${itemNode.rule.conditionsIn.join(',')}=>${itemNode.rule.conditionsOut.join(',')}, ${itemSuffixIn}=>${itemSuffixOut})`);
+                        } else {
+                            stack.push(`${itemText} (start)`);
+                        }
                     }
                     const message = `Cycle detected:\n  ${stack.join('\n  ')}`;
                     expect.soft(true, message).toEqual(false);