Language transformer (#582)

* Set up new deinflection data file

* Define types

* Test

* Add internal types

* Set up loading for transforms

* Add getPartOfSpeechFlags

* Convert static methods

* Add note

* Add transform function

* Update trace structure

* Add a language tag to the language transform descriptor

* Add clear function

* Add function for multiple parts of speech

* Clarify naming

* Add getConditionFlagsFromConditionType

* Add plural function

* Replace usages of Deinflector

* Update tests

* Update config

* Remove old

* Rename

* Rename files

4 files changed, 51 insertions, 41 deletions

diff --git a/test/data/json.json b/test/data/json.json
index 83b1b4e0..fe7df57d 100644
--- a/test/data/json.json
+++ b/test/data/json.json
@@ -30,11 +30,6 @@
             "type": "ManifestConfig"
         },
         {
-            "path": "ext/data/deinflect.json",
-            "typeFile": "types/ext/deinflector.d.ts",
-            "type": "ReasonsRaw"
-        },
-        {
             "path": "ext/data/pronunciation-style.json",
             "typeFile": "types/ext/css-style-applier.d.ts",
             "type": "RawStyleData"
@@ -95,6 +90,11 @@
             "type": "AjvSchema"
         },
         {
+            "path": "ext/data/language/japanese-transforms.json",
+            "typeFile": "types/ext/language-transformer.d.ts",
+            "type": "LanguageTransformDescriptor"
+        },
+        {
             "path": "test/data/translator-test-inputs.json",
             "typeFile": "types/test/translator.d.ts",
             "type": "TranslatorTestInputs",
diff --git a/test/fixtures/translator-test.js b/test/fixtures/translator-test.js
index 6562931c..58247b70 100644
--- a/test/fixtures/translator-test.js
+++ b/test/fixtures/translator-test.js
@@ -32,7 +32,7 @@ import {DictionaryImporterMediaLoader} from '../mocks/dictionary-importer-media-
 import {createDomTest} from './dom-test.js';
 
 const extDir = join(dirname(fileURLToPath(import.meta.url)), '../../ext');
-const deinflectionReasonsPath = join(extDir, 'data/deinflect.json');
+const languageTransformDescriptorPath = join(extDir, 'data/language/japanese-transforms.json');
 
 vi.stubGlobal('indexedDB', indexedDB);
 vi.stubGlobal('IDBKeyRange', IDBKeyRange);
@@ -65,8 +65,8 @@ async function createTranslatorContext(dictionaryDirectory, dictionaryName) {
 
     // Setup translator
     const translator = new Translator({database: dictionaryDatabase});
-    /** @type {import('deinflector').ReasonsRaw} */
-    const deinflectionReasons = parseJson(readFileSync(deinflectionReasonsPath, {encoding: 'utf8'}));
+    /** @type {import('language-transformer').LanguageTransformDescriptor} */
+    const deinflectionReasons = parseJson(readFileSync(languageTransformDescriptorPath, {encoding: 'utf8'}));
     translator.prepare(deinflectionReasons);
 
     // Assign properties
diff --git a/test/deinflection-cycles.test.js b/test/language-transformer-cycles.test.js
index a010d7a3..76bee83e 100644
--- a/test/deinflection-cycles.test.js
+++ b/test/language-transformer-cycles.test.js
@@ -20,19 +20,19 @@ import {join, dirname as pathDirname} from 'path';
 import {fileURLToPath} from 'url';
 import {describe, test} from 'vitest';
 import {parseJson} from '../dev/json.js';
-import {Deinflector} from '../ext/js/language/deinflector.js';
+import {LanguageTransformer} from '../ext/js/language/language-transformer.js';
 
 class DeinflectionNode {
     /**
      * @param {string} text
-     * @param {import('deinflector').ReasonTypeRaw[]} ruleNames
+     * @param {string[]} ruleNames
      * @param {?RuleNode} ruleNode
      * @param {?DeinflectionNode} previous
      */
     constructor(text, ruleNames, ruleNode, previous) {
         /** @type {string} */
         this.text = text;
-        /** @type {import('deinflector').ReasonTypeRaw[]} */
+        /** @type {string[]} */
         this.ruleNames = ruleNames;
         /** @type {?RuleNode} */
         this.ruleNode = ruleNode;
@@ -79,12 +79,12 @@ class DeinflectionNode {
 class RuleNode {
     /**
      * @param {string} groupName
-     * @param {import('deinflector').ReasonRaw} rule
+     * @param {import('language-transformer').Rule} rule
      */
     constructor(groupName, rule) {
         /** @type {string} */
         this.groupName = groupName;
-        /** @type {import('deinflector').ReasonRaw} */
+        /** @type {import('language-transformer').Rule} */
         this.rule = rule;
     }
 }
@@ -107,13 +107,15 @@ describe('Deinflection data', () => {
     test('Check for cycles', ({expect}) => {
         const dirname = pathDirname(fileURLToPath(import.meta.url));
 
-        /** @type {import('deinflector').ReasonsRaw} */
-        const deinflectionReasons = parseJson(readFileSync(join(dirname, '../ext/data/deinflect.json'), {encoding: 'utf8'}));
+        /** @type {import('language-transformer').LanguageTransformDescriptor} */
+        const descriptor = parseJson(readFileSync(join(dirname, '../ext/data/language/japanese-transforms.json'), {encoding: 'utf8'}));
+        const languageTransformer = new LanguageTransformer();
+        languageTransformer.addDescriptor(descriptor);
 
         /** @type {RuleNode[]} */
         const ruleNodes = [];
-        for (const [groupName, reasonInfo] of Object.entries(deinflectionReasons)) {
-            for (const rule of reasonInfo) {
+        for (const [groupName, reasonInfo] of Object.entries(descriptor.transforms)) {
+            for (const rule of reasonInfo.rules) {
                 ruleNodes.push(new RuleNode(groupName, rule));
             }
         }
@@ -121,24 +123,27 @@ describe('Deinflection data', () => {
         /** @type {DeinflectionNode[]} */
         const deinflectionNodes = [];
         for (const ruleNode of ruleNodes) {
-            deinflectionNodes.push(new DeinflectionNode(`?${ruleNode.rule.kanaIn}`, [], null, null));
+            deinflectionNodes.push(new DeinflectionNode(`?${ruleNode.rule.suffixIn}`, [], null, null));
         }
         for (let i = 0; i < deinflectionNodes.length; ++i) {
             const deinflectionNode = deinflectionNodes[i];
             const {text, ruleNames} = deinflectionNode;
             for (const ruleNode of ruleNodes) {
-                const {kanaIn, kanaOut, rulesIn, rulesOut} = ruleNode.rule;
+                const {suffixIn, suffixOut, conditionsIn, conditionsOut} = ruleNode.rule;
                 if (
-                    !Deinflector.rulesMatch(Deinflector.rulesToRuleFlags(ruleNames), Deinflector.rulesToRuleFlags(rulesIn)) ||
-                    !text.endsWith(kanaIn) ||
-                    (text.length - kanaIn.length + kanaOut.length) <= 0
+                    !LanguageTransformer.conditionsMatch(
+                        languageTransformer.getConditionFlagsFromConditionTypes(ruleNames),
+                        languageTransformer.getConditionFlagsFromConditionTypes(conditionsIn)
+                    ) ||
+                    !text.endsWith(suffixIn) ||
+                    (text.length - suffixIn.length + suffixOut.length) <= 0
                 ) {
                     continue;
                 }
 
                 const newDeinflectionNode = new DeinflectionNode(
-                    text.substring(0, text.length - kanaIn.length) + kanaOut,
-                    rulesOut,
+                    text.substring(0, text.length - suffixIn.length) + suffixOut,
+                    conditionsOut,
                     ruleNode,
                     deinflectionNode
                 );
@@ -150,7 +155,7 @@ describe('Deinflection data', () => {
                         stack.push(
                             item.ruleNode === null ?
                             `${item.text} (start)` :
-                            `${item.text} (${item.ruleNode.groupName}, ${item.ruleNode.rule.rulesIn.join(',')}=>${item.ruleNode.rule.rulesOut.join(',')}, ${item.ruleNode.rule.kanaIn}=>${item.ruleNode.rule.kanaOut})`
+                            `${item.text} (${item.ruleNode.groupName}, ${item.ruleNode.rule.conditionsIn.join(',')}=>${item.ruleNode.rule.conditionsOut.join(',')}, ${item.ruleNode.rule.suffixIn}=>${item.ruleNode.rule.suffixOut})`
                         );
                     }
                     const message = `Cycle detected:\n  ${stack.join('\n  ')}`;
diff --git a/test/deinflector.test.js b/test/language-transformer.test.js
index 71eb9349..b99c1bef 100644
--- a/test/deinflector.test.js
+++ b/test/language-transformer.test.js
@@ -21,37 +21,41 @@ import {fileURLToPath} from 'node:url';
 import path from 'path';
 import {describe, expect, test} from 'vitest';
 import {parseJson} from '../dev/json.js';
-import {Deinflector} from '../ext/js/language/deinflector.js';
+import {LanguageTransformer} from '../ext/js/language/language-transformer.js';
 
 const dirname = path.dirname(fileURLToPath(import.meta.url));
 
 /**
- * @param {Deinflector} deinflector
+ * @param {LanguageTransformer} languageTransformer
  * @param {string} source
  * @param {string} expectedTerm
- * @param {string|null} expectedRule
+ * @param {string|null} expectedConditionName
  * @param {string[]|null} expectedReasons
  * @returns {{has: false, reasons: null, rules: null}|{has: true, reasons: string[], rules: number}}
  */
-function hasTermReasons(deinflector, source, expectedTerm, expectedRule, expectedReasons) {
-    for (const {term, reasons, rules} of deinflector.deinflect(source)) {
-        if (term !== expectedTerm) { continue; }
-        if (expectedRule !== null) {
-            const expectedFlags = Deinflector.rulesToRuleFlags([expectedRule]);
-            if (!Deinflector.rulesMatch(rules, expectedFlags)) { continue; }
+function hasTermReasons(languageTransformer, source, expectedTerm, expectedConditionName, expectedReasons) {
+    for (const {text, conditions, trace} of languageTransformer.transform(source)) {
+        if (text !== expectedTerm) { continue; }
+        if (expectedConditionName !== null) {
+            const expectedConditions = languageTransformer.getConditionFlagsFromConditionType(expectedConditionName);
+            if (!LanguageTransformer.conditionsMatch(conditions, expectedConditions)) { continue; }
         }
         let okay = true;
         if (expectedReasons !== null) {
-            if (reasons.length !== expectedReasons.length) { continue; }
+            if (trace.length !== expectedReasons.length) { continue; }
             for (let i = 0, ii = expectedReasons.length; i < ii; ++i) {
-                if (expectedReasons[i] !== reasons[i]) {
+                if (expectedReasons[i] !== trace[i].transform) {
                     okay = false;
                     break;
                 }
             }
         }
         if (okay) {
-            return {has: true, reasons, rules};
+            return {
+                has: true,
+                reasons: trace.map((frame) => frame.transform),
+                rules: conditions
+            };
         }
     }
     return {has: false, reasons: null, rules: null};
@@ -1010,15 +1014,16 @@ function testDeinflections() {
     ];
     /* eslint-enable no-multi-spaces */
 
-    /** @type {import('deinflector').ReasonsRaw} */
-    const deinflectionReasons = parseJson(fs.readFileSync(path.join(dirname, '..', 'ext', 'data/deinflect.json'), {encoding: 'utf8'}));
-    const deinflector = new Deinflector(deinflectionReasons);
+    /** @type {import('language-transformer').LanguageTransformDescriptor} */
+    const descriptor = parseJson(fs.readFileSync(path.join(dirname, '..', 'ext', 'data/language/japanese-transforms.json'), {encoding: 'utf8'}));
+    const languageTransformer = new LanguageTransformer();
+    languageTransformer.addDescriptor(descriptor);
 
     describe('deinflections', () => {
         // for (const {valid, tests} of data) {
         describe.each(data)('$category', ({valid, tests}) => {
             for (const {source, term, rule, reasons} of tests) {
-                const {has} = hasTermReasons(deinflector, source, term, rule, reasons);
+                const {has} = hasTermReasons(languageTransformer, source, term, rule, reasons);
                 let message = `${source} ${valid ? 'has' : 'does not have'} term candidate ${JSON.stringify(term)}`;
                 if (typeof rule !== 'undefined') {
                     message += ` with rule ${JSON.stringify(rule)}`;