summaryrefslogtreecommitdiff
path: root/test
diff options
context:
space:
mode:
authortoasted-nutbread <toasted-nutbread@users.noreply.github.com>2024-01-31 08:28:05 -0500
committerGitHub <noreply@github.com>2024-01-31 13:28:05 +0000
commit3e419aa562aab03ca20421aaf7e4d1a39194a5b4 (patch)
tree15e8bfe81fa5e3fae55e54802f14d94a7502a469 /test
parent6807b05e9bd41f013364fae0cbcce83cf1ed37b6 (diff)
Language transformer (#582)
* Set up new deinflection data file * Define types * Test * Add internal types * Set up loading for transforms * Add getPartOfSpeechFlags * Convert static methods * Add note * Add transform function * Update trace structure * Add a language tag to the language transform descriptor * Add clear function * Add function for multiple parts of speech * Clarify naming * Add getConditionFlagsFromConditionType * Add plural function * Replace usages of Deinflector * Update tests * Update config * Remove old * Rename * Rename files
Diffstat (limited to 'test')
-rw-r--r--test/data/json.json10
-rw-r--r--test/fixtures/translator-test.js6
-rw-r--r--test/language-transformer-cycles.test.js (renamed from test/deinflection-cycles.test.js)39
-rw-r--r--test/language-transformer.test.js (renamed from test/deinflector.test.js)37
4 files changed, 51 insertions, 41 deletions
diff --git a/test/data/json.json b/test/data/json.json
index 83b1b4e0..fe7df57d 100644
--- a/test/data/json.json
+++ b/test/data/json.json
@@ -30,11 +30,6 @@
"type": "ManifestConfig"
},
{
- "path": "ext/data/deinflect.json",
- "typeFile": "types/ext/deinflector.d.ts",
- "type": "ReasonsRaw"
- },
- {
"path": "ext/data/pronunciation-style.json",
"typeFile": "types/ext/css-style-applier.d.ts",
"type": "RawStyleData"
@@ -95,6 +90,11 @@
"type": "AjvSchema"
},
{
+ "path": "ext/data/language/japanese-transforms.json",
+ "typeFile": "types/ext/language-transformer.d.ts",
+ "type": "LanguageTransformDescriptor"
+ },
+ {
"path": "test/data/translator-test-inputs.json",
"typeFile": "types/test/translator.d.ts",
"type": "TranslatorTestInputs",
diff --git a/test/fixtures/translator-test.js b/test/fixtures/translator-test.js
index 6562931c..58247b70 100644
--- a/test/fixtures/translator-test.js
+++ b/test/fixtures/translator-test.js
@@ -32,7 +32,7 @@ import {DictionaryImporterMediaLoader} from '../mocks/dictionary-importer-media-
import {createDomTest} from './dom-test.js';
const extDir = join(dirname(fileURLToPath(import.meta.url)), '../../ext');
-const deinflectionReasonsPath = join(extDir, 'data/deinflect.json');
+const languageTransformDescriptorPath = join(extDir, 'data/language/japanese-transforms.json');
vi.stubGlobal('indexedDB', indexedDB);
vi.stubGlobal('IDBKeyRange', IDBKeyRange);
@@ -65,8 +65,8 @@ async function createTranslatorContext(dictionaryDirectory, dictionaryName) {
// Setup translator
const translator = new Translator({database: dictionaryDatabase});
- /** @type {import('deinflector').ReasonsRaw} */
- const deinflectionReasons = parseJson(readFileSync(deinflectionReasonsPath, {encoding: 'utf8'}));
+ /** @type {import('language-transformer').LanguageTransformDescriptor} */
+ const deinflectionReasons = parseJson(readFileSync(languageTransformDescriptorPath, {encoding: 'utf8'}));
translator.prepare(deinflectionReasons);
// Assign properties
diff --git a/test/deinflection-cycles.test.js b/test/language-transformer-cycles.test.js
index a010d7a3..76bee83e 100644
--- a/test/deinflection-cycles.test.js
+++ b/test/language-transformer-cycles.test.js
@@ -20,19 +20,19 @@ import {join, dirname as pathDirname} from 'path';
import {fileURLToPath} from 'url';
import {describe, test} from 'vitest';
import {parseJson} from '../dev/json.js';
-import {Deinflector} from '../ext/js/language/deinflector.js';
+import {LanguageTransformer} from '../ext/js/language/language-transformer.js';
class DeinflectionNode {
/**
* @param {string} text
- * @param {import('deinflector').ReasonTypeRaw[]} ruleNames
+ * @param {string[]} ruleNames
* @param {?RuleNode} ruleNode
* @param {?DeinflectionNode} previous
*/
constructor(text, ruleNames, ruleNode, previous) {
/** @type {string} */
this.text = text;
- /** @type {import('deinflector').ReasonTypeRaw[]} */
+ /** @type {string[]} */
this.ruleNames = ruleNames;
/** @type {?RuleNode} */
this.ruleNode = ruleNode;
@@ -79,12 +79,12 @@ class DeinflectionNode {
class RuleNode {
/**
* @param {string} groupName
- * @param {import('deinflector').ReasonRaw} rule
+ * @param {import('language-transformer').Rule} rule
*/
constructor(groupName, rule) {
/** @type {string} */
this.groupName = groupName;
- /** @type {import('deinflector').ReasonRaw} */
+ /** @type {import('language-transformer').Rule} */
this.rule = rule;
}
}
@@ -107,13 +107,15 @@ describe('Deinflection data', () => {
test('Check for cycles', ({expect}) => {
const dirname = pathDirname(fileURLToPath(import.meta.url));
- /** @type {import('deinflector').ReasonsRaw} */
- const deinflectionReasons = parseJson(readFileSync(join(dirname, '../ext/data/deinflect.json'), {encoding: 'utf8'}));
+ /** @type {import('language-transformer').LanguageTransformDescriptor} */
+ const descriptor = parseJson(readFileSync(join(dirname, '../ext/data/language/japanese-transforms.json'), {encoding: 'utf8'}));
+ const languageTransformer = new LanguageTransformer();
+ languageTransformer.addDescriptor(descriptor);
/** @type {RuleNode[]} */
const ruleNodes = [];
- for (const [groupName, reasonInfo] of Object.entries(deinflectionReasons)) {
- for (const rule of reasonInfo) {
+ for (const [groupName, reasonInfo] of Object.entries(descriptor.transforms)) {
+ for (const rule of reasonInfo.rules) {
ruleNodes.push(new RuleNode(groupName, rule));
}
}
@@ -121,24 +123,27 @@ describe('Deinflection data', () => {
/** @type {DeinflectionNode[]} */
const deinflectionNodes = [];
for (const ruleNode of ruleNodes) {
- deinflectionNodes.push(new DeinflectionNode(`?${ruleNode.rule.kanaIn}`, [], null, null));
+ deinflectionNodes.push(new DeinflectionNode(`?${ruleNode.rule.suffixIn}`, [], null, null));
}
for (let i = 0; i < deinflectionNodes.length; ++i) {
const deinflectionNode = deinflectionNodes[i];
const {text, ruleNames} = deinflectionNode;
for (const ruleNode of ruleNodes) {
- const {kanaIn, kanaOut, rulesIn, rulesOut} = ruleNode.rule;
+ const {suffixIn, suffixOut, conditionsIn, conditionsOut} = ruleNode.rule;
if (
- !Deinflector.rulesMatch(Deinflector.rulesToRuleFlags(ruleNames), Deinflector.rulesToRuleFlags(rulesIn)) ||
- !text.endsWith(kanaIn) ||
- (text.length - kanaIn.length + kanaOut.length) <= 0
+ !LanguageTransformer.conditionsMatch(
+ languageTransformer.getConditionFlagsFromConditionTypes(ruleNames),
+ languageTransformer.getConditionFlagsFromConditionTypes(conditionsIn)
+ ) ||
+ !text.endsWith(suffixIn) ||
+ (text.length - suffixIn.length + suffixOut.length) <= 0
) {
continue;
}
const newDeinflectionNode = new DeinflectionNode(
- text.substring(0, text.length - kanaIn.length) + kanaOut,
- rulesOut,
+ text.substring(0, text.length - suffixIn.length) + suffixOut,
+ conditionsOut,
ruleNode,
deinflectionNode
);
@@ -150,7 +155,7 @@ describe('Deinflection data', () => {
stack.push(
item.ruleNode === null ?
`${item.text} (start)` :
- `${item.text} (${item.ruleNode.groupName}, ${item.ruleNode.rule.rulesIn.join(',')}=>${item.ruleNode.rule.rulesOut.join(',')}, ${item.ruleNode.rule.kanaIn}=>${item.ruleNode.rule.kanaOut})`
+ `${item.text} (${item.ruleNode.groupName}, ${item.ruleNode.rule.conditionsIn.join(',')}=>${item.ruleNode.rule.conditionsOut.join(',')}, ${item.ruleNode.rule.suffixIn}=>${item.ruleNode.rule.suffixOut})`
);
}
const message = `Cycle detected:\n ${stack.join('\n ')}`;
diff --git a/test/deinflector.test.js b/test/language-transformer.test.js
index 71eb9349..b99c1bef 100644
--- a/test/deinflector.test.js
+++ b/test/language-transformer.test.js
@@ -21,37 +21,41 @@ import {fileURLToPath} from 'node:url';
import path from 'path';
import {describe, expect, test} from 'vitest';
import {parseJson} from '../dev/json.js';
-import {Deinflector} from '../ext/js/language/deinflector.js';
+import {LanguageTransformer} from '../ext/js/language/language-transformer.js';
const dirname = path.dirname(fileURLToPath(import.meta.url));
/**
- * @param {Deinflector} deinflector
+ * @param {LanguageTransformer} languageTransformer
* @param {string} source
* @param {string} expectedTerm
- * @param {string|null} expectedRule
+ * @param {string|null} expectedConditionName
* @param {string[]|null} expectedReasons
* @returns {{has: false, reasons: null, rules: null}|{has: true, reasons: string[], rules: number}}
*/
-function hasTermReasons(deinflector, source, expectedTerm, expectedRule, expectedReasons) {
- for (const {term, reasons, rules} of deinflector.deinflect(source)) {
- if (term !== expectedTerm) { continue; }
- if (expectedRule !== null) {
- const expectedFlags = Deinflector.rulesToRuleFlags([expectedRule]);
- if (!Deinflector.rulesMatch(rules, expectedFlags)) { continue; }
+function hasTermReasons(languageTransformer, source, expectedTerm, expectedConditionName, expectedReasons) {
+ for (const {text, conditions, trace} of languageTransformer.transform(source)) {
+ if (text !== expectedTerm) { continue; }
+ if (expectedConditionName !== null) {
+ const expectedConditions = languageTransformer.getConditionFlagsFromConditionType(expectedConditionName);
+ if (!LanguageTransformer.conditionsMatch(conditions, expectedConditions)) { continue; }
}
let okay = true;
if (expectedReasons !== null) {
- if (reasons.length !== expectedReasons.length) { continue; }
+ if (trace.length !== expectedReasons.length) { continue; }
for (let i = 0, ii = expectedReasons.length; i < ii; ++i) {
- if (expectedReasons[i] !== reasons[i]) {
+ if (expectedReasons[i] !== trace[i].transform) {
okay = false;
break;
}
}
}
if (okay) {
- return {has: true, reasons, rules};
+ return {
+ has: true,
+ reasons: trace.map((frame) => frame.transform),
+ rules: conditions
+ };
}
}
return {has: false, reasons: null, rules: null};
@@ -1010,15 +1014,16 @@ function testDeinflections() {
];
/* eslint-enable no-multi-spaces */
- /** @type {import('deinflector').ReasonsRaw} */
- const deinflectionReasons = parseJson(fs.readFileSync(path.join(dirname, '..', 'ext', 'data/deinflect.json'), {encoding: 'utf8'}));
- const deinflector = new Deinflector(deinflectionReasons);
+ /** @type {import('language-transformer').LanguageTransformDescriptor} */
+ const descriptor = parseJson(fs.readFileSync(path.join(dirname, '..', 'ext', 'data/language/japanese-transforms.json'), {encoding: 'utf8'}));
+ const languageTransformer = new LanguageTransformer();
+ languageTransformer.addDescriptor(descriptor);
describe('deinflections', () => {
// for (const {valid, tests} of data) {
describe.each(data)('$category', ({valid, tests}) => {
for (const {source, term, rule, reasons} of tests) {
- const {has} = hasTermReasons(deinflector, source, term, rule, reasons);
+ const {has} = hasTermReasons(languageTransformer, source, term, rule, reasons);
let message = `${source} ${valid ? 'has' : 'does not have'} term candidate ${JSON.stringify(term)}`;
if (typeof rule !== 'undefined') {
message += ` with rule ${JSON.stringify(rule)}`;