diff options
| author | StefanVukovic99 <stefanvukovic44@gmail.com> | 2024-01-20 02:25:23 +0100 | 
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-01-20 01:25:23 +0000 | 
| commit | 2b87c919bcd879c7f356308bc522b95f33e35f3b (patch) | |
| tree | 1301aa8bf3d9b91ad96bbe9372a66dceba190346 /ext | |
| parent | 48f1d012ad5045319d4e492dfbefa39da92817b2 (diff) | |
Dictionary deinflections (#503)
* wip
* wip
* fix v3
* wip
* fix tests
* fix maxitems
* hide deinflection definitions
* fix anki template
* undo unnecessary change
* delete console.log
* refactor
* add set false to handlebars
* lint
* fix tests
* fix comments
* fix
* use Map in areArraysEqualIgnoreOrder
* move inflection source icons to css
* lint
* improve naming
* fix tests
* add test
* typescript
* use for of
* wip
* comments
* anki template upgrade
* update descriptions
Diffstat (limited to 'ext')
| -rw-r--r-- | ext/css/display.css | 30 | ||||
| -rw-r--r-- | ext/data/schemas/dictionary-term-bank-v3-schema.json | 20 | ||||
| -rw-r--r-- | ext/data/schemas/options-schema.json | 7 | ||||
| -rw-r--r-- | ext/data/templates/anki-field-templates-upgrade-v24.handlebars | 52 | ||||
| -rw-r--r-- | ext/data/templates/default-anki-field-templates.handlebars | 22 | ||||
| -rw-r--r-- | ext/display-templates.html | 3 | ||||
| -rw-r--r-- | ext/js/background/backend.js | 8 | ||||
| -rw-r--r-- | ext/js/data/options-util.js | 17 | ||||
| -rw-r--r-- | ext/js/data/sandbox/anki-note-data-creator.js | 4 | ||||
| -rw-r--r-- | ext/js/dictionary/dictionary-importer.js | 2 | ||||
| -rw-r--r-- | ext/js/display/display-generator.js | 46 | ||||
| -rw-r--r-- | ext/js/language/deinflector.js | 2 | ||||
| -rw-r--r-- | ext/js/language/translator.js | 235 | ||||
| -rw-r--r-- | ext/js/pages/settings/dictionary-controller.js | 10 | ||||
| -rw-r--r-- | ext/js/templates/sandbox/anki-template-renderer.js | 2 | ||||
| -rw-r--r-- | ext/settings.html | 18 | 
16 files changed, 411 insertions, 67 deletions
| diff --git a/ext/css/display.css b/ext/css/display.css index 49aeaaa5..e0b7ab6d 100644 --- a/ext/css/display.css +++ b/ext/css/display.css @@ -809,17 +809,39 @@ button.action-button:active {  /* Inflections */ -.inflection-list { -    display: inline-block; +.inflection-rule-chains { +    padding-inline-start: 0; +    list-style-type: none; +} +.inflection-rule-chain {      color: var(--reason-text-color);  } -.inflection-list:empty { +.inflection-rule-chain:empty {      display: none;  } -.inflection-list>.inflection+.inflection-separator+.inflection::before { +.inflection-rule-chain>.inflection+.inflection-separator+.inflection::before {      content: var(--inflection-separator);      padding: 0 0.25em;  } +.inflection-source-icon { +    display: inline-block; +    white-space: nowrap; +    text-align: center; +    width: 1.4em; +    margin-right: 0.2em; +} +.inflection-source-icon[data-inflection-source='dictionary']::after { +    content: '📖'; +} +.inflection-source-icon[data-inflection-source='algorithm']::after { +    content: '🧩'; +} +.inflection-source-icon[data-inflection-source='both'] { +    width: 2.8em; +} +.inflection-source-icon[data-inflection-source='both']::after { +    content: '🧩📖'; +}  /* Headwords */ diff --git a/ext/data/schemas/dictionary-term-bank-v3-schema.json b/ext/data/schemas/dictionary-term-bank-v3-schema.json index 8243f2a7..066229c3 100644 --- a/ext/data/schemas/dictionary-term-bank-v3-schema.json +++ b/ext/data/schemas/dictionary-term-bank-v3-schema.json @@ -400,7 +400,7 @@              },              {                  "type": "string", -                "description": "String of space-separated rule identifiers for the definition which is used to validate delinflection. Valid rule identifiers are: v1: ichidan verb; v5: godan verb; vs: suru verb; vk: kuru verb; adj-i: i-adjective. An empty string corresponds to words which aren't inflected, such as nouns." +                "description": "String of space-separated rule identifiers for the definition which is used to validate deinflection. An empty string should be used for words which aren't inflected."              },              {                  "type": "number", @@ -535,6 +535,24 @@                                      }                                  }                              ] +                        }, +                        { +                            "type": "array", +                            "description": "Deinflection of the term to an uninflected term.", +                            "items": [ +                                { +                                    "type": "string", +                                    "description": "The uninflected term." +                                }, +                                { +                                    "type": "array", +                                    "description": "A chain of inflection rules that produced the inflected term", +                                    "items": { +                                        "type": "string", +                                        "description": "A single inflection rule." +                                    } +                                } +                            ]                          }                      ]                  } diff --git a/ext/data/schemas/options-schema.json b/ext/data/schemas/options-schema.json index 8cf00400..24f3a6b0 100644 --- a/ext/data/schemas/options-schema.json +++ b/ext/data/schemas/options-schema.json @@ -823,7 +823,8 @@                                          "enabled",                                          "allowSecondarySearches",                                          "definitionsCollapsible", -                                        "partsOfSpeechFilter" +                                        "partsOfSpeechFilter", +                                        "useDeinflections"                                      ],                                      "properties": {                                          "name": { @@ -850,6 +851,10 @@                                          "partsOfSpeechFilter": {                                              "type": "boolean",                                              "default": true +                                        }, +                                        "useDeinflections": { +                                            "type": "boolean", +                                            "default": true                                          }                                      }                                  } diff --git a/ext/data/templates/anki-field-templates-upgrade-v24.handlebars b/ext/data/templates/anki-field-templates-upgrade-v24.handlebars new file mode 100644 index 00000000..2288737c --- /dev/null +++ b/ext/data/templates/anki-field-templates-upgrade-v24.handlebars @@ -0,0 +1,52 @@ +{{#*inline "phonetic-transcriptions"}} +    {{~#if (op ">" definition.phoneticTranscriptions.length 0)~}} +        <ul> +            {{~#each definition.phoneticTranscriptions~}} +                {{~#each phoneticTranscriptions~}} +                    <li> +                        {{~set "any" false~}}    +                        {{~#each tags~}} +                            {{~#if (get "any")}}, {{else}}<i>({{/if~}} +                            {{name}} +                            {{~set "any" true~}} +                        {{~/each~}} +                        {{~#if (get "any")}})</i> {{/if~}} +                        {{ipa~}} +                    </li> +                {{~/each~}} +            {{~/each~}} +        </ul> +    {{~/if~}} +{{/inline}} + +{{<<<<<<<}} +{{#*inline "conjugation"}} +    {{~#if (op ">" definition.inflectionRuleChainCandidates.length 0)~}} +        {{~set "multiple" false~}} +        {{~#if (op ">" definition.inflectionRuleChainCandidates.length 1)~}} +            {{~set "multiple" true~}} +        {{~/if~}} +        {{~#if (get "multiple")~}}<ul>{{/if~}} +            {{~#each definition.inflectionRuleChainCandidates~}} +                {{~#if (op ">" inflectionRules.length 0)~}} +                    {{~#if (get "multiple")~}}<li>{{/if~}} +                    {{~#each inflectionRules~}} +                        {{~#if (op ">" @index 0)}} « {{/if~}} +                        {{.}} +                    {{~/each~}} +                    {{~#if (get "multiple")~}}</li>{{/if~}} +                {{~/if~}} +            {{~/each~}} +        {{~#if (get "multiple")~}}</ul>{{/if~}} +    {{~/if~}} +{{/inline}} +{{=======}} +{{#*inline "conjugation"}} +    {{~#if definition.reasons~}} +        {{~#each definition.reasons~}} +            {{~#if (op ">" @index 0)}} « {{/if~}} +            {{.}} +        {{~/each~}} +    {{~/if~}} +{{/inline}} +{{>>>>>>>>}} diff --git a/ext/data/templates/default-anki-field-templates.handlebars b/ext/data/templates/default-anki-field-templates.handlebars index f23b9d0b..818677ce 100644 --- a/ext/data/templates/default-anki-field-templates.handlebars +++ b/ext/data/templates/default-anki-field-templates.handlebars @@ -261,11 +261,23 @@  {{/inline}}  {{#*inline "conjugation"}} -    {{~#if definition.reasons~}} -        {{~#each definition.reasons~}} -            {{~#if (op ">" @index 0)}} « {{/if~}} -            {{.}} -        {{~/each~}} +    {{~#if (op ">" definition.inflectionRuleChainCandidates.length 0)~}} +        {{~set "multiple" false~}} +        {{~#if (op ">" definition.inflectionRuleChainCandidates.length 1)~}} +            {{~set "multiple" true~}} +        {{~/if~}} +        {{~#if (get "multiple")~}}<ul>{{/if~}} +            {{~#each definition.inflectionRuleChainCandidates~}} +                {{~#if (op ">" inflectionRules.length 0)~}} +                    {{~#if (get "multiple")~}}<li>{{/if~}} +                    {{~#each inflectionRules~}} +                        {{~#if (op ">" @index 0)}} « {{/if~}} +                        {{.}} +                    {{~/each~}} +                    {{~#if (get "multiple")~}}</li>{{/if~}} +                {{~/if~}} +            {{~/each~}} +        {{~#if (get "multiple")~}}</ul>{{/if~}}      {{~/if~}}  {{/inline}} diff --git a/ext/display-templates.html b/ext/display-templates.html index ed0037bb..a50cea3b 100644 --- a/ext/display-templates.html +++ b/ext/display-templates.html @@ -32,7 +32,7 @@          <div class="headword-list"></div>          <div class="headword-list-details">              <div class="headword-list-tag-list tag-list"></div> -            <div class="inflection-list"></div> +            <ul class="inflection-rule-chains"></ul>          </div>      </div>      <div class="entry-body"> @@ -77,6 +77,7 @@  <template id="definition-disambiguation-template"><span class="definition-disambiguation"></span></template>  <template id="gloss-item-template"><li class="gloss-item click-scannable"><span class="gloss-separator"> </span><span class="gloss-content"></span></li></template>  <template id="gloss-item-image-description-template"> <span class="gloss-image-description"></span></template> +<template id="inflection-rule-chain-template"><li class="inflection-rule-chain"></li></template>  <template id="inflection-template"><span class="inflection"></span><span class="inflection-separator"> </span></template>  <!-- Frequency templates --> diff --git a/ext/js/background/backend.js b/ext/js/background/backend.js index a5a42272..bc4f222f 100644 --- a/ext/js/background/backend.js +++ b/ext/js/background/backend.js @@ -2428,7 +2428,8 @@ export class Backend {                  index: enabledDictionaryMap.size,                  priority: 0,                  allowSecondarySearches: false, -                partsOfSpeechFilter: true +                partsOfSpeechFilter: true, +                useDeinflections: true              });              excludeDictionaryDefinitions = new Set();              excludeDictionaryDefinitions.add(mainDictionary); @@ -2474,12 +2475,13 @@ export class Backend {          const enabledDictionaryMap = new Map();          for (const dictionary of options.dictionaries) {              if (!dictionary.enabled) { continue; } -            const {name, priority, allowSecondarySearches, partsOfSpeechFilter} = dictionary; +            const {name, priority, allowSecondarySearches, partsOfSpeechFilter, useDeinflections} = dictionary;              enabledDictionaryMap.set(name, {                  index: enabledDictionaryMap.size,                  priority,                  allowSecondarySearches, -                partsOfSpeechFilter +                partsOfSpeechFilter, +                useDeinflections              });          }          return enabledDictionaryMap; diff --git a/ext/js/data/options-util.js b/ext/js/data/options-util.js index c93e261d..0aabed6f 100644 --- a/ext/js/data/options-util.js +++ b/ext/js/data/options-util.js @@ -556,7 +556,8 @@ export class OptionsUtil {              this._updateVersion20,              this._updateVersion21,              this._updateVersion22, -            this._updateVersion23 +            this._updateVersion23, +            this._updateVersion24          ];          if (typeof targetVersion === 'number' && targetVersion < result.length) {              result.splice(targetVersion); @@ -1156,6 +1157,20 @@ export class OptionsUtil {      }      /** +     * - Added dictionaries[].useDeinflections. +     * @type {import('options-util').UpdateFunction} +     */ +    async _updateVersion24(options) { +        await this._applyAnkiFieldTemplatesPatch(options, '/data/templates/anki-field-templates-upgrade-v24.handlebars'); + +        for (const {options: profileOptions} of options.profiles) { +            for (const dictionary of profileOptions.dictionaries) { +                dictionary.useDeinflections = true; +            } +        } +    } + +    /**       * @param {string} url       * @returns {Promise<chrome.tabs.Tab>}       */ diff --git a/ext/js/data/sandbox/anki-note-data-creator.js b/ext/js/data/sandbox/anki-note-data-creator.js index c0a11869..77d6e357 100644 --- a/ext/js/data/sandbox/anki-note-data-creator.js +++ b/ext/js/data/sandbox/anki-note-data-creator.js @@ -376,7 +376,7 @@ export class AnkiNoteDataCreator {              case 'merge': type = 'termMerged'; break;          } -        const {inflections, score, dictionaryIndex, dictionaryPriority, sourceTermExactMatchCount, definitions} = dictionaryEntry; +        const {inflectionRuleChainCandidates, score, dictionaryIndex, dictionaryPriority, sourceTermExactMatchCount, definitions} = dictionaryEntry;          let {url} = context;          if (typeof url !== 'string') { url = ''; } @@ -401,7 +401,7 @@ export class AnkiNoteDataCreator {              source: (primarySource !== null ? primarySource.transformedText : null),              rawSource: (primarySource !== null ? primarySource.originalText : null),              sourceTerm: (type !== 'termMerged' ? (primarySource !== null ? primarySource.deinflectedText : null) : void 0), -            reasons: inflections, +            inflectionRuleChainCandidates,              score,              isPrimary: (type === 'term' ? dictionaryEntry.isPrimary : void 0),              get sequence() { return self.getCachedValue(sequence); }, diff --git a/ext/js/dictionary/dictionary-importer.js b/ext/js/dictionary/dictionary-importer.js index bfd7a8b2..8df7860e 100644 --- a/ext/js/dictionary/dictionary-importer.js +++ b/ext/js/dictionary/dictionary-importer.js @@ -160,7 +160,7 @@ export class DictionaryImporter {              const glossaryList = entry.glossary;              for (let j = 0, jj = glossaryList.length; j < jj; ++j) {                  const glossary = glossaryList[j]; -                if (typeof glossary !== 'object' || glossary === null) { continue; } +                if (typeof glossary !== 'object' || glossary === null || Array.isArray(glossary)) { continue; }                  glossaryList[j] = this._formatDictionaryTermGlossaryObject(glossary, entry, requirements);              }              if ((i % formatProgressInterval) === 0) { diff --git a/ext/js/display/display-generator.js b/ext/js/display/display-generator.js index 3a2a5621..521cbb41 100644 --- a/ext/js/display/display-generator.js +++ b/ext/js/display/display-generator.js @@ -67,13 +67,13 @@ export class DisplayGenerator {          const node = this._instantiate('term-entry');          const headwordsContainer = this._querySelector(node, '.headword-list'); -        const inflectionsContainer = this._querySelector(node, '.inflection-list'); +        const inflectionRuleChainsContainer = this._querySelector(node, '.inflection-rule-chains');          const groupedPronunciationsContainer = this._querySelector(node, '.pronunciation-group-list');          const frequencyGroupListContainer = this._querySelector(node, '.frequency-group-list');          const definitionsContainer = this._querySelector(node, '.definition-list');          const headwordTagsContainer = this._querySelector(node, '.headword-list-tag-list'); -        const {headwords, type, inflections, definitions, frequencies, pronunciations} = dictionaryEntry; +        const {headwords, type, inflectionRuleChainCandidates, definitions, frequencies, pronunciations} = dictionaryEntry;          const groupedPronunciations = DictionaryDataUtil.getGroupedPronunciations(dictionaryEntry);          const pronunciationCount = groupedPronunciations.reduce((i, v) => i + v.pronunciations.length, 0);          const groupedFrequencies = DictionaryDataUtil.groupTermFrequencies(dictionaryEntry); @@ -112,7 +112,7 @@ export class DisplayGenerator {          }          headwordsContainer.dataset.count = `${headwords.length}`; -        this._appendMultiple(inflectionsContainer, this._createTermInflection.bind(this), inflections); +        this._appendMultiple(inflectionRuleChainsContainer, this._createInflectionRuleChain.bind(this), inflectionRuleChainCandidates);          this._appendMultiple(frequencyGroupListContainer, this._createFrequencyGroup.bind(this), groupedFrequencies, false);          this._appendMultiple(groupedPronunciationsContainer, this._createGroupedPronunciation.bind(this), groupedPronunciations);          this._appendMultiple(headwordTagsContainer, this._createTermTag.bind(this), termTags, headwords.length); @@ -357,6 +357,44 @@ export class DisplayGenerator {      }      /** +     * @param {import('dictionary').InflectionRuleChainCandidate} inflectionRuleChain +     * @returns {?HTMLElement} +     */ +    _createInflectionRuleChain(inflectionRuleChain) { +        const {source, inflectionRules} = inflectionRuleChain; +        if (!Array.isArray(inflectionRules) || inflectionRules.length === 0) { return null; } +        const fragment = this._instantiate('inflection-rule-chain'); + +        const sourceIcon = this._getInflectionSourceIcon(source); + +        fragment.appendChild(sourceIcon); + +        this._appendMultiple(fragment, this._createTermInflection.bind(this), inflectionRules); +        return fragment; +    } + +    /** +     * @param {import('dictionary').InflectionSource} source +     * @returns {HTMLElement} +     */ +    _getInflectionSourceIcon(source) { +        const icon = document.createElement('span'); +        icon.classList.add('inflection-source-icon'); +        icon.dataset.inflectionSource = source; +        switch (source) { +            case 'dictionary': +                icon.title = 'Dictionary Deinflection'; +                return icon; +            case 'algorithm': +                icon.title = 'Algorithm Deinflection'; +                return icon; +            case 'both': +                icon.title = 'Dictionary and Algorithm Deinflection'; +                return icon; +        } +    } + +    /**       * @param {string} inflection       * @returns {DocumentFragment}       */ @@ -396,7 +434,7 @@ export class DisplayGenerator {      }      /** -     * @param {import('dictionary-data').TermGlossary} entry +     * @param {import('dictionary-data').TermGlossaryContent} entry       * @param {string} dictionary       * @returns {?HTMLElement}       */ diff --git a/ext/js/language/deinflector.js b/ext/js/language/deinflector.js index d2d92e53..e2b66cb4 100644 --- a/ext/js/language/deinflector.js +++ b/ext/js/language/deinflector.js @@ -80,7 +80,7 @@ export class Deinflector {      /**       * @param {string} term       * @param {import('translation-internal').DeinflectionRuleFlags} rules -     * @param {string[]} reasons +     * @param {import('dictionary').InflectionRuleChain} reasons       * @returns {import('translation-internal').Deinflection}       */      _createDeinflection(term, rules, reasons) { diff --git a/ext/js/language/translator.js b/ext/js/language/translator.js index 5441294b..89a3e5ec 100644 --- a/ext/js/language/translator.js +++ b/ext/js/language/translator.js @@ -70,7 +70,7 @@ export class Translator {      async findTerms(mode, text, options) {          const {enabledDictionaryMap, excludeDictionaryDefinitions, sortFrequencyDictionary, sortFrequencyDictionaryOrder} = options;          const tagAggregator = new TranslatorTagAggregator(); -        let {dictionaryEntries, originalTextLength} = await this._findTermsInternalWrapper(text, enabledDictionaryMap, options, tagAggregator); +        let {dictionaryEntries, originalTextLength} = await this._findTermsInternal(text, enabledDictionaryMap, options, tagAggregator);          switch (mode) {              case 'group': @@ -208,7 +208,7 @@ export class Translator {       * @param {TranslatorTagAggregator} tagAggregator       * @returns {Promise<import('translator').FindTermsResult>}       */ -    async _findTermsInternalWrapper(text, enabledDictionaryMap, options, tagAggregator) { +    async _findTermsInternal(text, enabledDictionaryMap, options, tagAggregator) {          if (options.removeNonJapaneseCharacters) {              text = this._getJapaneseOnlyText(text);          } @@ -216,18 +216,30 @@ export class Translator {              return {dictionaryEntries: [], originalTextLength: 0};          } -        const deinflections = await this._findTermsInternal(text, enabledDictionaryMap, options); +        const deinflections = await this._getDeinflections(text, enabledDictionaryMap, options);          let originalTextLength = 0; +        /** @type {import('dictionary').TermDictionaryEntry[]} */          const dictionaryEntries = [];          const ids = new Set(); -        for (const {databaseEntries, originalText, transformedText, deinflectedText, reasons} of deinflections) { +        for (const {databaseEntries, originalText, transformedText, deinflectedText, inflectionRuleChainCandidates} of deinflections) {              if (databaseEntries.length === 0) { continue; }              originalTextLength = Math.max(originalTextLength, originalText.length);              for (const databaseEntry of databaseEntries) {                  const {id} = databaseEntry; -                if (ids.has(id)) { continue; } -                const dictionaryEntry = this._createTermDictionaryEntryFromDatabaseEntry(databaseEntry, originalText, transformedText, deinflectedText, reasons, true, enabledDictionaryMap, tagAggregator); +                if (ids.has(id)) { +                    const existingEntry = dictionaryEntries.find((entry) => { +                        return entry.definitions.some((definition) => definition.id === id); +                    }); + +                    if (existingEntry && transformedText.length >= existingEntry.headwords[0].sources[0].transformedText.length) { +                        this._mergeInflectionRuleChains(existingEntry, inflectionRuleChainCandidates); +                    } + +                    continue; +                } + +                const dictionaryEntry = this._createTermDictionaryEntryFromDatabaseEntry(databaseEntry, originalText, transformedText, deinflectedText, inflectionRuleChainCandidates, true, enabledDictionaryMap, tagAggregator);                  dictionaryEntries.push(dictionaryEntry);                  ids.add(id);              } @@ -237,40 +249,163 @@ export class Translator {      }      /** +     * @param {import('dictionary').TermDictionaryEntry} existingEntry +     * @param {import('dictionary').InflectionRuleChainCandidate[]} inflectionRuleChainCandidates +     */ +    _mergeInflectionRuleChains(existingEntry, inflectionRuleChainCandidates) { +        const existingChains = existingEntry.inflectionRuleChainCandidates; + +        for (const {source, inflectionRules} of inflectionRuleChainCandidates) { +            const duplicate = existingChains.find((existingChain) => this._areArraysEqualIgnoreOrder(existingChain.inflectionRules, inflectionRules)); +            if (!duplicate) { +                existingEntry.inflectionRuleChainCandidates.push({source, inflectionRules}); +            } else if (duplicate.source !== source) { +                duplicate.source = 'both'; +            } +        } +    } + +    /** +     * @param {string[]} array1 +     * @param {string[]} array2 +     * @returns {boolean} +     */ +    _areArraysEqualIgnoreOrder(array1, array2) { +        if (array1.length !== array2.length) { +            return false; +        } + +        const frequencyCounter = new Map(); + +        for (const element of array1) { +            frequencyCounter.set(element, (frequencyCounter.get(element) || 0) + 1); +        } + +        for (const element of array2) { +            const frequency = frequencyCounter.get(element); +            if (!frequency) { +                return false; +            } +            frequencyCounter.set(element, frequency - 1); +        } + +        return true; +    } + + +    /**       * @param {string} text       * @param {Map<string, import('translation').FindTermDictionary>} enabledDictionaryMap       * @param {import('translation').FindTermsOptions} options       * @returns {Promise<import('translation-internal').DatabaseDeinflection[]>}       */ -    async _findTermsInternal(text, enabledDictionaryMap, options) { -        const deinflections = ( +    async _getDeinflections(text, enabledDictionaryMap, options) { +        let deinflections = (              options.deinflect ? -            this._getAllDeinflections(text, options) : +            this._getAlgorithmDeinflections(text, options) :              [this._createDeinflection(text, text, text, 0, [])]          );          if (deinflections.length === 0) { return []; } -        const uniqueDeinflectionTerms = []; -        const uniqueDeinflectionArrays = []; -        const uniqueDeinflectionsMap = new Map(); +        const {matchType} = options; + +        await this._addEntriesToDeinflections(deinflections, enabledDictionaryMap, matchType); + +        const dictionaryDeinflections = await this._getDictionaryDeinflections(deinflections, enabledDictionaryMap, matchType); +        deinflections.push(...dictionaryDeinflections); +          for (const deinflection of deinflections) { -            const term = deinflection.deinflectedText; -            let deinflectionArray = uniqueDeinflectionsMap.get(term); +            for (const entry of deinflection.databaseEntries) { +                entry.definitions = entry.definitions.filter((definition) => !Array.isArray(definition)); +            } +            deinflection.databaseEntries = deinflection.databaseEntries.filter((entry) => entry.definitions.length); +        } +        deinflections = deinflections.filter((deinflection) => deinflection.databaseEntries.length); + +        return deinflections; +    } + +    /** +     * @param {import('translation-internal').DatabaseDeinflection[]} deinflections +     * @param {Map<string, import('translation').FindTermDictionary>} enabledDictionaryMap +     * @param {import('dictionary').TermSourceMatchType} matchType +     * @returns {Promise<import('translation-internal').DatabaseDeinflection[]>} +     */ +    async _getDictionaryDeinflections(deinflections, enabledDictionaryMap, matchType) { +        /** @type {import('translation-internal').DatabaseDeinflection[]} */ +        const dictionaryDeinflections = []; +        for (const deinflection of deinflections) { +            const {originalText, transformedText, inflectionRuleChainCandidates: algorithmChains, databaseEntries} = deinflection; +            for (const entry of databaseEntries) { +                const {dictionary, definitions} = entry; +                const entryDictionary = enabledDictionaryMap.get(dictionary); +                const useDeinflections = entryDictionary?.useDeinflections ?? true; +                if (!useDeinflections) { continue; } +                for (const definition of definitions) { +                    if (Array.isArray(definition)) { +                        const [formOf, inflectionRules] = definition; +                        if (!formOf) { continue; } + +                        const inflectionRuleChainCandidates = algorithmChains.map(({inflectionRules: algInflections}) => { +                            return { +                                source: /** @type {import('dictionary').InflectionSource} */ (algInflections.length === 0 ? 'dictionary' : 'both'), +                                inflectionRules: [...algInflections, ...inflectionRules] +                            }; +                        }); + +                        const dictionaryDeinflection = this._createDeinflection(originalText, transformedText, formOf, 0, inflectionRuleChainCandidates); +                        dictionaryDeinflections.push(dictionaryDeinflection); +                    } +                } +            } +        } + +        await this._addEntriesToDeinflections(dictionaryDeinflections, enabledDictionaryMap, matchType); + +        return dictionaryDeinflections; +    } + +    /** +     * @param {import('translation-internal').DatabaseDeinflection[]} deinflections +     * @param {Map<string, import('translation').FindTermDictionary>} enabledDictionaryMap +     * @param {import('dictionary').TermSourceMatchType} matchType +     */ +    async _addEntriesToDeinflections(deinflections, enabledDictionaryMap, matchType) { +        const uniqueDeinflectionsMap = this._groupDeinflectionsByTerm(deinflections); +        const uniqueDeinflectionArrays = [...uniqueDeinflectionsMap.values()]; +        const uniqueDeinflectionTerms = [...uniqueDeinflectionsMap.keys()]; + +        const databaseEntries = await this._database.findTermsBulk(uniqueDeinflectionTerms, enabledDictionaryMap, matchType); +        this._matchEntriesToDeinflections(databaseEntries, uniqueDeinflectionArrays, enabledDictionaryMap); +    } + +    /** +     * @param {import('translation-internal').DatabaseDeinflection[]} deinflections +     * @returns {Map<string, import('translation-internal').DatabaseDeinflection[]>} +     */ +    _groupDeinflectionsByTerm(deinflections) { +        const result = new Map(); +        for (const deinflection of deinflections) { +            const {deinflectedText} = deinflection; +            let deinflectionArray = result.get(deinflectedText);              if (typeof deinflectionArray === 'undefined') {                  deinflectionArray = []; -                uniqueDeinflectionTerms.push(term); -                uniqueDeinflectionArrays.push(deinflectionArray); -                uniqueDeinflectionsMap.set(term, deinflectionArray); +                result.set(deinflectedText, deinflectionArray);              }              deinflectionArray.push(deinflection);          } +        return result; +    } -        const {matchType} = options; -        const databaseEntries = await this._database.findTermsBulk(uniqueDeinflectionTerms, enabledDictionaryMap, matchType); - +    /** +     * @param {import('dictionary-database').TermEntry[]} databaseEntries +     * @param {import('translation-internal').DatabaseDeinflection[][]} uniqueDeinflectionArrays +     * @param {Map<string, import('translation').FindTermDictionary>} enabledDictionaryMap +     */ +    _matchEntriesToDeinflections(databaseEntries, uniqueDeinflectionArrays, enabledDictionaryMap) {          for (const databaseEntry of databaseEntries) {              const entryDictionary = /** @type {import('translation').FindTermDictionary} */ (enabledDictionaryMap.get(databaseEntry.dictionary)); -            const partsOfSpeechFilter = entryDictionary.partsOfSpeechFilter; +            const {partsOfSpeechFilter} = entryDictionary;              const definitionRules = Deinflector.rulesToRuleFlags(databaseEntry.rules);              for (const deinflection of uniqueDeinflectionArrays[databaseEntry.index]) { @@ -280,8 +415,6 @@ export class Translator {                  }              }          } - -        return deinflections;      }      // Deinflections and text transformations @@ -291,7 +424,7 @@ export class Translator {       * @param {import('translation').FindTermsOptions} options       * @returns {import('translation-internal').DatabaseDeinflection[]}       */ -    _getAllDeinflections(text, options) { +    _getAlgorithmDeinflections(text, options) {          /** @type {import('translation-internal').TextDeinflectionOptionsArrays} */          const textOptionVariantArray = [              this._getTextReplacementsVariants(options), @@ -342,7 +475,12 @@ export class Translator {                  used.add(source);                  const rawSource = sourceMap.source.substring(0, sourceMap.getSourceLength(i));                  for (const {term, rules, reasons} of /** @type {Deinflector} */ (this._deinflector).deinflect(source)) { -                    deinflections.push(this._createDeinflection(rawSource, source, term, rules, reasons)); +                    /** @type {import('dictionary').InflectionRuleChainCandidate} */ +                    const inflectionRuleChainCandidate = { +                        source: 'algorithm', +                        inflectionRules: reasons +                    }; +                    deinflections.push(this._createDeinflection(rawSource, source, term, rules, [inflectionRuleChainCandidate]));                  }              }          } @@ -435,11 +573,11 @@ export class Translator {       * @param {string} transformedText       * @param {string} deinflectedText       * @param {import('translation-internal').DeinflectionRuleFlags} rules -     * @param {string[]} reasons +     * @param {import('dictionary').InflectionRuleChainCandidate[]} inflectionRuleChainCandidates       * @returns {import('translation-internal').DatabaseDeinflection}       */ -    _createDeinflection(originalText, transformedText, deinflectedText, rules, reasons) { -        return {originalText, transformedText, deinflectedText, rules, reasons, databaseEntries: []}; +    _createDeinflection(originalText, transformedText, deinflectedText, rules, inflectionRuleChainCandidates) { +        return {originalText, transformedText, deinflectedText, rules, inflectionRuleChainCandidates, databaseEntries: []};      }      // Term dictionary entry grouping @@ -597,8 +735,8 @@ export class Translator {      _groupDictionaryEntriesByHeadword(dictionaryEntries, tagAggregator) {          const groups = new Map();          for (const dictionaryEntry of dictionaryEntries) { -            const {inflections, headwords: [{term, reading}]} = dictionaryEntry; -            const key = this._createMapKey([term, reading, ...inflections]); +            const {inflectionRuleChainCandidates, headwords: [{term, reading}]} = dictionaryEntry; +            const key = this._createMapKey([term, reading, ...inflectionRuleChainCandidates]);              let groupDictionaryEntries = groups.get(key);              if (typeof groupDictionaryEntries === 'undefined') {                  groupDictionaryEntries = []; @@ -1370,7 +1508,7 @@ export class Translator {       * @param {number[]} sequences       * @param {boolean} isPrimary       * @param {import('dictionary').Tag[]} tags -     * @param {import('dictionary-data').TermGlossary[]} entries +     * @param {import('dictionary-data').TermGlossaryContent[]} entries       * @returns {import('dictionary').TermDefinition}       */      _createTermDefinition(index, headwordIndices, dictionary, dictionaryIndex, dictionaryPriority, id, score, sequences, isPrimary, tags, entries) { @@ -1421,7 +1559,7 @@ export class Translator {      /**       * @param {boolean} isPrimary -     * @param {string[]} inflections +     * @param {import('dictionary').InflectionRuleChainCandidate[]} inflectionRuleChainCandidates       * @param {number} score       * @param {number} dictionaryIndex       * @param {number} dictionaryPriority @@ -1431,11 +1569,11 @@ export class Translator {       * @param {import('dictionary').TermDefinition[]} definitions       * @returns {import('dictionary').TermDictionaryEntry}       */ -    _createTermDictionaryEntry(isPrimary, inflections, score, dictionaryIndex, dictionaryPriority, sourceTermExactMatchCount, maxTransformedTextLength, headwords, definitions) { +    _createTermDictionaryEntry(isPrimary, inflectionRuleChainCandidates, score, dictionaryIndex, dictionaryPriority, sourceTermExactMatchCount, maxTransformedTextLength, headwords, definitions) {          return {              type: 'term',              isPrimary, -            inflections, +            inflectionRuleChainCandidates,              score,              frequencyOrder: 0,              dictionaryIndex, @@ -1454,14 +1592,29 @@ export class Translator {       * @param {string} originalText       * @param {string} transformedText       * @param {string} deinflectedText -     * @param {string[]} reasons +     * @param {import('dictionary').InflectionRuleChainCandidate[]} inflectionRuleChainCandidates       * @param {boolean} isPrimary       * @param {Map<string, import('translation').FindTermDictionary>} enabledDictionaryMap       * @param {TranslatorTagAggregator} tagAggregator       * @returns {import('dictionary').TermDictionaryEntry}       */ -    _createTermDictionaryEntryFromDatabaseEntry(databaseEntry, originalText, transformedText, deinflectedText, reasons, isPrimary, enabledDictionaryMap, tagAggregator) { -        const {matchType, matchSource, term, reading: rawReading, definitionTags, termTags, definitions, score, dictionary, id, sequence: rawSequence, rules} = databaseEntry; +    _createTermDictionaryEntryFromDatabaseEntry(databaseEntry, originalText, transformedText, deinflectedText, inflectionRuleChainCandidates, isPrimary, enabledDictionaryMap, tagAggregator) { +        const { +            matchType, +            matchSource, +            term, +            reading: rawReading, +            definitionTags, +            termTags, +            definitions, +            score, +            dictionary, +            id, +            sequence: rawSequence, +            rules +        } = databaseEntry; +        // cast is safe because getDeinflections filters out deinflection definitions +        const contentDefinitions = /** @type {import('dictionary-data').TermGlossaryContent[]} */ (definitions);          const reading = (rawReading.length > 0 ? rawReading : term);          const {index: dictionaryIndex, priority: dictionaryPriority} = this._getDictionaryOrder(dictionary, enabledDictionaryMap);          const sourceTermExactMatchCount = (isPrimary && deinflectedText === term ? 1 : 0); @@ -1479,14 +1632,14 @@ export class Translator {          return this._createTermDictionaryEntry(              isPrimary, -            reasons, +            inflectionRuleChainCandidates,              score,              dictionaryIndex,              dictionaryPriority,              sourceTermExactMatchCount,              maxTransformedTextLength,              [this._createTermHeadword(0, term, reading, [source], headwordTagGroups, rules)], -            [this._createTermDefinition(0, [0], dictionary, dictionaryIndex, dictionaryPriority, id, score, [sequence], isPrimary, definitionTagGroups, definitions)] +            [this._createTermDefinition(0, [0], dictionary, dictionaryIndex, dictionaryPriority, id, score, [sequence], isPrimary, definitionTagGroups, contentDefinitions)]          );      } @@ -1530,7 +1683,7 @@ export class Translator {              if (dictionaryEntry.isPrimary) {                  isPrimary = true;                  maxTransformedTextLength = Math.max(maxTransformedTextLength, dictionaryEntry.maxTransformedTextLength); -                const dictionaryEntryInflections = dictionaryEntry.inflections; +                const dictionaryEntryInflections = dictionaryEntry.inflectionRuleChainCandidates;                  if (inflections === null || dictionaryEntryInflections.length < inflections.length) {                      inflections = dictionaryEntryInflections;                  } @@ -1742,7 +1895,7 @@ export class Translator {              if (i !== 0) { return i; }              // Sort by the number of inflection reasons -            i = v1.inflections.length - v2.inflections.length; +            i = v1.inflectionRuleChainCandidates.length - v2.inflectionRuleChainCandidates.length;              if (i !== 0) { return i; }              // Sort by how many terms exactly match the source (e.g. for exact kana prioritization) diff --git a/ext/js/pages/settings/dictionary-controller.js b/ext/js/pages/settings/dictionary-controller.js index 18a802be..1d3da532 100644 --- a/ext/js/pages/settings/dictionary-controller.js +++ b/ext/js/pages/settings/dictionary-controller.js @@ -185,6 +185,10 @@ class DictionaryEntry {          const partsOfSpeechFilterSetting = querySelectorNotNull(modal.node, '.dictionary-parts-of-speech-filter-setting');          /** @type {HTMLElement} */          const partsOfSpeechFilterToggle = querySelectorNotNull(partsOfSpeechFilterSetting, '.dictionary-parts-of-speech-filter-toggle'); +        /** @type {HTMLElement} */ +        const useDeinflectionsSetting = querySelectorNotNull(modal.node, '.dictionary-use-deinflections-setting'); +        /** @type {HTMLElement} */ +        const useDeinflectionsToggle = querySelectorNotNull(useDeinflectionsSetting, '.dictionary-use-deinflections-toggle');          titleElement.textContent = title;          versionElement.textContent = `rev.${revision}`; @@ -194,6 +198,9 @@ class DictionaryEntry {          partsOfSpeechFilterSetting.hidden = !counts.terms.total;          partsOfSpeechFilterToggle.dataset.setting = `dictionaries[${this._index}].partsOfSpeechFilter`; +        useDeinflectionsSetting.hidden = !counts.terms.total; +        useDeinflectionsToggle.dataset.setting = `dictionaries[${this._index}].useDeinflections`; +          this._setupDetails(detailsTableElement);          modal.setVisible(true); @@ -521,7 +528,8 @@ export class DictionaryController {              enabled,              allowSecondarySearches: false,              definitionsCollapsible: 'not-collapsible', -            partsOfSpeechFilter: true +            partsOfSpeechFilter: true, +            useDeinflections: true          };      } diff --git a/ext/js/templates/sandbox/anki-template-renderer.js b/ext/js/templates/sandbox/anki-template-renderer.js index 3311097f..e4822bee 100644 --- a/ext/js/templates/sandbox/anki-template-renderer.js +++ b/ext/js/templates/sandbox/anki-template-renderer.js @@ -675,7 +675,7 @@ export class AnkiTemplateRenderer {       * @type {import('template-renderer').HelperFunction<string>}       */      _formatGlossary(args, _context, options) { -        const [dictionary, content] = /** @type {[dictionary: string, content: import('dictionary-data').TermGlossary]} */ (args); +        const [dictionary, content] = /** @type {[dictionary: string, content: import('dictionary-data').TermGlossaryContent]} */ (args);          const data = options.data.root;          if (typeof content === 'string') { return this._stringToMultiLineHtml(this._escape(content)); }          if (!(typeof content === 'object' && content !== null)) { return ''; } diff --git a/ext/settings.html b/ext/settings.html index 2cc521d5..3a4c90ce 100644 --- a/ext/settings.html +++ b/ext/settings.html @@ -2691,6 +2691,24 @@              </div>          </div>          <hr> +        <div class="settings-item dictionary-use-deinflections-setting" hidden> +            <div class="settings-item-inner"> +                <div class="settings-item-left"> +                    <div class="settings-item-label"> +                        Use deinflections +                        <a tabindex="0" class="more-toggle more-only" data-parent-distance="4">(?)</a> +                    </div> +                </div> +                <div class="settings-item-right"> +                    <label class="toggle"><input type="checkbox" class="dictionary-use-deinflections-toggle"><span class="toggle-body"><span class="toggle-track"></span><span class="toggle-knob"></span></span></label> +                </div> +            </div> +            <div class="settings-item-children more" hidden> +                Deinflections from this dictionary will be used. +                <p><a tabindex="0" class="more-toggle" data-parent-distance="3">Hide…</a></p> +            </div> +        </div> +        <hr>          <div class="settings-item"><div class="settings-item-children">              <div class="dictionary-details-table"></div>              <div class="dictionary-counts"></div> |