From 7883d3949619fa7f1905c0d433fde82d7b84557c Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Thu, 9 Sep 2021 23:25:24 -0400 Subject: Normalize terms and readings during import (#1941) --- ext/js/language/dictionary-importer.js | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) (limited to 'ext') diff --git a/ext/js/language/dictionary-importer.js b/ext/js/language/dictionary-importer.js index 4144de66..0a10900d 100644 --- a/ext/js/language/dictionary-importer.js +++ b/ext/js/language/dictionary-importer.js @@ -476,12 +476,16 @@ class DictionaryImporter { } _convertTermBankEntryV1(entry, dictionary) { - const [expression, reading, definitionTags, rules, score, ...glossary] = entry; + let [expression, reading, definitionTags, rules, score, ...glossary] = entry; + expression = this._normalizeTermOrReading(expression); + reading = this._normalizeTermOrReading(reading.length > 0 ? reading : expression); return {expression, reading, definitionTags, rules, score, glossary, dictionary}; } _convertTermBankEntryV3(entry, dictionary) { - const [expression, reading, definitionTags, rules, score, glossary, sequence, termTags] = entry; + let [expression, reading, definitionTags, rules, score, glossary, sequence, termTags] = entry; + expression = this._normalizeTermOrReading(expression); + reading = this._normalizeTermOrReading(reading.length > 0 ? reading : expression); return {expression, reading, definitionTags, rules, score, glossary, sequence, termTags, dictionary}; } @@ -580,4 +584,12 @@ class DictionaryImporter { } return counts; } + + _normalizeTermOrReading(text) { + try { + return text.normalize('NFC'); + } catch (e) { + return text; + } + } } -- cgit v1.2.3