aboutsummaryrefslogtreecommitdiff
path: root/ext/js
diff options
context:
space:
mode:
authortoasted-nutbread <toasted-nutbread@users.noreply.github.com>2021-09-09 23:25:24 -0400
committerGitHub <noreply@github.com>2021-09-09 23:25:24 -0400
commit7883d3949619fa7f1905c0d433fde82d7b84557c (patch)
tree15a552f9346404adfd7cedb3697ab23f34b75341 /ext/js
parent069ca1a97b52a54c93d9cd15acde1a14dda595b1 (diff)
Normalize terms and readings during import (#1941)
Diffstat (limited to 'ext/js')
-rw-r--r--ext/js/language/dictionary-importer.js16
1 files changed, 14 insertions, 2 deletions
diff --git a/ext/js/language/dictionary-importer.js b/ext/js/language/dictionary-importer.js
index 4144de66..0a10900d 100644
--- a/ext/js/language/dictionary-importer.js
+++ b/ext/js/language/dictionary-importer.js
@@ -476,12 +476,16 @@ class DictionaryImporter {
}
_convertTermBankEntryV1(entry, dictionary) {
- const [expression, reading, definitionTags, rules, score, ...glossary] = entry;
+ let [expression, reading, definitionTags, rules, score, ...glossary] = entry;
+ expression = this._normalizeTermOrReading(expression);
+ reading = this._normalizeTermOrReading(reading.length > 0 ? reading : expression);
return {expression, reading, definitionTags, rules, score, glossary, dictionary};
}
_convertTermBankEntryV3(entry, dictionary) {
- const [expression, reading, definitionTags, rules, score, glossary, sequence, termTags] = entry;
+ let [expression, reading, definitionTags, rules, score, glossary, sequence, termTags] = entry;
+ expression = this._normalizeTermOrReading(expression);
+ reading = this._normalizeTermOrReading(reading.length > 0 ? reading : expression);
return {expression, reading, definitionTags, rules, score, glossary, sequence, termTags, dictionary};
}
@@ -580,4 +584,12 @@ class DictionaryImporter {
}
return counts;
}
+
+ _normalizeTermOrReading(text) {
+ try {
+ return text.normalize('NFC');
+ } catch (e) {
+ return text;
+ }
+ }
}