Simplify diacratic removal; modify Latin & Greek preprocessors (#724)

* Simplified diacratic removal and added preprocessors to LA and GRC

* linted

* Clarified the name of removeAlphabeticDiacritics

* Add comment to removeAlphabeticDiacritics

Signed-off-by: Darius Jahandarie <djahandarie@gmail.com>

* Change to NFD

Signed-off-by: Matttttt <18152455+martholomew@users.noreply.github.com>

* Remove trailing spaces in comment

Signed-off-by: Darius Jahandarie <djahandarie@gmail.com>

* Remove latin preprocessors .eslintrc.json

Signed-off-by: Matttttt <18152455+martholomew@users.noreply.github.com>

* fix tests

---------

Signed-off-by: Darius Jahandarie <djahandarie@gmail.com>
Signed-off-by: Matttttt <18152455+martholomew@users.noreply.github.com>
Co-authored-by: martholomew <martholomew@users.noreply.github.com>
Co-authored-by: Darius Jahandarie <djahandarie@gmail.com>
Co-authored-by: Stefan Vukovic <stefanvukovic44@gmail.com>

1 files changed, 14 insertions, 0 deletions

diff --git a/ext/js/language/text-preprocessors.js b/ext/js/language/text-preprocessors.js
index 12b3d1b6..e33fccda 100755
--- a/ext/js/language/text-preprocessors.js
+++ b/ext/js/language/text-preprocessors.js
@@ -33,3 +33,17 @@ export const capitalizeFirstLetter = {
     options: basicTextPreprocessorOptions,
     process: (str, setting) => (setting ? str.charAt(0).toUpperCase() + str.slice(1) : str)
 };
+
+/**
+ * WARNING: This should NOT be used with languages that use Han characters,
+ *          as it can result in undesirable normalization:
+ *            - '\u9038'.normalize('NFD') => '\u9038' (逸)
+ *            - '\ufa67'.normalize('NFD') => '\u9038' (逸 => 逸)
+ * @type {import('language').TextPreprocessor<boolean>}
+ */
+export const removeAlphabeticDiacritics = {
+    name: 'Remove Alphabetic Diacritics',
+    description: 'ἄήé -> αηe',
+    options: basicTextPreprocessorOptions,
+    process: (str, setting) => (setting ? str.normalize('NFD').replace(/[\u0300-\u036f]/g, '') : str)
+};