String frequency support (#1989)

* Restore support for string frequency values

* Add support for {value, displayValue} frequencies

* Update test data

* Improve number parsing of string frequencies

* Improve reading detection

* Expose a displayValue property for frequency information

* Update docs

* Expose displayValue to Anki note data

* Fix translator

* Update display generation

* Update test data

* Update counts

1 files changed, 45 insertions, 16 deletions

diff --git a/ext/js/language/translator.js b/ext/js/language/translator.js
index 056ff3a7..28e1cfcc 100644
--- a/ext/js/language/translator.js
+++ b/ext/js/language/translator.js
@@ -36,6 +36,7 @@ class Translator {
         this._deinflector = null;
         this._tagCache = new Map();
         this._stringComparer = new Intl.Collator('en-US'); // Invariant locale
+        this._numberRegex = /[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?/;
     }
 
     /**
@@ -853,12 +854,15 @@ class Translator {
                     case 'freq':
                         {
                             let frequency = data;
-                            const hasReading = (data !== null && typeof data === 'object');
+                            const hasReading = (data !== null && typeof data === 'object' && typeof data.reading === 'string');
                             if (hasReading) {
                                 if (data.reading !== reading) { continue; }
                                 frequency = data.frequency;
                             }
                             for (const {frequencies, headwordIndex} of targets) {
+                                let displayValue;
+                                let displayValueParsed;
+                                ({frequency, displayValue, displayValueParsed} = this._getFrequencyInfo(frequency));
                                 frequencies.push(this._createTermFrequency(
                                     frequencies.length,
                                     headwordIndex,
@@ -866,7 +870,9 @@ class Translator {
                                     dictionaryIndex,
                                     dictionaryPriority,
                                     hasReading,
-                                    this._convertFrequency(frequency)
+                                    frequency,
+                                    displayValue,
+                                    displayValueParsed
                                 ));
                             }
                         }
@@ -914,13 +920,16 @@ class Translator {
                 case 'freq':
                     {
                         const {frequencies} = dictionaryEntries[index];
+                        const {frequency, displayValue, displayValueParsed} = this._getFrequencyInfo(data);
                         frequencies.push(this._createKanjiFrequency(
                             frequencies.length,
                             dictionary,
                             dictionaryIndex,
                             dictionaryPriority,
                             character,
-                            this._convertFrequency(data)
+                            frequency,
+                            displayValue,
+                            displayValueParsed
                         ));
                     }
                     break;
@@ -971,16 +980,36 @@ class Translator {
         });
     }
 
-    _convertFrequency(value) {
-        switch (typeof value) {
-            case 'number':
-                return value;
-            case 'string':
-                value = Number.parseFloat(value);
-                return Number.isFinite(value) ? value : 0;
-            default:
-                return 0;
+    _convertStringToNumber(value) {
+        const match = this._numberRegex.exec(value);
+        if (match === null) { return 0; }
+        value = Number.parseFloat(match[0]);
+        return Number.isFinite(value) ? value : 0;
+    }
+
+    _getFrequencyInfo(frequency) {
+        let displayValue = null;
+        let displayValueParsed = false;
+        if (typeof frequency === 'object' && frequency !== null) {
+            ({value: frequency, displayValue} = frequency);
+            if (typeof frequency !== 'number') { frequency = 0; }
+            if (typeof displayValue !== 'string') { displayValue = null; }
+        } else {
+            switch (typeof frequency) {
+                case 'number':
+                    // No change
+                    break;
+                case 'string':
+                    displayValue = frequency;
+                    displayValueParsed = true;
+                    frequency = this._convertStringToNumber(frequency);
+                    break;
+                default:
+                    frequency = 0;
+                    break;
+            }
         }
+        return {frequency, displayValue, displayValueParsed};
     }
 
     // Helpers
@@ -1048,8 +1077,8 @@ class Translator {
         };
     }
 
-    _createKanjiFrequency(index, dictionary, dictionaryIndex, dictionaryPriority, character, frequency) {
-        return {index, dictionary, dictionaryIndex, dictionaryPriority, character, frequency};
+    _createKanjiFrequency(index, dictionary, dictionaryIndex, dictionaryPriority, character, frequency, displayValue, displayValueParsed) {
+        return {index, dictionary, dictionaryIndex, dictionaryPriority, character, frequency, displayValue, displayValueParsed};
     }
 
     _createKanjiDictionaryEntry(character, dictionary, onyomi, kunyomi, tags, stats, definitions) {
@@ -1114,8 +1143,8 @@ class Translator {
         return {index, headwordIndex, dictionary, dictionaryIndex, dictionaryPriority, pitches};
     }
 
-    _createTermFrequency(index, headwordIndex, dictionary, dictionaryIndex, dictionaryPriority, hasReading, frequency) {
-        return {index, headwordIndex, dictionary, dictionaryIndex, dictionaryPriority, hasReading, frequency};
+    _createTermFrequency(index, headwordIndex, dictionary, dictionaryIndex, dictionaryPriority, hasReading, frequency, displayValue, displayValueParsed) {
+        return {index, headwordIndex, dictionary, dictionaryIndex, dictionaryPriority, hasReading, frequency, displayValue, displayValueParsed};
     }
 
     _createTermDictionaryEntry(isPrimary, inflections, score, dictionaryIndex, dictionaryPriority, sourceTermExactMatchCount, maxTransformedTextLength, headwords, definitions) {