diff options
| -rw-r--r-- | .eslintrc.json | 1 | ||||
| -rw-r--r-- | ext/background.html | 1 | ||||
| -rw-r--r-- | ext/js/general/regex-util.js | 93 | ||||
| -rw-r--r-- | ext/js/language/translator.js | 90 | ||||
| -rw-r--r-- | ext/sw.js | 1 | ||||
| -rw-r--r-- | test/test-translator.js | 1 |
6 files changed, 99 insertions, 88 deletions
diff --git a/.eslintrc.json b/.eslintrc.json index 4bed4a0d..e7438971 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -208,6 +208,7 @@ "ext/js/dom/simple-dom-parser.js", "ext/js/general/cache-map.js", "ext/js/general/object-property-accessor.js", + "ext/js/general/regex-util.js", "ext/js/general/text-source-map.js", "ext/js/language/deinflector.js", "ext/js/language/dictionary-data-util.js", diff --git a/ext/background.html b/ext/background.html index 4ce7ddf0..15cd0b8c 100644 --- a/ext/background.html +++ b/ext/background.html @@ -39,6 +39,7 @@ <script src="/js/dom/native-simple-dom-parser.js"></script> <script src="/js/general/cache-map.js"></script> <script src="/js/general/object-property-accessor.js"></script> +<script src="/js/general/regex-util.js"></script> <script src="/js/general/text-source-map.js"></script> <script src="/js/language/deinflector.js"></script> <script src="/js/language/dictionary-data-util.js"></script> diff --git a/ext/js/general/regex-util.js b/ext/js/general/regex-util.js new file mode 100644 index 00000000..35b1c2b8 --- /dev/null +++ b/ext/js/general/regex-util.js @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2021 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +/** + * This class provides some general utility functions for regular expressions. + */ +class RegexUtil { + /** + * Applies string.replace using a regular expression and replacement string as arguments. + * A source map of the changes is also maintained. + * @param text A string of the text to replace. + * @param sourceMap An instance of `TextSourceMap` which corresponds to `text`. + * @param pattern A regular expression to use as the replacement. + * @param replacement A replacement string that follows the format of the standard + * JavaScript regular expression replacement string. + * @return A new string with the pattern replacements applied and the source map updated. + */ + static applyTextReplacement(text, sourceMap, pattern, replacement) { + const isGlobal = pattern.global; + if (isGlobal) { pattern.lastIndex = 0; } + for (let loop = true; loop; loop = isGlobal) { + const match = pattern.exec(text); + if (match === null) { break; } + + const matchText = match[0]; + const index = match.index; + const actualReplacement = this.applyMatchReplacement(replacement, match); + const actualReplacementLength = actualReplacement.length; + const delta = actualReplacementLength - (matchText.length > 0 ? matchText.length : -1); + + text = `${text.substring(0, index)}${actualReplacement}${text.substring(index + matchText.length)}`; + pattern.lastIndex += delta; + + if (actualReplacementLength > 0) { + sourceMap.insert(index, ...(new Array(actualReplacementLength).fill(0))); + sourceMap.combine(index - 1 + actualReplacementLength, matchText.length); + } else { + sourceMap.combine(index, matchText.length); + } + } + return text; + } + + /** + * Applies the replacement string for a given regular expression match. + * @param replacement The replacement string that follows the format of the standard + * JavaScript regular expression replacement string. + * @param match A match object returned from RegExp.match. + * @return A new string with the pattern replacement applied. + */ + static applyMatchReplacement(replacement, match) { + const pattern = this._matchReplacementPattern; + pattern.lastIndex = 0; + return replacement.replace(pattern, (g0, g1, g2) => { + if (typeof g1 !== 'undefined') { + const matchIndex = Number.parseInt(g1, 10); + if (matchIndex >= 1 && matchIndex <= match.length) { + return match[matchIndex]; + } + } else if (typeof g2 !== 'undefined') { + const {groups} = match; + if (typeof groups === 'object' && groups !== null && Object.prototype.hasOwnProperty.call(groups, g2)) { + return groups[g2]; + } + } else { + switch (g0) { + case '$': return '$'; + case '&': return match[0]; + case '`': return replacement.substring(0, match.index); + case '\'': return replacement.substring(match.index + g0.length); + } + } + return g0; + }); + } +} + +// eslint-disable-next-line no-underscore-dangle +RegexUtil._matchReplacementPattern = /\$(?:\$|&|`|'|(\d\d?)|<([^>]*)>)/g; diff --git a/ext/js/language/translator.js b/ext/js/language/translator.js index e8aba5b1..c885cd4d 100644 --- a/ext/js/language/translator.js +++ b/ext/js/language/translator.js @@ -17,6 +17,7 @@ /* global * Deinflector + * RegexUtil * TextSourceMap */ @@ -534,24 +535,6 @@ class Translator { ); } - _removeUsedDefinitions(definitions, termInfoMap, usedDefinitions) { - for (let i = 0, ii = definitions.length; i < ii; ++i) { - const definition = definitions[i]; - const {expression, reading} = definition; - const expressionMap = termInfoMap.get(expression); - if ( - typeof expressionMap !== 'undefined' && - typeof expressionMap.get(reading) !== 'undefined' - ) { - usedDefinitions.add(definition); - } else { - definitions.splice(i, 1); - --i; - --ii; - } - } - } - _getUniqueDefinitionTags(definitions) { const definitionTagsMap = new Map(); for (const {definitionTags} of definitions) { @@ -794,16 +777,6 @@ class Translator { // Simple helpers - _scoreToTermFrequency(score) { - if (score > 0) { - return 'popular'; - } else if (score < 0) { - return 'rare'; - } else { - return 'normal'; - } - } - _getNameBase(name) { const pos = name.indexOf(':'); return (pos >= 0 ? name.substring(0, pos) : name); @@ -974,14 +947,6 @@ class Translator { // Reduction functions - _getTermTagsScoreSum(termTags) { - let result = 0; - for (const {score} of termTags) { - result += score; - } - return result; - } - _getSourceTermMatchCountSum(definitions) { let result = 0; for (const {sourceTermExactMatchCount} of definitions) { @@ -1408,59 +1373,8 @@ class Translator { _applyTextReplacements(text, sourceMap, replacements) { for (const {pattern, replacement} of replacements) { - text = this._applyTextReplacement(text, sourceMap, pattern, replacement); + text = RegexUtil.applyTextReplacement(text, sourceMap, pattern, replacement); } return text; } - - _applyTextReplacement(text, sourceMap, pattern, replacement) { - const isGlobal = pattern.global; - if (isGlobal) { pattern.lastIndex = 0; } - for (let loop = true; loop; loop = isGlobal) { - const match = pattern.exec(text); - if (match === null) { break; } - - const matchText = match[0]; - const index = match.index; - const actualReplacement = this._applyMatchReplacement(replacement, match); - const actualReplacementLength = actualReplacement.length; - const delta = actualReplacementLength - (matchText.length > 0 ? matchText.length : -1); - - text = `${text.substring(0, index)}${actualReplacement}${text.substring(index + matchText.length)}`; - pattern.lastIndex += delta; - - if (actualReplacementLength > 0) { - sourceMap.insert(index, ...(new Array(actualReplacementLength).fill(0))); - sourceMap.combine(index - 1 + actualReplacementLength, matchText.length); - } else { - sourceMap.combine(index, matchText.length); - } - } - return text; - } - - _applyMatchReplacement(replacement, match) { - const pattern = /\$(?:\$|&|`|'|(\d\d?)|<([^>]*)>)/g; - return replacement.replace(pattern, (g0, g1, g2) => { - if (typeof g1 !== 'undefined') { - const matchIndex = Number.parseInt(g1, 10); - if (matchIndex >= 1 && matchIndex <= match.length) { - return match[matchIndex]; - } - } else if (typeof g2 !== 'undefined') { - const {groups} = match; - if (typeof groups === 'object' && groups !== null && Object.prototype.hasOwnProperty.call(groups, g2)) { - return groups[g2]; - } - } else { - switch (g0) { - case '$': return '$'; - case '&': return match[0]; - case '`': return replacement.substring(0, match.index); - case '\'': return replacement.substring(match.index + g0.length); - } - } - return g0; - }); - } } @@ -36,6 +36,7 @@ self.importScripts( '/js/dom/simple-dom-parser.js', '/js/general/cache-map.js', '/js/general/object-property-accessor.js', + '/js/general/regex-util.js', '/js/general/text-source-map.js', '/js/language/deinflector.js', '/js/language/dictionary-data-util.js', diff --git a/test/test-translator.js b/test/test-translator.js index 09f1ee62..307329d0 100644 --- a/test/test-translator.js +++ b/test/test-translator.js @@ -44,6 +44,7 @@ async function createVM() { 'js/language/dictionary-importer.js', 'js/data/database.js', 'js/language/dictionary-database.js', + 'js/general/regex-util.js', 'js/general/text-source-map.js', 'js/language/deinflector.js', 'js/language/translator.js' |