summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authortoasted-nutbread <toasted-nutbread@users.noreply.github.com>2021-04-28 21:17:05 -0400
committerGitHub <noreply@github.com>2021-04-28 21:17:05 -0400
commit512391346bd4f67b1933b768c64d4dac1b58b44d (patch)
tree3759ab3b64cdb52d3b4914cc974be6e3fbca665d
parent763c6c76aa5bceb1eb929aa3a87bacf91e37407f (diff)
Translator alphanumeric improvement (#1635)
* Simplify alphanumeric check * Update option name for clarity
-rw-r--r--ext/js/background/backend.js2
-rw-r--r--ext/js/language/translator.js17
-rw-r--r--test/data/translator-test-inputs.json12
3 files changed, 17 insertions, 14 deletions
diff --git a/ext/js/background/backend.js b/ext/js/background/backend.js
index 84ff2c0d..5b133d79 100644
--- a/ext/js/background/backend.js
+++ b/ext/js/background/backend.js
@@ -1896,7 +1896,7 @@ class Backend {
return {
wildcard,
mainDictionary,
- alphanumeric,
+ removeNonJapaneseCharacters: !alphanumeric,
convertHalfWidthCharacters,
convertNumericCharacters,
convertAlphabeticCharacters,
diff --git a/ext/js/language/translator.js b/ext/js/language/translator.js
index dc91d9eb..856763a5 100644
--- a/ext/js/language/translator.js
+++ b/ext/js/language/translator.js
@@ -64,7 +64,7 @@ class Translator {
* {
* wildcard: (enum: null, 'prefix', 'suffix'),
* mainDictionary: (string),
- * alphanumeric: (boolean),
+ * removeNonJapaneseCharacters: (boolean),
* convertHalfWidthCharacters: (enum: 'false', 'true', 'variant'),
* convertNumericCharacters: (enum: 'false', 'true', 'variant'),
* convertAlphabeticCharacters: (enum: 'false', 'true', 'variant'),
@@ -169,8 +169,10 @@ class Translator {
// Find terms internal implementation
async _findTermsInternal(text, enabledDictionaryMap, options) {
- const {alphanumeric, wildcard} = options;
- text = this._getSearchableText(text, alphanumeric);
+ const {wildcard} = options;
+ if (options.removeNonJapaneseCharacters) {
+ text = this._getJapaneseOnlyText(text);
+ }
if (text.length === 0) {
return {dictionaryEntries: [], originalTextLength: 0};
}
@@ -302,15 +304,16 @@ class Translator {
return text;
}
- _getSearchableText(text, allowAlphanumericCharacters) {
- if (allowAlphanumericCharacters) { return text; }
+ _getJapaneseOnlyText(text) {
const jp = this._japaneseUtil;
let length = 0;
for (const c of text) {
- if (!jp.isCodePointJapanese(c.codePointAt(0))) { break; }
+ if (!jp.isCodePointJapanese(c.codePointAt(0))) {
+ return text.substring(0, length);
+ }
length += c.length;
}
- return length >= text.length ? text : text.substring(0, length);
+ return text;
}
_getTextOptionEntryVariants(value) {
diff --git a/test/data/translator-test-inputs.json b/test/data/translator-test-inputs.json
index f3000389..5c47395c 100644
--- a/test/data/translator-test-inputs.json
+++ b/test/data/translator-test-inputs.json
@@ -14,7 +14,7 @@
"default": {
"wildcard": null,
"mainDictionary": "${title}",
- "alphanumeric": false,
+ "removeNonJapaneseCharacters": true,
"convertHalfWidthCharacters": false,
"convertNumericCharacters": false,
"convertAlphabeticCharacters": false,
@@ -182,7 +182,7 @@
"options": [
"default",
{
- "alphanumeric": true,
+ "removeNonJapaneseCharacters": false,
"textReplacements": [
null,
[
@@ -204,7 +204,7 @@
"options": [
"default",
{
- "alphanumeric": true,
+ "removeNonJapaneseCharacters": false,
"textReplacements": [
null,
[
@@ -226,7 +226,7 @@
"options": [
"default",
{
- "alphanumeric": true,
+ "removeNonJapaneseCharacters": false,
"textReplacements": [
null,
[
@@ -248,7 +248,7 @@
"options": [
"default",
{
- "alphanumeric": true,
+ "removeNonJapaneseCharacters": false,
"textReplacements": [
null,
[
@@ -270,7 +270,7 @@
"options": [
"default",
{
- "alphanumeric": true,
+ "removeNonJapaneseCharacters": false,
"textReplacements": [
null,
[