From ceb12ac41551aca11bc195e5fad9984a28a5e291 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sat, 11 Apr 2020 23:20:36 -0400 Subject: Add support for filtering frequency metadata based on readings --- .../dictionaries/valid-dictionary1/term_meta_bank_1.json | 6 ++++++ test/test-database.js | 16 ++++++++-------- 2 files changed, 14 insertions(+), 8 deletions(-) (limited to 'test') diff --git a/test/data/dictionaries/valid-dictionary1/term_meta_bank_1.json b/test/data/dictionaries/valid-dictionary1/term_meta_bank_1.json index 26922394..73d74e68 100644 --- a/test/data/dictionaries/valid-dictionary1/term_meta_bank_1.json +++ b/test/data/dictionaries/valid-dictionary1/term_meta_bank_1.json @@ -2,6 +2,12 @@ ["打", "freq", 1], ["打つ", "freq", 2], ["打ち込む", "freq", 3], + ["打", "freq", {"reading": "だ", "frequency": 4}], + ["打", "freq", {"reading": "ダース", "frequency": 5}], + ["打つ", "freq", {"reading": "うつ", "frequency": 6}], + ["打つ", "freq", {"reading": "ぶつ", "frequency": 7}], + ["打ち込む", "freq", {"reading": "うちこむ", "frequency": 8}], + ["打ち込む", "freq", {"reading": "ぶちこむ", "frequency": 9}], [ "打ち込む", "pitch", diff --git a/test/test-database.js b/test/test-database.js index d27f92e1..8b7a163a 100644 --- a/test/test-database.js +++ b/test/test-database.js @@ -235,8 +235,8 @@ async function testDatabase1() { true ); vm.assert.deepStrictEqual(counts, { - counts: [{kanji: 2, kanjiMeta: 2, terms: 32, termMeta: 6, tagMeta: 14}], - total: {kanji: 2, kanjiMeta: 2, terms: 32, termMeta: 6, tagMeta: 14} + counts: [{kanji: 2, kanjiMeta: 2, terms: 32, termMeta: 12, tagMeta: 14}], + total: {kanji: 2, kanjiMeta: 2, terms: 32, termMeta: 12, tagMeta: 14} }); // Test find* functions @@ -626,9 +626,9 @@ async function testFindTermMetaBulk1(database, titles) { } ], expectedResults: { - total: 1, + total: 3, modes: [ - ['freq', 1] + ['freq', 3] ] } }, @@ -639,9 +639,9 @@ async function testFindTermMetaBulk1(database, titles) { } ], expectedResults: { - total: 1, + total: 3, modes: [ - ['freq', 1] + ['freq', 3] ] } }, @@ -652,9 +652,9 @@ async function testFindTermMetaBulk1(database, titles) { } ], expectedResults: { - total: 3, + total: 5, modes: [ - ['freq', 1], + ['freq', 3], ['pitch', 2] ] } -- cgit v1.2.3 From fbaf50def1934ef6fe0967233f4419efc44f1c30 Mon Sep 17 00:00:00 2001 From: siikamiika Date: Sat, 18 Apr 2020 00:33:49 +0300 Subject: support iframes inside open shadow dom --- ext/fg/js/frame-offset-forwarder.js | 23 +++++++++++++++++++++-- test/data/html/test-document2.html | 19 ++++++++++++++++++- 2 files changed, 39 insertions(+), 3 deletions(-) (limited to 'test') diff --git a/ext/fg/js/frame-offset-forwarder.js b/ext/fg/js/frame-offset-forwarder.js index c658c55a..ac6e617d 100644 --- a/ext/fg/js/frame-offset-forwarder.js +++ b/ext/fg/js/frame-offset-forwarder.js @@ -79,9 +79,28 @@ class FrameOffsetForwarder { sourceFrame = frame; break; } + if (sourceFrame === null) { - this._forwardFrameOffsetOrigin(null, uniqueId); - return; + const getShadowRootElements = (documentOrElement) => { + const elements = Array.from(documentOrElement.querySelectorAll('*')) + .filter((el) => !!el.shadowRoot); + const childElements = elements + .map((el) => el.shadowRoot) + .map(getShadowRootElements); + elements.push(childElements.flat()); + + return elements.flat(); + }; + + sourceFrame = getShadowRootElements(document) + .map((el) => Array.from(el.shadowRoot.querySelectorAll('frame, iframe:not(.yomichan-float)'))) + .flat() + .find((el) => el.contentWindow === e.source); + + if (!sourceFrame) { + this._forwardFrameOffsetOrigin(null, uniqueId); + return; + } } const [forwardedX, forwardedY] = offset; diff --git a/test/data/html/test-document2.html b/test/data/html/test-document2.html index 3a22a5bf..b2046dfd 100644 --- a/test/data/html/test-document2.html +++ b/test/data/html/test-document2.html @@ -77,5 +77,22 @@ document.querySelector('#fullscreen-link1').addEventListener('click', () => togg +
+
<iframe> element inside of an open shadow DOM.
+
+ + +
+ - \ No newline at end of file + -- cgit v1.2.3 From c992e7f920f20c0c7cc55fddf6aba61e0f8b1641 Mon Sep 17 00:00:00 2001 From: siikamiika Date: Sat, 18 Apr 2020 03:35:00 +0300 Subject: add manual performance tests --- test/data/html/test-document3-frame1.html | 44 ++++++++++++++++++++++ test/data/html/test-document3-frame2.html | 62 +++++++++++++++++++++++++++++++ test/data/html/test-document3.html | 26 +++++++++++++ 3 files changed, 132 insertions(+) create mode 100644 test/data/html/test-document3-frame1.html create mode 100644 test/data/html/test-document3-frame2.html create mode 100644 test/data/html/test-document3.html (limited to 'test') diff --git a/test/data/html/test-document3-frame1.html b/test/data/html/test-document3-frame1.html new file mode 100644 index 00000000..2ae906d2 --- /dev/null +++ b/test/data/html/test-document3-frame1.html @@ -0,0 +1,44 @@ + + + + + + Yomichan Manual Performance Tests + + +
+ +
Add elements
+ +
+ 1000 + 10000 + 100000 + 1000000 + +
+ +
+
+ +
+ diff --git a/test/data/html/test-document3-frame2.html b/test/data/html/test-document3-frame2.html new file mode 100644 index 00000000..c486e04b --- /dev/null +++ b/test/data/html/test-document3-frame2.html @@ -0,0 +1,62 @@ + + + + + + Yomichan Manual Performance Tests + + +
+ +
<iframe> element inside of an open shadow DOM.
+ +
+ + + +
Add elements
+ +
+ 1000 + 10000 + 100000 + 1000000 +
+ +
+
+ + +
+ diff --git a/test/data/html/test-document3.html b/test/data/html/test-document3.html new file mode 100644 index 00000000..3e7d5236 --- /dev/null +++ b/test/data/html/test-document3.html @@ -0,0 +1,26 @@ + + + + + + Yomichan Manual Performance Tests + + + + + +

Yomichan Manual Performance Tests

+

Testing Yomichan performance with artificially demanding cases in a real browser

+ +
+
<iframe> element.
+ +
+ +
+
<iframe> element containing an <iframe> element inside of an open shadow DOM.
+ +
+ + + -- cgit v1.2.3 From 5f49f0fed25b421b0142b306a968f2cd435804e1 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sat, 11 Apr 2020 14:24:03 -0400 Subject: Add tests --- test/data/dictionaries/valid-dictionary1/image.gif | Bin 0 -> 45 bytes .../valid-dictionary1/term_bank_1.json | 3 +- test/test-database.js | 52 ++++++++++++++++++++- test/yomichan-test.js | 15 ++++-- 4 files changed, 62 insertions(+), 8 deletions(-) create mode 100644 test/data/dictionaries/valid-dictionary1/image.gif (limited to 'test') diff --git a/test/data/dictionaries/valid-dictionary1/image.gif b/test/data/dictionaries/valid-dictionary1/image.gif new file mode 100644 index 00000000..f089d07c Binary files /dev/null and b/test/data/dictionaries/valid-dictionary1/image.gif differ diff --git a/test/data/dictionaries/valid-dictionary1/term_bank_1.json b/test/data/dictionaries/valid-dictionary1/term_bank_1.json index 755d9f6a..a62ad117 100644 --- a/test/data/dictionaries/valid-dictionary1/term_bank_1.json +++ b/test/data/dictionaries/valid-dictionary1/term_bank_1.json @@ -30,5 +30,6 @@ ["打ち込む", "ぶちこむ", "tag1 tag2", "v5", 29, ["definition1a (打ち込む, ぶちこむ)", "definition1b (打ち込む, ぶちこむ)"], 6, "tag3 tag4 tag5"], ["打ち込む", "ぶちこむ", "tag1 tag2", "v5", 30, ["definition2a (打ち込む, ぶちこむ)", "definition2b (打ち込む, ぶちこむ)"], 6, "tag3 tag4 tag5"], ["打ち込む", "ぶちこむ", "tag1 tag2", "v5", 31, ["definition3a (打ち込む, ぶちこむ)", "definition3b (打ち込む, ぶちこむ)"], 6, "tag3 tag4 tag5"], - ["打ち込む", "ぶちこむ", "tag1 tag2", "v5", 32, ["definition4a (打ち込む, ぶちこむ)", "definition4b (打ち込む, ぶちこむ)"], 6, "tag3 tag4 tag5"] + ["打ち込む", "ぶちこむ", "tag1 tag2", "v5", 32, ["definition4a (打ち込む, ぶちこむ)", "definition4b (打ち込む, ぶちこむ)"], 6, "tag3 tag4 tag5"], + ["画像", "がぞう", "tag1 tag2", "", 33, ["definition1a (画像, がぞう)", {"type": "image", "path": "image.gif", "width": 350, "height": 350, "description": "An image", "pixelated": true}], 7, "tag3 tag4 tag5"] ] \ No newline at end of file diff --git a/test/test-database.js b/test/test-database.js index 8b7a163a..e9ec3f0b 100644 --- a/test/test-database.js +++ b/test/test-database.js @@ -92,9 +92,56 @@ class XMLHttpRequest { } } +class Image { + constructor() { + this._src = ''; + this._loadCallbacks = []; + } + + get src() { + return this._src; + } + + set src(value) { + this._src = value; + this._delayTriggerLoad(); + } + + get naturalWidth() { + return 100; + } + + get naturalHeight() { + return 100; + } + + addEventListener(eventName, callback) { + if (eventName === 'load') { + this._loadCallbacks.push(callback); + } + } + + removeEventListener(eventName, callback) { + if (eventName === 'load') { + const index = this._loadCallbacks.indexOf(callback); + if (index >= 0) { + this._loadCallbacks.splice(index, 1); + } + } + } + + async _delayTriggerLoad() { + await Promise.resolve(); + for (const callback of this._loadCallbacks) { + callback(); + } + } +} + const vm = new VM({ chrome, + Image, XMLHttpRequest, indexedDB: global.indexedDB, IDBKeyRange: global.IDBKeyRange, @@ -106,6 +153,7 @@ vm.execute([ 'bg/js/json-schema.js', 'bg/js/dictionary.js', 'mixed/js/core.js', + 'bg/js/media-utility.js', 'bg/js/request.js', 'bg/js/dictionary-importer.js', 'bg/js/database.js' @@ -235,8 +283,8 @@ async function testDatabase1() { true ); vm.assert.deepStrictEqual(counts, { - counts: [{kanji: 2, kanjiMeta: 2, terms: 32, termMeta: 12, tagMeta: 14}], - total: {kanji: 2, kanjiMeta: 2, terms: 32, termMeta: 12, tagMeta: 14} + counts: [{kanji: 2, kanjiMeta: 2, terms: 33, termMeta: 12, tagMeta: 14}], + total: {kanji: 2, kanjiMeta: 2, terms: 33, termMeta: 12, tagMeta: 14} }); // Test find* functions diff --git a/test/yomichan-test.js b/test/yomichan-test.js index 3351ecdf..0b340abb 100644 --- a/test/yomichan-test.js +++ b/test/yomichan-test.js @@ -38,12 +38,17 @@ function createTestDictionaryArchive(dictionary, dictionaryName) { const archive = new (getJSZip())(); for (const fileName of fileNames) { - const source = fs.readFileSync(path.join(dictionaryDirectory, fileName), {encoding: 'utf8'}); - const json = JSON.parse(source); - if (fileName === 'index.json' && typeof dictionaryName === 'string') { - json.title = dictionaryName; + if (/\.json$/.test(fileName)) { + const source = fs.readFileSync(path.join(dictionaryDirectory, fileName), {encoding: 'utf8'}); + const json = JSON.parse(source); + if (fileName === 'index.json' && typeof dictionaryName === 'string') { + json.title = dictionaryName; + } + archive.file(fileName, JSON.stringify(json, null, 0)); + } else { + const source = fs.readFileSync(path.join(dictionaryDirectory, fileName), {encoding: null}); + archive.file(fileName, source); } - archive.file(fileName, JSON.stringify(json, null, 0)); } return archive; -- cgit v1.2.3 From 99c1a6a6bc0ce55eb2f20703a7f7f69c4bcefd9d Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sun, 19 Apr 2020 10:57:33 -0400 Subject: Change some test variables using 'source' instead of 'content' --- test/yomichan-test.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'test') diff --git a/test/yomichan-test.js b/test/yomichan-test.js index 0b340abb..b4f5ac7c 100644 --- a/test/yomichan-test.js +++ b/test/yomichan-test.js @@ -39,15 +39,15 @@ function createTestDictionaryArchive(dictionary, dictionaryName) { for (const fileName of fileNames) { if (/\.json$/.test(fileName)) { - const source = fs.readFileSync(path.join(dictionaryDirectory, fileName), {encoding: 'utf8'}); - const json = JSON.parse(source); + const content = fs.readFileSync(path.join(dictionaryDirectory, fileName), {encoding: 'utf8'}); + const json = JSON.parse(content); if (fileName === 'index.json' && typeof dictionaryName === 'string') { json.title = dictionaryName; } archive.file(fileName, JSON.stringify(json, null, 0)); } else { - const source = fs.readFileSync(path.join(dictionaryDirectory, fileName), {encoding: null}); - archive.file(fileName, source); + const content = fs.readFileSync(path.join(dictionaryDirectory, fileName), {encoding: null}); + archive.file(fileName, content); } } -- cgit v1.2.3 From a761e420846c5a60ad70a5ad0df186c26f97ffd1 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Wed, 22 Apr 2020 21:37:28 -0400 Subject: Update tests --- test/test-japanese.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'test') diff --git a/test/test-japanese.js b/test/test-japanese.js index 87efdfad..321861d5 100644 --- a/test/test-japanese.js +++ b/test/test-japanese.js @@ -434,17 +434,17 @@ function testIsMoraPitchHigh() { [[2, 1], false], [[3, 1], false], - [[0, 2], true], + [[0, 2], false], [[1, 2], true], [[2, 2], false], [[3, 2], false], - [[0, 3], true], + [[0, 3], false], [[1, 3], true], [[2, 3], true], [[3, 3], false], - [[0, 4], true], + [[0, 4], false], [[1, 4], true], [[2, 4], true], [[3, 4], true] -- cgit v1.2.3 From 5b96559df819f496b39acb75c679f6b3d8c8e65d Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sun, 26 Apr 2020 16:55:25 -0400 Subject: Error logging refactoring (#454) * Create new logging methods on yomichan object * Use new yomichan.logError instead of global logError * Remove old logError * Handle unhandledrejection events * Add addEventListener stub * Update log function * Update error conversion to support more types * Add log event * Add API log function * Log errors to the backend * Make error/warning logs update the badge * Clear log error indicator on extension button click * Log correct URL on the background page * Fix incorrect error conversion * Remove unhandledrejection handling Firefox doesn't support it properly. * Remove unused argument type from log function * Improve function name * Change console.warn to yomichan.logWarning * Move log forwarding initialization into main scripts --- .eslintrc.json | 1 - ext/bg/js/backend.js | 50 ++++++++++- ext/bg/js/context-main.js | 5 ++ ext/bg/js/database.js | 2 +- ext/bg/js/mecab.js | 2 +- ext/bg/js/search-main.js | 2 + ext/bg/js/search-query-parser.js | 2 +- ext/bg/js/search.js | 2 +- ext/bg/js/settings/backup.js | 2 +- ext/bg/js/settings/dictionaries.js | 2 +- ext/bg/js/settings/main.js | 2 + ext/bg/js/settings/popup-preview-frame-main.js | 2 + ext/fg/js/content-script-main.js | 2 + ext/fg/js/float-main.js | 2 + ext/fg/js/float.js | 2 +- ext/fg/js/frontend-api-sender.js | 8 +- ext/fg/js/popup-proxy.js | 2 +- ext/mixed/js/api.js | 22 +++++ ext/mixed/js/core.js | 112 +++++++++++++++++++------ ext/mixed/js/text-scanner.js | 2 +- test/test-database.js | 5 +- 21 files changed, 186 insertions(+), 45 deletions(-) (limited to 'test') diff --git a/.eslintrc.json b/.eslintrc.json index 8882cb42..78fec27c 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -80,7 +80,6 @@ "yomichan": "readonly", "errorToJson": "readonly", "jsonToError": "readonly", - "logError": "readonly", "isObject": "readonly", "hasOwn": "readonly", "toIterable": "readonly", diff --git a/ext/bg/js/backend.js b/ext/bg/js/backend.js index 693a9ad6..3c47b14e 100644 --- a/ext/bg/js/backend.js +++ b/ext/bg/js/backend.js @@ -78,6 +78,7 @@ class Backend { this._isPrepared = false; this._prepareError = false; this._badgePrepareDelayTimer = null; + this._logErrorLevel = null; this._messageHandlers = new Map([ ['yomichanCoreReady', {handler: this._onApiYomichanCoreReady.bind(this), async: false}], @@ -112,7 +113,9 @@ class Backend { ['getDictionaryInfo', {handler: this._onApiGetDictionaryInfo.bind(this), async: true}], ['getDictionaryCounts', {handler: this._onApiGetDictionaryCounts.bind(this), async: true}], ['purgeDatabase', {handler: this._onApiPurgeDatabase.bind(this), async: true}], - ['getMedia', {handler: this._onApiGetMedia.bind(this), async: true}] + ['getMedia', {handler: this._onApiGetMedia.bind(this), async: true}], + ['log', {handler: this._onApiLog.bind(this), async: false}], + ['logIndicatorClear', {handler: this._onApiLogIndicatorClear.bind(this), async: false}] ]); this._commandHandlers = new Map([ @@ -164,7 +167,7 @@ class Backend { this._isPrepared = true; } catch (e) { this._prepareError = true; - logError(e); + yomichan.logError(e); throw e; } finally { if (this._badgePrepareDelayTimer !== null) { @@ -260,7 +263,7 @@ class Backend { this.options = JsonSchema.getValidValueOrDefault(this.optionsSchema, utilIsolate(options)); } catch (e) { // This shouldn't happen, but catch errors just in case of bugs - logError(e); + yomichan.logError(e); } } @@ -767,8 +770,34 @@ class Backend { return await this.database.getMedia(targets); } + _onApiLog({error, level, context}) { + yomichan.log(jsonToError(error), level, context); + + const levelValue = this._getErrorLevelValue(level); + if (levelValue <= this._getErrorLevelValue(this._logErrorLevel)) { return; } + + this._logErrorLevel = level; + this._updateBadge(); + } + + _onApiLogIndicatorClear() { + if (this._logErrorLevel === null) { return; } + this._logErrorLevel = null; + this._updateBadge(); + } + // Command handlers + _getErrorLevelValue(errorLevel) { + switch (errorLevel) { + case 'info': return 0; + case 'debug': return 0; + case 'warn': return 1; + case 'error': return 2; + default: return 0; + } + } + async _onCommandSearch(params) { const {mode='existingOrNewTab', query} = params || {}; @@ -890,7 +919,20 @@ class Backend { let color = null; let status = null; - if (!this._isPrepared) { + if (this._logErrorLevel !== null) { + switch (this._logErrorLevel) { + case 'error': + text = '!!'; + color = '#f04e4e'; + status = 'Error'; + break; + default: // 'warn' + text = '!'; + color = '#f0ad4e'; + status = 'Warning'; + break; + } + } else if (!this._isPrepared) { if (this._prepareError) { text = '!!'; color = '#f04e4e'; diff --git a/ext/bg/js/context-main.js b/ext/bg/js/context-main.js index e2086a96..dbba0272 100644 --- a/ext/bg/js/context-main.js +++ b/ext/bg/js/context-main.js @@ -17,7 +17,9 @@ /* global * apiCommandExec + * apiForwardLogsToBackend * apiGetEnvironmentInfo + * apiLogIndicatorClear * apiOptionsGet */ @@ -52,8 +54,11 @@ function setupButtonEvents(selector, command, url) { } async function mainInner() { + apiForwardLogsToBackend(); await yomichan.prepare(); + await apiLogIndicatorClear(); + showExtensionInfo(); apiGetEnvironmentInfo().then(({browser}) => { diff --git a/ext/bg/js/database.js b/ext/bg/js/database.js index 16612403..a94aa720 100644 --- a/ext/bg/js/database.js +++ b/ext/bg/js/database.js @@ -104,7 +104,7 @@ class Database { }); return true; } catch (e) { - logError(e); + yomichan.logError(e); return false; } } diff --git a/ext/bg/js/mecab.js b/ext/bg/js/mecab.js index 597dceae..815ee860 100644 --- a/ext/bg/js/mecab.js +++ b/ext/bg/js/mecab.js @@ -24,7 +24,7 @@ class Mecab { } onError(error) { - logError(error, false); + yomichan.logError(error); } async checkVersion() { diff --git a/ext/bg/js/search-main.js b/ext/bg/js/search-main.js index 38b6d99a..5e4d7a20 100644 --- a/ext/bg/js/search-main.js +++ b/ext/bg/js/search-main.js @@ -17,6 +17,7 @@ /* global * DisplaySearch + * apiForwardLogsToBackend * apiOptionsGet */ @@ -53,6 +54,7 @@ function injectSearchFrontend() { } (async () => { + apiForwardLogsToBackend(); await yomichan.prepare(); const displaySearch = new DisplaySearch(); diff --git a/ext/bg/js/search-query-parser.js b/ext/bg/js/search-query-parser.js index eb3b681c..0001c9ff 100644 --- a/ext/bg/js/search-query-parser.js +++ b/ext/bg/js/search-query-parser.js @@ -45,7 +45,7 @@ class QueryParser extends TextScanner { } onError(error) { - logError(error, false); + yomichan.logError(error); } onClick(e) { diff --git a/ext/bg/js/search.js b/ext/bg/js/search.js index a5484fc3..cbd7b562 100644 --- a/ext/bg/js/search.js +++ b/ext/bg/js/search.js @@ -122,7 +122,7 @@ class DisplaySearch extends Display { } onError(error) { - logError(error, true); + yomichan.logError(error); } onSearchClear() { diff --git a/ext/bg/js/settings/backup.js b/ext/bg/js/settings/backup.js index bdfef658..faf4e592 100644 --- a/ext/bg/js/settings/backup.js +++ b/ext/bg/js/settings/backup.js @@ -133,7 +133,7 @@ async function _settingsImportSetOptionsFull(optionsFull) { } function _showSettingsImportError(error) { - logError(error); + yomichan.logError(error); document.querySelector('#settings-import-error-modal-message').textContent = `${error}`; $('#settings-import-error-modal').modal('show'); } diff --git a/ext/bg/js/settings/dictionaries.js b/ext/bg/js/settings/dictionaries.js index 7eed4273..50add4c7 100644 --- a/ext/bg/js/settings/dictionaries.js +++ b/ext/bg/js/settings/dictionaries.js @@ -554,7 +554,7 @@ function dictionaryErrorsShow(errors) { if (errors !== null && errors.length > 0) { const uniqueErrors = new Map(); for (let e of errors) { - logError(e); + yomichan.logError(e); e = dictionaryErrorToString(e); let count = uniqueErrors.get(e); if (typeof count === 'undefined') { diff --git a/ext/bg/js/settings/main.js b/ext/bg/js/settings/main.js index 308e92eb..f03cc631 100644 --- a/ext/bg/js/settings/main.js +++ b/ext/bg/js/settings/main.js @@ -21,6 +21,7 @@ * ankiInitialize * ankiTemplatesInitialize * ankiTemplatesUpdateValue + * apiForwardLogsToBackend * apiOptionsSave * appearanceInitialize * audioSettingsInitialize @@ -284,6 +285,7 @@ function showExtensionInformation() { async function onReady() { + apiForwardLogsToBackend(); await yomichan.prepare(); showExtensionInformation(); diff --git a/ext/bg/js/settings/popup-preview-frame-main.js b/ext/bg/js/settings/popup-preview-frame-main.js index 2ab6af6b..8228125f 100644 --- a/ext/bg/js/settings/popup-preview-frame-main.js +++ b/ext/bg/js/settings/popup-preview-frame-main.js @@ -17,8 +17,10 @@ /* global * SettingsPopupPreview + * apiForwardLogsToBackend */ (() => { + apiForwardLogsToBackend(); new SettingsPopupPreview(); })(); diff --git a/ext/fg/js/content-script-main.js b/ext/fg/js/content-script-main.js index 0b852644..277e6567 100644 --- a/ext/fg/js/content-script-main.js +++ b/ext/fg/js/content-script-main.js @@ -22,6 +22,7 @@ * PopupProxy * PopupProxyHost * apiBroadcastTab + * apiForwardLogsToBackend * apiOptionsGet */ @@ -62,6 +63,7 @@ async function createPopupProxy(depth, id, parentFrameId) { } (async () => { + apiForwardLogsToBackend(); await yomichan.prepare(); const data = window.frontendInitializationData || {}; diff --git a/ext/fg/js/float-main.js b/ext/fg/js/float-main.js index f056f707..5ef4b07c 100644 --- a/ext/fg/js/float-main.js +++ b/ext/fg/js/float-main.js @@ -17,6 +17,7 @@ /* global * DisplayFloat + * apiForwardLogsToBackend * apiOptionsGet */ @@ -68,5 +69,6 @@ async function popupNestedInitialize(id, depth, parentFrameId, url) { } (async () => { + apiForwardLogsToBackend(); new DisplayFloat(); })(); diff --git a/ext/fg/js/float.js b/ext/fg/js/float.js index 2a5eba83..fd3b92cc 100644 --- a/ext/fg/js/float.js +++ b/ext/fg/js/float.js @@ -84,7 +84,7 @@ class DisplayFloat extends Display { if (this._orphaned) { this.setContent('orphaned'); } else { - logError(error, true); + yomichan.logError(error); } } diff --git a/ext/fg/js/frontend-api-sender.js b/ext/fg/js/frontend-api-sender.js index 1d539cab..0ad3f085 100644 --- a/ext/fg/js/frontend-api-sender.js +++ b/ext/fg/js/frontend-api-sender.js @@ -81,12 +81,12 @@ class FrontendApiSender { onAck(id) { const info = this.callbacks.get(id); if (typeof info === 'undefined') { - console.warn(`ID ${id} not found for ack`); + yomichan.logWarning(new Error(`ID ${id} not found for ack`)); return; } if (info.ack) { - console.warn(`Request ${id} already ack'd`); + yomichan.logWarning(new Error(`Request ${id} already ack'd`)); return; } @@ -98,12 +98,12 @@ class FrontendApiSender { onResult(id, data) { const info = this.callbacks.get(id); if (typeof info === 'undefined') { - console.warn(`ID ${id} not found`); + yomichan.logWarning(new Error(`ID ${id} not found`)); return; } if (!info.ack) { - console.warn(`Request ${id} not ack'd`); + yomichan.logWarning(new Error(`Request ${id} not ack'd`)); return; } diff --git a/ext/fg/js/popup-proxy.js b/ext/fg/js/popup-proxy.js index cd3c1bc9..93418202 100644 --- a/ext/fg/js/popup-proxy.js +++ b/ext/fg/js/popup-proxy.js @@ -148,7 +148,7 @@ class PopupProxy { } this._frameOffsetUpdatedAt = now; } catch (e) { - logError(e); + yomichan.logError(e); } finally { this._frameOffsetPromise = null; } diff --git a/ext/mixed/js/api.js b/ext/mixed/js/api.js index 52f41646..afd68aa2 100644 --- a/ext/mixed/js/api.js +++ b/ext/mixed/js/api.js @@ -144,6 +144,14 @@ function apiGetMedia(targets) { return _apiInvoke('getMedia', {targets}); } +function apiLog(error, level, context) { + return _apiInvoke('log', {error, level, context}); +} + +function apiLogIndicatorClear() { + return _apiInvoke('logIndicatorClear'); +} + function _apiInvoke(action, params={}) { const data = {action, params}; return new Promise((resolve, reject) => { @@ -171,3 +179,17 @@ function _apiInvoke(action, params={}) { function _apiCheckLastError() { // NOP } + +let _apiForwardLogsToBackendEnabled = false; +function apiForwardLogsToBackend() { + if (_apiForwardLogsToBackendEnabled) { return; } + _apiForwardLogsToBackendEnabled = true; + + yomichan.on('log', async ({error, level, context}) => { + try { + await apiLog(errorToJson(error), level, context); + } catch (e) { + // NOP + } + }); +} diff --git a/ext/mixed/js/core.js b/ext/mixed/js/core.js index 6a3298fc..fbe9943a 100644 --- a/ext/mixed/js/core.js +++ b/ext/mixed/js/core.js @@ -52,15 +52,28 @@ if (EXTENSION_IS_BROWSER_EDGE) { */ function errorToJson(error) { + try { + if (isObject(error)) { + return { + name: error.name, + message: error.message, + stack: error.stack, + data: error.data + }; + } + } catch (e) { + // NOP + } return { - name: error.name, - message: error.message, - stack: error.stack, - data: error.data + value: error, + hasValue: true }; } function jsonToError(jsonError) { + if (jsonError.hasValue) { + return jsonError.value; + } const error = new Error(jsonError.message); error.name = jsonError.name; error.stack = jsonError.stack; @@ -68,28 +81,6 @@ function jsonToError(jsonError) { return error; } -function logError(error, alert) { - const manifest = chrome.runtime.getManifest(); - let errorMessage = `${manifest.name} v${manifest.version} has encountered an error.\n`; - errorMessage += `Originating URL: ${window.location.href}\n`; - - const errorString = `${error.toString ? error.toString() : error}`; - const stack = `${error.stack}`.trimRight(); - if (!stack.startsWith(errorString)) { errorMessage += `${errorString}\n`; } - errorMessage += stack; - - const data = error.data; - if (typeof data !== 'undefined') { errorMessage += `\nData: ${JSON.stringify(data, null, 4)}`; } - - errorMessage += '\n\nIssues can be reported at https://github.com/FooSoft/yomichan/issues'; - - console.error(errorMessage); - - if (alert) { - window.alert(`${errorString}\n\nCheck the developer console for more details.`); - } -} - /* * Common helpers @@ -361,8 +352,77 @@ const yomichan = (() => { }); } + logWarning(error) { + this.log(error, 'warn'); + } + + logError(error) { + this.log(error, 'error'); + } + + log(error, level, context=null) { + if (!isObject(context)) { + context = this._getLogContext(); + } + + let errorString; + try { + errorString = error.toString(); + if (/^\[object \w+\]$/.test(errorString)) { + errorString = JSON.stringify(error); + } + } catch (e) { + errorString = `${error}`; + } + + let errorStack; + try { + errorStack = (typeof error.stack === 'string' ? error.stack.trimRight() : ''); + } catch (e) { + errorStack = ''; + } + + let errorData; + try { + errorData = error.data; + } catch (e) { + // NOP + } + + if (errorStack.startsWith(errorString)) { + errorString = errorStack; + } else if (errorStack.length > 0) { + errorString += `\n${errorStack}`; + } + + const manifest = chrome.runtime.getManifest(); + let message = `${manifest.name} v${manifest.version} has encountered a problem.`; + message += `\nOriginating URL: ${context.url}\n`; + message += errorString; + if (typeof errorData !== 'undefined') { + message += `\nData: ${JSON.stringify(errorData, null, 4)}`; + } + message += '\n\nIssues can be reported at https://github.com/FooSoft/yomichan/issues'; + + switch (level) { + case 'info': console.info(message); break; + case 'debug': console.debug(message); break; + case 'warn': console.warn(message); break; + case 'error': console.error(message); break; + default: console.log(message); break; + } + + this.trigger('log', {error, level, context}); + } + // Private + _getLogContext() { + return { + url: window.location.href + }; + } + _onMessage({action, params}, sender, callback) { const handler = this._messageHandlers.get(action); if (typeof handler !== 'function') { return false; } diff --git a/ext/mixed/js/text-scanner.js b/ext/mixed/js/text-scanner.js index 0cd12cd7..1c32714b 100644 --- a/ext/mixed/js/text-scanner.js +++ b/ext/mixed/js/text-scanner.js @@ -201,7 +201,7 @@ class TextScanner { } onError(error) { - logError(error, false); + yomichan.logError(error); } async scanTimerWait() { diff --git a/test/test-database.js b/test/test-database.js index e9ec3f0b..3684051b 100644 --- a/test/test-database.js +++ b/test/test-database.js @@ -145,7 +145,10 @@ const vm = new VM({ XMLHttpRequest, indexedDB: global.indexedDB, IDBKeyRange: global.IDBKeyRange, - JSZip: yomichanTest.JSZip + JSZip: yomichanTest.JSZip, + addEventListener() { + // NOP + } }); vm.context.window = vm.context; -- cgit v1.2.3 From 401fe9f8d027fe71f25d2f591f316781e1e0a436 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sat, 2 May 2020 12:57:42 -0400 Subject: Object property accessor API update (#485) * Simplify function names * Add delete and swap functions * Remove custom setter Not currently part of the expected use cases. * Add documentation * Update tests * Add delete test functions * Update tests to use fresh objects * Add swap test functions * Add empty tests * Disable delete on arrays --- ext/mixed/js/object-property-accessor.js | 125 ++++++++++++++++++--- test/test-object-property-accessor.js | 186 ++++++++++++++++++++++++++----- 2 files changed, 268 insertions(+), 43 deletions(-) (limited to 'test') diff --git a/ext/mixed/js/object-property-accessor.js b/ext/mixed/js/object-property-accessor.js index 349037b3..07b8df61 100644 --- a/ext/mixed/js/object-property-accessor.js +++ b/ext/mixed/js/object-property-accessor.js @@ -16,15 +16,27 @@ */ /** - * Class used to get and set generic properties of an object by using path strings. + * Class used to get and mutate generic properties of an object by using path strings. */ class ObjectPropertyAccessor { - constructor(target, setter=null) { + /** + * Create a new accessor for a specific object. + * @param target The object which the getter and mutation methods are applied to. + * @returns A new ObjectPropertyAccessor instance. + */ + constructor(target) { this._target = target; - this._setter = (typeof setter === 'function' ? setter : null); } - getProperty(pathArray, pathLength) { + /** + * Gets the value at the specified path. + * @param pathArray The path to the property on the target object. + * @param pathLength How many parts of the pathArray to use. + * This parameter is optional and defaults to the length of pathArray. + * @returns The value found at the path. + * @throws An error is thrown if pathArray is not valid for the target object. + */ + get(pathArray, pathLength) { let target = this._target; const ii = typeof pathLength === 'number' ? Math.min(pathArray.length, pathLength) : pathArray.length; for (let i = 0; i < ii; ++i) { @@ -37,24 +49,89 @@ class ObjectPropertyAccessor { return target; } - setProperty(pathArray, value) { - if (pathArray.length === 0) { - throw new Error('Invalid path'); + /** + * Sets the value at the specified path. + * @param pathArray The path to the property on the target object. + * @param value The value to assign to the property. + * @throws An error is thrown if pathArray is not valid for the target object. + */ + set(pathArray, value) { + const ii = pathArray.length - 1; + if (ii < 0) { throw new Error('Invalid path'); } + + const target = this.get(pathArray, ii); + const key = pathArray[ii]; + if (!ObjectPropertyAccessor.isValidPropertyType(target, key)) { + throw new Error(`Invalid path: ${ObjectPropertyAccessor.getPathString(pathArray)}`); } - const target = this.getProperty(pathArray, pathArray.length - 1); - const key = pathArray[pathArray.length - 1]; + target[key] = value; + } + + /** + * Deletes the property of the target object at the specified path. + * @param pathArray The path to the property on the target object. + * @throws An error is thrown if pathArray is not valid for the target object. + */ + delete(pathArray) { + const ii = pathArray.length - 1; + if (ii < 0) { throw new Error('Invalid path'); } + + const target = this.get(pathArray, ii); + const key = pathArray[ii]; if (!ObjectPropertyAccessor.isValidPropertyType(target, key)) { throw new Error(`Invalid path: ${ObjectPropertyAccessor.getPathString(pathArray)}`); } - if (this._setter !== null) { - this._setter(target, key, value, pathArray); - } else { - target[key] = value; + if (Array.isArray(target)) { + throw new Error('Invalid type'); + } + + delete target[key]; + } + + /** + * Swaps two properties of an object or array. + * @param pathArray1 The path to the first property on the target object. + * @param pathArray2 The path to the second property on the target object. + * @throws An error is thrown if pathArray1 or pathArray2 is not valid for the target object, + * or if the swap cannot be performed. + */ + swap(pathArray1, pathArray2) { + const ii1 = pathArray1.length - 1; + if (ii1 < 0) { throw new Error('Invalid path 1'); } + const target1 = this.get(pathArray1, ii1); + const key1 = pathArray1[ii1]; + if (!ObjectPropertyAccessor.isValidPropertyType(target1, key1)) { throw new Error(`Invalid path 1: ${ObjectPropertyAccessor.getPathString(pathArray1)}`); } + + const ii2 = pathArray2.length - 1; + if (ii2 < 0) { throw new Error('Invalid path 2'); } + const target2 = this.get(pathArray2, ii2); + const key2 = pathArray2[ii2]; + if (!ObjectPropertyAccessor.isValidPropertyType(target2, key2)) { throw new Error(`Invalid path 2: ${ObjectPropertyAccessor.getPathString(pathArray2)}`); } + + const value1 = target1[key1]; + const value2 = target2[key2]; + + target1[key1] = value2; + try { + target2[key2] = value1; + } catch (e) { + // Revert + try { + target1[key1] = value1; + } catch (e2) { + // NOP + } + throw e; } } + /** + * Converts a path string to a path array. + * @param pathArray The path array to convert. + * @returns A string representation of pathArray. + */ static getPathString(pathArray) { const regexShort = /^[a-zA-Z_][a-zA-Z0-9_]*$/; let pathString = ''; @@ -86,6 +163,12 @@ class ObjectPropertyAccessor { return pathString; } + /** + * Converts a path array to a path string. For the most part, the format of this string + * matches Javascript's notation for property access. + * @param pathString The path string to convert. + * @returns An array representation of pathString. + */ static getPathArray(pathString) { const pathArray = []; let state = 'empty'; @@ -201,6 +284,14 @@ class ObjectPropertyAccessor { return pathArray; } + /** + * Checks whether an object or array has the specified property. + * @param object The object to test. + * @param property The property to check for existence. + * This value should be a string if the object is a non-array object. + * For arrays, it should be an integer. + * @returns true if the property exists, otherwise false. + */ static hasProperty(object, property) { switch (typeof property) { case 'string': @@ -222,6 +313,14 @@ class ObjectPropertyAccessor { } } + /** + * Checks whether a property is valid for the given object + * @param object The object to test. + * @param property The property to check for existence. + * @returns true if the property is correct for the given object type, otherwise false. + * For arrays, this means that the property should be a positive integer. + * For non-array objects, the property should be a string. + */ static isValidPropertyType(object, property) { switch (typeof property) { case 'string': diff --git a/test/test-object-property-accessor.js b/test/test-object-property-accessor.js index 0773ba6e..1e694946 100644 --- a/test/test-object-property-accessor.js +++ b/test/test-object-property-accessor.js @@ -40,29 +40,30 @@ function createTestObject() { } -function testGetProperty1() { - const object = createTestObject(); - const accessor = new ObjectPropertyAccessor(object); - +function testGet1() { const data = [ - [[], object], - [['0'], object['0']], - [['value1'], object.value1], - [['value1', 'value2'], object.value1.value2], - [['value1', 'value3'], object.value1.value3], - [['value1', 'value4'], object.value1.value4], - [['value5'], object.value5], - [['value5', 0], object.value5[0]], - [['value5', 1], object.value5[1]], - [['value5', 2], object.value5[2]] + [[], (object) => object], + [['0'], (object) => object['0']], + [['value1'], (object) => object.value1], + [['value1', 'value2'], (object) => object.value1.value2], + [['value1', 'value3'], (object) => object.value1.value3], + [['value1', 'value4'], (object) => object.value1.value4], + [['value5'], (object) => object.value5], + [['value5', 0], (object) => object.value5[0]], + [['value5', 1], (object) => object.value5[1]], + [['value5', 2], (object) => object.value5[2]] ]; - for (const [pathArray, expected] of data) { - assert.strictEqual(accessor.getProperty(pathArray), expected); + for (const [pathArray, getExpected] of data) { + const object = createTestObject(); + const accessor = new ObjectPropertyAccessor(object); + const expected = getExpected(object); + + assert.strictEqual(accessor.get(pathArray), expected); } } -function testGetProperty2() { +function testGet2() { const object = createTestObject(); const accessor = new ObjectPropertyAccessor(object); @@ -89,15 +90,12 @@ function testGetProperty2() { ]; for (const [pathArray, message] of data) { - assert.throws(() => accessor.getProperty(pathArray), {message}); + assert.throws(() => accessor.get(pathArray), {message}); } } -function testSetProperty1() { - const object = createTestObject(); - const accessor = new ObjectPropertyAccessor(object); - +function testSet1() { const testValue = {}; const data = [ ['0'], @@ -112,17 +110,21 @@ function testSetProperty1() { ]; for (const pathArray of data) { - accessor.setProperty(pathArray, testValue); - assert.strictEqual(accessor.getProperty(pathArray), testValue); + const object = createTestObject(); + const accessor = new ObjectPropertyAccessor(object); + + accessor.set(pathArray, testValue); + assert.strictEqual(accessor.get(pathArray), testValue); } } -function testSetProperty2() { +function testSet2() { const object = createTestObject(); const accessor = new ObjectPropertyAccessor(object); const testValue = {}; const data = [ + [[], 'Invalid path'], [[0], 'Invalid path: [0]'], [['0', 'invalid'], 'Invalid path: ["0"].invalid'], [['value1', 'value2', 0], 'Invalid path: value1.value2[0]'], @@ -137,7 +139,127 @@ function testSetProperty2() { ]; for (const [pathArray, message] of data) { - assert.throws(() => accessor.setProperty(pathArray, testValue), {message}); + assert.throws(() => accessor.set(pathArray, testValue), {message}); + } +} + + +function testDelete1() { + const hasOwn = (object, property) => Object.prototype.hasOwnProperty.call(object, property); + + const data = [ + [['0'], (object) => !hasOwn(object, '0')], + [['value1', 'value2'], (object) => !hasOwn(object.value1, 'value2')], + [['value1', 'value3'], (object) => !hasOwn(object.value1, 'value3')], + [['value1', 'value4'], (object) => !hasOwn(object.value1, 'value4')], + [['value1'], (object) => !hasOwn(object, 'value1')], + [['value5'], (object) => !hasOwn(object, 'value5')] + ]; + + for (const [pathArray, validate] of data) { + const object = createTestObject(); + const accessor = new ObjectPropertyAccessor(object); + + accessor.delete(pathArray); + assert.ok(validate(object)); + } +} + +function testDelete2() { + const data = [ + [[], 'Invalid path'], + [[0], 'Invalid path: [0]'], + [['0', 'invalid'], 'Invalid path: ["0"].invalid'], + [['value1', 'value2', 0], 'Invalid path: value1.value2[0]'], + [['value1', 'value3', 'invalid'], 'Invalid path: value1.value3.invalid'], + [['value1', 'value4', 'invalid'], 'Invalid path: value1.value4.invalid'], + [['value1', 'value4', 0], 'Invalid path: value1.value4[0]'], + [['value5', 1, 'invalid'], 'Invalid path: value5[1].invalid'], + [['value5', 2, 'invalid'], 'Invalid path: value5[2].invalid'], + [['value5', 2, 0], 'Invalid path: value5[2][0]'], + [['value5', 2, 0, 'invalid'], 'Invalid path: value5[2][0]'], + [['value5', 2.5], 'Invalid index'], + [['value5', 0], 'Invalid type'], + [['value5', 1], 'Invalid type'], + [['value5', 2], 'Invalid type'] + ]; + + for (const [pathArray, message] of data) { + const object = createTestObject(); + const accessor = new ObjectPropertyAccessor(object); + + assert.throws(() => accessor.delete(pathArray), {message}); + } +} + + +function testSwap1() { + const data = [ + [['0'], true], + [['value1', 'value2'], true], + [['value1', 'value3'], true], + [['value1', 'value4'], true], + [['value1'], false], + [['value5', 0], true], + [['value5', 1], true], + [['value5', 2], true], + [['value5'], false] + ]; + + for (const [pathArray1, compareValues1] of data) { + for (const [pathArray2, compareValues2] of data) { + const object = createTestObject(); + const accessor = new ObjectPropertyAccessor(object); + + const value1a = accessor.get(pathArray1); + const value2a = accessor.get(pathArray2); + + accessor.swap(pathArray1, pathArray2); + + if (!compareValues1 || !compareValues2) { continue; } + + const value1b = accessor.get(pathArray1); + const value2b = accessor.get(pathArray2); + + assert.deepStrictEqual(value1a, value2b); + assert.deepStrictEqual(value2a, value1b); + } + } +} + +function testSwap2() { + const data = [ + [[], [], false, 'Invalid path 1'], + [['0'], [], false, 'Invalid path 2'], + [[], ['0'], false, 'Invalid path 1'], + [[0], ['0'], false, 'Invalid path 1: [0]'], + [['0'], [0], false, 'Invalid path 2: [0]'] + ]; + + for (const [pathArray1, pathArray2, checkRevert, message] of data) { + const object = createTestObject(); + const accessor = new ObjectPropertyAccessor(object); + + let value1a; + let value2a; + if (checkRevert) { + try { + value1a = accessor.get(pathArray1); + value2a = accessor.get(pathArray2); + } catch (e) { + // NOP + } + } + + assert.throws(() => accessor.swap(pathArray1, pathArray2), {message}); + + if (!checkRevert) { continue; } + + const value1b = accessor.get(pathArray1); + const value2b = accessor.get(pathArray2); + + assert.deepStrictEqual(value1a, value1b); + assert.deepStrictEqual(value2a, value2b); } } @@ -272,10 +394,14 @@ function testIsValidPropertyType() { function main() { - testGetProperty1(); - testGetProperty2(); - testSetProperty1(); - testSetProperty2(); + testGet1(); + testGet2(); + testSet1(); + testSet2(); + testDelete1(); + testDelete2(); + testSwap1(); + testSwap2(); testGetPathString1(); testGetPathString2(); testGetPathArray1(); -- cgit v1.2.3 From d4ae9aa501ece99ea6c5e6b8fb01c3005f5b7f03 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sat, 2 May 2020 13:05:43 -0400 Subject: DOMTextScanner (#458) * Create new class for scanning text in a document * Update test styles * Add tests --- ext/fg/js/dom-text-scanner.js | 538 ++++++++++++++++++++++++++++++ test/data/html/test-dom-text-scanner.html | 393 ++++++++++++++++++++++ test/data/html/test-stylesheet.css | 12 +- test/test-dom-text-scanner.js | 181 ++++++++++ 4 files changed, 1121 insertions(+), 3 deletions(-) create mode 100644 ext/fg/js/dom-text-scanner.js create mode 100644 test/data/html/test-dom-text-scanner.html create mode 100644 test/test-dom-text-scanner.js (limited to 'test') diff --git a/ext/fg/js/dom-text-scanner.js b/ext/fg/js/dom-text-scanner.js new file mode 100644 index 00000000..2de65041 --- /dev/null +++ b/ext/fg/js/dom-text-scanner.js @@ -0,0 +1,538 @@ +/* + * Copyright (C) 2020 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/** + * A class used to scan text in a document. + */ +class DOMTextScanner { + /** + * Creates a new instance of a DOMTextScanner. + * @param node The DOM Node to start at. + * @param offset The character offset in to start at when node is a text node. + * Use 0 for non-text nodes. + */ + constructor(node, offset, forcePreserveWhitespace=false, generateLayoutContent=true) { + const ruby = DOMTextScanner.getParentRubyElement(node); + const resetOffset = (ruby !== null); + if (resetOffset) { node = ruby; } + + this._node = node; + this._offset = offset; + this._content = ''; + this._remainder = 0; + this._resetOffset = resetOffset; + this._newlines = 0; + this._lineHasWhitespace = false; + this._lineHasContent = false; + this._forcePreserveWhitespace = forcePreserveWhitespace; + this._generateLayoutContent = generateLayoutContent; + } + + /** + * Gets the current node being scanned. + * @returns A DOM Node. + */ + get node() { + return this._node; + } + + /** + * Gets the current offset corresponding to the node being scanned. + * This value is only applicable for text nodes. + * @returns An integer. + */ + get offset() { + return this._offset; + } + + /** + * Gets the accumulated content string resulting from calls to seek(). + * @returns A string. + */ + get content() { + return this._content; + } + + /** + * Seeks a given length in the document and accumulates the text content. + * @param length A positive or negative integer corresponding to how many characters + * should be added to content. Content is only added to the accumulation string, + * never removed, so mixing seek calls with differently signed length values + * may give unexpected results. + * @returns this + */ + seek(length) { + const forward = (length >= 0); + this._remainder = (forward ? length : -length); + if (length === 0) { return this; } + + const TEXT_NODE = Node.TEXT_NODE; + const ELEMENT_NODE = Node.ELEMENT_NODE; + + const generateLayoutContent = this._generateLayoutContent; + let node = this._node; + let resetOffset = this._resetOffset; + let newlines = 0; + while (node !== null) { + let enterable = false; + const nodeType = node.nodeType; + + if (nodeType === TEXT_NODE) { + if (!( + forward ? + this._seekTextNodeForward(node, resetOffset) : + this._seekTextNodeBackward(node, resetOffset) + )) { + // Length reached + break; + } + } else if (nodeType === ELEMENT_NODE) { + [enterable, newlines] = DOMTextScanner.getElementSeekInfo(node); + if (newlines > this._newlines && generateLayoutContent) { + this._newlines = newlines; + } + } + + const exitedNodes = []; + node = DOMTextScanner.getNextNode(node, forward, enterable, exitedNodes); + + for (const exitedNode of exitedNodes) { + if (exitedNode.nodeType !== ELEMENT_NODE) { continue; } + newlines = DOMTextScanner.getElementSeekInfo(exitedNode)[1]; + if (newlines > this._newlines && generateLayoutContent) { + this._newlines = newlines; + } + } + + resetOffset = true; + } + + this._node = node; + this._resetOffset = resetOffset; + + return this; + } + + // Private + + /** + * Seeks forward in a text node. + * @param textNode The text node to use. + * @param resetOffset Whether or not the text offset should be reset. + * @returns true if scanning should continue, or false if the scan length has been reached. + */ + _seekTextNodeForward(textNode, resetOffset) { + const nodeValue = textNode.nodeValue; + const nodeValueLength = nodeValue.length; + const [preserveNewlines, preserveWhitespace] = ( + this._forcePreserveWhitespace ? + [true, true] : + DOMTextScanner.getWhitespaceSettings(textNode) + ); + + let lineHasWhitespace = this._lineHasWhitespace; + let lineHasContent = this._lineHasContent; + let content = this._content; + let offset = resetOffset ? 0 : this._offset; + let remainder = this._remainder; + let newlines = this._newlines; + + while (offset < nodeValueLength) { + const char = nodeValue[offset]; + const charAttributes = DOMTextScanner.getCharacterAttributes(char, preserveNewlines, preserveWhitespace); + ++offset; + + if (charAttributes === 0) { + // Character should be ignored + continue; + } else if (charAttributes === 1) { + // Character is collapsable whitespace + lineHasWhitespace = true; + } else { + // Character should be added to the content + if (newlines > 0) { + if (content.length > 0) { + const useNewlineCount = Math.min(remainder, newlines); + content += '\n'.repeat(useNewlineCount); + remainder -= useNewlineCount; + newlines -= useNewlineCount; + } else { + newlines = 0; + } + lineHasContent = false; + lineHasWhitespace = false; + if (remainder <= 0) { + --offset; // Revert character offset + break; + } + } + + lineHasContent = (charAttributes === 2); // 3 = character is a newline + + if (lineHasWhitespace) { + if (lineHasContent) { + content += ' '; + lineHasWhitespace = false; + if (--remainder <= 0) { + --offset; // Revert character offset + break; + } + } else { + lineHasWhitespace = false; + } + } + + content += char; + + if (--remainder <= 0) { break; } + } + } + + this._lineHasWhitespace = lineHasWhitespace; + this._lineHasContent = lineHasContent; + this._content = content; + this._offset = offset; + this._remainder = remainder; + this._newlines = newlines; + + return (remainder > 0); + } + + /** + * Seeks backward in a text node. + * This function is nearly the same as _seekTextNodeForward, with the following differences: + * - Iteration condition is reversed to check if offset is greater than 0. + * - offset is reset to nodeValueLength instead of 0. + * - offset is decremented instead of incremented. + * - offset is decremented before getting the character. + * - offset is reverted by incrementing instead of decrementing. + * - content string is prepended instead of appended. + * @param textNode The text node to use. + * @param resetOffset Whether or not the text offset should be reset. + * @returns true if scanning should continue, or false if the scan length has been reached. + */ + _seekTextNodeBackward(textNode, resetOffset) { + const nodeValue = textNode.nodeValue; + const nodeValueLength = nodeValue.length; + const [preserveNewlines, preserveWhitespace] = ( + this._forcePreserveWhitespace ? + [true, true] : + DOMTextScanner.getWhitespaceSettings(textNode) + ); + + let lineHasWhitespace = this._lineHasWhitespace; + let lineHasContent = this._lineHasContent; + let content = this._content; + let offset = resetOffset ? nodeValueLength : this._offset; + let remainder = this._remainder; + let newlines = this._newlines; + + while (offset > 0) { + --offset; + const char = nodeValue[offset]; + const charAttributes = DOMTextScanner.getCharacterAttributes(char, preserveNewlines, preserveWhitespace); + + if (charAttributes === 0) { + // Character should be ignored + continue; + } else if (charAttributes === 1) { + // Character is collapsable whitespace + lineHasWhitespace = true; + } else { + // Character should be added to the content + if (newlines > 0) { + if (content.length > 0) { + const useNewlineCount = Math.min(remainder, newlines); + content = '\n'.repeat(useNewlineCount) + content; + remainder -= useNewlineCount; + newlines -= useNewlineCount; + } else { + newlines = 0; + } + lineHasContent = false; + lineHasWhitespace = false; + if (remainder <= 0) { + ++offset; // Revert character offset + break; + } + } + + lineHasContent = (charAttributes === 2); // 3 = character is a newline + + if (lineHasWhitespace) { + if (lineHasContent) { + content = ' ' + content; + lineHasWhitespace = false; + if (--remainder <= 0) { + ++offset; // Revert character offset + break; + } + } else { + lineHasWhitespace = false; + } + } + + content = char + content; + + if (--remainder <= 0) { break; } + } + } + + this._lineHasWhitespace = lineHasWhitespace; + this._lineHasContent = lineHasContent; + this._content = content; + this._offset = offset; + this._remainder = remainder; + this._newlines = newlines; + + return (remainder > 0); + } + + // Static helpers + + /** + * Gets the next node in the document for a specified scanning direction. + * @param node The current DOM Node. + * @param forward Whether to scan forward in the document or backward. + * @param visitChildren Whether the children of the current node should be visited. + * @param exitedNodes An array which stores nodes which were exited. + * @returns The next node in the document, or null if there is no next node. + */ + static getNextNode(node, forward, visitChildren, exitedNodes) { + let next = visitChildren ? (forward ? node.firstChild : node.lastChild) : null; + if (next === null) { + while (true) { + exitedNodes.push(node); + + next = (forward ? node.nextSibling : node.previousSibling); + if (next !== null) { break; } + + next = node.parentNode; + if (next === null) { break; } + + node = next; + } + } + return next; + } + + /** + * Gets the parent element of a given Node. + * @param node The node to check. + * @returns The parent element if one exists, otherwise null. + */ + static getParentElement(node) { + while (node !== null && node.nodeType !== Node.ELEMENT_NODE) { + node = node.parentNode; + } + return node; + } + + /** + * Gets the parent element of a given node, if one exists. For efficiency purposes, + * this only checks the immediate parent elements and does not check all ancestors, so + * there are cases where the node may be in a ruby element but it is not returned. + * @param node The node to check. + * @returns A node if the input node is contained in one, otherwise null. + */ + static getParentRubyElement(node) { + node = DOMTextScanner.getParentElement(node); + if (node !== null && node.nodeName.toUpperCase() === 'RT') { + node = node.parentNode; + if (node !== null && node.nodeName.toUpperCase() === 'RUBY') { + return node; + } + } + return null; + } + + /** + * @returns [enterable: boolean, newlines: integer] + * The enterable value indicates whether the content of this node should be entered. + * The newlines value corresponds to the number of newline characters that should be added. + * 1 newline corresponds to a simple new line in the layout. + * 2 newlines corresponds to a significant visual distinction since the previous content. + */ + static getElementSeekInfo(element) { + let enterable = true; + switch (element.nodeName.toUpperCase()) { + case 'HEAD': + case 'RT': + case 'SCRIPT': + case 'STYLE': + return [false, 0]; + case 'BR': + return [false, 1]; + case 'TEXTAREA': + case 'INPUT': + case 'BUTTON': + enterable = false; + break; + } + + const style = window.getComputedStyle(element); + const display = style.display; + + const visible = (display !== 'none' && DOMTextScanner.isStyleVisible(style)); + let newlines = 0; + + if (!visible) { + enterable = false; + } else { + switch (style.position) { + case 'absolute': + case 'fixed': + case 'sticky': + newlines = 2; + break; + } + if (newlines === 0 && DOMTextScanner.doesCSSDisplayChangeLayout(display)) { + newlines = 1; + } + } + + return [enterable, newlines]; + } + + /** + * Gets information about how whitespace characters are treated. + * @param textNode The Text node to check. + * @returns [preserveNewlines: boolean, preserveWhitespace: boolean] + * The value of preserveNewlines indicates whether or not newline characters are treated as line breaks. + * The value of preserveWhitespace indicates whether or not sequences of whitespace characters are collapsed. + */ + static getWhitespaceSettings(textNode) { + const element = DOMTextScanner.getParentElement(textNode); + if (element !== null) { + const style = window.getComputedStyle(element); + switch (style.whiteSpace) { + case 'pre': + case 'pre-wrap': + case 'break-spaces': + return [true, true]; + case 'pre-line': + return [true, false]; + } + } + return [false, false]; + } + + /** + * Gets attributes for the specified character. + * @param character A string containing a single character. + * @returns An integer representing the attributes of the character. + * 0: Character should be ignored. + * 1: Character is collapsable whitespace. + * 2: Character should be added to the content. + * 3: Character should be added to the content and is a newline. + */ + static getCharacterAttributes(character, preserveNewlines, preserveWhitespace) { + switch (character.charCodeAt(0)) { + case 0x09: // Tab ('\t') + case 0x0c: // Form feed ('\f') + case 0x0d: // Carriage return ('\r') + case 0x20: // Space (' ') + return preserveWhitespace ? 2 : 1; + case 0x0a: // Line feed ('\n') + return preserveNewlines ? 3 : 1; + case 0x200c: // Zero-width non-joiner ('\u200c') + return 0; + default: // Other + return 2; + } + } + + /** + * Checks whether a given style is visible or not. + * This function does not check style.display === 'none'. + * @param style An object implementing the CSSStyleDeclaration interface. + * @returns true if the style should result in an element being visible, otherwise false. + */ + static isStyleVisible(style) { + return !( + style.visibility === 'hidden' || + parseFloat(style.opacity) <= 0 || + parseFloat(style.fontSize) <= 0 || + ( + !DOMTextScanner.isStyleSelectable(style) && + ( + DOMTextScanner.isCSSColorTransparent(style.color) || + DOMTextScanner.isCSSColorTransparent(style.webkitTextFillColor) + ) + ) + ); + } + + /** + * Checks whether a given style is selectable or not. + * @param style An object implementing the CSSStyleDeclaration interface. + * @returns true if the style is selectable, otherwise false. + */ + static isStyleSelectable(style) { + return !( + style.userSelect === 'none' || + style.webkitUserSelect === 'none' || + style.MozUserSelect === 'none' || + style.msUserSelect === 'none' + ); + } + + /** + * Checks whether a CSS color is transparent or not. + * @param cssColor A CSS color string, expected to be encoded in rgb(a) form. + * @returns true if the color is transparent, otherwise false. + */ + static isCSSColorTransparent(cssColor) { + return ( + typeof cssColor === 'string' && + cssColor.startsWith('rgba(') && + /,\s*0.?0*\)$/.test(cssColor) + ); + } + + /** + * Checks whether a CSS display value will cause a layout change for text. + * @param cssDisplay A CSS string corresponding to the value of the display property. + * @returns true if the layout is changed by this value, otherwise false. + */ + static doesCSSDisplayChangeLayout(cssDisplay) { + let pos = cssDisplay.indexOf(' '); + if (pos >= 0) { + // Truncate to part + cssDisplay = cssDisplay.substring(0, pos); + } + + pos = cssDisplay.indexOf('-'); + if (pos >= 0) { + // Truncate to first part of kebab-case value + cssDisplay = cssDisplay.substring(0, pos); + } + + switch (cssDisplay) { + case 'block': + case 'flex': + case 'grid': + case 'list': // list-item + case 'table': // table, table-* + return true; + case 'ruby': // rubt-* + return (pos >= 0); + default: + return false; + } + } +} diff --git a/test/data/html/test-dom-text-scanner.html b/test/data/html/test-dom-text-scanner.html new file mode 100644 index 00000000..6b78570a --- /dev/null +++ b/test/data/html/test-dom-text-scanner.html @@ -0,0 +1,393 @@ + + + + + + Yomichan DOMTextScanner Tests + + + + + +

Yomichan DOMTextScanner Tests

+ + + Layout newlines expected due to entering and exiting display:block nodes. +
小ぢん
まり1
+
小ぢん
まり2
+ + + + Layout newline expected due to sequential display:block elements. +
小ぢんまり1
小ぢんまり2
+ + + + Layout newline expected due to sequential display:block elements separated by a newline. +
小ぢんまり1
+
小ぢんまり2
+ + + + No newlines expected due to display:inline. +小ぢんまり1小ぢんまり2 + + + + No newlines expected due to white-space:normal. +小ぢんまり1 +小ぢんまり2 + + + + Newline expected due to white-space:pre. +
+小ぢんまり1
+小ぢんまり2
+
+ + + + No newlines expected due to display:inline-block. Actual layout flow cannot be determined by DOM/CSS alone. +小ぢんまり1小ぢんまり2 + + + + Single newline expected due to display:block layout. +
小ぢんまり1
小ぢんまり2
+ + + + Two newlines expected due to position:absolute causing a significant layout change. +
小ぢんまり1
小ぢんまり2
+ + + + Two newlines expected due to position:fixed causing a significant layout change. +
小ぢんまり1
小ぢんまり2
+ + + + Two newlines expected due to position:sticky being able to cause a significant layout change. +
小ぢんまり1
小ぢんまり2
+ + + + Scanning text starting in an <rt> element. Should start scanning at the start of the <ruby> tag instead. +
小(こ)ぢんまり1
+ + + + Skip <script> content. +
小ぢんまり1
+ + + + Skip <style> content. +
小ぢんまり1
+ + + + Skip <textarea> content. +
小ぢんまり1
+ + + + Skip <input> content. +
小ぢんまり1
+ + + + Skip <button> content. +
小ぢんまり1
+ + + + Skip content with font-size:0. +
小ぢんcontentまり1
+ + + + Skip content with opacity:0. +
小ぢんcontentまり1
+ + + + Skip content with visibility:hidden. +
小ぢんcontentまり1
+ + + + Skip content with display:none. +
小ぢんcontentまり1
+ + + + Don't skip content with user-select:none. +
小ぢんまり1
+ + + + Skip content with user-select:none and a transparent color. +
小ぢんcontentまり1
+ + + + \ No newline at end of file diff --git a/test/data/html/test-stylesheet.css b/test/data/html/test-stylesheet.css index f63d2481..2e9a2f52 100644 --- a/test/data/html/test-stylesheet.css +++ b/test/data/html/test-stylesheet.css @@ -28,7 +28,9 @@ a, a:visited { text-decoration: underline; } -.test { +.test, +y-test { + display: block; background-color: #ffffff; margin: 1em 0; padding: 0.5em; @@ -36,7 +38,8 @@ a, a:visited { border-radius: 4px; } -.test:before { +.test:before, +y-test:before { content: "Test " counter(test-id); display: block; counter-increment: test-id; @@ -45,7 +48,10 @@ a, a:visited { font-weight: bold; } -.description { +.description, +y-description { color: #444444; font-style: italic; + display: block; + padding-bottom: 0.5em; } diff --git a/test/test-dom-text-scanner.js b/test/test-dom-text-scanner.js new file mode 100644 index 00000000..41d6e307 --- /dev/null +++ b/test/test-dom-text-scanner.js @@ -0,0 +1,181 @@ +/* + * Copyright (C) 2020 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +const fs = require('fs'); +const path = require('path'); +const assert = require('assert'); +const {JSDOM} = require('jsdom'); +const {VM} = require('./yomichan-vm'); + + +function createJSDOM(fileName) { + const domSource = fs.readFileSync(fileName, {encoding: 'utf8'}); + return new JSDOM(domSource); +} + +function querySelectorTextNode(element, selector) { + let textIndex = -1; + const match = /::text$|::nth-text\((\d+)\)$/.exec(selector); + if (match !== null) { + textIndex = (match[1] ? parseInt(match[1], 10) - 1 : 0); + selector = selector.substring(0, selector.length - match[0].length); + } + const result = element.querySelector(selector); + if (textIndex < 0) { + return result; + } + for (let n = result.firstChild; n !== null; n = n.nextSibling) { + if (n.nodeType === n.constructor.TEXT_NODE) { + if (textIndex === 0) { + return n; + } + --textIndex; + } + } + return null; +} + + +function getComputedFontSizeInPixels(window, getComputedStyle, element) { + for (; element !== null; element = element.parentNode) { + if (element.nodeType === window.Node.ELEMENT_NODE) { + const fontSize = getComputedStyle(element).fontSize; + if (fontSize.endsWith('px')) { + const value = parseFloat(fontSize.substring(0, fontSize.length - 2)); + return value; + } + } + } + const defaultFontSize = 14; + return defaultFontSize; +} + +function createAbsoluteGetComputedStyle(window) { + // Wrapper to convert em units to px units + const getComputedStyleOld = window.getComputedStyle.bind(window); + return (element, ...args) => { + const style = getComputedStyleOld(element, ...args); + return new Proxy(style, { + get: (target, property) => { + let result = target[property]; + if (typeof result === 'string') { + result = result.replace(/([-+]?\d(?:\.\d)?(?:[eE][-+]?\d+)?)em/g, (g0, g1) => { + const fontSize = getComputedFontSizeInPixels(window, getComputedStyleOld, element); + return `${parseFloat(g1) * fontSize}px`; + }); + } + return result; + } + }); + }; +} + + +async function testDomTextScanner(dom, {DOMTextScanner}) { + const document = dom.window.document; + for (const testElement of document.querySelectorAll('y-test')) { + let testData = JSON.parse(testElement.dataset.testData); + if (!Array.isArray(testData)) { + testData = [testData]; + } + for (const testDataItem of testData) { + let { + node, + offset, + length, + forcePreserveWhitespace, + generateLayoutContent, + reversible, + expected: { + node: expectedNode, + offset: expectedOffset, + content: expectedContent + } + } = testDataItem; + + node = querySelectorTextNode(testElement, node); + expectedNode = querySelectorTextNode(testElement, expectedNode); + + // Standard test + { + const scanner = new DOMTextScanner(node, offset, forcePreserveWhitespace, generateLayoutContent); + scanner.seek(length); + + const {node: actualNode1, offset: actualOffset1, content: actualContent1} = scanner; + assert.strictEqual(actualContent1, expectedContent); + assert.strictEqual(actualOffset1, expectedOffset); + assert.strictEqual(actualNode1, expectedNode); + } + + // Substring tests + for (let i = 1; i <= length; ++i) { + const scanner = new DOMTextScanner(node, offset, forcePreserveWhitespace, generateLayoutContent); + scanner.seek(length - i); + + const {content: actualContent} = scanner; + assert.strictEqual(actualContent, expectedContent.substring(0, expectedContent.length - i)); + } + + if (reversible === false) { continue; } + + // Reversed test + { + const scanner = new DOMTextScanner(expectedNode, expectedOffset, forcePreserveWhitespace, generateLayoutContent); + scanner.seek(-length); + + const {content: actualContent} = scanner; + assert.strictEqual(actualContent, expectedContent); + } + + // Reversed substring tests + for (let i = 1; i <= length; ++i) { + const scanner = new DOMTextScanner(expectedNode, expectedOffset, forcePreserveWhitespace, generateLayoutContent); + scanner.seek(-(length - i)); + + const {content: actualContent} = scanner; + assert.strictEqual(actualContent, expectedContent.substring(i)); + } + } + } +} + + +async function testDocument1() { + const dom = createJSDOM(path.join(__dirname, 'data', 'html', 'test-dom-text-scanner.html')); + const window = dom.window; + try { + const {document, Node, Range} = window; + + window.getComputedStyle = createAbsoluteGetComputedStyle(window); + + const vm = new VM({document, window, Range, Node}); + vm.execute('fg/js/dom-text-scanner.js'); + const DOMTextScanner = vm.get('DOMTextScanner'); + + await testDomTextScanner(dom, {DOMTextScanner}); + } finally { + window.close(); + } +} + + +async function main() { + await testDocument1(); +} + + +if (require.main === module) { main(); } -- cgit v1.2.3 From 021ccb5ac3038f63d07ccc9575ee56480031a251 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Wed, 6 May 2020 19:28:26 -0400 Subject: Move util database modification functions (#499) * Update onProgress callback to handle multiple arguments * Add apiImportDictionaryArchive * Add apiDeleteDictionary * Make onProgress the last argument for consistency * Remove deprecated util functions * Fix issue with missing progress args * Remove function calls which modify the database from Translator * Update tests * Fix errors not being serialized correctly in _createActionListenerPort --- ext/bg/js/backend.js | 23 +++++++++++++++++++---- ext/bg/js/database.js | 2 +- ext/bg/js/dictionary-importer.js | 2 +- ext/bg/js/settings/dictionaries.js | 18 ++++++++++++++---- ext/bg/js/translator.js | 8 +------- ext/bg/js/util.js | 25 ------------------------- ext/mixed/js/api.js | 10 +++++++++- test/test-database.js | 18 +++++++++--------- 8 files changed, 54 insertions(+), 52 deletions(-) (limited to 'test') diff --git a/ext/bg/js/backend.js b/ext/bg/js/backend.js index c5173a2e..d454aa22 100644 --- a/ext/bg/js/backend.js +++ b/ext/bg/js/backend.js @@ -117,7 +117,10 @@ class Backend { ['logIndicatorClear', {handler: this._onApiLogIndicatorClear.bind(this), async: false}], ['createActionPort', {handler: this._onApiCreateActionPort.bind(this), async: false}] ]); - this._messageHandlersWithProgress = new Map(); + this._messageHandlersWithProgress = new Map([ + ['importDictionaryArchive', {handler: this._onApiImportDictionaryArchive.bind(this), async: true}], + ['deleteDictionary', {handler: this._onApiDeleteDictionary.bind(this), async: true}] + ]); this._commandHandlers = new Map([ ['search', this._onCommandSearch.bind(this)], @@ -771,7 +774,8 @@ class Backend { async _onApiPurgeDatabase(params, sender) { this._validatePrivilegedMessageSender(sender); - return await this.translator.purgeDatabase(); + this.translator.clearDatabaseCaches(); + await this.database.purge(); } async _onApiGetMedia({targets}) { @@ -814,12 +818,23 @@ class Backend { return portName; } + async _onApiImportDictionaryArchive({archiveContent, details}, sender, onProgress) { + this._validatePrivilegedMessageSender(sender); + return await this.dictionaryImporter.import(this.database, archiveContent, details, onProgress); + } + + async _onApiDeleteDictionary({dictionaryName}, sender, onProgress) { + this._validatePrivilegedMessageSender(sender); + this.translator.clearDatabaseCaches(); + await this.database.deleteDictionary(dictionaryName, {rate: 1000}, onProgress); + } + // Command handlers _createActionListenerPort(port, sender, handlers) { let hasStarted = false; - const onProgress = (data) => { + const onProgress = (...data) => { try { if (port === null) { return; } port.postMessage({type: 'progress', data}); @@ -847,7 +862,7 @@ class Backend { port.postMessage({type: 'complete', data: result}); } catch (e) { if (port !== null) { - port.postMessage({type: 'error', data: e}); + port.postMessage({type: 'error', data: errorToJson(e)}); } cleanup(); } diff --git a/ext/bg/js/database.js b/ext/bg/js/database.js index a94aa720..930cd0d0 100644 --- a/ext/bg/js/database.js +++ b/ext/bg/js/database.js @@ -129,7 +129,7 @@ class Database { await this.prepare(); } - async deleteDictionary(dictionaryName, onProgress, progressSettings) { + async deleteDictionary(dictionaryName, progressSettings, onProgress) { this._validate(); const targets = [ diff --git a/ext/bg/js/dictionary-importer.js b/ext/bg/js/dictionary-importer.js index 3727f7ee..10e30cec 100644 --- a/ext/bg/js/dictionary-importer.js +++ b/ext/bg/js/dictionary-importer.js @@ -27,7 +27,7 @@ class DictionaryImporter { this._schemas = new Map(); } - async import(database, archiveSource, onProgress, details) { + async import(database, archiveSource, details, onProgress) { if (!database) { throw new Error('Invalid database'); } diff --git a/ext/bg/js/settings/dictionaries.js b/ext/bg/js/settings/dictionaries.js index 50add4c7..632c01ea 100644 --- a/ext/bg/js/settings/dictionaries.js +++ b/ext/bg/js/settings/dictionaries.js @@ -17,8 +17,10 @@ /* global * PageExitPrevention + * apiDeleteDictionary * apiGetDictionaryCounts * apiGetDictionaryInfo + * apiImportDictionaryArchive * apiOptionsGet * apiOptionsGetFull * apiPurgeDatabase @@ -29,8 +31,6 @@ * storageEstimate * storageUpdateStats * utilBackgroundIsolate - * utilDatabaseDeleteDictionary - * utilDatabaseImport */ let dictionaryUI = null; @@ -312,7 +312,7 @@ class SettingsDictionaryEntryUI { progressBar.style.width = `${percent}%`; }; - await utilDatabaseDeleteDictionary(this.dictionaryInfo.title, onProgress, {rate: 1000}); + await apiDeleteDictionary(this.dictionaryInfo.title, onProgress); } catch (e) { dictionaryErrorsShow([e]); } finally { @@ -679,7 +679,8 @@ async function onDictionaryImport(e) { dictImportInfo.textContent = `(${i + 1} of ${ii})`; } - const {result, errors} = await utilDatabaseImport(files[i], updateProgress, importDetails); + const archiveContent = await dictReadFile(files[i]); + const {result, errors} = await apiImportDictionaryArchive(archiveContent, importDetails, updateProgress); for (const {options} of toIterable((await getOptionsFullMutable()).profiles)) { const dictionaryOptions = SettingsDictionaryListUI.createDictionaryOptions(); dictionaryOptions.enabled = true; @@ -713,6 +714,15 @@ async function onDictionaryImport(e) { } } +function dictReadFile(file) { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => resolve(reader.result); + reader.onerror = () => reject(reader.error); + reader.readAsBinaryString(file); + }); +} + async function onDatabaseEnablePrefixWildcardSearchesChanged(e) { const optionsFull = await getOptionsFullMutable(); diff --git a/ext/bg/js/translator.js b/ext/bg/js/translator.js index 8708e4d8..3fd329d1 100644 --- a/ext/bg/js/translator.js +++ b/ext/bg/js/translator.js @@ -45,14 +45,8 @@ class Translator { this.deinflector = new Deinflector(reasons); } - async purgeDatabase() { + clearDatabaseCaches() { this.tagCache.clear(); - await this.database.purge(); - } - - async deleteDictionary(dictionaryName) { - this.tagCache.clear(); - await this.database.deleteDictionary(dictionaryName); } async getSequencedDefinitions(definitions, mainDictionary) { diff --git a/ext/bg/js/util.js b/ext/bg/js/util.js index d2fb0e49..8f86e47a 100644 --- a/ext/bg/js/util.js +++ b/ext/bg/js/util.js @@ -66,31 +66,6 @@ function utilBackend() { return backend; } -async function utilDatabaseDeleteDictionary(dictionaryName, onProgress) { - return utilIsolate(await utilBackend().translator.database.deleteDictionary( - utilBackgroundIsolate(dictionaryName), - utilBackgroundFunctionIsolate(onProgress) - )); -} - -async function utilDatabaseImport(data, onProgress, details) { - data = await utilReadFile(data); - return utilIsolate(await utilBackend().importDictionary( - utilBackgroundIsolate(data), - utilBackgroundFunctionIsolate(onProgress), - utilBackgroundIsolate(details) - )); -} - -function utilReadFile(file) { - return new Promise((resolve, reject) => { - const reader = new FileReader(); - reader.onload = () => resolve(reader.result); - reader.onerror = () => reject(reader.error); - reader.readAsBinaryString(file); - }); -} - function utilReadFileArrayBuffer(file) { return new Promise((resolve, reject) => { const reader = new FileReader(); diff --git a/ext/mixed/js/api.js b/ext/mixed/js/api.js index ca4bdd6c..af97ac3d 100644 --- a/ext/mixed/js/api.js +++ b/ext/mixed/js/api.js @@ -152,6 +152,14 @@ function apiLogIndicatorClear() { return _apiInvoke('logIndicatorClear'); } +function apiImportDictionaryArchive(archiveContent, details, onProgress) { + return _apiInvokeWithProgress('importDictionaryArchive', {archiveContent, details}, onProgress); +} + +function apiDeleteDictionary(dictionaryName, onProgress) { + return _apiInvokeWithProgress('deleteDictionary', {dictionaryName}, onProgress); +} + function _apiCreateActionPort(timeout=5000) { return new Promise((resolve, reject) => { let timer = null; @@ -213,7 +221,7 @@ function _apiInvokeWithProgress(action, params, onProgress, timeout=5000) { break; case 'progress': try { - onProgress(message.data); + onProgress(...message.data); } catch (e) { // NOP } diff --git a/test/test-database.js b/test/test-database.js index 3684051b..e8a4a343 100644 --- a/test/test-database.js +++ b/test/test-database.js @@ -233,10 +233,10 @@ async function testDatabase1() { let progressEvent = false; await database.deleteDictionary( title, + {rate: 1000}, () => { progressEvent = true; - }, - {rate: 1000} + } ); assert.ok(progressEvent); @@ -267,10 +267,10 @@ async function testDatabase1() { const {result, errors} = await dictionaryImporter.import( database, testDictionarySource, + {prefixWildcardsSupported: true}, () => { progressEvent = true; - }, - {prefixWildcardsSupported: true} + } ); vm.assert.deepStrictEqual(errors, []); vm.assert.deepStrictEqual(result, expectedSummary); @@ -908,7 +908,7 @@ async function testDatabase2() { // Error: not prepared await assert.rejects(async () => await database.purge()); - await assert.rejects(async () => await database.deleteDictionary(title, () => {}, {})); + await assert.rejects(async () => await database.deleteDictionary(title, {}, () => {})); await assert.rejects(async () => await database.findTermsBulk(['?'], titles, null)); await assert.rejects(async () => await database.findTermsExactBulk(['?'], ['?'], titles)); await assert.rejects(async () => await database.findTermsBySequenceBulk([1], title)); @@ -919,17 +919,17 @@ async function testDatabase2() { await assert.rejects(async () => await database.findTagForTitle('tag', title)); await assert.rejects(async () => await database.getDictionaryInfo()); await assert.rejects(async () => await database.getDictionaryCounts(titles, true)); - await assert.rejects(async () => await dictionaryImporter.import(database, testDictionarySource, () => {}, {})); + await assert.rejects(async () => await dictionaryImporter.import(database, testDictionarySource, {}, () => {})); await database.prepare(); // Error: already prepared await assert.rejects(async () => await database.prepare()); - await dictionaryImporter.import(database, testDictionarySource, () => {}, {}); + await dictionaryImporter.import(database, testDictionarySource, {}, () => {}); // Error: dictionary already imported - await assert.rejects(async () => await dictionaryImporter.import(database, testDictionarySource, () => {}, {})); + await assert.rejects(async () => await dictionaryImporter.import(database, testDictionarySource, {}, () => {})); await database.close(); } @@ -956,7 +956,7 @@ async function testDatabase3() { let error = null; try { - await dictionaryImporter.import(database, testDictionarySource, () => {}, {}); + await dictionaryImporter.import(database, testDictionarySource, {}, () => {}); } catch (e) { error = e; } -- cgit v1.2.3 From f7df6254d6f71d5331b000dcbd27271bd2c3006f Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Wed, 6 May 2020 19:33:17 -0400 Subject: Dom text scanner fixes (#505) * Fix test case * Add test-dom-text-scanner * Ensure that DOMTextScanner._node never becomes null * Add remainder --- ext/fg/js/dom-text-scanner.js | 15 ++++++++++++++- package.json | 2 +- test/data/html/test-dom-text-scanner.html | 2 +- test/test-dom-text-scanner.js | 6 ++++-- 4 files changed, 20 insertions(+), 5 deletions(-) (limited to 'test') diff --git a/ext/fg/js/dom-text-scanner.js b/ext/fg/js/dom-text-scanner.js index 2de65041..8fa67ede 100644 --- a/ext/fg/js/dom-text-scanner.js +++ b/ext/fg/js/dom-text-scanner.js @@ -59,6 +59,15 @@ class DOMTextScanner { return this._offset; } + /** + * Gets the remaining number of characters that weren't scanned in the last seek() call. + * This value is usually 0 unless the end of the document was reached. + * @returns An integer. + */ + get remainder() { + return this._remainder; + } + /** * Gets the accumulated content string resulting from calls to seek(). * @returns A string. @@ -85,6 +94,7 @@ class DOMTextScanner { const generateLayoutContent = this._generateLayoutContent; let node = this._node; + let lastNode = node; let resetOffset = this._resetOffset; let newlines = 0; while (node !== null) { @@ -92,6 +102,7 @@ class DOMTextScanner { const nodeType = node.nodeType; if (nodeType === TEXT_NODE) { + lastNode = node; if (!( forward ? this._seekTextNodeForward(node, resetOffset) : @@ -101,6 +112,8 @@ class DOMTextScanner { break; } } else if (nodeType === ELEMENT_NODE) { + lastNode = node; + this._offset = 0; [enterable, newlines] = DOMTextScanner.getElementSeekInfo(node); if (newlines > this._newlines && generateLayoutContent) { this._newlines = newlines; @@ -121,7 +134,7 @@ class DOMTextScanner { resetOffset = true; } - this._node = node; + this._node = lastNode; this._resetOffset = resetOffset; return this; diff --git a/package.json b/package.json index 0729cda1..1aa6c856 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "scripts": { "test": "npm run test-lint && npm run test-code", "test-lint": "eslint . && node ./test/lint/global-declarations.js", - "test-code": "node ./test/test-schema.js && node ./test/test-dictionary.js && node ./test/test-database.js && node ./test/test-document.js && node ./test/test-object-property-accessor.js && node ./test/test-japanese.js && node ./test/test-text-source-map.js" + "test-code": "node ./test/test-schema.js && node ./test/test-dictionary.js && node ./test/test-database.js && node ./test/test-document.js && node ./test/test-object-property-accessor.js && node ./test/test-japanese.js && node ./test/test-text-source-map.js && node ./test/test-dom-text-scanner.js" }, "repository": { "type": "git", diff --git a/test/data/html/test-dom-text-scanner.html b/test/data/html/test-dom-text-scanner.html index 6b78570a..dc06eb64 100644 --- a/test/data/html/test-dom-text-scanner.html +++ b/test/data/html/test-dom-text-scanner.html @@ -85,7 +85,7 @@ "expected": { "node": "span:nth-of-type(2)::text", "offset": 6, - "content": "小ぢんまり1\n小ぢんまり2" + "content": "小ぢんまり1 小ぢんまり2" } }' > diff --git a/test/test-dom-text-scanner.js b/test/test-dom-text-scanner.js index 41d6e307..7374ff87 100644 --- a/test/test-dom-text-scanner.js +++ b/test/test-dom-text-scanner.js @@ -103,7 +103,8 @@ async function testDomTextScanner(dom, {DOMTextScanner}) { expected: { node: expectedNode, offset: expectedOffset, - content: expectedContent + content: expectedContent, + remainder: expectedRemainder } } = testDataItem; @@ -115,10 +116,11 @@ async function testDomTextScanner(dom, {DOMTextScanner}) { const scanner = new DOMTextScanner(node, offset, forcePreserveWhitespace, generateLayoutContent); scanner.seek(length); - const {node: actualNode1, offset: actualOffset1, content: actualContent1} = scanner; + const {node: actualNode1, offset: actualOffset1, content: actualContent1, remainder: actualRemainder1} = scanner; assert.strictEqual(actualContent1, expectedContent); assert.strictEqual(actualOffset1, expectedOffset); assert.strictEqual(actualNode1, expectedNode); + assert.strictEqual(actualRemainder1, expectedRemainder || 0); } // Substring tests -- cgit v1.2.3 From d6ae3229614fb11fee799387d17347ea97509c59 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Wed, 6 May 2020 19:34:32 -0400 Subject: Text scanning update (#507) * Fix unity test missing a parameter * Update docSentenceExtract to not rescan content --- ext/fg/js/document.js | 2 +- ext/fg/js/source.js | 10 +++++++--- test/data/html/test-document1.html | 1 + 3 files changed, 9 insertions(+), 4 deletions(-) (limited to 'test') diff --git a/ext/fg/js/document.js b/ext/fg/js/document.js index 6103c7c5..d639bc86 100644 --- a/ext/fg/js/document.js +++ b/ext/fg/js/document.js @@ -159,7 +159,7 @@ function docSentenceExtract(source, extent) { const sourceLocal = source.clone(); const position = sourceLocal.setStartOffset(extent); - sourceLocal.setEndOffset(position + extent); + sourceLocal.setEndOffset(extent * 2 - position, true); const content = sourceLocal.text(); let quoteStack = []; diff --git a/ext/fg/js/source.js b/ext/fg/js/source.js index b3119d40..c9d70215 100644 --- a/ext/fg/js/source.js +++ b/ext/fg/js/source.js @@ -46,10 +46,14 @@ class TextSourceRange { return this.content; } - setEndOffset(length) { - const state = TextSourceRange.seekForward(this.range.startContainer, this.range.startOffset, length); + setEndOffset(length, fromEnd=false) { + const state = ( + fromEnd ? + TextSourceRange.seekForward(this.range.endContainer, this.range.endOffset, length) : + TextSourceRange.seekForward(this.range.startContainer, this.range.startOffset, length) + ); this.range.setEnd(state.node, state.offset); - this.content = state.content; + this.content = (fromEnd ? this.content + state.content : state.content); return length - state.remainder; } diff --git a/test/data/html/test-document1.html b/test/data/html/test-document1.html index 0754a314..98a6fb44 100644 --- a/test/data/html/test-document1.html +++ b/test/data/html/test-document1.html @@ -103,6 +103,7 @@ data-end-node-selector="img" data-end-offset="0" data-result-type="TextSourceElement" + data-sentence-extent="100" data-sentence="よみちゃん" > よみちゃん -- cgit v1.2.3 From f361139d744e58a6c33841cee227d13d1970bb98 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Wed, 6 May 2020 19:37:36 -0400 Subject: Japanese util refactor (#510) * Convert mixed japanese.js to utility class * Copy functions from bg/js/japanese.js into mixed/js/japanese.js * Remove bg/js/japanese.js * Make wanakana dependency optional * Update tests --- ext/bg/background.html | 1 - ext/bg/js/japanese.js | 426 --------------------------------------- ext/bg/search.html | 1 - ext/bg/settings.html | 1 - ext/mixed/js/japanese.js | 507 ++++++++++++++++++++++++++++++++++++++++++----- test/test-japanese.js | 3 +- 6 files changed, 453 insertions(+), 486 deletions(-) delete mode 100644 ext/bg/js/japanese.js (limited to 'test') diff --git a/ext/bg/background.html b/ext/bg/background.html index 9c740adf..7cb76ec3 100644 --- a/ext/bg/background.html +++ b/ext/bg/background.html @@ -36,7 +36,6 @@ - diff --git a/ext/bg/js/japanese.js b/ext/bg/js/japanese.js deleted file mode 100644 index ac81acb5..00000000 --- a/ext/bg/js/japanese.js +++ /dev/null @@ -1,426 +0,0 @@ -/* - * Copyright (C) 2016-2020 Yomichan Authors - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -/* global - * jp - * wanakana - */ - -(() => { - const HALFWIDTH_KATAKANA_MAPPING = new Map([ - ['ｦ', 'ヲヺ-'], - ['ｧ', 'ァ--'], - ['ｨ', 'ィ--'], - ['ｩ', 'ゥ--'], - ['ｪ', 'ェ--'], - ['ｫ', 'ォ--'], - ['ｬ', 'ャ--'], - ['ｭ', 'ュ--'], - ['ｮ', 'ョ--'], - ['ｯ', 'ッ--'], - ['ｰ', 'ー--'], - ['ｱ', 'ア--'], - ['ｲ', 'イ--'], - ['ｳ', 'ウヴ-'], - ['ｴ', 'エ--'], - ['ｵ', 'オ--'], - ['ｶ', 'カガ-'], - ['ｷ', 'キギ-'], - ['ｸ', 'クグ-'], - ['ｹ', 'ケゲ-'], - ['ｺ', 'コゴ-'], - ['ｻ', 'サザ-'], - ['ｼ', 'シジ-'], - ['ｽ', 'スズ-'], - ['ｾ', 'セゼ-'], - ['ｿ', 'ソゾ-'], - ['ﾀ', 'タダ-'], - ['ﾁ', 'チヂ-'], - ['ﾂ', 'ツヅ-'], - ['ﾃ', 'テデ-'], - ['ﾄ', 'トド-'], - ['ﾅ', 'ナ--'], - ['ﾆ', 'ニ--'], - ['ﾇ', 'ヌ--'], - ['ﾈ', 'ネ--'], - ['ﾉ', 'ノ--'], - ['ﾊ', 'ハバパ'], - ['ﾋ', 'ヒビピ'], - ['ﾌ', 'フブプ'], - ['ﾍ', 'ヘベペ'], - ['ﾎ', 'ホボポ'], - ['ﾏ', 'マ--'], - ['ﾐ', 'ミ--'], - ['ﾑ', 'ム--'], - ['ﾒ', 'メ--'], - ['ﾓ', 'モ--'], - ['ﾔ', 'ヤ--'], - ['ﾕ', 'ユ--'], - ['ﾖ', 'ヨ--'], - ['ﾗ', 'ラ--'], - ['ﾘ', 'リ--'], - ['ﾙ', 'ル--'], - ['ﾚ', 'レ--'], - ['ﾛ', 'ロ--'], - ['ﾜ', 'ワ--'], - ['ﾝ', 'ン--'] - ]); - - const ITERATION_MARK_CODE_POINT = 0x3005; - - const HIRAGANA_SMALL_TSU_CODE_POINT = 0x3063; - const KATAKANA_SMALL_TSU_CODE_POINT = 0x30c3; - const KANA_PROLONGED_SOUND_MARK_CODE_POINT = 0x30fc; - - // Existing functions - - const isCodePointKanji = jp.isCodePointKanji; - const isStringEntirelyKana = jp.isStringEntirelyKana; - - - // Conversion functions - - function convertKatakanaToHiragana(text) { - let result = ''; - for (const c of text) { - if (wanakana.isKatakana(c)) { - result += wanakana.toHiragana(c); - } else { - result += c; - } - } - - return result; - } - - function convertHiraganaToKatakana(text) { - let result = ''; - for (const c of text) { - if (wanakana.isHiragana(c)) { - result += wanakana.toKatakana(c); - } else { - result += c; - } - } - - return result; - } - - function convertToRomaji(text) { - return wanakana.toRomaji(text); - } - - function convertReading(expression, reading, readingMode) { - switch (readingMode) { - case 'hiragana': - return convertKatakanaToHiragana(reading); - case 'katakana': - return convertHiraganaToKatakana(reading); - case 'romaji': - if (reading) { - return convertToRomaji(reading); - } else { - if (isStringEntirelyKana(expression)) { - return convertToRomaji(expression); - } - } - return reading; - case 'none': - return ''; - default: - return reading; - } - } - - function convertNumericToFullWidth(text) { - let result = ''; - for (const char of text) { - let c = char.codePointAt(0); - if (c >= 0x30 && c <= 0x39) { // ['0', '9'] - c += 0xff10 - 0x30; // 0xff10 = '0' full width - result += String.fromCodePoint(c); - } else { - result += char; - } - } - return result; - } - - function convertHalfWidthKanaToFullWidth(text, sourceMap=null) { - let result = ''; - - // This function is safe to use charCodeAt instead of codePointAt, since all - // the relevant characters are represented with a single UTF-16 character code. - for (let i = 0, ii = text.length; i < ii; ++i) { - const c = text[i]; - const mapping = HALFWIDTH_KATAKANA_MAPPING.get(c); - if (typeof mapping !== 'string') { - result += c; - continue; - } - - let index = 0; - switch (text.charCodeAt(i + 1)) { - case 0xff9e: // dakuten - index = 1; - break; - case 0xff9f: // handakuten - index = 2; - break; - } - - let c2 = mapping[index]; - if (index > 0) { - if (c2 === '-') { // invalid - index = 0; - c2 = mapping[0]; - } else { - ++i; - } - } - - if (sourceMap !== null && index > 0) { - sourceMap.combine(result.length, 1); - } - result += c2; - } - - return result; - } - - function convertAlphabeticToKana(text, sourceMap=null) { - let part = ''; - let result = ''; - - for (const char of text) { - // Note: 0x61 is the character code for 'a' - let c = char.codePointAt(0); - if (c >= 0x41 && c <= 0x5a) { // ['A', 'Z'] - c += (0x61 - 0x41); - } else if (c >= 0x61 && c <= 0x7a) { // ['a', 'z'] - // NOP; c += (0x61 - 0x61); - } else if (c >= 0xff21 && c <= 0xff3a) { // ['A', 'Z'] fullwidth - c += (0x61 - 0xff21); - } else if (c >= 0xff41 && c <= 0xff5a) { // ['a', 'z'] fullwidth - c += (0x61 - 0xff41); - } else if (c === 0x2d || c === 0xff0d) { // '-' or fullwidth dash - c = 0x2d; // '-' - } else { - if (part.length > 0) { - result += convertAlphabeticPartToKana(part, sourceMap, result.length); - part = ''; - } - result += char; - continue; - } - part += String.fromCodePoint(c); - } - - if (part.length > 0) { - result += convertAlphabeticPartToKana(part, sourceMap, result.length); - } - return result; - } - - function convertAlphabeticPartToKana(text, sourceMap, sourceMapStart) { - const result = wanakana.toHiragana(text); - - // Generate source mapping - if (sourceMap !== null) { - let i = 0; - let resultPos = 0; - const ii = text.length; - while (i < ii) { - // Find smallest matching substring - let iNext = i + 1; - let resultPosNext = result.length; - while (iNext < ii) { - const t = wanakana.toHiragana(text.substring(0, iNext)); - if (t === result.substring(0, t.length)) { - resultPosNext = t.length; - break; - } - ++iNext; - } - - // Merge characters - const removals = iNext - i - 1; - if (removals > 0) { - sourceMap.combine(sourceMapStart, removals); - } - ++sourceMapStart; - - // Empty elements - const additions = resultPosNext - resultPos - 1; - for (let j = 0; j < additions; ++j) { - sourceMap.insert(sourceMapStart, 0); - ++sourceMapStart; - } - - i = iNext; - resultPos = resultPosNext; - } - } - - return result; - } - - - // Furigana distribution - - function distributeFurigana(expression, reading) { - const fallback = [{furigana: reading, text: expression}]; - if (!reading) { - return fallback; - } - - let isAmbiguous = false; - const segmentize = (reading2, groups) => { - if (groups.length === 0 || isAmbiguous) { - return []; - } - - const group = groups[0]; - if (group.mode === 'kana') { - if (convertKatakanaToHiragana(reading2).startsWith(convertKatakanaToHiragana(group.text))) { - const readingLeft = reading2.substring(group.text.length); - const segs = segmentize(readingLeft, groups.splice(1)); - if (segs) { - return [{text: group.text, furigana: ''}].concat(segs); - } - } - } else { - let foundSegments = null; - for (let i = reading2.length; i >= group.text.length; --i) { - const readingUsed = reading2.substring(0, i); - const readingLeft = reading2.substring(i); - const segs = segmentize(readingLeft, groups.slice(1)); - if (segs) { - if (foundSegments !== null) { - // more than one way to segmentize the tail, mark as ambiguous - isAmbiguous = true; - return null; - } - foundSegments = [{text: group.text, furigana: readingUsed}].concat(segs); - } - // there is only one way to segmentize the last non-kana group - if (groups.length === 1) { - break; - } - } - return foundSegments; - } - }; - - const groups = []; - let modePrev = null; - for (const c of expression) { - const codePoint = c.codePointAt(0); - const modeCurr = isCodePointKanji(codePoint) || codePoint === ITERATION_MARK_CODE_POINT ? 'kanji' : 'kana'; - if (modeCurr === modePrev) { - groups[groups.length - 1].text += c; - } else { - groups.push({mode: modeCurr, text: c}); - modePrev = modeCurr; - } - } - - const segments = segmentize(reading, groups); - if (segments && !isAmbiguous) { - return segments; - } - return fallback; - } - - function distributeFuriganaInflected(expression, reading, source) { - const output = []; - - let stemLength = 0; - const shortest = Math.min(source.length, expression.length); - const sourceHiragana = convertKatakanaToHiragana(source); - const expressionHiragana = convertKatakanaToHiragana(expression); - while (stemLength < shortest && sourceHiragana[stemLength] === expressionHiragana[stemLength]) { - ++stemLength; - } - const offset = source.length - stemLength; - - const stemExpression = source.substring(0, source.length - offset); - const stemReading = reading.substring( - 0, - offset === 0 ? reading.length : reading.length - expression.length + stemLength - ); - for (const segment of distributeFurigana(stemExpression, stemReading)) { - output.push(segment); - } - - if (stemLength !== source.length) { - output.push({text: source.substring(stemLength), furigana: ''}); - } - - return output; - } - - - // Miscellaneous - - function collapseEmphaticSequences(text, fullCollapse, sourceMap=null) { - let result = ''; - let collapseCodePoint = -1; - const hasSourceMap = (sourceMap !== null); - for (const char of text) { - const c = char.codePointAt(0); - if ( - c === HIRAGANA_SMALL_TSU_CODE_POINT || - c === KATAKANA_SMALL_TSU_CODE_POINT || - c === KANA_PROLONGED_SOUND_MARK_CODE_POINT - ) { - if (collapseCodePoint !== c) { - collapseCodePoint = c; - if (!fullCollapse) { - result += char; - continue; - } - } - } else { - collapseCodePoint = -1; - result += char; - continue; - } - - if (hasSourceMap) { - sourceMap.combine(Math.max(0, result.length - 1), 1); - } - } - return result; - } - - - // Exports - - Object.assign(jp, { - convertKatakanaToHiragana, - convertHiraganaToKatakana, - convertToRomaji, - convertReading, - convertNumericToFullWidth, - convertHalfWidthKanaToFullWidth, - convertAlphabeticToKana, - distributeFurigana, - distributeFuriganaInflected, - collapseEmphaticSequences - }); -})(); diff --git a/ext/bg/search.html b/ext/bg/search.html index 52915b76..a30b1d60 100644 --- a/ext/bg/search.html +++ b/ext/bg/search.html @@ -79,7 +79,6 @@ - diff --git a/ext/bg/settings.html b/ext/bg/settings.html index b8477e46..a0981687 100644 --- a/ext/bg/settings.html +++ b/ext/bg/settings.html @@ -1139,7 +1139,6 @@ - diff --git a/ext/mixed/js/japanese.js b/ext/mixed/js/japanese.js index ced486dd..801dec84 100644 --- a/ext/mixed/js/japanese.js +++ b/ext/mixed/js/japanese.js @@ -16,6 +16,11 @@ */ const jp = (() => { + const ITERATION_MARK_CODE_POINT = 0x3005; + const HIRAGANA_SMALL_TSU_CODE_POINT = 0x3063; + const KATAKANA_SMALL_TSU_CODE_POINT = 0x30c3; + const KANA_PROLONGED_SOUND_MARK_CODE_POINT = 0x30fc; + const HIRAGANA_RANGE = [0x3040, 0x309f]; const KATAKANA_RANGE = [0x30a0, 0x30ff]; const KANA_RANGES = [HIRAGANA_RANGE, KATAKANA_RANGE]; @@ -65,20 +70,65 @@ const jp = (() => { const SMALL_KANA_SET = new Set(Array.from('ぁぃぅぇぉゃゅょゎァィゥェォャュョヮ')); + const HALFWIDTH_KATAKANA_MAPPING = new Map([ + ['ｦ', 'ヲヺ-'], + ['ｧ', 'ァ--'], + ['ｨ', 'ィ--'], + ['ｩ', 'ゥ--'], + ['ｪ', 'ェ--'], + ['ｫ', 'ォ--'], + ['ｬ', 'ャ--'], + ['ｭ', 'ュ--'], + ['ｮ', 'ョ--'], + ['ｯ', 'ッ--'], + ['ｰ', 'ー--'], + ['ｱ', 'ア--'], + ['ｲ', 'イ--'], + ['ｳ', 'ウヴ-'], + ['ｴ', 'エ--'], + ['ｵ', 'オ--'], + ['ｶ', 'カガ-'], + ['ｷ', 'キギ-'], + ['ｸ', 'クグ-'], + ['ｹ', 'ケゲ-'], + ['ｺ', 'コゴ-'], + ['ｻ', 'サザ-'], + ['ｼ', 'シジ-'], + ['ｽ', 'スズ-'], + ['ｾ', 'セゼ-'], + ['ｿ', 'ソゾ-'], + ['ﾀ', 'タダ-'], + ['ﾁ', 'チヂ-'], + ['ﾂ', 'ツヅ-'], + ['ﾃ', 'テデ-'], + ['ﾄ', 'トド-'], + ['ﾅ', 'ナ--'], + ['ﾆ', 'ニ--'], + ['ﾇ', 'ヌ--'], + ['ﾈ', 'ネ--'], + ['ﾉ', 'ノ--'], + ['ﾊ', 'ハバパ'], + ['ﾋ', 'ヒビピ'], + ['ﾌ', 'フブプ'], + ['ﾍ', 'ヘベペ'], + ['ﾎ', 'ホボポ'], + ['ﾏ', 'マ--'], + ['ﾐ', 'ミ--'], + ['ﾑ', 'ム--'], + ['ﾒ', 'メ--'], + ['ﾓ', 'モ--'], + ['ﾔ', 'ヤ--'], + ['ﾕ', 'ユ--'], + ['ﾖ', 'ヨ--'], + ['ﾗ', 'ラ--'], + ['ﾘ', 'リ--'], + ['ﾙ', 'ル--'], + ['ﾚ', 'レ--'], + ['ﾛ', 'ロ--'], + ['ﾜ', 'ワ--'], + ['ﾝ', 'ン--'] + ]); - // Character code testing functions - - function isCodePointKanji(codePoint) { - return isCodePointInRanges(codePoint, CJK_UNIFIED_IDEOGRAPHS_RANGES); - } - - function isCodePointKana(codePoint) { - return isCodePointInRanges(codePoint, KANA_RANGES); - } - - function isCodePointJapanese(codePoint) { - return isCodePointInRanges(codePoint, JAPANESE_RANGES); - } function isCodePointInRanges(codePoint, ranges) { for (const [min, max] of ranges) { @@ -89,63 +139,410 @@ const jp = (() => { return false; } + function getWanakana() { + try { + if (typeof wanakana !== 'undefined') { + // eslint-disable-next-line no-undef + return wanakana; + } + } catch (e) { + // NOP + } + return null; + } + - // String testing functions + class JapaneseUtil { + constructor(wanakana=null) { + this._wanakana = wanakana; + } - function isStringEntirelyKana(str) { - if (str.length === 0) { return false; } - for (const c of str) { - if (!isCodePointKana(c.codePointAt(0))) { - return false; + // Character code testing functions + + isCodePointKanji(codePoint) { + return isCodePointInRanges(codePoint, CJK_UNIFIED_IDEOGRAPHS_RANGES); + } + + isCodePointKana(codePoint) { + return isCodePointInRanges(codePoint, KANA_RANGES); + } + + isCodePointJapanese(codePoint) { + return isCodePointInRanges(codePoint, JAPANESE_RANGES); + } + + // String testing functions + + isStringEntirelyKana(str) { + if (str.length === 0) { return false; } + for (const c of str) { + if (!isCodePointInRanges(c.codePointAt(0), KANA_RANGES)) { + return false; + } } + return true; } - return true; - } - function isStringPartiallyJapanese(str) { - if (str.length === 0) { return false; } - for (const c of str) { - if (isCodePointJapanese(c.codePointAt(0))) { - return true; + isStringPartiallyJapanese(str) { + if (str.length === 0) { return false; } + for (const c of str) { + if (isCodePointInRanges(c.codePointAt(0), JAPANESE_RANGES)) { + return true; + } } + return false; } - return false; - } + // Mora functions - // Mora functions + isMoraPitchHigh(moraIndex, pitchAccentPosition) { + switch (pitchAccentPosition) { + case 0: return (moraIndex > 0); + case 1: return (moraIndex < 1); + default: return (moraIndex > 0 && moraIndex < pitchAccentPosition); + } + } - function isMoraPitchHigh(moraIndex, pitchAccentPosition) { - switch (pitchAccentPosition) { - case 0: return (moraIndex > 0); - case 1: return (moraIndex < 1); - default: return (moraIndex > 0 && moraIndex < pitchAccentPosition); + getKanaMorae(text) { + const morae = []; + let i; + for (const c of text) { + if (SMALL_KANA_SET.has(c) && (i = morae.length) > 0) { + morae[i - 1] += c; + } else { + morae.push(c); + } + } + return morae; } - } - function getKanaMorae(text) { - const morae = []; - let i; - for (const c of text) { - if (SMALL_KANA_SET.has(c) && (i = morae.length) > 0) { - morae[i - 1] += c; - } else { - morae.push(c); + // Conversion functions + + convertKatakanaToHiragana(text) { + const wanakana = this._getWanakana(); + let result = ''; + for (const c of text) { + if (wanakana.isKatakana(c)) { + result += wanakana.toHiragana(c); + } else { + result += c; + } } + + return result; } - return morae; - } + convertHiraganaToKatakana(text) { + const wanakana = this._getWanakana(); + let result = ''; + for (const c of text) { + if (wanakana.isHiragana(c)) { + result += wanakana.toKatakana(c); + } else { + result += c; + } + } + + return result; + } + + convertToRomaji(text) { + const wanakana = this._getWanakana(); + return wanakana.toRomaji(text); + } + + convertReading(expression, reading, readingMode) { + switch (readingMode) { + case 'hiragana': + return this.convertKatakanaToHiragana(reading); + case 'katakana': + return this.convertHiraganaToKatakana(reading); + case 'romaji': + if (reading) { + return this.convertToRomaji(reading); + } else { + if (this.isStringEntirelyKana(expression)) { + return this.convertToRomaji(expression); + } + } + return reading; + case 'none': + return ''; + default: + return reading; + } + } + + convertNumericToFullWidth(text) { + let result = ''; + for (const char of text) { + let c = char.codePointAt(0); + if (c >= 0x30 && c <= 0x39) { // ['0', '9'] + c += 0xff10 - 0x30; // 0xff10 = '0' full width + result += String.fromCodePoint(c); + } else { + result += char; + } + } + return result; + } + + convertHalfWidthKanaToFullWidth(text, sourceMap=null) { + let result = ''; + + // This function is safe to use charCodeAt instead of codePointAt, since all + // the relevant characters are represented with a single UTF-16 character code. + for (let i = 0, ii = text.length; i < ii; ++i) { + const c = text[i]; + const mapping = HALFWIDTH_KATAKANA_MAPPING.get(c); + if (typeof mapping !== 'string') { + result += c; + continue; + } + + let index = 0; + switch (text.charCodeAt(i + 1)) { + case 0xff9e: // dakuten + index = 1; + break; + case 0xff9f: // handakuten + index = 2; + break; + } + + let c2 = mapping[index]; + if (index > 0) { + if (c2 === '-') { // invalid + index = 0; + c2 = mapping[0]; + } else { + ++i; + } + } + + if (sourceMap !== null && index > 0) { + sourceMap.combine(result.length, 1); + } + result += c2; + } + + return result; + } + + convertAlphabeticToKana(text, sourceMap=null) { + let part = ''; + let result = ''; + + for (const char of text) { + // Note: 0x61 is the character code for 'a' + let c = char.codePointAt(0); + if (c >= 0x41 && c <= 0x5a) { // ['A', 'Z'] + c += (0x61 - 0x41); + } else if (c >= 0x61 && c <= 0x7a) { // ['a', 'z'] + // NOP; c += (0x61 - 0x61); + } else if (c >= 0xff21 && c <= 0xff3a) { // ['A', 'Z'] fullwidth + c += (0x61 - 0xff21); + } else if (c >= 0xff41 && c <= 0xff5a) { // ['a', 'z'] fullwidth + c += (0x61 - 0xff41); + } else if (c === 0x2d || c === 0xff0d) { // '-' or fullwidth dash + c = 0x2d; // '-' + } else { + if (part.length > 0) { + result += this._convertAlphabeticPartToKana(part, sourceMap, result.length); + part = ''; + } + result += char; + continue; + } + part += String.fromCodePoint(c); + } + + if (part.length > 0) { + result += this._convertAlphabeticPartToKana(part, sourceMap, result.length); + } + return result; + } + + // Furigana distribution + + distributeFurigana(expression, reading) { + const fallback = [{furigana: reading, text: expression}]; + if (!reading) { + return fallback; + } + + let isAmbiguous = false; + const segmentize = (reading2, groups) => { + if (groups.length === 0 || isAmbiguous) { + return []; + } + + const group = groups[0]; + if (group.mode === 'kana') { + if (this.convertKatakanaToHiragana(reading2).startsWith(this.convertKatakanaToHiragana(group.text))) { + const readingLeft = reading2.substring(group.text.length); + const segs = segmentize(readingLeft, groups.splice(1)); + if (segs) { + return [{text: group.text, furigana: ''}].concat(segs); + } + } + } else { + let foundSegments = null; + for (let i = reading2.length; i >= group.text.length; --i) { + const readingUsed = reading2.substring(0, i); + const readingLeft = reading2.substring(i); + const segs = segmentize(readingLeft, groups.slice(1)); + if (segs) { + if (foundSegments !== null) { + // more than one way to segmentize the tail, mark as ambiguous + isAmbiguous = true; + return null; + } + foundSegments = [{text: group.text, furigana: readingUsed}].concat(segs); + } + // there is only one way to segmentize the last non-kana group + if (groups.length === 1) { + break; + } + } + return foundSegments; + } + }; + + const groups = []; + let modePrev = null; + for (const c of expression) { + const codePoint = c.codePointAt(0); + const modeCurr = this.isCodePointKanji(codePoint) || codePoint === ITERATION_MARK_CODE_POINT ? 'kanji' : 'kana'; + if (modeCurr === modePrev) { + groups[groups.length - 1].text += c; + } else { + groups.push({mode: modeCurr, text: c}); + modePrev = modeCurr; + } + } + + const segments = segmentize(reading, groups); + if (segments && !isAmbiguous) { + return segments; + } + return fallback; + } + + distributeFuriganaInflected(expression, reading, source) { + const output = []; + + let stemLength = 0; + const shortest = Math.min(source.length, expression.length); + const sourceHiragana = this.convertKatakanaToHiragana(source); + const expressionHiragana = this.convertKatakanaToHiragana(expression); + while (stemLength < shortest && sourceHiragana[stemLength] === expressionHiragana[stemLength]) { + ++stemLength; + } + const offset = source.length - stemLength; + + const stemExpression = source.substring(0, source.length - offset); + const stemReading = reading.substring( + 0, + offset === 0 ? reading.length : reading.length - expression.length + stemLength + ); + for (const segment of this.distributeFurigana(stemExpression, stemReading)) { + output.push(segment); + } + + if (stemLength !== source.length) { + output.push({text: source.substring(stemLength), furigana: ''}); + } + + return output; + } + + // Miscellaneous + + collapseEmphaticSequences(text, fullCollapse, sourceMap=null) { + let result = ''; + let collapseCodePoint = -1; + const hasSourceMap = (sourceMap !== null); + for (const char of text) { + const c = char.codePointAt(0); + if ( + c === HIRAGANA_SMALL_TSU_CODE_POINT || + c === KATAKANA_SMALL_TSU_CODE_POINT || + c === KANA_PROLONGED_SOUND_MARK_CODE_POINT + ) { + if (collapseCodePoint !== c) { + collapseCodePoint = c; + if (!fullCollapse) { + result += char; + continue; + } + } + } else { + collapseCodePoint = -1; + result += char; + continue; + } + + if (hasSourceMap) { + sourceMap.combine(Math.max(0, result.length - 1), 1); + } + } + return result; + } + + // Private + + _getWanakana() { + const wanakana = this._wanakana; + if (wanakana === null) { throw new Error('Functions which use WanaKana are not supported in this context'); } + return wanakana; + } + + _convertAlphabeticPartToKana(text, sourceMap, sourceMapStart) { + const wanakana = this._getWanakana(); + const result = wanakana.toHiragana(text); + + // Generate source mapping + if (sourceMap !== null) { + let i = 0; + let resultPos = 0; + const ii = text.length; + while (i < ii) { + // Find smallest matching substring + let iNext = i + 1; + let resultPosNext = result.length; + while (iNext < ii) { + const t = wanakana.toHiragana(text.substring(0, iNext)); + if (t === result.substring(0, t.length)) { + resultPosNext = t.length; + break; + } + ++iNext; + } + + // Merge characters + const removals = iNext - i - 1; + if (removals > 0) { + sourceMap.combine(sourceMapStart, removals); + } + ++sourceMapStart; + + // Empty elements + const additions = resultPosNext - resultPos - 1; + for (let j = 0; j < additions; ++j) { + sourceMap.insert(sourceMapStart, 0); + ++sourceMapStart; + } + + i = iNext; + resultPos = resultPosNext; + } + } + + return result; + } + } - // Exports - return { - isCodePointKanji, - isCodePointKana, - isCodePointJapanese, - isStringEntirelyKana, - isStringPartiallyJapanese, - isMoraPitchHigh, - getKanaMorae - }; + return new JapaneseUtil(getWanakana()); })(); diff --git a/test/test-japanese.js b/test/test-japanese.js index 321861d5..39004128 100644 --- a/test/test-japanese.js +++ b/test/test-japanese.js @@ -22,8 +22,7 @@ const vm = new VM(); vm.execute([ 'mixed/lib/wanakana.min.js', 'mixed/js/japanese.js', - 'bg/js/text-source-map.js', - 'bg/js/japanese.js' + 'bg/js/text-source-map.js' ]); const jp = vm.get('jp'); const TextSourceMap = vm.get('TextSourceMap'); -- cgit v1.2.3 From f85508a25edb49de96bebb77a5832931f0819eff Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sun, 10 May 2020 14:09:04 -0400 Subject: Simplify shadow test and add an open shadow DOM test (#533) --- test/data/html/test-document2-frame1.html | 6 +- test/data/html/test-document2-script.js | 24 +++++++ test/data/html/test-document2.html | 106 ++++++++++++++---------------- test/data/html/test-stylesheet.css | 3 +- 4 files changed, 76 insertions(+), 63 deletions(-) (limited to 'test') diff --git a/test/data/html/test-document2-frame1.html b/test/data/html/test-document2-frame1.html index 3b85cdc5..e572e3c4 100644 --- a/test/data/html/test-document2-frame1.html +++ b/test/data/html/test-document2-frame1.html @@ -33,10 +33,8 @@ a, a:visited { ありがとう
- Toggle fullscreen - + Toggle fullscreen +
\ No newline at end of file diff --git a/test/data/html/test-document2-script.js b/test/data/html/test-document2-script.js index bd5a570d..ab516a4e 100644 --- a/test/data/html/test-document2-script.js +++ b/test/data/html/test-document2-script.js @@ -39,3 +39,27 @@ function toggleFullscreen(element) { requestFullscreen(element); } } + +function setup(container, fullscreenElement=null) { + const fullscreenLink = container.querySelector('.fullscreen-link'); + if (fullscreenLink !== null) { + if (fullscreenElement === null) { + fullscreenElement = container.querySelector('.fullscreen-element'); + } + fullscreenLink.addEventListener('click', (e) => { + toggleFullscreen(fullscreenElement); + e.preventDefault(); + return false; + }, false); + } + + const template = container.querySelector('template'); + const templateContentContainer = container.querySelector('.template-content-container'); + if (template !== null && templateContentContainer !== null) { + const mode = container.dataset.shadowMode; + const shadow = templateContentContainer.attachShadow({mode}); + const content = document.importNode(template.content, true); + setup(content); + shadow.appendChild(content); + } +} diff --git a/test/data/html/test-document2.html b/test/data/html/test-document2.html index b2046dfd..6d174571 100644 --- a/test/data/html/test-document2.html +++ b/test/data/html/test-document2.html @@ -11,88 +11,78 @@

Yomichan Manual Tests

-

Manual tests involving fullscreen elements, <iframe>s, and shadow DOMs.

+ Manual tests involving fullscreen elements, <iframe>s, and shadow DOMs. -
-
Standard content.
-
+ + Standard content. +
ありがとう
- -
+ -
-
Content inside of a shadow DOM.
-
-