From 8a7ff6a18c78bbc2048dd4017597ccc4f5ee4106 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sat, 13 Jun 2020 10:23:04 -0400 Subject: Replace XMLHttpRequest (#562) * Replace XMLHttpRequest with fetch * Implement fetch placeholder for tests --- test/test-database.js | 66 +++++++++------------------------------------------ 1 file changed, 11 insertions(+), 55 deletions(-) (limited to 'test') diff --git a/test/test-database.js b/test/test-database.js index e8a4a343..63989857 100644 --- a/test/test-database.js +++ b/test/test-database.js @@ -38,60 +38,6 @@ const chrome = { } }; -class XMLHttpRequest { - constructor() { - this._eventCallbacks = new Map(); - this._url = ''; - this._responseText = null; - } - - overrideMimeType() { - // NOP - } - - addEventListener(eventName, callback) { - let callbacks = this._eventCallbacks.get(eventName); - if (typeof callbacks === 'undefined') { - callbacks = []; - this._eventCallbacks.set(eventName, callbacks); - } - callbacks.push(callback); - } - - open(action, url2) { - this._url = url2; - } - - send() { - const filePath = url.fileURLToPath(this._url); - Promise.resolve() - .then(() => { - let source; - try { - source = fs.readFileSync(filePath, {encoding: 'utf8'}); - } catch (e) { - this._trigger('error'); - return; - } - this._responseText = source; - this._trigger('load'); - }); - } - - get responseText() { - return this._responseText; - } - - _trigger(eventName, ...args) { - const callbacks = this._eventCallbacks.get(eventName); - if (typeof callbacks === 'undefined') { return; } - - for (let i = 0, ii = callbacks.length; i < ii; ++i) { - callbacks[i](...args); - } - } -} - class Image { constructor() { this._src = ''; @@ -138,11 +84,21 @@ class Image { } } +async function fetch(url2) { + const filePath = url.fileURLToPath(url2); + await Promise.resolve(); + const content = fs.readFileSync(filePath, {encoding: null}); + return { + text: async () => Promise.resolve(content.toString('utf8')), + json: async () => Promise.resolve(JSON.parse(content.toString('utf8'))) + }; +} + const vm = new VM({ chrome, Image, - XMLHttpRequest, + fetch, indexedDB: global.indexedDB, IDBKeyRange: global.IDBKeyRange, JSZip: yomichanTest.JSZip, -- cgit v1.2.3 From e23504613f8526b90a497512c086ed48e66cde95 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sun, 21 Jun 2020 16:07:51 -0400 Subject: Use DOMTextScanner (#536) * Use DOMTextScanner instead of TextSourceRange.seek* * Move getNodesInRange to dom.js * Move anyNodeMatchesSelector to dom.js * Remove unused functions * Update tests * Add layoutAwareScan option * Use layoutAwareScan for source and sentence scanning * Remove unused IGNORE_TEXT_PATTERN --- ext/bg/data/options-schema.json | 7 +- ext/bg/js/options.js | 3 +- ext/bg/js/search-query-parser.js | 8 +- ext/bg/search.html | 1 + ext/bg/settings-popup-preview.html | 1 + ext/bg/settings.html | 4 + ext/fg/float.html | 1 + ext/fg/js/document.js | 11 +- ext/fg/js/frontend.js | 14 ++- ext/fg/js/source.js | 224 ++----------------------------------- ext/manifest.json | 1 + ext/mixed/js/display.js | 11 +- ext/mixed/js/dom.js | 38 +++++++ ext/mixed/js/text-scanner.js | 11 +- test/test-document.js | 14 ++- 15 files changed, 102 insertions(+), 247 deletions(-) (limited to 'test') diff --git a/ext/bg/data/options-schema.json b/ext/bg/data/options-schema.json index 0379fa75..5885e036 100644 --- a/ext/bg/data/options-schema.json +++ b/ext/bg/data/options-schema.json @@ -321,7 +321,8 @@ "enablePopupSearch", "enableOnPopupExpressions", "enableOnSearchPage", - "enableSearchTags" + "enableSearchTags", + "layoutAwareScan" ], "properties": { "middleMouse": { @@ -383,6 +384,10 @@ "enableSearchTags": { "type": "boolean", "default": false + }, + "layoutAwareScan": { + "type": "boolean", + "default": false } } }, diff --git a/ext/bg/js/options.js b/ext/bg/js/options.js index 97368a0b..170e4799 100644 --- a/ext/bg/js/options.js +++ b/ext/bg/js/options.js @@ -203,7 +203,8 @@ function profileOptionsCreateDefaults() { enablePopupSearch: false, enableOnPopupExpressions: false, enableOnSearchPage: true, - enableSearchTags: false + enableSearchTags: false, + layoutAwareScan: false }, translation: { diff --git a/ext/bg/js/search-query-parser.js b/ext/bg/js/search-query-parser.js index addfc686..97e98b40 100644 --- a/ext/bg/js/search-query-parser.js +++ b/ext/bg/js/search-query-parser.js @@ -75,15 +75,17 @@ class QueryParser { async _search(textSource, cause) { if (textSource === null) { return null; } - const searchText = this._textScanner.getTextSourceContent(textSource, this._options.scanning.length); + const {length: scanLength, layoutAwareScan} = this._options.scanning; + const searchText = this._textScanner.getTextSourceContent(textSource, scanLength, layoutAwareScan); if (searchText.length === 0) { return null; } const {definitions, length} = await api.termsFind(searchText, {}, this._getOptionsContext()); if (definitions.length === 0) { return null; } - const sentence = docSentenceExtract(textSource, this._options.anki.sentenceExt); + const sentenceExtent = this._options.anki.sentenceExt; + const sentence = docSentenceExtract(textSource, sentenceExtent, layoutAwareScan); - textSource.setEndOffset(length); + textSource.setEndOffset(length, layoutAwareScan); this._setContent('terms', {definitions, context: { focus: false, diff --git a/ext/bg/search.html b/ext/bg/search.html index de08cdae..4a28dd88 100644 --- a/ext/bg/search.html +++ b/ext/bg/search.html @@ -79,6 +79,7 @@ + diff --git a/ext/bg/settings-popup-preview.html b/ext/bg/settings-popup-preview.html index fe92f24f..5eecd005 100644 --- a/ext/bg/settings-popup-preview.html +++ b/ext/bg/settings-popup-preview.html @@ -126,6 +126,7 @@ + diff --git a/ext/bg/settings.html b/ext/bg/settings.html index 118a13b9..77b61aef 100644 --- a/ext/bg/settings.html +++ b/ext/bg/settings.html @@ -400,6 +400,10 @@ +
+ +
+
diff --git a/ext/fg/float.html b/ext/fg/float.html index 17dbcc6d..3e41cde5 100644 --- a/ext/fg/float.html +++ b/ext/fg/float.html @@ -46,6 +46,7 @@ + diff --git a/ext/fg/js/document.js b/ext/fg/js/document.js index d639bc86..c288502c 100644 --- a/ext/fg/js/document.js +++ b/ext/fg/js/document.js @@ -17,6 +17,7 @@ /* global * DOM + * DOMTextScanner * TextSourceElement * TextSourceRange */ @@ -152,14 +153,14 @@ function docRangeFromPoint(x, y, deepDomScan) { } } -function docSentenceExtract(source, extent) { +function docSentenceExtract(source, extent, layoutAwareScan) { const quotesFwd = {'「': '」', '『': '』', "'": "'", '"': '"'}; const quotesBwd = {'」': '「', '』': '『', "'": "'", '"': '"'}; const terminators = '…。．.？?！!'; const sourceLocal = source.clone(); - const position = sourceLocal.setStartOffset(extent); - sourceLocal.setEndOffset(extent * 2 - position, true); + const position = sourceLocal.setStartOffset(extent, layoutAwareScan); + sourceLocal.setEndOffset(extent * 2 - position, layoutAwareScan, true); const content = sourceLocal.text(); let quoteStack = []; @@ -232,7 +233,7 @@ function isPointInRange(x, y, range) { const nodePre = range.endContainer; const offsetPre = range.endOffset; try { - const {node, offset, content} = TextSourceRange.seekForward(range.endContainer, range.endOffset, 1); + const {node, offset, content} = new DOMTextScanner(range.endContainer, range.endOffset, true, false).seek(1); range.setEnd(node, offset); if (!isWhitespace(content) && DOM.isPointInAnyRect(x, y, range.getClientRects())) { @@ -243,7 +244,7 @@ function isPointInRange(x, y, range) { } // Scan backward - const {node, offset, content} = TextSourceRange.seekBackward(range.startContainer, range.startOffset, 1); + const {node, offset, content} = new DOMTextScanner(range.startContainer, range.startOffset, true, false).seek(-1); range.setStart(node, offset); if (!isWhitespace(content) && DOM.isPointInAnyRect(x, y, range.getClientRects())) { diff --git a/ext/fg/js/frontend.js b/ext/fg/js/frontend.js index 70bd8a48..ab455c09 100644 --- a/ext/fg/js/frontend.js +++ b/ext/fg/js/frontend.js @@ -258,32 +258,36 @@ class Frontend { } async _findTerms(textSource, optionsContext) { - const searchText = this._textScanner.getTextSourceContent(textSource, this._options.scanning.length); + const {length: scanLength, layoutAwareScan} = this._options.scanning; + const searchText = this._textScanner.getTextSourceContent(textSource, scanLength, layoutAwareScan); if (searchText.length === 0) { return null; } const {definitions, length} = await api.termsFind(searchText, {}, optionsContext); if (definitions.length === 0) { return null; } - textSource.setEndOffset(length); + textSource.setEndOffset(length, layoutAwareScan); return {definitions, type: 'terms'}; } async _findKanji(textSource, optionsContext) { - const searchText = this._textScanner.getTextSourceContent(textSource, 1); + const layoutAwareScan = this._options.scanning.layoutAwareScan; + const searchText = this._textScanner.getTextSourceContent(textSource, 1, layoutAwareScan); if (searchText.length === 0) { return null; } const definitions = await api.kanjiFind(searchText, optionsContext); if (definitions.length === 0) { return null; } - textSource.setEndOffset(1); + textSource.setEndOffset(1, layoutAwareScan); return {definitions, type: 'kanji'}; } _showContent(textSource, focus, definitions, type, optionsContext) { const {url} = optionsContext; - const sentence = docSentenceExtract(textSource, this._options.anki.sentenceExt); + const sentenceExtent = this._options.anki.sentenceExt; + const layoutAwareScan = this._options.scanning.layoutAwareScan; + const sentence = docSentenceExtract(textSource, sentenceExtent, layoutAwareScan); this._showPopupContent( textSource, optionsContext, diff --git a/ext/fg/js/source.js b/ext/fg/js/source.js index fa4706f2..38810f07 100644 --- a/ext/fg/js/source.js +++ b/ext/fg/js/source.js @@ -15,9 +15,9 @@ * along with this program. If not, see . */ -// \u200c (Zero-width non-joiner) appears on Google Docs from Chrome 76 onwards -const IGNORE_TEXT_PATTERN = /\u200c/; - +/* global + * DOMTextScanner + */ /* * TextSourceRange @@ -46,19 +46,19 @@ class TextSourceRange { return this.content; } - setEndOffset(length, fromEnd=false) { + setEndOffset(length, layoutAwareScan, fromEnd=false) { const state = ( fromEnd ? - TextSourceRange.seekForward(this.range.endContainer, this.range.endOffset, length) : - TextSourceRange.seekForward(this.range.startContainer, this.range.startOffset, length) + new DOMTextScanner(this.range.endContainer, this.range.endOffset, !layoutAwareScan, layoutAwareScan).seek(length) : + new DOMTextScanner(this.range.startContainer, this.range.startOffset, !layoutAwareScan, layoutAwareScan).seek(length) ); this.range.setEnd(state.node, state.offset); this.content = (fromEnd ? this.content + state.content : state.content); return length - state.remainder; } - setStartOffset(length) { - const state = TextSourceRange.seekBackward(this.range.startContainer, this.range.startOffset, length); + setStartOffset(length, layoutAwareScan) { + const state = new DOMTextScanner(this.range.startContainer, this.range.startOffset, !layoutAwareScan, layoutAwareScan).seek(-length); this.range.setStart(state.node, state.offset); this.rangeStartOffset = this.range.startOffset; this.content = state.content + this.content; @@ -110,154 +110,6 @@ class TextSourceRange { } } - static shouldEnter(node) { - switch (node.nodeName.toUpperCase()) { - case 'RT': - case 'SCRIPT': - case 'STYLE': - return false; - } - - const style = window.getComputedStyle(node); - return !( - style.visibility === 'hidden' || - style.display === 'none' || - parseFloat(style.fontSize) === 0 - ); - } - - static getRubyElement(node) { - node = TextSourceRange.getParentElement(node); - if (node !== null && node.nodeName.toUpperCase() === 'RT') { - node = node.parentNode; - return (node !== null && node.nodeName.toUpperCase() === 'RUBY') ? node : null; - } - return null; - } - - static seekForward(node, offset, length) { - const state = {node, offset, remainder: length, content: ''}; - if (length <= 0) { - return state; - } - - const TEXT_NODE = Node.TEXT_NODE; - const ELEMENT_NODE = Node.ELEMENT_NODE; - let resetOffset = false; - - const ruby = TextSourceRange.getRubyElement(node); - if (ruby !== null) { - node = ruby; - resetOffset = true; - } - - while (node !== null) { - let visitChildren = true; - const nodeType = node.nodeType; - - if (nodeType === TEXT_NODE) { - state.node = node; - if (TextSourceRange.seekForwardTextNode(state, resetOffset)) { - break; - } - resetOffset = true; - } else if (nodeType === ELEMENT_NODE) { - visitChildren = TextSourceRange.shouldEnter(node); - } - - node = TextSourceRange.getNextNode(node, visitChildren); - } - - return state; - } - - static seekForwardTextNode(state, resetOffset) { - const nodeValue = state.node.nodeValue; - const nodeValueLength = nodeValue.length; - let content = state.content; - let offset = resetOffset ? 0 : state.offset; - let remainder = state.remainder; - let result = false; - - for (; offset < nodeValueLength; ++offset) { - const c = nodeValue[offset]; - if (!IGNORE_TEXT_PATTERN.test(c)) { - content += c; - if (--remainder <= 0) { - result = true; - ++offset; - break; - } - } - } - - state.offset = offset; - state.content = content; - state.remainder = remainder; - return result; - } - - static seekBackward(node, offset, length) { - const state = {node, offset, remainder: length, content: ''}; - if (length <= 0) { - return state; - } - - const TEXT_NODE = Node.TEXT_NODE; - const ELEMENT_NODE = Node.ELEMENT_NODE; - let resetOffset = false; - - const ruby = TextSourceRange.getRubyElement(node); - if (ruby !== null) { - node = ruby; - resetOffset = true; - } - - while (node !== null) { - let visitChildren = true; - const nodeType = node.nodeType; - - if (nodeType === TEXT_NODE) { - state.node = node; - if (TextSourceRange.seekBackwardTextNode(state, resetOffset)) { - break; - } - resetOffset = true; - } else if (nodeType === ELEMENT_NODE) { - visitChildren = TextSourceRange.shouldEnter(node); - } - - node = TextSourceRange.getPreviousNode(node, visitChildren); - } - - return state; - } - - static seekBackwardTextNode(state, resetOffset) { - const nodeValue = state.node.nodeValue; - let content = state.content; - let offset = resetOffset ? nodeValue.length : state.offset; - let remainder = state.remainder; - let result = false; - - for (; offset > 0; --offset) { - const c = nodeValue[offset - 1]; - if (!IGNORE_TEXT_PATTERN.test(c)) { - content = c + content; - if (--remainder <= 0) { - result = true; - --offset; - break; - } - } - } - - state.offset = offset; - state.content = content; - state.remainder = remainder; - return result; - } - static getParentElement(node) { while (node !== null && node.nodeType !== Node.ELEMENT_NODE) { node = node.parentNode; @@ -290,66 +142,6 @@ class TextSourceRange { return writingMode; } } - - static getNodesInRange(range) { - const end = range.endContainer; - const nodes = []; - for (let node = range.startContainer; node !== null; node = TextSourceRange.getNextNode(node, true)) { - nodes.push(node); - if (node === end) { break; } - } - return nodes; - } - - static getNextNode(node, visitChildren) { - let next = visitChildren ? node.firstChild : null; - if (next === null) { - while (true) { - next = node.nextSibling; - if (next !== null) { break; } - - next = node.parentNode; - if (next === null) { break; } - - node = next; - } - } - return next; - } - - static getPreviousNode(node, visitChildren) { - let next = visitChildren ? node.lastChild : null; - if (next === null) { - while (true) { - next = node.previousSibling; - if (next !== null) { break; } - - next = node.parentNode; - if (next === null) { break; } - - node = next; - } - } - return next; - } - - static anyNodeMatchesSelector(nodeList, selector) { - for (const node of nodeList) { - if (TextSourceRange.nodeMatchesSelector(node, selector)) { - return true; - } - } - return false; - } - - static nodeMatchesSelector(node, selector) { - for (; node !== null; node = node.parentNode) { - if (node.nodeType === Node.ELEMENT_NODE) { - return node.matches(selector); - } - } - return false; - } } diff --git a/ext/manifest.json b/ext/manifest.json index 75334675..4d4f0c06 100644 --- a/ext/manifest.json +++ b/ext/manifest.json @@ -42,6 +42,7 @@ "mixed/js/dynamic-loader.js", "mixed/js/text-scanner.js", "fg/js/document.js", + "fg/js/dom-text-scanner.js", "fg/js/popup.js", "fg/js/source.js", "fg/js/popup-factory.js", diff --git a/ext/mixed/js/display.js b/ext/mixed/js/display.js index 90fd1037..1d699706 100644 --- a/ext/mixed/js/display.js +++ b/ext/mixed/js/display.js @@ -236,7 +236,9 @@ class Display { const {textSource, definitions} = termLookupResults; const scannedElement = e.target; - const sentence = docSentenceExtract(textSource, this.options.anki.sentenceExt); + const sentenceExtent = this.options.anki.sentenceExt; + const layoutAwareScan = this.options.scanning.layoutAwareScan; + const sentence = docSentenceExtract(textSource, sentenceExtent, layoutAwareScan); this.context.update({ index: this.entryIndexFind(scannedElement), @@ -273,21 +275,22 @@ class Display { try { e.preventDefault(); - const textSource = docRangeFromPoint(e.clientX, e.clientY, this.options.scanning.deepDomScan); + const {length: scanLength, deepDomScan: deepScan, layoutAwareScan} = this.options.scanning; + const textSource = docRangeFromPoint(e.clientX, e.clientY, deepScan); if (textSource === null) { return false; } let definitions, length; try { - textSource.setEndOffset(this.options.scanning.length); + textSource.setEndOffset(scanLength, layoutAwareScan); ({definitions, length} = await api.termsFind(textSource.text(), {}, this.getOptionsContext())); if (definitions.length === 0) { return false; } - textSource.setEndOffset(length); + textSource.setEndOffset(length, layoutAwareScan); } finally { textSource.cleanup(); } diff --git a/ext/mixed/js/dom.js b/ext/mixed/js/dom.js index 0e8f4462..05764443 100644 --- a/ext/mixed/js/dom.js +++ b/ext/mixed/js/dom.js @@ -86,4 +86,42 @@ class DOM { null ); } + + static getNodesInRange(range) { + const end = range.endContainer; + const nodes = []; + for (let node = range.startContainer; node !== null; node = DOM.getNextNode(node)) { + nodes.push(node); + if (node === end) { break; } + } + return nodes; + } + + static getNextNode(node) { + let next = node.firstChild; + if (next === null) { + while (true) { + next = node.nextSibling; + if (next !== null) { break; } + + next = node.parentNode; + if (next === null) { break; } + + node = next; + } + } + return next; + } + + static anyNodeMatchesSelector(nodes, selector) { + const ELEMENT_NODE = Node.ELEMENT_NODE; + for (let node of nodes) { + for (; node !== null; node = node.parentNode) { + if (node.nodeType !== ELEMENT_NODE) { continue; } + if (node.matches(selector)) { return true; } + break; + } + } + return false; + } } diff --git a/ext/mixed/js/text-scanner.js b/ext/mixed/js/text-scanner.js index b8688b08..fb275452 100644 --- a/ext/mixed/js/text-scanner.js +++ b/ext/mixed/js/text-scanner.js @@ -17,7 +17,6 @@ /* global * DOM - * TextSourceRange * docRangeFromPoint */ @@ -119,20 +118,20 @@ class TextScanner extends EventDispatcher { } } - getTextSourceContent(textSource, length) { + getTextSourceContent(textSource, length, layoutAwareScan) { const clonedTextSource = textSource.clone(); - clonedTextSource.setEndOffset(length); + clonedTextSource.setEndOffset(length, layoutAwareScan); if (this._ignoreNodes !== null && clonedTextSource.range) { length = clonedTextSource.text().length; while (clonedTextSource.range && length > 0) { - const nodes = TextSourceRange.getNodesInRange(clonedTextSource.range); - if (!TextSourceRange.anyNodeMatchesSelector(nodes, this._ignoreNodes)) { + const nodes = DOM.getNodesInRange(clonedTextSource.range); + if (!DOM.anyNodeMatchesSelector(nodes, this._ignoreNodes)) { break; } --length; - clonedTextSource.setEndOffset(length); + clonedTextSource.setEndOffset(length, layoutAwareScan); } } diff --git a/test/test-document.js b/test/test-document.js index 0d9026db..ba7acc49 100644 --- a/test/test-document.js +++ b/test/test-document.js @@ -94,10 +94,12 @@ async function testDocument1() { const vm = new VM({document, window, Range, Node}); vm.execute([ 'mixed/js/dom.js', + 'fg/js/dom-text-scanner.js', 'fg/js/source.js', 'fg/js/document.js' ]); - const [TextSourceRange, TextSourceElement, docRangeFromPoint, docSentenceExtract] = vm.get([ + const [DOMTextScanner, TextSourceRange, TextSourceElement, docRangeFromPoint, docSentenceExtract] = vm.get([ + 'DOMTextScanner', 'TextSourceRange', 'TextSourceElement', 'docRangeFromPoint', @@ -106,7 +108,7 @@ async function testDocument1() { try { await testDocumentTextScanningFunctions(dom, {docRangeFromPoint, docSentenceExtract, TextSourceRange, TextSourceElement}); - await testTextSourceRangeSeekFunctions(dom, {TextSourceRange}); + await testTextSourceRangeSeekFunctions(dom, {DOMTextScanner}); } finally { window.close(); } @@ -179,7 +181,7 @@ async function testDocumentTextScanningFunctions(dom, {docRangeFromPoint, docSen if (source === null) { continue; } // Test docSentenceExtract - const sentenceActual = docSentenceExtract(source, sentenceExtent).text; + const sentenceActual = docSentenceExtract(source, sentenceExtent, false).text; assert.strictEqual(sentenceActual, sentence); // Clean @@ -187,7 +189,7 @@ async function testDocumentTextScanningFunctions(dom, {docRangeFromPoint, docSen } } -async function testTextSourceRangeSeekFunctions(dom, {TextSourceRange}) { +async function testTextSourceRangeSeekFunctions(dom, {DOMTextScanner}) { const document = dom.window.document; for (const testElement of document.querySelectorAll('.test[data-test-type=text-source-range-seek]')) { @@ -220,8 +222,8 @@ async function testTextSourceRangeSeekFunctions(dom, {TextSourceRange}) { const {node, offset, content} = ( seekDirection === 'forward' ? - TextSourceRange.seekForward(seekNode, seekOffset, seekLength) : - TextSourceRange.seekBackward(seekNode, seekOffset, seekLength) + new DOMTextScanner(seekNode, seekOffset, true, false).seek(seekLength) : + new DOMTextScanner(seekNode, seekOffset, true, false).seek(-seekLength) ); assert.strictEqual(node, expectedResultNode); -- cgit v1.2.3 From 244ab31bb2edb53ff7aecb51d2dd60b50a24c194 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sun, 21 Jun 2020 16:12:56 -0400 Subject: Generic database (#600) * Update test * Rename db to _db * Create GenericDatabase class * Catch prepare error * Allow database to be purged even if it was not open * Remove unused functions * Change static functions to non-static * Delete and count using the media object store * Update tests --- ext/bg/background.html | 1 + ext/bg/js/backend.js | 6 +- ext/bg/js/database.js | 807 ++++++++++++++++-------------------------- ext/bg/js/generic-database.js | 327 +++++++++++++++++ test/test-database.js | 10 +- 5 files changed, 652 insertions(+), 499 deletions(-) create mode 100644 ext/bg/js/generic-database.js (limited to 'test') diff --git a/ext/bg/background.html b/ext/bg/background.html index f2d01e85..55380ae7 100644 --- a/ext/bg/background.html +++ b/ext/bg/background.html @@ -30,6 +30,7 @@ + diff --git a/ext/bg/js/backend.js b/ext/bg/js/backend.js index a8ee9f83..b89cb641 100644 --- a/ext/bg/js/backend.js +++ b/ext/bg/js/backend.js @@ -172,7 +172,11 @@ class Backend { yomichan.on('log', this._onLog.bind(this)); await this.environment.prepare(); - await this.database.prepare(); + try { + await this.database.prepare(); + } catch (e) { + yomichan.logError(e); + } await this.translator.prepare(); await profileConditionsDescriptorPromise; diff --git a/ext/bg/js/database.js b/ext/bg/js/database.js index 65e267ab..47f1ebdd 100644 --- a/ext/bg/js/database.js +++ b/ext/bg/js/database.js @@ -16,423 +16,422 @@ */ /* global + * GenericDatabase * dictFieldSplit */ class Database { constructor() { - this.db = null; + this._db = new GenericDatabase(); + this._dbName = 'dict'; this._schemas = new Map(); } // Public async prepare() { - if (this.db !== null) { - throw new Error('Database already initialized'); - } - - try { - this.db = await Database._open('dict', 6, (db, transaction, oldVersion) => { - Database._upgrade(db, transaction, oldVersion, [ - { - version: 2, - stores: { - terms: { - primaryKey: {keyPath: 'id', autoIncrement: true}, - indices: ['dictionary', 'expression', 'reading'] - }, - kanji: { - primaryKey: {autoIncrement: true}, - indices: ['dictionary', 'character'] - }, - tagMeta: { - primaryKey: {autoIncrement: true}, - indices: ['dictionary'] - }, - dictionaries: { - primaryKey: {autoIncrement: true}, - indices: ['title', 'version'] - } + await this._db.open( + this._dbName, + 60, + [ + { + version: 20, + stores: { + terms: { + primaryKey: {keyPath: 'id', autoIncrement: true}, + indices: ['dictionary', 'expression', 'reading'] + }, + kanji: { + primaryKey: {autoIncrement: true}, + indices: ['dictionary', 'character'] + }, + tagMeta: { + primaryKey: {autoIncrement: true}, + indices: ['dictionary'] + }, + dictionaries: { + primaryKey: {autoIncrement: true}, + indices: ['title', 'version'] } - }, - { - version: 3, - stores: { - termMeta: { - primaryKey: {autoIncrement: true}, - indices: ['dictionary', 'expression'] - }, - kanjiMeta: { - primaryKey: {autoIncrement: true}, - indices: ['dictionary', 'character'] - }, - tagMeta: { - primaryKey: {autoIncrement: true}, - indices: ['dictionary', 'name'] - } + } + }, + { + version: 30, + stores: { + termMeta: { + primaryKey: {autoIncrement: true}, + indices: ['dictionary', 'expression'] + }, + kanjiMeta: { + primaryKey: {autoIncrement: true}, + indices: ['dictionary', 'character'] + }, + tagMeta: { + primaryKey: {autoIncrement: true}, + indices: ['dictionary', 'name'] } - }, - { - version: 4, - stores: { - terms: { - primaryKey: {keyPath: 'id', autoIncrement: true}, - indices: ['dictionary', 'expression', 'reading', 'sequence'] - } + } + }, + { + version: 40, + stores: { + terms: { + primaryKey: {keyPath: 'id', autoIncrement: true}, + indices: ['dictionary', 'expression', 'reading', 'sequence'] } - }, - { - version: 5, - stores: { - terms: { - primaryKey: {keyPath: 'id', autoIncrement: true}, - indices: ['dictionary', 'expression', 'reading', 'sequence', 'expressionReverse', 'readingReverse'] - } + } + }, + { + version: 50, + stores: { + terms: { + primaryKey: {keyPath: 'id', autoIncrement: true}, + indices: ['dictionary', 'expression', 'reading', 'sequence', 'expressionReverse', 'readingReverse'] } - }, - { - version: 6, - stores: { - media: { - primaryKey: {keyPath: 'id', autoIncrement: true}, - indices: ['dictionary', 'path'] - } + } + }, + { + version: 60, + stores: { + media: { + primaryKey: {keyPath: 'id', autoIncrement: true}, + indices: ['dictionary', 'path'] } } - ]); - }); - return true; - } catch (e) { - yomichan.logError(e); - return false; - } + } + ] + ); } async close() { - this._validate(); - this.db.close(); - this.db = null; + this._db.close(); } isPrepared() { - return this.db !== null; + return this._db.isOpen(); } async purge() { - this._validate(); - - this.db.close(); - await Database._deleteDatabase(this.db.name); - this.db = null; - + if (this._db.isOpening()) { + throw new Error('Cannot purge database while opening'); + } + if (this._db.isOpen()) { + this._db.close(); + } + await GenericDatabase.deleteDatabase(this._dbName); await this.prepare(); } async deleteDictionary(dictionaryName, progressSettings, onProgress) { - this._validate(); - const targets = [ ['dictionaries', 'title'], ['kanji', 'dictionary'], ['kanjiMeta', 'dictionary'], ['terms', 'dictionary'], ['termMeta', 'dictionary'], - ['tagMeta', 'dictionary'] + ['tagMeta', 'dictionary'], + ['media', 'dictionary'] ]; - const promises = []; + + const {rate} = progressSettings; const progressData = { count: 0, processed: 0, storeCount: targets.length, storesProcesed: 0 }; - let progressRate = (typeof progressSettings === 'object' && progressSettings !== null ? progressSettings.rate : 0); - if (typeof progressRate !== 'number' || progressRate <= 0) { - progressRate = 1000; - } - - for (const [objectStoreName, index] of targets) { - const dbTransaction = this.db.transaction([objectStoreName], 'readwrite'); - const dbObjectStore = dbTransaction.objectStore(objectStoreName); - const dbIndex = dbObjectStore.index(index); - const only = IDBKeyRange.only(dictionaryName); - promises.push(Database._deleteValues(dbObjectStore, dbIndex, only, onProgress, progressData, progressRate)); - } - - await Promise.all(promises); - } - - async findTermsBulk(termList, dictionaries, wildcard) { - this._validate(); - const promises = []; - const visited = new Set(); - const results = []; - const processRow = (row, index) => { - if (dictionaries.has(row.dictionary) && !visited.has(row.id)) { - visited.add(row.id); - results.push(Database._createTerm(row, index)); + const filterKeys = (keys) => { + ++progressData.storesProcesed; + progressData.count += keys.length; + onProgress(progressData); + return keys; + }; + const onProgress2 = () => { + const processed = progressData.processed + 1; + progressData.processed = processed; + if ((processed % rate) === 0 || processed === progressData.count) { + onProgress(progressData); } }; - const useWildcard = !!wildcard; - const prefixWildcard = wildcard === 'prefix'; - - const dbTransaction = this.db.transaction(['terms'], 'readonly'); - const dbTerms = dbTransaction.objectStore('terms'); - const dbIndex1 = dbTerms.index(prefixWildcard ? 'expressionReverse' : 'expression'); - const dbIndex2 = dbTerms.index(prefixWildcard ? 'readingReverse' : 'reading'); - - for (let i = 0; i < termList.length; ++i) { - const term = prefixWildcard ? stringReverse(termList[i]) : termList[i]; - const query = useWildcard ? IDBKeyRange.bound(term, `${term}\uffff`, false, false) : IDBKeyRange.only(term); - promises.push( - Database._getAll(dbIndex1, query, i, processRow), - Database._getAll(dbIndex2, query, i, processRow) - ); + const promises = []; + for (const [objectStoreName, indexName] of targets) { + const query = IDBKeyRange.only(dictionaryName); + const promise = this._db.bulkDelete(objectStoreName, indexName, query, filterKeys, onProgress2); + promises.push(promise); } - await Promise.all(promises); - - return results; } - async findTermsExactBulk(termList, readingList, dictionaries) { - this._validate(); - - const promises = []; - const results = []; - const processRow = (row, index) => { - if (row.reading === readingList[index] && dictionaries.has(row.dictionary)) { - results.push(Database._createTerm(row, index)); + findTermsBulk(termList, dictionaries, wildcard) { + return new Promise((resolve, reject) => { + const results = []; + const count = termList.length; + if (count === 0) { + resolve(results); + return; } - }; - - const dbTransaction = this.db.transaction(['terms'], 'readonly'); - const dbTerms = dbTransaction.objectStore('terms'); - const dbIndex = dbTerms.index('expression'); - for (let i = 0; i < termList.length; ++i) { - const only = IDBKeyRange.only(termList[i]); - promises.push(Database._getAll(dbIndex, only, i, processRow)); - } - - await Promise.all(promises); + const visited = new Set(); + const useWildcard = !!wildcard; + const prefixWildcard = wildcard === 'prefix'; + + const transaction = this._db.transaction(['terms'], 'readonly'); + const terms = transaction.objectStore('terms'); + const index1 = terms.index(prefixWildcard ? 'expressionReverse' : 'expression'); + const index2 = terms.index(prefixWildcard ? 'readingReverse' : 'reading'); + + const count2 = count * 2; + let completeCount = 0; + for (let i = 0; i < count; ++i) { + const inputIndex = i; + const term = prefixWildcard ? stringReverse(termList[i]) : termList[i]; + const query = useWildcard ? IDBKeyRange.bound(term, `${term}\uffff`, false, false) : IDBKeyRange.only(term); + + const onGetAll = (rows) => { + for (const row of rows) { + if (dictionaries.has(row.dictionary) && !visited.has(row.id)) { + visited.add(row.id); + results.push(this._createTerm(row, inputIndex)); + } + } + if (++completeCount >= count2) { + resolve(results); + } + }; - return results; + this._db.getAll(index1, query, onGetAll, reject); + this._db.getAll(index2, query, onGetAll, reject); + } + }); } - async findTermsBySequenceBulk(sequenceList, mainDictionary) { - this._validate(); - - const promises = []; - const results = []; - const processRow = (row, index) => { - if (row.dictionary === mainDictionary) { - results.push(Database._createTerm(row, index)); + findTermsExactBulk(termList, readingList, dictionaries) { + return new Promise((resolve, reject) => { + const results = []; + const count = termList.length; + if (count === 0) { + resolve(results); + return; } - }; - - const dbTransaction = this.db.transaction(['terms'], 'readonly'); - const dbTerms = dbTransaction.objectStore('terms'); - const dbIndex = dbTerms.index('sequence'); - - for (let i = 0; i < sequenceList.length; ++i) { - const only = IDBKeyRange.only(sequenceList[i]); - promises.push(Database._getAll(dbIndex, only, i, processRow)); - } - - await Promise.all(promises); - return results; - } - - async findTermMetaBulk(termList, dictionaries) { - return this._findGenericBulk('termMeta', 'expression', termList, dictionaries, Database._createTermMeta); - } + const transaction = this._db.transaction(['terms'], 'readonly'); + const terms = transaction.objectStore('terms'); + const index = terms.index('expression'); - async findKanjiBulk(kanjiList, dictionaries) { - return this._findGenericBulk('kanji', 'character', kanjiList, dictionaries, Database._createKanji); - } + let completeCount = 0; + for (let i = 0; i < count; ++i) { + const inputIndex = i; + const reading = readingList[i]; + const query = IDBKeyRange.only(termList[i]); - async findKanjiMetaBulk(kanjiList, dictionaries) { - return this._findGenericBulk('kanjiMeta', 'character', kanjiList, dictionaries, Database._createKanjiMeta); - } + const onGetAll = (rows) => { + for (const row of rows) { + if (row.reading === reading && dictionaries.has(row.dictionary)) { + results.push(this._createTerm(row, inputIndex)); + } + } + if (++completeCount >= count) { + resolve(results); + } + }; - async findTagForTitle(name, title) { - this._validate(); - - let result = null; - const dbTransaction = this.db.transaction(['tagMeta'], 'readonly'); - const dbTerms = dbTransaction.objectStore('tagMeta'); - const dbIndex = dbTerms.index('name'); - const only = IDBKeyRange.only(name); - await Database._getAll(dbIndex, only, null, (row) => { - if (title === row.dictionary) { - result = row; + this._db.getAll(index, query, onGetAll, reject); } }); - - return result; } - async getMedia(targets) { - this._validate(); - - const count = targets.length; - const promises = []; - const results = new Array(count).fill(null); - const createResult = Database._createMedia; - const processRow = (row, [index, dictionaryName]) => { - if (row.dictionary === dictionaryName) { - results[index] = createResult(row, index); + findTermsBySequenceBulk(sequenceList, mainDictionary) { + return new Promise((resolve, reject) => { + const results = []; + const count = sequenceList.length; + if (count === 0) { + resolve(results); + return; } - }; - const transaction = this.db.transaction(['media'], 'readonly'); - const objectStore = transaction.objectStore('media'); - const index = objectStore.index('path'); + const transaction = this._db.transaction(['terms'], 'readonly'); + const terms = transaction.objectStore('terms'); + const index = terms.index('sequence'); - for (let i = 0; i < count; ++i) { - const {path, dictionaryName} = targets[i]; - const only = IDBKeyRange.only(path); - promises.push(Database._getAll(index, only, [i, dictionaryName], processRow)); - } + let completeCount = 0; + for (let i = 0; i < count; ++i) { + const inputIndex = i; + const query = IDBKeyRange.only(sequenceList[i]); - await Promise.all(promises); + const onGetAll = (rows) => { + for (const row of rows) { + if (row.dictionary === mainDictionary) { + results.push(this._createTerm(row, inputIndex)); + } + } + if (++completeCount >= count) { + resolve(results); + } + }; - return results; + this._db.getAll(index, query, onGetAll, reject); + } + }); } - async getDictionaryInfo() { - this._validate(); + findTermMetaBulk(termList, dictionaries) { + return this._findGenericBulk('termMeta', 'expression', termList, dictionaries, this._createTermMeta.bind(this)); + } - const results = []; - const dbTransaction = this.db.transaction(['dictionaries'], 'readonly'); - const dbDictionaries = dbTransaction.objectStore('dictionaries'); + findKanjiBulk(kanjiList, dictionaries) { + return this._findGenericBulk('kanji', 'character', kanjiList, dictionaries, this._createKanji.bind(this)); + } - await Database._getAll(dbDictionaries, null, null, (info) => results.push(info)); + findKanjiMetaBulk(kanjiList, dictionaries) { + return this._findGenericBulk('kanjiMeta', 'character', kanjiList, dictionaries, this._createKanjiMeta.bind(this)); + } - return results; + findTagForTitle(name, title) { + const query = IDBKeyRange.only(name); + return this._db.find('tagMeta', 'name', query, (row) => (row.dictionary === title), null); } - async getDictionaryCounts(dictionaryNames, getTotal) { - this._validate(); + getMedia(targets) { + return new Promise((resolve, reject) => { + const count = targets.length; + const results = new Array(count).fill(null); + if (count === 0) { + resolve(results); + return; + } - const objectStoreNames = [ - 'kanji', - 'kanjiMeta', - 'terms', - 'termMeta', - 'tagMeta' - ]; - const dbCountTransaction = this.db.transaction(objectStoreNames, 'readonly'); - - const targets = []; - for (const objectStoreName of objectStoreNames) { - targets.push([ - objectStoreName, - dbCountTransaction.objectStore(objectStoreName).index('dictionary') - ]); - } + let completeCount = 0; + const transaction = this._db.transaction(['media'], 'readonly'); + const objectStore = transaction.objectStore('media'); + const index = objectStore.index('path'); - // Query is required for Edge, otherwise index.count throws an exception. - const query1 = IDBKeyRange.lowerBound('', false); - const totalPromise = getTotal ? Database._getCounts(targets, query1) : null; - - const counts = []; - const countPromises = []; - for (let i = 0; i < dictionaryNames.length; ++i) { - counts.push(null); - const index = i; - const query2 = IDBKeyRange.only(dictionaryNames[i]); - const countPromise = Database._getCounts(targets, query2).then((v) => counts[index] = v); - countPromises.push(countPromise); - } - await Promise.all(countPromises); + for (let i = 0; i < count; ++i) { + const inputIndex = i; + const {path, dictionaryName} = targets[i]; + const query = IDBKeyRange.only(path); - const result = {counts}; - if (totalPromise !== null) { - result.total = await totalPromise; - } - return result; + const onGetAll = (rows) => { + for (const row of rows) { + if (row.dictionary !== dictionaryName) { continue; } + results[inputIndex] = this._createMedia(row, inputIndex); + } + if (++completeCount >= count) { + resolve(results); + } + }; + + this._db.getAll(index, query, onGetAll, reject); + } + }); } - async dictionaryExists(title) { - this._validate(); - const transaction = this.db.transaction(['dictionaries'], 'readonly'); - const index = transaction.objectStore('dictionaries').index('title'); - const query = IDBKeyRange.only(title); - const count = await Database._getCount(index, query); - return count > 0; + getDictionaryInfo() { + return new Promise((resolve, reject) => { + const transaction = this._db.transaction(['dictionaries'], 'readonly'); + const objectStore = transaction.objectStore('dictionaries'); + this._db.getAll(objectStore, null, resolve, reject); + }); } - bulkAdd(objectStoreName, items, start, count) { + getDictionaryCounts(dictionaryNames, getTotal) { return new Promise((resolve, reject) => { - const transaction = this.db.transaction([objectStoreName], 'readwrite'); - const objectStore = transaction.objectStore(objectStoreName); + const targets = [ + ['kanji', 'dictionary'], + ['kanjiMeta', 'dictionary'], + ['terms', 'dictionary'], + ['termMeta', 'dictionary'], + ['tagMeta', 'dictionary'], + ['media', 'dictionary'] + ]; + const objectStoreNames = targets.map(([objectStoreName]) => objectStoreName); + const transaction = this._db.transaction(objectStoreNames, 'readonly'); + const databaseTargets = targets.map(([objectStoreName, indexName]) => { + const objectStore = transaction.objectStore(objectStoreName); + const index = objectStore.index(indexName); + return {objectStore, index}; + }); - if (start + count > items.length) { - count = items.length - start; + const countTargets = []; + if (getTotal) { + for (const {objectStore} of databaseTargets) { + countTargets.push([objectStore, null]); + } } - - if (count <= 0) { - resolve(); - return; + for (const dictionaryName of dictionaryNames) { + const query = IDBKeyRange.only(dictionaryName); + for (const {index} of databaseTargets) { + countTargets.push([index, query]); + } } - const end = start + count; - let completedCount = 0; - const onError = (e) => reject(e); - const onSuccess = () => { - if (++completedCount >= count) { - resolve(); + const onCountComplete = (results) => { + const resultCount = results.length; + const targetCount = targets.length; + const counts = []; + for (let i = 0; i < resultCount; i += targetCount) { + const countGroup = {}; + for (let j = 0; j < targetCount; ++j) { + countGroup[targets[j][0]] = results[i + j]; + } + counts.push(countGroup); } + const total = getTotal ? counts.shift() : null; + resolve({total, counts}); }; - for (let i = start; i < end; ++i) { - const request = objectStore.add(items[i]); - request.onerror = onError; - request.onsuccess = onSuccess; - } + this._db.bulkCount(countTargets, onCountComplete, reject); }); } - // Private + async dictionaryExists(title) { + const query = IDBKeyRange.only(title); + const result = await this._db.find('dictionaries', 'title', query); + return typeof result !== 'undefined'; + } - _validate() { - if (this.db === null) { - throw new Error('Database not initialized'); - } + bulkAdd(objectStoreName, items, start, count) { + return this._db.bulkAdd(objectStoreName, items, start, count); } - async _findGenericBulk(tableName, indexName, indexValueList, dictionaries, createResult) { - this._validate(); + // Private - const promises = []; - const results = []; - const processRow = (row, index) => { - if (dictionaries.has(row.dictionary)) { - results.push(createResult(row, index)); + async _findGenericBulk(objectStoreName, indexName, indexValueList, dictionaries, createResult) { + return new Promise((resolve, reject) => { + const results = []; + const count = indexValueList.length; + if (count === 0) { + resolve(results); + return; } - }; - const dbTransaction = this.db.transaction([tableName], 'readonly'); - const dbTerms = dbTransaction.objectStore(tableName); - const dbIndex = dbTerms.index(indexName); + const transaction = this._db.transaction([objectStoreName], 'readonly'); + const terms = transaction.objectStore(objectStoreName); + const index = terms.index(indexName); - for (let i = 0; i < indexValueList.length; ++i) { - const only = IDBKeyRange.only(indexValueList[i]); - promises.push(Database._getAll(dbIndex, only, i, processRow)); - } + let completeCount = 0; + for (let i = 0; i < count; ++i) { + const inputIndex = i; + const query = IDBKeyRange.only(indexValueList[i]); - await Promise.all(promises); + const onGetAll = (rows) => { + for (const row of rows) { + if (dictionaries.has(row.dictionary)) { + results.push(createResult(row, inputIndex)); + } + } + if (++completeCount >= count) { + resolve(results); + } + }; - return results; + this._db.getAll(index, query, onGetAll, reject); + } + }); } - static _createTerm(row, index) { + _createTerm(row, index) { return { index, expression: row.expression, @@ -448,7 +447,7 @@ class Database { }; } - static _createKanji(row, index) { + _createKanji(row, index) { return { index, character: row.character, @@ -461,193 +460,15 @@ class Database { }; } - static _createTermMeta({expression, mode, data, dictionary}, index) { + _createTermMeta({expression, mode, data, dictionary}, index) { return {expression, mode, data, dictionary, index}; } - static _createKanjiMeta({character, mode, data, dictionary}, index) { + _createKanjiMeta({character, mode, data, dictionary}, index) { return {character, mode, data, dictionary, index}; } - static _createMedia(row, index) { + _createMedia(row, index) { return Object.assign({}, row, {index}); } - - static _getAll(dbIndex, query, context, processRow) { - const fn = typeof dbIndex.getAll === 'function' ? Database._getAllFast : Database._getAllUsingCursor; - return fn(dbIndex, query, context, processRow); - } - - static _getAllFast(dbIndex, query, context, processRow) { - return new Promise((resolve, reject) => { - const request = dbIndex.getAll(query); - request.onerror = (e) => reject(e); - request.onsuccess = (e) => { - for (const row of e.target.result) { - processRow(row, context); - } - resolve(); - }; - }); - } - - static _getAllUsingCursor(dbIndex, query, context, processRow) { - return new Promise((resolve, reject) => { - const request = dbIndex.openCursor(query, 'next'); - request.onerror = (e) => reject(e); - request.onsuccess = (e) => { - const cursor = e.target.result; - if (cursor) { - processRow(cursor.value, context); - cursor.continue(); - } else { - resolve(); - } - }; - }); - } - - static _getCounts(targets, query) { - const countPromises = []; - const counts = {}; - for (const [objectStoreName, index] of targets) { - const n = objectStoreName; - const countPromise = Database._getCount(index, query).then((count) => counts[n] = count); - countPromises.push(countPromise); - } - return Promise.all(countPromises).then(() => counts); - } - - static _getCount(dbIndex, query) { - return new Promise((resolve, reject) => { - const request = dbIndex.count(query); - request.onerror = (e) => reject(e); - request.onsuccess = (e) => resolve(e.target.result); - }); - } - - static _getAllKeys(dbIndex, query) { - const fn = typeof dbIndex.getAllKeys === 'function' ? Database._getAllKeysFast : Database._getAllKeysUsingCursor; - return fn(dbIndex, query); - } - - static _getAllKeysFast(dbIndex, query) { - return new Promise((resolve, reject) => { - const request = dbIndex.getAllKeys(query); - request.onerror = (e) => reject(e); - request.onsuccess = (e) => resolve(e.target.result); - }); - } - - static _getAllKeysUsingCursor(dbIndex, query) { - return new Promise((resolve, reject) => { - const primaryKeys = []; - const request = dbIndex.openKeyCursor(query, 'next'); - request.onerror = (e) => reject(e); - request.onsuccess = (e) => { - const cursor = e.target.result; - if (cursor) { - primaryKeys.push(cursor.primaryKey); - cursor.continue(); - } else { - resolve(primaryKeys); - } - }; - }); - } - - static async _deleteValues(dbObjectStore, dbIndex, query, onProgress, progressData, progressRate) { - const hasProgress = (typeof onProgress === 'function'); - const count = await Database._getCount(dbIndex, query); - ++progressData.storesProcesed; - progressData.count += count; - if (hasProgress) { - onProgress(progressData); - } - - const onValueDeleted = ( - hasProgress ? - () => { - const p = ++progressData.processed; - if ((p % progressRate) === 0 || p === progressData.count) { - onProgress(progressData); - } - } : - () => {} - ); - - const promises = []; - const primaryKeys = await Database._getAllKeys(dbIndex, query); - for (const key of primaryKeys) { - const promise = Database._deleteValue(dbObjectStore, key).then(onValueDeleted); - promises.push(promise); - } - - await Promise.all(promises); - } - - static _deleteValue(dbObjectStore, key) { - return new Promise((resolve, reject) => { - const request = dbObjectStore.delete(key); - request.onerror = (e) => reject(e); - request.onsuccess = () => resolve(); - }); - } - - static _open(name, version, onUpgradeNeeded) { - return new Promise((resolve, reject) => { - const request = indexedDB.open(name, version * 10); - - request.onupgradeneeded = (event) => { - try { - request.transaction.onerror = (e) => reject(e); - onUpgradeNeeded(request.result, request.transaction, event.oldVersion / 10, event.newVersion / 10); - } catch (e) { - reject(e); - } - }; - - request.onerror = (e) => reject(e); - request.onsuccess = () => resolve(request.result); - }); - } - - static _upgrade(db, transaction, oldVersion, upgrades) { - for (const {version, stores} of upgrades) { - if (oldVersion >= version) { continue; } - - const objectStoreNames = Object.keys(stores); - for (const objectStoreName of objectStoreNames) { - const {primaryKey, indices} = stores[objectStoreName]; - - const objectStoreNames2 = transaction.objectStoreNames || db.objectStoreNames; - const objectStore = ( - Database._listContains(objectStoreNames2, objectStoreName) ? - transaction.objectStore(objectStoreName) : - db.createObjectStore(objectStoreName, primaryKey) - ); - - for (const indexName of indices) { - if (Database._listContains(objectStore.indexNames, indexName)) { continue; } - - objectStore.createIndex(indexName, indexName, {}); - } - } - } - } - - static _deleteDatabase(dbName) { - return new Promise((resolve, reject) => { - const request = indexedDB.deleteDatabase(dbName); - request.onerror = (e) => reject(e); - request.onsuccess = () => resolve(); - }); - } - - static _listContains(list, value) { - for (let i = 0, ii = list.length; i < ii; ++i) { - if (list[i] === value) { return true; } - } - return false; - } } diff --git a/ext/bg/js/generic-database.js b/ext/bg/js/generic-database.js new file mode 100644 index 00000000..a82ad650 --- /dev/null +++ b/ext/bg/js/generic-database.js @@ -0,0 +1,327 @@ +/* + * Copyright (C) 2020 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +class GenericDatabase { + constructor() { + this._db = null; + this._isOpening = false; + } + + // Public + + async open(databaseName, version, structure) { + if (this._db !== null) { + throw new Error('Database already open'); + } + if (this._isOpening) { + throw new Error('Already opening'); + } + + try { + this._isOpening = true; + this._db = await this._open(databaseName, version, (db, transaction, oldVersion) => { + this._upgrade(db, transaction, oldVersion, structure); + }); + } finally { + this._isOpening = false; + } + } + + close() { + if (this._db === null) { + throw new Error('Database is not open'); + } + + this._db.close(); + this._db = null; + } + + isOpening() { + return this._isOpening; + } + + isOpen() { + return this._db !== null; + } + + transaction(storeNames, mode) { + if (this._db === null) { + throw new Error(this._isOpening ? 'Database not ready' : 'Database not open'); + } + return this._db.transaction(storeNames, mode); + } + + bulkAdd(objectStoreName, items, start, count) { + return new Promise((resolve, reject) => { + if (start + count > items.length) { + count = items.length - start; + } + + if (count <= 0) { + resolve(); + return; + } + + const end = start + count; + let completedCount = 0; + const onError = (e) => reject(e.target.error); + const onSuccess = () => { + if (++completedCount >= count) { + resolve(); + } + }; + + const transaction = this.transaction([objectStoreName], 'readwrite'); + const objectStore = transaction.objectStore(objectStoreName); + for (let i = start; i < end; ++i) { + const request = objectStore.add(items[i]); + request.onerror = onError; + request.onsuccess = onSuccess; + } + }); + } + + getAll(objectStoreOrIndex, query, resolve, reject) { + if (typeof objectStoreOrIndex.getAll === 'function') { + this._getAllFast(objectStoreOrIndex, query, resolve, reject); + } else { + this._getAllUsingCursor(objectStoreOrIndex, query, resolve, reject); + } + } + + getAllKeys(objectStoreOrIndex, query, resolve, reject) { + if (typeof objectStoreOrIndex.getAll === 'function') { + this._getAllKeysFast(objectStoreOrIndex, query, resolve, reject); + } else { + this._getAllKeysUsingCursor(objectStoreOrIndex, query, resolve, reject); + } + } + + find(objectStoreName, indexName, query, predicate=null, defaultValue) { + return new Promise((resolve, reject) => { + const transaction = this.transaction([objectStoreName], 'readonly'); + const objectStore = transaction.objectStore(objectStoreName); + const objectStoreOrIndex = indexName !== null ? objectStore.index(indexName) : objectStore; + const request = objectStoreOrIndex.openCursor(query, 'next'); + request.onerror = (e) => reject(e.target.error); + request.onsuccess = (e) => { + const cursor = e.target.result; + if (cursor) { + const value = cursor.value; + if (typeof predicate !== 'function' || predicate(value)) { + resolve(value); + } else { + cursor.continue(); + } + } else { + resolve(defaultValue); + } + }; + }); + } + + bulkCount(targets, resolve, reject) { + const targetCount = targets.length; + if (targetCount <= 0) { + resolve(); + return; + } + + let completedCount = 0; + const results = new Array(targetCount).fill(null); + + const onError = (e) => reject(e.target.error); + const onSuccess = (e, index) => { + const count = e.target.result; + results[index] = count; + if (++completedCount >= targetCount) { + resolve(results); + } + }; + + for (let i = 0; i < targetCount; ++i) { + const index = i; + const [objectStoreOrIndex, query] = targets[i]; + const request = objectStoreOrIndex.count(query); + request.onerror = onError; + request.onsuccess = (e) => onSuccess(e, index); + } + } + + delete(objectStoreName, key) { + return new Promise((resolve, reject) => { + const transaction = this.transaction([objectStoreName], 'readwrite'); + const objectStore = transaction.objectStore(objectStoreName); + const request = objectStore.delete(key); + request.onerror = (e) => reject(e.target.error); + request.onsuccess = () => resolve(); + }); + } + + bulkDelete(objectStoreName, indexName, query, filterKeys=null, onProgress=null) { + return new Promise((resolve, reject) => { + const transaction = this.transaction([objectStoreName], 'readwrite'); + const objectStore = transaction.objectStore(objectStoreName); + const objectStoreOrIndex = indexName !== null ? objectStore.index(indexName) : objectStore; + + const onGetKeys = (keys) => { + try { + if (typeof filterKeys === 'function') { + keys = filterKeys(keys); + } + this._bulkDeleteInternal(objectStore, keys, onProgress, resolve, reject); + } catch (e) { + reject(e); + } + }; + + this.getAllKeys(objectStoreOrIndex, query, onGetKeys, reject); + }); + } + + static deleteDatabase(databaseName) { + return new Promise((resolve, reject) => { + const request = indexedDB.deleteDatabase(databaseName); + request.onerror = (e) => reject(e.target.error); + request.onsuccess = () => resolve(); + request.onblocked = () => reject(new Error('Database deletion blocked')); + }); + } + + // Private + + _open(name, version, onUpgradeNeeded) { + return new Promise((resolve, reject) => { + const request = indexedDB.open(name, version); + + request.onupgradeneeded = (event) => { + try { + request.transaction.onerror = (e) => reject(e.target.error); + onUpgradeNeeded(request.result, request.transaction, event.oldVersion, event.newVersion); + } catch (e) { + reject(e); + } + }; + + request.onerror = (e) => reject(e.target.error); + request.onsuccess = () => resolve(request.result); + }); + } + + _upgrade(db, transaction, oldVersion, upgrades) { + for (const {version, stores} of upgrades) { + if (oldVersion >= version) { continue; } + + for (const [objectStoreName, {primaryKey, indices}] of Object.entries(stores)) { + const existingObjectStoreNames = transaction.objectStoreNames || db.objectStoreNames; + const objectStore = ( + this._listContains(existingObjectStoreNames, objectStoreName) ? + transaction.objectStore(objectStoreName) : + db.createObjectStore(objectStoreName, primaryKey) + ); + const existingIndexNames = objectStore.indexNames; + + for (const indexName of indices) { + if (this._listContains(existingIndexNames, indexName)) { continue; } + + objectStore.createIndex(indexName, indexName, {}); + } + } + } + } + + _listContains(list, value) { + for (let i = 0, ii = list.length; i < ii; ++i) { + if (list[i] === value) { return true; } + } + return false; + } + + _getAllFast(objectStoreOrIndex, query, resolve, reject) { + const request = objectStoreOrIndex.getAll(query); + request.onerror = (e) => reject(e.target.error); + request.onsuccess = (e) => resolve(e.target.result); + } + + _getAllUsingCursor(objectStoreOrIndex, query, resolve, reject) { + const results = []; + const request = objectStoreOrIndex.openCursor(query, 'next'); + request.onerror = (e) => reject(e.target.error); + request.onsuccess = (e) => { + const cursor = e.target.result; + if (cursor) { + results.push(cursor.value); + cursor.continue(); + } else { + resolve(results); + } + }; + } + + _getAllKeysFast(objectStoreOrIndex, query, resolve, reject) { + const request = objectStoreOrIndex.getAllKeys(query); + request.onerror = (e) => reject(e.target.error); + request.onsuccess = (e) => resolve(e.target.result); + } + + _getAllKeysUsingCursor(objectStoreOrIndex, query, resolve, reject) { + const results = []; + const request = objectStoreOrIndex.openKeyCursor(query, 'next'); + request.onerror = (e) => reject(e.target.error); + request.onsuccess = (e) => { + const cursor = e.target.result; + if (cursor) { + results.push(cursor.primaryKey); + cursor.continue(); + } else { + resolve(results); + } + }; + } + + _bulkDeleteInternal(objectStore, keys, onProgress, resolve, reject) { + const count = keys.length; + if (count === 0) { + resolve(); + return; + } + + let completedCount = 0; + const hasProgress = (typeof onProgress === 'function'); + + const onError = (e) => reject(e.target.error); + const onSuccess = () => { + ++completedCount; + if (hasProgress) { + try { + onProgress(completedCount, count); + } catch (e) { + // NOP + } + } + if (completedCount >= count) { + resolve(); + } + }; + + for (const key of keys) { + const request = objectStore.delete(key); + request.onerror = onError; + request.onsuccess = onSuccess; + } + } +} diff --git a/test/test-database.js b/test/test-database.js index 63989857..03b2bd3b 100644 --- a/test/test-database.js +++ b/test/test-database.js @@ -115,6 +115,7 @@ vm.execute([ 'bg/js/media-utility.js', 'bg/js/request.js', 'bg/js/dictionary-importer.js', + 'bg/js/generic-database.js', 'bg/js/database.js' ]); const DictionaryImporter = vm.get('DictionaryImporter'); @@ -242,8 +243,8 @@ async function testDatabase1() { true ); vm.assert.deepStrictEqual(counts, { - counts: [{kanji: 2, kanjiMeta: 2, terms: 33, termMeta: 12, tagMeta: 14}], - total: {kanji: 2, kanjiMeta: 2, terms: 33, termMeta: 12, tagMeta: 14} + counts: [{kanji: 2, kanjiMeta: 2, terms: 33, termMeta: 12, tagMeta: 14, media: 1}], + total: {kanji: 2, kanjiMeta: 2, terms: 33, termMeta: 12, tagMeta: 14, media: 1} }); // Test find* functions @@ -269,7 +270,7 @@ async function testDatabaseEmpty1(database) { const counts = await database.getDictionaryCounts([], true); vm.assert.deepStrictEqual(counts, { counts: [], - total: {kanji: 0, kanjiMeta: 0, terms: 0, termMeta: 0, tagMeta: 0} + total: {kanji: 0, kanjiMeta: 0, terms: 0, termMeta: 0, tagMeta: 0, media: 0} }); } @@ -863,8 +864,7 @@ async function testDatabase2() { const database = new Database(); // Error: not prepared - await assert.rejects(async () => await database.purge()); - await assert.rejects(async () => await database.deleteDictionary(title, {}, () => {})); + await assert.rejects(async () => await database.deleteDictionary(title, {rate: 1000}, () => {})); await assert.rejects(async () => await database.findTermsBulk(['?'], titles, null)); await assert.rejects(async () => await database.findTermsExactBulk(['?'], ['?'], titles)); await assert.rejects(async () => await database.findTermsBySequenceBulk([1], title)); -- cgit v1.2.3