diff options
| -rw-r--r-- | ext/bg/background.html | 1 | ||||
| -rw-r--r-- | ext/bg/js/backend.js | 6 | ||||
| -rw-r--r-- | ext/bg/js/database.js | 807 | ||||
| -rw-r--r-- | ext/bg/js/generic-database.js | 327 | ||||
| -rw-r--r-- | test/test-database.js | 10 |
5 files changed, 652 insertions, 499 deletions
diff --git a/ext/bg/background.html b/ext/bg/background.html index f2d01e85..55380ae7 100644 --- a/ext/bg/background.html +++ b/ext/bg/background.html @@ -30,6 +30,7 @@ <script src="/bg/js/audio-uri-builder.js"></script> <script src="/bg/js/clipboard-monitor.js"></script> <script src="/bg/js/conditions.js"></script> + <script src="/bg/js/generic-database.js"></script> <script src="/bg/js/database.js"></script> <script src="/bg/js/dictionary-importer.js"></script> <script src="/bg/js/deinflector.js"></script> diff --git a/ext/bg/js/backend.js b/ext/bg/js/backend.js index a8ee9f83..b89cb641 100644 --- a/ext/bg/js/backend.js +++ b/ext/bg/js/backend.js @@ -172,7 +172,11 @@ class Backend { yomichan.on('log', this._onLog.bind(this)); await this.environment.prepare(); - await this.database.prepare(); + try { + await this.database.prepare(); + } catch (e) { + yomichan.logError(e); + } await this.translator.prepare(); await profileConditionsDescriptorPromise; diff --git a/ext/bg/js/database.js b/ext/bg/js/database.js index 65e267ab..47f1ebdd 100644 --- a/ext/bg/js/database.js +++ b/ext/bg/js/database.js @@ -16,423 +16,422 @@ */ /* global + * GenericDatabase * dictFieldSplit */ class Database { constructor() { - this.db = null; + this._db = new GenericDatabase(); + this._dbName = 'dict'; this._schemas = new Map(); } // Public async prepare() { - if (this.db !== null) { - throw new Error('Database already initialized'); - } - - try { - this.db = await Database._open('dict', 6, (db, transaction, oldVersion) => { - Database._upgrade(db, transaction, oldVersion, [ - { - version: 2, - stores: { - terms: { - primaryKey: {keyPath: 'id', autoIncrement: true}, - indices: ['dictionary', 'expression', 'reading'] - }, - kanji: { - primaryKey: {autoIncrement: true}, - indices: ['dictionary', 'character'] - }, - tagMeta: { - primaryKey: {autoIncrement: true}, - indices: ['dictionary'] - }, - dictionaries: { - primaryKey: {autoIncrement: true}, - indices: ['title', 'version'] - } + await this._db.open( + this._dbName, + 60, + [ + { + version: 20, + stores: { + terms: { + primaryKey: {keyPath: 'id', autoIncrement: true}, + indices: ['dictionary', 'expression', 'reading'] + }, + kanji: { + primaryKey: {autoIncrement: true}, + indices: ['dictionary', 'character'] + }, + tagMeta: { + primaryKey: {autoIncrement: true}, + indices: ['dictionary'] + }, + dictionaries: { + primaryKey: {autoIncrement: true}, + indices: ['title', 'version'] } - }, - { - version: 3, - stores: { - termMeta: { - primaryKey: {autoIncrement: true}, - indices: ['dictionary', 'expression'] - }, - kanjiMeta: { - primaryKey: {autoIncrement: true}, - indices: ['dictionary', 'character'] - }, - tagMeta: { - primaryKey: {autoIncrement: true}, - indices: ['dictionary', 'name'] - } + } + }, + { + version: 30, + stores: { + termMeta: { + primaryKey: {autoIncrement: true}, + indices: ['dictionary', 'expression'] + }, + kanjiMeta: { + primaryKey: {autoIncrement: true}, + indices: ['dictionary', 'character'] + }, + tagMeta: { + primaryKey: {autoIncrement: true}, + indices: ['dictionary', 'name'] } - }, - { - version: 4, - stores: { - terms: { - primaryKey: {keyPath: 'id', autoIncrement: true}, - indices: ['dictionary', 'expression', 'reading', 'sequence'] - } + } + }, + { + version: 40, + stores: { + terms: { + primaryKey: {keyPath: 'id', autoIncrement: true}, + indices: ['dictionary', 'expression', 'reading', 'sequence'] } - }, - { - version: 5, - stores: { - terms: { - primaryKey: {keyPath: 'id', autoIncrement: true}, - indices: ['dictionary', 'expression', 'reading', 'sequence', 'expressionReverse', 'readingReverse'] - } + } + }, + { + version: 50, + stores: { + terms: { + primaryKey: {keyPath: 'id', autoIncrement: true}, + indices: ['dictionary', 'expression', 'reading', 'sequence', 'expressionReverse', 'readingReverse'] } - }, - { - version: 6, - stores: { - media: { - primaryKey: {keyPath: 'id', autoIncrement: true}, - indices: ['dictionary', 'path'] - } + } + }, + { + version: 60, + stores: { + media: { + primaryKey: {keyPath: 'id', autoIncrement: true}, + indices: ['dictionary', 'path'] } } - ]); - }); - return true; - } catch (e) { - yomichan.logError(e); - return false; - } + } + ] + ); } async close() { - this._validate(); - this.db.close(); - this.db = null; + this._db.close(); } isPrepared() { - return this.db !== null; + return this._db.isOpen(); } async purge() { - this._validate(); - - this.db.close(); - await Database._deleteDatabase(this.db.name); - this.db = null; - + if (this._db.isOpening()) { + throw new Error('Cannot purge database while opening'); + } + if (this._db.isOpen()) { + this._db.close(); + } + await GenericDatabase.deleteDatabase(this._dbName); await this.prepare(); } async deleteDictionary(dictionaryName, progressSettings, onProgress) { - this._validate(); - const targets = [ ['dictionaries', 'title'], ['kanji', 'dictionary'], ['kanjiMeta', 'dictionary'], ['terms', 'dictionary'], ['termMeta', 'dictionary'], - ['tagMeta', 'dictionary'] + ['tagMeta', 'dictionary'], + ['media', 'dictionary'] ]; - const promises = []; + + const {rate} = progressSettings; const progressData = { count: 0, processed: 0, storeCount: targets.length, storesProcesed: 0 }; - let progressRate = (typeof progressSettings === 'object' && progressSettings !== null ? progressSettings.rate : 0); - if (typeof progressRate !== 'number' || progressRate <= 0) { - progressRate = 1000; - } - - for (const [objectStoreName, index] of targets) { - const dbTransaction = this.db.transaction([objectStoreName], 'readwrite'); - const dbObjectStore = dbTransaction.objectStore(objectStoreName); - const dbIndex = dbObjectStore.index(index); - const only = IDBKeyRange.only(dictionaryName); - promises.push(Database._deleteValues(dbObjectStore, dbIndex, only, onProgress, progressData, progressRate)); - } - - await Promise.all(promises); - } - - async findTermsBulk(termList, dictionaries, wildcard) { - this._validate(); - const promises = []; - const visited = new Set(); - const results = []; - const processRow = (row, index) => { - if (dictionaries.has(row.dictionary) && !visited.has(row.id)) { - visited.add(row.id); - results.push(Database._createTerm(row, index)); + const filterKeys = (keys) => { + ++progressData.storesProcesed; + progressData.count += keys.length; + onProgress(progressData); + return keys; + }; + const onProgress2 = () => { + const processed = progressData.processed + 1; + progressData.processed = processed; + if ((processed % rate) === 0 || processed === progressData.count) { + onProgress(progressData); } }; - const useWildcard = !!wildcard; - const prefixWildcard = wildcard === 'prefix'; - - const dbTransaction = this.db.transaction(['terms'], 'readonly'); - const dbTerms = dbTransaction.objectStore('terms'); - const dbIndex1 = dbTerms.index(prefixWildcard ? 'expressionReverse' : 'expression'); - const dbIndex2 = dbTerms.index(prefixWildcard ? 'readingReverse' : 'reading'); - - for (let i = 0; i < termList.length; ++i) { - const term = prefixWildcard ? stringReverse(termList[i]) : termList[i]; - const query = useWildcard ? IDBKeyRange.bound(term, `${term}\uffff`, false, false) : IDBKeyRange.only(term); - promises.push( - Database._getAll(dbIndex1, query, i, processRow), - Database._getAll(dbIndex2, query, i, processRow) - ); + const promises = []; + for (const [objectStoreName, indexName] of targets) { + const query = IDBKeyRange.only(dictionaryName); + const promise = this._db.bulkDelete(objectStoreName, indexName, query, filterKeys, onProgress2); + promises.push(promise); } - await Promise.all(promises); - - return results; } - async findTermsExactBulk(termList, readingList, dictionaries) { - this._validate(); - - const promises = []; - const results = []; - const processRow = (row, index) => { - if (row.reading === readingList[index] && dictionaries.has(row.dictionary)) { - results.push(Database._createTerm(row, index)); + findTermsBulk(termList, dictionaries, wildcard) { + return new Promise((resolve, reject) => { + const results = []; + const count = termList.length; + if (count === 0) { + resolve(results); + return; } - }; - - const dbTransaction = this.db.transaction(['terms'], 'readonly'); - const dbTerms = dbTransaction.objectStore('terms'); - const dbIndex = dbTerms.index('expression'); - for (let i = 0; i < termList.length; ++i) { - const only = IDBKeyRange.only(termList[i]); - promises.push(Database._getAll(dbIndex, only, i, processRow)); - } - - await Promise.all(promises); + const visited = new Set(); + const useWildcard = !!wildcard; + const prefixWildcard = wildcard === 'prefix'; + + const transaction = this._db.transaction(['terms'], 'readonly'); + const terms = transaction.objectStore('terms'); + const index1 = terms.index(prefixWildcard ? 'expressionReverse' : 'expression'); + const index2 = terms.index(prefixWildcard ? 'readingReverse' : 'reading'); + + const count2 = count * 2; + let completeCount = 0; + for (let i = 0; i < count; ++i) { + const inputIndex = i; + const term = prefixWildcard ? stringReverse(termList[i]) : termList[i]; + const query = useWildcard ? IDBKeyRange.bound(term, `${term}\uffff`, false, false) : IDBKeyRange.only(term); + + const onGetAll = (rows) => { + for (const row of rows) { + if (dictionaries.has(row.dictionary) && !visited.has(row.id)) { + visited.add(row.id); + results.push(this._createTerm(row, inputIndex)); + } + } + if (++completeCount >= count2) { + resolve(results); + } + }; - return results; + this._db.getAll(index1, query, onGetAll, reject); + this._db.getAll(index2, query, onGetAll, reject); + } + }); } - async findTermsBySequenceBulk(sequenceList, mainDictionary) { - this._validate(); - - const promises = []; - const results = []; - const processRow = (row, index) => { - if (row.dictionary === mainDictionary) { - results.push(Database._createTerm(row, index)); + findTermsExactBulk(termList, readingList, dictionaries) { + return new Promise((resolve, reject) => { + const results = []; + const count = termList.length; + if (count === 0) { + resolve(results); + return; } - }; - - const dbTransaction = this.db.transaction(['terms'], 'readonly'); - const dbTerms = dbTransaction.objectStore('terms'); - const dbIndex = dbTerms.index('sequence'); - - for (let i = 0; i < sequenceList.length; ++i) { - const only = IDBKeyRange.only(sequenceList[i]); - promises.push(Database._getAll(dbIndex, only, i, processRow)); - } - - await Promise.all(promises); - return results; - } - - async findTermMetaBulk(termList, dictionaries) { - return this._findGenericBulk('termMeta', 'expression', termList, dictionaries, Database._createTermMeta); - } + const transaction = this._db.transaction(['terms'], 'readonly'); + const terms = transaction.objectStore('terms'); + const index = terms.index('expression'); - async findKanjiBulk(kanjiList, dictionaries) { - return this._findGenericBulk('kanji', 'character', kanjiList, dictionaries, Database._createKanji); - } + let completeCount = 0; + for (let i = 0; i < count; ++i) { + const inputIndex = i; + const reading = readingList[i]; + const query = IDBKeyRange.only(termList[i]); - async findKanjiMetaBulk(kanjiList, dictionaries) { - return this._findGenericBulk('kanjiMeta', 'character', kanjiList, dictionaries, Database._createKanjiMeta); - } + const onGetAll = (rows) => { + for (const row of rows) { + if (row.reading === reading && dictionaries.has(row.dictionary)) { + results.push(this._createTerm(row, inputIndex)); + } + } + if (++completeCount >= count) { + resolve(results); + } + }; - async findTagForTitle(name, title) { - this._validate(); - - let result = null; - const dbTransaction = this.db.transaction(['tagMeta'], 'readonly'); - const dbTerms = dbTransaction.objectStore('tagMeta'); - const dbIndex = dbTerms.index('name'); - const only = IDBKeyRange.only(name); - await Database._getAll(dbIndex, only, null, (row) => { - if (title === row.dictionary) { - result = row; + this._db.getAll(index, query, onGetAll, reject); } }); - - return result; } - async getMedia(targets) { - this._validate(); - - const count = targets.length; - const promises = []; - const results = new Array(count).fill(null); - const createResult = Database._createMedia; - const processRow = (row, [index, dictionaryName]) => { - if (row.dictionary === dictionaryName) { - results[index] = createResult(row, index); + findTermsBySequenceBulk(sequenceList, mainDictionary) { + return new Promise((resolve, reject) => { + const results = []; + const count = sequenceList.length; + if (count === 0) { + resolve(results); + return; } - }; - const transaction = this.db.transaction(['media'], 'readonly'); - const objectStore = transaction.objectStore('media'); - const index = objectStore.index('path'); + const transaction = this._db.transaction(['terms'], 'readonly'); + const terms = transaction.objectStore('terms'); + const index = terms.index('sequence'); - for (let i = 0; i < count; ++i) { - const {path, dictionaryName} = targets[i]; - const only = IDBKeyRange.only(path); - promises.push(Database._getAll(index, only, [i, dictionaryName], processRow)); - } + let completeCount = 0; + for (let i = 0; i < count; ++i) { + const inputIndex = i; + const query = IDBKeyRange.only(sequenceList[i]); - await Promise.all(promises); + const onGetAll = (rows) => { + for (const row of rows) { + if (row.dictionary === mainDictionary) { + results.push(this._createTerm(row, inputIndex)); + } + } + if (++completeCount >= count) { + resolve(results); + } + }; - return results; + this._db.getAll(index, query, onGetAll, reject); + } + }); } - async getDictionaryInfo() { - this._validate(); + findTermMetaBulk(termList, dictionaries) { + return this._findGenericBulk('termMeta', 'expression', termList, dictionaries, this._createTermMeta.bind(this)); + } - const results = []; - const dbTransaction = this.db.transaction(['dictionaries'], 'readonly'); - const dbDictionaries = dbTransaction.objectStore('dictionaries'); + findKanjiBulk(kanjiList, dictionaries) { + return this._findGenericBulk('kanji', 'character', kanjiList, dictionaries, this._createKanji.bind(this)); + } - await Database._getAll(dbDictionaries, null, null, (info) => results.push(info)); + findKanjiMetaBulk(kanjiList, dictionaries) { + return this._findGenericBulk('kanjiMeta', 'character', kanjiList, dictionaries, this._createKanjiMeta.bind(this)); + } - return results; + findTagForTitle(name, title) { + const query = IDBKeyRange.only(name); + return this._db.find('tagMeta', 'name', query, (row) => (row.dictionary === title), null); } - async getDictionaryCounts(dictionaryNames, getTotal) { - this._validate(); + getMedia(targets) { + return new Promise((resolve, reject) => { + const count = targets.length; + const results = new Array(count).fill(null); + if (count === 0) { + resolve(results); + return; + } - const objectStoreNames = [ - 'kanji', - 'kanjiMeta', - 'terms', - 'termMeta', - 'tagMeta' - ]; - const dbCountTransaction = this.db.transaction(objectStoreNames, 'readonly'); - - const targets = []; - for (const objectStoreName of objectStoreNames) { - targets.push([ - objectStoreName, - dbCountTransaction.objectStore(objectStoreName).index('dictionary') - ]); - } + let completeCount = 0; + const transaction = this._db.transaction(['media'], 'readonly'); + const objectStore = transaction.objectStore('media'); + const index = objectStore.index('path'); - // Query is required for Edge, otherwise index.count throws an exception. - const query1 = IDBKeyRange.lowerBound('', false); - const totalPromise = getTotal ? Database._getCounts(targets, query1) : null; - - const counts = []; - const countPromises = []; - for (let i = 0; i < dictionaryNames.length; ++i) { - counts.push(null); - const index = i; - const query2 = IDBKeyRange.only(dictionaryNames[i]); - const countPromise = Database._getCounts(targets, query2).then((v) => counts[index] = v); - countPromises.push(countPromise); - } - await Promise.all(countPromises); + for (let i = 0; i < count; ++i) { + const inputIndex = i; + const {path, dictionaryName} = targets[i]; + const query = IDBKeyRange.only(path); - const result = {counts}; - if (totalPromise !== null) { - result.total = await totalPromise; - } - return result; + const onGetAll = (rows) => { + for (const row of rows) { + if (row.dictionary !== dictionaryName) { continue; } + results[inputIndex] = this._createMedia(row, inputIndex); + } + if (++completeCount >= count) { + resolve(results); + } + }; + + this._db.getAll(index, query, onGetAll, reject); + } + }); } - async dictionaryExists(title) { - this._validate(); - const transaction = this.db.transaction(['dictionaries'], 'readonly'); - const index = transaction.objectStore('dictionaries').index('title'); - const query = IDBKeyRange.only(title); - const count = await Database._getCount(index, query); - return count > 0; + getDictionaryInfo() { + return new Promise((resolve, reject) => { + const transaction = this._db.transaction(['dictionaries'], 'readonly'); + const objectStore = transaction.objectStore('dictionaries'); + this._db.getAll(objectStore, null, resolve, reject); + }); } - bulkAdd(objectStoreName, items, start, count) { + getDictionaryCounts(dictionaryNames, getTotal) { return new Promise((resolve, reject) => { - const transaction = this.db.transaction([objectStoreName], 'readwrite'); - const objectStore = transaction.objectStore(objectStoreName); + const targets = [ + ['kanji', 'dictionary'], + ['kanjiMeta', 'dictionary'], + ['terms', 'dictionary'], + ['termMeta', 'dictionary'], + ['tagMeta', 'dictionary'], + ['media', 'dictionary'] + ]; + const objectStoreNames = targets.map(([objectStoreName]) => objectStoreName); + const transaction = this._db.transaction(objectStoreNames, 'readonly'); + const databaseTargets = targets.map(([objectStoreName, indexName]) => { + const objectStore = transaction.objectStore(objectStoreName); + const index = objectStore.index(indexName); + return {objectStore, index}; + }); - if (start + count > items.length) { - count = items.length - start; + const countTargets = []; + if (getTotal) { + for (const {objectStore} of databaseTargets) { + countTargets.push([objectStore, null]); + } } - - if (count <= 0) { - resolve(); - return; + for (const dictionaryName of dictionaryNames) { + const query = IDBKeyRange.only(dictionaryName); + for (const {index} of databaseTargets) { + countTargets.push([index, query]); + } } - const end = start + count; - let completedCount = 0; - const onError = (e) => reject(e); - const onSuccess = () => { - if (++completedCount >= count) { - resolve(); + const onCountComplete = (results) => { + const resultCount = results.length; + const targetCount = targets.length; + const counts = []; + for (let i = 0; i < resultCount; i += targetCount) { + const countGroup = {}; + for (let j = 0; j < targetCount; ++j) { + countGroup[targets[j][0]] = results[i + j]; + } + counts.push(countGroup); } + const total = getTotal ? counts.shift() : null; + resolve({total, counts}); }; - for (let i = start; i < end; ++i) { - const request = objectStore.add(items[i]); - request.onerror = onError; - request.onsuccess = onSuccess; - } + this._db.bulkCount(countTargets, onCountComplete, reject); }); } - // Private + async dictionaryExists(title) { + const query = IDBKeyRange.only(title); + const result = await this._db.find('dictionaries', 'title', query); + return typeof result !== 'undefined'; + } - _validate() { - if (this.db === null) { - throw new Error('Database not initialized'); - } + bulkAdd(objectStoreName, items, start, count) { + return this._db.bulkAdd(objectStoreName, items, start, count); } - async _findGenericBulk(tableName, indexName, indexValueList, dictionaries, createResult) { - this._validate(); + // Private - const promises = []; - const results = []; - const processRow = (row, index) => { - if (dictionaries.has(row.dictionary)) { - results.push(createResult(row, index)); + async _findGenericBulk(objectStoreName, indexName, indexValueList, dictionaries, createResult) { + return new Promise((resolve, reject) => { + const results = []; + const count = indexValueList.length; + if (count === 0) { + resolve(results); + return; } - }; - const dbTransaction = this.db.transaction([tableName], 'readonly'); - const dbTerms = dbTransaction.objectStore(tableName); - const dbIndex = dbTerms.index(indexName); + const transaction = this._db.transaction([objectStoreName], 'readonly'); + const terms = transaction.objectStore(objectStoreName); + const index = terms.index(indexName); - for (let i = 0; i < indexValueList.length; ++i) { - const only = IDBKeyRange.only(indexValueList[i]); - promises.push(Database._getAll(dbIndex, only, i, processRow)); - } + let completeCount = 0; + for (let i = 0; i < count; ++i) { + const inputIndex = i; + const query = IDBKeyRange.only(indexValueList[i]); - await Promise.all(promises); + const onGetAll = (rows) => { + for (const row of rows) { + if (dictionaries.has(row.dictionary)) { + results.push(createResult(row, inputIndex)); + } + } + if (++completeCount >= count) { + resolve(results); + } + }; - return results; + this._db.getAll(index, query, onGetAll, reject); + } + }); } - static _createTerm(row, index) { + _createTerm(row, index) { return { index, expression: row.expression, @@ -448,7 +447,7 @@ class Database { }; } - static _createKanji(row, index) { + _createKanji(row, index) { return { index, character: row.character, @@ -461,193 +460,15 @@ class Database { }; } - static _createTermMeta({expression, mode, data, dictionary}, index) { + _createTermMeta({expression, mode, data, dictionary}, index) { return {expression, mode, data, dictionary, index}; } - static _createKanjiMeta({character, mode, data, dictionary}, index) { + _createKanjiMeta({character, mode, data, dictionary}, index) { return {character, mode, data, dictionary, index}; } - static _createMedia(row, index) { + _createMedia(row, index) { return Object.assign({}, row, {index}); } - - static _getAll(dbIndex, query, context, processRow) { - const fn = typeof dbIndex.getAll === 'function' ? Database._getAllFast : Database._getAllUsingCursor; - return fn(dbIndex, query, context, processRow); - } - - static _getAllFast(dbIndex, query, context, processRow) { - return new Promise((resolve, reject) => { - const request = dbIndex.getAll(query); - request.onerror = (e) => reject(e); - request.onsuccess = (e) => { - for (const row of e.target.result) { - processRow(row, context); - } - resolve(); - }; - }); - } - - static _getAllUsingCursor(dbIndex, query, context, processRow) { - return new Promise((resolve, reject) => { - const request = dbIndex.openCursor(query, 'next'); - request.onerror = (e) => reject(e); - request.onsuccess = (e) => { - const cursor = e.target.result; - if (cursor) { - processRow(cursor.value, context); - cursor.continue(); - } else { - resolve(); - } - }; - }); - } - - static _getCounts(targets, query) { - const countPromises = []; - const counts = {}; - for (const [objectStoreName, index] of targets) { - const n = objectStoreName; - const countPromise = Database._getCount(index, query).then((count) => counts[n] = count); - countPromises.push(countPromise); - } - return Promise.all(countPromises).then(() => counts); - } - - static _getCount(dbIndex, query) { - return new Promise((resolve, reject) => { - const request = dbIndex.count(query); - request.onerror = (e) => reject(e); - request.onsuccess = (e) => resolve(e.target.result); - }); - } - - static _getAllKeys(dbIndex, query) { - const fn = typeof dbIndex.getAllKeys === 'function' ? Database._getAllKeysFast : Database._getAllKeysUsingCursor; - return fn(dbIndex, query); - } - - static _getAllKeysFast(dbIndex, query) { - return new Promise((resolve, reject) => { - const request = dbIndex.getAllKeys(query); - request.onerror = (e) => reject(e); - request.onsuccess = (e) => resolve(e.target.result); - }); - } - - static _getAllKeysUsingCursor(dbIndex, query) { - return new Promise((resolve, reject) => { - const primaryKeys = []; - const request = dbIndex.openKeyCursor(query, 'next'); - request.onerror = (e) => reject(e); - request.onsuccess = (e) => { - const cursor = e.target.result; - if (cursor) { - primaryKeys.push(cursor.primaryKey); - cursor.continue(); - } else { - resolve(primaryKeys); - } - }; - }); - } - - static async _deleteValues(dbObjectStore, dbIndex, query, onProgress, progressData, progressRate) { - const hasProgress = (typeof onProgress === 'function'); - const count = await Database._getCount(dbIndex, query); - ++progressData.storesProcesed; - progressData.count += count; - if (hasProgress) { - onProgress(progressData); - } - - const onValueDeleted = ( - hasProgress ? - () => { - const p = ++progressData.processed; - if ((p % progressRate) === 0 || p === progressData.count) { - onProgress(progressData); - } - } : - () => {} - ); - - const promises = []; - const primaryKeys = await Database._getAllKeys(dbIndex, query); - for (const key of primaryKeys) { - const promise = Database._deleteValue(dbObjectStore, key).then(onValueDeleted); - promises.push(promise); - } - - await Promise.all(promises); - } - - static _deleteValue(dbObjectStore, key) { - return new Promise((resolve, reject) => { - const request = dbObjectStore.delete(key); - request.onerror = (e) => reject(e); - request.onsuccess = () => resolve(); - }); - } - - static _open(name, version, onUpgradeNeeded) { - return new Promise((resolve, reject) => { - const request = indexedDB.open(name, version * 10); - - request.onupgradeneeded = (event) => { - try { - request.transaction.onerror = (e) => reject(e); - onUpgradeNeeded(request.result, request.transaction, event.oldVersion / 10, event.newVersion / 10); - } catch (e) { - reject(e); - } - }; - - request.onerror = (e) => reject(e); - request.onsuccess = () => resolve(request.result); - }); - } - - static _upgrade(db, transaction, oldVersion, upgrades) { - for (const {version, stores} of upgrades) { - if (oldVersion >= version) { continue; } - - const objectStoreNames = Object.keys(stores); - for (const objectStoreName of objectStoreNames) { - const {primaryKey, indices} = stores[objectStoreName]; - - const objectStoreNames2 = transaction.objectStoreNames || db.objectStoreNames; - const objectStore = ( - Database._listContains(objectStoreNames2, objectStoreName) ? - transaction.objectStore(objectStoreName) : - db.createObjectStore(objectStoreName, primaryKey) - ); - - for (const indexName of indices) { - if (Database._listContains(objectStore.indexNames, indexName)) { continue; } - - objectStore.createIndex(indexName, indexName, {}); - } - } - } - } - - static _deleteDatabase(dbName) { - return new Promise((resolve, reject) => { - const request = indexedDB.deleteDatabase(dbName); - request.onerror = (e) => reject(e); - request.onsuccess = () => resolve(); - }); - } - - static _listContains(list, value) { - for (let i = 0, ii = list.length; i < ii; ++i) { - if (list[i] === value) { return true; } - } - return false; - } } diff --git a/ext/bg/js/generic-database.js b/ext/bg/js/generic-database.js new file mode 100644 index 00000000..a82ad650 --- /dev/null +++ b/ext/bg/js/generic-database.js @@ -0,0 +1,327 @@ +/* + * Copyright (C) 2020 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +class GenericDatabase { + constructor() { + this._db = null; + this._isOpening = false; + } + + // Public + + async open(databaseName, version, structure) { + if (this._db !== null) { + throw new Error('Database already open'); + } + if (this._isOpening) { + throw new Error('Already opening'); + } + + try { + this._isOpening = true; + this._db = await this._open(databaseName, version, (db, transaction, oldVersion) => { + this._upgrade(db, transaction, oldVersion, structure); + }); + } finally { + this._isOpening = false; + } + } + + close() { + if (this._db === null) { + throw new Error('Database is not open'); + } + + this._db.close(); + this._db = null; + } + + isOpening() { + return this._isOpening; + } + + isOpen() { + return this._db !== null; + } + + transaction(storeNames, mode) { + if (this._db === null) { + throw new Error(this._isOpening ? 'Database not ready' : 'Database not open'); + } + return this._db.transaction(storeNames, mode); + } + + bulkAdd(objectStoreName, items, start, count) { + return new Promise((resolve, reject) => { + if (start + count > items.length) { + count = items.length - start; + } + + if (count <= 0) { + resolve(); + return; + } + + const end = start + count; + let completedCount = 0; + const onError = (e) => reject(e.target.error); + const onSuccess = () => { + if (++completedCount >= count) { + resolve(); + } + }; + + const transaction = this.transaction([objectStoreName], 'readwrite'); + const objectStore = transaction.objectStore(objectStoreName); + for (let i = start; i < end; ++i) { + const request = objectStore.add(items[i]); + request.onerror = onError; + request.onsuccess = onSuccess; + } + }); + } + + getAll(objectStoreOrIndex, query, resolve, reject) { + if (typeof objectStoreOrIndex.getAll === 'function') { + this._getAllFast(objectStoreOrIndex, query, resolve, reject); + } else { + this._getAllUsingCursor(objectStoreOrIndex, query, resolve, reject); + } + } + + getAllKeys(objectStoreOrIndex, query, resolve, reject) { + if (typeof objectStoreOrIndex.getAll === 'function') { + this._getAllKeysFast(objectStoreOrIndex, query, resolve, reject); + } else { + this._getAllKeysUsingCursor(objectStoreOrIndex, query, resolve, reject); + } + } + + find(objectStoreName, indexName, query, predicate=null, defaultValue) { + return new Promise((resolve, reject) => { + const transaction = this.transaction([objectStoreName], 'readonly'); + const objectStore = transaction.objectStore(objectStoreName); + const objectStoreOrIndex = indexName !== null ? objectStore.index(indexName) : objectStore; + const request = objectStoreOrIndex.openCursor(query, 'next'); + request.onerror = (e) => reject(e.target.error); + request.onsuccess = (e) => { + const cursor = e.target.result; + if (cursor) { + const value = cursor.value; + if (typeof predicate !== 'function' || predicate(value)) { + resolve(value); + } else { + cursor.continue(); + } + } else { + resolve(defaultValue); + } + }; + }); + } + + bulkCount(targets, resolve, reject) { + const targetCount = targets.length; + if (targetCount <= 0) { + resolve(); + return; + } + + let completedCount = 0; + const results = new Array(targetCount).fill(null); + + const onError = (e) => reject(e.target.error); + const onSuccess = (e, index) => { + const count = e.target.result; + results[index] = count; + if (++completedCount >= targetCount) { + resolve(results); + } + }; + + for (let i = 0; i < targetCount; ++i) { + const index = i; + const [objectStoreOrIndex, query] = targets[i]; + const request = objectStoreOrIndex.count(query); + request.onerror = onError; + request.onsuccess = (e) => onSuccess(e, index); + } + } + + delete(objectStoreName, key) { + return new Promise((resolve, reject) => { + const transaction = this.transaction([objectStoreName], 'readwrite'); + const objectStore = transaction.objectStore(objectStoreName); + const request = objectStore.delete(key); + request.onerror = (e) => reject(e.target.error); + request.onsuccess = () => resolve(); + }); + } + + bulkDelete(objectStoreName, indexName, query, filterKeys=null, onProgress=null) { + return new Promise((resolve, reject) => { + const transaction = this.transaction([objectStoreName], 'readwrite'); + const objectStore = transaction.objectStore(objectStoreName); + const objectStoreOrIndex = indexName !== null ? objectStore.index(indexName) : objectStore; + + const onGetKeys = (keys) => { + try { + if (typeof filterKeys === 'function') { + keys = filterKeys(keys); + } + this._bulkDeleteInternal(objectStore, keys, onProgress, resolve, reject); + } catch (e) { + reject(e); + } + }; + + this.getAllKeys(objectStoreOrIndex, query, onGetKeys, reject); + }); + } + + static deleteDatabase(databaseName) { + return new Promise((resolve, reject) => { + const request = indexedDB.deleteDatabase(databaseName); + request.onerror = (e) => reject(e.target.error); + request.onsuccess = () => resolve(); + request.onblocked = () => reject(new Error('Database deletion blocked')); + }); + } + + // Private + + _open(name, version, onUpgradeNeeded) { + return new Promise((resolve, reject) => { + const request = indexedDB.open(name, version); + + request.onupgradeneeded = (event) => { + try { + request.transaction.onerror = (e) => reject(e.target.error); + onUpgradeNeeded(request.result, request.transaction, event.oldVersion, event.newVersion); + } catch (e) { + reject(e); + } + }; + + request.onerror = (e) => reject(e.target.error); + request.onsuccess = () => resolve(request.result); + }); + } + + _upgrade(db, transaction, oldVersion, upgrades) { + for (const {version, stores} of upgrades) { + if (oldVersion >= version) { continue; } + + for (const [objectStoreName, {primaryKey, indices}] of Object.entries(stores)) { + const existingObjectStoreNames = transaction.objectStoreNames || db.objectStoreNames; + const objectStore = ( + this._listContains(existingObjectStoreNames, objectStoreName) ? + transaction.objectStore(objectStoreName) : + db.createObjectStore(objectStoreName, primaryKey) + ); + const existingIndexNames = objectStore.indexNames; + + for (const indexName of indices) { + if (this._listContains(existingIndexNames, indexName)) { continue; } + + objectStore.createIndex(indexName, indexName, {}); + } + } + } + } + + _listContains(list, value) { + for (let i = 0, ii = list.length; i < ii; ++i) { + if (list[i] === value) { return true; } + } + return false; + } + + _getAllFast(objectStoreOrIndex, query, resolve, reject) { + const request = objectStoreOrIndex.getAll(query); + request.onerror = (e) => reject(e.target.error); + request.onsuccess = (e) => resolve(e.target.result); + } + + _getAllUsingCursor(objectStoreOrIndex, query, resolve, reject) { + const results = []; + const request = objectStoreOrIndex.openCursor(query, 'next'); + request.onerror = (e) => reject(e.target.error); + request.onsuccess = (e) => { + const cursor = e.target.result; + if (cursor) { + results.push(cursor.value); + cursor.continue(); + } else { + resolve(results); + } + }; + } + + _getAllKeysFast(objectStoreOrIndex, query, resolve, reject) { + const request = objectStoreOrIndex.getAllKeys(query); + request.onerror = (e) => reject(e.target.error); + request.onsuccess = (e) => resolve(e.target.result); + } + + _getAllKeysUsingCursor(objectStoreOrIndex, query, resolve, reject) { + const results = []; + const request = objectStoreOrIndex.openKeyCursor(query, 'next'); + request.onerror = (e) => reject(e.target.error); + request.onsuccess = (e) => { + const cursor = e.target.result; + if (cursor) { + results.push(cursor.primaryKey); + cursor.continue(); + } else { + resolve(results); + } + }; + } + + _bulkDeleteInternal(objectStore, keys, onProgress, resolve, reject) { + const count = keys.length; + if (count === 0) { + resolve(); + return; + } + + let completedCount = 0; + const hasProgress = (typeof onProgress === 'function'); + + const onError = (e) => reject(e.target.error); + const onSuccess = () => { + ++completedCount; + if (hasProgress) { + try { + onProgress(completedCount, count); + } catch (e) { + // NOP + } + } + if (completedCount >= count) { + resolve(); + } + }; + + for (const key of keys) { + const request = objectStore.delete(key); + request.onerror = onError; + request.onsuccess = onSuccess; + } + } +} diff --git a/test/test-database.js b/test/test-database.js index 63989857..03b2bd3b 100644 --- a/test/test-database.js +++ b/test/test-database.js @@ -115,6 +115,7 @@ vm.execute([ 'bg/js/media-utility.js', 'bg/js/request.js', 'bg/js/dictionary-importer.js', + 'bg/js/generic-database.js', 'bg/js/database.js' ]); const DictionaryImporter = vm.get('DictionaryImporter'); @@ -242,8 +243,8 @@ async function testDatabase1() { true ); vm.assert.deepStrictEqual(counts, { - counts: [{kanji: 2, kanjiMeta: 2, terms: 33, termMeta: 12, tagMeta: 14}], - total: {kanji: 2, kanjiMeta: 2, terms: 33, termMeta: 12, tagMeta: 14} + counts: [{kanji: 2, kanjiMeta: 2, terms: 33, termMeta: 12, tagMeta: 14, media: 1}], + total: {kanji: 2, kanjiMeta: 2, terms: 33, termMeta: 12, tagMeta: 14, media: 1} }); // Test find* functions @@ -269,7 +270,7 @@ async function testDatabaseEmpty1(database) { const counts = await database.getDictionaryCounts([], true); vm.assert.deepStrictEqual(counts, { counts: [], - total: {kanji: 0, kanjiMeta: 0, terms: 0, termMeta: 0, tagMeta: 0} + total: {kanji: 0, kanjiMeta: 0, terms: 0, termMeta: 0, tagMeta: 0, media: 0} }); } @@ -863,8 +864,7 @@ async function testDatabase2() { const database = new Database(); // Error: not prepared - await assert.rejects(async () => await database.purge()); - await assert.rejects(async () => await database.deleteDictionary(title, {}, () => {})); + await assert.rejects(async () => await database.deleteDictionary(title, {rate: 1000}, () => {})); await assert.rejects(async () => await database.findTermsBulk(['?'], titles, null)); await assert.rejects(async () => await database.findTermsExactBulk(['?'], ['?'], titles)); await assert.rejects(async () => await database.findTermsBySequenceBulk([1], title)); |