diff options
Diffstat (limited to 'ext/bg/js/database.js')
| -rw-r--r-- | ext/bg/js/database.js | 428 |
1 files changed, 356 insertions, 72 deletions
diff --git a/ext/bg/js/database.js b/ext/bg/js/database.js index 9f477b24..a20d5f15 100644 --- a/ext/bg/js/database.js +++ b/ext/bg/js/database.js @@ -23,63 +23,137 @@ class Database { } async prepare() { - if (this.db) { + if (this.db !== null) { throw new Error('Database already initialized'); } - this.db = new Dexie('dict'); - this.db.version(2).stores({ - terms: '++id,dictionary,expression,reading', - kanji: '++,dictionary,character', - tagMeta: '++,dictionary', - dictionaries: '++,title,version' - }); - this.db.version(3).stores({ - termMeta: '++,dictionary,expression', - kanjiMeta: '++,dictionary,character', - tagMeta: '++,dictionary,name' - }); - this.db.version(4).stores({ - terms: '++id,dictionary,expression,reading,sequence' - }); - - await this.db.open(); + try { + this.db = await Database.open('dict', 4, (db, transaction, oldVersion) => { + Database.upgrade(db, transaction, oldVersion, [ + { + version: 2, + stores: { + terms: { + primaryKey: {keyPath: 'id', autoIncrement: true}, + indices: ['dictionary', 'expression', 'reading'] + }, + kanji: { + primaryKey: {autoIncrement: true}, + indices: ['dictionary', 'character'] + }, + tagMeta: { + primaryKey: {autoIncrement: true}, + indices: ['dictionary'] + }, + dictionaries: { + primaryKey: {autoIncrement: true}, + indices: ['title', 'version'] + } + } + }, + { + version: 3, + stores: { + termMeta: { + primaryKey: {autoIncrement: true}, + indices: ['dictionary', 'expression'] + }, + kanjiMeta: { + primaryKey: {autoIncrement: true}, + indices: ['dictionary', 'character'] + }, + tagMeta: { + primaryKey: {autoIncrement: true}, + indices: ['dictionary', 'name'] + } + } + }, + { + version: 4, + stores: { + terms: { + primaryKey: {keyPath: 'id', autoIncrement: true}, + indices: ['dictionary', 'expression', 'reading', 'sequence'] + } + } + } + ]); + }); + return true; + } catch (e) { + console.error(e); + return false; + } } async purge() { this.validate(); this.db.close(); - await this.db.delete(); + await Database.deleteDatabase(this.db.name); this.db = null; await this.prepare(); } - async findTermsBulk(termList, titles) { + async deleteDictionary(dictionaryName, onProgress, progressSettings) { + this.validate(); + + const targets = [ + ['dictionaries', 'title'], + ['kanji', 'dictionary'], + ['kanjiMeta', 'dictionary'], + ['terms', 'dictionary'], + ['termMeta', 'dictionary'], + ['tagMeta', 'dictionary'] + ]; + const promises = []; + const progressData = { + count: 0, + processed: 0, + storeCount: targets.length, + storesProcesed: 0 + }; + let progressRate = (typeof progressSettings === 'object' && progressSettings !== null ? progressSettings.rate : 0); + if (typeof progressRate !== 'number' || progressRate <= 0) { + progressRate = 1000; + } + + for (const [objectStoreName, index] of targets) { + const dbTransaction = this.db.transaction([objectStoreName], 'readwrite'); + const dbObjectStore = dbTransaction.objectStore(objectStoreName); + const dbIndex = dbObjectStore.index(index); + const only = IDBKeyRange.only(dictionaryName); + promises.push(Database.deleteValues(dbObjectStore, dbIndex, only, onProgress, progressData, progressRate)); + } + + await Promise.all(promises); + } + + async findTermsBulk(termList, titles, wildcard) { this.validate(); const promises = []; const visited = {}; const results = []; const processRow = (row, index) => { - if (titles.includes(row.dictionary) && !visited.hasOwnProperty(row.id)) { + if (titles.includes(row.dictionary) && !hasOwn(visited, row.id)) { visited[row.id] = true; results.push(Database.createTerm(row, index)); } }; - const db = this.db.backendDB(); - const dbTransaction = db.transaction(['terms'], 'readonly'); + const dbTransaction = this.db.transaction(['terms'], 'readonly'); const dbTerms = dbTransaction.objectStore('terms'); const dbIndex1 = dbTerms.index('expression'); const dbIndex2 = dbTerms.index('reading'); for (let i = 0; i < termList.length; ++i) { - const only = IDBKeyRange.only(termList[i]); + const term = termList[i]; + const query = wildcard ? IDBKeyRange.bound(term, `${term}\uffff`, false, false) : IDBKeyRange.only(term); promises.push( - Database.getAll(dbIndex1, only, i, processRow), - Database.getAll(dbIndex2, only, i, processRow) + Database.getAll(dbIndex1, query, i, processRow), + Database.getAll(dbIndex2, query, i, processRow) ); } @@ -99,8 +173,7 @@ class Database { } }; - const db = this.db.backendDB(); - const dbTransaction = db.transaction(['terms'], 'readonly'); + const dbTransaction = this.db.transaction(['terms'], 'readonly'); const dbTerms = dbTransaction.objectStore('terms'); const dbIndex = dbTerms.index('expression'); @@ -125,8 +198,7 @@ class Database { } }; - const db = this.db.backendDB(); - const dbTransaction = db.transaction(['terms'], 'readonly'); + const dbTransaction = this.db.transaction(['terms'], 'readonly'); const dbTerms = dbTransaction.objectStore('terms'); const dbIndex = dbTerms.index('sequence'); @@ -163,8 +235,7 @@ class Database { } }; - const db = this.db.backendDB(); - const dbTransaction = db.transaction([tableName], 'readonly'); + const dbTransaction = this.db.transaction([tableName], 'readonly'); const dbTerms = dbTransaction.objectStore(tableName); const dbIndex = dbTerms.index(indexName); @@ -182,12 +253,11 @@ class Database { this.validate(); let result = null; - const db = this.db.backendDB(); - const dbTransaction = db.transaction(['tagMeta'], 'readonly'); + const dbTransaction = this.db.transaction(['tagMeta'], 'readonly'); const dbTerms = dbTransaction.objectStore('tagMeta'); const dbIndex = dbTerms.index('name'); const only = IDBKeyRange.only(name); - await Database.getAll(dbIndex, only, null, row => { + await Database.getAll(dbIndex, only, null, (row) => { if (title === row.dictionary) { result = row; } @@ -196,24 +266,76 @@ class Database { return result; } - async summarize() { + async getDictionaryInfo() { + this.validate(); + + const results = []; + const dbTransaction = this.db.transaction(['dictionaries'], 'readonly'); + const dbDictionaries = dbTransaction.objectStore('dictionaries'); + + await Database.getAll(dbDictionaries, null, null, (info) => results.push(info)); + + return results; + } + + async getDictionaryCounts(dictionaryNames, getTotal) { this.validate(); - return this.db.dictionaries.toArray(); + const objectStoreNames = [ + 'kanji', + 'kanjiMeta', + 'terms', + 'termMeta', + 'tagMeta' + ]; + const dbCountTransaction = this.db.transaction(objectStoreNames, 'readonly'); + + const targets = []; + for (const objectStoreName of objectStoreNames) { + targets.push([ + objectStoreName, + dbCountTransaction.objectStore(objectStoreName).index('dictionary') + ]); + } + + // Query is required for Edge, otherwise index.count throws an exception. + const query1 = IDBKeyRange.lowerBound('', false); + const totalPromise = getTotal ? Database.getCounts(targets, query1) : null; + + const counts = []; + const countPromises = []; + for (let i = 0; i < dictionaryNames.length; ++i) { + counts.push(null); + const index = i; + const query2 = IDBKeyRange.only(dictionaryNames[i]); + const countPromise = Database.getCounts(targets, query2).then((v) => counts[index] = v); + countPromises.push(countPromise); + } + await Promise.all(countPromises); + + const result = {counts}; + if (totalPromise !== null) { + result.total = await totalPromise; + } + return result; } async importDictionary(archive, progressCallback, exceptions) { this.validate(); const maxTransactionLength = 1000; - const bulkAdd = async (table, items, total, current) => { - if (items.length < maxTransactionLength) { + const bulkAdd = async (objectStoreName, items, total, current) => { + const db = this.db; + for (let i = 0; i < items.length; i += maxTransactionLength) { if (progressCallback) { - progressCallback(total, current); + progressCallback(total, current + i / items.length); } try { - await table.bulkAdd(items); + const count = Math.min(maxTransactionLength, items.length - i); + const transaction = db.transaction([objectStoreName], 'readwrite'); + const objectStore = transaction.objectStore(objectStoreName); + await Database.bulkAdd(objectStore, items, i, count); } catch (e) { if (exceptions) { exceptions.push(e); @@ -221,37 +343,27 @@ class Database { throw e; } } - } else { - for (let i = 0; i < items.length; i += maxTransactionLength) { - if (progressCallback) { - progressCallback(total, current + i / items.length); - } - - let count = Math.min(maxTransactionLength, items.length - i); - try { - await table.bulkAdd(items.slice(i, i + count)); - } catch (e) { - if (exceptions) { - exceptions.push(e); - } else { - throw e; - } - } - } } }; - const indexDataLoaded = async summary => { + const indexDataLoaded = async (summary) => { if (summary.version > 3) { throw new Error('Unsupported dictionary version'); } - const count = await this.db.dictionaries.where('title').equals(summary.title).count(); + const db = this.db; + const dbCountTransaction = db.transaction(['dictionaries'], 'readonly'); + const dbIndex = dbCountTransaction.objectStore('dictionaries').index('title'); + const only = IDBKeyRange.only(summary.title); + const count = await Database.getCount(dbIndex, only); + if (count > 0) { throw new Error('Dictionary is already imported'); } - await this.db.dictionaries.add(summary); + const transaction = db.transaction(['dictionaries'], 'readwrite'); + const objectStore = transaction.objectStore('dictionaries'); + await Database.bulkAdd(objectStore, [summary], 0, 1); }; const termDataLoaded = async (summary, entries, total, current) => { @@ -284,7 +396,7 @@ class Database { } } - await bulkAdd(this.db.terms, rows, total, current); + await bulkAdd('terms', rows, total, current); }; const termMetaDataLoaded = async (summary, entries, total, current) => { @@ -298,7 +410,7 @@ class Database { }); } - await bulkAdd(this.db.termMeta, rows, total, current); + await bulkAdd('termMeta', rows, total, current); }; const kanjiDataLoaded = async (summary, entries, total, current) => { @@ -328,7 +440,7 @@ class Database { } } - await bulkAdd(this.db.kanji, rows, total, current); + await bulkAdd('kanji', rows, total, current); }; const kanjiMetaDataLoaded = async (summary, entries, total, current) => { @@ -342,7 +454,7 @@ class Database { }); } - await bulkAdd(this.db.kanjiMeta, rows, total, current); + await bulkAdd('kanjiMeta', rows, total, current); }; const tagDataLoaded = async (summary, entries, total, current) => { @@ -360,7 +472,7 @@ class Database { rows.push(row); } - await bulkAdd(this.db.tagMeta, rows, total, current); + await bulkAdd('tagMeta', rows, total, current); }; return await Database.importDictionaryZip( @@ -410,13 +522,13 @@ class Database { await indexDataLoaded(summary); - const buildTermBankName = index => `term_bank_${index + 1}.json`; - const buildTermMetaBankName = index => `term_meta_bank_${index + 1}.json`; - const buildKanjiBankName = index => `kanji_bank_${index + 1}.json`; - const buildKanjiMetaBankName = index => `kanji_meta_bank_${index + 1}.json`; - const buildTagBankName = index => `tag_bank_${index + 1}.json`; + const buildTermBankName = (index) => `term_bank_${index + 1}.json`; + const buildTermMetaBankName = (index) => `term_meta_bank_${index + 1}.json`; + const buildKanjiBankName = (index) => `kanji_bank_${index + 1}.json`; + const buildKanjiMetaBankName = (index) => `kanji_meta_bank_${index + 1}.json`; + const buildTagBankName = (index) => `tag_bank_${index + 1}.json`; - const countBanks = namer => { + const countBanks = (namer) => { let count = 0; while (zip.files[namer(count)]) { ++count; @@ -539,4 +651,176 @@ class Database { }; }); } + + static getCounts(targets, query) { + const countPromises = []; + const counts = {}; + for (const [objectStoreName, index] of targets) { + const n = objectStoreName; + const countPromise = Database.getCount(index, query).then((count) => counts[n] = count); + countPromises.push(countPromise); + } + return Promise.all(countPromises).then(() => counts); + } + + static getCount(dbIndex, query) { + return new Promise((resolve, reject) => { + const request = dbIndex.count(query); + request.onerror = (e) => reject(e); + request.onsuccess = (e) => resolve(e.target.result); + }); + } + + static getAllKeys(dbIndex, query) { + const fn = typeof dbIndex.getAllKeys === 'function' ? Database.getAllKeysFast : Database.getAllKeysUsingCursor; + return fn(dbIndex, query); + } + + static getAllKeysFast(dbIndex, query) { + return new Promise((resolve, reject) => { + const request = dbIndex.getAllKeys(query); + request.onerror = (e) => reject(e); + request.onsuccess = (e) => resolve(e.target.result); + }); + } + + static getAllKeysUsingCursor(dbIndex, query) { + return new Promise((resolve, reject) => { + const primaryKeys = []; + const request = dbIndex.openKeyCursor(query, 'next'); + request.onerror = (e) => reject(e); + request.onsuccess = (e) => { + const cursor = e.target.result; + if (cursor) { + primaryKeys.push(cursor.primaryKey); + cursor.continue(); + } else { + resolve(primaryKeys); + } + }; + }); + } + + static async deleteValues(dbObjectStore, dbIndex, query, onProgress, progressData, progressRate) { + const hasProgress = (typeof onProgress === 'function'); + const count = await Database.getCount(dbIndex, query); + ++progressData.storesProcesed; + progressData.count += count; + if (hasProgress) { + onProgress(progressData); + } + + const onValueDeleted = ( + hasProgress ? + () => { + const p = ++progressData.processed; + if ((p % progressRate) === 0 || p === progressData.count) { + onProgress(progressData); + } + } : + () => {} + ); + + const promises = []; + const primaryKeys = await Database.getAllKeys(dbIndex, query); + for (const key of primaryKeys) { + const promise = Database.deleteValue(dbObjectStore, key).then(onValueDeleted); + promises.push(promise); + } + + await Promise.all(promises); + } + + static deleteValue(dbObjectStore, key) { + return new Promise((resolve, reject) => { + const request = dbObjectStore.delete(key); + request.onerror = (e) => reject(e); + request.onsuccess = () => resolve(); + }); + } + + static bulkAdd(objectStore, items, start, count) { + return new Promise((resolve, reject) => { + if (start + count > items.length) { + count = items.length - start; + } + + if (count <= 0) { + resolve(); + return; + } + + const end = start + count; + let completedCount = 0; + const onError = (e) => reject(e); + const onSuccess = () => { + if (++completedCount >= count) { + resolve(); + } + }; + + for (let i = start; i < end; ++i) { + const request = objectStore.add(items[i]); + request.onerror = onError; + request.onsuccess = onSuccess; + } + }); + } + + static open(name, version, onUpgradeNeeded) { + return new Promise((resolve, reject) => { + const request = window.indexedDB.open(name, version * 10); + + request.onupgradeneeded = (event) => { + try { + request.transaction.onerror = (e) => reject(e); + onUpgradeNeeded(request.result, request.transaction, event.oldVersion / 10, event.newVersion / 10); + } catch (e) { + reject(e); + } + }; + + request.onerror = (e) => reject(e); + request.onsuccess = () => resolve(request.result); + }); + } + + static upgrade(db, transaction, oldVersion, upgrades) { + for (const {version, stores} of upgrades) { + if (oldVersion >= version) { continue; } + + const objectStoreNames = Object.keys(stores); + for (const objectStoreName of objectStoreNames) { + const {primaryKey, indices} = stores[objectStoreName]; + + const objectStoreNames = transaction.objectStoreNames || db.objectStoreNames; + const objectStore = ( + Database.listContains(objectStoreNames, objectStoreName) ? + transaction.objectStore(objectStoreName) : + db.createObjectStore(objectStoreName, primaryKey) + ); + + for (const indexName of indices) { + if (Database.listContains(objectStore.indexNames, indexName)) { continue; } + + objectStore.createIndex(indexName, indexName, {}); + } + } + } + } + + static deleteDatabase(dbName) { + return new Promise((resolve, reject) => { + const request = indexedDB.deleteDatabase(dbName); + request.onerror = (e) => reject(e); + request.onsuccess = () => resolve(); + }); + } + + static listContains(list, value) { + for (let i = 0, ii = list.length; i < ii; ++i) { + if (list[i] === value) { return true; } + } + return false; + } } |