diff options
| -rw-r--r-- | ext/bg/js/database.js | 172 |
1 files changed, 172 insertions, 0 deletions
diff --git a/ext/bg/js/database.js b/ext/bg/js/database.js index 7d3019bf..51e8d359 100644 --- a/ext/bg/js/database.js +++ b/ext/bg/js/database.js @@ -316,6 +316,169 @@ class Database { return result; } + async importDictionaryNew(archiveSource, onProgress, details) { + this._validate(); + const db = this.db; + const hasOnProgress = (typeof onProgress === 'function'); + + // Read archive + const archive = await JSZip.loadAsync(archiveSource); + + // Read and validate index + const indexFile = archive.files['index.json']; + if (!indexFile) { + throw new Error('No dictionary index found in archive'); + } + + const index = JSON.parse(await indexFile.async('string')); + + const dictionaryTitle = index.title; + const version = index.format || index.version; + + if (!dictionaryTitle || !index.revision) { + throw new Error('Unrecognized dictionary format'); + } + + // Verify database is not already imported + if (await this._dictionaryExists(dictionaryTitle)) { + throw new Error('Dictionary is already imported'); + } + + // Data format converters + const convertTermBankEntry = (entry) => { + if (version === 1) { + const [expression, reading, definitionTags, rules, score, ...glossary] = entry; + return {expression, reading, definitionTags, rules, score, glossary}; + } else { + const [expression, reading, definitionTags, rules, score, glossary, sequence, termTags] = entry; + return {expression, reading, definitionTags, rules, score, glossary, sequence, termTags}; + } + }; + + const convertTermMetaBankEntry = (entry) => { + const [expression, mode, data] = entry; + return {expression, mode, data}; + }; + + const convertKanjiBankEntry = (entry) => { + if (version === 1) { + const [character, onyomi, kunyomi, tags, ...meanings] = entry; + return {character, onyomi, kunyomi, tags, meanings}; + } else { + const [character, onyomi, kunyomi, tags, meanings, stats] = entry; + return {character, onyomi, kunyomi, tags, meanings, stats}; + } + }; + + const convertKanjiMetaBankEntry = (entry) => { + const [character, mode, data] = entry; + return {character, mode, data}; + }; + + const convertTagBankEntry = (entry) => { + const [name, category, order, notes, score] = entry; + return {name, category, order, notes, score}; + }; + + // Archive file reading + const readFileSequence = async (fileNameFormat, convertEntry) => { + const results = []; + for (let i = 1; true; ++i) { + const fileName = fileNameFormat.replace(/\?/, `${i}`); + const file = archive.files[fileName]; + if (!file) { break; } + + const entries = JSON.parse(await file.async('string')); + for (let entry of entries) { + entry = convertEntry(entry); + entry.dictionary = dictionaryTitle; + results.push(entry); + } + } + return results; + }; + + // Load data + const termList = await readFileSequence('term_bank_?.json', convertTermBankEntry); + const termMetaList = await readFileSequence('term_meta_bank_?.json', convertTermMetaBankEntry); + const kanjiList = await readFileSequence('kanji_bank_?.json', convertKanjiBankEntry); + const kanjiMetaList = await readFileSequence('kanji_meta_bank_?.json', convertKanjiMetaBankEntry); + const tagList = await readFileSequence('tag_bank_?.json', convertTagBankEntry); + + // Old tags + const indexTagMeta = index.tagMeta; + if (typeof indexTagMeta === 'object' && indexTagMeta !== null) { + for (const name of Object.keys(indexTagMeta)) { + const {category, order, notes, score} = indexTagMeta[name]; + tagList.push({name, category, order, notes, score}); + } + } + + // Prefix wildcard support + const prefixWildcardsSupported = !!details.prefixWildcardsSupported; + if (prefixWildcardsSupported) { + for (const entry of termList) { + entry.expressionReverse = stringReverse(entry.expression); + entry.readingReverse = stringReverse(entry.reading); + } + } + + // Add dictionary + const summary = { + title: dictionaryTitle, + revision: index.revision, + sequenced: index.sequenced, + version, + prefixWildcardsSupported + }; + + { + const transaction = db.transaction(['dictionaries'], 'readwrite'); + const objectStore = transaction.objectStore('dictionaries'); + await Database._bulkAdd(objectStore, [summary], 0, 1); + } + + // Add data + const errors = []; + const total = ( + termList.length + + termMetaList.length + + kanjiList.length + + kanjiMetaList.length + + tagList.length + ); + let loadedCount = 0; + const maxTransactionLength = 1000; + + const bulkAdd = async (objectStoreName, entries) => { + const ii = entries.length; + for (let i = 0; i < ii; i += maxTransactionLength) { + const count = Math.min(maxTransactionLength, ii - i); + + try { + const transaction = db.transaction([objectStoreName], 'readwrite'); + const objectStore = transaction.objectStore(objectStoreName); + await Database._bulkAdd(objectStore, entries, i, count); + } catch (e) { + errors.push(e); + } + + loadedCount += count; + if (hasOnProgress) { + onProgress(total, loadedCount); + } + } + }; + + await bulkAdd('terms', termList); + await bulkAdd('termMeta', termMetaList); + await bulkAdd('kanji', kanjiList); + await bulkAdd('kanjiMeta', kanjiMetaList); + await bulkAdd('tagMeta', tagList); + + return {result: summary, errors}; + } + async importDictionary(archive, progressCallback, details) { this._validate(); @@ -499,6 +662,15 @@ class Database { } } + async _dictionaryExists(title) { + const db = this.db; + const dbCountTransaction = db.transaction(['dictionaries'], 'readonly'); + const dbIndex = dbCountTransaction.objectStore('dictionaries').index('title'); + const only = IDBKeyRange.only(title); + const count = await Database._getCount(dbIndex, only); + return count > 0; + } + async _findGenericBulk(tableName, indexName, indexValueList, titles, createResult) { this._validate(); |