summaryrefslogtreecommitdiff
path: root/ext/bg
diff options
context:
space:
mode:
authortoasted-nutbread <toasted-nutbread@users.noreply.github.com>2020-02-17 18:17:12 -0500
committertoasted-nutbread <toasted-nutbread@users.noreply.github.com>2020-02-22 12:09:33 -0500
commit177bca4865f615dff5618aff13248777d34081a6 (patch)
treeef95c151e8a67fb12700c784c3469af023819caf /ext/bg
parent212e5428e81ad98f8af64eb3a6d2ab9d5b93f645 (diff)
Create new dictionary import function
Diffstat (limited to 'ext/bg')
-rw-r--r--ext/bg/js/database.js172
1 files changed, 172 insertions, 0 deletions
diff --git a/ext/bg/js/database.js b/ext/bg/js/database.js
index 7d3019bf..51e8d359 100644
--- a/ext/bg/js/database.js
+++ b/ext/bg/js/database.js
@@ -316,6 +316,169 @@ class Database {
return result;
}
+ async importDictionaryNew(archiveSource, onProgress, details) {
+ this._validate();
+ const db = this.db;
+ const hasOnProgress = (typeof onProgress === 'function');
+
+ // Read archive
+ const archive = await JSZip.loadAsync(archiveSource);
+
+ // Read and validate index
+ const indexFile = archive.files['index.json'];
+ if (!indexFile) {
+ throw new Error('No dictionary index found in archive');
+ }
+
+ const index = JSON.parse(await indexFile.async('string'));
+
+ const dictionaryTitle = index.title;
+ const version = index.format || index.version;
+
+ if (!dictionaryTitle || !index.revision) {
+ throw new Error('Unrecognized dictionary format');
+ }
+
+ // Verify database is not already imported
+ if (await this._dictionaryExists(dictionaryTitle)) {
+ throw new Error('Dictionary is already imported');
+ }
+
+ // Data format converters
+ const convertTermBankEntry = (entry) => {
+ if (version === 1) {
+ const [expression, reading, definitionTags, rules, score, ...glossary] = entry;
+ return {expression, reading, definitionTags, rules, score, glossary};
+ } else {
+ const [expression, reading, definitionTags, rules, score, glossary, sequence, termTags] = entry;
+ return {expression, reading, definitionTags, rules, score, glossary, sequence, termTags};
+ }
+ };
+
+ const convertTermMetaBankEntry = (entry) => {
+ const [expression, mode, data] = entry;
+ return {expression, mode, data};
+ };
+
+ const convertKanjiBankEntry = (entry) => {
+ if (version === 1) {
+ const [character, onyomi, kunyomi, tags, ...meanings] = entry;
+ return {character, onyomi, kunyomi, tags, meanings};
+ } else {
+ const [character, onyomi, kunyomi, tags, meanings, stats] = entry;
+ return {character, onyomi, kunyomi, tags, meanings, stats};
+ }
+ };
+
+ const convertKanjiMetaBankEntry = (entry) => {
+ const [character, mode, data] = entry;
+ return {character, mode, data};
+ };
+
+ const convertTagBankEntry = (entry) => {
+ const [name, category, order, notes, score] = entry;
+ return {name, category, order, notes, score};
+ };
+
+ // Archive file reading
+ const readFileSequence = async (fileNameFormat, convertEntry) => {
+ const results = [];
+ for (let i = 1; true; ++i) {
+ const fileName = fileNameFormat.replace(/\?/, `${i}`);
+ const file = archive.files[fileName];
+ if (!file) { break; }
+
+ const entries = JSON.parse(await file.async('string'));
+ for (let entry of entries) {
+ entry = convertEntry(entry);
+ entry.dictionary = dictionaryTitle;
+ results.push(entry);
+ }
+ }
+ return results;
+ };
+
+ // Load data
+ const termList = await readFileSequence('term_bank_?.json', convertTermBankEntry);
+ const termMetaList = await readFileSequence('term_meta_bank_?.json', convertTermMetaBankEntry);
+ const kanjiList = await readFileSequence('kanji_bank_?.json', convertKanjiBankEntry);
+ const kanjiMetaList = await readFileSequence('kanji_meta_bank_?.json', convertKanjiMetaBankEntry);
+ const tagList = await readFileSequence('tag_bank_?.json', convertTagBankEntry);
+
+ // Old tags
+ const indexTagMeta = index.tagMeta;
+ if (typeof indexTagMeta === 'object' && indexTagMeta !== null) {
+ for (const name of Object.keys(indexTagMeta)) {
+ const {category, order, notes, score} = indexTagMeta[name];
+ tagList.push({name, category, order, notes, score});
+ }
+ }
+
+ // Prefix wildcard support
+ const prefixWildcardsSupported = !!details.prefixWildcardsSupported;
+ if (prefixWildcardsSupported) {
+ for (const entry of termList) {
+ entry.expressionReverse = stringReverse(entry.expression);
+ entry.readingReverse = stringReverse(entry.reading);
+ }
+ }
+
+ // Add dictionary
+ const summary = {
+ title: dictionaryTitle,
+ revision: index.revision,
+ sequenced: index.sequenced,
+ version,
+ prefixWildcardsSupported
+ };
+
+ {
+ const transaction = db.transaction(['dictionaries'], 'readwrite');
+ const objectStore = transaction.objectStore('dictionaries');
+ await Database._bulkAdd(objectStore, [summary], 0, 1);
+ }
+
+ // Add data
+ const errors = [];
+ const total = (
+ termList.length +
+ termMetaList.length +
+ kanjiList.length +
+ kanjiMetaList.length +
+ tagList.length
+ );
+ let loadedCount = 0;
+ const maxTransactionLength = 1000;
+
+ const bulkAdd = async (objectStoreName, entries) => {
+ const ii = entries.length;
+ for (let i = 0; i < ii; i += maxTransactionLength) {
+ const count = Math.min(maxTransactionLength, ii - i);
+
+ try {
+ const transaction = db.transaction([objectStoreName], 'readwrite');
+ const objectStore = transaction.objectStore(objectStoreName);
+ await Database._bulkAdd(objectStore, entries, i, count);
+ } catch (e) {
+ errors.push(e);
+ }
+
+ loadedCount += count;
+ if (hasOnProgress) {
+ onProgress(total, loadedCount);
+ }
+ }
+ };
+
+ await bulkAdd('terms', termList);
+ await bulkAdd('termMeta', termMetaList);
+ await bulkAdd('kanji', kanjiList);
+ await bulkAdd('kanjiMeta', kanjiMetaList);
+ await bulkAdd('tagMeta', tagList);
+
+ return {result: summary, errors};
+ }
+
async importDictionary(archive, progressCallback, details) {
this._validate();
@@ -499,6 +662,15 @@ class Database {
}
}
+ async _dictionaryExists(title) {
+ const db = this.db;
+ const dbCountTransaction = db.transaction(['dictionaries'], 'readonly');
+ const dbIndex = dbCountTransaction.objectStore('dictionaries').index('title');
+ const only = IDBKeyRange.only(title);
+ const count = await Database._getCount(dbIndex, only);
+ return count > 0;
+ }
+
async _findGenericBulk(tableName, indexName, indexValueList, titles, createResult) {
this._validate();