aboutsummaryrefslogtreecommitdiff
path: root/ext/bg/js/database.js
diff options
context:
space:
mode:
authorAlex Yatskov <alex@foosoft.net>2017-09-24 11:01:40 -0700
committerAlex Yatskov <alex@foosoft.net>2017-09-24 11:01:40 -0700
commit00c20aed4253da6506194d90a6102920d3235ced (patch)
treecfb2fd51a441241cb333a208b3ce3130add93953 /ext/bg/js/database.js
parent3a1aad07d61411f634e86f905babd6fbbac2eae1 (diff)
parent29812c1b036d3eae9b5b222219f556aec712c3e8 (diff)
Merge branch 'master' into firefox-amo
Diffstat (limited to 'ext/bg/js/database.js')
-rw-r--r--ext/bg/js/database.js366
1 files changed, 254 insertions, 112 deletions
diff --git a/ext/bg/js/database.js b/ext/bg/js/database.js
index e00cb7a3..6ceb3ec8 100644
--- a/ext/bg/js/database.js
+++ b/ext/bg/js/database.js
@@ -20,44 +20,33 @@
class Database {
constructor() {
this.db = null;
- this.version = 2;
this.tagCache = {};
}
- async sanitize() {
- try {
- const db = new Dexie('dict');
- await db.open();
- db.close();
- if (db.verno !== this.version) {
- await db.delete();
- }
- } catch(e) {
- // NOP
- }
- }
-
async prepare() {
if (this.db) {
- throw 'database already initialized';
+ throw 'Database already initialized';
}
- await this.sanitize();
-
this.db = new Dexie('dict');
- this.db.version(this.version).stores({
- terms: '++id,dictionary,expression,reading',
- kanji: '++,dictionary,character',
- tagMeta: '++,dictionary',
+ this.db.version(2).stores({
+ terms: '++id,dictionary,expression,reading',
+ kanji: '++,dictionary,character',
+ tagMeta: '++,dictionary',
dictionaries: '++,title,version'
});
+ this.db.version(3).stores({
+ termMeta: '++,dictionary,expression',
+ kanjiMeta: '++,dictionary,character',
+ tagMeta: '++,dictionary,name'
+ });
await this.db.open();
}
async purge() {
if (!this.db) {
- throw 'database not initialized';
+ throw 'Database not initialized';
}
this.db.close();
@@ -70,7 +59,7 @@ class Database {
async findTerms(term, titles) {
if (!this.db) {
- throw 'database not initialized';
+ throw 'Database not initialized';
}
const results = [];
@@ -89,17 +78,31 @@ class Database {
}
});
- await this.cacheTagMeta(titles);
- for (const result of results) {
- result.tagMeta = this.tagCache[result.dictionary] || {};
+ return results;
+ }
+
+ async findTermMeta(term, titles) {
+ if (!this.db) {
+ throw 'Database not initialized';
}
+ const results = [];
+ await this.db.termMeta.where('expression').equals(term).each(row => {
+ if (titles.includes(row.dictionary)) {
+ results.push({
+ mode: row.mode,
+ data: row.data,
+ dictionary: row.dictionary
+ });
+ }
+ });
+
return results;
}
async findKanji(kanji, titles) {
if (!this.db) {
- return Promise.reject('database not initialized');
+ throw 'Database not initialized';
}
const results = [];
@@ -111,165 +114,304 @@ class Database {
kunyomi: dictFieldSplit(row.kunyomi),
tags: dictFieldSplit(row.tags),
glossary: row.meanings,
+ stats: row.stats,
dictionary: row.dictionary
});
}
});
- await this.cacheTagMeta(titles);
- for (const result of results) {
- result.tagMeta = this.tagCache[result.dictionary] || {};
+ return results;
+ }
+
+ async findKanjiMeta(kanji, titles) {
+ if (!this.db) {
+ throw 'Database not initialized';
}
+ const results = [];
+ await this.db.kanjiMeta.where('character').equals(kanji).each(row => {
+ if (titles.includes(row.dictionary)) {
+ results.push({
+ mode: row.mode,
+ data: row.data,
+ dictionary: row.dictionary
+ });
+ }
+ });
+
return results;
}
- async cacheTagMeta(titles) {
+ async findTagForTitle(name, title) {
if (!this.db) {
- throw 'database not initialized';
+ throw 'Database not initialized';
}
- for (const title of titles) {
- if (!this.tagCache[title]) {
- const tagMeta = {};
- await this.db.tagMeta.where('dictionary').equals(title).each(row => {
- tagMeta[row.name] = {category: row.category, notes: row.notes, order: row.order};
- });
+ this.tagCache[title] = this.tagCache[title] || {};
- this.tagCache[title] = tagMeta;
- }
+ let result = this.tagCache[title][name];
+ if (!result) {
+ await this.db.tagMeta.where('name').equals(name).each(row => {
+ if (title === row.dictionary) {
+ result = row;
+ }
+ });
+
+ this.tagCache[title][name] = result;
}
+
+ return result;
}
- async getDictionaries() {
+ async getTitles() {
if (this.db) {
return this.db.dictionaries.toArray();
} else {
- throw 'database not initialized';
+ throw 'Database not initialized';
}
}
async importDictionary(archive, callback) {
if (!this.db) {
- return Promise.reject('database not initialized');
+ throw 'Database not initialized';
}
- let summary = null;
- const indexLoaded = async (title, version, revision, tagMeta, hasTerms, hasKanji) => {
- summary = {title, version, revision, hasTerms, hasKanji};
+ const indexDataLoaded = async summary => {
+ if (summary.version > 2) {
+ throw 'Unsupported dictionary version';
+ }
- const count = await this.db.dictionaries.where('title').equals(title).count();
+ const count = await this.db.dictionaries.where('title').equals(summary.title).count();
if (count > 0) {
- throw `dictionary "${title}" is already imported`;
+ throw 'Dictionary is already imported';
}
- await this.db.dictionaries.add({title, version, revision, hasTerms, hasKanji});
+ await this.db.dictionaries.add(summary);
+ };
- const rows = [];
- for (const tag in tagMeta || {}) {
- const meta = tagMeta[tag];
- const row = dictTagSanitize({
- name: tag,
- category: meta.category,
- notes: meta.notes,
- order: meta.order,
- dictionary: title
- });
+ const termDataLoaded = async (summary, entries, total, current) => {
+ if (callback) {
+ callback(total, current);
+ }
- rows.push(row);
+ const rows = [];
+ if (summary.version === 1) {
+ for (const [expression, reading, tags, rules, score, ...glossary] of entries) {
+ rows.push({
+ expression,
+ reading,
+ tags,
+ rules,
+ score,
+ glossary,
+ dictionary: summary.title
+ });
+ }
+ } else {
+ for (const [expression, reading, tags, rules, score, glossary] of entries) {
+ rows.push({
+ expression,
+ reading,
+ tags,
+ rules,
+ score,
+ glossary,
+ dictionary: summary.title
+ });
+ }
}
- await this.db.tagMeta.bulkAdd(rows);
+ await this.db.terms.bulkAdd(rows);
};
- const termsLoaded = async (title, entries, total, current) => {
+ const termMetaDataLoaded = async (summary, entries, total, current) => {
if (callback) {
callback(total, current);
}
const rows = [];
- for (const [expression, reading, tags, rules, score, ...glossary] of entries) {
+ for (const [expression, mode, data] of entries) {
rows.push({
expression,
- reading,
- tags,
- rules,
- score,
- glossary,
- dictionary: title
+ mode,
+ data,
+ dictionary: summary.title
});
}
- await this.db.terms.bulkAdd(rows);
+ await this.db.termMeta.bulkAdd(rows);
+ };
+
+ const kanjiDataLoaded = async (summary, entries, total, current) => {
+ if (callback) {
+ callback(total, current);
+ }
+
+ const rows = [];
+ if (summary.version === 1) {
+ for (const [character, onyomi, kunyomi, tags, ...meanings] of entries) {
+ rows.push({
+ character,
+ onyomi,
+ kunyomi,
+ tags,
+ meanings,
+ dictionary: summary.title
+ });
+ }
+ } else {
+ for (const [character, onyomi, kunyomi, tags, meanings, stats] of entries) {
+ rows.push({
+ character,
+ onyomi,
+ kunyomi,
+ tags,
+ meanings,
+ stats,
+ dictionary: summary.title
+ });
+ }
+ }
+
+ await this.db.kanji.bulkAdd(rows);
};
- const kanjiLoaded = async (title, entries, total, current) => {
+ const kanjiMetaDataLoaded = async (summary, entries, total, current) => {
if (callback) {
callback(total, current);
}
const rows = [];
- for (const [character, onyomi, kunyomi, tags, ...meanings] of entries) {
+ for (const [character, mode, data] of entries) {
rows.push({
character,
- onyomi,
- kunyomi,
- tags,
- meanings,
- dictionary: title
+ mode,
+ data,
+ dictionary: summary.title
});
}
- await this.db.kanji.bulkAdd(rows);
+ await this.db.kanjiMeta.bulkAdd(rows);
};
- await Database.importDictionaryZip(archive, indexLoaded, termsLoaded, kanjiLoaded);
- return summary;
- }
+ const tagDataLoaded = async (summary, entries, total, current) => {
+ if (callback) {
+ callback(total, current);
+ }
- static async importDictionaryZip(archive, indexLoaded, termsLoaded, kanjiLoaded) {
- const files = (await JSZip.loadAsync(archive)).files;
+ const rows = [];
+ for (const [name, category, order, notes] of entries) {
+ const row = dictTagSanitize({
+ name,
+ category,
+ order,
+ notes,
+ dictionary: summary.title
+ });
+
+ rows.push(row);
+ }
- const indexFile = files['index.json'];
+ await this.db.tagMeta.bulkAdd(rows);
+ };
+
+ return await Database.importDictionaryZip(
+ archive,
+ indexDataLoaded,
+ termDataLoaded,
+ termMetaDataLoaded,
+ kanjiDataLoaded,
+ kanjiMetaDataLoaded,
+ tagDataLoaded
+ );
+ }
+
+ static async importDictionaryZip(
+ archive,
+ indexDataLoaded,
+ termDataLoaded,
+ termMetaDataLoaded,
+ kanjiDataLoaded,
+ kanjiMetaDataLoaded,
+ tagDataLoaded
+ ) {
+ const zip = await JSZip.loadAsync(archive);
+
+ const indexFile = zip.files['index.json'];
if (!indexFile) {
- throw 'no dictionary index found in archive';
+ throw 'No dictionary index found in archive';
}
const index = JSON.parse(await indexFile.async('string'));
- if (!index.title || !index.version || !index.revision) {
- throw 'unrecognized dictionary format';
+ if (!index.title || !index.revision) {
+ throw 'Unrecognized dictionary format';
}
- await indexLoaded(
- index.title,
- index.version,
- index.revision,
- index.tagMeta || {},
- index.termBanks > 0,
- index.kanjiBanks > 0
- );
+ const summary = {
+ title: index.title,
+ revision: index.revision,
+ version: index.format || index.version
+ };
+
+ if (indexDataLoaded) {
+ await indexDataLoaded(summary);
+ }
- const banksTotal = index.termBanks + index.kanjiBanks;
- let banksLoaded = 0;
+ const buildTermBankName = index => `term_bank_${index + 1}.json`;
+ const buildTermMetaBankName = index => `term_meta_bank_${index + 1}.json`;
+ const buildKanjiBankName = index => `kanji_bank_${index + 1}.json`;
+ const buildKanjiMetaBankName = index => `kanji_meta_bank_${index + 1}.json`;
+ const buildTagBankName = index => `tag_bank_${index + 1}.json`;
- for (let i = 1; i <= index.termBanks; ++i) {
- const bankFile = files[`term_bank_${i}.json`];
- if (bankFile) {
- const bank = JSON.parse(await bankFile.async('string'));
- await termsLoaded(index.title, bank, banksTotal, banksLoaded++);
- } else {
- throw 'missing term bank file';
+ const countBanks = namer => {
+ let count = 0;
+ while (zip.files[namer(count)]) {
+ ++count;
}
- }
- for (let i = 1; i <= index.kanjiBanks; ++i) {
- const bankFile = files[`kanji_bank_${i}.json`];
- if (bankFile) {
- const bank = JSON.parse(await bankFile.async('string'));
- await kanjiLoaded(index.title, bank, banksTotal, banksLoaded++);
- } else {
- throw 'missing kanji bank file';
+ return count;
+ };
+
+ const termBankCount = countBanks(buildTermBankName);
+ const termMetaBankCount = countBanks(buildTermMetaBankName);
+ const kanjiBankCount = countBanks(buildKanjiBankName);
+ const kanjiMetaBankCount = countBanks(buildKanjiMetaBankName);
+ const tagBankCount = countBanks(buildTagBankName);
+
+ let bankLoadedCount = 0;
+ let bankTotalCount =
+ termBankCount +
+ termMetaBankCount +
+ kanjiBankCount +
+ kanjiMetaBankCount +
+ tagBankCount;
+
+ if (tagDataLoaded && index.tagMeta) {
+ const bank = [];
+ for (const name in index.tagMeta) {
+ const tag = index.tagMeta[name];
+ bank.push([name, tag.category, tag.order, tag.notes]);
}
+
+ tagDataLoaded(summary, bank, ++bankTotalCount, bankLoadedCount++);
}
+
+ const loadBank = async (summary, namer, count, callback) => {
+ if (callback) {
+ for (let i = 0; i < count; ++i) {
+ const bankFile = zip.files[namer(i)];
+ const bank = JSON.parse(await bankFile.async('string'));
+ await callback(summary, bank, bankTotalCount, bankLoadedCount++);
+ }
+ }
+ };
+
+ await loadBank(summary, buildTermBankName, termBankCount, termDataLoaded);
+ await loadBank(summary, buildTermMetaBankName, termMetaBankCount, termMetaDataLoaded);
+ await loadBank(summary, buildKanjiBankName, kanjiBankCount, kanjiDataLoaded);
+ await loadBank(summary, buildKanjiMetaBankName, kanjiMetaBankCount, kanjiMetaDataLoaded);
+ await loadBank(summary, buildTagBankName, tagBankCount, tagDataLoaded);
+
+ return summary;
}
}