aboutsummaryrefslogtreecommitdiff
path: root/ext/js/language/dictionary-database.js
diff options
context:
space:
mode:
authortoasted-nutbread <toasted-nutbread@users.noreply.github.com>2021-02-14 11:19:54 -0500
committerGitHub <noreply@github.com>2021-02-14 11:19:54 -0500
commite419a418f6f03ef0a24330b67e7b76c5e3a7c22d (patch)
treea4c27bdfabc9280d9f6262d93d5152a58de8bc15 /ext/js/language/dictionary-database.js
parent43d1457ebfe23196348649c245dfb942a0f00a1a (diff)
Move bg/js (#1387)
* Move bg/js/anki.js to js/comm/anki.js * Move bg/js/mecab.js to js/comm/mecab.js * Move bg/js/search-main.js to js/display/search-main.js * Move bg/js/template-patcher.js to js/templates/template-patcher.js * Move bg/js/template-renderer-frame-api.js to js/templates/template-renderer-frame-api.js * Move bg/js/template-renderer-frame-main.js to js/templates/template-renderer-frame-main.js * Move bg/js/template-renderer-proxy.js to js/templates/template-renderer-proxy.js * Move bg/js/template-renderer.js to js/templates/template-renderer.js * Move bg/js/media-utility.js to js/media/media-utility.js * Move bg/js/native-simple-dom-parser.js to js/dom/native-simple-dom-parser.js * Move bg/js/simple-dom-parser.js to js/dom/simple-dom-parser.js * Move bg/js/audio-downloader.js to js/media/audio-downloader.js * Move bg/js/deinflector.js to js/language/deinflector.js * Move bg/js/backend.js to js/background/backend.js * Move bg/js/translator.js to js/language/translator.js * Move bg/js/search-display-controller.js to js/display/search-display-controller.js * Move bg/js/request-builder.js to js/background/request-builder.js * Move bg/js/text-source-map.js to js/general/text-source-map.js * Move bg/js/clipboard-reader.js to js/comm/clipboard-reader.js * Move bg/js/clipboard-monitor.js to js/comm/clipboard-monitor.js * Move bg/js/query-parser.js to js/display/query-parser.js * Move bg/js/profile-conditions.js to js/background/profile-conditions.js * Move bg/js/dictionary-database.js to js/language/dictionary-database.js * Move bg/js/dictionary-importer.js to js/language/dictionary-importer.js * Move bg/js/anki-note-builder.js to js/data/anki-note-builder.js * Move bg/js/anki-note-data.js to js/data/anki-note-data.js * Move bg/js/database.js to js/data/database.js * Move bg/js/json-schema.js to js/data/json-schema.js * Move bg/js/options.js to js/data/options-util.js * Move bg/js/background-main.js to js/background/background-main.js * Move bg/js/permissions-util.js to js/data/permissions-util.js * Move bg/js/context-main.js to js/pages/action-popup-main.js * Move bg/js/generic-page-main.js to js/pages/generic-page-main.js * Move bg/js/info-main.js to js/pages/info-main.js * Move bg/js/permissions-main.js to js/pages/permissions-main.js * Move bg/js/welcome-main.js to js/pages/welcome-main.js
Diffstat (limited to 'ext/js/language/dictionary-database.js')
-rw-r--r--ext/js/language/dictionary-database.js484
1 files changed, 484 insertions, 0 deletions
diff --git a/ext/js/language/dictionary-database.js b/ext/js/language/dictionary-database.js
new file mode 100644
index 00000000..b363ed25
--- /dev/null
+++ b/ext/js/language/dictionary-database.js
@@ -0,0 +1,484 @@
+/*
+ * Copyright (C) 2016-2021 Yomichan Authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/* global
+ * Database
+ */
+
+class DictionaryDatabase {
+ constructor() {
+ this._db = new Database();
+ this._dbName = 'dict';
+ this._schemas = new Map();
+ }
+
+ // Public
+
+ async prepare() {
+ await this._db.open(
+ this._dbName,
+ 60,
+ [
+ {
+ version: 20,
+ stores: {
+ terms: {
+ primaryKey: {keyPath: 'id', autoIncrement: true},
+ indices: ['dictionary', 'expression', 'reading']
+ },
+ kanji: {
+ primaryKey: {autoIncrement: true},
+ indices: ['dictionary', 'character']
+ },
+ tagMeta: {
+ primaryKey: {autoIncrement: true},
+ indices: ['dictionary']
+ },
+ dictionaries: {
+ primaryKey: {autoIncrement: true},
+ indices: ['title', 'version']
+ }
+ }
+ },
+ {
+ version: 30,
+ stores: {
+ termMeta: {
+ primaryKey: {autoIncrement: true},
+ indices: ['dictionary', 'expression']
+ },
+ kanjiMeta: {
+ primaryKey: {autoIncrement: true},
+ indices: ['dictionary', 'character']
+ },
+ tagMeta: {
+ primaryKey: {autoIncrement: true},
+ indices: ['dictionary', 'name']
+ }
+ }
+ },
+ {
+ version: 40,
+ stores: {
+ terms: {
+ primaryKey: {keyPath: 'id', autoIncrement: true},
+ indices: ['dictionary', 'expression', 'reading', 'sequence']
+ }
+ }
+ },
+ {
+ version: 50,
+ stores: {
+ terms: {
+ primaryKey: {keyPath: 'id', autoIncrement: true},
+ indices: ['dictionary', 'expression', 'reading', 'sequence', 'expressionReverse', 'readingReverse']
+ }
+ }
+ },
+ {
+ version: 60,
+ stores: {
+ media: {
+ primaryKey: {keyPath: 'id', autoIncrement: true},
+ indices: ['dictionary', 'path']
+ }
+ }
+ }
+ ]
+ );
+ }
+
+ async close() {
+ this._db.close();
+ }
+
+ isPrepared() {
+ return this._db.isOpen();
+ }
+
+ async purge() {
+ if (this._db.isOpening()) {
+ throw new Error('Cannot purge database while opening');
+ }
+ if (this._db.isOpen()) {
+ this._db.close();
+ }
+ let result = false;
+ try {
+ await Database.deleteDatabase(this._dbName);
+ result = true;
+ } catch (e) {
+ yomichan.logError(e);
+ }
+ await this.prepare();
+ return result;
+ }
+
+ async deleteDictionary(dictionaryName, progressSettings, onProgress) {
+ const targets = [
+ ['dictionaries', 'title'],
+ ['kanji', 'dictionary'],
+ ['kanjiMeta', 'dictionary'],
+ ['terms', 'dictionary'],
+ ['termMeta', 'dictionary'],
+ ['tagMeta', 'dictionary'],
+ ['media', 'dictionary']
+ ];
+
+ const {rate} = progressSettings;
+ const progressData = {
+ count: 0,
+ processed: 0,
+ storeCount: targets.length,
+ storesProcesed: 0
+ };
+
+ const filterKeys = (keys) => {
+ ++progressData.storesProcesed;
+ progressData.count += keys.length;
+ onProgress(progressData);
+ return keys;
+ };
+ const onProgress2 = () => {
+ const processed = progressData.processed + 1;
+ progressData.processed = processed;
+ if ((processed % rate) === 0 || processed === progressData.count) {
+ onProgress(progressData);
+ }
+ };
+
+ const promises = [];
+ for (const [objectStoreName, indexName] of targets) {
+ const query = IDBKeyRange.only(dictionaryName);
+ const promise = this._db.bulkDelete(objectStoreName, indexName, query, filterKeys, onProgress2);
+ promises.push(promise);
+ }
+ await Promise.all(promises);
+ }
+
+ findTermsBulk(termList, dictionaries, wildcard) {
+ return new Promise((resolve, reject) => {
+ const results = [];
+ const count = termList.length;
+ if (count === 0) {
+ resolve(results);
+ return;
+ }
+
+ const visited = new Set();
+ const useWildcard = !!wildcard;
+ const prefixWildcard = wildcard === 'prefix';
+
+ const transaction = this._db.transaction(['terms'], 'readonly');
+ const terms = transaction.objectStore('terms');
+ const index1 = terms.index(prefixWildcard ? 'expressionReverse' : 'expression');
+ const index2 = terms.index(prefixWildcard ? 'readingReverse' : 'reading');
+
+ const count2 = count * 2;
+ let completeCount = 0;
+ for (let i = 0; i < count; ++i) {
+ const inputIndex = i;
+ const term = prefixWildcard ? stringReverse(termList[i]) : termList[i];
+ const query = useWildcard ? IDBKeyRange.bound(term, `${term}\uffff`, false, false) : IDBKeyRange.only(term);
+
+ const onGetAll = (rows) => {
+ for (const row of rows) {
+ if (dictionaries.has(row.dictionary) && !visited.has(row.id)) {
+ visited.add(row.id);
+ results.push(this._createTerm(row, inputIndex));
+ }
+ }
+ if (++completeCount >= count2) {
+ resolve(results);
+ }
+ };
+
+ this._db.getAll(index1, query, onGetAll, reject);
+ this._db.getAll(index2, query, onGetAll, reject);
+ }
+ });
+ }
+
+ findTermsExactBulk(termList, readingList, dictionaries) {
+ return new Promise((resolve, reject) => {
+ const results = [];
+ const count = termList.length;
+ if (count === 0) {
+ resolve(results);
+ return;
+ }
+
+ const transaction = this._db.transaction(['terms'], 'readonly');
+ const terms = transaction.objectStore('terms');
+ const index = terms.index('expression');
+
+ let completeCount = 0;
+ for (let i = 0; i < count; ++i) {
+ const inputIndex = i;
+ const reading = readingList[i];
+ const query = IDBKeyRange.only(termList[i]);
+
+ const onGetAll = (rows) => {
+ for (const row of rows) {
+ if (row.reading === reading && dictionaries.has(row.dictionary)) {
+ results.push(this._createTerm(row, inputIndex));
+ }
+ }
+ if (++completeCount >= count) {
+ resolve(results);
+ }
+ };
+
+ this._db.getAll(index, query, onGetAll, reject);
+ }
+ });
+ }
+
+ findTermsBySequenceBulk(sequenceList, mainDictionary) {
+ return new Promise((resolve, reject) => {
+ const results = [];
+ const count = sequenceList.length;
+ if (count === 0) {
+ resolve(results);
+ return;
+ }
+
+ const transaction = this._db.transaction(['terms'], 'readonly');
+ const terms = transaction.objectStore('terms');
+ const index = terms.index('sequence');
+
+ let completeCount = 0;
+ for (let i = 0; i < count; ++i) {
+ const inputIndex = i;
+ const query = IDBKeyRange.only(sequenceList[i]);
+
+ const onGetAll = (rows) => {
+ for (const row of rows) {
+ if (row.dictionary === mainDictionary) {
+ results.push(this._createTerm(row, inputIndex));
+ }
+ }
+ if (++completeCount >= count) {
+ resolve(results);
+ }
+ };
+
+ this._db.getAll(index, query, onGetAll, reject);
+ }
+ });
+ }
+
+ findTermMetaBulk(termList, dictionaries) {
+ return this._findGenericBulk('termMeta', 'expression', termList, dictionaries, this._createTermMeta.bind(this));
+ }
+
+ findKanjiBulk(kanjiList, dictionaries) {
+ return this._findGenericBulk('kanji', 'character', kanjiList, dictionaries, this._createKanji.bind(this));
+ }
+
+ findKanjiMetaBulk(kanjiList, dictionaries) {
+ return this._findGenericBulk('kanjiMeta', 'character', kanjiList, dictionaries, this._createKanjiMeta.bind(this));
+ }
+
+ findTagForTitle(name, title) {
+ const query = IDBKeyRange.only(name);
+ return this._db.find('tagMeta', 'name', query, (row) => (row.dictionary === title), null);
+ }
+
+ getMedia(targets) {
+ return new Promise((resolve, reject) => {
+ const count = targets.length;
+ const results = new Array(count).fill(null);
+ if (count === 0) {
+ resolve(results);
+ return;
+ }
+
+ let completeCount = 0;
+ const transaction = this._db.transaction(['media'], 'readonly');
+ const objectStore = transaction.objectStore('media');
+ const index = objectStore.index('path');
+
+ for (let i = 0; i < count; ++i) {
+ const inputIndex = i;
+ const {path, dictionaryName} = targets[i];
+ const query = IDBKeyRange.only(path);
+
+ const onGetAll = (rows) => {
+ for (const row of rows) {
+ if (row.dictionary !== dictionaryName) { continue; }
+ results[inputIndex] = this._createMedia(row, inputIndex);
+ }
+ if (++completeCount >= count) {
+ resolve(results);
+ }
+ };
+
+ this._db.getAll(index, query, onGetAll, reject);
+ }
+ });
+ }
+
+ getDictionaryInfo() {
+ return new Promise((resolve, reject) => {
+ const transaction = this._db.transaction(['dictionaries'], 'readonly');
+ const objectStore = transaction.objectStore('dictionaries');
+ this._db.getAll(objectStore, null, resolve, reject);
+ });
+ }
+
+ getDictionaryCounts(dictionaryNames, getTotal) {
+ return new Promise((resolve, reject) => {
+ const targets = [
+ ['kanji', 'dictionary'],
+ ['kanjiMeta', 'dictionary'],
+ ['terms', 'dictionary'],
+ ['termMeta', 'dictionary'],
+ ['tagMeta', 'dictionary'],
+ ['media', 'dictionary']
+ ];
+ const objectStoreNames = targets.map(([objectStoreName]) => objectStoreName);
+ const transaction = this._db.transaction(objectStoreNames, 'readonly');
+ const databaseTargets = targets.map(([objectStoreName, indexName]) => {
+ const objectStore = transaction.objectStore(objectStoreName);
+ const index = objectStore.index(indexName);
+ return {objectStore, index};
+ });
+
+ const countTargets = [];
+ if (getTotal) {
+ for (const {objectStore} of databaseTargets) {
+ countTargets.push([objectStore, null]);
+ }
+ }
+ for (const dictionaryName of dictionaryNames) {
+ const query = IDBKeyRange.only(dictionaryName);
+ for (const {index} of databaseTargets) {
+ countTargets.push([index, query]);
+ }
+ }
+
+ const onCountComplete = (results) => {
+ const resultCount = results.length;
+ const targetCount = targets.length;
+ const counts = [];
+ for (let i = 0; i < resultCount; i += targetCount) {
+ const countGroup = {};
+ for (let j = 0; j < targetCount; ++j) {
+ countGroup[targets[j][0]] = results[i + j];
+ }
+ counts.push(countGroup);
+ }
+ const total = getTotal ? counts.shift() : null;
+ resolve({total, counts});
+ };
+
+ this._db.bulkCount(countTargets, onCountComplete, reject);
+ });
+ }
+
+ async dictionaryExists(title) {
+ const query = IDBKeyRange.only(title);
+ const result = await this._db.find('dictionaries', 'title', query);
+ return typeof result !== 'undefined';
+ }
+
+ bulkAdd(objectStoreName, items, start, count) {
+ return this._db.bulkAdd(objectStoreName, items, start, count);
+ }
+
+ // Private
+
+ async _findGenericBulk(objectStoreName, indexName, indexValueList, dictionaries, createResult) {
+ return new Promise((resolve, reject) => {
+ const results = [];
+ const count = indexValueList.length;
+ if (count === 0) {
+ resolve(results);
+ return;
+ }
+
+ const transaction = this._db.transaction([objectStoreName], 'readonly');
+ const terms = transaction.objectStore(objectStoreName);
+ const index = terms.index(indexName);
+
+ let completeCount = 0;
+ for (let i = 0; i < count; ++i) {
+ const inputIndex = i;
+ const query = IDBKeyRange.only(indexValueList[i]);
+
+ const onGetAll = (rows) => {
+ for (const row of rows) {
+ if (dictionaries.has(row.dictionary)) {
+ results.push(createResult(row, inputIndex));
+ }
+ }
+ if (++completeCount >= count) {
+ resolve(results);
+ }
+ };
+
+ this._db.getAll(index, query, onGetAll, reject);
+ }
+ });
+ }
+
+ _createTerm(row, index) {
+ return {
+ index,
+ expression: row.expression,
+ reading: row.reading,
+ definitionTags: this._splitField(row.definitionTags || row.tags || ''),
+ termTags: this._splitField(row.termTags || ''),
+ rules: this._splitField(row.rules),
+ glossary: row.glossary,
+ score: row.score,
+ dictionary: row.dictionary,
+ id: row.id,
+ sequence: typeof row.sequence === 'undefined' ? -1 : row.sequence
+ };
+ }
+
+ _createKanji(row, index) {
+ return {
+ index,
+ character: row.character,
+ onyomi: this._splitField(row.onyomi),
+ kunyomi: this._splitField(row.kunyomi),
+ tags: this._splitField(row.tags),
+ glossary: row.meanings,
+ stats: row.stats,
+ dictionary: row.dictionary
+ };
+ }
+
+ _createTermMeta({expression, mode, data, dictionary}, index) {
+ return {expression, mode, data, dictionary, index};
+ }
+
+ _createKanjiMeta({character, mode, data, dictionary}, index) {
+ return {character, mode, data, dictionary, index};
+ }
+
+ _createMedia(row, index) {
+ return Object.assign({}, row, {index});
+ }
+
+ _splitField(field) {
+ return field.length === 0 ? [] : field.split(' ');
+ }
+}