Dictionary database improvements (#1527)

* Update formatting

* Add _findMultiBulk

* Update implementation of findTermsBySequenceBulk

* Update tests

* Generalize query creation

* Remove _findGenericBulk

* Reduce function creation

* Add more bindings

* Simplify findTermsExactBulk implementation

* Update var names

* Update _findMultiBulk to support multiple index queries

* Update findTermsBulk

* Update getMedia implementation

* Pass data arg to getAll and findFirst to avoid having multiple closures

4 files changed, 119 insertions, 208 deletions

diff --git a/ext/js/data/database.js b/ext/js/data/database.js
index f44ea1d9..a0a1804a 100644
--- a/ext/js/data/database.js
+++ b/ext/js/data/database.js
@@ -95,11 +95,11 @@ class Database {
         });
     }
 
-    getAll(objectStoreOrIndex, query, resolve, reject) {
+    getAll(objectStoreOrIndex, query, resolve, reject, data) {
         if (typeof objectStoreOrIndex.getAll === 'function') {
-            this._getAllFast(objectStoreOrIndex, query, resolve, reject);
+            this._getAllFast(objectStoreOrIndex, query, resolve, reject, data);
         } else {
-            this._getAllUsingCursor(objectStoreOrIndex, query, resolve, reject);
+            this._getAllUsingCursor(objectStoreOrIndex, query, resolve, reject, data);
         }
     }
 
@@ -116,25 +116,25 @@ class Database {
             const transaction = this.transaction([objectStoreName], 'readonly');
             const objectStore = transaction.objectStore(objectStoreName);
             const objectStoreOrIndex = indexName !== null ? objectStore.index(indexName) : objectStore;
-            this.findFirst(objectStoreOrIndex, query, resolve, reject, predicate, predicateArg, defaultValue);
+            this.findFirst(objectStoreOrIndex, query, resolve, reject, null, predicate, predicateArg, defaultValue);
         });
     }
 
-    findFirst(objectStoreOrIndex, query, resolve, reject, predicate, predicateArg, defaultValue) {
+    findFirst(objectStoreOrIndex, query, resolve, reject, data, predicate, predicateArg, defaultValue) {
         const noPredicate = (typeof predicate !== 'function');
         const request = objectStoreOrIndex.openCursor(query, 'next');
-        request.onerror = (e) => reject(e.target.error);
+        request.onerror = (e) => reject(e.target.error, data);
         request.onsuccess = (e) => {
             const cursor = e.target.result;
             if (cursor) {
                 const {value} = cursor;
                 if (noPredicate || predicate(value, predicateArg)) {
-                    resolve(value);
+                    resolve(value, data);
                 } else {
                     cursor.continue();
                 }
             } else {
-                resolve(defaultValue);
+                resolve(defaultValue, data);
             }
         };
     }
@@ -256,23 +256,23 @@ class Database {
         return false;
     }
 
-    _getAllFast(objectStoreOrIndex, query, resolve, reject) {
+    _getAllFast(objectStoreOrIndex, query, resolve, reject, data) {
         const request = objectStoreOrIndex.getAll(query);
-        request.onerror = (e) => reject(e.target.error);
-        request.onsuccess = (e) => resolve(e.target.result);
+        request.onerror = (e) => reject(e.target.error, data);
+        request.onsuccess = (e) => resolve(e.target.result, data);
     }
 
-    _getAllUsingCursor(objectStoreOrIndex, query, resolve, reject) {
+    _getAllUsingCursor(objectStoreOrIndex, query, resolve, reject, data) {
         const results = [];
         const request = objectStoreOrIndex.openCursor(query, 'next');
-        request.onerror = (e) => reject(e.target.error);
+        request.onerror = (e) => reject(e.target.error, data);
         request.onsuccess = (e) => {
             const cursor = e.target.result;
             if (cursor) {
                 results.push(cursor.value);
                 cursor.continue();
             } else {
-                resolve(results);
+                resolve(results, data);
             }
         };
     }
diff --git a/ext/js/language/dictionary-database.js b/ext/js/language/dictionary-database.js
index b1c1a3aa..fc4336c5 100644
--- a/ext/js/language/dictionary-database.js
+++ b/ext/js/language/dictionary-database.js
@@ -24,6 +24,17 @@ class DictionaryDatabase {
         this._db = new Database();
         this._dbName = 'dict';
         this._schemas = new Map();
+        this._createOnlyQuery1 = (item) => IDBKeyRange.only(item);
+        this._createOnlyQuery2 = (item) => IDBKeyRange.only(item.query);
+        this._createOnlyQuery3 = (item) => IDBKeyRange.only(item.expression);
+        this._createOnlyQuery4 = (item) => IDBKeyRange.only(item.path);
+        this._createBoundQuery1 = (item) => IDBKeyRange.bound(item, `${item}\uffff`, false, false);
+        this._createBoundQuery2 = (item) => { item = stringReverse(item); return IDBKeyRange.bound(item, `${item}\uffff`, false, false); };
+        this._createTermBind = this._createTerm.bind(this);
+        this._createTermMetaBind = this._createTermMeta.bind(this);
+        this._createKanjiBind = this._createKanji.bind(this);
+        this._createKanjiMetaBind = this._createKanjiMeta.bind(this);
+        this._createMediaBind = this._createMedia.bind(this);
     }
 
     // Public
@@ -171,132 +182,61 @@ class DictionaryDatabase {
     }
 
     findTermsBulk(termList, dictionaries, wildcard) {
-        return new Promise((resolve, reject) => {
-            const results = [];
-            const count = termList.length;
-            if (count === 0) {
-                resolve(results);
-                return;
-            }
-
-            const visited = new Set();
-            const useWildcard = !!wildcard;
-            const prefixWildcard = wildcard === 'prefix';
-
-            const transaction = this._db.transaction(['terms'], 'readonly');
-            const terms = transaction.objectStore('terms');
-            const index1 = terms.index(prefixWildcard ? 'expressionReverse' : 'expression');
-            const index2 = terms.index(prefixWildcard ? 'readingReverse' : 'reading');
+        const visited = new Set();
+        const predicate = (row) => {
+            if (!dictionaries.has(row.dictionary)) { return false; }
+            const {id} = row;
+            if (visited.has(id)) { return false; }
+            visited.add(id);
+            return true;
+        };
 
-            const count2 = count * 2;
-            let completeCount = 0;
-            for (let i = 0; i < count; ++i) {
-                const inputIndex = i;
-                const term = prefixWildcard ? stringReverse(termList[i]) : termList[i];
-                const query = useWildcard ? IDBKeyRange.bound(term, `${term}\uffff`, false, false) : IDBKeyRange.only(term);
-
-                const onGetAll = (rows) => {
-                    for (const row of rows) {
-                        if (dictionaries.has(row.dictionary) && !visited.has(row.id)) {
-                            visited.add(row.id);
-                            results.push(this._createTerm(row, inputIndex));
-                        }
-                    }
-                    if (++completeCount >= count2) {
-                        resolve(results);
-                    }
-                };
+        const indexNames = (wildcard === 'prefix') ? ['expressionReverse', 'readingReverse'] : ['expression', 'reading'];
+
+        let createQuery;
+        switch (wildcard) {
+            case 'suffix':
+                createQuery = this._createBoundQuery1;
+                break;
+            case 'prefix':
+                createQuery = this._createBoundQuery2;
+                break;
+            default:
+                createQuery = this._createOnlyQuery1;
+                break;
+        }
 
-                this._db.getAll(index1, query, onGetAll, reject);
-                this._db.getAll(index2, query, onGetAll, reject);
-            }
-        });
+        return this._findMultiBulk('terms', indexNames, termList, createQuery, predicate, this._createTermBind);
     }
 
-    findTermsExactBulk(termList, readingList, dictionaries) {
-        return new Promise((resolve, reject) => {
-            const results = [];
-            const count = termList.length;
-            if (count === 0) {
-                resolve(results);
-                return;
-            }
-
-            const transaction = this._db.transaction(['terms'], 'readonly');
-            const terms = transaction.objectStore('terms');
-            const index = terms.index('expression');
-
-            let completeCount = 0;
-            for (let i = 0; i < count; ++i) {
-                const inputIndex = i;
-                const reading = readingList[i];
-                const query = IDBKeyRange.only(termList[i]);
-
-                const onGetAll = (rows) => {
-                    for (const row of rows) {
-                        if (row.reading === reading && dictionaries.has(row.dictionary)) {
-                            results.push(this._createTerm(row, inputIndex));
-                        }
-                    }
-                    if (++completeCount >= count) {
-                        resolve(results);
-                    }
-                };
-
-                this._db.getAll(index, query, onGetAll, reject);
-            }
-        });
+    findTermsExactBulk(termList, dictionaries) {
+        const predicate = (row, item) => (row.reading === item.reading && dictionaries.has(row.dictionary));
+        return this._findMultiBulk('terms', ['expression'], termList, this._createOnlyQuery3, predicate, this._createTermBind);
     }
 
-    findTermsBySequenceBulk(sequenceList, mainDictionary) {
-        return new Promise((resolve, reject) => {
-            const results = [];
-            const count = sequenceList.length;
-            if (count === 0) {
-                resolve(results);
-                return;
-            }
-
-            const transaction = this._db.transaction(['terms'], 'readonly');
-            const terms = transaction.objectStore('terms');
-            const index = terms.index('sequence');
-
-            let completeCount = 0;
-            for (let i = 0; i < count; ++i) {
-                const inputIndex = i;
-                const query = IDBKeyRange.only(sequenceList[i]);
-
-                const onGetAll = (rows) => {
-                    for (const row of rows) {
-                        if (row.dictionary === mainDictionary) {
-                            results.push(this._createTerm(row, inputIndex));
-                        }
-                    }
-                    if (++completeCount >= count) {
-                        resolve(results);
-                    }
-                };
-
-                this._db.getAll(index, query, onGetAll, reject);
-            }
-        });
+    findTermsBySequenceBulk(items) {
+        const predicate = (row, item) => (row.dictionary === item.dictionary);
+        return this._findMultiBulk('terms', ['sequence'], items, this._createOnlyQuery2, predicate, this._createTermBind);
     }
 
     findTermMetaBulk(termList, dictionaries) {
-        return this._findGenericBulk('termMeta', 'expression', termList, dictionaries, this._createTermMeta.bind(this));
+        const predicate = (row) => dictionaries.has(row.dictionary);
+        return this._findMultiBulk('termMeta', ['expression'], termList, this._createOnlyQuery1, predicate, this._createTermMetaBind);
     }
 
     findKanjiBulk(kanjiList, dictionaries) {
-        return this._findGenericBulk('kanji', 'character', kanjiList, dictionaries, this._createKanji.bind(this));
+        const predicate = (row) => dictionaries.has(row.dictionary);
+        return this._findMultiBulk('kanji', ['character'], kanjiList, this._createOnlyQuery1, predicate, this._createKanjiBind);
     }
 
     findKanjiMetaBulk(kanjiList, dictionaries) {
-        return this._findGenericBulk('kanjiMeta', 'character', kanjiList, dictionaries, this._createKanjiMeta.bind(this));
+        const predicate = (row) => dictionaries.has(row.dictionary);
+        return this._findMultiBulk('kanjiMeta', ['character'], kanjiList, this._createOnlyQuery1, predicate, this._createKanjiMetaBind);
     }
 
     findTagMetaBulk(items) {
         const predicate = (row, item) => (row.dictionary === item.dictionary);
-        return this._findFirstBulk('tagMeta', 'name', items, predicate);
+        return this._findFirstBulk('tagMeta', 'name', items, this._createOnlyQuery2, predicate);
     }
 
     findTagForTitle(name, title) {
@@ -304,38 +244,9 @@ class DictionaryDatabase {
         return this._db.find('tagMeta', 'name', query, (row) => (row.dictionary === title), null, null);
     }
 
-    getMedia(targets) {
-        return new Promise((resolve, reject) => {
-            const count = targets.length;
-            const results = new Array(count).fill(null);
-            if (count === 0) {
-                resolve(results);
-                return;
-            }
-
-            let completeCount = 0;
-            const transaction = this._db.transaction(['media'], 'readonly');
-            const objectStore = transaction.objectStore('media');
-            const index = objectStore.index('path');
-
-            for (let i = 0; i < count; ++i) {
-                const inputIndex = i;
-                const {path, dictionaryName} = targets[i];
-                const query = IDBKeyRange.only(path);
-
-                const onGetAll = (rows) => {
-                    for (const row of rows) {
-                        if (row.dictionary !== dictionaryName) { continue; }
-                        results[inputIndex] = this._createMedia(row, inputIndex);
-                    }
-                    if (++completeCount >= count) {
-                        resolve(results);
-                    }
-                };
-
-                this._db.getAll(index, query, onGetAll, reject);
-            }
-        });
+    getMedia(items) {
+        const predicate = (row, item) => (row.dictionary === item.dictionary);
+        return this._findMultiBulk('media', ['path'], items, this._createOnlyQuery4, predicate, this._createMediaBind);
     }
 
     getDictionaryInfo() {
@@ -408,66 +319,67 @@ class DictionaryDatabase {
 
     // Private
 
-    async _findGenericBulk(objectStoreName, indexName, indexValueList, dictionaries, createResult) {
+    _findMultiBulk(objectStoreName, indexNames, items, createQuery, predicate, createResult) {
         return new Promise((resolve, reject) => {
+            const itemCount = items.length;
+            const indexCount = indexNames.length;
             const results = [];
-            const count = indexValueList.length;
-            if (count === 0) {
+            if (itemCount === 0 || indexCount === 0) {
                 resolve(results);
                 return;
             }
 
             const transaction = this._db.transaction([objectStoreName], 'readonly');
-            const terms = transaction.objectStore(objectStoreName);
-            const index = terms.index(indexName);
-
+            const objectStore = transaction.objectStore(objectStoreName);
+            const indexList = [];
+            for (const indexName of indexNames) {
+                indexList.push(objectStore.index(indexName));
+            }
             let completeCount = 0;
-            for (let i = 0; i < count; ++i) {
-                const inputIndex = i;
-                const query = IDBKeyRange.only(indexValueList[i]);
-
-                const onGetAll = (rows) => {
-                    for (const row of rows) {
-                        if (dictionaries.has(row.dictionary)) {
-                            results.push(createResult(row, inputIndex));
-                        }
+            const requiredCompleteCount = itemCount * indexCount;
+            const onGetAll = (rows, {item, itemIndex}) => {
+                for (const row of rows) {
+                    if (predicate(row, item)) {
+                        results.push(createResult(row, itemIndex));
                     }
-                    if (++completeCount >= count) {
-                        resolve(results);
-                    }
-                };
-
-                this._db.getAll(index, query, onGetAll, reject);
+                }
+                if (++completeCount >= requiredCompleteCount) {
+                    resolve(results);
+                }
+            };
+            for (let i = 0; i < itemCount; ++i) {
+                const item = items[i];
+                const query = createQuery(item);
+                for (let j = 0; j < indexCount; ++j) {
+                    this._db.getAll(indexList[j], query, onGetAll, reject, {item, itemIndex: i});
+                }
             }
         });
     }
 
-    _findFirstBulk(objectStoreName, indexName, items, predicate) {
+    _findFirstBulk(objectStoreName, indexName, items, createQuery, predicate) {
         return new Promise((resolve, reject) => {
-            const count = items.length;
-            const results = new Array(count);
-            if (count === 0) {
+            const itemCount = items.length;
+            const results = new Array(itemCount);
+            if (itemCount === 0) {
                 resolve(results);
                 return;
             }
 
             const transaction = this._db.transaction([objectStoreName], 'readonly');
-            const terms = transaction.objectStore(objectStoreName);
-            const index = terms.index(indexName);
-
+            const objectStore = transaction.objectStore(objectStoreName);
+            const index = objectStore.index(indexName);
             let completeCount = 0;
-            for (let i = 0; i < count; ++i) {
-                const itemIndex = i;
+            const onFind = (row, itemIndex) => {
+                results[itemIndex] = row;
+                if (++completeCount >= itemCount) {
+                    resolve(results);
+                }
+            };
+            for (let i = 0; i < itemCount; ++i) {
                 const item = items[i];
-                const query = IDBKeyRange.only(item.query);
-
-                const onFind = (row) => {
-                    results[itemIndex] = row;
-                    if (++completeCount >= count) {
-                        resolve(results);
-                    }
-                };
-                this._db.findFirst(index, query, onFind, reject, predicate, item, void 0);
+                const query = createQuery(item);
+                this._db.findFirst(index, query, onFind, reject, i, predicate, item, void 0);
             }
         });
     }
diff --git a/ext/js/language/translator.js b/ext/js/language/translator.js
index 761fac6b..151b1172 100644
--- a/ext/js/language/translator.js
+++ b/ext/js/language/translator.js
@@ -392,7 +392,8 @@ class Translator {
     }
 
     async _addRelatedDefinitions(sequencedDefinitions, unsequencedDefinitions, sequenceList, mainDictionary, enabledDictionaryMap) {
-        const databaseDefinitions = await this._database.findTermsBySequenceBulk(sequenceList, mainDictionary);
+        const items = sequenceList.map((query) => ({query, dictionary: mainDictionary}));
+        const databaseDefinitions = await this._database.findTermsBySequenceBulk(items);
         for (const databaseDefinition of databaseDefinitions) {
             const {relatedDefinitions, definitionIds} = sequencedDefinitions[databaseDefinition.index];
             const {id} = databaseDefinition;
@@ -410,8 +411,7 @@ class Translator {
         if (unsequencedDefinitions.length === 0 && secondarySearchDictionaryMap.size === 0) { return; }
 
         // Prepare grouping info
-        const expressionList = [];
-        const readingList = [];
+        const termList = [];
         const targetList = [];
         const targetMap = new Map();
 
@@ -431,8 +431,7 @@ class Translator {
                 target.sequencedDefinitions.push(sequencedDefinition);
                 if (!definition.isPrimary && !target.searchSecondary) {
                     target.searchSecondary = true;
-                    expressionList.push(expression);
-                    readingList.push(reading);
+                    termList.push({expression, reading});
                     targetList.push(target);
                 }
             }
@@ -456,14 +455,14 @@ class Translator {
         }
 
         // Search database for additional secondary terms
-        if (expressionList.length === 0 || secondarySearchDictionaryMap.size === 0) { return; }
+        if (termList.length === 0 || secondarySearchDictionaryMap.size === 0) { return; }
 
-        const databaseDefinitions = await this._database.findTermsExactBulk(expressionList, readingList, secondarySearchDictionaryMap);
+        const databaseDefinitions = await this._database.findTermsExactBulk(termList, secondarySearchDictionaryMap);
         this._sortDatabaseDefinitionsByIndex(databaseDefinitions);
 
         for (const databaseDefinition of databaseDefinitions) {
             const {index, id} = databaseDefinition;
-            const source = expressionList[index];
+            const source = termList[index].expression;
             const target = targetList[index];
             for (const {definitionIds, secondaryDefinitions} of target.sequencedDefinitions) {
                 if (definitionIds.has(id)) { continue; }
diff --git a/ext/js/media/media-loader.js b/ext/js/media/media-loader.js
index 4ac733c5..7dafc2a3 100644
--- a/ext/js/media/media-loader.js
+++ b/ext/js/media/media-loader.js
@@ -26,13 +26,13 @@ class MediaLoader {
         this._loadMediaData = [];
     }
 
-    async loadMedia(path, dictionaryName, onLoad, onUnload) {
+    async loadMedia(path, dictionary, onLoad, onUnload) {
         const token = this._token;
         const data = {onUnload, loaded: false};
 
         this._loadMediaData.push(data);
 
-        const media = await this.getMedia(path, dictionaryName);
+        const media = await this.getMedia(path, dictionary);
         if (token !== this._token) { return; }
 
         onLoad(media.url);
@@ -59,14 +59,14 @@ class MediaLoader {
         this._token = {};
     }
 
-    async getMedia(path, dictionaryName) {
+    async getMedia(path, dictionary) {
         let cachedData;
-        let dictionaryCache = this._mediaCache.get(dictionaryName);
+        let dictionaryCache = this._mediaCache.get(dictionary);
         if (typeof dictionaryCache !== 'undefined') {
             cachedData = dictionaryCache.get(path);
         } else {
             dictionaryCache = new Map();
-            this._mediaCache.set(dictionaryName, dictionaryCache);
+            this._mediaCache.set(dictionary, dictionaryCache);
         }
 
         if (typeof cachedData === 'undefined') {
@@ -76,15 +76,15 @@ class MediaLoader {
                 url: null
             };
             dictionaryCache.set(path, cachedData);
-            cachedData.promise = this._getMediaData(path, dictionaryName, cachedData);
+            cachedData.promise = this._getMediaData(path, dictionary, cachedData);
         }
 
         return cachedData.promise;
     }
 
-    async _getMediaData(path, dictionaryName, cachedData) {
+    async _getMediaData(path, dictionary, cachedData) {
         const token = this._token;
-        const data = (await yomichan.api.getMedia([{path, dictionaryName}]))[0];
+        const data = (await yomichan.api.getMedia([{path, dictionary}]))[0];
         if (token === this._token && data !== null) {
             const blob = MediaUtil.createBlobFromBase64Content(data.content, data.mediaType);
             const url = URL.createObjectURL(blob);