Merge branch 'master' into testing

1 files changed, 356 insertions, 72 deletions

diff --git a/ext/bg/js/database.js b/ext/bg/js/database.js
index 9f477b24..a20d5f15 100644
--- a/ext/bg/js/database.js
+++ b/ext/bg/js/database.js
@@ -23,63 +23,137 @@ class Database {
     }
 
     async prepare() {
-        if (this.db) {
+        if (this.db !== null) {
             throw new Error('Database already initialized');
         }
 
-        this.db = new Dexie('dict');
-        this.db.version(2).stores({
-            terms:        '++id,dictionary,expression,reading',
-            kanji:        '++,dictionary,character',
-            tagMeta:      '++,dictionary',
-            dictionaries: '++,title,version'
-        });
-        this.db.version(3).stores({
-            termMeta:  '++,dictionary,expression',
-            kanjiMeta: '++,dictionary,character',
-            tagMeta:   '++,dictionary,name'
-        });
-        this.db.version(4).stores({
-            terms: '++id,dictionary,expression,reading,sequence'
-        });
-
-        await this.db.open();
+        try {
+            this.db = await Database.open('dict', 4, (db, transaction, oldVersion) => {
+                Database.upgrade(db, transaction, oldVersion, [
+                    {
+                        version: 2,
+                        stores: {
+                            terms: {
+                                primaryKey: {keyPath: 'id', autoIncrement: true},
+                                indices: ['dictionary', 'expression', 'reading']
+                            },
+                            kanji: {
+                                primaryKey: {autoIncrement: true},
+                                indices: ['dictionary', 'character']
+                            },
+                            tagMeta: {
+                                primaryKey: {autoIncrement: true},
+                                indices: ['dictionary']
+                            },
+                            dictionaries: {
+                                primaryKey: {autoIncrement: true},
+                                indices: ['title', 'version']
+                            }
+                        }
+                    },
+                    {
+                        version: 3,
+                        stores: {
+                            termMeta: {
+                                primaryKey: {autoIncrement: true},
+                                indices: ['dictionary', 'expression']
+                            },
+                            kanjiMeta: {
+                                primaryKey: {autoIncrement: true},
+                                indices: ['dictionary', 'character']
+                            },
+                            tagMeta: {
+                                primaryKey: {autoIncrement: true},
+                                indices: ['dictionary', 'name']
+                            }
+                        }
+                    },
+                    {
+                        version: 4,
+                        stores: {
+                            terms: {
+                                primaryKey: {keyPath: 'id', autoIncrement: true},
+                                indices: ['dictionary', 'expression', 'reading', 'sequence']
+                            }
+                        }
+                    }
+                ]);
+            });
+            return true;
+        } catch (e) {
+            console.error(e);
+            return false;
+        }
     }
 
     async purge() {
         this.validate();
 
         this.db.close();
-        await this.db.delete();
+        await Database.deleteDatabase(this.db.name);
         this.db = null;
 
         await this.prepare();
     }
 
-    async findTermsBulk(termList, titles) {
+    async deleteDictionary(dictionaryName, onProgress, progressSettings) {
+        this.validate();
+
+        const targets = [
+            ['dictionaries', 'title'],
+            ['kanji', 'dictionary'],
+            ['kanjiMeta', 'dictionary'],
+            ['terms', 'dictionary'],
+            ['termMeta', 'dictionary'],
+            ['tagMeta', 'dictionary']
+        ];
+        const promises = [];
+        const progressData = {
+            count: 0,
+            processed: 0,
+            storeCount: targets.length,
+            storesProcesed: 0
+        };
+        let progressRate = (typeof progressSettings === 'object' && progressSettings !== null ? progressSettings.rate : 0);
+        if (typeof progressRate !== 'number' || progressRate <= 0) {
+            progressRate = 1000;
+        }
+
+        for (const [objectStoreName, index] of targets) {
+            const dbTransaction = this.db.transaction([objectStoreName], 'readwrite');
+            const dbObjectStore = dbTransaction.objectStore(objectStoreName);
+            const dbIndex = dbObjectStore.index(index);
+            const only = IDBKeyRange.only(dictionaryName);
+            promises.push(Database.deleteValues(dbObjectStore, dbIndex, only, onProgress, progressData, progressRate));
+        }
+
+        await Promise.all(promises);
+    }
+
+    async findTermsBulk(termList, titles, wildcard) {
         this.validate();
 
         const promises = [];
         const visited = {};
         const results = [];
         const processRow = (row, index) => {
-            if (titles.includes(row.dictionary) && !visited.hasOwnProperty(row.id)) {
+            if (titles.includes(row.dictionary) && !hasOwn(visited, row.id)) {
                 visited[row.id] = true;
                 results.push(Database.createTerm(row, index));
             }
         };
 
-        const db = this.db.backendDB();
-        const dbTransaction = db.transaction(['terms'], 'readonly');
+        const dbTransaction = this.db.transaction(['terms'], 'readonly');
         const dbTerms = dbTransaction.objectStore('terms');
         const dbIndex1 = dbTerms.index('expression');
         const dbIndex2 = dbTerms.index('reading');
 
         for (let i = 0; i < termList.length; ++i) {
-            const only = IDBKeyRange.only(termList[i]);
+            const term = termList[i];
+            const query = wildcard ? IDBKeyRange.bound(term, `${term}\uffff`, false, false) : IDBKeyRange.only(term);
             promises.push(
-                Database.getAll(dbIndex1, only, i, processRow),
-                Database.getAll(dbIndex2, only, i, processRow)
+                Database.getAll(dbIndex1, query, i, processRow),
+                Database.getAll(dbIndex2, query, i, processRow)
             );
         }
 
@@ -99,8 +173,7 @@ class Database {
             }
         };
 
-        const db = this.db.backendDB();
-        const dbTransaction = db.transaction(['terms'], 'readonly');
+        const dbTransaction = this.db.transaction(['terms'], 'readonly');
         const dbTerms = dbTransaction.objectStore('terms');
         const dbIndex = dbTerms.index('expression');
 
@@ -125,8 +198,7 @@ class Database {
             }
         };
 
-        const db = this.db.backendDB();
-        const dbTransaction = db.transaction(['terms'], 'readonly');
+        const dbTransaction = this.db.transaction(['terms'], 'readonly');
         const dbTerms = dbTransaction.objectStore('terms');
         const dbIndex = dbTerms.index('sequence');
 
@@ -163,8 +235,7 @@ class Database {
             }
         };
 
-        const db = this.db.backendDB();
-        const dbTransaction = db.transaction([tableName], 'readonly');
+        const dbTransaction = this.db.transaction([tableName], 'readonly');
         const dbTerms = dbTransaction.objectStore(tableName);
         const dbIndex = dbTerms.index(indexName);
 
@@ -182,12 +253,11 @@ class Database {
         this.validate();
 
         let result = null;
-        const db = this.db.backendDB();
-        const dbTransaction = db.transaction(['tagMeta'], 'readonly');
+        const dbTransaction = this.db.transaction(['tagMeta'], 'readonly');
         const dbTerms = dbTransaction.objectStore('tagMeta');
         const dbIndex = dbTerms.index('name');
         const only = IDBKeyRange.only(name);
-        await Database.getAll(dbIndex, only, null, row => {
+        await Database.getAll(dbIndex, only, null, (row) => {
             if (title === row.dictionary) {
                 result = row;
             }
@@ -196,24 +266,76 @@ class Database {
         return result;
     }
 
-    async summarize() {
+    async getDictionaryInfo() {
+        this.validate();
+
+        const results = [];
+        const dbTransaction = this.db.transaction(['dictionaries'], 'readonly');
+        const dbDictionaries = dbTransaction.objectStore('dictionaries');
+
+        await Database.getAll(dbDictionaries, null, null, (info) => results.push(info));
+
+        return results;
+    }
+
+    async getDictionaryCounts(dictionaryNames, getTotal) {
         this.validate();
 
-        return this.db.dictionaries.toArray();
+        const objectStoreNames = [
+            'kanji',
+            'kanjiMeta',
+            'terms',
+            'termMeta',
+            'tagMeta'
+        ];
+        const dbCountTransaction = this.db.transaction(objectStoreNames, 'readonly');
+
+        const targets = [];
+        for (const objectStoreName of objectStoreNames) {
+            targets.push([
+                objectStoreName,
+                dbCountTransaction.objectStore(objectStoreName).index('dictionary')
+            ]);
+        }
+
+        // Query is required for Edge, otherwise index.count throws an exception.
+        const query1 = IDBKeyRange.lowerBound('', false);
+        const totalPromise = getTotal ? Database.getCounts(targets, query1) : null;
+
+        const counts = [];
+        const countPromises = [];
+        for (let i = 0; i < dictionaryNames.length; ++i) {
+            counts.push(null);
+            const index = i;
+            const query2 = IDBKeyRange.only(dictionaryNames[i]);
+            const countPromise = Database.getCounts(targets, query2).then((v) => counts[index] = v);
+            countPromises.push(countPromise);
+        }
+        await Promise.all(countPromises);
+
+        const result = {counts};
+        if (totalPromise !== null) {
+            result.total = await totalPromise;
+        }
+        return result;
     }
 
     async importDictionary(archive, progressCallback, exceptions) {
         this.validate();
 
         const maxTransactionLength = 1000;
-        const bulkAdd = async (table, items, total, current) => {
-            if (items.length < maxTransactionLength) {
+        const bulkAdd = async (objectStoreName, items, total, current) => {
+            const db = this.db;
+            for (let i = 0; i < items.length; i += maxTransactionLength) {
                 if (progressCallback) {
-                    progressCallback(total, current);
+                    progressCallback(total, current + i / items.length);
                 }
 
                 try {
-                    await table.bulkAdd(items);
+                    const count = Math.min(maxTransactionLength, items.length - i);
+                    const transaction = db.transaction([objectStoreName], 'readwrite');
+                    const objectStore = transaction.objectStore(objectStoreName);
+                    await Database.bulkAdd(objectStore, items, i, count);
                 } catch (e) {
                     if (exceptions) {
                         exceptions.push(e);
@@ -221,37 +343,27 @@ class Database {
                         throw e;
                     }
                 }
-            } else {
-                for (let i = 0; i < items.length; i += maxTransactionLength) {
-                    if (progressCallback) {
-                        progressCallback(total, current + i / items.length);
-                    }
-
-                    let count = Math.min(maxTransactionLength, items.length - i);
-                    try {
-                        await table.bulkAdd(items.slice(i, i + count));
-                    } catch (e) {
-                        if (exceptions) {
-                            exceptions.push(e);
-                        } else {
-                            throw e;
-                        }
-                    }
-                }
             }
         };
 
-        const indexDataLoaded = async summary => {
+        const indexDataLoaded = async (summary) => {
             if (summary.version > 3) {
                 throw new Error('Unsupported dictionary version');
             }
 
-            const count = await this.db.dictionaries.where('title').equals(summary.title).count();
+            const db = this.db;
+            const dbCountTransaction = db.transaction(['dictionaries'], 'readonly');
+            const dbIndex = dbCountTransaction.objectStore('dictionaries').index('title');
+            const only = IDBKeyRange.only(summary.title);
+            const count = await Database.getCount(dbIndex, only);
+
             if (count > 0) {
                 throw new Error('Dictionary is already imported');
             }
 
-            await this.db.dictionaries.add(summary);
+            const transaction = db.transaction(['dictionaries'], 'readwrite');
+            const objectStore = transaction.objectStore('dictionaries');
+            await Database.bulkAdd(objectStore, [summary], 0, 1);
         };
 
         const termDataLoaded = async (summary, entries, total, current) => {
@@ -284,7 +396,7 @@ class Database {
                 }
             }
 
-            await bulkAdd(this.db.terms, rows, total, current);
+            await bulkAdd('terms', rows, total, current);
         };
 
         const termMetaDataLoaded = async (summary, entries, total, current) => {
@@ -298,7 +410,7 @@ class Database {
                 });
             }
 
-            await bulkAdd(this.db.termMeta, rows, total, current);
+            await bulkAdd('termMeta', rows, total, current);
         };
 
         const kanjiDataLoaded = async (summary, entries, total, current)  => {
@@ -328,7 +440,7 @@ class Database {
                 }
             }
 
-            await bulkAdd(this.db.kanji, rows, total, current);
+            await bulkAdd('kanji', rows, total, current);
         };
 
         const kanjiMetaDataLoaded = async (summary, entries, total, current) => {
@@ -342,7 +454,7 @@ class Database {
                 });
             }
 
-            await bulkAdd(this.db.kanjiMeta, rows, total, current);
+            await bulkAdd('kanjiMeta', rows, total, current);
         };
 
         const tagDataLoaded = async (summary, entries, total, current) => {
@@ -360,7 +472,7 @@ class Database {
                 rows.push(row);
             }
 
-            await bulkAdd(this.db.tagMeta, rows, total, current);
+            await bulkAdd('tagMeta', rows, total, current);
         };
 
         return await Database.importDictionaryZip(
@@ -410,13 +522,13 @@ class Database {
 
         await indexDataLoaded(summary);
 
-        const buildTermBankName      = index => `term_bank_${index + 1}.json`;
-        const buildTermMetaBankName  = index => `term_meta_bank_${index + 1}.json`;
-        const buildKanjiBankName     = index => `kanji_bank_${index + 1}.json`;
-        const buildKanjiMetaBankName = index => `kanji_meta_bank_${index + 1}.json`;
-        const buildTagBankName       = index => `tag_bank_${index + 1}.json`;
+        const buildTermBankName      = (index) => `term_bank_${index + 1}.json`;
+        const buildTermMetaBankName  = (index) => `term_meta_bank_${index + 1}.json`;
+        const buildKanjiBankName     = (index) => `kanji_bank_${index + 1}.json`;
+        const buildKanjiMetaBankName = (index) => `kanji_meta_bank_${index + 1}.json`;
+        const buildTagBankName       = (index) => `tag_bank_${index + 1}.json`;
 
-        const countBanks = namer => {
+        const countBanks = (namer) => {
             let count = 0;
             while (zip.files[namer(count)]) {
                 ++count;
@@ -539,4 +651,176 @@ class Database {
             };
         });
     }
+
+    static getCounts(targets, query) {
+        const countPromises = [];
+        const counts = {};
+        for (const [objectStoreName, index] of targets) {
+            const n = objectStoreName;
+            const countPromise = Database.getCount(index, query).then((count) => counts[n] = count);
+            countPromises.push(countPromise);
+        }
+        return Promise.all(countPromises).then(() => counts);
+    }
+
+    static getCount(dbIndex, query) {
+        return new Promise((resolve, reject) => {
+            const request = dbIndex.count(query);
+            request.onerror = (e) => reject(e);
+            request.onsuccess = (e) => resolve(e.target.result);
+        });
+    }
+
+    static getAllKeys(dbIndex, query) {
+        const fn = typeof dbIndex.getAllKeys === 'function' ? Database.getAllKeysFast : Database.getAllKeysUsingCursor;
+        return fn(dbIndex, query);
+    }
+
+    static getAllKeysFast(dbIndex, query) {
+        return new Promise((resolve, reject) => {
+            const request = dbIndex.getAllKeys(query);
+            request.onerror = (e) => reject(e);
+            request.onsuccess = (e) => resolve(e.target.result);
+        });
+    }
+
+    static getAllKeysUsingCursor(dbIndex, query) {
+        return new Promise((resolve, reject) => {
+            const primaryKeys = [];
+            const request = dbIndex.openKeyCursor(query, 'next');
+            request.onerror = (e) => reject(e);
+            request.onsuccess = (e) => {
+                const cursor = e.target.result;
+                if (cursor) {
+                    primaryKeys.push(cursor.primaryKey);
+                    cursor.continue();
+                } else {
+                    resolve(primaryKeys);
+                }
+            };
+        });
+    }
+
+    static async deleteValues(dbObjectStore, dbIndex, query, onProgress, progressData, progressRate) {
+        const hasProgress = (typeof onProgress === 'function');
+        const count = await Database.getCount(dbIndex, query);
+        ++progressData.storesProcesed;
+        progressData.count += count;
+        if (hasProgress) {
+            onProgress(progressData);
+        }
+
+        const onValueDeleted = (
+            hasProgress ?
+            () => {
+                const p = ++progressData.processed;
+                if ((p % progressRate) === 0 || p === progressData.count) {
+                    onProgress(progressData);
+                }
+            } :
+            () => {}
+        );
+
+        const promises = [];
+        const primaryKeys = await Database.getAllKeys(dbIndex, query);
+        for (const key of primaryKeys) {
+            const promise = Database.deleteValue(dbObjectStore, key).then(onValueDeleted);
+            promises.push(promise);
+        }
+
+        await Promise.all(promises);
+    }
+
+    static deleteValue(dbObjectStore, key) {
+        return new Promise((resolve, reject) => {
+            const request = dbObjectStore.delete(key);
+            request.onerror = (e) => reject(e);
+            request.onsuccess = () => resolve();
+        });
+    }
+
+    static bulkAdd(objectStore, items, start, count) {
+        return new Promise((resolve, reject) => {
+            if (start + count > items.length) {
+                count = items.length - start;
+            }
+
+            if (count <= 0) {
+                resolve();
+                return;
+            }
+
+            const end = start + count;
+            let completedCount = 0;
+            const onError = (e) => reject(e);
+            const onSuccess = () => {
+                if (++completedCount >= count) {
+                    resolve();
+                }
+            };
+
+            for (let i = start; i < end; ++i) {
+                const request = objectStore.add(items[i]);
+                request.onerror = onError;
+                request.onsuccess = onSuccess;
+            }
+        });
+    }
+
+    static open(name, version, onUpgradeNeeded) {
+        return new Promise((resolve, reject) => {
+            const request = window.indexedDB.open(name, version * 10);
+
+            request.onupgradeneeded = (event) => {
+                try {
+                    request.transaction.onerror = (e) => reject(e);
+                    onUpgradeNeeded(request.result, request.transaction, event.oldVersion / 10, event.newVersion / 10);
+                } catch (e) {
+                    reject(e);
+                }
+            };
+
+            request.onerror = (e) => reject(e);
+            request.onsuccess = () => resolve(request.result);
+        });
+    }
+
+    static upgrade(db, transaction, oldVersion, upgrades) {
+        for (const {version, stores} of upgrades) {
+            if (oldVersion >= version) { continue; }
+
+            const objectStoreNames = Object.keys(stores);
+            for (const objectStoreName of objectStoreNames) {
+                const {primaryKey, indices} = stores[objectStoreName];
+
+                const objectStoreNames = transaction.objectStoreNames || db.objectStoreNames;
+                const objectStore = (
+                    Database.listContains(objectStoreNames, objectStoreName) ?
+                    transaction.objectStore(objectStoreName) :
+                    db.createObjectStore(objectStoreName, primaryKey)
+                );
+
+                for (const indexName of indices) {
+                    if (Database.listContains(objectStore.indexNames, indexName)) { continue; }
+
+                    objectStore.createIndex(indexName, indexName, {});
+                }
+            }
+        }
+    }
+
+    static deleteDatabase(dbName) {
+        return new Promise((resolve, reject) => {
+            const request = indexedDB.deleteDatabase(dbName);
+            request.onerror = (e) => reject(e);
+            request.onsuccess = () => resolve();
+        });
+    }
+
+    static listContains(list, value) {
+        for (let i = 0, ii = list.length; i < ii; ++i) {
+            if (list[i] === value) { return true; }
+        }
+        return false;
+    }
 }