Passively handle database errors

3 files changed, 108 insertions, 58 deletions

diff --git a/ext/bg/js/database.js b/ext/bg/js/database.js
index 3c7f6aab..093ec102 100644
--- a/ext/bg/js/database.js
+++ b/ext/bg/js/database.js
@@ -228,11 +228,47 @@ class Database {
         }
     }
 
-    async importDictionary(archive, callback) {
+    async importDictionary(archive, progressCallback, exceptions) {
         if (!this.db) {
             throw 'Database not initialized';
         }
 
+        const maxTransactionLength = 1000;
+        const bulkAdd = async (table, items, total, current) => {
+            if (items.length < maxTransactionLength) {
+                if (progressCallback) {
+                    progressCallback(total, current);
+                }
+
+                try {
+                    await table.bulkAdd(items);
+                } catch (e) {
+                    if (exceptions) {
+                        exceptions.push(e);
+                    } else {
+                        throw e;
+                    }
+                }
+            } else {
+                for (let i = 0; i < items.length; i += maxTransactionLength) {
+                    if (progressCallback) {
+                        progressCallback(total, current + i / items.length);
+                    }
+
+                    let count = Math.min(maxTransactionLength, items.length - i);
+                    try {
+                        await table.bulkAdd(items.slice(i, i + count));
+                    } catch (e) {
+                        if (exceptions) {
+                            exceptions.push(e);
+                        } else {
+                            throw e;
+                        }
+                    }
+                }
+            }
+        };
+
         const indexDataLoaded = async summary => {
             if (summary.version > 3) {
                 throw 'Unsupported dictionary version';
@@ -247,10 +283,6 @@ class Database {
         };
 
         const termDataLoaded = async (summary, entries, total, current) => {
-            if (callback) {
-                callback(total, current);
-            }
-
             const rows = [];
             if (summary.version === 1) {
                 for (const [expression, reading, definitionTags, rules, score, ...glossary] of entries) {
@@ -280,14 +312,10 @@ class Database {
                 }
             }
 
-            await this.db.terms.bulkAdd(rows);
+            await bulkAdd(this.db.terms, rows, total, current);
         };
 
         const termMetaDataLoaded = async (summary, entries, total, current) => {
-            if (callback) {
-                callback(total, current);
-            }
-
             const rows = [];
             for (const [expression, mode, data] of entries) {
                 rows.push({
@@ -298,14 +326,10 @@ class Database {
                 });
             }
 
-            await this.db.termMeta.bulkAdd(rows);
+            await bulkAdd(this.db.termMeta, rows, total, current);
         };
 
         const kanjiDataLoaded = async (summary, entries, total, current)  => {
-            if (callback) {
-                callback(total, current);
-            }
-
             const rows = [];
             if (summary.version === 1) {
                 for (const [character, onyomi, kunyomi, tags, ...meanings] of entries) {
@@ -332,14 +356,10 @@ class Database {
                 }
             }
 
-            await this.db.kanji.bulkAdd(rows);
+            await bulkAdd(this.db.kanji, rows, total, current);
         };
 
         const kanjiMetaDataLoaded = async (summary, entries, total, current) => {
-            if (callback) {
-                callback(total, current);
-            }
-
             const rows = [];
             for (const [character, mode, data] of entries) {
                 rows.push({
@@ -350,14 +370,10 @@ class Database {
                 });
             }
 
-            await this.db.kanjiMeta.bulkAdd(rows);
+            await bulkAdd(this.db.kanjiMeta, rows, total, current);
         };
 
         const tagDataLoaded = async (summary, entries, total, current) => {
-            if (callback) {
-                callback(total, current);
-            }
-
             const rows = [];
             for (const [name, category, order, notes, score] of entries) {
                 const row = dictTagSanitize({
@@ -372,7 +388,7 @@ class Database {
                 rows.push(row);
             }
 
-            await this.db.tagMeta.bulkAdd(rows);
+            await bulkAdd(this.db.tagMeta, rows, total, current);
         };
 
         return await Database.importDictionaryZip(
diff --git a/ext/bg/js/settings.js b/ext/bg/js/settings.js
index 579af085..99463593 100644
--- a/ext/bg/js/settings.js
+++ b/ext/bg/js/settings.js
@@ -193,7 +193,7 @@ async function onReady() {
         await dictionaryGroupsPopulate(options);
         await formMainDictionaryOptionsPopulate(options);
     } catch (e) {
-        dictionaryErrorShow(e);
+        dictionaryErrorsShow([e]);
     }
 
     try {
@@ -214,36 +214,63 @@ $(document).ready(utilAsync(onReady));
  * Dictionary
  */
 
-function dictionaryErrorShow(error) {
+function dictionaryErrorToString(error) {
+    if (error.toString) {
+        error = error.toString();
+    } else {
+        error = `${error}`;
+    }
+
+    for (const [match, subst] of dictionaryErrorToString.overrides) {
+        if (error.includes(match)) {
+            error = subst;
+            break;
+        }
+    }
+
+    return error;
+}
+dictionaryErrorToString.overrides = [
+    [
+        'A mutation operation was attempted on a database that did not allow mutations.',
+        'Access to IndexedDB appears to be restricted. Firefox seems to require that the history preference is set to "Remember history" before IndexedDB use of any kind is allowed.'
+    ],
+    [
+        'The operation failed for reasons unrelated to the database itself and not covered by any other error code.',
+        'Unable to access IndexedDB due to a possibly corrupt user profile. Try using the "Refresh Firefox" feature to reset your user profile.'
+    ],
+    [
+        'BulkError',
+        'Unable to finish importing dictionary data into IndexedDB. This may indicate that you do not have sufficient disk space available to complete this operation.'
+    ]
+];
+
+function dictionaryErrorsShow(errors) {
     const dialog = $('#dict-error');
-    if (error) {
-        const overrides = [
-            [
-                'A mutation operation was attempted on a database that did not allow mutations.',
-                'Access to IndexedDB appears to be restricted. Firefox seems to require that the history preference is set to "Remember history" before IndexedDB use of any kind is allowed.'
-            ],
-            [
-                'The operation failed for reasons unrelated to the database itself and not covered by any other error code.',
-                'Unable to access IndexedDB due to a possibly corrupt user profile. Try using the "Refresh Firefox" feature to reset your user profile.'
-            ],
-            [
-                'BulkError',
-                'Unable to finish importing dictionary data into IndexedDB. This may indicate that you do not have sufficient disk space available to complete this operation.'
-            ]
-        ];
-
-        if (error.toString) {
-            error = error.toString();
+    dialog.show().text('');
+
+    if (errors !== null && errors.length > 0) {
+        const uniqueErrors = {};
+        for (let e of errors) {
+            e = dictionaryErrorToString(e);
+            uniqueErrors[e] = uniqueErrors.hasOwnProperty(e) ? uniqueErrors[e] + 1 : 1;
         }
 
-        for (const [match, subst] of overrides) {
-            if (error.includes(match)) {
-                error = subst;
-                break;
+        for (const e in uniqueErrors) {
+            const count = uniqueErrors[e];
+            const div = document.createElement('p');
+            if (count > 1) {
+                div.textContent = `${e} `;
+                const em = document.createElement('em');
+                em.textContent = `(${count})`;
+                div.appendChild(em);
+            } else {
+                div.textContent = `${e}`;
             }
+            dialog.append($(div));
         }
 
-        dialog.show().text(error);
+        dialog.show();
     } else {
         dialog.hide();
     }
@@ -319,7 +346,7 @@ async function onDictionaryPurge(e) {
     const dictProgress = $('#dict-purge').show();
 
     try {
-        dictionaryErrorShow();
+        dictionaryErrorsShow(null);
         dictionarySpinnerShow(true);
 
         await utilDatabasePurge();
@@ -331,7 +358,7 @@ async function onDictionaryPurge(e) {
         await dictionaryGroupsPopulate(options);
         await formMainDictionaryOptionsPopulate(options);
     } catch (e) {
-        dictionaryErrorShow(e);
+        dictionaryErrorsShow([e]);
     } finally {
         dictionarySpinnerShow(false);
 
@@ -346,25 +373,32 @@ async function onDictionaryImport(e) {
     const dictProgress = $('#dict-import-progress').show();
 
     try {
-        dictionaryErrorShow();
+        dictionaryErrorsShow(null);
         dictionarySpinnerShow(true);
 
         const setProgress = percent => dictProgress.find('.progress-bar').css('width', `${percent}%`);
         const updateProgress = (total, current) => setProgress(current / total * 100.0);
         setProgress(0.0);
 
+        const exceptions = [];
         const options = await optionsLoad();
-        const summary = await utilDatabaseImport(e.target.files[0], updateProgress);
+        const summary = await utilDatabaseImport(e.target.files[0], updateProgress, exceptions);
         options.dictionaries[summary.title] = {enabled: true, priority: 0, allowSecondarySearches: false};
         if (summary.sequenced && options.general.mainDictionary === '') {
             options.general.mainDictionary = summary.title;
         }
+
+        if (exceptions.length > 0) {
+            exceptions.push(`Dictionary may not have been imported properly: ${exceptions.length} error${exceptions.length === 1 ? '' : 's'} reported.`);
+            dictionaryErrorsShow(exceptions);
+        }
+
         await optionsSave(options);
 
         await dictionaryGroupsPopulate(options);
         await formMainDictionaryOptionsPopulate(options);
     } catch (e) {
-        dictionaryErrorShow(e);
+        dictionaryErrorsShow([e]);
     } finally {
         dictionarySpinnerShow(false);
 
diff --git a/ext/bg/js/util.js b/ext/bg/js/util.js
index 216cef3f..34b06ddb 100644
--- a/ext/bg/js/util.js
+++ b/ext/bg/js/util.js
@@ -87,6 +87,6 @@ function utilDatabasePurge() {
     return utilBackend().translator.database.purge();
 }
 
-function utilDatabaseImport(data, progress) {
-    return utilBackend().translator.database.importDictionary(data, progress);
+function utilDatabaseImport(data, progress, exceptions) {
+    return utilBackend().translator.database.importDictionary(data, progress, exceptions);
 }