Merge branch 'dev'

8 files changed, 516 insertions, 275 deletions

diff --git a/ext/bg/js/deinflector.js b/ext/bg/js/deinflector.js
index 0eabd0f3..8b9f88e2 100644
--- a/ext/bg/js/deinflector.js
+++ b/ext/bg/js/deinflector.js
@@ -26,32 +26,36 @@ class Deinflection {
     }
 
     validate(validator) {
-        for (const tags of validator(this.term)) {
-            if (this.tags.length === 0) {
-                return true;
-            }
-
-            for (const tag of this.tags) {
-                if (tags.indexOf(tag) !== -1) {
+        return validator(this.term).then(sets => {
+            for (const tags of sets) {
+                if (this.tags.length === 0) {
                     return true;
                 }
+
+                for (const tag of this.tags) {
+                    if (tags.includes(tag)) {
+                        return true;
+                    }
+                }
             }
-        }
 
-        return false;
+            return false;
+        });
     }
 
     deinflect(validator, rules) {
-        if (this.validate(validator)) {
-            const child = new Deinflection(this.term, this.tags);
-            this.children.push(child);
-        }
+        const promises = [
+            this.validate(validator).then(valid => {
+                const child = new Deinflection(this.term, this.tags);
+                this.children.push(child);
+            })
+        ];
 
         for (const rule in rules) {
             for (const variant of rules[rule]) {
                 let allowed = this.tags.length === 0;
                 for (const tag of this.tags) {
-                    if (variant.ti.indexOf(tag) !== -1) {
+                    if (variant.ti.includes(tag)) {
                         allowed = true;
                         break;
                     }
@@ -62,14 +66,24 @@ class Deinflection {
                 }
 
                 const term = this.term.slice(0, -variant.ki.length) + variant.ko;
-                const child = new Deinflection(term, variant.to, rule);
-                if (child.deinflect(validator, rules)) {
-                    this.children.push(child);
+                if (term.length === 0) {
+                    continue;
                 }
+
+                const child = new Deinflection(term, variant.to, rule);
+                promises.push(
+                    child.deinflect(validator, rules).then(valid => {
+                        if (valid) {
+                            this.children.push(child);
+                        }
+                    }
+                ));
             }
         }
 
-        return this.children.length > 0;
+        return Promise.all(promises).then(() => {
+            return this.children.length > 0;
+        });
     }
 
     gather() {
@@ -105,10 +119,6 @@ class Deinflector {
 
     deinflect(term, validator) {
         const node = new Deinflection(term);
-        if (node.deinflect(validator, this.rules)) {
-            return node.gather();
-        }
-
-        return null;
+        return node.deinflect(validator, this.rules).then(success => success ? node.gather() : []);
     }
 }
diff --git a/ext/bg/js/dictionary.js b/ext/bg/js/dictionary.js
index a6438523..4562c821 100644
--- a/ext/bg/js/dictionary.js
+++ b/ext/bg/js/dictionary.js
@@ -19,63 +19,199 @@
 
 class Dictionary {
     constructor() {
-        this.termDicts = {};
-        this.kanjiDicts = {};
+        this.db = null;
+        this.dbVer = 1;
+        this.entities = null;
     }
 
-    addTermDict(name, dict) {
-        this.termDicts[name] = dict;
-    }
+    initDb() {
+        if (this.db !== null) {
+            return Promise.reject('database already initialized');
+        }
 
-    addKanjiDict(name, dict) {
-        this.kanjiDicts[name] = dict;
+        this.db = new Dexie('dict');
+        this.db.version(1).stores({
+            terms: '++id,expression,reading',
+            entities: '++,name',
+            kanji: '++,character',
+            meta: 'name,value',
+        });
     }
 
-    findTerm(term) {
-        let results = [];
+    prepareDb() {
+        this.initDb();
 
-        for (let name in this.termDicts) {
-            const dict = this.termDicts[name];
-            if (!(term in dict.i)) {
-                continue;
+        return this.db.meta.get('version').then(row => {
+            return row ? row.value : 0;
+        }).catch(() => {
+            return 0;
+        }).then(version => {
+            if (this.dbVer === version) {
+                return true;
             }
 
-            const indices = dict.i[term].split(' ').map(Number);
-            results = results.concat(
-                indices.map(index => {
-                    const [e, r, t, ...g] = dict.d[index];
-                    return {
-                        expression: e,
-                        reading:    r,
-                        tags:       t.split(' '),
-                        glossary:   g,
-                        entities:   dict.e,
-                        id:         index
-                    };
-                })
-            );
+            const db = this.db;
+            this.db.close();
+            this.db = null;
+
+            return db.delete().then(() => {
+                this.initDb();
+                return false;
+            });
+        });
+    }
+
+    sealDb() {
+        if (this.db === null) {
+            return Promise.reject('database not initialized');
         }
 
-        return results;
+        return this.db.meta.put({name: 'version', value: this.dbVer});
+    }
+
+    findTerm(term) {
+        if (this.db === null) {
+            return Promise.reject('database not initialized');
+        }
+
+        const results = [];
+        return this.db.terms.where('expression').equals(term).or('reading').equals(term).each(row => {
+            results.push({
+                expression: row.expression,
+                reading: row.reading,
+                tags: row.tags.split(' '),
+                glossary: row.glossary,
+                id: row.id
+            });
+        }).then(() => {
+            return this.getEntities();
+        }).then(entities => {
+            for (const result of results) {
+                result.entities = entities;
+            }
+
+            return results;
+        });
     }
 
     findKanji(kanji) {
+        if (this.db === null) {
+            return Promise.reject('database not initialized');
+        }
+
         const results = [];
+        return this.db.kanji.where('character').equals(kanji).each(row => {
+            results.push({
+                character: row.character,
+                onyomi: row.onyomi.split(' '),
+                kunyomi: row.kunyomi.split(' '),
+                tags: row.tags.split(' '),
+                glossary: row.meanings
+            });
+        }).then(() => results);
+    }
 
-        for (let name in this.kanjiDicts) {
-            const def = this.kanjiDicts[name].c[kanji];
-            if (def) {
-                const [k, o, t, ...g] = def;
-                results.push({
-                    character: kanji,
-                    kunyomi:   k.split(' '),
-                    onyomi:    o.split(' '),
-                    tags:      t.split(' '),
-                    glossary:  g
-                });
+    getEntities(tags) {
+        if (this.db === null) {
+            return Promise.reject('database not initialized');
+        }
+
+        if (this.entities !== null) {
+            return Promise.resolve(this.entities);
+        }
+
+        return this.db.entities.toArray(rows => {
+            this.entities = {};
+            for (const row of rows) {
+                this.entities[row.name] = row.value;
+            }
+
+            return this.entities;
+        });
+    }
+
+    importTermDict(indexUrl, callback) {
+        if (this.db === null) {
+            return Promise.reject('database not initialized');
+        }
+
+        const indexDir = indexUrl.slice(0, indexUrl.lastIndexOf('/'));
+        return loadJson(indexUrl).then(index => {
+            const entities = [];
+            for (const [name, value] of index.ents) {
+                entities.push({name, value});
             }
+
+            return this.db.entities.bulkAdd(entities).then(() => {
+                if (this.entities === null) {
+                    this.entities = {};
+                }
+
+                for (const entity of entities) {
+                    this.entities[entity.name] = entity.value;
+                }
+            }).then(() => {
+                const loaders = [];
+                for (let i = 1; i <= index.banks; ++i) {
+                    const bankUrl = `${indexDir}/bank_${i}.json`;
+                    loaders.push(() => {
+                        return loadJson(bankUrl).then(definitions => {
+                            const rows = [];
+                            for (const [expression, reading, tags, ...glossary] of definitions) {
+                                rows.push({expression, reading, tags, glossary});
+                            }
+
+                            return this.db.terms.bulkAdd(rows).then(() => {
+                                if (callback) {
+                                    callback(i, index.banks, indexUrl);
+                                }
+                            });
+                        });
+                    });
+                }
+
+                let chain = Promise.resolve();
+                for (const loader of loaders) {
+                    chain = chain.then(loader);
+                }
+
+                return chain;
+            });
+        });
+    }
+
+    importKanjiDict(indexUrl, callback) {
+        if (this.db === null) {
+            return Promise.reject('database not initialized');
         }
 
-        return results;
+        const indexDir = indexUrl.slice(0, indexUrl.lastIndexOf('/'));
+        return loadJson(indexUrl).then(index => {
+            const loaders = [];
+            for (let i = 1; i <= index.banks; ++i) {
+                const bankUrl = `${indexDir}/bank_${i}.json`;
+                loaders.push(() => {
+                    return loadJson(bankUrl).then(definitions => {
+                        const rows = [];
+                        for (const [character, onyomi, kunyomi, tags, ...meanings] of definitions) {
+                            rows.push({character, onyomi, kunyomi, tags, meanings});
+                        }
+
+                        return this.db.kanji.bulkAdd(rows).then(() => {
+                            if (callback) {
+                                callback(i, index.banks, indexUrl);
+                            }
+                        });
+                    });
+                });
+            }
+
+            let chain = Promise.resolve();
+            for (const loader of loaders) {
+                chain = chain.then(loader);
+            }
+
+            return chain;
+        });
     }
 }
diff --git a/ext/bg/js/import.js b/ext/bg/js/import.js
new file mode 100644
index 00000000..0601cb9f
--- /dev/null
+++ b/ext/bg/js/import.js
@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2016  Alex Yatskov <alex@foosoft.net>
+ * Author: Alex Yatskov <alex@foosoft.net>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+function api_setProgress(progress) {
+    $('.progress-bar').css('width', `${progress}%`);
+
+    if (progress === 100.0) {
+        $('.progress').hide();
+        $('.alert').show();
+    }
+}
+
+chrome.runtime.onMessage.addListener(({action, params}, sender, callback) => {
+    const method = this['api_' + action];
+    if (typeof(method) === 'function') {
+        method.call(this, params);
+    }
+
+    callback();
+});
diff --git a/ext/bg/js/options-form.js b/ext/bg/js/options-form.js
index 3dab0a87..1cd050b4 100644
--- a/ext/bg/js/options-form.js
+++ b/ext/bg/js/options-form.js
@@ -32,7 +32,7 @@ function fieldsToDict(selection) {
 
 function modelIdToFieldOptKey(id) {
     return {
-        'anki-term-model':  'ankiTermFields',
+        'anki-term-model': 'ankiTermFields',
         'anki-kanji-model': 'ankiKanjiFields'
     }[id];
 }
@@ -60,15 +60,14 @@ function modelIdToMarkers(id) {
     }[id];
 }
 
-function formToOptions(section, callback) {
-    loadOptions((optsOld) => {
+function formToOptions(section) {
+    return loadOptions().then(optsOld => {
         const optsNew = $.extend({}, optsOld);
 
         switch (section) {
             case 'general':
                 optsNew.scanLength = parseInt($('#scan-length').val(), 10);
                 optsNew.activateOnStartup = $('#activate-on-startup').prop('checked');
-                optsNew.loadEnamDict = $('#load-enamdict').prop('checked');
                 optsNew.selectMatchedText = $('#select-matched-text').prop('checked');
                 optsNew.showAdvancedOptions = $('#show-advanced-options').prop('checked');
                 optsNew.enableAudioPlayback = $('#enable-audio-playback').prop('checked');
@@ -86,7 +85,10 @@ function formToOptions(section, callback) {
                 break;
         }
 
-        callback(sanitizeOptions(optsNew), sanitizeOptions(optsOld));
+        return {
+            optsNew: sanitizeOptions(optsNew),
+            optsOld: sanitizeOptions(optsOld)
+        };
     });
 }
 
@@ -95,9 +97,9 @@ function populateAnkiDeckAndModel(opts) {
 
     const ankiDeck = $('.anki-deck');
     ankiDeck.find('option').remove();
-    yomi.api_getDeckNames({callback: (names) => {
+    yomi.api_getDeckNames({callback: names => {
         if (names !== null) {
-            names.forEach((name) => ankiDeck.append($('<option/>', {value: name, text: name})));
+            names.forEach(name => ankiDeck.append($('<option/>', {value: name, text: name})));
         }
 
         $('#anki-term-deck').val(opts.ankiTermDeck);
@@ -106,9 +108,9 @@ function populateAnkiDeckAndModel(opts) {
 
     const ankiModel = $('.anki-model');
     ankiModel.find('option').remove();
-    yomi.api_getModelNames({callback: (names) => {
+    yomi.api_getModelNames({callback: names => {
         if (names !== null) {
-            names.forEach((name) => ankiModel.append($('<option/>', {value: name, text: name})));
+            names.forEach(name => ankiModel.append($('<option/>', {value: name, text: name})));
         }
 
         populateAnkiFields($('#anki-term-model').val(opts.ankiTermModel), opts);
@@ -119,7 +121,7 @@ function populateAnkiDeckAndModel(opts) {
 function updateAnkiStatus() {
     $('.error-dlg').hide();
 
-    yomichan().api_getVersion({callback: (version) => {
+    yomichan().api_getVersion({callback: version => {
         if (version === null) {
             $('.error-dlg-connection').show();
             $('.options-anki-controls').hide();
@@ -142,19 +144,19 @@ function populateAnkiFields(element, opts) {
     const optKey = modelIdToFieldOptKey(modelId);
     const markers = modelIdToMarkers(modelId);
 
-    yomichan().api_getModelFieldNames({modelName, callback: (names) => {
+    yomichan().api_getModelFieldNames({modelName, callback: names => {
         const table = element.closest('.tab-pane').find('.anki-fields');
         table.find('tbody').remove();
 
         const tbody = $('<tbody>');
-        names.forEach((name) => {
+        names.forEach(name => {
             const button = $('<button>', {type: 'button', class: 'btn btn-default dropdown-toggle'});
             button.attr('data-toggle', 'dropdown').dropdown();
 
             const markerItems = $('<ul>', {class: 'dropdown-menu dropdown-menu-right'});
             for (const marker of markers) {
                 const link = $('<a>', {href: '#'}).text(`{${marker}}`);
-                link.click((e) => {
+                link.click(e => {
                     e.preventDefault();
                     link.closest('.input-group').find('.anki-field-value').val(link.text()).trigger('change');
                 });
@@ -185,8 +187,8 @@ function onOptionsGeneralChanged(e) {
         return;
     }
 
-    formToOptions('general', (optsNew, optsOld) => {
-        saveOptions(optsNew, () => {
+    formToOptions('general').then(({optsNew, optsOld}) => {
+        saveOptions(optsNew).then(() => {
             yomichan().setOptions(optsNew);
             if (!optsOld.enableAnkiConnect && optsNew.enableAnkiConnect) {
                 updateAnkiStatus();
@@ -210,30 +212,29 @@ function onOptionsAnkiChanged(e) {
         return;
     }
 
-    formToOptions('anki', (opts) => {
-        saveOptions(opts, () => yomichan().setOptions(opts));
+    formToOptions('anki').then(({optsNew, optsOld}) => {
+        saveOptions(optsNew).then(() => yomichan().setOptions(optsNew));
     });
 }
 
 function onAnkiModelChanged(e) {
     if (e.originalEvent) {
-        formToOptions('anki', (opts) => {
-            opts[modelIdToFieldOptKey($(this).id)] = {};
-            populateAnkiFields($(this), opts);
-            saveOptions(opts, () => yomichan().setOptions(opts));
+        formToOptions('anki').then(({optsNew, optsOld}) => {
+            optsNew[modelIdToFieldOptKey($(this).id)] = {};
+            populateAnkiFields($(this), optsNew);
+            saveOptions(optsNew).then(() => yomichan().setOptions(optsNew));
         });
     }
 }
 
 $(document).ready(() => {
-    loadOptions((opts) => {
-        $('#scan-length').val(opts.scanLength);
+    loadOptions().then(opts => {
         $('#activate-on-startup').prop('checked', opts.activateOnStartup);
-        $('#load-enamdict').prop('checked', opts.loadEnamDict);
         $('#select-matched-text').prop('checked', opts.selectMatchedText);
-        $('#show-advanced-options').prop('checked', opts.showAdvancedOptions);
         $('#enable-audio-playback').prop('checked', opts.enableAudioPlayback);
         $('#enable-anki-connect').prop('checked', opts.enableAnkiConnect);
+        $('#show-advanced-options').prop('checked', opts.showAdvancedOptions);
+        $('#scan-length').val(opts.scanLength);
 
         $('#anki-card-tags').val(opts.ankiCardTags.join(' '));
         $('#sentence-extent').val(opts.sentenceExtent);
diff --git a/ext/bg/js/options.js b/ext/bg/js/options.js
index 831bb817..915164c7 100644
--- a/ext/bg/js/options.js
+++ b/ext/bg/js/options.js
@@ -19,21 +19,22 @@
 
 function sanitizeOptions(options) {
     const defaults = {
-        scanLength:          20,
-        activateOnStartup:   false,
-        selectMatchedText:   true,
-        showAdvancedOptions: false,
-        loadEnamDict:        false,
+        activateOnStartup: true,
+        selectMatchedText: true,
         enableAudioPlayback: true,
-        enableAnkiConnect:   false,
-        ankiCardTags:        ['yomichan'],
-        sentenceExtent:      200,
-        ankiTermDeck:        '',
-        ankiTermModel:       '',
-        ankiTermFields:      {},
-        ankiKanjiDeck:       '',
-        ankiKanjiModel:      '',
-        ankiKanjiFields:     {}
+        enableAnkiConnect: false,
+        showAdvancedOptions: false,
+        scanLength: 20,
+
+        ankiCardTags: ['yomichan'],
+        sentenceExtent: 200,
+
+        ankiTermDeck: '',
+        ankiTermModel: '',
+        ankiTermFields: {},
+        ankiKanjiDeck: '',
+        ankiKanjiModel: '',
+        ankiKanjiFields: {}
     };
 
     for (const key in defaults) {
@@ -45,10 +46,16 @@ function sanitizeOptions(options) {
     return options;
 }
 
-function loadOptions(callback) {
-    chrome.storage.sync.get(null, (items) => callback(sanitizeOptions(items)));
+function loadOptions() {
+    return new Promise((resolve, reject) => {
+        chrome.storage.sync.get(null, opts => {
+            resolve(sanitizeOptions(opts));
+        });
+    });
 }
 
-function saveOptions(opts, callback) {
-    chrome.storage.sync.set(sanitizeOptions(opts), callback);
+function saveOptions(opts) {
+    return new Promise((resolve, reject) => {
+        chrome.storage.sync.set(sanitizeOptions(opts), resolve);
+    });
 }
diff --git a/ext/bg/js/translator.js b/ext/bg/js/translator.js
index fd414847..e534e0cb 100644
--- a/ext/bg/js/translator.js
+++ b/ext/bg/js/translator.js
@@ -25,171 +25,199 @@ class Translator {
         this.deinflector = new Deinflector();
     }
 
-    loadData({loadEnamDict=true}, callback) {
+    loadData(callback) {
         if (this.loaded) {
-            callback();
-            return;
+            return Promise.resolve();
         }
 
-        Translator.loadData('bg/data/rules.json')
-            .then((response) => {
-                this.deinflector.setRules(JSON.parse(response));
-                return Translator.loadData('bg/data/tags.json');
-            })
-            .then((response) => {
-                this.tagMeta = JSON.parse(response);
-                return Translator.loadData('bg/data/edict.json');
-            })
-            .then((response) => {
-                this.dictionary.addTermDict('edict', JSON.parse(response));
-                return Translator.loadData('bg/data/kanjidic.json');
-            })
-            .then((response) => {
-                this.dictionary.addKanjiDict('kanjidic', JSON.parse(response));
-                return loadEnamDict ? Translator.loadData('bg/data/enamdict.json') : Promise.resolve(null);
-            })
-            .then((response) => {
-                if (response !== null) {
-                    this.dictionary.addTermDict('enamdict', JSON.parse(response));
+        return loadJson('bg/data/rules.json').then(rules => {
+            this.deinflector.setRules(rules);
+            return loadJson('bg/data/tags.json');
+        }).then(tagMeta => {
+            this.tagMeta = tagMeta;
+            return this.dictionary.prepareDb();
+        }).then(exists => {
+            if (exists) {
+                return;
+            }
+
+            if (callback) {
+                callback({state: 'begin', progress: 0});
+            }
+
+            const banks = {};
+            const bankCallback = (loaded, total, indexUrl) => {
+                banks[indexUrl] = {loaded, total};
+
+                let percent = 0.0;
+                for (const url in banks) {
+                    percent += banks[url].loaded / banks[url].total;
                 }
 
-                this.loaded = true;
-                callback();
-            });
-    }
+                percent /= 3.0;
 
-    findTerm(text) {
-        const groups = {};
-        for (let i = text.length; i > 0; --i) {
-            const term = text.slice(0, i);
-            const dfs = this.deinflector.deinflect(term, t => {
-                const tags = [];
-                for (const d of this.dictionary.findTerm(t)) {
-                    tags.push(d.tags);
+                if (callback) {
+                    callback({state: 'update', progress: Math.ceil(100.0 * percent)});
                 }
+            };
 
-                return tags;
+            return Promise.all([
+                this.dictionary.importTermDict('bg/data/edict/index.json', bankCallback),
+                this.dictionary.importTermDict('bg/data/enamdict/index.json', bankCallback),
+                this.dictionary.importKanjiDict('bg/data/kanjidic/index.json', bankCallback),
+            ]).then(() => {
+                return this.dictionary.sealDb();
+            }).then(() => {
+                if (callback) {
+                    callback({state: 'end', progress: 100.0});
+                }
             });
+        }).then(() => {
+            this.loaded = true;
+        });
+    }
 
-            if (dfs === null) {
-                continue;
-            }
+    findTermGroups(text) {
+        const deinflectGroups = {};
+        const deinflectPromises = [];
 
-            for (const df of dfs) {
-                this.processTerm(groups, df.source, df.tags, df.rules, df.root);
-            }
+        for (let i = text.length; i > 0; --i) {
+            deinflectPromises.push(
+                this.deinflector.deinflect(text.slice(0, i), term => {
+                    return this.dictionary.findTerm(term).then(definitions => definitions.map(definition => definition.tags));
+                }).then(deinflects => {
+                    const processPromises = [];
+                    for (const deinflect of deinflects) {
+                        processPromises.push(this.processTerm(
+                            deinflectGroups,
+                            deinflect.source,
+                            deinflect.tags,
+                            deinflect.rules,
+                            deinflect.root
+                        ));
+                    }
+
+                    return Promise.all(processPromises);
+                })
+            );
         }
 
-        let definitions = [];
-        for (const key in groups) {
-            definitions.push(groups[key]);
-        }
+        return Promise.all(deinflectPromises).then(() => deinflectGroups);
+    }
 
-        definitions = definitions.sort((v1, v2) => {
-            const sl1 = v1.source.length;
-            const sl2 = v2.source.length;
-            if (sl1 > sl2) {
-                return -1;
-            } else if (sl1 < sl2) {
-                return 1;
+    findTerm(text) {
+        return this.findTermGroups(text).then(deinflectGroups => {
+            let definitions = [];
+            for (const key in deinflectGroups) {
+                definitions.push(deinflectGroups[key]);
             }
 
-            const s1 = v1.score;
-            const s2 = v2.score;
-            if (s1 > s2) {
-                return -1;
-            } else if (s1 < s2) {
-                return 1;
-            }
+            definitions = definitions.sort((v1, v2) => {
+                const sl1 = v1.source.length;
+                const sl2 = v2.source.length;
+                if (sl1 > sl2) {
+                    return -1;
+                } else if (sl1 < sl2) {
+                    return 1;
+                }
 
-            const rl1 = v1.rules.length;
-            const rl2 = v2.rules.length;
-            if (rl1 < rl2) {
-                return -1;
-            } else if (rl1 > rl2) {
-                return 1;
-            }
+                const s1 = v1.score;
+                const s2 = v2.score;
+                if (s1 > s2) {
+                    return -1;
+                } else if (s1 < s2) {
+                    return 1;
+                }
 
-            return v2.expression.localeCompare(v1.expression);
-        });
+                const rl1 = v1.rules.length;
+                const rl2 = v2.rules.length;
+                if (rl1 < rl2) {
+                    return -1;
+                } else if (rl1 > rl2) {
+                    return 1;
+                }
 
-        let length = 0;
-        for (const result of definitions) {
-            length = Math.max(length, result.source.length);
-        }
+                return v2.expression.localeCompare(v1.expression);
+            });
+
+            let length = 0;
+            for (const result of definitions) {
+                length = Math.max(length, result.source.length);
+            }
 
-        return {definitions: definitions, length: length};
+            return {definitions, length};
+        });
     }
 
     findKanji(text) {
-        let definitions = [];
         const processed = {};
+        const promises = [];
 
         for (const c of text) {
             if (!processed[c]) {
-                definitions = definitions.concat(this.dictionary.findKanji(c));
+                promises.push(this.dictionary.findKanji(c).then((definitions) => definitions));
                 processed[c] = true;
             }
         }
 
-        return this.processKanji(definitions);
+        return Promise.all(promises).then(sets => this.processKanji(sets.reduce((a, b) => a.concat(b))));
     }
 
     processTerm(groups, source, tags, rules, root) {
-        for (const entry of this.dictionary.findTerm(root)) {
-            if (entry.id in groups) {
-                continue;
-            }
+        return this.dictionary.findTerm(root).then(definitions => {
+            for (const definition of definitions) {
+                if (definition.id in groups) {
+                    continue;
+                }
 
-            let matched = tags.length === 0;
-            for (const tag of tags) {
-                if (entry.tags.indexOf(tag) !== -1) {
-                    matched = true;
-                    break;
+                let matched = tags.length === 0;
+                for (const tag of tags) {
+                    if (definition.tags.includes(tag)) {
+                        matched = true;
+                        break;
+                    }
                 }
-            }
 
-            if (!matched) {
-                continue;
-            }
+                if (!matched) {
+                    continue;
+                }
 
-            const tagItems = [];
-            for (const tag of entry.tags) {
-                const tagItem = {
-                    name: tag,
-                    class: 'default',
-                    order: Number.MAX_SAFE_INTEGER,
-                    score: 0,
-                    desc: entry.entities[tag] || '',
-                };
+                const tagItems = [];
+                for (const tag of definition.tags) {
+                    const tagItem = {
+                        name: tag,
+                        class: 'default',
+                        order: Number.MAX_SAFE_INTEGER,
+                        score: 0,
+                        desc: definition.entities[tag] || '',
+                    };
+
+                    this.applyTagMeta(tagItem);
+                    tagItems.push(tagItem);
+                }
 
-                this.applyTagMeta(tagItem);
-                tagItems.push(tagItem);
-            }
+                let score = 0;
+                for (const tagItem of tagItems) {
+                    score += tagItem.score;
+                }
 
-            let score = 0;
-            for (const tagItem of tagItems) {
-                score += tagItem.score;
+                groups[definition.id] = {
+                    score,
+                    source,
+                    rules,
+                    expression: definition.expression,
+                    reading: definition.reading,
+                    glossary: definition.glossary,
+                    tags: Translator.sortTags(tagItems)
+                };
             }
-
-            groups[entry.id] = {
-                score,
-                source,
-                rules,
-                expression: entry.expression,
-                reading: entry.reading,
-                glossary: entry.glossary,
-                tags: Translator.sortTags(tagItems)
-            };
-        }
+        });
     }
 
-    processKanji(entries) {
-        const results = [];
-
-        for (const entry of entries) {
+    processKanji(definitions) {
+        for (const definition of definitions) {
             const tagItems = [];
-            for (const tag of entry.tags) {
+            for (const tag of definition.tags) {
                 const tagItem = {
                     name: tag,
                     class: 'default',
@@ -201,16 +229,10 @@ class Translator {
                 tagItems.push(tagItem);
             }
 
-            results.push({
-                character: entry.character,
-                kunyomi: entry.kunyomi,
-                onyomi: entry.onyomi,
-                glossary: entry.glossary,
-                tags: Translator.sortTags(tagItems)
-            });
+            definition.tags = Translator.sortTags(tagItems);
         }
 
-        return results;
+        return definitions;
     }
 
     applyTagMeta(tag) {
@@ -241,18 +263,4 @@ class Translator {
             return 0;
         });
     }
-
-    static isKanji(c) {
-        const code = c.charCodeAt(0);
-        return code >= 0x4e00 && code < 0x9fb0 || code >= 0x3400 && code < 0x4dc0;
-    }
-
-    static loadData(url) {
-        return new Promise((resolve, reject) => {
-            const xhr = new XMLHttpRequest();
-            xhr.addEventListener('load', () => resolve(xhr.responseText));
-            xhr.open('GET', chrome.extension.getURL(url), true);
-            xhr.send();
-        });
-    }
 }
diff --git a/ext/bg/js/util.js b/ext/bg/js/util.js
new file mode 100644
index 00000000..5583502d
--- /dev/null
+++ b/ext/bg/js/util.js
@@ -0,0 +1,46 @@
+/*
+ * Copyright (C) 2016  Alex Yatskov <alex@foosoft.net>
+ * Author: Alex Yatskov <alex@foosoft.net>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+function kanjiLinks(options) {
+    let result = '';
+    for (const c of options.fn(this)) {
+        if (isKanji(c)) {
+            result += Handlebars.templates['kanji-link.html']({kanji: c}).trim();
+        } else {
+            result += c;
+        }
+    }
+
+    return result;
+}
+
+function loadJson(url) {
+    return new Promise((resolve, reject) => {
+        const xhr = new XMLHttpRequest();
+        xhr.addEventListener('load', () => resolve(JSON.parse(xhr.responseText)));
+        xhr.open('GET', chrome.extension.getURL(url), true);
+        xhr.send();
+    });
+}
+
+function isKanji(c) {
+    const code = c.charCodeAt(0);
+    return code >= 0x4e00 && code < 0x9fb0 || code >= 0x3400 && code < 0x4dc0;
+}
+
diff --git a/ext/bg/js/yomichan.js b/ext/bg/js/yomichan.js
index fd9b84d3..f1b3ffc4 100644
--- a/ext/bg/js/yomichan.js
+++ b/ext/bg/js/yomichan.js
@@ -20,31 +20,20 @@
 class Yomichan {
     constructor() {
         Handlebars.partials = Handlebars.templates;
-        Handlebars.registerHelper('kanjiLinks', function(options) {
-            let result = '';
-            for (const c of options.fn(this)) {
-                if (Translator.isKanji(c)) {
-                    result += Handlebars.templates['kanji-link.html']({kanji: c}).trim();
-                } else {
-                    result += c;
-                }
-            }
-
-            return result;
-        });
+        Handlebars.registerHelper('kanjiLinks', kanjiLinks);
 
         this.translator = new Translator();
+        this.importTabId = null;
         this.asyncPools = {};
         this.ankiConnectVer = 0;
         this.setState('disabled');
 
-        chrome.runtime.onInstalled.addListener(this.onInstalled.bind(this));
         chrome.runtime.onMessage.addListener(this.onMessage.bind(this));
         chrome.browserAction.onClicked.addListener(this.onBrowserAction.bind(this));
-        chrome.tabs.onCreated.addListener((tab) => this.onTabReady(tab.id));
+        chrome.tabs.onCreated.addListener(tab => this.onTabReady(tab.id));
         chrome.tabs.onUpdated.addListener(this.onTabReady.bind(this));
 
-        loadOptions((opts) => {
+        loadOptions().then(opts => {
             this.setOptions(opts);
             if (this.options.activateOnStartup) {
                 this.setState('loading');
@@ -52,9 +41,17 @@ class Yomichan {
         });
     }
 
-    onInstalled(details) {
-        if (details.reason === 'install') {
-            chrome.tabs.create({url: chrome.extension.getURL('bg/guide.html')});
+    onImport({state, progress}) {
+        if (state === 'begin') {
+            chrome.tabs.create({url: chrome.extension.getURL('bg/import.html')}, tab => this.importTabId = tab.id);
+        }
+
+        if (this.importTabId !== null) {
+            this.tabInvoke(this.importTabId, 'setProgress', progress);
+        }
+
+        if (state === 'end') {
+            this.importTabId = null;
         }
     }
 
@@ -101,7 +98,7 @@ class Yomichan {
                 break;
             case 'loading':
                 chrome.browserAction.setBadgeText({text: '...'});
-                this.translator.loadData({loadEnamDict: this.options.loadEnamDict}, () => this.setState('enabled'));
+                this.translator.loadData(this.onImport.bind(this)).then(() => this.setState('enabled'));
                 break;
         }
 
@@ -118,7 +115,7 @@ class Yomichan {
     }
 
     tabInvokeAll(action, params) {
-        chrome.tabs.query({}, (tabs) => {
+        chrome.tabs.query({}, tabs => {
             for (const tab of tabs) {
                 this.tabInvoke(tab.id, action, params);
             }
@@ -133,7 +130,7 @@ class Yomichan {
         if (this.ankiConnectVer === this.getApiVersion()) {
             this.ankiInvoke(action, params, pool, callback);
         } else {
-            this.api_getVersion({callback: (version) => {
+            this.api_getVersion({callback: version => {
                 if (version === this.getApiVersion()) {
                     this.ankiConnectVer = version;
                     this.ankiInvoke(action, params, pool, callback);
@@ -209,7 +206,7 @@ class Yomichan {
                     break;
                 case 'tags':
                     if (definition.tags) {
-                        value = definition.tags.map((t) => t.name);
+                        value = definition.tags.map(t => t.name);
                     }
                     break;
             }
@@ -244,7 +241,7 @@ class Yomichan {
             };
 
             for (const name in fields) {
-                if (fields[name].indexOf('{audio}') !== -1) {
+                if (fields[name].includes('{audio}')) {
                     audio.fields.push(name);
                 }
             }
@@ -274,7 +271,7 @@ class Yomichan {
             }
         }
 
-        this.ankiInvokeSafe('canAddNotes', {notes}, 'notes', (results) => {
+        this.ankiInvokeSafe('canAddNotes', {notes}, 'notes', results => {
             const states = [];
 
             if (results !== null) {
@@ -293,11 +290,11 @@ class Yomichan {
     }
 
     api_findKanji({text, callback}) {
-        callback(this.translator.findKanji(text));
+        this.translator.findKanji(text).then(result => callback(result));
     }
 
     api_findTerm({text, callback}) {
-        callback(this.translator.findTerm(text));
+        this.translator.findTerm(text).then(result => callback(result));
     }
 
     api_getDeckNames({callback}) {