merged mode: implement missing stuff, refactoring

- use correct tags
- indicate popular and rare terms
- indicate definitions restricted to specific terms
- frequencies (Innocent Corpus)

6 files changed, 308 insertions, 154 deletions

diff --git a/ext/bg/js/database.js b/ext/bg/js/database.js
index 9787b0f8..0de0505d 100644
--- a/ext/bg/js/database.js
+++ b/ext/bg/js/database.js
@@ -30,7 +30,7 @@ class Database {
 
         this.db = new Dexie('dict');
         this.db.version(2).stores({
-            terms:        '++id,dictionary,expression,reading,sequence',
+            terms:        '++id,dictionary,expression,reading',
             kanji:        '++,dictionary,character',
             tagMeta:      '++,dictionary',
             dictionaries: '++,title,version'
@@ -40,6 +40,9 @@ class Database {
             kanjiMeta: '++,dictionary,character',
             tagMeta:   '++,dictionary,name'
         });
+        this.db.version(4).stores({
+            terms: '++id,dictionary,expression,reading,sequence'
+        });
 
         await this.db.open();
     }
@@ -74,7 +77,7 @@ class Database {
                     score: row.score,
                     dictionary: row.dictionary,
                     id: row.id,
-                    sequence: row.sequence
+                    sequence: typeof row.sequence === 'undefined' ? -1 : row.sequence
                 });
             }
         });
@@ -82,14 +85,15 @@ class Database {
         return results;
     }
 
-    async findEntry(sequence) {
+    async findTermsBySequence(sequence, dictionary) {
         if (!this.db) {
             throw 'Database not initialized';
         }
 
-        const entry = [];
+        const results = [];
         await this.db.terms.where('sequence').equals(sequence).each(row => {
-            entry.push({
+            // if (dictionary === row.dictionary) {
+            results.push({
                 expression: row.expression,
                 reading: row.reading,
                 tags: dictFieldSplit(row.tags),
@@ -97,11 +101,12 @@ class Database {
                 glossary: row.glossary,
                 score: row.score,
                 dictionary: row.dictionary,
-                id: row.id
+                id: row.id,
+                sequence: typeof row.sequence === 'undefined' ? -1 : row.sequence
             });
         });
 
-        return entry;
+        return results;
     }
 
     async findTermMeta(term, titles) {
diff --git a/ext/bg/js/dictionary.js b/ext/bg/js/dictionary.js
index f3f573d3..2b289a23 100644
--- a/ext/bg/js/dictionary.js
+++ b/ext/bg/js/dictionary.js
@@ -144,6 +144,77 @@ function dictTermsGroup(definitions, dictionaries) {
     return dictTermsSort(results);
 }
 
+function dictTermsMergeBySequence(definitions) {
+    const definitionsBySequence = {'-1': []};
+    for (const definition of definitions) {
+        if (definition.sequence > 0) {
+            if (!definitionsBySequence[definition.sequence]) {
+                definitionsBySequence[definition.sequence] = {
+                    reasons: definition.reasons,
+                    score: Number.MIN_SAFE_INTEGER,
+                    expression: new Set(),
+                    reading: new Set(),
+                    expressions: new Map(),
+                    source: definition.source,
+                    dictionary: definition.dictionary,
+                    definitions: []
+                };
+            }
+            const score = Math.max(definitionsBySequence[definition.sequence].score, definition.score);
+            definitionsBySequence[definition.sequence].score = score;
+        } else {
+            definitionsBySequence['-1'].push(definition);
+        }
+    }
+
+    return definitionsBySequence;
+}
+
+function dictTermsMergeByGloss(result, definitions) {
+    const definitionsByGloss = {};
+    for (const definition of definitions) {
+
+        const gloss = JSON.stringify(definition.glossary);
+        if (!definitionsByGloss[gloss]) {
+            definitionsByGloss[gloss] = {
+                expression: new Set(),
+                reading: new Set(),
+                tags: new Set(),
+                source: result.source,
+                reasons: [],
+                score: definition.score,
+                id: definition.id,
+                dictionary: definition.dictionary
+            };
+        }
+
+        definitionsByGloss[gloss].expression.add(definition.expression);
+        definitionsByGloss[gloss].reading.add(definition.reading);
+
+        result.expression.add(definition.expression);
+        result.reading.add(definition.reading);
+
+        // result->expressions[ Expression1[ Reading1[ Tag1, Tag2 ] ], Expression2, ... ]
+        if (!result.expressions.has(definition.expression)) {
+            result.expressions.set(definition.expression, new Map());
+        }
+        if (!result.expressions.get(definition.expression).has(definition.reading)) {
+            result.expressions.get(definition.expression).set(definition.reading, new Set());
+        }
+
+        for (const tag of definition.tags) {
+            if (dictIsJmdictTermTag(tag)) {
+                // TODO: expand tags
+                result.expressions.get(definition.expression).get(definition.reading).add(tag);
+            } else {
+                definitionsByGloss[gloss].tags.add(tag);
+            }
+        }
+    }
+
+    return definitionsByGloss;
+}
+
 function dictTagBuildSource(name) {
     return dictTagSanitize({name, category: 'dictionary', order: 100});
 }
@@ -178,6 +249,45 @@ function dictTagsSort(tags) {
     });
 }
 
+function dictIsJmdictTermTag(tag) {
+    return [
+        'P',
+        'news',
+        'ichi',
+        'spec',
+        'gai',
+        'ik',
+        'iK',
+        'ok',
+        'oK',
+        'ek',
+        'eK',
+        'io',
+        'oik',
+        'ateji',
+        'gikun'
+    ].includes(tag);
+}
+
+function dictJmdictTermTagsRare(tags) {
+    const rareTags = [
+        'ik',
+        'iK',
+        'ok',
+        'oK',
+        'ek',
+        'eK',
+        'io',
+        'oik'
+    ];
+    for (const tag of tags) {
+        if (rareTags.includes(tag)) {
+            return true;
+        }
+    }
+    return false;
+}
+
 function dictFieldSplit(field) {
     return field.length === 0 ? [] : field.split(' ');
 }
diff --git a/ext/bg/js/handlebars.js b/ext/bg/js/handlebars.js
index 08304d43..66d5fa2b 100644
--- a/ext/bg/js/handlebars.js
+++ b/ext/bg/js/handlebars.js
@@ -71,16 +71,6 @@ function handlebarsKanjiLinks(options) {
     return result;
 }
 
-function handlebarsExpressions(options) {
-    const definition = options.fn(this);
-    return definition.expression;
-}
-
-function handlebarsReadings(options) {
-    const definition = options.fn(this);
-    return definition.reading;
-}
-
 function handlebarsMultiLine(options) {
     return options.fn(this).split('\n').join('<br>');
 }
@@ -93,8 +83,6 @@ function handlebarsRegisterHelpers() {
         Handlebars.registerHelper('furiganaPlain', handlebarsFuriganaPlain);
         Handlebars.registerHelper('kanjiLinks', handlebarsKanjiLinks);
         Handlebars.registerHelper('multiLine', handlebarsMultiLine);
-        Handlebars.registerHelper('expressions', handlebarsExpressions);
-        Handlebars.registerHelper('readings', handlebarsReadings);
     }
 }
 
diff --git a/ext/bg/js/templates.js b/ext/bg/js/templates.js
index 2cea8b07..b9303ee4 100644
--- a/ext/bg/js/templates.js
+++ b/ext/bg/js/templates.js
@@ -204,15 +204,30 @@ templates['model.html'] = template({"1":function(container,depth0,helpers,partia
 templates['terms.html'] = template({"1":function(container,depth0,helpers,partials,data) {
     var stack1, alias1=depth0 != null ? depth0 : (container.nullContext || {});
 
-  return ((stack1 = helpers["if"].call(alias1,(depth0 != null ? depth0.tags : depth0),{"name":"if","hash":{},"fn":container.program(2, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "")
-    + ((stack1 = helpers["if"].call(alias1,((stack1 = (depth0 != null ? depth0.glossary : depth0)) != null ? stack1["1"] : stack1),{"name":"if","hash":{},"fn":container.program(5, data, 0),"inverse":container.program(9, data, 0),"data":data})) != null ? stack1 : "");
+  return ((stack1 = helpers["if"].call(alias1,(depth0 != null ? depth0.only : depth0),{"name":"if","hash":{},"fn":container.program(2, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "")
+    + ((stack1 = helpers["if"].call(alias1,(depth0 != null ? depth0.tags : depth0),{"name":"if","hash":{},"fn":container.program(6, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "")
+    + ((stack1 = helpers["if"].call(alias1,((stack1 = (depth0 != null ? depth0.glossary : depth0)) != null ? stack1["1"] : stack1),{"name":"if","hash":{},"fn":container.program(9, data, 0),"inverse":container.program(13, data, 0),"data":data})) != null ? stack1 : "");
 },"2":function(container,depth0,helpers,partials,data) {
     var stack1;
 
+  return "<div>\n    ("
+    + ((stack1 = helpers.each.call(depth0 != null ? depth0 : (container.nullContext || {}),(depth0 != null ? depth0.only : depth0),{"name":"each","hash":{},"fn":container.program(3, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "")
+    + "    only)\n</div>\n";
+},"3":function(container,depth0,helpers,partials,data) {
+    var stack1;
+
+  return ((stack1 = container.lambda(depth0, depth0)) != null ? stack1 : "")
+    + ((stack1 = helpers.unless.call(depth0 != null ? depth0 : (container.nullContext || {}),(data && data.last),{"name":"unless","hash":{},"fn":container.program(4, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "")
+    + "\n";
+},"4":function(container,depth0,helpers,partials,data) {
+    return ", ";
+},"6":function(container,depth0,helpers,partials,data) {
+    var stack1;
+
   return "<div>\n"
-    + ((stack1 = helpers.each.call(depth0 != null ? depth0 : (container.nullContext || {}),(depth0 != null ? depth0.tags : depth0),{"name":"each","hash":{},"fn":container.program(3, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "")
+    + ((stack1 = helpers.each.call(depth0 != null ? depth0 : (container.nullContext || {}),(depth0 != null ? depth0.tags : depth0),{"name":"each","hash":{},"fn":container.program(7, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "")
     + "</div>\n";
-},"3":function(container,depth0,helpers,partials,data) {
+},"7":function(container,depth0,helpers,partials,data) {
     var helper, alias1=depth0 != null ? depth0 : (container.nullContext || {}), alias2=helpers.helperMissing, alias3="function", alias4=container.escapeExpression;
 
   return "    <span class=\"label label-default tag-"
@@ -222,112 +237,111 @@ templates['terms.html'] = template({"1":function(container,depth0,helpers,partia
     + "\">"
     + alias4(((helper = (helper = helpers.name || (depth0 != null ? depth0.name : depth0)) != null ? helper : alias2),(typeof helper === alias3 ? helper.call(alias1,{"name":"name","hash":{},"data":data}) : helper)))
     + "</span>\n";
-},"5":function(container,depth0,helpers,partials,data) {
+},"9":function(container,depth0,helpers,partials,data) {
     var stack1;
 
   return "<ul>\n"
-    + ((stack1 = helpers.each.call(depth0 != null ? depth0 : (container.nullContext || {}),(depth0 != null ? depth0.glossary : depth0),{"name":"each","hash":{},"fn":container.program(6, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "")
+    + ((stack1 = helpers.each.call(depth0 != null ? depth0 : (container.nullContext || {}),(depth0 != null ? depth0.glossary : depth0),{"name":"each","hash":{},"fn":container.program(10, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "")
     + "</ul>\n";
-},"6":function(container,depth0,helpers,partials,data) {
+},"10":function(container,depth0,helpers,partials,data) {
     var stack1, helper, options, buffer = 
   "    <li><span class=\"glossary-item\">";
-  stack1 = ((helper = (helper = helpers.multiLine || (depth0 != null ? depth0.multiLine : depth0)) != null ? helper : helpers.helperMissing),(options={"name":"multiLine","hash":{},"fn":container.program(7, data, 0),"inverse":container.noop,"data":data}),(typeof helper === "function" ? helper.call(depth0 != null ? depth0 : (container.nullContext || {}),options) : helper));
+  stack1 = ((helper = (helper = helpers.multiLine || (depth0 != null ? depth0.multiLine : depth0)) != null ? helper : helpers.helperMissing),(options={"name":"multiLine","hash":{},"fn":container.program(11, data, 0),"inverse":container.noop,"data":data}),(typeof helper === "function" ? helper.call(depth0 != null ? depth0 : (container.nullContext || {}),options) : helper));
   if (!helpers.multiLine) { stack1 = helpers.blockHelperMissing.call(depth0,stack1,options)}
   if (stack1 != null) { buffer += stack1; }
   return buffer + "</span></li>\n";
-},"7":function(container,depth0,helpers,partials,data) {
+},"11":function(container,depth0,helpers,partials,data) {
     return container.escapeExpression(container.lambda(depth0, depth0));
-},"9":function(container,depth0,helpers,partials,data) {
+},"13":function(container,depth0,helpers,partials,data) {
     var stack1, helper, options, buffer = 
   "<div class=\"glossary-item\">";
-  stack1 = ((helper = (helper = helpers.multiLine || (depth0 != null ? depth0.multiLine : depth0)) != null ? helper : helpers.helperMissing),(options={"name":"multiLine","hash":{},"fn":container.program(10, data, 0),"inverse":container.noop,"data":data}),(typeof helper === "function" ? helper.call(depth0 != null ? depth0 : (container.nullContext || {}),options) : helper));
+  stack1 = ((helper = (helper = helpers.multiLine || (depth0 != null ? depth0.multiLine : depth0)) != null ? helper : helpers.helperMissing),(options={"name":"multiLine","hash":{},"fn":container.program(14, data, 0),"inverse":container.noop,"data":data}),(typeof helper === "function" ? helper.call(depth0 != null ? depth0 : (container.nullContext || {}),options) : helper));
   if (!helpers.multiLine) { stack1 = helpers.blockHelperMissing.call(depth0,stack1,options)}
   if (stack1 != null) { buffer += stack1; }
   return buffer + "</div>\n";
-},"10":function(container,depth0,helpers,partials,data) {
+},"14":function(container,depth0,helpers,partials,data) {
     var stack1;
 
   return container.escapeExpression(container.lambda(((stack1 = (depth0 != null ? depth0.glossary : depth0)) != null ? stack1["0"] : stack1), depth0));
-},"12":function(container,depth0,helpers,partials,data) {
+},"16":function(container,depth0,helpers,partials,data) {
     var stack1, alias1=depth0 != null ? depth0 : (container.nullContext || {});
 
   return "<div class=\"entry\" data-type=\"term\">\n    <div class=\"actions\">\n"
-    + ((stack1 = helpers["if"].call(alias1,(depth0 != null ? depth0.addable : depth0),{"name":"if","hash":{},"fn":container.program(13, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "")
-    + ((stack1 = helpers["if"].call(alias1,(depth0 != null ? depth0.playback : depth0),{"name":"if","hash":{},"fn":container.program(15, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "")
+    + ((stack1 = helpers["if"].call(alias1,(depth0 != null ? depth0.addable : depth0),{"name":"if","hash":{},"fn":container.program(17, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "")
+    + ((stack1 = helpers["if"].call(alias1,(depth0 != null ? depth0.playback : depth0),{"name":"if","hash":{},"fn":container.program(19, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "")
     + "        <img src=\"/mixed/img/entry-current.png\" class=\"current\" title=\"Current entry (Alt + Up/Down/Home/End/PgUp/PgDn)\" alt>\n    </div>\n\n"
-    + ((stack1 = helpers["if"].call(alias1,(depth0 != null ? depth0.merged : depth0),{"name":"if","hash":{},"fn":container.program(17, data, 0),"inverse":container.program(21, data, 0),"data":data})) != null ? stack1 : "")
+    + ((stack1 = helpers["if"].call(alias1,(depth0 != null ? depth0.merged : depth0),{"name":"if","hash":{},"fn":container.program(21, data, 0),"inverse":container.program(28, data, 0),"data":data})) != null ? stack1 : "")
     + "\n"
-    + ((stack1 = helpers["if"].call(alias1,(depth0 != null ? depth0.reasons : depth0),{"name":"if","hash":{},"fn":container.program(24, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "")
+    + ((stack1 = helpers["if"].call(alias1,(depth0 != null ? depth0.reasons : depth0),{"name":"if","hash":{},"fn":container.program(30, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "")
     + "\n"
-    + ((stack1 = helpers["if"].call(alias1,(depth0 != null ? depth0.frequencies : depth0),{"name":"if","hash":{},"fn":container.program(28, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "")
+    + ((stack1 = helpers["if"].call(alias1,(depth0 != null ? depth0.frequencies : depth0),{"name":"if","hash":{},"fn":container.program(34, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "")
     + "\n    <div class=\"glossary\">\n"
-    + ((stack1 = helpers["if"].call(alias1,(depth0 != null ? depth0.grouped : depth0),{"name":"if","hash":{},"fn":container.program(31, data, 0),"inverse":container.program(37, data, 0),"data":data})) != null ? stack1 : "")
+    + ((stack1 = helpers["if"].call(alias1,(depth0 != null ? depth0.grouped : depth0),{"name":"if","hash":{},"fn":container.program(37, data, 0),"inverse":container.program(43, data, 0),"data":data})) != null ? stack1 : "")
     + "    </div>\n\n"
-    + ((stack1 = helpers["if"].call(alias1,(depth0 != null ? depth0.debug : depth0),{"name":"if","hash":{},"fn":container.program(40, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "")
+    + ((stack1 = helpers["if"].call(alias1,(depth0 != null ? depth0.debug : depth0),{"name":"if","hash":{},"fn":container.program(46, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "")
     + "</div>\n";
-},"13":function(container,depth0,helpers,partials,data) {
+},"17":function(container,depth0,helpers,partials,data) {
     return "        <a href=\"#\" class=\"action-view-note pending disabled\"><img src=\"/mixed/img/view-note.png\" title=\"View added note (Alt + V)\" alt></a>\n        <a href=\"#\" class=\"action-add-note pending disabled\" data-mode=\"term-kanji\"><img src=\"/mixed/img/add-term-kanji.png\" title=\"Add expression (Alt + E)\" alt></a>\n        <a href=\"#\" class=\"action-add-note pending disabled\" data-mode=\"term-kana\"><img src=\"/mixed/img/add-term-kana.png\" title=\"Add reading (Alt + R)\" alt></a>\n";
-},"15":function(container,depth0,helpers,partials,data) {
+},"19":function(container,depth0,helpers,partials,data) {
     return "        <a href=\"#\" class=\"action-play-audio\"><img src=\"/mixed/img/play-audio.png\" title=\"Play audio (Alt + P)\" alt></a>\n";
-},"17":function(container,depth0,helpers,partials,data) {
-    var stack1, helper, options, alias1=depth0 != null ? depth0 : (container.nullContext || {}), alias2=helpers.helperMissing, alias3="function", alias4=helpers.blockHelperMissing, buffer = 
-  "    <div class=\"expression\">";
-  stack1 = ((helper = (helper = helpers.kanjiLinks || (depth0 != null ? depth0.kanjiLinks : depth0)) != null ? helper : alias2),(options={"name":"kanjiLinks","hash":{},"fn":container.program(18, data, 0),"inverse":container.noop,"data":data}),(typeof helper === alias3 ? helper.call(alias1,options) : helper));
-  if (!helpers.kanjiLinks) { stack1 = alias4.call(depth0,stack1,options)}
-  if (stack1 != null) { buffer += stack1; }
-  buffer += "</div>\n    <div>";
-  stack1 = ((helper = (helper = helpers.readings || (depth0 != null ? depth0.readings : depth0)) != null ? helper : alias2),(options={"name":"readings","hash":{},"fn":container.program(19, data, 0),"inverse":container.noop,"data":data}),(typeof helper === alias3 ? helper.call(alias1,options) : helper));
-  if (!helpers.readings) { stack1 = alias4.call(depth0,stack1,options)}
+},"21":function(container,depth0,helpers,partials,data) {
+    var stack1;
+
+  return ((stack1 = helpers.each.call(depth0 != null ? depth0 : (container.nullContext || {}),(depth0 != null ? depth0.expressions : depth0),{"name":"each","hash":{},"fn":container.program(22, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "");
+},"22":function(container,depth0,helpers,partials,data) {
+    var stack1, helper, options, alias1=depth0 != null ? depth0 : (container.nullContext || {}), alias2=helpers.helperMissing, alias3="function", buffer = 
+  "    <div class=\"expression\">\n        <span class=\"expression-"
+    + container.escapeExpression(((helper = (helper = helpers.jmdictTermFrequency || (depth0 != null ? depth0.jmdictTermFrequency : depth0)) != null ? helper : alias2),(typeof helper === alias3 ? helper.call(alias1,{"name":"jmdictTermFrequency","hash":{},"data":data}) : helper)))
+    + "\">";
+  stack1 = ((helper = (helper = helpers.kanjiLinks || (depth0 != null ? depth0.kanjiLinks : depth0)) != null ? helper : alias2),(options={"name":"kanjiLinks","hash":{},"fn":container.program(23, data, 0),"inverse":container.noop,"data":data}),(typeof helper === alias3 ? helper.call(alias1,options) : helper));
+  if (!helpers.kanjiLinks) { stack1 = helpers.blockHelperMissing.call(depth0,stack1,options)}
   if (stack1 != null) { buffer += stack1; }
-  return buffer + "</div>\n";
-},"18":function(container,depth0,helpers,partials,data) {
+  return buffer + "</span>\n        "
+    + ((stack1 = helpers.unless.call(alias1,(data && data.last),{"name":"unless","hash":{},"fn":container.program(26, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "")
+    + "\n    </div>\n";
+},"23":function(container,depth0,helpers,partials,data) {
     var stack1, helper, options;
 
-  stack1 = ((helper = (helper = helpers.expressions || (depth0 != null ? depth0.expressions : depth0)) != null ? helper : helpers.helperMissing),(options={"name":"expressions","hash":{},"fn":container.program(19, data, 0),"inverse":container.noop,"data":data}),(typeof helper === "function" ? helper.call(depth0 != null ? depth0 : (container.nullContext || {}),options) : helper));
-  if (!helpers.expressions) { stack1 = helpers.blockHelperMissing.call(depth0,stack1,options)}
+  stack1 = ((helper = (helper = helpers.furigana || (depth0 != null ? depth0.furigana : depth0)) != null ? helper : helpers.helperMissing),(options={"name":"furigana","hash":{},"fn":container.program(24, data, 0),"inverse":container.noop,"data":data}),(typeof helper === "function" ? helper.call(depth0 != null ? depth0 : (container.nullContext || {}),options) : helper));
+  if (!helpers.furigana) { stack1 = helpers.blockHelperMissing.call(depth0,stack1,options)}
   if (stack1 != null) { return stack1; }
   else { return ''; }
-},"19":function(container,depth0,helpers,partials,data) {
+},"24":function(container,depth0,helpers,partials,data) {
     var stack1;
 
   return ((stack1 = container.lambda(depth0, depth0)) != null ? stack1 : "");
-},"21":function(container,depth0,helpers,partials,data) {
+},"26":function(container,depth0,helpers,partials,data) {
+    return "、";
+},"28":function(container,depth0,helpers,partials,data) {
     var stack1, helper, options, buffer = 
   "    <div class=\"expression\">";
-  stack1 = ((helper = (helper = helpers.kanjiLinks || (depth0 != null ? depth0.kanjiLinks : depth0)) != null ? helper : helpers.helperMissing),(options={"name":"kanjiLinks","hash":{},"fn":container.program(22, data, 0),"inverse":container.noop,"data":data}),(typeof helper === "function" ? helper.call(depth0 != null ? depth0 : (container.nullContext || {}),options) : helper));
+  stack1 = ((helper = (helper = helpers.kanjiLinks || (depth0 != null ? depth0.kanjiLinks : depth0)) != null ? helper : helpers.helperMissing),(options={"name":"kanjiLinks","hash":{},"fn":container.program(23, data, 0),"inverse":container.noop,"data":data}),(typeof helper === "function" ? helper.call(depth0 != null ? depth0 : (container.nullContext || {}),options) : helper));
   if (!helpers.kanjiLinks) { stack1 = helpers.blockHelperMissing.call(depth0,stack1,options)}
   if (stack1 != null) { buffer += stack1; }
   return buffer + "</div>\n";
-},"22":function(container,depth0,helpers,partials,data) {
-    var stack1, helper, options;
-
-  stack1 = ((helper = (helper = helpers.furigana || (depth0 != null ? depth0.furigana : depth0)) != null ? helper : helpers.helperMissing),(options={"name":"furigana","hash":{},"fn":container.program(19, data, 0),"inverse":container.noop,"data":data}),(typeof helper === "function" ? helper.call(depth0 != null ? depth0 : (container.nullContext || {}),options) : helper));
-  if (!helpers.furigana) { stack1 = helpers.blockHelperMissing.call(depth0,stack1,options)}
-  if (stack1 != null) { return stack1; }
-  else { return ''; }
-},"24":function(container,depth0,helpers,partials,data) {
+},"30":function(container,depth0,helpers,partials,data) {
     var stack1;
 
   return "    <div class=\"reasons\">\n"
-    + ((stack1 = helpers.each.call(depth0 != null ? depth0 : (container.nullContext || {}),(depth0 != null ? depth0.reasons : depth0),{"name":"each","hash":{},"fn":container.program(25, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "")
+    + ((stack1 = helpers.each.call(depth0 != null ? depth0 : (container.nullContext || {}),(depth0 != null ? depth0.reasons : depth0),{"name":"each","hash":{},"fn":container.program(31, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "")
     + "    </div>\n";
-},"25":function(container,depth0,helpers,partials,data) {
+},"31":function(container,depth0,helpers,partials,data) {
     var stack1;
 
   return "        <span class=\"reasons\">"
     + container.escapeExpression(container.lambda(depth0, depth0))
     + "</span> "
-    + ((stack1 = helpers.unless.call(depth0 != null ? depth0 : (container.nullContext || {}),(data && data.last),{"name":"unless","hash":{},"fn":container.program(26, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "")
+    + ((stack1 = helpers.unless.call(depth0 != null ? depth0 : (container.nullContext || {}),(data && data.last),{"name":"unless","hash":{},"fn":container.program(32, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "")
     + "\n";
-},"26":function(container,depth0,helpers,partials,data) {
+},"32":function(container,depth0,helpers,partials,data) {
     return "&laquo;";
-},"28":function(container,depth0,helpers,partials,data) {
+},"34":function(container,depth0,helpers,partials,data) {
     var stack1;
 
   return "    <div>\n"
-    + ((stack1 = helpers.each.call(depth0 != null ? depth0 : (container.nullContext || {}),(depth0 != null ? depth0.frequencies : depth0),{"name":"each","hash":{},"fn":container.program(29, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "")
+    + ((stack1 = helpers.each.call(depth0 != null ? depth0 : (container.nullContext || {}),(depth0 != null ? depth0.frequencies : depth0),{"name":"each","hash":{},"fn":container.program(35, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "")
     + "    </div>\n";
-},"29":function(container,depth0,helpers,partials,data) {
+},"35":function(container,depth0,helpers,partials,data) {
     var helper, alias1=depth0 != null ? depth0 : (container.nullContext || {}), alias2=helpers.helperMissing, alias3="function", alias4=container.escapeExpression;
 
   return "        <span class=\"label label-default tag-frequency\">"
@@ -335,67 +349,67 @@ templates['terms.html'] = template({"1":function(container,depth0,helpers,partia
     + ":"
     + alias4(((helper = (helper = helpers.frequency || (depth0 != null ? depth0.frequency : depth0)) != null ? helper : alias2),(typeof helper === alias3 ? helper.call(alias1,{"name":"frequency","hash":{},"data":data}) : helper)))
     + "</span>\n";
-},"31":function(container,depth0,helpers,partials,data) {
+},"37":function(container,depth0,helpers,partials,data) {
     var stack1;
 
-  return ((stack1 = helpers["if"].call(depth0 != null ? depth0 : (container.nullContext || {}),((stack1 = (depth0 != null ? depth0.definitions : depth0)) != null ? stack1["1"] : stack1),{"name":"if","hash":{},"fn":container.program(32, data, 0),"inverse":container.program(35, data, 0),"data":data})) != null ? stack1 : "");
-},"32":function(container,depth0,helpers,partials,data) {
+  return ((stack1 = helpers["if"].call(depth0 != null ? depth0 : (container.nullContext || {}),((stack1 = (depth0 != null ? depth0.definitions : depth0)) != null ? stack1["1"] : stack1),{"name":"if","hash":{},"fn":container.program(38, data, 0),"inverse":container.program(41, data, 0),"data":data})) != null ? stack1 : "");
+},"38":function(container,depth0,helpers,partials,data) {
     var stack1;
 
   return "        <ol>\n"
-    + ((stack1 = helpers.each.call(depth0 != null ? depth0 : (container.nullContext || {}),(depth0 != null ? depth0.definitions : depth0),{"name":"each","hash":{},"fn":container.program(33, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "")
+    + ((stack1 = helpers.each.call(depth0 != null ? depth0 : (container.nullContext || {}),(depth0 != null ? depth0.definitions : depth0),{"name":"each","hash":{},"fn":container.program(39, data, 0),"inverse":container.noop,"data":data})) != null ? stack1 : "")
     + "        </ol>\n";
-},"33":function(container,depth0,helpers,partials,data) {
+},"39":function(container,depth0,helpers,partials,data) {
     var stack1;
 
   return "            <li>"
     + ((stack1 = container.invokePartial(partials.definition,depth0,{"name":"definition","data":data,"helpers":helpers,"partials":partials,"decorators":container.decorators})) != null ? stack1 : "")
     + "</li>\n";
-},"35":function(container,depth0,helpers,partials,data) {
+},"41":function(container,depth0,helpers,partials,data) {
     var stack1;
 
   return ((stack1 = container.invokePartial(partials.definition,((stack1 = (depth0 != null ? depth0.definitions : depth0)) != null ? stack1["0"] : stack1),{"name":"definition","data":data,"indent":"        ","helpers":helpers,"partials":partials,"decorators":container.decorators})) != null ? stack1 : "");
-},"37":function(container,depth0,helpers,partials,data) {
+},"43":function(container,depth0,helpers,partials,data) {
     var stack1;
 
-  return ((stack1 = helpers["if"].call(depth0 != null ? depth0 : (container.nullContext || {}),(depth0 != null ? depth0.merged : depth0),{"name":"if","hash":{},"fn":container.program(31, data, 0),"inverse":container.program(38, data, 0),"data":data})) != null ? stack1 : "");
-},"38":function(container,depth0,helpers,partials,data) {
+  return ((stack1 = helpers["if"].call(depth0 != null ? depth0 : (container.nullContext || {}),(depth0 != null ? depth0.merged : depth0),{"name":"if","hash":{},"fn":container.program(37, data, 0),"inverse":container.program(44, data, 0),"data":data})) != null ? stack1 : "");
+},"44":function(container,depth0,helpers,partials,data) {
     var stack1;
 
   return ((stack1 = container.invokePartial(partials.definition,depth0,{"name":"definition","data":data,"indent":"        ","helpers":helpers,"partials":partials,"decorators":container.decorators})) != null ? stack1 : "")
     + "        ";
-},"40":function(container,depth0,helpers,partials,data) {
+},"46":function(container,depth0,helpers,partials,data) {
     var stack1, helper, options, buffer = 
   "    <pre>";
-  stack1 = ((helper = (helper = helpers.dumpObject || (depth0 != null ? depth0.dumpObject : depth0)) != null ? helper : helpers.helperMissing),(options={"name":"dumpObject","hash":{},"fn":container.program(19, data, 0),"inverse":container.noop,"data":data}),(typeof helper === "function" ? helper.call(depth0 != null ? depth0 : (container.nullContext || {}),options) : helper));
+  stack1 = ((helper = (helper = helpers.dumpObject || (depth0 != null ? depth0.dumpObject : depth0)) != null ? helper : helpers.helperMissing),(options={"name":"dumpObject","hash":{},"fn":container.program(24, data, 0),"inverse":container.noop,"data":data}),(typeof helper === "function" ? helper.call(depth0 != null ? depth0 : (container.nullContext || {}),options) : helper));
   if (!helpers.dumpObject) { stack1 = helpers.blockHelperMissing.call(depth0,stack1,options)}
   if (stack1 != null) { buffer += stack1; }
   return buffer + "</pre>\n";
-},"42":function(container,depth0,helpers,partials,data,blockParams,depths) {
+},"48":function(container,depth0,helpers,partials,data,blockParams,depths) {
     var stack1;
 
-  return ((stack1 = helpers.each.call(depth0 != null ? depth0 : (container.nullContext || {}),(depth0 != null ? depth0.definitions : depth0),{"name":"each","hash":{},"fn":container.program(43, data, 0, blockParams, depths),"inverse":container.noop,"data":data})) != null ? stack1 : "");
-},"43":function(container,depth0,helpers,partials,data,blockParams,depths) {
+  return ((stack1 = helpers.each.call(depth0 != null ? depth0 : (container.nullContext || {}),(depth0 != null ? depth0.definitions : depth0),{"name":"each","hash":{},"fn":container.program(49, data, 0, blockParams, depths),"inverse":container.noop,"data":data})) != null ? stack1 : "");
+},"49":function(container,depth0,helpers,partials,data,blockParams,depths) {
     var stack1;
 
-  return ((stack1 = helpers.unless.call(depth0 != null ? depth0 : (container.nullContext || {}),(data && data.first),{"name":"unless","hash":{},"fn":container.program(44, data, 0, blockParams, depths),"inverse":container.noop,"data":data})) != null ? stack1 : "")
+  return ((stack1 = helpers.unless.call(depth0 != null ? depth0 : (container.nullContext || {}),(data && data.first),{"name":"unless","hash":{},"fn":container.program(50, data, 0, blockParams, depths),"inverse":container.noop,"data":data})) != null ? stack1 : "")
     + "\n"
     + ((stack1 = container.invokePartial(partials.term,depth0,{"name":"term","hash":{"playback":(depths[1] != null ? depths[1].playback : depths[1]),"addable":(depths[1] != null ? depths[1].addable : depths[1]),"merged":(depths[1] != null ? depths[1].merged : depths[1]),"grouped":(depths[1] != null ? depths[1].grouped : depths[1]),"debug":(depths[1] != null ? depths[1].debug : depths[1])},"data":data,"helpers":helpers,"partials":partials,"decorators":container.decorators})) != null ? stack1 : "");
-},"44":function(container,depth0,helpers,partials,data) {
+},"50":function(container,depth0,helpers,partials,data) {
     return "<hr>";
-},"46":function(container,depth0,helpers,partials,data) {
+},"52":function(container,depth0,helpers,partials,data) {
     return "<p class=\"note\">No results found.</p>\n";
 },"compiler":[7,">= 4.0.0"],"main":function(container,depth0,helpers,partials,data,blockParams,depths) {
     var stack1;
 
   return "\n\n"
-    + ((stack1 = helpers["if"].call(depth0 != null ? depth0 : (container.nullContext || {}),(depth0 != null ? depth0.definitions : depth0),{"name":"if","hash":{},"fn":container.program(42, data, 0, blockParams, depths),"inverse":container.program(46, data, 0, blockParams, depths),"data":data})) != null ? stack1 : "");
+    + ((stack1 = helpers["if"].call(depth0 != null ? depth0 : (container.nullContext || {}),(depth0 != null ? depth0.definitions : depth0),{"name":"if","hash":{},"fn":container.program(48, data, 0, blockParams, depths),"inverse":container.program(52, data, 0, blockParams, depths),"data":data})) != null ? stack1 : "");
 },"main_d":  function(fn, props, container, depth0, data, blockParams, depths) {
 
   var decorators = container.decorators;
 
   fn = decorators.inline(fn,props,container,{"name":"inline","hash":{},"fn":container.program(1, data, 0, blockParams, depths),"inverse":container.noop,"args":["definition"],"data":data}) || fn;
-  fn = decorators.inline(fn,props,container,{"name":"inline","hash":{},"fn":container.program(12, data, 0, blockParams, depths),"inverse":container.noop,"args":["term"],"data":data}) || fn;
+  fn = decorators.inline(fn,props,container,{"name":"inline","hash":{},"fn":container.program(16, data, 0, blockParams, depths),"inverse":container.noop,"args":["term"],"data":data}) || fn;
   return fn;
   }
 
diff --git a/ext/bg/js/translator.js b/ext/bg/js/translator.js
index db287d63..81253374 100644
--- a/ext/bg/js/translator.js
+++ b/ext/bg/js/translator.js
@@ -49,77 +49,79 @@ class Translator {
     }
 
     async findTermsMerged(text, dictionaries, alphanumeric) {
-        // const titles = Object.keys(dictionaries);
+        const titles = Object.keys(dictionaries);
         const {length, definitions} = await this.findTerms(text, dictionaries, alphanumeric);
 
-        // const definitionsMerged = dictTermsMerge(definitions, dictionaries, this.database);
-        // for (const definition of definitionsMerged) {
-        //     await this.buildTermFrequencies(definition, titles);
-        // }
+        const definitionsBySequence = dictTermsMergeBySequence(definitions);
 
-        const sequences = {};
-        const stray = [];
-        for (const definition of definitions) {
-            if (typeof definition.sequence !== 'undefined') {
-                if (!sequences[definition.sequence]) {
-                    sequences[definition.sequence] = {
-                        reasons: definition.reasons,
-                        score: Number.MIN_SAFE_INTEGER,
-                        expression: new Set(),
-                        reading: new Set(),
-                        source: definition.source,
-                        definitions: []
-                    };
-                }
-                const seq = sequences[definition.sequence];
-                seq.score = Math.max(seq.score, definition.score);
-            } else {
-                stray.push(definition);
+        const definitionsMerged = dictTermsGroup(definitionsBySequence['-1'], dictionaries);
+        for (const sequence in definitionsBySequence) {
+            if (!(sequence > 0)) {
+                continue;
             }
-        }
 
-        const definitionsMerged = dictTermsGroup(stray, dictionaries);
-        for (const sequence in sequences) {
-            const entry = await this.database.findEntry(Number(sequence));
-
-            const result = sequences[sequence];
-            const glossaries = new Map();
-            for (const definition of entry) {
-
-                const gloss = definition.glossary.join('||');
-                if (!glossaries.get(gloss)) {
-                    const tags = await this.expandTags(definition.tags, definition.dictionary);
-                    tags.push(dictTagBuildSource(definition.dictionary));
-                    glossaries.set(gloss, {
-                        expressions: new Set(),
-                        readings: new Set(),
-                        tags: dictTagsSort(tags), // TODO: use correct tags
-                        source: result.source,
-                        reasons: [],
-                        score: definition.score,
-                        id: definition.id,
-                        dictionary: definition.dictionary
-                    });
+            const result = definitionsBySequence[sequence];
+
+            const rawDefinitionsBySequence = await this.database.findTermsBySequence(Number(sequence));
+            const definitionsByGloss = dictTermsMergeByGloss(result, rawDefinitionsBySequence);
+
+            // postprocess glossaries
+            for (const gloss in definitionsByGloss) {
+                const definition = definitionsByGloss[gloss];
+                definition.glossary = JSON.parse(gloss);
+
+                const tags = await this.expandTags(definition.tags, definition.dictionary);
+                tags.push(dictTagBuildSource(definition.dictionary));
+                definition.tags = dictTagsSort(tags);
+
+                definition.only = [];
+                if (!utilSetEqual(definition.expression, result.expression)) {
+                    for (const expression of utilSetIntersection(definition.expression, result.expression)) {
+                        definition.only.push(expression);
+                    }
+                }
+                if (!utilSetEqual(definition.reading, result.reading)) {
+                    for (const reading of utilSetIntersection(definition.reading, result.reading)) {
+                        definition.only.push(reading);
+                    }
                 }
-                glossaries.get(gloss).expressions.add(definition.expression);
-                glossaries.get(gloss).readings.add(definition.reading);
 
-                result.expression.add(definition.expression);
-                result.reading.add(definition.reading);
+                result.definitions.push(definition);
             }
 
-            for (const gloss of glossaries.keys()) {
-                const definition = glossaries.get(gloss);
-                definition.glossary = gloss.split('||');
-                result.definitions.push(definition);
+            result.definitions.sort(definition => -definition.id);
+
+            // turn the Map()/Set() mess to [{expression: E1, reading: R1}, {...}] and tag popular/normal/rare instead of actual tags
+            const expressions = [];
+            for (const expression of result.expressions.keys()) {
+                for (const reading of result.expressions.get(expression).keys()) {
+                    expressions.push({
+                        expression: expression,
+                        reading: reading,
+                        jmdictTermFrequency: (tags => {
+                            if (tags.has('P')) {
+                                return 'popular';
+                            } else if (dictJmdictTermTagsRare(tags)) {
+                                return 'rare';
+                            } else {
+                                return 'normal';
+                            }
+                        })(result.expressions.get(expression).get(reading))
+                    });
+                }
             }
-            //dictTermsSort(groupDefs, dictionaries)
+
+            result.expressions = expressions;
 
             result.expression = Array.from(result.expression).join(', ');
             result.reading = Array.from(result.reading).join(', ');
             definitionsMerged.push(result);
         }
 
+        for (const definition of definitionsMerged) {
+            await this.buildTermFrequencies(definition, titles);
+        }
+
         return {length, definitions: dictTermsSort(definitionsMerged)};
     }
 
@@ -234,14 +236,23 @@ class Translator {
     }
 
     async buildTermFrequencies(definition, titles) {
-        definition.frequencies = [];
-        for (const meta of await this.database.findTermMeta(definition.expression, titles)) {
-            if (meta.mode === 'freq') {
-                definition.frequencies.push({
-                    expression: meta.expression,
-                    frequency: meta.data,
-                    dictionary: meta.dictionary
-                });
+        let terms = [];
+        if (definition.expressions) {
+            terms = terms.concat(definition.expressions);
+        } else {
+            terms.push(definition);
+        }
+
+        for (const term of terms) {
+            term.frequencies = [];
+            for (const meta of await this.database.findTermMeta(term.expression, titles)) {
+                if (meta.mode === 'freq') {
+                    term.frequencies.push({
+                        expression: meta.expression,
+                        frequency: meta.data,
+                        dictionary: meta.dictionary
+                    });
+                }
             }
         }
     }
diff --git a/ext/bg/js/util.js b/ext/bg/js/util.js
index f44582eb..9a3a2a3e 100644
--- a/ext/bg/js/util.js
+++ b/ext/bg/js/util.js
@@ -26,6 +26,32 @@ function utilIsolate(data) {
     return JSON.parse(JSON.stringify(data));
 }
 
+function utilSetEqual(setA, setB) {
+    if (setA.size !== setB.size) {
+        return false;
+    }
+
+    for (const value of setA) {
+        if (!setB.has(value)) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+function utilSetIntersection(setA, setB) {
+    return new Set(
+        [...setA].filter(value => setB.has(value))
+    );
+}
+
+function utilSetDifference(setA, setB) {
+    return new Set(
+        [...setA].filter(value => !setB.has(value))
+    );
+}
+
 function utilBackend() {
     return chrome.extension.getBackgroundPage().yomichan_backend;
 }