Anki support for structured-content (#1786)

* Update how glossary text is formatted

* Update structured content and image generation

* Pass root data to _createStructuredContentGenerator

* Implement media URLs

* Update documentation

* Update options util

* Apply styles to content

* Improve HTML normalization

* Update DatabaseVM.fetch function

* Update test

* Update test data

7 files changed, 173 insertions, 15 deletions

diff --git a/ext/data/templates/anki-field-templates-upgrade-v13.handlebars b/ext/data/templates/anki-field-templates-upgrade-v13.handlebars
new file mode 100644
index 00000000..04cc855a
--- /dev/null
+++ b/ext/data/templates/anki-field-templates-upgrade-v13.handlebars
@@ -0,0 +1,17 @@
+{{<<<<<<<}}
+{{#each glossary}}{{#multiLine}}{{.}}{{/multiLine}}{{/each}}
+{{=======}}
+{{#each glossary}}{{#formatGlossary ../dictionary}}{{{.}}}{{/formatGlossary}}{{/each}}
+{{>>>>>>>}}
+
+{{<<<<<<<}}
+{{#each glossary}}{{#multiLine}}{{.}}{{/multiLine}}{{#unless @last}} | {{/unless}}{{/each}}
+{{=======}}
+{{#each glossary}}{{#formatGlossary ../dictionary}}{{{.}}}{{/formatGlossary}}{{#unless @last}} | {{/unless}}{{/each}}
+{{>>>>>>>}}
+
+{{<<<<<<<}}
+{{#each glossary}}<li>{{#multiLine}}{{.}}{{/multiLine}}</li>{{/each}}
+{{=======}}
+{{#each glossary}}<li>{{#formatGlossary ../dictionary}}{{{.}}}{{/formatGlossary}}</li>{{/each}}
+{{>>>>>>>}}
diff --git a/ext/data/templates/default-anki-field-templates.handlebars b/ext/data/templates/default-anki-field-templates.handlebars
index 97359aa0..67547732 100644
--- a/ext/data/templates/default-anki-field-templates.handlebars
+++ b/ext/data/templates/default-anki-field-templates.handlebars
@@ -21,11 +21,11 @@
         {{~#if only~}}({{#each only}}{{.}}{{#unless @last}}, {{/unless}}{{/each}} only) {{/if~}}
     {{~/unless~}}
     {{~#if (op "<=" glossary.length 1)~}}
-        {{#each glossary}}{{#multiLine}}{{.}}{{/multiLine}}{{/each}}
+        {{#each glossary}}{{#formatGlossary ../dictionary}}{{{.}}}{{/formatGlossary}}{{/each}}
     {{~else if @root.compactGlossaries~}}
-        {{#each glossary}}{{#multiLine}}{{.}}{{/multiLine}}{{#unless @last}} | {{/unless}}{{/each}}
+        {{#each glossary}}{{#formatGlossary ../dictionary}}{{{.}}}{{/formatGlossary}}{{#unless @last}} | {{/unless}}{{/each}}
     {{~else~}}
-        <ul>{{#each glossary}}<li>{{#multiLine}}{{.}}{{/multiLine}}</li>{{/each}}</ul>
+        <ul>{{#each glossary}}<li>{{#formatGlossary ../dictionary}}{{{.}}}{{/formatGlossary}}</li>{{/each}}</ul>
     {{~/if~}}
     {{~#set "previousDictionary" dictionary~}}{{~/set~}}
 {{/inline}}
diff --git a/ext/js/data/options-util.js b/ext/js/data/options-util.js
index eb29dae4..42175d35 100644
--- a/ext/js/data/options-util.js
+++ b/ext/js/data/options-util.js
@@ -461,7 +461,8 @@ class OptionsUtil {
             {async: false, update: this._updateVersion9.bind(this)},
             {async: true,  update: this._updateVersion10.bind(this)},
             {async: false, update: this._updateVersion11.bind(this)},
-            {async: true,  update: this._updateVersion12.bind(this)}
+            {async: true,  update: this._updateVersion12.bind(this)},
+            {async: true,  update: this._updateVersion13.bind(this)}
         ];
     }
 
@@ -844,4 +845,11 @@ class OptionsUtil {
         }
         return options;
     }
+
+    async _updateVersion13(options) {
+        // Version 13 changes:
+        //  Handlebars templates updated to use formatGlossary.
+        await this._applyAnkiFieldTemplatesPatch(options, '/data/templates/anki-field-templates-upgrade-v13.handlebars');
+        return options;
+    }
 }
diff --git a/ext/js/dom/css-style-applier.js b/ext/js/dom/css-style-applier.js
index 593e7a46..c617fead 100644
--- a/ext/js/dom/css-style-applier.js
+++ b/ext/js/dom/css-style-applier.js
@@ -72,12 +72,6 @@ class CssStyleApplier {
                 element.removeAttribute('style');
             }
         }
-        for (const element of elements) {
-            const {dataset} = element;
-            for (const key of Object.keys(dataset)) {
-                delete dataset[key];
-            }
-        }
     }
 
     // Private
diff --git a/ext/js/templates/template-renderer-frame-main.js b/ext/js/templates/template-renderer-frame-main.js
index 3d61295d..bb9cac3a 100644
--- a/ext/js/templates/template-renderer-frame-main.js
+++ b/ext/js/templates/template-renderer-frame-main.js
@@ -17,14 +17,17 @@
 
 /* globals
  * AnkiNoteDataCreator
+ * CssStyleApplier
  * JapaneseUtil
  * TemplateRenderer
  * TemplateRendererFrameApi
  */
 
-(() => {
+(async () => {
+    const cssStyleApplier = new CssStyleApplier('/data/structured-content-style.json');
+    await cssStyleApplier.prepare();
     const japaneseUtil = new JapaneseUtil(null);
-    const templateRenderer = new TemplateRenderer(japaneseUtil);
+    const templateRenderer = new TemplateRenderer(japaneseUtil, cssStyleApplier);
     const ankiNoteDataCreator = new AnkiNoteDataCreator(japaneseUtil);
     templateRenderer.registerDataType('ankiNote', {
         modifier: ({marker, commonData}) => ankiNoteDataCreator.create(marker, commonData),
diff --git a/ext/js/templates/template-renderer.js b/ext/js/templates/template-renderer.js
index a06298aa..928ec3c4 100644
--- a/ext/js/templates/template-renderer.js
+++ b/ext/js/templates/template-renderer.js
@@ -18,11 +18,13 @@
 /* global
  * DictionaryDataUtil
  * Handlebars
+ * StructuredContentGenerator
  */
 
 class TemplateRenderer {
-    constructor(japaneseUtil) {
+    constructor(japaneseUtil, cssStyleApplier) {
         this._japaneseUtil = japaneseUtil;
+        this._cssStyleApplier = cssStyleApplier;
         this._cache = new Map();
         this._cacheMaxSize = 5;
         this._helpersRegistered = false;
@@ -31,6 +33,7 @@ class TemplateRenderer {
         this._requirements = null;
         this._cleanupCallbacks = null;
         this._customData = null;
+        this._temporaryElement = null;
     }
 
     registerDataType(name, {modifier=null, composeData=null}) {
@@ -161,7 +164,8 @@ class TemplateRenderer {
             ['typeof',           this._getTypeof.bind(this)],
             ['join',             this._join.bind(this)],
             ['concat',           this._concat.bind(this)],
-            ['pitchCategories',  this._pitchCategories.bind(this)]
+            ['pitchCategories',  this._pitchCategories.bind(this)],
+            ['formatGlossary',   this._formatGlossary.bind(this)]
         ];
 
         for (const [name, helper] of helpers) {
@@ -244,8 +248,12 @@ class TemplateRenderer {
         return result;
     }
 
+    _stringToMultiLineHtml(string) {
+        return string.split('\n').join('<br>');
+    }
+
     _multiLine(context, options) {
-        return options.fn(context).split('\n').join('<br>');
+        return this._stringToMultiLineHtml(options.fn(context));
     }
 
     _sanitizeCssClass(context, options) {
@@ -497,4 +505,130 @@ class TemplateRenderer {
         }
         return [...categories];
     }
+
+    _getTemporaryElement() {
+        let element = this._temporaryElement;
+        if (element === null) {
+            element = document.createElement('div');
+            this._temporaryElement = element;
+        }
+        return element;
+    }
+
+    _getHtml(node) {
+        const container = this._getTemporaryElement();
+        container.appendChild(node);
+        this._normalizeHtml(container);
+        const result = container.innerHTML;
+        container.textContent = '';
+        return result;
+    }
+
+    _normalizeHtml(root) {
+        const {ELEMENT_NODE, TEXT_NODE} = Node;
+        const treeWalker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT | NodeFilter.SHOW_TEXT);
+        const elements = [];
+        const textNodes = [];
+        while (true) {
+            const node = treeWalker.nextNode();
+            if (node === null) { break; }
+            switch (node.nodeType) {
+                case ELEMENT_NODE:
+                    elements.push(node);
+                    break;
+                case TEXT_NODE:
+                    textNodes.push(node);
+                    break;
+            }
+        }
+        this._cssStyleApplier.applyClassStyles(elements);
+        for (const element of elements) {
+            const {dataset} = element;
+            for (const key of Object.keys(dataset)) {
+                delete dataset[key];
+            }
+        }
+        for (const textNode of textNodes) {
+            this._replaceNewlines(textNode);
+        }
+    }
+
+    _replaceNewlines(textNode) {
+        const parts = textNode.nodeValue.split('\n');
+        if (parts.length <= 1) { return; }
+        const {parentNode} = textNode;
+        if (parentNode === null) { return; }
+        const fragment = document.createDocumentFragment();
+        for (let i = 0, ii = parts.length; i < ii; ++i) {
+            if (i > 0) { fragment.appendChild(document.createElement('br')); }
+            fragment.appendChild(document.createTextNode(parts[i]));
+        }
+        parentNode.replaceChild(fragment, textNode);
+    }
+
+    _getDictionaryMedia(data, dictionary, path) {
+        const {media} = data;
+        if (typeof media === 'object' && media !== null && Object.prototype.hasOwnProperty.call(media, 'dictionaryMedia')) {
+            const {dictionaryMedia} = media;
+            if (typeof dictionaryMedia === 'object' && dictionaryMedia !== null && Object.prototype.hasOwnProperty.call(dictionaryMedia, dictionary)) {
+                const dictionaryMedia2 = dictionaryMedia[dictionary];
+                if (Object.prototype.hasOwnProperty.call(dictionaryMedia2, path)) {
+                    return dictionaryMedia2[path];
+                }
+            }
+        }
+        return null;
+    }
+
+    _createStructuredContentGenerator(data) {
+        const mediaLoader = {
+            loadMedia: async (path, dictionary, onLoad, onUnload) => {
+                const imageUrl = this._getDictionaryMedia(data, dictionary, path);
+                if (imageUrl !== null) {
+                    onLoad(imageUrl);
+                    this._cleanupCallbacks.push(() => onUnload(true));
+                } else {
+                    let set = this._customData.requiredDictionaryMedia;
+                    if (typeof set === 'undefined') {
+                        set = new Set();
+                        this._customData.requiredDictionaryMedia = set;
+                    }
+                    const key = JSON.stringify([dictionary, path]);
+                    if (!set.has(key)) {
+                        set.add(key);
+                        this._requirements.push({
+                            type: 'dictionaryMedia',
+                            dictionary,
+                            path
+                        });
+                    }
+                }
+            }
+        };
+        return new StructuredContentGenerator(mediaLoader, document);
+    }
+
+    _formatGlossary(context, dictionary, options) {
+        const data = options.data.root;
+        const content = options.fn(context);
+        if (typeof content === 'string') { return this._stringToMultiLineHtml(content); }
+        if (!(typeof content === 'object' && content !== null)) { return ''; }
+        switch (content.type) {
+            case 'image': return this._formatGlossaryImage(content, dictionary, data);
+            case 'structured-content': return this._formatStructuredContent(content, dictionary, data);
+        }
+        return '';
+    }
+
+    _formatGlossaryImage(content, dictionary, data) {
+        const structuredContentGenerator = this._createStructuredContentGenerator(data);
+        const node = structuredContentGenerator.createDefinitionImage(content, dictionary);
+        return this._getHtml(node);
+    }
+
+    _formatStructuredContent(content, dictionary, data) {
+        const structuredContentGenerator = this._createStructuredContentGenerator(data);
+        const node = structuredContentGenerator.createStructuredContent(content.content, dictionary);
+        return node !== null ? this._getHtml(node) : '';
+    }
 }
diff --git a/ext/template-renderer.html b/ext/template-renderer.html
index f9667acd..f01b5b9a 100644
--- a/ext/template-renderer.html
+++ b/ext/template-renderer.html
@@ -18,6 +18,8 @@
 <script src="/lib/handlebars.min.js"></script>
 
 <script src="/js/data/anki-note-data-creator.js"></script>
+<script src="/js/display/structured-content-generator.js"></script>
+<script src="/js/dom/css-style-applier.js"></script>
 <script src="/js/language/dictionary-data-util.js"></script>
 <script src="/js/language/japanese-util.js"></script>
 <script src="/js/templates/template-renderer.js"></script>