From ca97e38bd22875e23cfe6f70d1803ea31d6f0e89 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Fri, 2 Jul 2021 22:46:38 -0400 Subject: Anki support for structured-content (#1786) * Update how glossary text is formatted * Update structured content and image generation * Pass root data to _createStructuredContentGenerator * Implement media URLs * Update documentation * Update options util * Apply styles to content * Improve HTML normalization * Update DatabaseVM.fetch function * Update test * Update test data --- dev/database-vm.js | 8 +- docs/templates.md | 29 +++++ .../anki-field-templates-upgrade-v13.handlebars | 17 +++ .../default-anki-field-templates.handlebars | 6 +- ext/js/data/options-util.js | 10 +- ext/js/dom/css-style-applier.js | 6 - ext/js/templates/template-renderer-frame-main.js | 7 +- ext/js/templates/template-renderer.js | 140 ++++++++++++++++++++- ext/template-renderer.html | 2 + test/data/anki-note-builder-test-results.json | 12 +- test/test-anki-note-builder.js | 21 +++- test/test-options-util.js | 32 ++++- 12 files changed, 257 insertions(+), 33 deletions(-) create mode 100644 ext/data/templates/anki-field-templates-upgrade-v13.handlebars diff --git a/dev/database-vm.js b/dev/database-vm.js index 014c989f..0127bc8f 100644 --- a/dev/database-vm.js +++ b/dev/database-vm.js @@ -84,7 +84,13 @@ class Blob { } async function fetch(url2) { - const filePath = url.fileURLToPath(url2); + const extDir = path.join(__dirname, '..', 'ext'); + let filePath; + try { + filePath = url.fileURLToPath(url2); + } catch (e) { + filePath = path.resolve(extDir, url2.replace(/^[/\\]/, '')); + } await Promise.resolve(); const content = fs.readFileSync(filePath, {encoding: null}); return { diff --git a/docs/templates.md b/docs/templates.md index 81259a3f..95155bea 100644 --- a/docs/templates.md +++ b/docs/templates.md @@ -645,6 +645,35 @@ Returns an array representing the different pitch categories for a specific term +### `formatGlossary` + +Formats a glossary entry to a HTML content string. This helper handles image and +structured-content generation. + +
+ Syntax: + + {{#formatGlossary dictionary}}{{{definitionEntry}}}{{/pitchCategories}}
+ + * _`@dictionary`_
+ The dictionary that the glossary entry belongs to. + * _`@definitionEntry`_
+ The definition entry object in raw form. +
+
+ Example: + + ```handlebars + {{#each glossary}}{{#formatGlossary ../dictionary}}{{{.}}}{{/formatGlossary}}{{/each}} + ``` + + Output: + ```html + Here is the content of a gloss, which may include formatted HTML. + ``` +
+ + ## Legacy Helpers Yomichan has historically used Handlebars templates to generate the HTML used on the search page and results popup. diff --git a/ext/data/templates/anki-field-templates-upgrade-v13.handlebars b/ext/data/templates/anki-field-templates-upgrade-v13.handlebars new file mode 100644 index 00000000..04cc855a --- /dev/null +++ b/ext/data/templates/anki-field-templates-upgrade-v13.handlebars @@ -0,0 +1,17 @@ +{{<<<<<<<}} +{{#each glossary}}{{#multiLine}}{{.}}{{/multiLine}}{{/each}} +{{=======}} +{{#each glossary}}{{#formatGlossary ../dictionary}}{{{.}}}{{/formatGlossary}}{{/each}} +{{>>>>>>>}} + +{{<<<<<<<}} +{{#each glossary}}{{#multiLine}}{{.}}{{/multiLine}}{{#unless @last}} | {{/unless}}{{/each}} +{{=======}} +{{#each glossary}}{{#formatGlossary ../dictionary}}{{{.}}}{{/formatGlossary}}{{#unless @last}} | {{/unless}}{{/each}} +{{>>>>>>>}} + +{{<<<<<<<}} +{{#each glossary}}
  • {{#multiLine}}{{.}}{{/multiLine}}
    • {{/each}} +{{=======}} +{{#each glossary}}
  • {{#formatGlossary ../dictionary}}{{{.}}}{{/formatGlossary}}
    • {{/each}} +{{>>>>>>>}} diff --git a/ext/data/templates/default-anki-field-templates.handlebars b/ext/data/templates/default-anki-field-templates.handlebars index 97359aa0..67547732 100644 --- a/ext/data/templates/default-anki-field-templates.handlebars +++ b/ext/data/templates/default-anki-field-templates.handlebars @@ -21,11 +21,11 @@ {{~#if only~}}({{#each only}}{{.}}{{#unless @last}}, {{/unless}}{{/each}} only) {{/if~}} {{~/unless~}} {{~#if (op "<=" glossary.length 1)~}} - {{#each glossary}}{{#multiLine}}{{.}}{{/multiLine}}{{/each}} + {{#each glossary}}{{#formatGlossary ../dictionary}}{{{.}}}{{/formatGlossary}}{{/each}} {{~else if @root.compactGlossaries~}} - {{#each glossary}}{{#multiLine}}{{.}}{{/multiLine}}{{#unless @last}} | {{/unless}}{{/each}} + {{#each glossary}}{{#formatGlossary ../dictionary}}{{{.}}}{{/formatGlossary}}{{#unless @last}} | {{/unless}}{{/each}} {{~else~}} - + {{~/if~}} {{~#set "previousDictionary" dictionary~}}{{~/set~}} {{/inline}} diff --git a/ext/js/data/options-util.js b/ext/js/data/options-util.js index eb29dae4..42175d35 100644 --- a/ext/js/data/options-util.js +++ b/ext/js/data/options-util.js @@ -461,7 +461,8 @@ class OptionsUtil { {async: false, update: this._updateVersion9.bind(this)}, {async: true, update: this._updateVersion10.bind(this)}, {async: false, update: this._updateVersion11.bind(this)}, - {async: true, update: this._updateVersion12.bind(this)} + {async: true, update: this._updateVersion12.bind(this)}, + {async: true, update: this._updateVersion13.bind(this)} ]; } @@ -844,4 +845,11 @@ class OptionsUtil { } return options; } + + async _updateVersion13(options) { + // Version 13 changes: + // Handlebars templates updated to use formatGlossary. + await this._applyAnkiFieldTemplatesPatch(options, '/data/templates/anki-field-templates-upgrade-v13.handlebars'); + return options; + } } diff --git a/ext/js/dom/css-style-applier.js b/ext/js/dom/css-style-applier.js index 593e7a46..c617fead 100644 --- a/ext/js/dom/css-style-applier.js +++ b/ext/js/dom/css-style-applier.js @@ -72,12 +72,6 @@ class CssStyleApplier { element.removeAttribute('style'); } } - for (const element of elements) { - const {dataset} = element; - for (const key of Object.keys(dataset)) { - delete dataset[key]; - } - } } // Private diff --git a/ext/js/templates/template-renderer-frame-main.js b/ext/js/templates/template-renderer-frame-main.js index 3d61295d..bb9cac3a 100644 --- a/ext/js/templates/template-renderer-frame-main.js +++ b/ext/js/templates/template-renderer-frame-main.js @@ -17,14 +17,17 @@ /* globals * AnkiNoteDataCreator + * CssStyleApplier * JapaneseUtil * TemplateRenderer * TemplateRendererFrameApi */ -(() => { +(async () => { + const cssStyleApplier = new CssStyleApplier('/data/structured-content-style.json'); + await cssStyleApplier.prepare(); const japaneseUtil = new JapaneseUtil(null); - const templateRenderer = new TemplateRenderer(japaneseUtil); + const templateRenderer = new TemplateRenderer(japaneseUtil, cssStyleApplier); const ankiNoteDataCreator = new AnkiNoteDataCreator(japaneseUtil); templateRenderer.registerDataType('ankiNote', { modifier: ({marker, commonData}) => ankiNoteDataCreator.create(marker, commonData), diff --git a/ext/js/templates/template-renderer.js b/ext/js/templates/template-renderer.js index a06298aa..928ec3c4 100644 --- a/ext/js/templates/template-renderer.js +++ b/ext/js/templates/template-renderer.js @@ -18,11 +18,13 @@ /* global * DictionaryDataUtil * Handlebars + * StructuredContentGenerator */ class TemplateRenderer { - constructor(japaneseUtil) { + constructor(japaneseUtil, cssStyleApplier) { this._japaneseUtil = japaneseUtil; + this._cssStyleApplier = cssStyleApplier; this._cache = new Map(); this._cacheMaxSize = 5; this._helpersRegistered = false; @@ -31,6 +33,7 @@ class TemplateRenderer { this._requirements = null; this._cleanupCallbacks = null; this._customData = null; + this._temporaryElement = null; } registerDataType(name, {modifier=null, composeData=null}) { @@ -161,7 +164,8 @@ class TemplateRenderer { ['typeof', this._getTypeof.bind(this)], ['join', this._join.bind(this)], ['concat', this._concat.bind(this)], - ['pitchCategories', this._pitchCategories.bind(this)] + ['pitchCategories', this._pitchCategories.bind(this)], + ['formatGlossary', this._formatGlossary.bind(this)] ]; for (const [name, helper] of helpers) { @@ -244,8 +248,12 @@ class TemplateRenderer { return result; } + _stringToMultiLineHtml(string) { + return string.split('\n').join('
    '); + } + _multiLine(context, options) { - return options.fn(context).split('\n').join('
    '); + return this._stringToMultiLineHtml(options.fn(context)); } _sanitizeCssClass(context, options) { @@ -497,4 +505,130 @@ class TemplateRenderer { } return [...categories]; } + + _getTemporaryElement() { + let element = this._temporaryElement; + if (element === null) { + element = document.createElement('div'); + this._temporaryElement = element; + } + return element; + } + + _getHtml(node) { + const container = this._getTemporaryElement(); + container.appendChild(node); + this._normalizeHtml(container); + const result = container.innerHTML; + container.textContent = ''; + return result; + } + + _normalizeHtml(root) { + const {ELEMENT_NODE, TEXT_NODE} = Node; + const treeWalker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT | NodeFilter.SHOW_TEXT); + const elements = []; + const textNodes = []; + while (true) { + const node = treeWalker.nextNode(); + if (node === null) { break; } + switch (node.nodeType) { + case ELEMENT_NODE: + elements.push(node); + break; + case TEXT_NODE: + textNodes.push(node); + break; + } + } + this._cssStyleApplier.applyClassStyles(elements); + for (const element of elements) { + const {dataset} = element; + for (const key of Object.keys(dataset)) { + delete dataset[key]; + } + } + for (const textNode of textNodes) { + this._replaceNewlines(textNode); + } + } + + _replaceNewlines(textNode) { + const parts = textNode.nodeValue.split('\n'); + if (parts.length <= 1) { return; } + const {parentNode} = textNode; + if (parentNode === null) { return; } + const fragment = document.createDocumentFragment(); + for (let i = 0, ii = parts.length; i < ii; ++i) { + if (i > 0) { fragment.appendChild(document.createElement('br')); } + fragment.appendChild(document.createTextNode(parts[i])); + } + parentNode.replaceChild(fragment, textNode); + } + + _getDictionaryMedia(data, dictionary, path) { + const {media} = data; + if (typeof media === 'object' && media !== null && Object.prototype.hasOwnProperty.call(media, 'dictionaryMedia')) { + const {dictionaryMedia} = media; + if (typeof dictionaryMedia === 'object' && dictionaryMedia !== null && Object.prototype.hasOwnProperty.call(dictionaryMedia, dictionary)) { + const dictionaryMedia2 = dictionaryMedia[dictionary]; + if (Object.prototype.hasOwnProperty.call(dictionaryMedia2, path)) { + return dictionaryMedia2[path]; + } + } + } + return null; + } + + _createStructuredContentGenerator(data) { + const mediaLoader = { + loadMedia: async (path, dictionary, onLoad, onUnload) => { + const imageUrl = this._getDictionaryMedia(data, dictionary, path); + if (imageUrl !== null) { + onLoad(imageUrl); + this._cleanupCallbacks.push(() => onUnload(true)); + } else { + let set = this._customData.requiredDictionaryMedia; + if (typeof set === 'undefined') { + set = new Set(); + this._customData.requiredDictionaryMedia = set; + } + const key = JSON.stringify([dictionary, path]); + if (!set.has(key)) { + set.add(key); + this._requirements.push({ + type: 'dictionaryMedia', + dictionary, + path + }); + } + } + } + }; + return new StructuredContentGenerator(mediaLoader, document); + } + + _formatGlossary(context, dictionary, options) { + const data = options.data.root; + const content = options.fn(context); + if (typeof content === 'string') { return this._stringToMultiLineHtml(content); } + if (!(typeof content === 'object' && content !== null)) { return ''; } + switch (content.type) { + case 'image': return this._formatGlossaryImage(content, dictionary, data); + case 'structured-content': return this._formatStructuredContent(content, dictionary, data); + } + return ''; + } + + _formatGlossaryImage(content, dictionary, data) { + const structuredContentGenerator = this._createStructuredContentGenerator(data); + const node = structuredContentGenerator.createDefinitionImage(content, dictionary); + return this._getHtml(node); + } + + _formatStructuredContent(content, dictionary, data) { + const structuredContentGenerator = this._createStructuredContentGenerator(data); + const node = structuredContentGenerator.createStructuredContent(content.content, dictionary); + return node !== null ? this._getHtml(node) : ''; + } } diff --git a/ext/template-renderer.html b/ext/template-renderer.html index f9667acd..f01b5b9a 100644 --- a/ext/template-renderer.html +++ b/ext/template-renderer.html @@ -18,6 +18,8 @@ + + diff --git a/test/data/anki-note-builder-test-results.json b/test/data/anki-note-builder-test-results.json index c6405439..c8357cb3 100644 --- a/test/data/anki-note-builder-test-results.json +++ b/test/data/anki-note-builder-test-results.json @@ -587,9 +587,9 @@ "frequencies": "", "furigana": "画像がぞう", "furigana-plain": "画像[がぞう]", - "glossary": "
    (n, Test Dictionary 2)
    ", - "glossary-brief": "
    ", - "glossary-no-dictionary": "
    (n)
    ", + "glossary": "
    (n, Test Dictionary 2)
    ", + "glossary-brief": "
    ", + "glossary-no-dictionary": "
    (n)
    ", "part-of-speech": "Noun", "pitch-accents": "No pitch accent data", "pitch-accent-graphs": "No pitch accent data", @@ -1042,9 +1042,9 @@ "frequencies": "", "furigana": "画像がぞう", "furigana-plain": "画像[がぞう]", - "glossary": "
    (n, Test Dictionary 2)
    ", - "glossary-brief": "
    ", - "glossary-no-dictionary": "
    (n)
    ", + "glossary": "
    (n, Test Dictionary 2)
    ", + "glossary-brief": "
    ", + "glossary-no-dictionary": "
    (n)
    ", "part-of-speech": "Noun", "pitch-accents": "No pitch accent data", "pitch-accent-graphs": "No pitch accent data", diff --git a/test/test-anki-note-builder.js b/test/test-anki-note-builder.js index d7045254..2517e521 100644 --- a/test/test-anki-note-builder.js +++ b/test/test-anki-note-builder.js @@ -29,9 +29,13 @@ function clone(value) { async function createVM() { const dom = new JSDOM(); - const {document} = dom.window; + const {Node, NodeFilter, document} = dom.window; - const vm = new TranslatorVM({document}); + const vm = new TranslatorVM({ + Node, + NodeFilter, + document + }); const dictionaryDirectory = path.join(__dirname, 'data', 'dictionaries', 'valid-dictionary1'); await vm.prepare(dictionaryDirectory, 'Test Dictionary 2'); @@ -39,6 +43,8 @@ async function createVM() { vm.execute([ 'js/data/anki-note-builder.js', 'js/data/anki-util.js', + 'js/dom/css-style-applier.js', + 'js/display/structured-content-generator.js', 'js/templates/template-renderer.js', 'lib/handlebars.min.js' ]); @@ -46,11 +52,13 @@ async function createVM() { const [ JapaneseUtil, TemplateRenderer, - AnkiNoteBuilder + AnkiNoteBuilder, + CssStyleApplier ] = vm.get([ 'JapaneseUtil', 'TemplateRenderer', - 'AnkiNoteBuilder' + 'AnkiNoteBuilder', + 'CssStyleApplier' ]); const ankiNoteDataCreator = vm.ankiNoteDataCreator; @@ -58,7 +66,8 @@ async function createVM() { constructor() { this._preparePromise = null; this._japaneseUtil = new JapaneseUtil(null); - this._templateRenderer = new TemplateRenderer(this._japaneseUtil); + this._cssStyleApplier = new CssStyleApplier('/data/structured-content-style.json'); + this._templateRenderer = new TemplateRenderer(this._japaneseUtil, this._cssStyleApplier); this._templateRenderer.registerDataType('ankiNote', { modifier: ({marker, commonData}) => ankiNoteDataCreator.create(marker, commonData), composeData: (marker, commonData) => ({marker, commonData}) @@ -83,7 +92,7 @@ async function createVM() { } async _prepareInternal() { - // Empty + await this._cssStyleApplier.prepare(); } _serializeError(error) { diff --git a/test/test-options-util.js b/test/test-options-util.js index 47b09b94..7d86743b 100644 --- a/test/test-options-util.js +++ b/test/test-options-util.js @@ -589,7 +589,7 @@ function createOptionsUpdatedTestData1() { } ], profileCurrent: 0, - version: 12, + version: 13, global: { database: { prefixWildcardsSupported: false @@ -655,7 +655,8 @@ async function testFieldTemplatesUpdate(extDir) { {version: 6, changes: loadDataFile('data/templates/anki-field-templates-upgrade-v6.handlebars')}, {version: 8, changes: loadDataFile('data/templates/anki-field-templates-upgrade-v8.handlebars')}, {version: 10, changes: loadDataFile('data/templates/anki-field-templates-upgrade-v10.handlebars')}, - {version: 12, changes: loadDataFile('data/templates/anki-field-templates-upgrade-v12.handlebars')} + {version: 12, changes: loadDataFile('data/templates/anki-field-templates-upgrade-v12.handlebars')}, + {version: 13, changes: loadDataFile('data/templates/anki-field-templates-upgrade-v13.handlebars')} ]; const getUpdateAdditions = (startVersion=0) => { let value = ''; @@ -875,11 +876,11 @@ ${getUpdateAdditions()} {{~#if only~}}({{#each only}}{{.}}{{#unless @last}}, {{/unless}}{{/each}} only) {{/if~}} {{~/unless~}} {{~#if (op "<=" glossary.length 1)~}} - {{#each glossary}}{{#multiLine}}{{.}}{{/multiLine}}{{/each}} + {{#each glossary}}{{#formatGlossary ../dictionary}}{{{.}}}{{/formatGlossary}}{{/each}} {{~else if @root.compactGlossaries~}} - {{#each glossary}}{{#multiLine}}{{.}}{{/multiLine}}{{#unless @last}} | {{/unless}}{{/each}} + {{#each glossary}}{{#formatGlossary ../dictionary}}{{{.}}}{{/formatGlossary}}{{#unless @last}} | {{/unless}}{{/each}} {{~else~}} - + {{~/if~}} {{~#set "previousDictionary" dictionary~}}{{~/set~}} {{/inline}} @@ -920,6 +921,27 @@ ${getUpdateAdditions()} ${getUpdateAdditions(7)} {{~> (lookup . "marker") ~}}`.trimStart() + }, + // formatGlossary update + { + oldVersion: 12, + old: ` + {{~#if (op "<=" glossary.length 1)~}} + {{#each glossary}}{{#multiLine}}{{.}}{{/multiLine}}{{/each}} + {{~else if @root.compactGlossaries~}} + {{#each glossary}}{{#multiLine}}{{.}}{{/multiLine}}{{#unless @last}} | {{/unless}}{{/each}} + {{~else~}} + + {{~/if~}}`.trimStart(), + + expected: ` + {{~#if (op "<=" glossary.length 1)~}} + {{#each glossary}}{{#formatGlossary ../dictionary}}{{{.}}}{{/formatGlossary}}{{/each}} + {{~else if @root.compactGlossaries~}} + {{#each glossary}}{{#formatGlossary ../dictionary}}{{{.}}}{{/formatGlossary}}{{#unless @last}} | {{/unless}}{{/each}} + {{~else~}} + + {{~/if~}}`.trimStart() } ]; -- cgit v1.2.3