From 376151096431d4362e4baaacf0cef4a534e169f7 Mon Sep 17 00:00:00 2001 From: Darius Jahandarie Date: Fri, 3 Nov 2023 23:32:33 +0900 Subject: Replace JsonSchema with ajv for dictionary validation --- ext/lib/ucs2length.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 ext/lib/ucs2length.js (limited to 'ext/lib') diff --git a/ext/lib/ucs2length.js b/ext/lib/ucs2length.js new file mode 100644 index 00000000..120a64d4 --- /dev/null +++ b/ext/lib/ucs2length.js @@ -0,0 +1,16 @@ +export default function ucs2length(str) { + const len = str.length; + let length = 0; + let pos = 0; + let value; + while (pos < len) { + length++; + value = str.charCodeAt(pos++); + if (value >= 0xd800 && value <= 0xdbff && pos < len) { + // high surrogate, and there is a next character + value = str.charCodeAt(pos); + if ((value & 0xfc00) === 0xdc00) pos++; // low surrogate + } + } + return length; +} -- cgit v1.2.3 From ef79eab44bfd000792c610b968b5ceefd41e76a0 Mon Sep 17 00:00:00 2001 From: Darius Jahandarie Date: Sat, 4 Nov 2023 18:45:57 +0900 Subject: Modernize codebase - Use ES modules - Remove vendored libs and build them from npm using esbuild - Switch from JSZip to zip.js --- .eslintrc.json | 33 +- .gitignore | 2 +- .vscode/settings.json | 7 +- dev/build-libs.js | 60 +- dev/build.js | 2 + dev/data/manifest-variants.json | 33 +- dev/lib/dexie-export-import.js | 17 + dev/lib/dexie.js | 17 + dev/lib/handlebars.js | 18 + dev/lib/handlebars/LICENSE | 29 + dev/lib/handlebars/README.md | 164 + .../handlebars/__snapshots__/index.test.ts.snap | 91 + dev/lib/handlebars/index.test.ts | 567 ++ dev/lib/handlebars/index.ts | 33 + dev/lib/handlebars/jest.config.js | 10 + dev/lib/handlebars/kibana.jsonc | 5 + dev/lib/handlebars/package.json | 6 + .../scripts/check_for_upstream_updates.sh | 45 + dev/lib/handlebars/scripts/print_ast.js | 64 + .../handlebars/scripts/update_upstream_git_hash.sh | 24 + dev/lib/handlebars/src/__jest__/test_bench.ts | 207 + dev/lib/handlebars/src/handlebars.ts | 47 + dev/lib/handlebars/src/spec/.upstream_git_hash | 1 + dev/lib/handlebars/src/spec/index.basic.test.ts | 481 ++ dev/lib/handlebars/src/spec/index.blocks.test.ts | 366 + dev/lib/handlebars/src/spec/index.builtins.test.ts | 676 ++ dev/lib/handlebars/src/spec/index.compiler.test.ts | 86 + dev/lib/handlebars/src/spec/index.data.test.ts | 269 + dev/lib/handlebars/src/spec/index.helpers.test.ts | 958 +++ dev/lib/handlebars/src/spec/index.partials.test.ts | 591 ++ .../handlebars/src/spec/index.regressions.test.ts | 379 + dev/lib/handlebars/src/spec/index.security.test.ts | 132 + dev/lib/handlebars/src/spec/index.strict.test.ts | 164 + .../src/spec/index.subexpressions.test.ts | 214 + dev/lib/handlebars/src/spec/index.utils.test.ts | 24 + .../src/spec/index.whitespace_control.test.ts | 88 + dev/lib/handlebars/src/symbols.ts | 8 + dev/lib/handlebars/src/types.ts | 225 + dev/lib/handlebars/src/utils.ts | 69 + dev/lib/handlebars/src/visitor.ts | 778 ++ dev/lib/handlebars/tsconfig.json | 15 + dev/lib/parse5.js | 17 + dev/lib/ucs2length.js | 18 + dev/lib/wanakana.js | 17 + dev/lib/zip.js | 17 + ext/action-popup.html | 13 +- ext/background.html | 38 +- ext/info.html | 16 +- ext/issues.html | 6 +- ext/js/accessibility/accessibility-controller.js | 5 +- ext/js/accessibility/google-docs-util.js | 9 +- ext/js/app/content-script-main.js | 10 +- ext/js/app/content-script-wrapper.js | 24 + ext/js/app/frontend.js | 18 +- ext/js/app/popup-factory.js | 14 +- ext/js/app/popup-proxy.js | 7 +- ext/js/app/popup-window.js | 6 +- ext/js/app/popup.js | 14 +- ext/js/app/theme-controller.js | 2 +- ext/js/background/backend.js | 46 +- ext/js/background/background-main.js | 5 +- ext/js/background/offscreen-main.js | 4 +- ext/js/background/offscreen.js | 8 +- ext/js/background/profile-conditions-util.js | 6 +- ext/js/background/request-builder.js | 2 +- ext/js/background/script-manager.js | 3 +- ext/js/comm/anki-connect.js | 7 +- ext/js/comm/api.js | 4 +- ext/js/comm/clipboard-monitor.js | 4 +- ext/js/comm/clipboard-reader.js | 6 +- ext/js/comm/cross-frame-api.js | 5 +- ext/js/comm/frame-ancestry-handler.js | 5 +- ext/js/comm/frame-client.js | 4 +- ext/js/comm/frame-endpoint.js | 5 +- ext/js/comm/frame-offset-forwarder.js | 7 +- ext/js/comm/mecab.js | 4 +- ext/js/core.js | 34 +- ext/js/data/anki-note-builder.js | 10 +- ext/js/data/anki-util.js | 4 +- ext/js/data/database.js | 2 +- ext/js/data/json-schema.js | 7 +- ext/js/data/options-util.js | 9 +- ext/js/data/permissions-util.js | 6 +- ext/js/data/sandbox/anki-note-data-creator.js | 6 +- ext/js/data/sandbox/array-buffer-util.js | 2 +- ext/js/data/sandbox/string-util.js | 2 +- ext/js/debug/timer.js | 2 +- ext/js/display/display-anki.js | 12 +- ext/js/display/display-audio.js | 7 +- ext/js/display/display-content-manager.js | 8 +- ext/js/display/display-generator.js | 16 +- ext/js/display/display-history.js | 4 +- ext/js/display/display-notification.js | 4 +- ext/js/display/display-profile-selection.js | 8 +- ext/js/display/display-resizer.js | 4 +- ext/js/display/display.js | 42 +- ext/js/display/element-overflow-controller.js | 4 +- ext/js/display/option-toggle-hotkey-handler.js | 5 +- ext/js/display/popup-main.js | 11 + ext/js/display/query-parser.js | 8 +- ext/js/display/sandbox/pronunciation-generator.js | 2 +- .../sandbox/structured-content-generator.js | 2 +- ext/js/display/search-action-popup-controller.js | 2 +- ext/js/display/search-display-controller.js | 10 +- ext/js/display/search-main.js | 24 +- .../display/search-persistent-state-controller.js | 4 +- ext/js/dom/document-focus-controller.js | 2 +- ext/js/dom/document-util.js | 11 +- ext/js/dom/dom-data-binder.js | 10 +- ext/js/dom/dom-text-scanner.js | 6 +- ext/js/dom/html-template-collection.js | 2 +- ext/js/dom/native-simple-dom-parser.js | 2 +- ext/js/dom/panel-element.js | 4 +- ext/js/dom/popup-menu.js | 4 +- ext/js/dom/sandbox/css-style-applier.js | 2 +- ext/js/dom/scroll-element.js | 2 +- ext/js/dom/selector-observer.js | 2 +- ext/js/dom/simple-dom-parser.js | 6 +- ext/js/dom/text-source-element.js | 9 +- ext/js/dom/text-source-range.js | 9 +- ext/js/extension/environment.js | 2 +- ext/js/general/cache-map.js | 3 +- ext/js/general/object-property-accessor.js | 2 +- ext/js/general/regex-util.js | 3 +- ext/js/general/task-accumulator.js | 4 +- ext/js/general/text-source-map.js | 2 +- ext/js/input/hotkey-handler.js | 8 +- ext/js/input/hotkey-help-controller.js | 8 +- ext/js/input/hotkey-util.js | 2 +- ext/js/language/deinflector.js | 2 +- ext/js/language/dictionary-database.js | 7 +- .../language/dictionary-importer-media-loader.js | 4 +- ext/js/language/dictionary-importer.js | 83 +- ext/js/language/dictionary-worker-handler.js | 11 +- ext/js/language/dictionary-worker-main.js | 18 +- ext/js/language/dictionary-worker-media-loader.js | 4 +- ext/js/language/dictionary-worker.js | 9 +- ext/js/language/sandbox/dictionary-data-util.js | 2 +- ext/js/language/sandbox/japanese-util.js | 2 +- ext/js/language/text-scanner.js | 8 +- ext/js/language/translator.js | 11 +- ext/js/media/audio-downloader.js | 14 +- ext/js/media/audio-system.js | 7 +- ext/js/media/media-util.js | 2 +- ext/js/media/text-to-speech-audio.js | 2 +- ext/js/pages/action-popup-main.js | 9 +- .../pages/common/extension-content-controller.js | 6 +- ext/js/pages/generic-page-main.js | 6 +- ext/js/pages/info-main.js | 10 +- ext/js/pages/permissions-main.js | 20 +- ext/js/pages/settings/anki-controller.js | 16 +- ext/js/pages/settings/anki-templates-controller.js | 10 +- ext/js/pages/settings/audio-controller.js | 7 +- ext/js/pages/settings/backup-controller.js | 16 +- .../settings/collapsible-dictionary-controller.js | 7 +- ext/js/pages/settings/dictionary-controller.js | 6 +- .../pages/settings/dictionary-import-controller.js | 10 +- .../extension-keyboard-shortcuts-controller.js | 10 +- .../pages/settings/generic-setting-controller.js | 9 +- .../pages/settings/keyboard-mouse-input-field.js | 9 +- .../settings/keyboard-shortcuts-controller.js | 12 +- ext/js/pages/settings/mecab-controller.js | 4 +- ext/js/pages/settings/modal-controller.js | 6 +- ext/js/pages/settings/modal.js | 6 +- ext/js/pages/settings/nested-popups-controller.js | 6 +- .../settings/permissions-origin-controller.js | 4 +- .../settings/permissions-toggle-controller.js | 6 +- .../settings/persistent-storage-controller.js | 5 +- ext/js/pages/settings/popup-preview-controller.js | 2 +- ext/js/pages/settings/popup-preview-frame-main.js | 6 + ext/js/pages/settings/popup-preview-frame.js | 11 +- ext/js/pages/settings/popup-window-controller.js | 4 +- ext/js/pages/settings/profile-conditions-ui.js | 7 +- ext/js/pages/settings/profile-controller.js | 8 +- .../settings/recommended-permissions-controller.js | 4 +- ext/js/pages/settings/scan-inputs-controller.js | 8 +- .../settings/scan-inputs-simple-controller.js | 11 +- .../secondary-search-dictionary-controller.js | 7 +- .../sentence-termination-characters-controller.js | 4 +- ext/js/pages/settings/settings-controller.js | 12 +- .../pages/settings/settings-display-controller.js | 10 +- ext/js/pages/settings/settings-main.js | 64 +- .../sort-frequency-dictionary-controller.js | 4 +- ext/js/pages/settings/status-footer.js | 6 +- ext/js/pages/settings/storage-controller.js | 4 +- .../translation-text-replacements-controller.js | 4 +- ext/js/pages/welcome-main.js | 26 +- ext/js/script/dynamic-loader-sentinel.js | 2 + ext/js/script/dynamic-loader.js | 5 +- .../anki-template-renderer-content-manager.js | 2 +- ext/js/templates/sandbox/anki-template-renderer.js | 12 +- .../sandbox/template-renderer-frame-api.js | 2 +- .../sandbox/template-renderer-frame-main.js | 6 +- .../sandbox/template-renderer-media-provider.js | 2 +- ext/js/templates/sandbox/template-renderer.js | 10 +- ext/js/templates/template-patcher.js | 2 +- ext/js/templates/template-renderer-proxy.js | 4 +- ext/js/yomichan.js | 9 +- ext/legal.html | 6 +- ext/lib/dexie-export-import.js | 3457 -------- ext/lib/dexie-export-import.js.map | 1 - ext/lib/dexie.min.js | 2 - ext/lib/dexie.min.js.map | 1 - ext/lib/handlebars.min.js | 25 - ext/lib/handlebars.min.js.map | 7 - ext/lib/jszip.min.js | 13 - ext/lib/parse5.js | 8557 -------------------- ext/lib/ucs2length.js | 16 - ext/lib/wanakana.min.js | 2 - ext/lib/wanakana.min.js.map | 1 - ext/offscreen.html | 9 +- ext/permissions.html | 26 +- ext/popup-preview.html | 27 +- ext/popup.html | 49 +- ext/search.html | 51 +- ext/settings.html | 75 +- ext/sw.js | 39 +- ext/template-renderer.html | 16 +- ext/welcome.html | 38 +- jsconfig.json | 11 + package-lock.json | 806 +- package.json | 9 +- 222 files changed, 8491 insertions(+), 13144 deletions(-) create mode 100644 dev/lib/dexie-export-import.js create mode 100644 dev/lib/dexie.js create mode 100644 dev/lib/handlebars.js create mode 100644 dev/lib/handlebars/LICENSE create mode 100644 dev/lib/handlebars/README.md create mode 100644 dev/lib/handlebars/__snapshots__/index.test.ts.snap create mode 100644 dev/lib/handlebars/index.test.ts create mode 100644 dev/lib/handlebars/index.ts create mode 100644 dev/lib/handlebars/jest.config.js create mode 100644 dev/lib/handlebars/kibana.jsonc create mode 100644 dev/lib/handlebars/package.json create mode 100755 dev/lib/handlebars/scripts/check_for_upstream_updates.sh create mode 100755 dev/lib/handlebars/scripts/print_ast.js create mode 100755 dev/lib/handlebars/scripts/update_upstream_git_hash.sh create mode 100644 dev/lib/handlebars/src/__jest__/test_bench.ts create mode 100644 dev/lib/handlebars/src/handlebars.ts create mode 100644 dev/lib/handlebars/src/spec/.upstream_git_hash create mode 100644 dev/lib/handlebars/src/spec/index.basic.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.blocks.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.builtins.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.compiler.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.data.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.helpers.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.partials.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.regressions.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.security.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.strict.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.subexpressions.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.utils.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.whitespace_control.test.ts create mode 100644 dev/lib/handlebars/src/symbols.ts create mode 100644 dev/lib/handlebars/src/types.ts create mode 100644 dev/lib/handlebars/src/utils.ts create mode 100644 dev/lib/handlebars/src/visitor.ts create mode 100644 dev/lib/handlebars/tsconfig.json create mode 100644 dev/lib/parse5.js create mode 100644 dev/lib/ucs2length.js create mode 100644 dev/lib/wanakana.js create mode 100644 dev/lib/zip.js create mode 100644 ext/js/app/content-script-wrapper.js delete mode 100644 ext/lib/dexie-export-import.js delete mode 100644 ext/lib/dexie-export-import.js.map delete mode 100644 ext/lib/dexie.min.js delete mode 100644 ext/lib/dexie.min.js.map delete mode 100644 ext/lib/handlebars.min.js delete mode 100644 ext/lib/handlebars.min.js.map delete mode 100644 ext/lib/jszip.min.js delete mode 100644 ext/lib/parse5.js delete mode 100644 ext/lib/ucs2length.js delete mode 100644 ext/lib/wanakana.min.js delete mode 100644 ext/lib/wanakana.min.js.map create mode 100644 jsconfig.json (limited to 'ext/lib') diff --git a/.eslintrc.json b/.eslintrc.json index a7fb842b..99c2383a 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -6,7 +6,7 @@ ], "parserOptions": { "ecmaVersion": 11, - "sourceType": "script", + "sourceType": "module", "ecmaFeatures": { "globalReturn": false, "impliedStrict": true @@ -50,6 +50,7 @@ "no-case-declarations": "error", "no-const-assign": "error", "no-constant-condition": "off", + "no-console": "warn", "no-global-assign": "error", "no-param-reassign": "off", "no-prototype-builtins": "error", @@ -250,6 +251,7 @@ "jsdoc/multiline-blocks": "error", "jsdoc/no-bad-blocks": "error", "jsdoc/no-multi-asterisks": "error", + "jsdoc/no-undefined-types": 1, "jsdoc/require-asterisk-prefix": "error", "jsdoc/require-hyphen-before-param-description": [ "error", @@ -384,26 +386,7 @@ "ext/js/accessibility/google-docs.js", "ext/js/**/sandbox/**/*.js" ], - "globals": { - "serializeError": "readonly", - "deserializeError": "readonly", - "isObject": "readonly", - "stringReverse": "readonly", - "promiseTimeout": "readonly", - "escapeRegExp": "readonly", - "deferPromise": "readonly", - "clone": "readonly", - "deepEqual": "readonly", - "generateId": "readonly", - "promiseAnimationFrame": "readonly", - "invokeMessageHandler": "readonly", - "log": "readonly", - "DynamicProperty": "readonly", - "EventDispatcher": "readonly", - "EventListenerCollection": "readonly", - "Logger": "readonly", - "import": "readonly" - } + "globals": {} }, { "files": [ @@ -446,6 +429,14 @@ "webextensions": false } }, + { + "files": [ + "ext/js/language/dictionary-worker-main.js" + ], + "parserOptions": { + "sourceType": "module" + } + }, { "files": [ "playwright.config.js" diff --git a/.gitignore b/.gitignore index 426db4ad..7d085ebf 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,4 @@ dictionaries/ /playwright/.cache/ /test/playwright/__screenshots__/ ext/manifest.json -ext/lib/validate-schemas.js +ext/lib/ diff --git a/.vscode/settings.json b/.vscode/settings.json index 9318d9f6..2480961b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,10 +1,13 @@ { "markdown.extension.toc.levels": "1..3", "editor.codeActionsOnSave": { - "source.fixAll.eslint": true + "source.addMissingImports": true, + "source.organizeImports": true, + "source.fixAll.eslint": true, }, "eslint.format.enable": true, "playwright.env": { "PW_EXPERIMENTAL_SERVICE_WORKER_NETWORK_EVENTS": 1 - } + }, + "javascript.preferences.importModuleSpecifierEnding": "js", } diff --git a/dev/build-libs.js b/dev/build-libs.js index 36c07edd..497206c9 100644 --- a/dev/build-libs.js +++ b/dev/build-libs.js @@ -18,49 +18,35 @@ const fs = require('fs'); const path = require('path'); -const browserify = require('browserify'); +const esbuild = require('esbuild'); -async function buildParse5() { - const parse5Path = require.resolve('parse5'); - const cwd = process.cwd(); - try { - const baseDir = path.dirname(parse5Path); - process.chdir(baseDir); // This is necessary to ensure relative source map file names are consistent - return await new Promise((resolve, reject) => { - browserify({ - entries: [parse5Path], - standalone: 'parse5', - debug: true, - baseDir - }).bundle((error, result) => { - if (error) { - reject(error); - } else { - resolve(result); - } - }); - }); - } finally { - process.chdir(cwd); - } -} - -function getBuildTargets() { - const extLibPath = path.join(__dirname, '..', 'ext', 'lib'); - return [ - {path: path.join(extLibPath, 'parse5.js'), build: buildParse5} - ]; +async function buildLib(p) { + await esbuild.build({ + entryPoints: [p], + bundle: true, + minify: false, + sourcemap: true, + target: 'es2020', + format: 'esm', + outfile: path.join(__dirname, '..', 'ext', 'lib', path.basename(p)), + external: ['fs'] + }); } -async function main() { - for (const {path: path2, build} of getBuildTargets()) { - const content = await build(); - fs.writeFileSync(path2, content); +async function buildLibs() { + const devLibPath = path.join(__dirname, 'lib'); + const files = await fs.promises.readdir(devLibPath, { + withFileTypes: true + }); + for (const f of files) { + if (f.isFile()) { + await buildLib(path.join(devLibPath, f.name)); + } } } -if (require.main === module) { main(); } +if (require.main === module) { buildLibs(); } module.exports = { - getBuildTargets + buildLibs }; diff --git a/dev/build.js b/dev/build.js index 3bfb5418..1e6ef1d0 100644 --- a/dev/build.js +++ b/dev/build.js @@ -26,6 +26,7 @@ const {getAllFiles, getArgs, testMain} = util; const {ManifestUtil} = require('./manifest-util'); const Ajv = require('ajv'); const standaloneCode = require('ajv/dist/standalone').default; +const buildLibs = require('./build-libs.js').buildLibs; async function createZip(directory, excludeFiles, outputFileName, sevenZipExes, onUpdate, dryRun) { try { @@ -215,6 +216,7 @@ async function main(argv) { const manifestPath = path.join(extDir, 'manifest.json'); try { + await buildLibs(); const variantNames = ( argv.length === 0 || args.get('all') ? manifestUtil.getVariants().filter(({buildable}) => buildable !== false).map(({name}) => name) : diff --git a/dev/data/manifest-variants.json b/dev/data/manifest-variants.json index 1eae2112..d44251e1 100644 --- a/dev/data/manifest-variants.json +++ b/dev/data/manifest-variants.json @@ -28,7 +28,8 @@ "default_popup": "action-popup.html" }, "background": { - "service_worker": "sw.js" + "service_worker": "sw.js", + "type": "module" }, "content_scripts": [ { @@ -41,28 +42,7 @@ "match_about_blank": true, "all_frames": true, "js": [ - "js/core.js", - "js/yomichan.js", - "js/app/frontend.js", - "js/app/popup.js", - "js/app/popup-factory.js", - "js/app/popup-proxy.js", - "js/app/popup-window.js", - "js/app/theme-controller.js", - "js/comm/api.js", - "js/comm/cross-frame-api.js", - "js/comm/frame-ancestry-handler.js", - "js/comm/frame-client.js", - "js/comm/frame-offset-forwarder.js", - "js/data/sandbox/string-util.js", - "js/dom/dom-text-scanner.js", - "js/dom/document-util.js", - "js/dom/text-source-element.js", - "js/dom/text-source-range.js", - "js/input/hotkey-handler.js", - "js/language/text-scanner.js", - "js/script/dynamic-loader.js", - "js/app/content-script-main.js" + "js/app/content-script-wrapper.js" ] } ], @@ -118,7 +98,8 @@ { "resources": [ "popup.html", - "template-renderer.html" + "template-renderer.html", + "js/*" ], "matches": [ "" @@ -187,7 +168,9 @@ "path": [ "permissions" ], - "items": ["clipboardRead"] + "items": [ + "clipboardRead" + ] } ] }, diff --git a/dev/lib/dexie-export-import.js b/dev/lib/dexie-export-import.js new file mode 100644 index 00000000..8d2ec206 --- /dev/null +++ b/dev/lib/dexie-export-import.js @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2023 Yomitan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +export * from 'dexie-export-import'; diff --git a/dev/lib/dexie.js b/dev/lib/dexie.js new file mode 100644 index 00000000..aa3f2b7d --- /dev/null +++ b/dev/lib/dexie.js @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2023 Yomitan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +export * from 'dexie'; diff --git a/dev/lib/handlebars.js b/dev/lib/handlebars.js new file mode 100644 index 00000000..5b57efdd --- /dev/null +++ b/dev/lib/handlebars.js @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2023 Yomitan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +export {Handlebars} from './handlebars/src/handlebars.js'; + diff --git a/dev/lib/handlebars/LICENSE b/dev/lib/handlebars/LICENSE new file mode 100644 index 00000000..5d971a17 --- /dev/null +++ b/dev/lib/handlebars/LICENSE @@ -0,0 +1,29 @@ +The MIT License (MIT) + +Copyright (c) Elasticsearch BV +Copyright (c) Copyright (C) 2011-2019 by Yehuda Katz + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +This software consists of voluntary contributions made by many +individuals. For exact contribution history, see the revision history +available at the following locations: + - https://github.com/handlebars-lang/handlebars.js + - https://github.com/elastic/kibana/tree/main/packages/kbn-handlebars diff --git a/dev/lib/handlebars/README.md b/dev/lib/handlebars/README.md new file mode 100644 index 00000000..cc151645 --- /dev/null +++ b/dev/lib/handlebars/README.md @@ -0,0 +1,164 @@ +# @kbn/handlebars + +A custom version of the handlebars package which, to improve security, does not use `eval` or `new Function`. This means that templates can't be compiled into JavaScript functions in advance and hence, rendering the templates is a lot slower. + +## Limitations + +- Only the following compile options are supported: + - `data` + - `knownHelpers` + - `knownHelpersOnly` + - `noEscape` + - `strict` + - `assumeObjects` + - `preventIndent` + - `explicitPartialContext` + +- Only the following runtime options are supported: + - `data` + - `helpers` + - `partials` + - `decorators` (not documented in the official Handlebars [runtime options documentation](https://handlebarsjs.com/api-reference/runtime-options.html)) + - `blockParams` (not documented in the official Handlebars [runtime options documentation](https://handlebarsjs.com/api-reference/runtime-options.html)) + +## Implementation differences + +The standard `handlebars` implementation: + +1. When given a template string, e.g. `Hello {{x}}`, return a "render" function which takes an "input" object, e.g. `{ x: 'World' }`. +1. The first time the "render" function is called the following happens: + 1. Turn the template string into an Abstract Syntax Tree (AST). + 1. Convert the AST into a hyper optimized JavaScript function which takes the input object as an argument. + 1. Call the generate JavaScript function with the given "input" object to produce and return the final output string (`Hello World`). +1. Subsequent calls to the "render" function will re-use the already generated JavaScript function. + +The custom `@kbn/handlebars` implementation: + +1. When given a template string, e.g. `Hello {{x}}`, return a "render" function which takes an "input" object, e.g. `{ x: 'World' }`. +1. The first time the "render" function is called the following happens: + 1. Turn the template string into an Abstract Syntax Tree (AST). + 1. Process the AST with the given "input" object to produce and return the final output string (`Hello World`). +1. Subsequent calls to the "render" function will re-use the already generated AST. + +_Note: Not parsing of the template string until the first call to the "render" function is deliberate as it mimics the original `handlebars` implementation. This means that any errors that occur due to an invalid template string will not be thrown until the first call to the "render" function._ + +## Technical details + +The `handlebars` library exposes the API for both [generating the AST](https://github.com/handlebars-lang/handlebars.js/blob/master/docs/compiler-api.md#ast) and walking it by implementing the [Visitor API](https://github.com/handlebars-lang/handlebars.js/blob/master/docs/compiler-api.md#ast-visitor). We can leverage that to our advantage and create our own "render" function, which internally calls this API to generate the AST and then the API to walk the AST. + +The `@kbn/handlebars` implementation of the `Visitor` class implements all the necessary methods called by the parent `Visitor` code when instructed to walk the AST. They all start with an upppercase letter, e.g. `MustacheStatement` or `SubExpression`. We call this class `ElasticHandlebarsVisitor`. + +To parse the template string to an AST representation, we call `Handlebars.parse(templateString)`, which returns an AST object. + +The AST object contains a bunch of nodes, one for each element of the template string, all arranged in a tree-like structure. The root of the AST object is a node of type `Program`. This is a special node, which we do not need to worry about, but each of its direct children has a type named like the method which will be called when the walking algorithm reaches that node, e.g. `ContentStatement` or `BlockStatement`. These are the methods that our `Visitor` implementation implements. + +To instruct our `ElasticHandlebarsVisitor` class to start walking the AST object, we call the `accept()` method inherited from the parent `Visitor` class with the main AST object. The `Visitor` will walk each node in turn that is directly attached to the root `Program` node. For each node it traverses, it will call the matching method in our `ElasticHandlebarsVisitor` class. + +To instruct the `Visitor` code to traverse any child nodes of a given node, our implementation needs to manually call `accept(childNode)`, `acceptArray(arrayOfChildNodes)`, `acceptKey(node, childKeyName)`, or `acceptRequired(node, childKeyName)` from within any of the "node" methods, otherwise the child nodes are ignored. + +### State + +We keep state internally in the `ElasticHandlebarsVisitor` object using the following private properties: + +- `contexts`: An array (stack) of `context` objects. In a simple template this array will always only contain a single element: The main `context` object. In more complicated scenarios, new `context` objects will be pushed and popped to and from the `contexts` stack as needed. +- `output`: An array containing the "rendered" output of each node (normally just one element per node). In the most simple template, this is simply joined together into a the final output string after the AST has been traversed. In more complicated templates, we use this array temporarily to collect parameters to give to helper functions (see the `getParams` function). + +## Testing + +The tests for `@kbn/handlebars` are integrated into the regular test suite of Kibana and are all jest tests. To run them all, simply do: + +```sh +node scripts/jest packages/kbn-handlebars +``` + +By default, each test will run both the original `handlebars` code and the modified `@kbn/handlebars` code to compare if the output of the two are identical. To isolate a test run to just one or the other, you can use the following environment variables: + +- `EVAL=1` - Set to only run the original `handlebars` implementation that uses `eval`. +- `AST=1` - Set to only run the modified `@kbn/handlebars` implementation that doesn't use `eval`. + +## Development + +Some of the tests have been copied from the upstream `handlebars` project and modified to fit our use-case, test-suite, and coding conventions. They are all located under the `packages/kbn-handlebars/src/spec` directory. To check if any of the copied files have received updates upstream that we might want to include in our copies, you can run the following script: + +```sh +./packages/kbn-handlebars/scripts/check_for_upstream_updates.sh +``` + +_Note: This will look for changes in the `4.x` branch of the `handlebars.js` repo only. Changes in the `master` branch are ignored._ + +Once all updates have been manually merged with our versions of the files, run the following script to "lock" us into the new updates: + +```sh +./packages/kbn-handlebars/scripts/update_upstream_git_hash.sh +``` + +This will update file `packages/kbn-handlebars/src/spec/.upstream_git_hash`. Make sure to commit changes to this file as well. + +## Debugging + +### Print AST + +To output the generated AST object structure in a somewhat readable form, use the following script: + +```sh +./packages/kbn-handlebars/scripts/print_ast.js +``` + +Example: + +```sh +./packages/kbn-handlebars/scripts/print_ast.js '{{value}}' +``` + +Output: + +```js +{ + type: 'Program', + body: [ + { + type: 'MustacheStatement', + path: { + type: 'PathExpression', + data: false, + depth: 0, + parts: [ 'value' ], + original: 'value' + }, + params: [], + hash: undefined, + escaped: true + } + ] +} +``` + +By default certain properties will be hidden in the output. +For more control over the output, check out the options by running the script without any arguments. + +### Print generated code + +It's possible to see the generated JavaScript code that `handlebars` create for a given template using the following command line tool: + +```sh +./node_modules/handlebars/print-script