From ef79eab44bfd000792c610b968b5ceefd41e76a0 Mon Sep 17 00:00:00 2001 From: Darius Jahandarie Date: Sat, 4 Nov 2023 18:45:57 +0900 Subject: Modernize codebase - Use ES modules - Remove vendored libs and build them from npm using esbuild - Switch from JSZip to zip.js --- dev/build-libs.js | 60 +- dev/build.js | 2 + dev/data/manifest-variants.json | 33 +- dev/lib/dexie-export-import.js | 17 + dev/lib/dexie.js | 17 + dev/lib/handlebars.js | 18 + dev/lib/handlebars/LICENSE | 29 + dev/lib/handlebars/README.md | 164 ++++ .../handlebars/__snapshots__/index.test.ts.snap | 91 ++ dev/lib/handlebars/index.test.ts | 567 ++++++++++++ dev/lib/handlebars/index.ts | 33 + dev/lib/handlebars/jest.config.js | 10 + dev/lib/handlebars/kibana.jsonc | 5 + dev/lib/handlebars/package.json | 6 + .../scripts/check_for_upstream_updates.sh | 45 + dev/lib/handlebars/scripts/print_ast.js | 64 ++ .../handlebars/scripts/update_upstream_git_hash.sh | 24 + dev/lib/handlebars/src/__jest__/test_bench.ts | 207 +++++ dev/lib/handlebars/src/handlebars.ts | 47 + dev/lib/handlebars/src/spec/.upstream_git_hash | 1 + dev/lib/handlebars/src/spec/index.basic.test.ts | 481 +++++++++++ dev/lib/handlebars/src/spec/index.blocks.test.ts | 366 ++++++++ dev/lib/handlebars/src/spec/index.builtins.test.ts | 676 +++++++++++++++ dev/lib/handlebars/src/spec/index.compiler.test.ts | 86 ++ dev/lib/handlebars/src/spec/index.data.test.ts | 269 ++++++ dev/lib/handlebars/src/spec/index.helpers.test.ts | 958 +++++++++++++++++++++ dev/lib/handlebars/src/spec/index.partials.test.ts | 591 +++++++++++++ .../handlebars/src/spec/index.regressions.test.ts | 379 ++++++++ dev/lib/handlebars/src/spec/index.security.test.ts | 132 +++ dev/lib/handlebars/src/spec/index.strict.test.ts | 164 ++++ .../src/spec/index.subexpressions.test.ts | 214 +++++ dev/lib/handlebars/src/spec/index.utils.test.ts | 24 + .../src/spec/index.whitespace_control.test.ts | 88 ++ dev/lib/handlebars/src/symbols.ts | 8 + dev/lib/handlebars/src/types.ts | 225 +++++ dev/lib/handlebars/src/utils.ts | 69 ++ dev/lib/handlebars/src/visitor.ts | 778 +++++++++++++++++ dev/lib/handlebars/tsconfig.json | 15 + dev/lib/parse5.js | 17 + dev/lib/ucs2length.js | 18 + dev/lib/wanakana.js | 17 + dev/lib/zip.js | 17 + 42 files changed, 6970 insertions(+), 62 deletions(-) create mode 100644 dev/lib/dexie-export-import.js create mode 100644 dev/lib/dexie.js create mode 100644 dev/lib/handlebars.js create mode 100644 dev/lib/handlebars/LICENSE create mode 100644 dev/lib/handlebars/README.md create mode 100644 dev/lib/handlebars/__snapshots__/index.test.ts.snap create mode 100644 dev/lib/handlebars/index.test.ts create mode 100644 dev/lib/handlebars/index.ts create mode 100644 dev/lib/handlebars/jest.config.js create mode 100644 dev/lib/handlebars/kibana.jsonc create mode 100644 dev/lib/handlebars/package.json create mode 100755 dev/lib/handlebars/scripts/check_for_upstream_updates.sh create mode 100755 dev/lib/handlebars/scripts/print_ast.js create mode 100755 dev/lib/handlebars/scripts/update_upstream_git_hash.sh create mode 100644 dev/lib/handlebars/src/__jest__/test_bench.ts create mode 100644 dev/lib/handlebars/src/handlebars.ts create mode 100644 dev/lib/handlebars/src/spec/.upstream_git_hash create mode 100644 dev/lib/handlebars/src/spec/index.basic.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.blocks.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.builtins.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.compiler.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.data.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.helpers.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.partials.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.regressions.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.security.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.strict.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.subexpressions.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.utils.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.whitespace_control.test.ts create mode 100644 dev/lib/handlebars/src/symbols.ts create mode 100644 dev/lib/handlebars/src/types.ts create mode 100644 dev/lib/handlebars/src/utils.ts create mode 100644 dev/lib/handlebars/src/visitor.ts create mode 100644 dev/lib/handlebars/tsconfig.json create mode 100644 dev/lib/parse5.js create mode 100644 dev/lib/ucs2length.js create mode 100644 dev/lib/wanakana.js create mode 100644 dev/lib/zip.js (limited to 'dev') diff --git a/dev/build-libs.js b/dev/build-libs.js index 36c07edd..497206c9 100644 --- a/dev/build-libs.js +++ b/dev/build-libs.js @@ -18,49 +18,35 @@ const fs = require('fs'); const path = require('path'); -const browserify = require('browserify'); +const esbuild = require('esbuild'); -async function buildParse5() { - const parse5Path = require.resolve('parse5'); - const cwd = process.cwd(); - try { - const baseDir = path.dirname(parse5Path); - process.chdir(baseDir); // This is necessary to ensure relative source map file names are consistent - return await new Promise((resolve, reject) => { - browserify({ - entries: [parse5Path], - standalone: 'parse5', - debug: true, - baseDir - }).bundle((error, result) => { - if (error) { - reject(error); - } else { - resolve(result); - } - }); - }); - } finally { - process.chdir(cwd); - } -} - -function getBuildTargets() { - const extLibPath = path.join(__dirname, '..', 'ext', 'lib'); - return [ - {path: path.join(extLibPath, 'parse5.js'), build: buildParse5} - ]; +async function buildLib(p) { + await esbuild.build({ + entryPoints: [p], + bundle: true, + minify: false, + sourcemap: true, + target: 'es2020', + format: 'esm', + outfile: path.join(__dirname, '..', 'ext', 'lib', path.basename(p)), + external: ['fs'] + }); } -async function main() { - for (const {path: path2, build} of getBuildTargets()) { - const content = await build(); - fs.writeFileSync(path2, content); +async function buildLibs() { + const devLibPath = path.join(__dirname, 'lib'); + const files = await fs.promises.readdir(devLibPath, { + withFileTypes: true + }); + for (const f of files) { + if (f.isFile()) { + await buildLib(path.join(devLibPath, f.name)); + } } } -if (require.main === module) { main(); } +if (require.main === module) { buildLibs(); } module.exports = { - getBuildTargets + buildLibs }; diff --git a/dev/build.js b/dev/build.js index 3bfb5418..1e6ef1d0 100644 --- a/dev/build.js +++ b/dev/build.js @@ -26,6 +26,7 @@ const {getAllFiles, getArgs, testMain} = util; const {ManifestUtil} = require('./manifest-util'); const Ajv = require('ajv'); const standaloneCode = require('ajv/dist/standalone').default; +const buildLibs = require('./build-libs.js').buildLibs; async function createZip(directory, excludeFiles, outputFileName, sevenZipExes, onUpdate, dryRun) { try { @@ -215,6 +216,7 @@ async function main(argv) { const manifestPath = path.join(extDir, 'manifest.json'); try { + await buildLibs(); const variantNames = ( argv.length === 0 || args.get('all') ? manifestUtil.getVariants().filter(({buildable}) => buildable !== false).map(({name}) => name) : diff --git a/dev/data/manifest-variants.json b/dev/data/manifest-variants.json index 1eae2112..d44251e1 100644 --- a/dev/data/manifest-variants.json +++ b/dev/data/manifest-variants.json @@ -28,7 +28,8 @@ "default_popup": "action-popup.html" }, "background": { - "service_worker": "sw.js" + "service_worker": "sw.js", + "type": "module" }, "content_scripts": [ { @@ -41,28 +42,7 @@ "match_about_blank": true, "all_frames": true, "js": [ - "js/core.js", - "js/yomichan.js", - "js/app/frontend.js", - "js/app/popup.js", - "js/app/popup-factory.js", - "js/app/popup-proxy.js", - "js/app/popup-window.js", - "js/app/theme-controller.js", - "js/comm/api.js", - "js/comm/cross-frame-api.js", - "js/comm/frame-ancestry-handler.js", - "js/comm/frame-client.js", - "js/comm/frame-offset-forwarder.js", - "js/data/sandbox/string-util.js", - "js/dom/dom-text-scanner.js", - "js/dom/document-util.js", - "js/dom/text-source-element.js", - "js/dom/text-source-range.js", - "js/input/hotkey-handler.js", - "js/language/text-scanner.js", - "js/script/dynamic-loader.js", - "js/app/content-script-main.js" + "js/app/content-script-wrapper.js" ] } ], @@ -118,7 +98,8 @@ { "resources": [ "popup.html", - "template-renderer.html" + "template-renderer.html", + "js/*" ], "matches": [ "" @@ -187,7 +168,9 @@ "path": [ "permissions" ], - "items": ["clipboardRead"] + "items": [ + "clipboardRead" + ] } ] }, diff --git a/dev/lib/dexie-export-import.js b/dev/lib/dexie-export-import.js new file mode 100644 index 00000000..8d2ec206 --- /dev/null +++ b/dev/lib/dexie-export-import.js @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2023 Yomitan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +export * from 'dexie-export-import'; diff --git a/dev/lib/dexie.js b/dev/lib/dexie.js new file mode 100644 index 00000000..aa3f2b7d --- /dev/null +++ b/dev/lib/dexie.js @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2023 Yomitan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +export * from 'dexie'; diff --git a/dev/lib/handlebars.js b/dev/lib/handlebars.js new file mode 100644 index 00000000..5b57efdd --- /dev/null +++ b/dev/lib/handlebars.js @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2023 Yomitan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +export {Handlebars} from './handlebars/src/handlebars.js'; + diff --git a/dev/lib/handlebars/LICENSE b/dev/lib/handlebars/LICENSE new file mode 100644 index 00000000..5d971a17 --- /dev/null +++ b/dev/lib/handlebars/LICENSE @@ -0,0 +1,29 @@ +The MIT License (MIT) + +Copyright (c) Elasticsearch BV +Copyright (c) Copyright (C) 2011-2019 by Yehuda Katz + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +This software consists of voluntary contributions made by many +individuals. For exact contribution history, see the revision history +available at the following locations: + - https://github.com/handlebars-lang/handlebars.js + - https://github.com/elastic/kibana/tree/main/packages/kbn-handlebars diff --git a/dev/lib/handlebars/README.md b/dev/lib/handlebars/README.md new file mode 100644 index 00000000..cc151645 --- /dev/null +++ b/dev/lib/handlebars/README.md @@ -0,0 +1,164 @@ +# @kbn/handlebars + +A custom version of the handlebars package which, to improve security, does not use `eval` or `new Function`. This means that templates can't be compiled into JavaScript functions in advance and hence, rendering the templates is a lot slower. + +## Limitations + +- Only the following compile options are supported: + - `data` + - `knownHelpers` + - `knownHelpersOnly` + - `noEscape` + - `strict` + - `assumeObjects` + - `preventIndent` + - `explicitPartialContext` + +- Only the following runtime options are supported: + - `data` + - `helpers` + - `partials` + - `decorators` (not documented in the official Handlebars [runtime options documentation](https://handlebarsjs.com/api-reference/runtime-options.html)) + - `blockParams` (not documented in the official Handlebars [runtime options documentation](https://handlebarsjs.com/api-reference/runtime-options.html)) + +## Implementation differences + +The standard `handlebars` implementation: + +1. When given a template string, e.g. `Hello {{x}}`, return a "render" function which takes an "input" object, e.g. `{ x: 'World' }`. +1. The first time the "render" function is called the following happens: + 1. Turn the template string into an Abstract Syntax Tree (AST). + 1. Convert the AST into a hyper optimized JavaScript function which takes the input object as an argument. + 1. Call the generate JavaScript function with the given "input" object to produce and return the final output string (`Hello World`). +1. Subsequent calls to the "render" function will re-use the already generated JavaScript function. + +The custom `@kbn/handlebars` implementation: + +1. When given a template string, e.g. `Hello {{x}}`, return a "render" function which takes an "input" object, e.g. `{ x: 'World' }`. +1. The first time the "render" function is called the following happens: + 1. Turn the template string into an Abstract Syntax Tree (AST). + 1. Process the AST with the given "input" object to produce and return the final output string (`Hello World`). +1. Subsequent calls to the "render" function will re-use the already generated AST. + +_Note: Not parsing of the template string until the first call to the "render" function is deliberate as it mimics the original `handlebars` implementation. This means that any errors that occur due to an invalid template string will not be thrown until the first call to the "render" function._ + +## Technical details + +The `handlebars` library exposes the API for both [generating the AST](https://github.com/handlebars-lang/handlebars.js/blob/master/docs/compiler-api.md#ast) and walking it by implementing the [Visitor API](https://github.com/handlebars-lang/handlebars.js/blob/master/docs/compiler-api.md#ast-visitor). We can leverage that to our advantage and create our own "render" function, which internally calls this API to generate the AST and then the API to walk the AST. + +The `@kbn/handlebars` implementation of the `Visitor` class implements all the necessary methods called by the parent `Visitor` code when instructed to walk the AST. They all start with an upppercase letter, e.g. `MustacheStatement` or `SubExpression`. We call this class `ElasticHandlebarsVisitor`. + +To parse the template string to an AST representation, we call `Handlebars.parse(templateString)`, which returns an AST object. + +The AST object contains a bunch of nodes, one for each element of the template string, all arranged in a tree-like structure. The root of the AST object is a node of type `Program`. This is a special node, which we do not need to worry about, but each of its direct children has a type named like the method which will be called when the walking algorithm reaches that node, e.g. `ContentStatement` or `BlockStatement`. These are the methods that our `Visitor` implementation implements. + +To instruct our `ElasticHandlebarsVisitor` class to start walking the AST object, we call the `accept()` method inherited from the parent `Visitor` class with the main AST object. The `Visitor` will walk each node in turn that is directly attached to the root `Program` node. For each node it traverses, it will call the matching method in our `ElasticHandlebarsVisitor` class. + +To instruct the `Visitor` code to traverse any child nodes of a given node, our implementation needs to manually call `accept(childNode)`, `acceptArray(arrayOfChildNodes)`, `acceptKey(node, childKeyName)`, or `acceptRequired(node, childKeyName)` from within any of the "node" methods, otherwise the child nodes are ignored. + +### State + +We keep state internally in the `ElasticHandlebarsVisitor` object using the following private properties: + +- `contexts`: An array (stack) of `context` objects. In a simple template this array will always only contain a single element: The main `context` object. In more complicated scenarios, new `context` objects will be pushed and popped to and from the `contexts` stack as needed. +- `output`: An array containing the "rendered" output of each node (normally just one element per node). In the most simple template, this is simply joined together into a the final output string after the AST has been traversed. In more complicated templates, we use this array temporarily to collect parameters to give to helper functions (see the `getParams` function). + +## Testing + +The tests for `@kbn/handlebars` are integrated into the regular test suite of Kibana and are all jest tests. To run them all, simply do: + +```sh +node scripts/jest packages/kbn-handlebars +``` + +By default, each test will run both the original `handlebars` code and the modified `@kbn/handlebars` code to compare if the output of the two are identical. To isolate a test run to just one or the other, you can use the following environment variables: + +- `EVAL=1` - Set to only run the original `handlebars` implementation that uses `eval`. +- `AST=1` - Set to only run the modified `@kbn/handlebars` implementation that doesn't use `eval`. + +## Development + +Some of the tests have been copied from the upstream `handlebars` project and modified to fit our use-case, test-suite, and coding conventions. They are all located under the `packages/kbn-handlebars/src/spec` directory. To check if any of the copied files have received updates upstream that we might want to include in our copies, you can run the following script: + +```sh +./packages/kbn-handlebars/scripts/check_for_upstream_updates.sh +``` + +_Note: This will look for changes in the `4.x` branch of the `handlebars.js` repo only. Changes in the `master` branch are ignored._ + +Once all updates have been manually merged with our versions of the files, run the following script to "lock" us into the new updates: + +```sh +./packages/kbn-handlebars/scripts/update_upstream_git_hash.sh +``` + +This will update file `packages/kbn-handlebars/src/spec/.upstream_git_hash`. Make sure to commit changes to this file as well. + +## Debugging + +### Print AST + +To output the generated AST object structure in a somewhat readable form, use the following script: + +```sh +./packages/kbn-handlebars/scripts/print_ast.js +``` + +Example: + +```sh +./packages/kbn-handlebars/scripts/print_ast.js '{{value}}' +``` + +Output: + +```js +{ + type: 'Program', + body: [ + { + type: 'MustacheStatement', + path: { + type: 'PathExpression', + data: false, + depth: 0, + parts: [ 'value' ], + original: 'value' + }, + params: [], + hash: undefined, + escaped: true + } + ] +} +``` + +By default certain properties will be hidden in the output. +For more control over the output, check out the options by running the script without any arguments. + +### Print generated code + +It's possible to see the generated JavaScript code that `handlebars` create for a given template using the following command line tool: + +```sh +./node_modules/handlebars/print-script