From 376151096431d4362e4baaacf0cef4a534e169f7 Mon Sep 17 00:00:00 2001 From: Darius Jahandarie Date: Fri, 3 Nov 2023 23:32:33 +0900 Subject: Replace JsonSchema with ajv for dictionary validation --- ext/js/language/dictionary-importer.js | 99 +++++++++------------------------- 1 file changed, 25 insertions(+), 74 deletions(-) (limited to 'ext/js') diff --git a/ext/js/language/dictionary-importer.js b/ext/js/language/dictionary-importer.js index 718d9f1c..0cf3d5f5 100644 --- a/ext/js/language/dictionary-importer.js +++ b/ext/js/language/dictionary-importer.js @@ -18,7 +18,6 @@ /* global * JSZip - * JsonSchema * MediaUtil */ @@ -51,8 +50,10 @@ class DictionaryImporter { const index = JSON.parse(await indexFile.async('string')); - const indexSchema = await this._getSchema('/data/schemas/dictionary-index-schema.json'); - this._validateJsonSchema(index, indexSchema, indexFileName); + const ajvSchemas = await import('/lib/validate-schemas.js'); + if (!ajvSchemas.dictionaryIndex(index)) { + throw this._formatAjvSchemaError(ajvSchemas.dictionaryIndex, indexFileName); + } const dictionaryTitle = index.title; const version = index.format || index.version; @@ -75,8 +76,7 @@ class DictionaryImporter { // Load schemas this._progressNextStep(0); - const dataBankSchemaPaths = this._getDataBankSchemaPaths(version); - const dataBankSchemas = await Promise.all(dataBankSchemaPaths.map((path) => this._getSchema(path))); + const dataBankSchemas = this._getDataBankSchemas(version); // Files const termFiles = this._getArchiveFiles(archive, 'term_bank_?.json'); @@ -87,11 +87,11 @@ class DictionaryImporter { // Load data this._progressNextStep(termFiles.length + termMetaFiles.length + kanjiFiles.length + kanjiMetaFiles.length + tagFiles.length); - const termList = await this._readFileSequence(termFiles, convertTermBankEntry, dataBankSchemas[0], dictionaryTitle); - const termMetaList = await this._readFileSequence(termMetaFiles, convertTermMetaBankEntry, dataBankSchemas[1], dictionaryTitle); - const kanjiList = await this._readFileSequence(kanjiFiles, convertKanjiBankEntry, dataBankSchemas[2], dictionaryTitle); - const kanjiMetaList = await this._readFileSequence(kanjiMetaFiles, convertKanjiMetaBankEntry, dataBankSchemas[3], dictionaryTitle); - const tagList = await this._readFileSequence(tagFiles, convertTagBankEntry, dataBankSchemas[4], dictionaryTitle); + const termList = await this._readFileSequence(ajvSchemas, termFiles, convertTermBankEntry, dataBankSchemas[0], dictionaryTitle); + const termMetaList = await this._readFileSequence(ajvSchemas, termMetaFiles, convertTermMetaBankEntry, dataBankSchemas[1], dictionaryTitle); + const kanjiList = await this._readFileSequence(ajvSchemas, kanjiFiles, convertKanjiBankEntry, dataBankSchemas[2], dictionaryTitle); + const kanjiMetaList = await this._readFileSequence(ajvSchemas, kanjiMetaFiles, convertKanjiMetaBankEntry, dataBankSchemas[3], dictionaryTitle); + const tagList = await this._readFileSequence(ajvSchemas, tagFiles, convertTagBankEntry, dataBankSchemas[4], dictionaryTitle); this._addOldIndexTags(index, tagList, dictionaryTitle); // Prefix wildcard support @@ -214,68 +214,27 @@ class DictionaryImporter { return summary; } - async _getSchema(fileName) { - const schema = await this._fetchJsonAsset(fileName); - return new JsonSchema(schema); - } - - _validateJsonSchema(value, schema, fileName) { - try { - schema.validate(value); - } catch (e) { - throw this._formatSchemaError(e, fileName); - } - } - - _formatSchemaError(e, fileName) { - const valuePathString = this._getSchemaErrorPathString(e.valueStack, 'dictionary'); - const schemaPathString = this._getSchemaErrorPathString(e.schemaStack, 'schema'); - - const e2 = new Error(`Dictionary has invalid data in '${fileName}' for value '${valuePathString}', validated against '${schemaPathString}': ${e.message}`); - e2.data = e; + _formatAjvSchemaError(schema, fileName) { + const e2 = new Error(`Dictionary has invalid data in '${fileName}'`); + e2.data = schema.errors; return e2; } - _getSchemaErrorPathString(infoList, base='') { - let result = base; - for (const {path} of infoList) { - const pathArray = Array.isArray(path) ? path : [path]; - for (const pathPart of pathArray) { - if (pathPart === null) { - result = base; - } else { - switch (typeof pathPart) { - case 'string': - if (result.length > 0) { - result += '.'; - } - result += pathPart; - break; - case 'number': - result += `[${pathPart}]`; - break; - } - } - } - } - return result; - } - - _getDataBankSchemaPaths(version) { + _getDataBankSchemas(version) { const termBank = ( version === 1 ? - '/data/schemas/dictionary-term-bank-v1-schema.json' : - '/data/schemas/dictionary-term-bank-v3-schema.json' + 'dictionaryTermBankV1' : + 'dictionaryTermBankV3' ); - const termMetaBank = '/data/schemas/dictionary-term-meta-bank-v3-schema.json'; + const termMetaBank = 'dictionaryTermMetaBankV3'; const kanjiBank = ( version === 1 ? - '/data/schemas/dictionary-kanji-bank-v1-schema.json' : - '/data/schemas/dictionary-kanji-bank-v3-schema.json' + 'dictionaryKanjiBankV1' : + 'dictionaryKanjiBankV3' ); - const kanjiMetaBank = '/data/schemas/dictionary-kanji-meta-bank-v3-schema.json'; - const tagBank = '/data/schemas/dictionary-tag-bank-v3-schema.json'; + const kanjiMetaBank = 'dictionaryKanjiMetaBankV3'; + const tagBank = 'dictionaryTagBankV3'; return [termBank, termMetaBank, kanjiBank, kanjiMetaBank, tagBank]; } @@ -539,28 +498,20 @@ class DictionaryImporter { return results; } - async _readFileSequence(files, convertEntry, schema, dictionaryTitle) { + async _readFileSequence(ajvSchemas, files, convertEntry, schemaName, dictionaryTitle) { const progressData = this._progressData; - let count = 0; let startIndex = 0; - if (typeof this._onProgress === 'function') { - schema.progressInterval = 1000; - schema.progress = (s) => { - const index = s.getValueStackLength() > 1 ? s.getValueStackItem(1).path : 0; - progressData.index = startIndex + (index / count); - this._progress(); - }; - } const results = []; for (const file of files) { const entries = JSON.parse(await file.async('string')); - count = Array.isArray(entries) ? Math.max(entries.length, 1) : 1; startIndex = progressData.index; this._progress(); - this._validateJsonSchema(entries, schema, file.name); + if (!ajvSchemas[schemaName](entries)) { + throw this._formatAjvSchemaError(ajvSchemas[schemaName], file.name); + } progressData.index = startIndex + 1; this._progress(); -- cgit v1.2.3 From ef79eab44bfd000792c610b968b5ceefd41e76a0 Mon Sep 17 00:00:00 2001 From: Darius Jahandarie Date: Sat, 4 Nov 2023 18:45:57 +0900 Subject: Modernize codebase - Use ES modules - Remove vendored libs and build them from npm using esbuild - Switch from JSZip to zip.js --- .eslintrc.json | 33 +- .gitignore | 2 +- .vscode/settings.json | 7 +- dev/build-libs.js | 60 +- dev/build.js | 2 + dev/data/manifest-variants.json | 33 +- dev/lib/dexie-export-import.js | 17 + dev/lib/dexie.js | 17 + dev/lib/handlebars.js | 18 + dev/lib/handlebars/LICENSE | 29 + dev/lib/handlebars/README.md | 164 + .../handlebars/__snapshots__/index.test.ts.snap | 91 + dev/lib/handlebars/index.test.ts | 567 ++ dev/lib/handlebars/index.ts | 33 + dev/lib/handlebars/jest.config.js | 10 + dev/lib/handlebars/kibana.jsonc | 5 + dev/lib/handlebars/package.json | 6 + .../scripts/check_for_upstream_updates.sh | 45 + dev/lib/handlebars/scripts/print_ast.js | 64 + .../handlebars/scripts/update_upstream_git_hash.sh | 24 + dev/lib/handlebars/src/__jest__/test_bench.ts | 207 + dev/lib/handlebars/src/handlebars.ts | 47 + dev/lib/handlebars/src/spec/.upstream_git_hash | 1 + dev/lib/handlebars/src/spec/index.basic.test.ts | 481 ++ dev/lib/handlebars/src/spec/index.blocks.test.ts | 366 + dev/lib/handlebars/src/spec/index.builtins.test.ts | 676 ++ dev/lib/handlebars/src/spec/index.compiler.test.ts | 86 + dev/lib/handlebars/src/spec/index.data.test.ts | 269 + dev/lib/handlebars/src/spec/index.helpers.test.ts | 958 +++ dev/lib/handlebars/src/spec/index.partials.test.ts | 591 ++ .../handlebars/src/spec/index.regressions.test.ts | 379 + dev/lib/handlebars/src/spec/index.security.test.ts | 132 + dev/lib/handlebars/src/spec/index.strict.test.ts | 164 + .../src/spec/index.subexpressions.test.ts | 214 + dev/lib/handlebars/src/spec/index.utils.test.ts | 24 + .../src/spec/index.whitespace_control.test.ts | 88 + dev/lib/handlebars/src/symbols.ts | 8 + dev/lib/handlebars/src/types.ts | 225 + dev/lib/handlebars/src/utils.ts | 69 + dev/lib/handlebars/src/visitor.ts | 778 ++ dev/lib/handlebars/tsconfig.json | 15 + dev/lib/parse5.js | 17 + dev/lib/ucs2length.js | 18 + dev/lib/wanakana.js | 17 + dev/lib/zip.js | 17 + ext/action-popup.html | 13 +- ext/background.html | 38 +- ext/info.html | 16 +- ext/issues.html | 6 +- ext/js/accessibility/accessibility-controller.js | 5 +- ext/js/accessibility/google-docs-util.js | 9 +- ext/js/app/content-script-main.js | 10 +- ext/js/app/content-script-wrapper.js | 24 + ext/js/app/frontend.js | 18 +- ext/js/app/popup-factory.js | 14 +- ext/js/app/popup-proxy.js | 7 +- ext/js/app/popup-window.js | 6 +- ext/js/app/popup.js | 14 +- ext/js/app/theme-controller.js | 2 +- ext/js/background/backend.js | 46 +- ext/js/background/background-main.js | 5 +- ext/js/background/offscreen-main.js | 4 +- ext/js/background/offscreen.js | 8 +- ext/js/background/profile-conditions-util.js | 6 +- ext/js/background/request-builder.js | 2 +- ext/js/background/script-manager.js | 3 +- ext/js/comm/anki-connect.js | 7 +- ext/js/comm/api.js | 4 +- ext/js/comm/clipboard-monitor.js | 4 +- ext/js/comm/clipboard-reader.js | 6 +- ext/js/comm/cross-frame-api.js | 5 +- ext/js/comm/frame-ancestry-handler.js | 5 +- ext/js/comm/frame-client.js | 4 +- ext/js/comm/frame-endpoint.js | 5 +- ext/js/comm/frame-offset-forwarder.js | 7 +- ext/js/comm/mecab.js | 4 +- ext/js/core.js | 34 +- ext/js/data/anki-note-builder.js | 10 +- ext/js/data/anki-util.js | 4 +- ext/js/data/database.js | 2 +- ext/js/data/json-schema.js | 7 +- ext/js/data/options-util.js | 9 +- ext/js/data/permissions-util.js | 6 +- ext/js/data/sandbox/anki-note-data-creator.js | 6 +- ext/js/data/sandbox/array-buffer-util.js | 2 +- ext/js/data/sandbox/string-util.js | 2 +- ext/js/debug/timer.js | 2 +- ext/js/display/display-anki.js | 12 +- ext/js/display/display-audio.js | 7 +- ext/js/display/display-content-manager.js | 8 +- ext/js/display/display-generator.js | 16 +- ext/js/display/display-history.js | 4 +- ext/js/display/display-notification.js | 4 +- ext/js/display/display-profile-selection.js | 8 +- ext/js/display/display-resizer.js | 4 +- ext/js/display/display.js | 42 +- ext/js/display/element-overflow-controller.js | 4 +- ext/js/display/option-toggle-hotkey-handler.js | 5 +- ext/js/display/popup-main.js | 11 + ext/js/display/query-parser.js | 8 +- ext/js/display/sandbox/pronunciation-generator.js | 2 +- .../sandbox/structured-content-generator.js | 2 +- ext/js/display/search-action-popup-controller.js | 2 +- ext/js/display/search-display-controller.js | 10 +- ext/js/display/search-main.js | 24 +- .../display/search-persistent-state-controller.js | 4 +- ext/js/dom/document-focus-controller.js | 2 +- ext/js/dom/document-util.js | 11 +- ext/js/dom/dom-data-binder.js | 10 +- ext/js/dom/dom-text-scanner.js | 6 +- ext/js/dom/html-template-collection.js | 2 +- ext/js/dom/native-simple-dom-parser.js | 2 +- ext/js/dom/panel-element.js | 4 +- ext/js/dom/popup-menu.js | 4 +- ext/js/dom/sandbox/css-style-applier.js | 2 +- ext/js/dom/scroll-element.js | 2 +- ext/js/dom/selector-observer.js | 2 +- ext/js/dom/simple-dom-parser.js | 6 +- ext/js/dom/text-source-element.js | 9 +- ext/js/dom/text-source-range.js | 9 +- ext/js/extension/environment.js | 2 +- ext/js/general/cache-map.js | 3 +- ext/js/general/object-property-accessor.js | 2 +- ext/js/general/regex-util.js | 3 +- ext/js/general/task-accumulator.js | 4 +- ext/js/general/text-source-map.js | 2 +- ext/js/input/hotkey-handler.js | 8 +- ext/js/input/hotkey-help-controller.js | 8 +- ext/js/input/hotkey-util.js | 2 +- ext/js/language/deinflector.js | 2 +- ext/js/language/dictionary-database.js | 7 +- .../language/dictionary-importer-media-loader.js | 4 +- ext/js/language/dictionary-importer.js | 83 +- ext/js/language/dictionary-worker-handler.js | 11 +- ext/js/language/dictionary-worker-main.js | 18 +- ext/js/language/dictionary-worker-media-loader.js | 4 +- ext/js/language/dictionary-worker.js | 9 +- ext/js/language/sandbox/dictionary-data-util.js | 2 +- ext/js/language/sandbox/japanese-util.js | 2 +- ext/js/language/text-scanner.js | 8 +- ext/js/language/translator.js | 11 +- ext/js/media/audio-downloader.js | 14 +- ext/js/media/audio-system.js | 7 +- ext/js/media/media-util.js | 2 +- ext/js/media/text-to-speech-audio.js | 2 +- ext/js/pages/action-popup-main.js | 9 +- .../pages/common/extension-content-controller.js | 6 +- ext/js/pages/generic-page-main.js | 6 +- ext/js/pages/info-main.js | 10 +- ext/js/pages/permissions-main.js | 20 +- ext/js/pages/settings/anki-controller.js | 16 +- ext/js/pages/settings/anki-templates-controller.js | 10 +- ext/js/pages/settings/audio-controller.js | 7 +- ext/js/pages/settings/backup-controller.js | 16 +- .../settings/collapsible-dictionary-controller.js | 7 +- ext/js/pages/settings/dictionary-controller.js | 6 +- .../pages/settings/dictionary-import-controller.js | 10 +- .../extension-keyboard-shortcuts-controller.js | 10 +- .../pages/settings/generic-setting-controller.js | 9 +- .../pages/settings/keyboard-mouse-input-field.js | 9 +- .../settings/keyboard-shortcuts-controller.js | 12 +- ext/js/pages/settings/mecab-controller.js | 4 +- ext/js/pages/settings/modal-controller.js | 6 +- ext/js/pages/settings/modal.js | 6 +- ext/js/pages/settings/nested-popups-controller.js | 6 +- .../settings/permissions-origin-controller.js | 4 +- .../settings/permissions-toggle-controller.js | 6 +- .../settings/persistent-storage-controller.js | 5 +- ext/js/pages/settings/popup-preview-controller.js | 2 +- ext/js/pages/settings/popup-preview-frame-main.js | 6 + ext/js/pages/settings/popup-preview-frame.js | 11 +- ext/js/pages/settings/popup-window-controller.js | 4 +- ext/js/pages/settings/profile-conditions-ui.js | 7 +- ext/js/pages/settings/profile-controller.js | 8 +- .../settings/recommended-permissions-controller.js | 4 +- ext/js/pages/settings/scan-inputs-controller.js | 8 +- .../settings/scan-inputs-simple-controller.js | 11 +- .../secondary-search-dictionary-controller.js | 7 +- .../sentence-termination-characters-controller.js | 4 +- ext/js/pages/settings/settings-controller.js | 12 +- .../pages/settings/settings-display-controller.js | 10 +- ext/js/pages/settings/settings-main.js | 64 +- .../sort-frequency-dictionary-controller.js | 4 +- ext/js/pages/settings/status-footer.js | 6 +- ext/js/pages/settings/storage-controller.js | 4 +- .../translation-text-replacements-controller.js | 4 +- ext/js/pages/welcome-main.js | 26 +- ext/js/script/dynamic-loader-sentinel.js | 2 + ext/js/script/dynamic-loader.js | 5 +- .../anki-template-renderer-content-manager.js | 2 +- ext/js/templates/sandbox/anki-template-renderer.js | 12 +- .../sandbox/template-renderer-frame-api.js | 2 +- .../sandbox/template-renderer-frame-main.js | 6 +- .../sandbox/template-renderer-media-provider.js | 2 +- ext/js/templates/sandbox/template-renderer.js | 10 +- ext/js/templates/template-patcher.js | 2 +- ext/js/templates/template-renderer-proxy.js | 4 +- ext/js/yomichan.js | 9 +- ext/legal.html | 6 +- ext/lib/dexie-export-import.js | 3457 -------- ext/lib/dexie-export-import.js.map | 1 - ext/lib/dexie.min.js | 2 - ext/lib/dexie.min.js.map | 1 - ext/lib/handlebars.min.js | 25 - ext/lib/handlebars.min.js.map | 7 - ext/lib/jszip.min.js | 13 - ext/lib/parse5.js | 8557 -------------------- ext/lib/ucs2length.js | 16 - ext/lib/wanakana.min.js | 2 - ext/lib/wanakana.min.js.map | 1 - ext/offscreen.html | 9 +- ext/permissions.html | 26 +- ext/popup-preview.html | 27 +- ext/popup.html | 49 +- ext/search.html | 51 +- ext/settings.html | 75 +- ext/sw.js | 39 +- ext/template-renderer.html | 16 +- ext/welcome.html | 38 +- jsconfig.json | 11 + package-lock.json | 806 +- package.json | 9 +- 222 files changed, 8491 insertions(+), 13144 deletions(-) create mode 100644 dev/lib/dexie-export-import.js create mode 100644 dev/lib/dexie.js create mode 100644 dev/lib/handlebars.js create mode 100644 dev/lib/handlebars/LICENSE create mode 100644 dev/lib/handlebars/README.md create mode 100644 dev/lib/handlebars/__snapshots__/index.test.ts.snap create mode 100644 dev/lib/handlebars/index.test.ts create mode 100644 dev/lib/handlebars/index.ts create mode 100644 dev/lib/handlebars/jest.config.js create mode 100644 dev/lib/handlebars/kibana.jsonc create mode 100644 dev/lib/handlebars/package.json create mode 100755 dev/lib/handlebars/scripts/check_for_upstream_updates.sh create mode 100755 dev/lib/handlebars/scripts/print_ast.js create mode 100755 dev/lib/handlebars/scripts/update_upstream_git_hash.sh create mode 100644 dev/lib/handlebars/src/__jest__/test_bench.ts create mode 100644 dev/lib/handlebars/src/handlebars.ts create mode 100644 dev/lib/handlebars/src/spec/.upstream_git_hash create mode 100644 dev/lib/handlebars/src/spec/index.basic.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.blocks.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.builtins.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.compiler.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.data.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.helpers.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.partials.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.regressions.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.security.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.strict.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.subexpressions.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.utils.test.ts create mode 100644 dev/lib/handlebars/src/spec/index.whitespace_control.test.ts create mode 100644 dev/lib/handlebars/src/symbols.ts create mode 100644 dev/lib/handlebars/src/types.ts create mode 100644 dev/lib/handlebars/src/utils.ts create mode 100644 dev/lib/handlebars/src/visitor.ts create mode 100644 dev/lib/handlebars/tsconfig.json create mode 100644 dev/lib/parse5.js create mode 100644 dev/lib/ucs2length.js create mode 100644 dev/lib/wanakana.js create mode 100644 dev/lib/zip.js create mode 100644 ext/js/app/content-script-wrapper.js delete mode 100644 ext/lib/dexie-export-import.js delete mode 100644 ext/lib/dexie-export-import.js.map delete mode 100644 ext/lib/dexie.min.js delete mode 100644 ext/lib/dexie.min.js.map delete mode 100644 ext/lib/handlebars.min.js delete mode 100644 ext/lib/handlebars.min.js.map delete mode 100644 ext/lib/jszip.min.js delete mode 100644 ext/lib/parse5.js delete mode 100644 ext/lib/ucs2length.js delete mode 100644 ext/lib/wanakana.min.js delete mode 100644 ext/lib/wanakana.min.js.map create mode 100644 jsconfig.json (limited to 'ext/js') diff --git a/.eslintrc.json b/.eslintrc.json index a7fb842b..99c2383a 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -6,7 +6,7 @@ ], "parserOptions": { "ecmaVersion": 11, - "sourceType": "script", + "sourceType": "module", "ecmaFeatures": { "globalReturn": false, "impliedStrict": true @@ -50,6 +50,7 @@ "no-case-declarations": "error", "no-const-assign": "error", "no-constant-condition": "off", + "no-console": "warn", "no-global-assign": "error", "no-param-reassign": "off", "no-prototype-builtins": "error", @@ -250,6 +251,7 @@ "jsdoc/multiline-blocks": "error", "jsdoc/no-bad-blocks": "error", "jsdoc/no-multi-asterisks": "error", + "jsdoc/no-undefined-types": 1, "jsdoc/require-asterisk-prefix": "error", "jsdoc/require-hyphen-before-param-description": [ "error", @@ -384,26 +386,7 @@ "ext/js/accessibility/google-docs.js", "ext/js/**/sandbox/**/*.js" ], - "globals": { - "serializeError": "readonly", - "deserializeError": "readonly", - "isObject": "readonly", - "stringReverse": "readonly", - "promiseTimeout": "readonly", - "escapeRegExp": "readonly", - "deferPromise": "readonly", - "clone": "readonly", - "deepEqual": "readonly", - "generateId": "readonly", - "promiseAnimationFrame": "readonly", - "invokeMessageHandler": "readonly", - "log": "readonly", - "DynamicProperty": "readonly", - "EventDispatcher": "readonly", - "EventListenerCollection": "readonly", - "Logger": "readonly", - "import": "readonly" - } + "globals": {} }, { "files": [ @@ -446,6 +429,14 @@ "webextensions": false } }, + { + "files": [ + "ext/js/language/dictionary-worker-main.js" + ], + "parserOptions": { + "sourceType": "module" + } + }, { "files": [ "playwright.config.js" diff --git a/.gitignore b/.gitignore index 426db4ad..7d085ebf 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,4 @@ dictionaries/ /playwright/.cache/ /test/playwright/__screenshots__/ ext/manifest.json -ext/lib/validate-schemas.js +ext/lib/ diff --git a/.vscode/settings.json b/.vscode/settings.json index 9318d9f6..2480961b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,10 +1,13 @@ { "markdown.extension.toc.levels": "1..3", "editor.codeActionsOnSave": { - "source.fixAll.eslint": true + "source.addMissingImports": true, + "source.organizeImports": true, + "source.fixAll.eslint": true, }, "eslint.format.enable": true, "playwright.env": { "PW_EXPERIMENTAL_SERVICE_WORKER_NETWORK_EVENTS": 1 - } + }, + "javascript.preferences.importModuleSpecifierEnding": "js", } diff --git a/dev/build-libs.js b/dev/build-libs.js index 36c07edd..497206c9 100644 --- a/dev/build-libs.js +++ b/dev/build-libs.js @@ -18,49 +18,35 @@ const fs = require('fs'); const path = require('path'); -const browserify = require('browserify'); +const esbuild = require('esbuild'); -async function buildParse5() { - const parse5Path = require.resolve('parse5'); - const cwd = process.cwd(); - try { - const baseDir = path.dirname(parse5Path); - process.chdir(baseDir); // This is necessary to ensure relative source map file names are consistent - return await new Promise((resolve, reject) => { - browserify({ - entries: [parse5Path], - standalone: 'parse5', - debug: true, - baseDir - }).bundle((error, result) => { - if (error) { - reject(error); - } else { - resolve(result); - } - }); - }); - } finally { - process.chdir(cwd); - } -} - -function getBuildTargets() { - const extLibPath = path.join(__dirname, '..', 'ext', 'lib'); - return [ - {path: path.join(extLibPath, 'parse5.js'), build: buildParse5} - ]; +async function buildLib(p) { + await esbuild.build({ + entryPoints: [p], + bundle: true, + minify: false, + sourcemap: true, + target: 'es2020', + format: 'esm', + outfile: path.join(__dirname, '..', 'ext', 'lib', path.basename(p)), + external: ['fs'] + }); } -async function main() { - for (const {path: path2, build} of getBuildTargets()) { - const content = await build(); - fs.writeFileSync(path2, content); +async function buildLibs() { + const devLibPath = path.join(__dirname, 'lib'); + const files = await fs.promises.readdir(devLibPath, { + withFileTypes: true + }); + for (const f of files) { + if (f.isFile()) { + await buildLib(path.join(devLibPath, f.name)); + } } } -if (require.main === module) { main(); } +if (require.main === module) { buildLibs(); } module.exports = { - getBuildTargets + buildLibs }; diff --git a/dev/build.js b/dev/build.js index 3bfb5418..1e6ef1d0 100644 --- a/dev/build.js +++ b/dev/build.js @@ -26,6 +26,7 @@ const {getAllFiles, getArgs, testMain} = util; const {ManifestUtil} = require('./manifest-util'); const Ajv = require('ajv'); const standaloneCode = require('ajv/dist/standalone').default; +const buildLibs = require('./build-libs.js').buildLibs; async function createZip(directory, excludeFiles, outputFileName, sevenZipExes, onUpdate, dryRun) { try { @@ -215,6 +216,7 @@ async function main(argv) { const manifestPath = path.join(extDir, 'manifest.json'); try { + await buildLibs(); const variantNames = ( argv.length === 0 || args.get('all') ? manifestUtil.getVariants().filter(({buildable}) => buildable !== false).map(({name}) => name) : diff --git a/dev/data/manifest-variants.json b/dev/data/manifest-variants.json index 1eae2112..d44251e1 100644 --- a/dev/data/manifest-variants.json +++ b/dev/data/manifest-variants.json @@ -28,7 +28,8 @@ "default_popup": "action-popup.html" }, "background": { - "service_worker": "sw.js" + "service_worker": "sw.js", + "type": "module" }, "content_scripts": [ { @@ -41,28 +42,7 @@ "match_about_blank": true, "all_frames": true, "js": [ - "js/core.js", - "js/yomichan.js", - "js/app/frontend.js", - "js/app/popup.js", - "js/app/popup-factory.js", - "js/app/popup-proxy.js", - "js/app/popup-window.js", - "js/app/theme-controller.js", - "js/comm/api.js", - "js/comm/cross-frame-api.js", - "js/comm/frame-ancestry-handler.js", - "js/comm/frame-client.js", - "js/comm/frame-offset-forwarder.js", - "js/data/sandbox/string-util.js", - "js/dom/dom-text-scanner.js", - "js/dom/document-util.js", - "js/dom/text-source-element.js", - "js/dom/text-source-range.js", - "js/input/hotkey-handler.js", - "js/language/text-scanner.js", - "js/script/dynamic-loader.js", - "js/app/content-script-main.js" + "js/app/content-script-wrapper.js" ] } ], @@ -118,7 +98,8 @@ { "resources": [ "popup.html", - "template-renderer.html" + "template-renderer.html", + "js/*" ], "matches": [ "" @@ -187,7 +168,9 @@ "path": [ "permissions" ], - "items": ["clipboardRead"] + "items": [ + "clipboardRead" + ] } ] }, diff --git a/dev/lib/dexie-export-import.js b/dev/lib/dexie-export-import.js new file mode 100644 index 00000000..8d2ec206 --- /dev/null +++ b/dev/lib/dexie-export-import.js @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2023 Yomitan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +export * from 'dexie-export-import'; diff --git a/dev/lib/dexie.js b/dev/lib/dexie.js new file mode 100644 index 00000000..aa3f2b7d --- /dev/null +++ b/dev/lib/dexie.js @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2023 Yomitan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +export * from 'dexie'; diff --git a/dev/lib/handlebars.js b/dev/lib/handlebars.js new file mode 100644 index 00000000..5b57efdd --- /dev/null +++ b/dev/lib/handlebars.js @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2023 Yomitan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +export {Handlebars} from './handlebars/src/handlebars.js'; + diff --git a/dev/lib/handlebars/LICENSE b/dev/lib/handlebars/LICENSE new file mode 100644 index 00000000..5d971a17 --- /dev/null +++ b/dev/lib/handlebars/LICENSE @@ -0,0 +1,29 @@ +The MIT License (MIT) + +Copyright (c) Elasticsearch BV +Copyright (c) Copyright (C) 2011-2019 by Yehuda Katz + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +This software consists of voluntary contributions made by many +individuals. For exact contribution history, see the revision history +available at the following locations: + - https://github.com/handlebars-lang/handlebars.js + - https://github.com/elastic/kibana/tree/main/packages/kbn-handlebars diff --git a/dev/lib/handlebars/README.md b/dev/lib/handlebars/README.md new file mode 100644 index 00000000..cc151645 --- /dev/null +++ b/dev/lib/handlebars/README.md @@ -0,0 +1,164 @@ +# @kbn/handlebars + +A custom version of the handlebars package which, to improve security, does not use `eval` or `new Function`. This means that templates can't be compiled into JavaScript functions in advance and hence, rendering the templates is a lot slower. + +## Limitations + +- Only the following compile options are supported: + - `data` + - `knownHelpers` + - `knownHelpersOnly` + - `noEscape` + - `strict` + - `assumeObjects` + - `preventIndent` + - `explicitPartialContext` + +- Only the following runtime options are supported: + - `data` + - `helpers` + - `partials` + - `decorators` (not documented in the official Handlebars [runtime options documentation](https://handlebarsjs.com/api-reference/runtime-options.html)) + - `blockParams` (not documented in the official Handlebars [runtime options documentation](https://handlebarsjs.com/api-reference/runtime-options.html)) + +## Implementation differences + +The standard `handlebars` implementation: + +1. When given a template string, e.g. `Hello {{x}}`, return a "render" function which takes an "input" object, e.g. `{ x: 'World' }`. +1. The first time the "render" function is called the following happens: + 1. Turn the template string into an Abstract Syntax Tree (AST). + 1. Convert the AST into a hyper optimized JavaScript function which takes the input object as an argument. + 1. Call the generate JavaScript function with the given "input" object to produce and return the final output string (`Hello World`). +1. Subsequent calls to the "render" function will re-use the already generated JavaScript function. + +The custom `@kbn/handlebars` implementation: + +1. When given a template string, e.g. `Hello {{x}}`, return a "render" function which takes an "input" object, e.g. `{ x: 'World' }`. +1. The first time the "render" function is called the following happens: + 1. Turn the template string into an Abstract Syntax Tree (AST). + 1. Process the AST with the given "input" object to produce and return the final output string (`Hello World`). +1. Subsequent calls to the "render" function will re-use the already generated AST. + +_Note: Not parsing of the template string until the first call to the "render" function is deliberate as it mimics the original `handlebars` implementation. This means that any errors that occur due to an invalid template string will not be thrown until the first call to the "render" function._ + +## Technical details + +The `handlebars` library exposes the API for both [generating the AST](https://github.com/handlebars-lang/handlebars.js/blob/master/docs/compiler-api.md#ast) and walking it by implementing the [Visitor API](https://github.com/handlebars-lang/handlebars.js/blob/master/docs/compiler-api.md#ast-visitor). We can leverage that to our advantage and create our own "render" function, which internally calls this API to generate the AST and then the API to walk the AST. + +The `@kbn/handlebars` implementation of the `Visitor` class implements all the necessary methods called by the parent `Visitor` code when instructed to walk the AST. They all start with an upppercase letter, e.g. `MustacheStatement` or `SubExpression`. We call this class `ElasticHandlebarsVisitor`. + +To parse the template string to an AST representation, we call `Handlebars.parse(templateString)`, which returns an AST object. + +The AST object contains a bunch of nodes, one for each element of the template string, all arranged in a tree-like structure. The root of the AST object is a node of type `Program`. This is a special node, which we do not need to worry about, but each of its direct children has a type named like the method which will be called when the walking algorithm reaches that node, e.g. `ContentStatement` or `BlockStatement`. These are the methods that our `Visitor` implementation implements. + +To instruct our `ElasticHandlebarsVisitor` class to start walking the AST object, we call the `accept()` method inherited from the parent `Visitor` class with the main AST object. The `Visitor` will walk each node in turn that is directly attached to the root `Program` node. For each node it traverses, it will call the matching method in our `ElasticHandlebarsVisitor` class. + +To instruct the `Visitor` code to traverse any child nodes of a given node, our implementation needs to manually call `accept(childNode)`, `acceptArray(arrayOfChildNodes)`, `acceptKey(node, childKeyName)`, or `acceptRequired(node, childKeyName)` from within any of the "node" methods, otherwise the child nodes are ignored. + +### State + +We keep state internally in the `ElasticHandlebarsVisitor` object using the following private properties: + +- `contexts`: An array (stack) of `context` objects. In a simple template this array will always only contain a single element: The main `context` object. In more complicated scenarios, new `context` objects will be pushed and popped to and from the `contexts` stack as needed. +- `output`: An array containing the "rendered" output of each node (normally just one element per node). In the most simple template, this is simply joined together into a the final output string after the AST has been traversed. In more complicated templates, we use this array temporarily to collect parameters to give to helper functions (see the `getParams` function). + +## Testing + +The tests for `@kbn/handlebars` are integrated into the regular test suite of Kibana and are all jest tests. To run them all, simply do: + +```sh +node scripts/jest packages/kbn-handlebars +``` + +By default, each test will run both the original `handlebars` code and the modified `@kbn/handlebars` code to compare if the output of the two are identical. To isolate a test run to just one or the other, you can use the following environment variables: + +- `EVAL=1` - Set to only run the original `handlebars` implementation that uses `eval`. +- `AST=1` - Set to only run the modified `@kbn/handlebars` implementation that doesn't use `eval`. + +## Development + +Some of the tests have been copied from the upstream `handlebars` project and modified to fit our use-case, test-suite, and coding conventions. They are all located under the `packages/kbn-handlebars/src/spec` directory. To check if any of the copied files have received updates upstream that we might want to include in our copies, you can run the following script: + +```sh +./packages/kbn-handlebars/scripts/check_for_upstream_updates.sh +``` + +_Note: This will look for changes in the `4.x` branch of the `handlebars.js` repo only. Changes in the `master` branch are ignored._ + +Once all updates have been manually merged with our versions of the files, run the following script to "lock" us into the new updates: + +```sh +./packages/kbn-handlebars/scripts/update_upstream_git_hash.sh +``` + +This will update file `packages/kbn-handlebars/src/spec/.upstream_git_hash`. Make sure to commit changes to this file as well. + +## Debugging + +### Print AST + +To output the generated AST object structure in a somewhat readable form, use the following script: + +```sh +./packages/kbn-handlebars/scripts/print_ast.js +``` + +Example: + +```sh +./packages/kbn-handlebars/scripts/print_ast.js '{{value}}' +``` + +Output: + +```js +{ + type: 'Program', + body: [ + { + type: 'MustacheStatement', + path: { + type: 'PathExpression', + data: false, + depth: 0, + parts: [ 'value' ], + original: 'value' + }, + params: [], + hash: undefined, + escaped: true + } + ] +} +``` + +By default certain properties will be hidden in the output. +For more control over the output, check out the options by running the script without any arguments. + +### Print generated code + +It's possible to see the generated JavaScript code that `handlebars` create for a given template using the following command line tool: + +```sh +./node_modules/handlebars/print-script