diff options
author | Darius Jahandarie <djahandarie@gmail.com> | 2023-11-08 03:11:35 +0900 |
---|---|---|
committer | Darius Jahandarie <djahandarie@gmail.com> | 2023-11-08 03:23:17 +0900 |
commit | 0f4d36938fd0d844f548aa5a7f7e7842df8dfb41 (patch) | |
tree | 5b6be3620a557d0b9177047003f6d742d9d2a32d /dev | |
parent | ef79eab44bfd000792c610b968b5ceefd41e76a0 (diff) |
Switch to vitest for ESM support; other fixes
Diffstat (limited to 'dev')
-rw-r--r-- | dev/bin/build-libs.js | 21 | ||||
-rw-r--r-- | dev/bin/build.js (renamed from dev/build.js) | 51 | ||||
-rw-r--r-- | dev/bin/dictionary-validate.js | 40 | ||||
-rw-r--r-- | dev/bin/generate-css-json.js | 29 | ||||
-rw-r--r-- | dev/bin/schema-validate.js | 60 | ||||
-rw-r--r-- | dev/build-libs.js | 33 | ||||
-rw-r--r-- | dev/css-to-json-util.js | 172 | ||||
-rw-r--r-- | dev/data/manifest-variants.json | 22 | ||||
-rw-r--r-- | dev/database-vm.js | 82 | ||||
-rw-r--r-- | dev/dictionary-validate.js | 49 | ||||
-rw-r--r-- | dev/generate-css-json.js | 157 | ||||
-rw-r--r-- | dev/lib/ucs2length.js | 4 | ||||
-rw-r--r-- | dev/lib/z-worker.js | 17 | ||||
-rw-r--r-- | dev/lib/zip.js | 2 | ||||
-rw-r--r-- | dev/lint/global-declarations.js | 133 | ||||
-rw-r--r-- | dev/lint/html-scripts.js | 173 | ||||
-rw-r--r-- | dev/manifest-util.js | 18 | ||||
-rw-r--r-- | dev/patch-dependencies.js | 47 | ||||
-rw-r--r-- | dev/schema-validate.js | 60 | ||||
-rw-r--r-- | dev/translator-vm.js | 84 | ||||
-rw-r--r-- | dev/util.js | 57 | ||||
-rw-r--r-- | dev/vm.js | 204 |
22 files changed, 439 insertions, 1076 deletions
diff --git a/dev/bin/build-libs.js b/dev/bin/build-libs.js new file mode 100644 index 00000000..07d27188 --- /dev/null +++ b/dev/bin/build-libs.js @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2023 Yomitan Authors + * Copyright (C) 2020-2022 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +import {buildLibs} from '../build-libs.js'; + +buildLibs(); diff --git a/dev/build.js b/dev/bin/build.js index 1e6ef1d0..282f0414 100644 --- a/dev/build.js +++ b/dev/bin/build.js @@ -16,17 +16,17 @@ * along with this program. If not, see <https://www.gnu.org/licenses/>. */ -const fs = require('fs'); -const path = require('path'); -const assert = require('assert'); -const readline = require('readline'); -const childProcess = require('child_process'); -const util = require('./util'); -const {getAllFiles, getArgs, testMain} = util; -const {ManifestUtil} = require('./manifest-util'); -const Ajv = require('ajv'); -const standaloneCode = require('ajv/dist/standalone').default; -const buildLibs = require('./build-libs.js').buildLibs; +import assert from 'assert'; +import childProcess from 'child_process'; +import fs from 'fs'; +import path from 'path'; +import readline from 'readline'; +import {fileURLToPath} from 'url'; +import {buildLibs} from '../build-libs.js'; +import {ManifestUtil} from '../manifest-util.js'; +import {getAllFiles, getArgs, testMain} from '../util.js'; + +const dirname = path.dirname(fileURLToPath(import.meta.url)); async function createZip(directory, excludeFiles, outputFileName, sevenZipExes, onUpdate, dryRun) { try { @@ -61,7 +61,7 @@ async function createZip(directory, excludeFiles, outputFileName, sevenZipExes, } async function createJSZip(directory, excludeFiles, outputFileName, onUpdate, dryRun) { - const JSZip = util.JSZip; + const JSZip = null; const files = getAllFiles(directory); removeItemsFromArray(files, excludeFiles); const zip = new JSZip(); @@ -132,19 +132,6 @@ async function build(buildDir, extDir, manifestUtil, variantNames, manifestPath, process.stdout.write(message); }; - process.stdout.write('Building schema validators using ajv\n'); - const schemaDir = path.join(extDir, 'data/schemas/'); - const schemaFileNames = fs.readdirSync(schemaDir); - const schemas = schemaFileNames.map((schemaFileName) => JSON.parse(fs.readFileSync(path.join(schemaDir, schemaFileName)))); - const ajv = new Ajv({schemas: schemas, code: {source: true, esm: true}}); - const moduleCode = standaloneCode(ajv); - - // https://github.com/ajv-validator/ajv/issues/2209 - const patchedModuleCode = moduleCode.replaceAll('require("ajv/dist/runtime/ucs2length").default', 'import("/lib/ucs2length.js").default'); - - fs.writeFileSync(path.join(extDir, 'lib/validate-schemas.js'), patchedModuleCode); - - process.stdout.write(`Version: ${yomitanVersion}...\n`); for (const variantName of variantNames) { @@ -193,7 +180,7 @@ function ensureFilesExist(directory, files) { } -async function main(argv) { +export async function main(argv) { const args = getArgs(argv, new Map([ ['all', false], ['default', false], @@ -210,7 +197,7 @@ async function main(argv) { const manifestUtil = new ManifestUtil(); - const rootDir = path.join(__dirname, '..'); + const rootDir = path.join(dirname, '..', '..'); const extDir = path.join(rootDir, 'ext'); const buildDir = path.join(rootDir, 'builds'); const manifestPath = path.join(extDir, 'manifest.json'); @@ -234,12 +221,4 @@ async function main(argv) { } } - -if (require.main === module) { - testMain(main, process.argv.slice(2)); -} - - -module.exports = { - main -}; +testMain(main, process.argv.slice(2)); diff --git a/dev/bin/dictionary-validate.js b/dev/bin/dictionary-validate.js new file mode 100644 index 00000000..78ad5198 --- /dev/null +++ b/dev/bin/dictionary-validate.js @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2023 Yomitan Authors + * Copyright (C) 2020-2022 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +import {testDictionaryFiles} from '../dictionary-validate.js'; + +async function main() { + const dictionaryFileNames = process.argv.slice(2); + if (dictionaryFileNames.length === 0) { + console.log([ + 'Usage:', + ' node dictionary-validate [--ajv] <dictionary-file-names>...' + ].join('\n')); + return; + } + + let mode = null; + if (dictionaryFileNames[0] === '--ajv') { + mode = 'ajv'; + dictionaryFileNames.splice(0, 1); + } + + await testDictionaryFiles(mode, dictionaryFileNames); +} + +main(); diff --git a/dev/bin/generate-css-json.js b/dev/bin/generate-css-json.js new file mode 100644 index 00000000..48b42c65 --- /dev/null +++ b/dev/bin/generate-css-json.js @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2023 Yomitan Authors + * Copyright (C) 2020-2022 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +import fs from 'fs'; +import {formatRulesJson, generateRules, getTargets} from '../generate-css-json.js'; + +function main() { + for (const {cssFile, overridesCssFile, outputPath} of getTargets()) { + const json = formatRulesJson(generateRules(cssFile, overridesCssFile)); + fs.writeFileSync(outputPath, json, {encoding: 'utf8'}); + } +} + +main(); diff --git a/dev/bin/schema-validate.js b/dev/bin/schema-validate.js new file mode 100644 index 00000000..86cfebae --- /dev/null +++ b/dev/bin/schema-validate.js @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2023 Yomitan Authors + * Copyright (C) 2020-2022 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +import fs from 'fs'; +import performance from 'perf_hooks'; +import {createJsonSchema} from '../util.js'; + +function main() { + const args = process.argv.slice(2); + if (args.length < 2) { + console.log([ + 'Usage:', + ' node schema-validate [--ajv] <schema-file-name> <data-file-names>...' + ].join('\n')); + return; + } + + let mode = null; + if (args[0] === '--ajv') { + mode = 'ajv'; + args.splice(0, 1); + } + + const schemaSource = fs.readFileSync(args[0], {encoding: 'utf8'}); + const schema = JSON.parse(schemaSource); + + for (const dataFileName of args.slice(1)) { + const start = performance.now(); + try { + console.log(`Validating ${dataFileName}...`); + const dataSource = fs.readFileSync(dataFileName, {encoding: 'utf8'}); + const data = JSON.parse(dataSource); + createJsonSchema(mode, schema).validate(data); + const end = performance.now(); + console.log(`No issues detected (${((end - start) / 1000).toFixed(2)}s)`); + } catch (e) { + const end = performance.now(); + console.log(`Encountered an error (${((end - start) / 1000).toFixed(2)}s)`); + console.warn(e); + } + } +} + + +main(); diff --git a/dev/build-libs.js b/dev/build-libs.js index 497206c9..8320a947 100644 --- a/dev/build-libs.js +++ b/dev/build-libs.js @@ -16,9 +16,15 @@ * along with this program. If not, see <https://www.gnu.org/licenses/>. */ -const fs = require('fs'); -const path = require('path'); -const esbuild = require('esbuild'); +import Ajv from 'ajv'; +import standaloneCode from 'ajv/dist/standalone/index.js'; +import esbuild from 'esbuild'; +import fs from 'fs'; +import path from 'path'; +import {fileURLToPath} from 'url'; + +const dirname = path.dirname(fileURLToPath(import.meta.url)); +const extDir = path.join(dirname, '..', 'ext'); async function buildLib(p) { await esbuild.build({ @@ -28,13 +34,13 @@ async function buildLib(p) { sourcemap: true, target: 'es2020', format: 'esm', - outfile: path.join(__dirname, '..', 'ext', 'lib', path.basename(p)), + outfile: path.join(extDir, 'lib', path.basename(p)), external: ['fs'] }); } -async function buildLibs() { - const devLibPath = path.join(__dirname, 'lib'); +export async function buildLibs() { + const devLibPath = path.join(dirname, 'lib'); const files = await fs.promises.readdir(devLibPath, { withFileTypes: true }); @@ -43,10 +49,15 @@ async function buildLibs() { await buildLib(path.join(devLibPath, f.name)); } } -} -if (require.main === module) { buildLibs(); } + const schemaDir = path.join(extDir, 'data/schemas/'); + const schemaFileNames = fs.readdirSync(schemaDir); + const schemas = schemaFileNames.map((schemaFileName) => JSON.parse(fs.readFileSync(path.join(schemaDir, schemaFileName)))); + const ajv = new Ajv({schemas: schemas, code: {source: true, esm: true}}); + const moduleCode = standaloneCode(ajv); -module.exports = { - buildLibs -}; + // https://github.com/ajv-validator/ajv/issues/2209 + const patchedModuleCode = "import {ucs2length} from './ucs2length.js';" + moduleCode.replaceAll('require("ajv/dist/runtime/ucs2length").default', 'ucs2length'); + + fs.writeFileSync(path.join(extDir, 'lib/validate-schemas.js'), patchedModuleCode); +} diff --git a/dev/css-to-json-util.js b/dev/css-to-json-util.js deleted file mode 100644 index 79aae3c9..00000000 --- a/dev/css-to-json-util.js +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright (C) 2023 Yomitan Authors - * Copyright (C) 2021-2022 Yomichan Authors - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <https://www.gnu.org/licenses/>. - */ - -const fs = require('fs'); -const css = require('css'); - -function indexOfRule(rules, selectors) { - const jj = selectors.length; - for (let i = 0, ii = rules.length; i < ii; ++i) { - const ruleSelectors = rules[i].selectors; - if (ruleSelectors.length !== jj) { continue; } - let okay = true; - for (let j = 0; j < jj; ++j) { - if (selectors[j] !== ruleSelectors[j]) { - okay = false; - break; - } - } - if (okay) { return i; } - } - return -1; -} - -function removeProperty(styles, property, removedProperties) { - let removeCount = removedProperties.get(property); - if (typeof removeCount !== 'undefined') { return removeCount; } - removeCount = 0; - for (let i = 0, ii = styles.length; i < ii; ++i) { - const key = styles[i][0]; - if (key !== property) { continue; } - styles.splice(i, 1); - --i; - --ii; - ++removeCount; - } - removedProperties.set(property, removeCount); - return removeCount; -} - -function formatRulesJson(rules) { - // Manually format JSON, for improved compactness - // return JSON.stringify(rules, null, 4); - const indent1 = ' '; - const indent2 = indent1.repeat(2); - const indent3 = indent1.repeat(3); - let result = ''; - result += '['; - let index1 = 0; - for (const {selectors, styles} of rules) { - if (index1 > 0) { result += ','; } - result += `\n${indent1}{\n${indent2}"selectors": `; - if (selectors.length === 1) { - result += `[${JSON.stringify(selectors[0], null, 4)}]`; - } else { - result += JSON.stringify(selectors, null, 4).replace(/\n/g, '\n' + indent2); - } - result += `,\n${indent2}"styles": [`; - let index2 = 0; - for (const [key, value] of styles) { - if (index2 > 0) { result += ','; } - result += `\n${indent3}[${JSON.stringify(key)}, ${JSON.stringify(value)}]`; - ++index2; - } - if (index2 > 0) { result += `\n${indent2}`; } - result += `]\n${indent1}}`; - ++index1; - } - if (index1 > 0) { result += '\n'; } - result += ']'; - return result; -} - -function generateRules(cssFile, overridesCssFile) { - const content1 = fs.readFileSync(cssFile, {encoding: 'utf8'}); - const content2 = fs.readFileSync(overridesCssFile, {encoding: 'utf8'}); - const stylesheet1 = css.parse(content1, {}).stylesheet; - const stylesheet2 = css.parse(content2, {}).stylesheet; - - const removePropertyPattern = /^remove-property\s+([\w\W]+)$/; - const removeRulePattern = /^remove-rule$/; - const propertySeparator = /\s+/; - - const rules = []; - - // Default stylesheet - for (const rule of stylesheet1.rules) { - if (rule.type !== 'rule') { continue; } - const {selectors, declarations} = rule; - const styles = []; - for (const declaration of declarations) { - if (declaration.type !== 'declaration') { console.log(declaration); continue; } - const {property, value} = declaration; - styles.push([property, value]); - } - if (styles.length > 0) { - rules.push({selectors, styles}); - } - } - - // Overrides - for (const rule of stylesheet2.rules) { - if (rule.type !== 'rule') { continue; } - const {selectors, declarations} = rule; - const removedProperties = new Map(); - for (const declaration of declarations) { - switch (declaration.type) { - case 'declaration': - { - const index = indexOfRule(rules, selectors); - let entry; - if (index >= 0) { - entry = rules[index]; - } else { - entry = {selectors, styles: []}; - rules.push(entry); - } - const {property, value} = declaration; - removeProperty(entry.styles, property, removedProperties); - entry.styles.push([property, value]); - } - break; - case 'comment': - { - const index = indexOfRule(rules, selectors); - if (index < 0) { throw new Error('Could not find rule with matching selectors'); } - const comment = declaration.comment.trim(); - let m; - if ((m = removePropertyPattern.exec(comment)) !== null) { - for (const property of m[1].split(propertySeparator)) { - const removeCount = removeProperty(rules[index].styles, property, removedProperties); - if (removeCount === 0) { throw new Error(`Property removal is unnecessary; ${property} does not exist`); } - } - } else if (removeRulePattern.test(comment)) { - rules.splice(index, 1); - } - } - break; - } - } - } - - // Remove empty - for (let i = 0, ii = rules.length; i < ii; ++i) { - if (rules[i].styles.length > 0) { continue; } - rules.splice(i, 1); - --i; - --ii; - } - - return rules; -} - - -module.exports = { - formatRulesJson, - generateRules -}; diff --git a/dev/data/manifest-variants.json b/dev/data/manifest-variants.json index d44251e1..e6113b75 100644 --- a/dev/data/manifest-variants.json +++ b/dev/data/manifest-variants.json @@ -122,8 +122,7 @@ "inherit": "base", "fileName": "yomitan-chrome.zip", "excludeFiles": [ - "background.html", - "js/dom/native-simple-dom-parser.js" + "background.html" ] }, { @@ -187,6 +186,13 @@ ] }, { + "action": "delete", + "path": [ + "background", + "type" + ] + }, + { "action": "set", "path": [ "background", @@ -251,9 +257,7 @@ "sw.js", "offscreen.html", "js/background/offscreen.js", - "js/background/offscreen-main.js", - "js/dom/simple-dom-parser.js", - "lib/parse5.js" + "js/background/offscreen-main.js" ] }, { @@ -302,9 +306,7 @@ "sw.js", "offscreen.html", "js/background/offscreen.js", - "js/background/offscreen-main.js", - "js/dom/simple-dom-parser.js", - "lib/parse5.js" + "js/background/offscreen-main.js" ] }, { @@ -351,9 +353,7 @@ "sw.js", "offscreen.html", "js/background/offscreen.js", - "js/background/offscreen-main.js", - "js/dom/simple-dom-parser.js", - "lib/parse5.js" + "js/background/offscreen-main.js" ] } ] diff --git a/dev/database-vm.js b/dev/database-vm.js deleted file mode 100644 index d5570691..00000000 --- a/dev/database-vm.js +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (C) 2023 Yomitan Authors - * Copyright (C) 2020-2022 Yomichan Authors - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <https://www.gnu.org/licenses/>. - */ - -const fs = require('fs'); -const url = require('url'); -const path = require('path'); -const {JSZip} = require('./util'); -const {VM} = require('./vm'); -require('fake-indexeddb/auto'); - -const chrome = { - runtime: { - getURL: (path2) => { - return url.pathToFileURL(path.join(__dirname, '..', 'ext', path2.replace(/^\//, ''))).href; - } - } -}; - -async function fetch(url2) { - const extDir = path.join(__dirname, '..', 'ext'); - let filePath; - try { - filePath = url.fileURLToPath(url2); - } catch (e) { - filePath = path.resolve(extDir, url2.replace(/^[/\\]/, '')); - } - await Promise.resolve(); - const content = fs.readFileSync(filePath, {encoding: null}); - return { - ok: true, - status: 200, - statusText: 'OK', - text: async () => Promise.resolve(content.toString('utf8')), - json: async () => Promise.resolve(JSON.parse(content.toString('utf8'))) - }; -} - -function atob(data) { - return Buffer.from(data, 'base64').toString('ascii'); -} - -class DatabaseVM extends VM { - constructor(globals={}) { - super(Object.assign({ - chrome, - fetch, - indexedDB: global.indexedDB, - IDBKeyRange: global.IDBKeyRange, - JSZip, - atob - }, globals)); - this.context.window = this.context; - this.indexedDB = global.indexedDB; - } -} - -class DatabaseVMDictionaryImporterMediaLoader { - async getImageDetails(content) { - // Placeholder values - return {content, width: 100, height: 100}; - } -} - -module.exports = { - DatabaseVM, - DatabaseVMDictionaryImporterMediaLoader -}; diff --git a/dev/dictionary-validate.js b/dev/dictionary-validate.js index 0c926acc..eb40beda 100644 --- a/dev/dictionary-validate.js +++ b/dev/dictionary-validate.js @@ -16,12 +16,11 @@ * along with this program. If not, see <https://www.gnu.org/licenses/>. */ -const fs = require('fs'); -const path = require('path'); -const {performance} = require('perf_hooks'); -const {JSZip} = require('./util'); -const {createJsonSchema} = require('./schema-validate'); - +import fs from 'fs'; +import JSZip from 'jszip'; +import path from 'path'; +import {performance} from 'perf_hooks'; +import {createJsonSchema} from './schema-validate.js'; function readSchema(relativeFileName) { const fileName = path.join(__dirname, relativeFileName); @@ -29,7 +28,6 @@ function readSchema(relativeFileName) { return JSON.parse(source); } - async function validateDictionaryBanks(mode, zip, fileNameFormat, schema) { let jsonSchema; try { @@ -57,7 +55,7 @@ async function validateDictionaryBanks(mode, zip, fileNameFormat, schema) { } } -async function validateDictionary(mode, archive, schemas) { +export async function validateDictionary(mode, archive, schemas) { const fileName = 'index.json'; const indexFile = archive.files[fileName]; if (!indexFile) { @@ -82,7 +80,7 @@ async function validateDictionary(mode, archive, schemas) { await validateDictionaryBanks(mode, archive, 'tag_bank_?.json', schemas.tagBankV3); } -function getSchemas() { +export function getSchemas() { return { index: readSchema('../ext/data/schemas/dictionary-index-schema.json'), kanjiBankV1: readSchema('../ext/data/schemas/dictionary-kanji-bank-v1-schema.json'), @@ -95,8 +93,7 @@ function getSchemas() { }; } - -async function testDictionaryFiles(mode, dictionaryFileNames) { +export async function testDictionaryFiles(mode, dictionaryFileNames) { const schemas = getSchemas(); for (const dictionaryFileName of dictionaryFileNames) { @@ -115,33 +112,3 @@ async function testDictionaryFiles(mode, dictionaryFileNames) { } } } - - -async function main() { - const dictionaryFileNames = process.argv.slice(2); - if (dictionaryFileNames.length === 0) { - console.log([ - 'Usage:', - ' node dictionary-validate [--ajv] <dictionary-file-names>...' - ].join('\n')); - return; - } - - let mode = null; - if (dictionaryFileNames[0] === '--ajv') { - mode = 'ajv'; - dictionaryFileNames.splice(0, 1); - } - - await testDictionaryFiles(mode, dictionaryFileNames); -} - - -if (require.main === module) { main(); } - - -module.exports = { - getSchemas, - validateDictionary, - testDictionaryFiles -}; diff --git a/dev/generate-css-json.js b/dev/generate-css-json.js index 787173ab..914c1452 100644 --- a/dev/generate-css-json.js +++ b/dev/generate-css-json.js @@ -16,12 +16,10 @@ * along with this program. If not, see <https://www.gnu.org/licenses/>. */ -const fs = require('fs'); -const path = require('path'); -const {testMain} = require('./util'); -const {formatRulesJson, generateRules} = require('./css-to-json-util'); +import fs from 'fs'; +import path from 'path'; -function getTargets() { +export function getTargets() { return [ { cssFile: path.join(__dirname, '..', 'ext/css/structured-content.css'), @@ -36,19 +34,150 @@ function getTargets() { ]; } -function main() { - for (const {cssFile, overridesCssFile, outputPath} of getTargets()) { - const json = formatRulesJson(generateRules(cssFile, overridesCssFile)); - fs.writeFileSync(outputPath, json, {encoding: 'utf8'}); +import css from 'css'; + +function indexOfRule(rules, selectors) { + const jj = selectors.length; + for (let i = 0, ii = rules.length; i < ii; ++i) { + const ruleSelectors = rules[i].selectors; + if (ruleSelectors.length !== jj) { continue; } + let okay = true; + for (let j = 0; j < jj; ++j) { + if (selectors[j] !== ruleSelectors[j]) { + okay = false; + break; + } + } + if (okay) { return i; } } + return -1; } +function removeProperty(styles, property, removedProperties) { + let removeCount = removedProperties.get(property); + if (typeof removeCount !== 'undefined') { return removeCount; } + removeCount = 0; + for (let i = 0, ii = styles.length; i < ii; ++i) { + const key = styles[i][0]; + if (key !== property) { continue; } + styles.splice(i, 1); + --i; + --ii; + ++removeCount; + } + removedProperties.set(property, removeCount); + return removeCount; +} -if (require.main === module) { - testMain(main, process.argv.slice(2)); +export function formatRulesJson(rules) { + // Manually format JSON, for improved compactness + // return JSON.stringify(rules, null, 4); + const indent1 = ' '; + const indent2 = indent1.repeat(2); + const indent3 = indent1.repeat(3); + let result = ''; + result += '['; + let index1 = 0; + for (const {selectors, styles} of rules) { + if (index1 > 0) { result += ','; } + result += `\n${indent1}{\n${indent2}"selectors": `; + if (selectors.length === 1) { + result += `[${JSON.stringify(selectors[0], null, 4)}]`; + } else { + result += JSON.stringify(selectors, null, 4).replace(/\n/g, '\n' + indent2); + } + result += `,\n${indent2}"styles": [`; + let index2 = 0; + for (const [key, value] of styles) { + if (index2 > 0) { result += ','; } + result += `\n${indent3}[${JSON.stringify(key)}, ${JSON.stringify(value)}]`; + ++index2; + } + if (index2 > 0) { result += `\n${indent2}`; } + result += `]\n${indent1}}`; + ++index1; + } + if (index1 > 0) { result += '\n'; } + result += ']'; + return result; } +export function generateRules(cssFile, overridesCssFile) { + const content1 = fs.readFileSync(cssFile, {encoding: 'utf8'}); + const content2 = fs.readFileSync(overridesCssFile, {encoding: 'utf8'}); + const stylesheet1 = css.parse(content1, {}).stylesheet; + const stylesheet2 = css.parse(content2, {}).stylesheet; + + const removePropertyPattern = /^remove-property\s+([\w\W]+)$/; + const removeRulePattern = /^remove-rule$/; + const propertySeparator = /\s+/; -module.exports = { - getTargets -}; + const rules = []; + + // Default stylesheet + for (const rule of stylesheet1.rules) { + if (rule.type !== 'rule') { continue; } + const {selectors, declarations} = rule; + const styles = []; + for (const declaration of declarations) { + if (declaration.type !== 'declaration') { console.log(declaration); continue; } + const {property, value} = declaration; + styles.push([property, value]); + } + if (styles.length > 0) { + rules.push({selectors, styles}); + } + } + + // Overrides + for (const rule of stylesheet2.rules) { + if (rule.type !== 'rule') { continue; } + const {selectors, declarations} = rule; + const removedProperties = new Map(); + for (const declaration of declarations) { + switch (declaration.type) { + case 'declaration': + { + const index = indexOfRule(rules, selectors); + let entry; + if (index >= 0) { + entry = rules[index]; + } else { + entry = {selectors, styles: []}; + rules.push(entry); + } + const {property, value} = declaration; + removeProperty(entry.styles, property, removedProperties); + entry.styles.push([property, value]); + } + break; + case 'comment': + { + const index = indexOfRule(rules, selectors); + if (index < 0) { throw new Error('Could not find rule with matching selectors'); } + const comment = declaration.comment.trim(); + let m; + if ((m = removePropertyPattern.exec(comment)) !== null) { + for (const property of m[1].split(propertySeparator)) { + const removeCount = removeProperty(rules[index].styles, property, removedProperties); + if (removeCount === 0) { throw new Error(`Property removal is unnecessary; ${property} does not exist`); } + } + } else if (removeRulePattern.test(comment)) { + rules.splice(index, 1); + } + } + break; + } + } + } + + // Remove empty + for (let i = 0, ii = rules.length; i < ii; ++i) { + if (rules[i].styles.length > 0) { continue; } + rules.splice(i, 1); + --i; + --ii; + } + + return rules; +} diff --git a/dev/lib/ucs2length.js b/dev/lib/ucs2length.js index 2e4a01cd..3b370493 100644 --- a/dev/lib/ucs2length.js +++ b/dev/lib/ucs2length.js @@ -14,5 +14,7 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see <https://www.gnu.org/licenses/>. */ -export {ucs2length} from 'ajv/dist/runtime/ucs2length'; +import ucs2length from 'ajv/dist/runtime/ucs2length.js'; +const ucs2length2 = ucs2length.default; +export {ucs2length2 as ucs2length}; diff --git a/dev/lib/z-worker.js b/dev/lib/z-worker.js new file mode 100644 index 00000000..f6a95ed3 --- /dev/null +++ b/dev/lib/z-worker.js @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2023 Yomitan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + */ +import '../../node_modules/@zip.js/zip.js/lib/z-worker.js'; diff --git a/dev/lib/zip.js b/dev/lib/zip.js index 7560f5f8..b6e85451 100644 --- a/dev/lib/zip.js +++ b/dev/lib/zip.js @@ -14,4 +14,4 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see <https://www.gnu.org/licenses/>. */ -export * from '@zip.js/zip.js/lib/zip-full.js'; +export * from '@zip.js/zip.js/lib/zip.js'; diff --git a/dev/lint/global-declarations.js b/dev/lint/global-declarations.js deleted file mode 100644 index 7f90d227..00000000 --- a/dev/lint/global-declarations.js +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright (C) 2023 Yomitan Authors - * Copyright (C) 2020-2022 Yomichan Authors - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <https://www.gnu.org/licenses/>. - */ - -const fs = require('fs'); -const path = require('path'); -const assert = require('assert'); -const {getAllFiles} = require('../util'); - - -function escapeRegExp(string) { - return string.replace(/[.*+\-?^${}()|[\]\\]/g, '\\$&'); -} - -function countOccurences(string, pattern) { - return (string.match(pattern) || []).length; -} - -function getNewline(string) { - const count1 = countOccurences(string, /(?:^|[^\r])\n/g); - const count2 = countOccurences(string, /\r\n/g); - const count3 = countOccurences(string, /\r(?:[^\n]|$)/g); - if (count2 > count1) { - return (count3 > count2) ? '\r' : '\r\n'; - } else { - return (count3 > count1) ? '\r' : '\n'; - } -} - -function getSubstringCount(string, substring) { - let count = 0; - const pattern = new RegExp(`\\b${escapeRegExp(substring)}\\b`, 'g'); - while (true) { - const match = pattern.exec(string); - if (match === null) { break; } - ++count; - } - return count; -} - - -function validateGlobals(fileName, fix) { - const pattern = /\/\*\s*global\s+([\w\W]*?)\*\//g; - const trimPattern = /^[\s,*]+|[\s,*]+$/g; - const splitPattern = /[\s,*]+/; - const source = fs.readFileSync(fileName, {encoding: 'utf8'}); - let match; - let first = true; - let endIndex = 0; - let newSource = ''; - const allGlobals = []; - const newline = getNewline(source); - while ((match = pattern.exec(source)) !== null) { - if (!first) { - console.error(`Encountered more than one global declaration in ${fileName}`); - return false; - } - first = false; - - const parts = match[1].replace(trimPattern, '').split(splitPattern); - parts.sort(); - - const actual = match[0]; - const expected = `/* global${parts.map((v) => `${newline} * ${v}`).join('')}${newline} */`; - - try { - assert.strictEqual(actual, expected); - } catch (e) { - console.error(`Global declaration error encountered in ${fileName}:`); - console.error(e.message); - if (!fix) { - return false; - } - } - - newSource += source.substring(0, match.index); - newSource += expected; - endIndex = match.index + match[0].length; - - allGlobals.push(...parts); - } - - newSource += source.substring(endIndex); - - // This is an approximate check to see if a global variable is unused. - // If the global appears in a comment, string, or similar, the check will pass. - let errorCount = 0; - for (const global of allGlobals) { - if (getSubstringCount(newSource, global) <= 1) { - console.error(`Global variable ${global} appears to be unused in ${fileName}`); - ++errorCount; - } - } - - if (fix) { - fs.writeFileSync(fileName, newSource, {encoding: 'utf8'}); - } - - return errorCount === 0; -} - - -function main() { - const fix = (process.argv.length >= 2 && process.argv[2] === '--fix'); - const directory = path.resolve(__dirname, '..', '..', 'ext'); - const pattern = /\.js$/; - const ignorePattern = /^lib[\\/]/; - const fileNames = getAllFiles(directory, (f) => pattern.test(f) && !ignorePattern.test(f)); - for (const fileName of fileNames) { - if (!validateGlobals(path.join(directory, fileName), fix)) { - process.exit(-1); - return; - } - } - process.exit(0); -} - - -if (require.main === module) { main(); } diff --git a/dev/lint/html-scripts.js b/dev/lint/html-scripts.js deleted file mode 100644 index db6e6ca4..00000000 --- a/dev/lint/html-scripts.js +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Copyright (C) 2023 Yomitan Authors - * Copyright (C) 2020-2022 Yomichan Authors - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <https://www.gnu.org/licenses/>. - */ - -const fs = require('fs'); -const path = require('path'); -const assert = require('assert'); -const {JSDOM} = require('jsdom'); -const {getAllFiles} = require('../util'); - - -function lstatSyncSafe(fileName) { - try { - return fs.lstatSync(fileName); - } catch (e) { - return null; - } -} - -function validatePath(src, fileName, extDir) { - assert.ok(typeof src === 'string', `<script> missing src attribute in ${fileName}`); - assert.ok(src.startsWith('/'), `<script> src attribute is not absolute in ${fileName} (src=${JSON.stringify(src)})`); - const relativeSrc = src.substring(1); - assert.ok(!path.isAbsolute(relativeSrc), `<script> src attribute is invalid in ${fileName} (src=${JSON.stringify(src)})`); - const fullSrc = path.join(extDir, relativeSrc); - const stats = lstatSyncSafe(fullSrc); - assert.ok(stats !== null, `<script> src file not found in ${fileName} (src=${JSON.stringify(src)})`); - assert.ok(stats.isFile(), `<script> src file invalid in ${fileName} (src=${JSON.stringify(src)})`); -} - -function getSubstringCount(string, pattern) { - let count = 0; - while (true) { - const match = pattern.exec(string); - if (match === null) { break; } - ++count; - } - return count; -} - -function getSortedScriptPaths(scriptPaths) { - // Sort file names without the extension - const extensionPattern = /\.[^.]*$/; - scriptPaths = scriptPaths.map((value) => { - const match = extensionPattern.exec(value); - let ext = ''; - if (match !== null) { - ext = match[0]; - value = value.substring(0, value.length - ext.length); - } - return {value, ext}; - }); - - const stringComparer = new Intl.Collator('en-US'); // Invariant locale - scriptPaths.sort((a, b) => stringComparer.compare(a.value, b.value)); - - scriptPaths = scriptPaths.map(({value, ext}) => `${value}${ext}`); - return scriptPaths; -} - -function validateScriptOrder(fileName, window) { - const {document, Node: {ELEMENT_NODE, TEXT_NODE}, NodeFilter} = window; - - const scriptElements = document.querySelectorAll('script'); - if (scriptElements.length === 0) { return; } - - // Assert all scripts are siblings - const scriptContainerElement = scriptElements[0].parentNode; - for (const element of scriptElements) { - if (element.parentNode !== scriptContainerElement) { - assert.fail('All script nodes are not contained within the same element'); - } - } - - // Get script groupings and order - const scriptGroups = []; - const newlinePattern = /\n/g; - let separatingText = ''; - const walker = document.createTreeWalker(scriptContainerElement, NodeFilter.SHOW_ELEMENT | NodeFilter.SHOW_TEXT); - walker.firstChild(); - for (let node = walker.currentNode; node !== null; node = walker.nextSibling()) { - switch (node.nodeType) { - case ELEMENT_NODE: - if (node.tagName.toLowerCase() === 'script') { - let scriptGroup; - if (scriptGroups.length === 0 || getSubstringCount(separatingText, newlinePattern) >= 2) { - scriptGroup = []; - scriptGroups.push(scriptGroup); - } else { - scriptGroup = scriptGroups[scriptGroups.length - 1]; - } - scriptGroup.push(node.src); - separatingText = ''; - } - break; - case TEXT_NODE: - separatingText += node.nodeValue; - break; - } - } - - // Ensure core.js is first (if it is present) - const ignorePattern = /^\/lib\//; - const index = scriptGroups.flat() - .filter((value) => !ignorePattern.test(value)) - .findIndex((value) => (value === '/js/core.js')); - assert.ok(index <= 0, 'core.js is not the first included script'); - - // Check script order - for (let i = 0, ii = scriptGroups.length; i < ii; ++i) { - const scriptGroup = scriptGroups[i]; - try { - assert.deepStrictEqual(scriptGroup, getSortedScriptPaths(scriptGroup)); - } catch (e) { - console.error(`Script order for group ${i + 1} in file ${fileName} is not correct:`); - throw e; - } - } -} - -function validateHtmlScripts(fileName, extDir) { - const fullFileName = path.join(extDir, fileName); - const domSource = fs.readFileSync(fullFileName, {encoding: 'utf8'}); - const dom = new JSDOM(domSource); - const {window} = dom; - const {document} = window; - try { - for (const {src} of document.querySelectorAll('script')) { - validatePath(src, fullFileName, extDir); - } - for (const {href} of document.querySelectorAll('link')) { - validatePath(href, fullFileName, extDir); - } - validateScriptOrder(fileName, window); - } finally { - window.close(); - } -} - - -function main() { - try { - const extDir = path.resolve(__dirname, '..', '..', 'ext'); - const pattern = /\.html$/; - const ignorePattern = /^lib[\\/]/; - const fileNames = getAllFiles(extDir, (f) => pattern.test(f) && !ignorePattern.test(f)); - for (const fileName of fileNames) { - validateHtmlScripts(fileName, extDir); - } - } catch (e) { - console.error(e); - process.exit(-1); - return; - } - process.exit(0); -} - - -if (require.main === module) { main(); } diff --git a/dev/manifest-util.js b/dev/manifest-util.js index 082cf57c..15175e7f 100644 --- a/dev/manifest-util.js +++ b/dev/manifest-util.js @@ -16,19 +16,21 @@ * along with this program. If not, see <https://www.gnu.org/licenses/>. */ -const fs = require('fs'); -const path = require('path'); -const childProcess = require('child_process'); +import childProcess from 'child_process'; +import fs from 'fs'; +import {fileURLToPath} from 'node:url'; +import path from 'path'; +const dirname = path.dirname(fileURLToPath(import.meta.url)); function clone(value) { return JSON.parse(JSON.stringify(value)); } -class ManifestUtil { +export class ManifestUtil { constructor() { - const fileName = path.join(__dirname, 'data', 'manifest-variants.json'); + const fileName = path.join(dirname, 'data', 'manifest-variants.json'); const {manifest, variants, defaultVariant} = JSON.parse(fs.readFileSync(fileName)); this._manifest = manifest; this._variants = variants; @@ -74,7 +76,7 @@ class ManifestUtil { _evaluateModificationCommand(data) { const {command, args, trim} = data; const {stdout, stderr, status} = childProcess.spawnSync(command, args, { - cwd: __dirname, + cwd: dirname, stdio: 'pipe', shell: false }); @@ -263,7 +265,3 @@ class ManifestUtil { } } - -module.exports = { - ManifestUtil -}; diff --git a/dev/patch-dependencies.js b/dev/patch-dependencies.js deleted file mode 100644 index 81572c5c..00000000 --- a/dev/patch-dependencies.js +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (C) 2023 Yomitan Authors - * Copyright (C) 2021-2022 Yomichan Authors - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <https://www.gnu.org/licenses/>. - */ - -const fs = require('fs'); -const assert = require('assert'); - -/** - * This function patches the following bug: - * - https://github.com/jsdom/jsdom/issues/3211 - * - https://github.com/dperini/nwsapi/issues/48 - */ -function patchNwsapi() { - const nwsapiPath = require.resolve('nwsapi'); - const nwsapiSource = fs.readFileSync(nwsapiPath, {encoding: 'utf8'}); - const pattern = /(if|while)(\()(?:e&&)?(\(e=e\.parentElement\)\)\{)/g; - let modifications = 0; - const nwsapiSourceNew = nwsapiSource.replace(pattern, (g0, g1, g2, g3) => { - ++modifications; - return `${g1}${g2}e&&${g3}`; - }); - assert.strictEqual(modifications, 2); - fs.writeFileSync(nwsapiPath, nwsapiSourceNew, {encoding: 'utf8'}); - // nwsapi is used by JSDOM - const {testJSDOM} = require('../test/test-jsdom'); - testJSDOM(); -} - -function main() { - patchNwsapi(); -} - -if (require.main === module) { main(); } diff --git a/dev/schema-validate.js b/dev/schema-validate.js index 1d7607b7..fbd6b06a 100644 --- a/dev/schema-validate.js +++ b/dev/schema-validate.js @@ -16,21 +16,11 @@ * along with this program. If not, see <https://www.gnu.org/licenses/>. */ -const fs = require('fs'); -const {performance} = require('perf_hooks'); -const {VM} = require('./vm'); - -const vm = new VM(); -vm.execute([ - 'js/core.js', - 'js/general/cache-map.js', - 'js/data/json-schema.js' -]); -const JsonSchema = vm.get('JsonSchema'); +import Ajv from 'ajv'; +import {JsonSchema} from '../ext/js/data/json-schema.js'; class JsonSchemaAjv { constructor(schema) { - const Ajv = require('ajv'); const ajv = new Ajv({ meta: false, strictTuples: false, @@ -49,53 +39,9 @@ class JsonSchemaAjv { } } -function createJsonSchema(mode, schema) { +export function createJsonSchema(mode, schema) { switch (mode) { case 'ajv': return new JsonSchemaAjv(schema); default: return new JsonSchema(schema); } } - -function main() { - const args = process.argv.slice(2); - if (args.length < 2) { - console.log([ - 'Usage:', - ' node schema-validate [--ajv] <schema-file-name> <data-file-names>...' - ].join('\n')); - return; - } - - let mode = null; - if (args[0] === '--ajv') { - mode = 'ajv'; - args.splice(0, 1); - } - - const schemaSource = fs.readFileSync(args[0], {encoding: 'utf8'}); - const schema = JSON.parse(schemaSource); - - for (const dataFileName of args.slice(1)) { - const start = performance.now(); - try { - console.log(`Validating ${dataFileName}...`); - const dataSource = fs.readFileSync(dataFileName, {encoding: 'utf8'}); - const data = JSON.parse(dataSource); - createJsonSchema(mode, schema).validate(data); - const end = performance.now(); - console.log(`No issues detected (${((end - start) / 1000).toFixed(2)}s)`); - } catch (e) { - const end = performance.now(); - console.log(`Encountered an error (${((end - start) / 1000).toFixed(2)}s)`); - console.warn(e); - } - } -} - - -if (require.main === module) { main(); } - - -module.exports = { - createJsonSchema -}; diff --git a/dev/translator-vm.js b/dev/translator-vm.js index 2a51ab8c..9f14523e 100644 --- a/dev/translator-vm.js +++ b/dev/translator-vm.js @@ -16,19 +16,32 @@ * along with this program. If not, see <https://www.gnu.org/licenses/>. */ -const fs = require('fs'); -const path = require('path'); -const assert = require('assert'); -const {DatabaseVM, DatabaseVMDictionaryImporterMediaLoader} = require('./database-vm'); -const {createDictionaryArchive} = require('./util'); - -function clone(value) { - return JSON.parse(JSON.stringify(value)); -} +import fs from 'fs'; +import url, {fileURLToPath} from 'node:url'; +import path from 'path'; +import {expect, vi} from 'vitest'; +import {AnkiNoteDataCreator} from '../ext/js/data/sandbox/anki-note-data-creator.js'; +import {DictionaryDatabase} from '../ext/js/language/dictionary-database.js'; +import {DictionaryImporterMediaLoader} from '../ext/js/language/dictionary-importer-media-loader.js'; +import {DictionaryImporter} from '../ext/js/language/dictionary-importer.js'; +import {JapaneseUtil} from '../ext/js/language/sandbox/japanese-util.js'; +import {Translator} from '../ext/js/language/translator.js'; +import {createDictionaryArchive} from './util.js'; + +vi.mock('../ext/js/language/dictionary-importer-media-loader.js'); + +const dirname = path.dirname(fileURLToPath(import.meta.url)); + +export class TranslatorVM { + constructor() { + global.chrome = { + runtime: { + getURL: (path2) => { + return url.pathToFileURL(path.join(dirname, '..', 'ext', path2.replace(/^\//, ''))).href; + } + } + }; -class TranslatorVM extends DatabaseVM { - constructor(globals) { - super(globals); this._japaneseUtil = null; this._translator = null; this._ankiNoteDataCreator = null; @@ -40,43 +53,14 @@ class TranslatorVM extends DatabaseVM { } async prepare(dictionaryDirectory, dictionaryName) { - this.execute([ - 'js/core.js', - 'js/data/sandbox/anki-note-data-creator.js', - 'js/data/database.js', - 'js/data/json-schema.js', - 'js/general/cache-map.js', - 'js/general/regex-util.js', - 'js/general/text-source-map.js', - 'js/language/deinflector.js', - 'js/language/sandbox/dictionary-data-util.js', - 'js/language/dictionary-importer.js', - 'js/language/dictionary-database.js', - 'js/language/sandbox/japanese-util.js', - 'js/language/translator.js', - 'js/media/media-util.js' - ]); - const [ - DictionaryImporter, - DictionaryDatabase, - JapaneseUtil, - Translator, - AnkiNoteDataCreator - ] = this.get([ - 'DictionaryImporter', - 'DictionaryDatabase', - 'JapaneseUtil', - 'Translator', - 'AnkiNoteDataCreator' - ]); - // Dictionary this._dictionaryName = dictionaryName; const testDictionary = createDictionaryArchive(dictionaryDirectory, dictionaryName); + // const testDictionaryContent = await testDictionary.arrayBuffer(); const testDictionaryContent = await testDictionary.generateAsync({type: 'arraybuffer'}); // Setup database - const dictionaryImporterMediaLoader = new DatabaseVMDictionaryImporterMediaLoader(); + const dictionaryImporterMediaLoader = new DictionaryImporterMediaLoader(); const dictionaryImporter = new DictionaryImporter(dictionaryImporterMediaLoader, null); const dictionaryDatabase = new DictionaryDatabase(); await dictionaryDatabase.prepare(); @@ -87,7 +71,9 @@ class TranslatorVM extends DatabaseVM { {prefixWildcardsSupported: true} ); - assert.deepStrictEqual(errors.length, 0); + expect(errors.length).toEqual(0); + + const myDirname = path.dirname(fileURLToPath(import.meta.url)); // Setup translator this._japaneseUtil = new JapaneseUtil(null); @@ -95,7 +81,7 @@ class TranslatorVM extends DatabaseVM { japaneseUtil: this._japaneseUtil, database: dictionaryDatabase }); - const deinflectionReasons = JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'ext', 'data/deinflect.json'))); + const deinflectionReasons = JSON.parse(fs.readFileSync(path.join(myDirname, '..', 'ext', 'data/deinflect.json'))); this._translator.prepare(deinflectionReasons); // Assign properties @@ -132,10 +118,10 @@ class TranslatorVM extends DatabaseVM { if (!Object.prototype.hasOwnProperty.call(optionsPresets, entry)) { throw new Error('Invalid options preset'); } - Object.assign(options, clone(optionsPresets[entry])); + Object.assign(options, structuredClone(optionsPresets[entry])); break; case 'object': - Object.assign(options, clone(entry)); + Object.assign(options, structuredClone(entry)); break; default: throw new Error('Invalid options type'); @@ -177,7 +163,3 @@ class TranslatorVM extends DatabaseVM { return options; } } - -module.exports = { - TranslatorVM -}; diff --git a/dev/util.js b/dev/util.js index 65b1d982..cabc40aa 100644 --- a/dev/util.js +++ b/dev/util.js @@ -16,24 +16,11 @@ * along with this program. If not, see <https://www.gnu.org/licenses/>. */ -const fs = require('fs'); -const path = require('path'); +import fs from 'fs'; +import JSZip from 'jszip'; +import path from 'path'; - -let JSZip = null; - - -function getJSZip() { - if (JSZip === null) { - process.noDeprecation = true; // Suppress a warning about JSZip - JSZip = require(path.join(__dirname, '../ext/lib/jszip.min.js')); - process.noDeprecation = false; - } - return JSZip; -} - - -function getArgs(args, argMap) { +export function getArgs(args, argMap) { let key = null; let canKey = true; let onKey = false; @@ -77,7 +64,7 @@ function getArgs(args, argMap) { return argMap; } -function getAllFiles(baseDirectory, predicate=null) { +export function getAllFiles(baseDirectory, predicate=null) { const results = []; const directories = [baseDirectory]; while (directories.length > 0) { @@ -99,11 +86,12 @@ function getAllFiles(baseDirectory, predicate=null) { return results; } -function createDictionaryArchive(dictionaryDirectory, dictionaryName) { +export function createDictionaryArchive(dictionaryDirectory, dictionaryName) { const fileNames = fs.readdirSync(dictionaryDirectory); - const JSZip2 = getJSZip(); - const archive = new JSZip2(); + // const zipFileWriter = new BlobWriter(); + // const zipWriter = new ZipWriter(zipFileWriter); + const archive = new JSZip(); for (const fileName of fileNames) { if (/\.json$/.test(fileName)) { @@ -113,17 +101,31 @@ function createDictionaryArchive(dictionaryDirectory, dictionaryName) { json.title = dictionaryName; } archive.file(fileName, JSON.stringify(json, null, 0)); + + // await zipWriter.add(fileName, new TextReader(JSON.stringify(json, null, 0))); } else { const content = fs.readFileSync(path.join(dictionaryDirectory, fileName), {encoding: null}); archive.file(fileName, content); + + // console.log('adding'); + // const r = new TextReader(content); + // console.log(r.readUint8Array(0, 10)); + // console.log('reader done'); + // await zipWriter.add(fileName, r); + // console.log('??'); } } + // await zipWriter.close(); + // Retrieves the Blob object containing the zip content into `zipFileBlob`. It + // is also returned by zipWriter.close() for more convenience. + // const zipFileBlob = await zipFileWriter.getData(); return archive; -} + // return zipFileBlob; +} -async function testMain(func, ...args) { +export async function testMain(func, ...args) { try { await func(...args); } catch (e) { @@ -131,12 +133,3 @@ async function testMain(func, ...args) { process.exit(-1); } } - - -module.exports = { - get JSZip() { return getJSZip(); }, - getArgs, - getAllFiles, - createDictionaryArchive, - testMain -}; diff --git a/dev/vm.js b/dev/vm.js deleted file mode 100644 index c3266443..00000000 --- a/dev/vm.js +++ /dev/null @@ -1,204 +0,0 @@ -/* - * Copyright (C) 2023 Yomitan Authors - * Copyright (C) 2020-2022 Yomichan Authors - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <https://www.gnu.org/licenses/>. - */ - -const fs = require('fs'); -const vm = require('vm'); -const path = require('path'); -const assert = require('assert'); -const crypto = require('crypto'); - - -function getContextEnvironmentRecords(context, names) { - // Enables export of values from the declarative environment record - if (!Array.isArray(names) || names.length === 0) { - return []; - } - - let scriptSource = '(() => {\n "use strict";\n const results = [];'; - for (const name of names) { - scriptSource += `\n try { results.push(${name}); } catch (e) { results.push(void 0); }`; - } - scriptSource += '\n return results;\n})();'; - - const script = new vm.Script(scriptSource, {filename: 'getContextEnvironmentRecords'}); - - const contextHasNames = Object.prototype.hasOwnProperty.call(context, 'names'); - const contextNames = context.names; - context.names = names; - - const results = script.runInContext(context, {}); - - if (contextHasNames) { - context.names = contextNames; - } else { - delete context.names; - } - - return Array.from(results); -} - -function isDeepStrictEqual(val1, val2) { - if (val1 === val2) { return true; } - - if (Array.isArray(val1)) { - if (Array.isArray(val2)) { - return isArrayDeepStrictEqual(val1, val2); - } - } else if (typeof val1 === 'object' && val1 !== null) { - if (typeof val2 === 'object' && val2 !== null) { - return isObjectDeepStrictEqual(val1, val2); - } - } - - return false; -} - -function isArrayDeepStrictEqual(val1, val2) { - const ii = val1.length; - if (ii !== val2.length) { return false; } - - for (let i = 0; i < ii; ++i) { - if (!isDeepStrictEqual(val1[i], val2[i])) { - return false; - } - } - - return true; -} - -function isObjectDeepStrictEqual(val1, val2) { - const keys1 = Object.keys(val1); - const keys2 = Object.keys(val2); - - if (keys1.length !== keys2.length) { return false; } - - const keySet = new Set(keys1); - for (const key of keys2) { - if (!keySet.delete(key)) { return false; } - } - - for (const key of keys1) { - if (!isDeepStrictEqual(val1[key], val2[key])) { - return false; - } - } - - const tag1 = Object.prototype.toString.call(val1); - const tag2 = Object.prototype.toString.call(val2); - if (tag1 !== tag2) { return false; } - - return true; -} - -function deepStrictEqual(actual, expected) { - try { - // This will fail on prototype === comparison on cross context objects - assert.deepStrictEqual(actual, expected); - } catch (e) { - if (!isDeepStrictEqual(actual, expected)) { - throw e; - } - } -} - - -function createURLClass() { - const BaseURL = URL; - const result = function URL(url) { - const u = new BaseURL(url); - this.hash = u.hash; - this.host = u.host; - this.hostname = u.hostname; - this.href = u.href; - this.origin = u.origin; - this.password = u.password; - this.pathname = u.pathname; - this.port = u.port; - this.protocol = u.protocol; - this.search = u.search; - this.searchParams = u.searchParams; - this.username = u.username; - }; - return result; -} - - -class VM { - constructor(context={}) { - context.URL = createURLClass(); - context.crypto = { - getRandomValues: (array) => { - const buffer = crypto.randomBytes(array.byteLength); - buffer.copy(array); - return array; - } - }; - this._context = vm.createContext(context); - this._assert = { - deepStrictEqual - }; - } - - get context() { - return this._context; - } - - get assert() { - return this._assert; - } - - get(names) { - if (typeof names === 'string') { - return getContextEnvironmentRecords(this._context, [names])[0]; - } else if (Array.isArray(names)) { - return getContextEnvironmentRecords(this._context, names); - } else { - throw new Error('Invalid argument'); - } - } - - set(values) { - if (typeof values === 'object' && values !== null) { - Object.assign(this._context, values); - } else { - throw new Error('Invalid argument'); - } - } - - execute(fileNames) { - const single = !Array.isArray(fileNames); - if (single) { - fileNames = [fileNames]; - } - - const results = []; - for (const fileName of fileNames) { - const absoluteFileName = path.resolve(__dirname, '..', 'ext', fileName); - const source = fs.readFileSync(absoluteFileName, {encoding: 'utf8'}); - const script = new vm.Script(source, {filename: absoluteFileName}); - results.push(script.runInContext(this._context, {})); - } - - return single ? results[0] : results; - } -} - - -module.exports = { - VM -}; |