From 0f4d36938fd0d844f548aa5a7f7e7842df8dfb41 Mon Sep 17 00:00:00 2001 From: Darius Jahandarie Date: Wed, 8 Nov 2023 03:11:35 +0900 Subject: Switch to vitest for ESM support; other fixes --- dev/bin/build-libs.js | 21 ++++ dev/bin/build.js | 224 ++++++++++++++++++++++++++++++++++++ dev/bin/dictionary-validate.js | 40 +++++++ dev/bin/generate-css-json.js | 29 +++++ dev/bin/schema-validate.js | 60 ++++++++++ dev/build-libs.js | 33 ++++-- dev/build.js | 245 ---------------------------------------- dev/css-to-json-util.js | 172 ---------------------------- dev/data/manifest-variants.json | 22 ++-- dev/database-vm.js | 82 -------------- dev/dictionary-validate.js | 49 ++------ dev/generate-css-json.js | 157 ++++++++++++++++++++++--- dev/lib/ucs2length.js | 4 +- dev/lib/z-worker.js | 17 +++ dev/lib/zip.js | 2 +- dev/lint/global-declarations.js | 133 ---------------------- dev/lint/html-scripts.js | 173 ---------------------------- dev/manifest-util.js | 18 ++- dev/patch-dependencies.js | 47 -------- dev/schema-validate.js | 60 +--------- dev/translator-vm.js | 84 ++++++-------- dev/util.js | 57 ++++------ dev/vm.js | 204 --------------------------------- 23 files changed, 648 insertions(+), 1285 deletions(-) create mode 100644 dev/bin/build-libs.js create mode 100644 dev/bin/build.js create mode 100644 dev/bin/dictionary-validate.js create mode 100644 dev/bin/generate-css-json.js create mode 100644 dev/bin/schema-validate.js delete mode 100644 dev/build.js delete mode 100644 dev/css-to-json-util.js delete mode 100644 dev/database-vm.js create mode 100644 dev/lib/z-worker.js delete mode 100644 dev/lint/global-declarations.js delete mode 100644 dev/lint/html-scripts.js delete mode 100644 dev/patch-dependencies.js delete mode 100644 dev/vm.js (limited to 'dev') diff --git a/dev/bin/build-libs.js b/dev/bin/build-libs.js new file mode 100644 index 00000000..07d27188 --- /dev/null +++ b/dev/bin/build-libs.js @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2023 Yomitan Authors + * Copyright (C) 2020-2022 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +import {buildLibs} from '../build-libs.js'; + +buildLibs(); diff --git a/dev/bin/build.js b/dev/bin/build.js new file mode 100644 index 00000000..282f0414 --- /dev/null +++ b/dev/bin/build.js @@ -0,0 +1,224 @@ +/* + * Copyright (C) 2023 Yomitan Authors + * Copyright (C) 2020-2022 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +import assert from 'assert'; +import childProcess from 'child_process'; +import fs from 'fs'; +import path from 'path'; +import readline from 'readline'; +import {fileURLToPath} from 'url'; +import {buildLibs} from '../build-libs.js'; +import {ManifestUtil} from '../manifest-util.js'; +import {getAllFiles, getArgs, testMain} from '../util.js'; + +const dirname = path.dirname(fileURLToPath(import.meta.url)); + +async function createZip(directory, excludeFiles, outputFileName, sevenZipExes, onUpdate, dryRun) { + try { + fs.unlinkSync(outputFileName); + } catch (e) { + // NOP + } + + if (!dryRun) { + for (const exe of sevenZipExes) { + try { + const excludeArguments = excludeFiles.map((excludeFilePath) => `-x!${excludeFilePath}`); + childProcess.execFileSync( + exe, + [ + 'a', + outputFileName, + '.', + ...excludeArguments + ], + { + cwd: directory + } + ); + return; + } catch (e) { + // NOP + } + } + } + return await createJSZip(directory, excludeFiles, outputFileName, onUpdate, dryRun); +} + +async function createJSZip(directory, excludeFiles, outputFileName, onUpdate, dryRun) { + const JSZip = null; + const files = getAllFiles(directory); + removeItemsFromArray(files, excludeFiles); + const zip = new JSZip(); + for (const fileName of files) { + zip.file( + fileName.replace(/\\/g, '/'), + fs.readFileSync(path.join(directory, fileName), {encoding: null, flag: 'r'}), + {} + ); + } + + if (typeof onUpdate !== 'function') { + onUpdate = () => {}; // NOP + } + + const data = await zip.generateAsync({ + type: 'nodebuffer', + compression: 'DEFLATE', + compressionOptions: {level: 9} + }, onUpdate); + process.stdout.write('\n'); + + if (!dryRun) { + fs.writeFileSync(outputFileName, data, {encoding: null, flag: 'w'}); + } +} + +function removeItemsFromArray(array, removeItems) { + for (const item of removeItems) { + const index = getIndexOfFilePath(array, item); + if (index >= 0) { + array.splice(index, 1); + } + } +} + +function getIndexOfFilePath(array, item) { + const pattern = /\\/g; + const separator = '/'; + item = item.replace(pattern, separator); + for (let i = 0, ii = array.length; i < ii; ++i) { + if (array[i].replace(pattern, separator) === item) { + return i; + } + } + return -1; +} + +async function build(buildDir, extDir, manifestUtil, variantNames, manifestPath, dryRun, dryRunBuildZip, yomitanVersion) { + const sevenZipExes = ['7za', '7z']; + + // Create build directory + if (!fs.existsSync(buildDir) && !dryRun) { + fs.mkdirSync(buildDir, {recursive: true}); + } + + const dontLogOnUpdate = !process.stdout.isTTY; + const onUpdate = (metadata) => { + if (dontLogOnUpdate) { return; } + + let message = `Progress: ${metadata.percent.toFixed(2)}%`; + if (metadata.currentFile) { + message += ` (${metadata.currentFile})`; + } + + readline.clearLine(process.stdout); + readline.cursorTo(process.stdout, 0); + process.stdout.write(message); + }; + + process.stdout.write(`Version: ${yomitanVersion}...\n`); + + for (const variantName of variantNames) { + const variant = manifestUtil.getVariant(variantName); + if (typeof variant === 'undefined' || variant.buildable === false) { continue; } + + const {name, fileName, fileCopies} = variant; + let {excludeFiles} = variant; + if (!Array.isArray(excludeFiles)) { excludeFiles = []; } + + process.stdout.write(`Building ${name}...\n`); + + const modifiedManifest = manifestUtil.getManifest(variant.name); + + ensureFilesExist(extDir, excludeFiles); + + if (typeof fileName === 'string') { + const fileNameSafe = path.basename(fileName); + const fullFileName = path.join(buildDir, fileNameSafe); + if (!dryRun) { + fs.writeFileSync(manifestPath, ManifestUtil.createManifestString(modifiedManifest).replace('$YOMITAN_VERSION', yomitanVersion)); + } + + if (!dryRun || dryRunBuildZip) { + await createZip(extDir, excludeFiles, fullFileName, sevenZipExes, onUpdate, dryRun); + } + + if (!dryRun) { + if (Array.isArray(fileCopies)) { + for (const fileName2 of fileCopies) { + const fileName2Safe = path.basename(fileName2); + fs.copyFileSync(fullFileName, path.join(buildDir, fileName2Safe)); + } + } + } + } + + process.stdout.write('\n'); + } +} + +function ensureFilesExist(directory, files) { + for (const file of files) { + assert.ok(fs.existsSync(path.join(directory, file))); + } +} + + +export async function main(argv) { + const args = getArgs(argv, new Map([ + ['all', false], + ['default', false], + ['manifest', null], + ['dry-run', false], + ['dry-run-build-zip', false], + ['yomitan-version', '0.0.0.0'], + [null, []] + ])); + + const dryRun = args.get('dry-run'); + const dryRunBuildZip = args.get('dry-run-build-zip'); + const yomitanVersion = args.get('yomitan-version'); + + const manifestUtil = new ManifestUtil(); + + const rootDir = path.join(dirname, '..', '..'); + const extDir = path.join(rootDir, 'ext'); + const buildDir = path.join(rootDir, 'builds'); + const manifestPath = path.join(extDir, 'manifest.json'); + + try { + await buildLibs(); + const variantNames = ( + argv.length === 0 || args.get('all') ? + manifestUtil.getVariants().filter(({buildable}) => buildable !== false).map(({name}) => name) : + args.get(null) + ); + await build(buildDir, extDir, manifestUtil, variantNames, manifestPath, dryRun, dryRunBuildZip, yomitanVersion); + } finally { + // Restore manifest + const manifestName = (!args.get('default') && args.get('manifest') !== null) ? args.get('manifest') : null; + const restoreManifest = manifestUtil.getManifest(manifestName); + process.stdout.write('Restoring manifest...\n'); + if (!dryRun) { + fs.writeFileSync(manifestPath, ManifestUtil.createManifestString(restoreManifest).replace('$YOMITAN_VERSION', yomitanVersion)); + } + } +} + +testMain(main, process.argv.slice(2)); diff --git a/dev/bin/dictionary-validate.js b/dev/bin/dictionary-validate.js new file mode 100644 index 00000000..78ad5198 --- /dev/null +++ b/dev/bin/dictionary-validate.js @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2023 Yomitan Authors + * Copyright (C) 2020-2022 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +import {testDictionaryFiles} from '../dictionary-validate.js'; + +async function main() { + const dictionaryFileNames = process.argv.slice(2); + if (dictionaryFileNames.length === 0) { + console.log([ + 'Usage:', + ' node dictionary-validate [--ajv] ...' + ].join('\n')); + return; + } + + let mode = null; + if (dictionaryFileNames[0] === '--ajv') { + mode = 'ajv'; + dictionaryFileNames.splice(0, 1); + } + + await testDictionaryFiles(mode, dictionaryFileNames); +} + +main(); diff --git a/dev/bin/generate-css-json.js b/dev/bin/generate-css-json.js new file mode 100644 index 00000000..48b42c65 --- /dev/null +++ b/dev/bin/generate-css-json.js @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2023 Yomitan Authors + * Copyright (C) 2020-2022 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +import fs from 'fs'; +import {formatRulesJson, generateRules, getTargets} from '../generate-css-json.js'; + +function main() { + for (const {cssFile, overridesCssFile, outputPath} of getTargets()) { + const json = formatRulesJson(generateRules(cssFile, overridesCssFile)); + fs.writeFileSync(outputPath, json, {encoding: 'utf8'}); + } +} + +main(); diff --git a/dev/bin/schema-validate.js b/dev/bin/schema-validate.js new file mode 100644 index 00000000..86cfebae --- /dev/null +++ b/dev/bin/schema-validate.js @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2023 Yomitan Authors + * Copyright (C) 2020-2022 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +import fs from 'fs'; +import performance from 'perf_hooks'; +import {createJsonSchema} from '../util.js'; + +function main() { + const args = process.argv.slice(2); + if (args.length < 2) { + console.log([ + 'Usage:', + ' node schema-validate [--ajv] ...' + ].join('\n')); + return; + } + + let mode = null; + if (args[0] === '--ajv') { + mode = 'ajv'; + args.splice(0, 1); + } + + const schemaSource = fs.readFileSync(args[0], {encoding: 'utf8'}); + const schema = JSON.parse(schemaSource); + + for (const dataFileName of args.slice(1)) { + const start = performance.now(); + try { + console.log(`Validating ${dataFileName}...`); + const dataSource = fs.readFileSync(dataFileName, {encoding: 'utf8'}); + const data = JSON.parse(dataSource); + createJsonSchema(mode, schema).validate(data); + const end = performance.now(); + console.log(`No issues detected (${((end - start) / 1000).toFixed(2)}s)`); + } catch (e) { + const end = performance.now(); + console.log(`Encountered an error (${((end - start) / 1000).toFixed(2)}s)`); + console.warn(e); + } + } +} + + +main(); diff --git a/dev/build-libs.js b/dev/build-libs.js index 497206c9..8320a947 100644 --- a/dev/build-libs.js +++ b/dev/build-libs.js @@ -16,9 +16,15 @@ * along with this program. If not, see . */ -const fs = require('fs'); -const path = require('path'); -const esbuild = require('esbuild'); +import Ajv from 'ajv'; +import standaloneCode from 'ajv/dist/standalone/index.js'; +import esbuild from 'esbuild'; +import fs from 'fs'; +import path from 'path'; +import {fileURLToPath} from 'url'; + +const dirname = path.dirname(fileURLToPath(import.meta.url)); +const extDir = path.join(dirname, '..', 'ext'); async function buildLib(p) { await esbuild.build({ @@ -28,13 +34,13 @@ async function buildLib(p) { sourcemap: true, target: 'es2020', format: 'esm', - outfile: path.join(__dirname, '..', 'ext', 'lib', path.basename(p)), + outfile: path.join(extDir, 'lib', path.basename(p)), external: ['fs'] }); } -async function buildLibs() { - const devLibPath = path.join(__dirname, 'lib'); +export async function buildLibs() { + const devLibPath = path.join(dirname, 'lib'); const files = await fs.promises.readdir(devLibPath, { withFileTypes: true }); @@ -43,10 +49,15 @@ async function buildLibs() { await buildLib(path.join(devLibPath, f.name)); } } -} -if (require.main === module) { buildLibs(); } + const schemaDir = path.join(extDir, 'data/schemas/'); + const schemaFileNames = fs.readdirSync(schemaDir); + const schemas = schemaFileNames.map((schemaFileName) => JSON.parse(fs.readFileSync(path.join(schemaDir, schemaFileName)))); + const ajv = new Ajv({schemas: schemas, code: {source: true, esm: true}}); + const moduleCode = standaloneCode(ajv); -module.exports = { - buildLibs -}; + // https://github.com/ajv-validator/ajv/issues/2209 + const patchedModuleCode = "import {ucs2length} from './ucs2length.js';" + moduleCode.replaceAll('require("ajv/dist/runtime/ucs2length").default', 'ucs2length'); + + fs.writeFileSync(path.join(extDir, 'lib/validate-schemas.js'), patchedModuleCode); +} diff --git a/dev/build.js b/dev/build.js deleted file mode 100644 index 1e6ef1d0..00000000 --- a/dev/build.js +++ /dev/null @@ -1,245 +0,0 @@ -/* - * Copyright (C) 2023 Yomitan Authors - * Copyright (C) 2020-2022 Yomichan Authors - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -const fs = require('fs'); -const path = require('path'); -const assert = require('assert'); -const readline = require('readline'); -const childProcess = require('child_process'); -const util = require('./util'); -const {getAllFiles, getArgs, testMain} = util; -const {ManifestUtil} = require('./manifest-util'); -const Ajv = require('ajv'); -const standaloneCode = require('ajv/dist/standalone').default; -const buildLibs = require('./build-libs.js').buildLibs; - -async function createZip(directory, excludeFiles, outputFileName, sevenZipExes, onUpdate, dryRun) { - try { - fs.unlinkSync(outputFileName); - } catch (e) { - // NOP - } - - if (!dryRun) { - for (const exe of sevenZipExes) { - try { - const excludeArguments = excludeFiles.map((excludeFilePath) => `-x!${excludeFilePath}`); - childProcess.execFileSync( - exe, - [ - 'a', - outputFileName, - '.', - ...excludeArguments - ], - { - cwd: directory - } - ); - return; - } catch (e) { - // NOP - } - } - } - return await createJSZip(directory, excludeFiles, outputFileName, onUpdate, dryRun); -} - -async function createJSZip(directory, excludeFiles, outputFileName, onUpdate, dryRun) { - const JSZip = util.JSZip; - const files = getAllFiles(directory); - removeItemsFromArray(files, excludeFiles); - const zip = new JSZip(); - for (const fileName of files) { - zip.file( - fileName.replace(/\\/g, '/'), - fs.readFileSync(path.join(directory, fileName), {encoding: null, flag: 'r'}), - {} - ); - } - - if (typeof onUpdate !== 'function') { - onUpdate = () => {}; // NOP - } - - const data = await zip.generateAsync({ - type: 'nodebuffer', - compression: 'DEFLATE', - compressionOptions: {level: 9} - }, onUpdate); - process.stdout.write('\n'); - - if (!dryRun) { - fs.writeFileSync(outputFileName, data, {encoding: null, flag: 'w'}); - } -} - -function removeItemsFromArray(array, removeItems) { - for (const item of removeItems) { - const index = getIndexOfFilePath(array, item); - if (index >= 0) { - array.splice(index, 1); - } - } -} - -function getIndexOfFilePath(array, item) { - const pattern = /\\/g; - const separator = '/'; - item = item.replace(pattern, separator); - for (let i = 0, ii = array.length; i < ii; ++i) { - if (array[i].replace(pattern, separator) === item) { - return i; - } - } - return -1; -} - -async function build(buildDir, extDir, manifestUtil, variantNames, manifestPath, dryRun, dryRunBuildZip, yomitanVersion) { - const sevenZipExes = ['7za', '7z']; - - // Create build directory - if (!fs.existsSync(buildDir) && !dryRun) { - fs.mkdirSync(buildDir, {recursive: true}); - } - - const dontLogOnUpdate = !process.stdout.isTTY; - const onUpdate = (metadata) => { - if (dontLogOnUpdate) { return; } - - let message = `Progress: ${metadata.percent.toFixed(2)}%`; - if (metadata.currentFile) { - message += ` (${metadata.currentFile})`; - } - - readline.clearLine(process.stdout); - readline.cursorTo(process.stdout, 0); - process.stdout.write(message); - }; - - process.stdout.write('Building schema validators using ajv\n'); - const schemaDir = path.join(extDir, 'data/schemas/'); - const schemaFileNames = fs.readdirSync(schemaDir); - const schemas = schemaFileNames.map((schemaFileName) => JSON.parse(fs.readFileSync(path.join(schemaDir, schemaFileName)))); - const ajv = new Ajv({schemas: schemas, code: {source: true, esm: true}}); - const moduleCode = standaloneCode(ajv); - - // https://github.com/ajv-validator/ajv/issues/2209 - const patchedModuleCode = moduleCode.replaceAll('require("ajv/dist/runtime/ucs2length").default', 'import("/lib/ucs2length.js").default'); - - fs.writeFileSync(path.join(extDir, 'lib/validate-schemas.js'), patchedModuleCode); - - - process.stdout.write(`Version: ${yomitanVersion}...\n`); - - for (const variantName of variantNames) { - const variant = manifestUtil.getVariant(variantName); - if (typeof variant === 'undefined' || variant.buildable === false) { continue; } - - const {name, fileName, fileCopies} = variant; - let {excludeFiles} = variant; - if (!Array.isArray(excludeFiles)) { excludeFiles = []; } - - process.stdout.write(`Building ${name}...\n`); - - const modifiedManifest = manifestUtil.getManifest(variant.name); - - ensureFilesExist(extDir, excludeFiles); - - if (typeof fileName === 'string') { - const fileNameSafe = path.basename(fileName); - const fullFileName = path.join(buildDir, fileNameSafe); - if (!dryRun) { - fs.writeFileSync(manifestPath, ManifestUtil.createManifestString(modifiedManifest).replace('$YOMITAN_VERSION', yomitanVersion)); - } - - if (!dryRun || dryRunBuildZip) { - await createZip(extDir, excludeFiles, fullFileName, sevenZipExes, onUpdate, dryRun); - } - - if (!dryRun) { - if (Array.isArray(fileCopies)) { - for (const fileName2 of fileCopies) { - const fileName2Safe = path.basename(fileName2); - fs.copyFileSync(fullFileName, path.join(buildDir, fileName2Safe)); - } - } - } - } - - process.stdout.write('\n'); - } -} - -function ensureFilesExist(directory, files) { - for (const file of files) { - assert.ok(fs.existsSync(path.join(directory, file))); - } -} - - -async function main(argv) { - const args = getArgs(argv, new Map([ - ['all', false], - ['default', false], - ['manifest', null], - ['dry-run', false], - ['dry-run-build-zip', false], - ['yomitan-version', '0.0.0.0'], - [null, []] - ])); - - const dryRun = args.get('dry-run'); - const dryRunBuildZip = args.get('dry-run-build-zip'); - const yomitanVersion = args.get('yomitan-version'); - - const manifestUtil = new ManifestUtil(); - - const rootDir = path.join(__dirname, '..'); - const extDir = path.join(rootDir, 'ext'); - const buildDir = path.join(rootDir, 'builds'); - const manifestPath = path.join(extDir, 'manifest.json'); - - try { - await buildLibs(); - const variantNames = ( - argv.length === 0 || args.get('all') ? - manifestUtil.getVariants().filter(({buildable}) => buildable !== false).map(({name}) => name) : - args.get(null) - ); - await build(buildDir, extDir, manifestUtil, variantNames, manifestPath, dryRun, dryRunBuildZip, yomitanVersion); - } finally { - // Restore manifest - const manifestName = (!args.get('default') && args.get('manifest') !== null) ? args.get('manifest') : null; - const restoreManifest = manifestUtil.getManifest(manifestName); - process.stdout.write('Restoring manifest...\n'); - if (!dryRun) { - fs.writeFileSync(manifestPath, ManifestUtil.createManifestString(restoreManifest).replace('$YOMITAN_VERSION', yomitanVersion)); - } - } -} - - -if (require.main === module) { - testMain(main, process.argv.slice(2)); -} - - -module.exports = { - main -}; diff --git a/dev/css-to-json-util.js b/dev/css-to-json-util.js deleted file mode 100644 index 79aae3c9..00000000 --- a/dev/css-to-json-util.js +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright (C) 2023 Yomitan Authors - * Copyright (C) 2021-2022 Yomichan Authors - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -const fs = require('fs'); -const css = require('css'); - -function indexOfRule(rules, selectors) { - const jj = selectors.length; - for (let i = 0, ii = rules.length; i < ii; ++i) { - const ruleSelectors = rules[i].selectors; - if (ruleSelectors.length !== jj) { continue; } - let okay = true; - for (let j = 0; j < jj; ++j) { - if (selectors[j] !== ruleSelectors[j]) { - okay = false; - break; - } - } - if (okay) { return i; } - } - return -1; -} - -function removeProperty(styles, property, removedProperties) { - let removeCount = removedProperties.get(property); - if (typeof removeCount !== 'undefined') { return removeCount; } - removeCount = 0; - for (let i = 0, ii = styles.length; i < ii; ++i) { - const key = styles[i][0]; - if (key !== property) { continue; } - styles.splice(i, 1); - --i; - --ii; - ++removeCount; - } - removedProperties.set(property, removeCount); - return removeCount; -} - -function formatRulesJson(rules) { - // Manually format JSON, for improved compactness - // return JSON.stringify(rules, null, 4); - const indent1 = ' '; - const indent2 = indent1.repeat(2); - const indent3 = indent1.repeat(3); - let result = ''; - result += '['; - let index1 = 0; - for (const {selectors, styles} of rules) { - if (index1 > 0) { result += ','; } - result += `\n${indent1}{\n${indent2}"selectors": `; - if (selectors.length === 1) { - result += `[${JSON.stringify(selectors[0], null, 4)}]`; - } else { - result += JSON.stringify(selectors, null, 4).replace(/\n/g, '\n' + indent2); - } - result += `,\n${indent2}"styles": [`; - let index2 = 0; - for (const [key, value] of styles) { - if (index2 > 0) { result += ','; } - result += `\n${indent3}[${JSON.stringify(key)}, ${JSON.stringify(value)}]`; - ++index2; - } - if (index2 > 0) { result += `\n${indent2}`; } - result += `]\n${indent1}}`; - ++index1; - } - if (index1 > 0) { result += '\n'; } - result += ']'; - return result; -} - -function generateRules(cssFile, overridesCssFile) { - const content1 = fs.readFileSync(cssFile, {encoding: 'utf8'}); - const content2 = fs.readFileSync(overridesCssFile, {encoding: 'utf8'}); - const stylesheet1 = css.parse(content1, {}).stylesheet; - const stylesheet2 = css.parse(content2, {}).stylesheet; - - const removePropertyPattern = /^remove-property\s+([\w\W]+)$/; - const removeRulePattern = /^remove-rule$/; - const propertySeparator = /\s+/; - - const rules = []; - - // Default stylesheet - for (const rule of stylesheet1.rules) { - if (rule.type !== 'rule') { continue; } - const {selectors, declarations} = rule; - const styles = []; - for (const declaration of declarations) { - if (declaration.type !== 'declaration') { console.log(declaration); continue; } - const {property, value} = declaration; - styles.push([property, value]); - } - if (styles.length > 0) { - rules.push({selectors, styles}); - } - } - - // Overrides - for (const rule of stylesheet2.rules) { - if (rule.type !== 'rule') { continue; } - const {selectors, declarations} = rule; - const removedProperties = new Map(); - for (const declaration of declarations) { - switch (declaration.type) { - case 'declaration': - { - const index = indexOfRule(rules, selectors); - let entry; - if (index >= 0) { - entry = rules[index]; - } else { - entry = {selectors, styles: []}; - rules.push(entry); - } - const {property, value} = declaration; - removeProperty(entry.styles, property, removedProperties); - entry.styles.push([property, value]); - } - break; - case 'comment': - { - const index = indexOfRule(rules, selectors); - if (index < 0) { throw new Error('Could not find rule with matching selectors'); } - const comment = declaration.comment.trim(); - let m; - if ((m = removePropertyPattern.exec(comment)) !== null) { - for (const property of m[1].split(propertySeparator)) { - const removeCount = removeProperty(rules[index].styles, property, removedProperties); - if (removeCount === 0) { throw new Error(`Property removal is unnecessary; ${property} does not exist`); } - } - } else if (removeRulePattern.test(comment)) { - rules.splice(index, 1); - } - } - break; - } - } - } - - // Remove empty - for (let i = 0, ii = rules.length; i < ii; ++i) { - if (rules[i].styles.length > 0) { continue; } - rules.splice(i, 1); - --i; - --ii; - } - - return rules; -} - - -module.exports = { - formatRulesJson, - generateRules -}; diff --git a/dev/data/manifest-variants.json b/dev/data/manifest-variants.json index d44251e1..e6113b75 100644 --- a/dev/data/manifest-variants.json +++ b/dev/data/manifest-variants.json @@ -122,8 +122,7 @@ "inherit": "base", "fileName": "yomitan-chrome.zip", "excludeFiles": [ - "background.html", - "js/dom/native-simple-dom-parser.js" + "background.html" ] }, { @@ -186,6 +185,13 @@ "service_worker" ] }, + { + "action": "delete", + "path": [ + "background", + "type" + ] + }, { "action": "set", "path": [ @@ -251,9 +257,7 @@ "sw.js", "offscreen.html", "js/background/offscreen.js", - "js/background/offscreen-main.js", - "js/dom/simple-dom-parser.js", - "lib/parse5.js" + "js/background/offscreen-main.js" ] }, { @@ -302,9 +306,7 @@ "sw.js", "offscreen.html", "js/background/offscreen.js", - "js/background/offscreen-main.js", - "js/dom/simple-dom-parser.js", - "lib/parse5.js" + "js/background/offscreen-main.js" ] }, { @@ -351,9 +353,7 @@ "sw.js", "offscreen.html", "js/background/offscreen.js", - "js/background/offscreen-main.js", - "js/dom/simple-dom-parser.js", - "lib/parse5.js" + "js/background/offscreen-main.js" ] } ] diff --git a/dev/database-vm.js b/dev/database-vm.js deleted file mode 100644 index d5570691..00000000 --- a/dev/database-vm.js +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (C) 2023 Yomitan Authors - * Copyright (C) 2020-2022 Yomichan Authors - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -const fs = require('fs'); -const url = require('url'); -const path = require('path'); -const {JSZip} = require('./util'); -const {VM} = require('./vm'); -require('fake-indexeddb/auto'); - -const chrome = { - runtime: { - getURL: (path2) => { - return url.pathToFileURL(path.join(__dirname, '..', 'ext', path2.replace(/^\//, ''))).href; - } - } -}; - -async function fetch(url2) { - const extDir = path.join(__dirname, '..', 'ext'); - let filePath; - try { - filePath = url.fileURLToPath(url2); - } catch (e) { - filePath = path.resolve(extDir, url2.replace(/^[/\\]/, '')); - } - await Promise.resolve(); - const content = fs.readFileSync(filePath, {encoding: null}); - return { - ok: true, - status: 200, - statusText: 'OK', - text: async () => Promise.resolve(content.toString('utf8')), - json: async () => Promise.resolve(JSON.parse(content.toString('utf8'))) - }; -} - -function atob(data) { - return Buffer.from(data, 'base64').toString('ascii'); -} - -class DatabaseVM extends VM { - constructor(globals={}) { - super(Object.assign({ - chrome, - fetch, - indexedDB: global.indexedDB, - IDBKeyRange: global.IDBKeyRange, - JSZip, - atob - }, globals)); - this.context.window = this.context; - this.indexedDB = global.indexedDB; - } -} - -class DatabaseVMDictionaryImporterMediaLoader { - async getImageDetails(content) { - // Placeholder values - return {content, width: 100, height: 100}; - } -} - -module.exports = { - DatabaseVM, - DatabaseVMDictionaryImporterMediaLoader -}; diff --git a/dev/dictionary-validate.js b/dev/dictionary-validate.js index 0c926acc..eb40beda 100644 --- a/dev/dictionary-validate.js +++ b/dev/dictionary-validate.js @@ -16,12 +16,11 @@ * along with this program. If not, see . */ -const fs = require('fs'); -const path = require('path'); -const {performance} = require('perf_hooks'); -const {JSZip} = require('./util'); -const {createJsonSchema} = require('./schema-validate'); - +import fs from 'fs'; +import JSZip from 'jszip'; +import path from 'path'; +import {performance} from 'perf_hooks'; +import {createJsonSchema} from './schema-validate.js'; function readSchema(relativeFileName) { const fileName = path.join(__dirname, relativeFileName); @@ -29,7 +28,6 @@ function readSchema(relativeFileName) { return JSON.parse(source); } - async function validateDictionaryBanks(mode, zip, fileNameFormat, schema) { let jsonSchema; try { @@ -57,7 +55,7 @@ async function validateDictionaryBanks(mode, zip, fileNameFormat, schema) { } } -async function validateDictionary(mode, archive, schemas) { +export async function validateDictionary(mode, archive, schemas) { const fileName = 'index.json'; const indexFile = archive.files[fileName]; if (!indexFile) { @@ -82,7 +80,7 @@ async function validateDictionary(mode, archive, schemas) { await validateDictionaryBanks(mode, archive, 'tag_bank_?.json', schemas.tagBankV3); } -function getSchemas() { +export function getSchemas() { return { index: readSchema('../ext/data/schemas/dictionary-index-schema.json'), kanjiBankV1: readSchema('../ext/data/schemas/dictionary-kanji-bank-v1-schema.json'), @@ -95,8 +93,7 @@ function getSchemas() { }; } - -async function testDictionaryFiles(mode, dictionaryFileNames) { +export async function testDictionaryFiles(mode, dictionaryFileNames) { const schemas = getSchemas(); for (const dictionaryFileName of dictionaryFileNames) { @@ -115,33 +112,3 @@ async function testDictionaryFiles(mode, dictionaryFileNames) { } } } - - -async function main() { - const dictionaryFileNames = process.argv.slice(2); - if (dictionaryFileNames.length === 0) { - console.log([ - 'Usage:', - ' node dictionary-validate [--ajv] ...' - ].join('\n')); - return; - } - - let mode = null; - if (dictionaryFileNames[0] === '--ajv') { - mode = 'ajv'; - dictionaryFileNames.splice(0, 1); - } - - await testDictionaryFiles(mode, dictionaryFileNames); -} - - -if (require.main === module) { main(); } - - -module.exports = { - getSchemas, - validateDictionary, - testDictionaryFiles -}; diff --git a/dev/generate-css-json.js b/dev/generate-css-json.js index 787173ab..914c1452 100644 --- a/dev/generate-css-json.js +++ b/dev/generate-css-json.js @@ -16,12 +16,10 @@ * along with this program. If not, see . */ -const fs = require('fs'); -const path = require('path'); -const {testMain} = require('./util'); -const {formatRulesJson, generateRules} = require('./css-to-json-util'); +import fs from 'fs'; +import path from 'path'; -function getTargets() { +export function getTargets() { return [ { cssFile: path.join(__dirname, '..', 'ext/css/structured-content.css'), @@ -36,19 +34,150 @@ function getTargets() { ]; } -function main() { - for (const {cssFile, overridesCssFile, outputPath} of getTargets()) { - const json = formatRulesJson(generateRules(cssFile, overridesCssFile)); - fs.writeFileSync(outputPath, json, {encoding: 'utf8'}); +import css from 'css'; + +function indexOfRule(rules, selectors) { + const jj = selectors.length; + for (let i = 0, ii = rules.length; i < ii; ++i) { + const ruleSelectors = rules[i].selectors; + if (ruleSelectors.length !== jj) { continue; } + let okay = true; + for (let j = 0; j < jj; ++j) { + if (selectors[j] !== ruleSelectors[j]) { + okay = false; + break; + } + } + if (okay) { return i; } } + return -1; } +function removeProperty(styles, property, removedProperties) { + let removeCount = removedProperties.get(property); + if (typeof removeCount !== 'undefined') { return removeCount; } + removeCount = 0; + for (let i = 0, ii = styles.length; i < ii; ++i) { + const key = styles[i][0]; + if (key !== property) { continue; } + styles.splice(i, 1); + --i; + --ii; + ++removeCount; + } + removedProperties.set(property, removeCount); + return removeCount; +} -if (require.main === module) { - testMain(main, process.argv.slice(2)); +export function formatRulesJson(rules) { + // Manually format JSON, for improved compactness + // return JSON.stringify(rules, null, 4); + const indent1 = ' '; + const indent2 = indent1.repeat(2); + const indent3 = indent1.repeat(3); + let result = ''; + result += '['; + let index1 = 0; + for (const {selectors, styles} of rules) { + if (index1 > 0) { result += ','; } + result += `\n${indent1}{\n${indent2}"selectors": `; + if (selectors.length === 1) { + result += `[${JSON.stringify(selectors[0], null, 4)}]`; + } else { + result += JSON.stringify(selectors, null, 4).replace(/\n/g, '\n' + indent2); + } + result += `,\n${indent2}"styles": [`; + let index2 = 0; + for (const [key, value] of styles) { + if (index2 > 0) { result += ','; } + result += `\n${indent3}[${JSON.stringify(key)}, ${JSON.stringify(value)}]`; + ++index2; + } + if (index2 > 0) { result += `\n${indent2}`; } + result += `]\n${indent1}}`; + ++index1; + } + if (index1 > 0) { result += '\n'; } + result += ']'; + return result; } +export function generateRules(cssFile, overridesCssFile) { + const content1 = fs.readFileSync(cssFile, {encoding: 'utf8'}); + const content2 = fs.readFileSync(overridesCssFile, {encoding: 'utf8'}); + const stylesheet1 = css.parse(content1, {}).stylesheet; + const stylesheet2 = css.parse(content2, {}).stylesheet; + + const removePropertyPattern = /^remove-property\s+([\w\W]+)$/; + const removeRulePattern = /^remove-rule$/; + const propertySeparator = /\s+/; -module.exports = { - getTargets -}; + const rules = []; + + // Default stylesheet + for (const rule of stylesheet1.rules) { + if (rule.type !== 'rule') { continue; } + const {selectors, declarations} = rule; + const styles = []; + for (const declaration of declarations) { + if (declaration.type !== 'declaration') { console.log(declaration); continue; } + const {property, value} = declaration; + styles.push([property, value]); + } + if (styles.length > 0) { + rules.push({selectors, styles}); + } + } + + // Overrides + for (const rule of stylesheet2.rules) { + if (rule.type !== 'rule') { continue; } + const {selectors, declarations} = rule; + const removedProperties = new Map(); + for (const declaration of declarations) { + switch (declaration.type) { + case 'declaration': + { + const index = indexOfRule(rules, selectors); + let entry; + if (index >= 0) { + entry = rules[index]; + } else { + entry = {selectors, styles: []}; + rules.push(entry); + } + const {property, value} = declaration; + removeProperty(entry.styles, property, removedProperties); + entry.styles.push([property, value]); + } + break; + case 'comment': + { + const index = indexOfRule(rules, selectors); + if (index < 0) { throw new Error('Could not find rule with matching selectors'); } + const comment = declaration.comment.trim(); + let m; + if ((m = removePropertyPattern.exec(comment)) !== null) { + for (const property of m[1].split(propertySeparator)) { + const removeCount = removeProperty(rules[index].styles, property, removedProperties); + if (removeCount === 0) { throw new Error(`Property removal is unnecessary; ${property} does not exist`); } + } + } else if (removeRulePattern.test(comment)) { + rules.splice(index, 1); + } + } + break; + } + } + } + + // Remove empty + for (let i = 0, ii = rules.length; i < ii; ++i) { + if (rules[i].styles.length > 0) { continue; } + rules.splice(i, 1); + --i; + --ii; + } + + return rules; +} diff --git a/dev/lib/ucs2length.js b/dev/lib/ucs2length.js index 2e4a01cd..3b370493 100644 --- a/dev/lib/ucs2length.js +++ b/dev/lib/ucs2length.js @@ -14,5 +14,7 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -export {ucs2length} from 'ajv/dist/runtime/ucs2length'; +import ucs2length from 'ajv/dist/runtime/ucs2length.js'; +const ucs2length2 = ucs2length.default; +export {ucs2length2 as ucs2length}; diff --git a/dev/lib/z-worker.js b/dev/lib/z-worker.js new file mode 100644 index 00000000..f6a95ed3 --- /dev/null +++ b/dev/lib/z-worker.js @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2023 Yomitan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +import '../../node_modules/@zip.js/zip.js/lib/z-worker.js'; diff --git a/dev/lib/zip.js b/dev/lib/zip.js index 7560f5f8..b6e85451 100644 --- a/dev/lib/zip.js +++ b/dev/lib/zip.js @@ -14,4 +14,4 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -export * from '@zip.js/zip.js/lib/zip-full.js'; +export * from '@zip.js/zip.js/lib/zip.js'; diff --git a/dev/lint/global-declarations.js b/dev/lint/global-declarations.js deleted file mode 100644 index 7f90d227..00000000 --- a/dev/lint/global-declarations.js +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright (C) 2023 Yomitan Authors - * Copyright (C) 2020-2022 Yomichan Authors - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -const fs = require('fs'); -const path = require('path'); -const assert = require('assert'); -const {getAllFiles} = require('../util'); - - -function escapeRegExp(string) { - return string.replace(/[.*+\-?^${}()|[\]\\]/g, '\\$&'); -} - -function countOccurences(string, pattern) { - return (string.match(pattern) || []).length; -} - -function getNewline(string) { - const count1 = countOccurences(string, /(?:^|[^\r])\n/g); - const count2 = countOccurences(string, /\r\n/g); - const count3 = countOccurences(string, /\r(?:[^\n]|$)/g); - if (count2 > count1) { - return (count3 > count2) ? '\r' : '\r\n'; - } else { - return (count3 > count1) ? '\r' : '\n'; - } -} - -function getSubstringCount(string, substring) { - let count = 0; - const pattern = new RegExp(`\\b${escapeRegExp(substring)}\\b`, 'g'); - while (true) { - const match = pattern.exec(string); - if (match === null) { break; } - ++count; - } - return count; -} - - -function validateGlobals(fileName, fix) { - const pattern = /\/\*\s*global\s+([\w\W]*?)\*\//g; - const trimPattern = /^[\s,*]+|[\s,*]+$/g; - const splitPattern = /[\s,*]+/; - const source = fs.readFileSync(fileName, {encoding: 'utf8'}); - let match; - let first = true; - let endIndex = 0; - let newSource = ''; - const allGlobals = []; - const newline = getNewline(source); - while ((match = pattern.exec(source)) !== null) { - if (!first) { - console.error(`Encountered more than one global declaration in ${fileName}`); - return false; - } - first = false; - - const parts = match[1].replace(trimPattern, '').split(splitPattern); - parts.sort(); - - const actual = match[0]; - const expected = `/* global${parts.map((v) => `${newline} * ${v}`).join('')}${newline} */`; - - try { - assert.strictEqual(actual, expected); - } catch (e) { - console.error(`Global declaration error encountered in ${fileName}:`); - console.error(e.message); - if (!fix) { - return false; - } - } - - newSource += source.substring(0, match.index); - newSource += expected; - endIndex = match.index + match[0].length; - - allGlobals.push(...parts); - } - - newSource += source.substring(endIndex); - - // This is an approximate check to see if a global variable is unused. - // If the global appears in a comment, string, or similar, the check will pass. - let errorCount = 0; - for (const global of allGlobals) { - if (getSubstringCount(newSource, global) <= 1) { - console.error(`Global variable ${global} appears to be unused in ${fileName}`); - ++errorCount; - } - } - - if (fix) { - fs.writeFileSync(fileName, newSource, {encoding: 'utf8'}); - } - - return errorCount === 0; -} - - -function main() { - const fix = (process.argv.length >= 2 && process.argv[2] === '--fix'); - const directory = path.resolve(__dirname, '..', '..', 'ext'); - const pattern = /\.js$/; - const ignorePattern = /^lib[\\/]/; - const fileNames = getAllFiles(directory, (f) => pattern.test(f) && !ignorePattern.test(f)); - for (const fileName of fileNames) { - if (!validateGlobals(path.join(directory, fileName), fix)) { - process.exit(-1); - return; - } - } - process.exit(0); -} - - -if (require.main === module) { main(); } diff --git a/dev/lint/html-scripts.js b/dev/lint/html-scripts.js deleted file mode 100644 index db6e6ca4..00000000 --- a/dev/lint/html-scripts.js +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Copyright (C) 2023 Yomitan Authors - * Copyright (C) 2020-2022 Yomichan Authors - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -const fs = require('fs'); -const path = require('path'); -const assert = require('assert'); -const {JSDOM} = require('jsdom'); -const {getAllFiles} = require('../util'); - - -function lstatSyncSafe(fileName) { - try { - return fs.lstatSync(fileName); - } catch (e) { - return null; - } -} - -function validatePath(src, fileName, extDir) { - assert.ok(typeof src === 'string', `