diff options
Diffstat (limited to 'dev')
| -rw-r--r-- | dev/build.js | 50 | ||||
| -rw-r--r-- | dev/dictionary-validate.js | 118 | ||||
| -rw-r--r-- | dev/lint/global-declarations.js | 130 | ||||
| -rw-r--r-- | dev/schema-validate.js | 57 | ||||
| -rw-r--r-- | dev/util.js (renamed from dev/yomichan-util.js) | 71 | ||||
| -rw-r--r-- | dev/vm.js | 194 |
6 files changed, 572 insertions, 48 deletions
diff --git a/dev/build.js b/dev/build.js index 6afb82ae..b76a769b 100644 --- a/dev/build.js +++ b/dev/build.js @@ -19,8 +19,8 @@ const fs = require('fs'); const path = require('path'); const readline = require('readline'); const childProcess = require('child_process'); -const util = require('./yomichan-util'); -const {getAllFiles, getDefaultManifestAndVariants, createManifestString} = util; +const util = require('./util'); +const {getAllFiles, getDefaultManifestAndVariants, createManifestString, getArgs} = util; function clone(value) { @@ -208,54 +208,10 @@ async function build(manifest, buildDir, extDir, manifestPath, variantMap, varia } } -function getArs(args, argMap) { - let key = null; - let canKey = true; - let onKey = false; - for (const arg of args) { - onKey = false; - - if (canKey && arg.startsWith('--')) { - if (arg.length === 2) { - canKey = false; - key = null; - onKey = false; - } else { - key = arg.substring(2); - onKey = true; - } - } - - const target = argMap.get(key); - if (typeof target === 'boolean') { - argMap.set(key, true); - key = null; - } else if (typeof target === 'number') { - argMap.set(key, target + 1); - key = null; - } else if (target === null || typeof target === 'string') { - if (!onKey) { - argMap.set(key, arg); - key = null; - } - } else if (Array.isArray(target)) { - if (!onKey) { - target.push(arg); - key = null; - } - } else { - console.error(`Unknown argument: ${arg}`); - key = null; - } - } - - return argMap; -} - async function main() { const argv = process.argv.slice(2); - const args = getArs(argv, new Map([ + const args = getArgs(argv, new Map([ ['all', false], ['default', false], ['manifest', null], diff --git a/dev/dictionary-validate.js b/dev/dictionary-validate.js new file mode 100644 index 00000000..cf449b09 --- /dev/null +++ b/dev/dictionary-validate.js @@ -0,0 +1,118 @@ +/* + * Copyright (C) 2020 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +const fs = require('fs'); +const path = require('path'); +const {JSZip} = require('./util'); +const {VM} = require('./vm'); + +const vm = new VM(); +vm.execute([ + 'mixed/js/core.js', + 'mixed/js/cache-map.js', + 'bg/js/json-schema.js' +]); +const JsonSchemaValidator = vm.get('JsonSchemaValidator'); + + +function readSchema(relativeFileName) { + const fileName = path.join(__dirname, relativeFileName); + const source = fs.readFileSync(fileName, {encoding: 'utf8'}); + return JSON.parse(source); +} + + +async function validateDictionaryBanks(zip, fileNameFormat, schema) { + let index = 1; + while (true) { + const fileName = fileNameFormat.replace(/\?/, index); + + const file = zip.files[fileName]; + if (!file) { break; } + + const data = JSON.parse(await file.async('string')); + new JsonSchemaValidator().validate(data, schema); + + ++index; + } +} + +async function validateDictionary(archive, schemas) { + const indexFile = archive.files['index.json']; + if (!indexFile) { + throw new Error('No dictionary index found in archive'); + } + + const index = JSON.parse(await indexFile.async('string')); + const version = index.format || index.version; + + new JsonSchemaValidator().validate(index, schemas.index); + + await validateDictionaryBanks(archive, 'term_bank_?.json', version === 1 ? schemas.termBankV1 : schemas.termBankV3); + await validateDictionaryBanks(archive, 'term_meta_bank_?.json', schemas.termMetaBankV3); + await validateDictionaryBanks(archive, 'kanji_bank_?.json', version === 1 ? schemas.kanjiBankV1 : schemas.kanjiBankV3); + await validateDictionaryBanks(archive, 'kanji_meta_bank_?.json', schemas.kanjiMetaBankV3); + await validateDictionaryBanks(archive, 'tag_bank_?.json', schemas.tagBankV3); +} + +function getSchemas() { + return { + index: readSchema('../ext/bg/data/dictionary-index-schema.json'), + kanjiBankV1: readSchema('../ext/bg/data/dictionary-kanji-bank-v1-schema.json'), + kanjiBankV3: readSchema('../ext/bg/data/dictionary-kanji-bank-v3-schema.json'), + kanjiMetaBankV3: readSchema('../ext/bg/data/dictionary-kanji-meta-bank-v3-schema.json'), + tagBankV3: readSchema('../ext/bg/data/dictionary-tag-bank-v3-schema.json'), + termBankV1: readSchema('../ext/bg/data/dictionary-term-bank-v1-schema.json'), + termBankV3: readSchema('../ext/bg/data/dictionary-term-bank-v3-schema.json'), + termMetaBankV3: readSchema('../ext/bg/data/dictionary-term-meta-bank-v3-schema.json') + }; +} + + +async function main() { + const dictionaryFileNames = process.argv.slice(2); + if (dictionaryFileNames.length === 0) { + console.log([ + 'Usage:', + ' node dictionary-validate <dictionary-file-names>...' + ].join('\n')); + return; + } + + const schemas = getSchemas(); + + for (const dictionaryFileName of dictionaryFileNames) { + try { + console.log(`Validating ${dictionaryFileName}...`); + const source = fs.readFileSync(dictionaryFileName); + const archive = await JSZip.loadAsync(source); + await validateDictionary(archive, schemas); + console.log('No issues found'); + } catch (e) { + console.warn(e); + } + } +} + + +if (require.main === module) { main(); } + + +module.exports = { + getSchemas, + validateDictionary +}; diff --git a/dev/lint/global-declarations.js b/dev/lint/global-declarations.js new file mode 100644 index 00000000..5448df85 --- /dev/null +++ b/dev/lint/global-declarations.js @@ -0,0 +1,130 @@ +/* + * Copyright (C) 2020 Yomichan Authors + * Author: Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +const fs = require('fs'); +const path = require('path'); +const assert = require('assert'); +const {getAllFiles} = require('../util'); + + +function countOccurences(string, pattern) { + return (string.match(pattern) || []).length; +} + +function getNewline(string) { + const count1 = countOccurences(string, /(?:^|[^\r])\n/g); + const count2 = countOccurences(string, /\r\n/g); + const count3 = countOccurences(string, /\r(?:[^\n]|$)/g); + if (count2 > count1) { + return (count3 > count2) ? '\r' : '\r\n'; + } else { + return (count3 > count1) ? '\r' : '\n'; + } +} + +function getSubstringCount(string, substring) { + let start = 0; + let count = 0; + while (true) { + const pos = string.indexOf(substring, start); + if (pos < 0) { break; } + ++count; + start = pos + substring.length; + } + return count; +} + + +function validateGlobals(fileName, fix) { + const pattern = /\/\*\s*global\s+([\w\W]*?)\*\//g; + const trimPattern = /^[\s,*]+|[\s,*]+$/g; + const splitPattern = /[\s,*]+/; + const source = fs.readFileSync(fileName, {encoding: 'utf8'}); + let match; + let first = true; + let endIndex = 0; + let newSource = ''; + const allGlobals = []; + const newline = getNewline(source); + while ((match = pattern.exec(source)) !== null) { + if (!first) { + console.error(`Encountered more than one global declaration in ${fileName}`); + return false; + } + first = false; + + const parts = match[1].replace(trimPattern, '').split(splitPattern); + parts.sort(); + + const actual = match[0]; + const expected = `/* global${parts.map((v) => `${newline} * ${v}`).join('')}${newline} */`; + + try { + assert.strictEqual(actual, expected); + } catch (e) { + console.error(`Global declaration error encountered in ${fileName}:`); + console.error(e.message); + if (!fix) { + return false; + } + } + + newSource += source.substring(0, match.index); + newSource += expected; + endIndex = match.index + match[0].length; + + allGlobals.push(...parts); + } + + newSource += source.substring(endIndex); + + // This is an approximate check to see if a global variable is unused. + // If the global appears in a comment, string, or similar, the check will pass. + let errorCount = 0; + for (const global of allGlobals) { + if (getSubstringCount(newSource, global) <= 1) { + console.error(`Global variable ${global} appears to be unused in ${fileName}`); + ++errorCount; + } + } + + if (fix) { + fs.writeFileSync(fileName, newSource, {encoding: 'utf8'}); + } + + return errorCount === 0; +} + + +function main() { + const fix = (process.argv.length >= 2 && process.argv[2] === '--fix'); + const directory = path.resolve(__dirname, '..', '..', 'ext'); + const pattern = /\.js$/; + const ignorePattern = /[\\/]ext[\\/]mixed[\\/]lib[\\/]/; + const fileNames = getAllFiles(directory, null, (f) => pattern.test(f) && !ignorePattern.test(f)); + for (const fileName of fileNames) { + if (!validateGlobals(fileName, fix)) { + process.exit(-1); + return; + } + } + process.exit(0); +} + + +if (require.main === module) { main(); } diff --git a/dev/schema-validate.js b/dev/schema-validate.js new file mode 100644 index 00000000..7b6bb5e7 --- /dev/null +++ b/dev/schema-validate.js @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2020 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +const fs = require('fs'); +const {VM} = require('./vm'); + +const vm = new VM(); +vm.execute([ + 'mixed/js/core.js', + 'mixed/js/cache-map.js', + 'bg/js/json-schema.js' +]); +const JsonSchemaValidator = vm.get('JsonSchemaValidator'); + + +function main() { + const args = process.argv.slice(2); + if (args.length < 2) { + console.log([ + 'Usage:', + ' node schema-validate <schema-file-name> <data-file-names>...' + ].join('\n')); + return; + } + + const schemaSource = fs.readFileSync(args[0], {encoding: 'utf8'}); + const schema = JSON.parse(schemaSource); + + for (const dataFileName of args.slice(1)) { + try { + console.log(`Validating ${dataFileName}...`); + const dataSource = fs.readFileSync(dataFileName, {encoding: 'utf8'}); + const data = JSON.parse(dataSource); + new JsonSchemaValidator().validate(data, schema); + console.log('No issues found'); + } catch (e) { + console.warn(e); + } + } +} + + +if (require.main === module) { main(); } diff --git a/dev/yomichan-util.js b/dev/util.js index 2d8f31ea..971837af 100644 --- a/dev/yomichan-util.js +++ b/dev/util.js @@ -32,6 +32,50 @@ function getJSZip() { } +function getArgs(args, argMap) { + let key = null; + let canKey = true; + let onKey = false; + for (const arg of args) { + onKey = false; + + if (canKey && arg.startsWith('--')) { + if (arg.length === 2) { + canKey = false; + key = null; + onKey = false; + } else { + key = arg.substring(2); + onKey = true; + } + } + + const target = argMap.get(key); + if (typeof target === 'boolean') { + argMap.set(key, true); + key = null; + } else if (typeof target === 'number') { + argMap.set(key, target + 1); + key = null; + } else if (target === null || typeof target === 'string') { + if (!onKey) { + argMap.set(key, arg); + key = null; + } + } else if (Array.isArray(target)) { + if (!onKey) { + target.push(arg); + key = null; + } + } else { + console.error(`Unknown argument: ${arg}`); + key = null; + } + } + + return argMap; +} + function getAllFiles(baseDirectory, relativeTo=null, predicate=null) { const results = []; const directories = [baseDirectory]; @@ -69,11 +113,36 @@ function createManifestString(manifest) { return JSON.stringify(manifest, null, 4) + '\n'; } +function createDictionaryArchive(dictionaryDirectory, dictionaryName) { + const fileNames = fs.readdirSync(dictionaryDirectory); + + const JSZip2 = getJSZip(); + const archive = new JSZip2(); + + for (const fileName of fileNames) { + if (/\.json$/.test(fileName)) { + const content = fs.readFileSync(path.join(dictionaryDirectory, fileName), {encoding: 'utf8'}); + const json = JSON.parse(content); + if (fileName === 'index.json' && typeof dictionaryName === 'string') { + json.title = dictionaryName; + } + archive.file(fileName, JSON.stringify(json, null, 0)); + } else { + const content = fs.readFileSync(path.join(dictionaryDirectory, fileName), {encoding: null}); + archive.file(fileName, content); + } + } + + return archive; +} + module.exports = { get JSZip() { return getJSZip(); }, + getArgs, getAllFiles, getDefaultManifest, getDefaultManifestAndVariants, - createManifestString + createManifestString, + createDictionaryArchive }; diff --git a/dev/vm.js b/dev/vm.js new file mode 100644 index 00000000..79e92772 --- /dev/null +++ b/dev/vm.js @@ -0,0 +1,194 @@ +/* + * Copyright (C) 2020 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +const fs = require('fs'); +const vm = require('vm'); +const path = require('path'); +const assert = require('assert'); + + +function getContextEnvironmentRecords(context, names) { + // Enables export of values from the declarative environment record + if (!Array.isArray(names) || names.length === 0) { + return []; + } + + let scriptSource = '(() => {\n "use strict";\n const results = [];'; + for (const name of names) { + scriptSource += `\n try { results.push(${name}); } catch (e) { results.push(void 0); }`; + } + scriptSource += '\n return results;\n})();'; + + const script = new vm.Script(scriptSource, {filename: 'getContextEnvironmentRecords'}); + + const contextHasNames = Object.prototype.hasOwnProperty.call(context, 'names'); + const contextNames = context.names; + context.names = names; + + const results = script.runInContext(context, {}); + + if (contextHasNames) { + context.names = contextNames; + } else { + delete context.names; + } + + return Array.from(results); +} + +function isDeepStrictEqual(val1, val2) { + if (val1 === val2) { return true; } + + if (Array.isArray(val1)) { + if (Array.isArray(val2)) { + return isArrayDeepStrictEqual(val1, val2); + } + } else if (typeof val1 === 'object' && val1 !== null) { + if (typeof val2 === 'object' && val2 !== null) { + return isObjectDeepStrictEqual(val1, val2); + } + } + + return false; +} + +function isArrayDeepStrictEqual(val1, val2) { + const ii = val1.length; + if (ii !== val2.length) { return false; } + + for (let i = 0; i < ii; ++i) { + if (!isDeepStrictEqual(val1[i], val2[i])) { + return false; + } + } + + return true; +} + +function isObjectDeepStrictEqual(val1, val2) { + const keys1 = Object.keys(val1); + const keys2 = Object.keys(val2); + + if (keys1.length !== keys2.length) { return false; } + + const keySet = new Set(keys1); + for (const key of keys2) { + if (!keySet.delete(key)) { return false; } + } + + for (const key of keys1) { + if (!isDeepStrictEqual(val1[key], val2[key])) { + return false; + } + } + + const tag1 = Object.prototype.toString.call(val1); + const tag2 = Object.prototype.toString.call(val2); + if (tag1 !== tag2) { return false; } + + return true; +} + +function deepStrictEqual(actual, expected) { + try { + // This will fail on prototype === comparison on cross context objects + assert.deepStrictEqual(actual, expected); + } catch (e) { + if (!isDeepStrictEqual(actual, expected)) { + throw e; + } + } +} + + +function createURLClass() { + const BaseURL = URL; + return function URL(url) { + const u = new BaseURL(url); + this.hash = u.hash; + this.host = u.host; + this.hostname = u.hostname; + this.href = u.href; + this.origin = u.origin; + this.password = u.password; + this.pathname = u.pathname; + this.port = u.port; + this.protocol = u.protocol; + this.search = u.search; + this.searchParams = u.searchParams; + this.username = u.username; + }; +} + + +class VM { + constructor(context={}) { + context.URL = createURLClass(); + this._context = vm.createContext(context); + this._assert = { + deepStrictEqual + }; + } + + get context() { + return this._context; + } + + get assert() { + return this._assert; + } + + get(names) { + if (typeof names === 'string') { + return getContextEnvironmentRecords(this._context, [names])[0]; + } else if (Array.isArray(names)) { + return getContextEnvironmentRecords(this._context, names); + } else { + throw new Error('Invalid argument'); + } + } + + set(values) { + if (typeof values === 'object' && values !== null) { + Object.assign(this._context, values); + } else { + throw new Error('Invalid argument'); + } + } + + execute(fileNames) { + const single = !Array.isArray(fileNames); + if (single) { + fileNames = [fileNames]; + } + + const results = []; + for (const fileName of fileNames) { + const absoluteFileName = path.resolve(__dirname, '..', 'ext', fileName); + const source = fs.readFileSync(absoluteFileName, {encoding: 'utf8'}); + const script = new vm.Script(source, {filename: absoluteFileName}); + results.push(script.runInContext(this._context, {})); + } + + return single ? results[0] : results; + } +} + + +module.exports = { + VM +}; |