Dev/test script organization (#846)

* Move lint scripts

* Move dictionary-validate.js

* Move schema-validate.js

* Move createTestDictionaryArchive, remove yomichan-test.js

* Rename yomichan-util.js to util.js

* Move test/yomichan-vm.js to dev/vm.js

* Move getArgs into util.js (and fix name)

* Create test-all.js

* Update test-code script

6 files changed, 572 insertions, 48 deletions

diff --git a/dev/build.js b/dev/build.js
index 6afb82ae..b76a769b 100644
--- a/dev/build.js
+++ b/dev/build.js
@@ -19,8 +19,8 @@ const fs = require('fs');
 const path = require('path');
 const readline = require('readline');
 const childProcess = require('child_process');
-const util = require('./yomichan-util');
-const {getAllFiles, getDefaultManifestAndVariants, createManifestString} = util;
+const util = require('./util');
+const {getAllFiles, getDefaultManifestAndVariants, createManifestString, getArgs} = util;
 
 
 function clone(value) {
@@ -208,54 +208,10 @@ async function build(manifest, buildDir, extDir, manifestPath, variantMap, varia
     }
 }
 
-function getArs(args, argMap) {
-    let key = null;
-    let canKey = true;
-    let onKey = false;
-    for (const arg of args) {
-        onKey = false;
-
-        if (canKey && arg.startsWith('--')) {
-            if (arg.length === 2) {
-                canKey = false;
-                key = null;
-                onKey = false;
-            } else {
-                key = arg.substring(2);
-                onKey = true;
-            }
-        }
-
-        const target = argMap.get(key);
-        if (typeof target === 'boolean') {
-            argMap.set(key, true);
-            key = null;
-        } else if (typeof target === 'number') {
-            argMap.set(key, target + 1);
-            key = null;
-        } else if (target === null || typeof target === 'string') {
-            if (!onKey) {
-                argMap.set(key, arg);
-                key = null;
-            }
-        } else if (Array.isArray(target)) {
-            if (!onKey) {
-                target.push(arg);
-                key = null;
-            }
-        } else {
-            console.error(`Unknown argument: ${arg}`);
-            key = null;
-        }
-    }
-
-    return argMap;
-}
-
 
 async function main() {
     const argv = process.argv.slice(2);
-    const args = getArs(argv, new Map([
+    const args = getArgs(argv, new Map([
         ['all', false],
         ['default', false],
         ['manifest', null],
diff --git a/dev/dictionary-validate.js b/dev/dictionary-validate.js
new file mode 100644
index 00000000..cf449b09
--- /dev/null
+++ b/dev/dictionary-validate.js
@@ -0,0 +1,118 @@
+/*
+ * Copyright (C) 2020  Yomichan Authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const {JSZip} = require('./util');
+const {VM} = require('./vm');
+
+const vm = new VM();
+vm.execute([
+    'mixed/js/core.js',
+    'mixed/js/cache-map.js',
+    'bg/js/json-schema.js'
+]);
+const JsonSchemaValidator = vm.get('JsonSchemaValidator');
+
+
+function readSchema(relativeFileName) {
+    const fileName = path.join(__dirname, relativeFileName);
+    const source = fs.readFileSync(fileName, {encoding: 'utf8'});
+    return JSON.parse(source);
+}
+
+
+async function validateDictionaryBanks(zip, fileNameFormat, schema) {
+    let index = 1;
+    while (true) {
+        const fileName = fileNameFormat.replace(/\?/, index);
+
+        const file = zip.files[fileName];
+        if (!file) { break; }
+
+        const data = JSON.parse(await file.async('string'));
+        new JsonSchemaValidator().validate(data, schema);
+
+        ++index;
+    }
+}
+
+async function validateDictionary(archive, schemas) {
+    const indexFile = archive.files['index.json'];
+    if (!indexFile) {
+        throw new Error('No dictionary index found in archive');
+    }
+
+    const index = JSON.parse(await indexFile.async('string'));
+    const version = index.format || index.version;
+
+    new JsonSchemaValidator().validate(index, schemas.index);
+
+    await validateDictionaryBanks(archive, 'term_bank_?.json', version === 1 ? schemas.termBankV1 : schemas.termBankV3);
+    await validateDictionaryBanks(archive, 'term_meta_bank_?.json', schemas.termMetaBankV3);
+    await validateDictionaryBanks(archive, 'kanji_bank_?.json', version === 1 ? schemas.kanjiBankV1 : schemas.kanjiBankV3);
+    await validateDictionaryBanks(archive, 'kanji_meta_bank_?.json', schemas.kanjiMetaBankV3);
+    await validateDictionaryBanks(archive, 'tag_bank_?.json', schemas.tagBankV3);
+}
+
+function getSchemas() {
+    return {
+        index: readSchema('../ext/bg/data/dictionary-index-schema.json'),
+        kanjiBankV1: readSchema('../ext/bg/data/dictionary-kanji-bank-v1-schema.json'),
+        kanjiBankV3: readSchema('../ext/bg/data/dictionary-kanji-bank-v3-schema.json'),
+        kanjiMetaBankV3: readSchema('../ext/bg/data/dictionary-kanji-meta-bank-v3-schema.json'),
+        tagBankV3: readSchema('../ext/bg/data/dictionary-tag-bank-v3-schema.json'),
+        termBankV1: readSchema('../ext/bg/data/dictionary-term-bank-v1-schema.json'),
+        termBankV3: readSchema('../ext/bg/data/dictionary-term-bank-v3-schema.json'),
+        termMetaBankV3: readSchema('../ext/bg/data/dictionary-term-meta-bank-v3-schema.json')
+    };
+}
+
+
+async function main() {
+    const dictionaryFileNames = process.argv.slice(2);
+    if (dictionaryFileNames.length === 0) {
+        console.log([
+            'Usage:',
+            '  node dictionary-validate <dictionary-file-names>...'
+        ].join('\n'));
+        return;
+    }
+
+    const schemas = getSchemas();
+
+    for (const dictionaryFileName of dictionaryFileNames) {
+        try {
+            console.log(`Validating ${dictionaryFileName}...`);
+            const source = fs.readFileSync(dictionaryFileName);
+            const archive = await JSZip.loadAsync(source);
+            await validateDictionary(archive, schemas);
+            console.log('No issues found');
+        } catch (e) {
+            console.warn(e);
+        }
+    }
+}
+
+
+if (require.main === module) { main(); }
+
+
+module.exports = {
+    getSchemas,
+    validateDictionary
+};
diff --git a/dev/lint/global-declarations.js b/dev/lint/global-declarations.js
new file mode 100644
index 00000000..5448df85
--- /dev/null
+++ b/dev/lint/global-declarations.js
@@ -0,0 +1,130 @@
+/*
+ * Copyright (C) 2020  Yomichan Authors
+ * Author: Yomichan Authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const assert = require('assert');
+const {getAllFiles} = require('../util');
+
+
+function countOccurences(string, pattern) {
+    return (string.match(pattern) || []).length;
+}
+
+function getNewline(string) {
+    const count1 = countOccurences(string, /(?:^|[^\r])\n/g);
+    const count2 = countOccurences(string, /\r\n/g);
+    const count3 = countOccurences(string, /\r(?:[^\n]|$)/g);
+    if (count2 > count1) {
+        return (count3 > count2) ? '\r' : '\r\n';
+    } else {
+        return (count3 > count1) ? '\r' : '\n';
+    }
+}
+
+function getSubstringCount(string, substring) {
+    let start = 0;
+    let count = 0;
+    while (true) {
+        const pos = string.indexOf(substring, start);
+        if (pos < 0) { break; }
+        ++count;
+        start = pos + substring.length;
+    }
+    return count;
+}
+
+
+function validateGlobals(fileName, fix) {
+    const pattern = /\/\*\s*global\s+([\w\W]*?)\*\//g;
+    const trimPattern = /^[\s,*]+|[\s,*]+$/g;
+    const splitPattern = /[\s,*]+/;
+    const source = fs.readFileSync(fileName, {encoding: 'utf8'});
+    let match;
+    let first = true;
+    let endIndex = 0;
+    let newSource = '';
+    const allGlobals = [];
+    const newline = getNewline(source);
+    while ((match = pattern.exec(source)) !== null) {
+        if (!first) {
+            console.error(`Encountered more than one global declaration in ${fileName}`);
+            return false;
+        }
+        first = false;
+
+        const parts = match[1].replace(trimPattern, '').split(splitPattern);
+        parts.sort();
+
+        const actual = match[0];
+        const expected = `/* global${parts.map((v) => `${newline} * ${v}`).join('')}${newline} */`;
+
+        try {
+            assert.strictEqual(actual, expected);
+        } catch (e) {
+            console.error(`Global declaration error encountered in ${fileName}:`);
+            console.error(e.message);
+            if (!fix) {
+                return false;
+            }
+        }
+
+        newSource += source.substring(0, match.index);
+        newSource += expected;
+        endIndex = match.index + match[0].length;
+
+        allGlobals.push(...parts);
+    }
+
+    newSource += source.substring(endIndex);
+
+    // This is an approximate check to see if a global variable is unused.
+    // If the global appears in a comment, string, or similar, the check will pass.
+    let errorCount = 0;
+    for (const global of allGlobals) {
+        if (getSubstringCount(newSource, global) <= 1) {
+            console.error(`Global variable ${global} appears to be unused in ${fileName}`);
+            ++errorCount;
+        }
+    }
+
+    if (fix) {
+        fs.writeFileSync(fileName, newSource, {encoding: 'utf8'});
+    }
+
+    return errorCount === 0;
+}
+
+
+function main() {
+    const fix = (process.argv.length >= 2 && process.argv[2] === '--fix');
+    const directory = path.resolve(__dirname, '..', '..', 'ext');
+    const pattern = /\.js$/;
+    const ignorePattern = /[\\/]ext[\\/]mixed[\\/]lib[\\/]/;
+    const fileNames = getAllFiles(directory, null, (f) => pattern.test(f) && !ignorePattern.test(f));
+    for (const fileName of fileNames) {
+        if (!validateGlobals(fileName, fix)) {
+            process.exit(-1);
+            return;
+        }
+    }
+    process.exit(0);
+}
+
+
+if (require.main === module) { main(); }
diff --git a/dev/schema-validate.js b/dev/schema-validate.js
new file mode 100644
index 00000000..7b6bb5e7
--- /dev/null
+++ b/dev/schema-validate.js
@@ -0,0 +1,57 @@
+/*
+ * Copyright (C) 2020  Yomichan Authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+const fs = require('fs');
+const {VM} = require('./vm');
+
+const vm = new VM();
+vm.execute([
+    'mixed/js/core.js',
+    'mixed/js/cache-map.js',
+    'bg/js/json-schema.js'
+]);
+const JsonSchemaValidator = vm.get('JsonSchemaValidator');
+
+
+function main() {
+    const args = process.argv.slice(2);
+    if (args.length < 2) {
+        console.log([
+            'Usage:',
+            '  node schema-validate <schema-file-name> <data-file-names>...'
+        ].join('\n'));
+        return;
+    }
+
+    const schemaSource = fs.readFileSync(args[0], {encoding: 'utf8'});
+    const schema = JSON.parse(schemaSource);
+
+    for (const dataFileName of args.slice(1)) {
+        try {
+            console.log(`Validating ${dataFileName}...`);
+            const dataSource = fs.readFileSync(dataFileName, {encoding: 'utf8'});
+            const data = JSON.parse(dataSource);
+            new JsonSchemaValidator().validate(data, schema);
+            console.log('No issues found');
+        } catch (e) {
+            console.warn(e);
+        }
+    }
+}
+
+
+if (require.main === module) { main(); }
diff --git a/dev/yomichan-util.js b/dev/util.js
index 2d8f31ea..971837af 100644
--- a/dev/yomichan-util.js
+++ b/dev/util.js
@@ -32,6 +32,50 @@ function getJSZip() {
 }
 
 
+function getArgs(args, argMap) {
+    let key = null;
+    let canKey = true;
+    let onKey = false;
+    for (const arg of args) {
+        onKey = false;
+
+        if (canKey && arg.startsWith('--')) {
+            if (arg.length === 2) {
+                canKey = false;
+                key = null;
+                onKey = false;
+            } else {
+                key = arg.substring(2);
+                onKey = true;
+            }
+        }
+
+        const target = argMap.get(key);
+        if (typeof target === 'boolean') {
+            argMap.set(key, true);
+            key = null;
+        } else if (typeof target === 'number') {
+            argMap.set(key, target + 1);
+            key = null;
+        } else if (target === null || typeof target === 'string') {
+            if (!onKey) {
+                argMap.set(key, arg);
+                key = null;
+            }
+        } else if (Array.isArray(target)) {
+            if (!onKey) {
+                target.push(arg);
+                key = null;
+            }
+        } else {
+            console.error(`Unknown argument: ${arg}`);
+            key = null;
+        }
+    }
+
+    return argMap;
+}
+
 function getAllFiles(baseDirectory, relativeTo=null, predicate=null) {
     const results = [];
     const directories = [baseDirectory];
@@ -69,11 +113,36 @@ function createManifestString(manifest) {
     return JSON.stringify(manifest, null, 4) + '\n';
 }
 
+function createDictionaryArchive(dictionaryDirectory, dictionaryName) {
+    const fileNames = fs.readdirSync(dictionaryDirectory);
+
+    const JSZip2 = getJSZip();
+    const archive = new JSZip2();
+
+    for (const fileName of fileNames) {
+        if (/\.json$/.test(fileName)) {
+            const content = fs.readFileSync(path.join(dictionaryDirectory, fileName), {encoding: 'utf8'});
+            const json = JSON.parse(content);
+            if (fileName === 'index.json' && typeof dictionaryName === 'string') {
+                json.title = dictionaryName;
+            }
+            archive.file(fileName, JSON.stringify(json, null, 0));
+        } else {
+            const content = fs.readFileSync(path.join(dictionaryDirectory, fileName), {encoding: null});
+            archive.file(fileName, content);
+        }
+    }
+
+    return archive;
+}
+
 
 module.exports = {
     get JSZip() { return getJSZip(); },
+    getArgs,
     getAllFiles,
     getDefaultManifest,
     getDefaultManifestAndVariants,
-    createManifestString
+    createManifestString,
+    createDictionaryArchive
 };
diff --git a/dev/vm.js b/dev/vm.js
new file mode 100644
index 00000000..79e92772
--- /dev/null
+++ b/dev/vm.js
@@ -0,0 +1,194 @@
+/*
+ * Copyright (C) 2020  Yomichan Authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+const fs = require('fs');
+const vm = require('vm');
+const path = require('path');
+const assert = require('assert');
+
+
+function getContextEnvironmentRecords(context, names) {
+    // Enables export of values from the declarative environment record
+    if (!Array.isArray(names) || names.length === 0) {
+        return [];
+    }
+
+    let scriptSource = '(() => {\n    "use strict";\n    const results = [];';
+    for (const name of names) {
+        scriptSource += `\n    try { results.push(${name}); } catch (e) { results.push(void 0); }`;
+    }
+    scriptSource += '\n    return results;\n})();';
+
+    const script = new vm.Script(scriptSource, {filename: 'getContextEnvironmentRecords'});
+
+    const contextHasNames = Object.prototype.hasOwnProperty.call(context, 'names');
+    const contextNames = context.names;
+    context.names = names;
+
+    const results = script.runInContext(context, {});
+
+    if (contextHasNames) {
+        context.names = contextNames;
+    } else {
+        delete context.names;
+    }
+
+    return Array.from(results);
+}
+
+function isDeepStrictEqual(val1, val2) {
+    if (val1 === val2) { return true; }
+
+    if (Array.isArray(val1)) {
+        if (Array.isArray(val2)) {
+            return isArrayDeepStrictEqual(val1, val2);
+        }
+    } else if (typeof val1 === 'object' && val1 !== null) {
+        if (typeof val2 === 'object' && val2 !== null) {
+            return isObjectDeepStrictEqual(val1, val2);
+        }
+    }
+
+    return false;
+}
+
+function isArrayDeepStrictEqual(val1, val2) {
+    const ii = val1.length;
+    if (ii !== val2.length) { return false; }
+
+    for (let i = 0; i < ii; ++i) {
+        if (!isDeepStrictEqual(val1[i], val2[i])) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+function isObjectDeepStrictEqual(val1, val2) {
+    const keys1 = Object.keys(val1);
+    const keys2 = Object.keys(val2);
+
+    if (keys1.length !== keys2.length) { return false; }
+
+    const keySet = new Set(keys1);
+    for (const key of keys2) {
+        if (!keySet.delete(key)) { return false; }
+    }
+
+    for (const key of keys1) {
+        if (!isDeepStrictEqual(val1[key], val2[key])) {
+            return false;
+        }
+    }
+
+    const tag1 = Object.prototype.toString.call(val1);
+    const tag2 = Object.prototype.toString.call(val2);
+    if (tag1 !== tag2) { return false; }
+
+    return true;
+}
+
+function deepStrictEqual(actual, expected) {
+    try {
+        // This will fail on prototype === comparison on cross context objects
+        assert.deepStrictEqual(actual, expected);
+    } catch (e) {
+        if (!isDeepStrictEqual(actual, expected)) {
+            throw e;
+        }
+    }
+}
+
+
+function createURLClass() {
+    const BaseURL = URL;
+    return function URL(url) {
+        const u = new BaseURL(url);
+        this.hash = u.hash;
+        this.host = u.host;
+        this.hostname = u.hostname;
+        this.href = u.href;
+        this.origin = u.origin;
+        this.password = u.password;
+        this.pathname = u.pathname;
+        this.port = u.port;
+        this.protocol = u.protocol;
+        this.search = u.search;
+        this.searchParams = u.searchParams;
+        this.username = u.username;
+    };
+}
+
+
+class VM {
+    constructor(context={}) {
+        context.URL = createURLClass();
+        this._context = vm.createContext(context);
+        this._assert = {
+            deepStrictEqual
+        };
+    }
+
+    get context() {
+        return this._context;
+    }
+
+    get assert() {
+        return this._assert;
+    }
+
+    get(names) {
+        if (typeof names === 'string') {
+            return getContextEnvironmentRecords(this._context, [names])[0];
+        } else if (Array.isArray(names)) {
+            return getContextEnvironmentRecords(this._context, names);
+        } else {
+            throw new Error('Invalid argument');
+        }
+    }
+
+    set(values) {
+        if (typeof values === 'object' && values !== null) {
+            Object.assign(this._context, values);
+        } else {
+            throw new Error('Invalid argument');
+        }
+    }
+
+    execute(fileNames) {
+        const single = !Array.isArray(fileNames);
+        if (single) {
+            fileNames = [fileNames];
+        }
+
+        const results = [];
+        for (const fileName of fileNames) {
+            const absoluteFileName = path.resolve(__dirname, '..', 'ext', fileName);
+            const source = fs.readFileSync(absoluteFileName, {encoding: 'utf8'});
+            const script = new vm.Script(source, {filename: absoluteFileName});
+            results.push(script.runInContext(this._context, {}));
+        }
+
+        return single ? results[0] : results;
+    }
+}
+
+
+module.exports = {
+    VM
+};