From 8733e324ecbe10bcb4bc9f1a0b9568c7f32429d3 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sat, 1 Feb 2020 22:41:02 -0500 Subject: Create script to validate dictionary files --- test/dictionary-validate.js | 90 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 test/dictionary-validate.js (limited to 'test/dictionary-validate.js') diff --git a/test/dictionary-validate.js b/test/dictionary-validate.js new file mode 100644 index 00000000..971c4971 --- /dev/null +++ b/test/dictionary-validate.js @@ -0,0 +1,90 @@ +const fs = require('fs'); +const path = require('path'); + +process.noDeprecation = true; // Suppress a warning about JSZip +const JSZip = require(path.join(__dirname, '../ext/mixed/lib/jszip.min.js')); +process.noDeprecation = false; + +const jsonSchemaFileName = path.join(__dirname, '../ext/bg/js/json-schema.js'); +const jsonSchemaFileSource = fs.readFileSync(jsonSchemaFileName, {encoding: 'utf8'}); +const JsonSchema = Function(`'use strict';${jsonSchemaFileSource};return JsonSchema;`)(); + + +function readSchema(relativeFileName) { + const fileName = path.join(__dirname, relativeFileName); + const source = fs.readFileSync(fileName, {encoding: 'utf8'}); + return JSON.parse(source); +} + + +async function validateDictionaryBanks(zip, fileNameFormat, schema) { + let index = 1; + while (true) { + const fileName = fileNameFormat.replace(/%s/, index); + + const file = zip.files[fileName]; + if (!file) { break; } + + const data = JSON.parse(await file.async('string')); + JsonSchema.validate(data, schema); + + ++index; + } +} + +async function validateDictionary(fileName, schemas) { + const source = fs.readFileSync(fileName); + const zip = await JSZip.loadAsync(source); + + const indexFile = zip.files['index.json']; + if (!indexFile) { + throw new Error('No dictionary index found in archive'); + } + + const index = JSON.parse(await indexFile.async('string')); + const version = index.format || index.version; + + JsonSchema.validate(index, schemas.index); + + await validateDictionaryBanks(zip, 'term_bank_%s.json', version === 1 ? schemas.termBankV1 : schemas.termBankV3); + await validateDictionaryBanks(zip, 'term_meta_bank_%s.json', schemas.termMetaBankV3); + await validateDictionaryBanks(zip, 'kanji_bank_%s.json', version === 1 ? schemas.kanjiBankV1 : schemas.kanjiBankV3); + await validateDictionaryBanks(zip, 'kanji_meta_bank_%s.json', schemas.kanjiMetaBankV3); + await validateDictionaryBanks(zip, 'tag_bank_%s.json', schemas.tagBankV3); +} + + +async function main() { + const dictionaryFileNames = process.argv.slice(2); + if (dictionaryFileNames.length === 0) { + console.log([ + 'Usage:', + ' node dictionary-validate ...' + ].join('\n')); + return; + } + + const schemas = { + index: readSchema('../ext/bg/data/dictionary-index-schema.json'), + kanjiBankV1: readSchema('../ext/bg/data/dictionary-kanji-bank-v1-schema.json'), + kanjiBankV3: readSchema('../ext/bg/data/dictionary-kanji-bank-v3-schema.json'), + kanjiMetaBankV3: readSchema('../ext/bg/data/dictionary-kanji-meta-bank-v3-schema.json'), + tagBankV3: readSchema('../ext/bg/data/dictionary-tag-bank-v3-schema.json'), + termBankV1: readSchema('../ext/bg/data/dictionary-term-bank-v1-schema.json'), + termBankV3: readSchema('../ext/bg/data/dictionary-term-bank-v3-schema.json'), + termMetaBankV3: readSchema('../ext/bg/data/dictionary-term-meta-bank-v3-schema.json') + }; + + for (const dictionaryFileName of dictionaryFileNames) { + try { + console.log(`Validating ${dictionaryFileName}...`); + await validateDictionary(dictionaryFileName, schemas); + console.log('No issues found'); + } catch (e) { + console.warn(e); + } + } +} + + +main(); -- cgit v1.2.3 From 7cbebf6897efb32c80c5e258f594d18861974bad Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Mon, 17 Feb 2020 22:30:16 -0500 Subject: Update how main is invoked --- test/dictionary-validate.js | 2 +- test/schema-validate.js | 2 +- test/test-schema.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'test/dictionary-validate.js') diff --git a/test/dictionary-validate.js b/test/dictionary-validate.js index 971c4971..dca3bcf5 100644 --- a/test/dictionary-validate.js +++ b/test/dictionary-validate.js @@ -87,4 +87,4 @@ async function main() { } -main(); +if (require.main === module) { main(); } diff --git a/test/schema-validate.js b/test/schema-validate.js index ac5a8a85..309adf83 100644 --- a/test/schema-validate.js +++ b/test/schema-validate.js @@ -33,4 +33,4 @@ function main() { } -main(); +if (require.main === module) { main(); } diff --git a/test/test-schema.js b/test/test-schema.js index 2f294e43..762b8509 100644 --- a/test/test-schema.js +++ b/test/test-schema.js @@ -233,4 +233,4 @@ function main() { } -main(); +if (require.main === module) { main(); } -- cgit v1.2.3 From 06480751a9d9ff1471455f31e3efc99d552a4975 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Mon, 17 Feb 2020 22:31:28 -0500 Subject: Create yomichan-test script to reduce repeated code --- test/dictionary-validate.js | 10 +++------- test/schema-validate.js | 6 ++---- test/test-schema.js | 7 ++----- test/yomichan-test.js | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 40 insertions(+), 16 deletions(-) create mode 100644 test/yomichan-test.js (limited to 'test/dictionary-validate.js') diff --git a/test/dictionary-validate.js b/test/dictionary-validate.js index dca3bcf5..082f0aea 100644 --- a/test/dictionary-validate.js +++ b/test/dictionary-validate.js @@ -1,13 +1,9 @@ const fs = require('fs'); const path = require('path'); +const yomichanTest = require('./yomichan-test'); -process.noDeprecation = true; // Suppress a warning about JSZip -const JSZip = require(path.join(__dirname, '../ext/mixed/lib/jszip.min.js')); -process.noDeprecation = false; - -const jsonSchemaFileName = path.join(__dirname, '../ext/bg/js/json-schema.js'); -const jsonSchemaFileSource = fs.readFileSync(jsonSchemaFileName, {encoding: 'utf8'}); -const JsonSchema = Function(`'use strict';${jsonSchemaFileSource};return JsonSchema;`)(); +const JSZip = yomichanTest.JSZip; +const {JsonSchema} = yomichanTest.requireScript('ext/bg/js/json-schema.js', ['JsonSchema']); function readSchema(relativeFileName) { diff --git a/test/schema-validate.js b/test/schema-validate.js index 309adf83..1271a611 100644 --- a/test/schema-validate.js +++ b/test/schema-validate.js @@ -1,9 +1,7 @@ const fs = require('fs'); -const path = require('path'); +const yomichanTest = require('./yomichan-test'); -const jsonSchemaFileName = path.join(__dirname, '../ext/bg/js/json-schema.js'); -const jsonSchemaFileSource = fs.readFileSync(jsonSchemaFileName, {encoding: 'utf8'}); -const JsonSchema = Function(`'use strict';${jsonSchemaFileSource};return JsonSchema;`)(); +const {JsonSchema} = yomichanTest.requireScript('ext/bg/js/json-schema.js', ['JsonSchema']); function main() { diff --git a/test/test-schema.js b/test/test-schema.js index 762b8509..ca4f56dd 100644 --- a/test/test-schema.js +++ b/test/test-schema.js @@ -1,10 +1,7 @@ -const fs = require('fs'); -const path = require('path'); const assert = require('assert'); +const yomichanTest = require('./yomichan-test'); -const jsonSchemaFileName = path.join(__dirname, '../ext/bg/js/json-schema.js'); -const jsonSchemaFileSource = fs.readFileSync(jsonSchemaFileName, {encoding: 'utf8'}); -const JsonSchema = Function(`'use strict';${jsonSchemaFileSource};return JsonSchema;`)(); +const {JsonSchema} = yomichanTest.requireScript('ext/bg/js/json-schema.js', ['JsonSchema']); function testValidate1() { diff --git a/test/yomichan-test.js b/test/yomichan-test.js new file mode 100644 index 00000000..602b8d78 --- /dev/null +++ b/test/yomichan-test.js @@ -0,0 +1,33 @@ +const fs = require('fs'); +const path = require('path'); + + +let JSZip = null; + +function requireScript(fileName, exportNames, variables) { + const absoluteFileName = path.join(__dirname, '..', fileName); + const source = fs.readFileSync(absoluteFileName, {encoding: 'utf8'}); + const exportNamesString = Array.isArray(exportNames) ? exportNames.join(',') : ''; + const variablesArgumentName = '__variables__'; + let variableString = ''; + if (typeof variables === 'object' && variables !== null) { + variableString = Object.keys(variables).join(','); + variableString = `const {${variableString}} = ${variablesArgumentName};`; + } + return Function(variablesArgumentName, `'use strict';${variableString}${source}\n;return {${exportNamesString}};`)(variables); +} + +function getJSZip() { + if (JSZip === null) { + process.noDeprecation = true; // Suppress a warning about JSZip + JSZip = require(path.join(__dirname, '../ext/mixed/lib/jszip.min.js')); + process.noDeprecation = false; + } + return JSZip; +} + + +module.exports = { + requireScript, + get JSZip() { return getJSZip(); } +}; -- cgit v1.2.3 From dce5f7e1e532240a2ce3ab8cb8a2ee846572c0b9 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Mon, 17 Feb 2020 22:38:48 -0500 Subject: Allow dictionary-validate to be require'd --- test/dictionary-validate.js | 51 ++++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 21 deletions(-) (limited to 'test/dictionary-validate.js') diff --git a/test/dictionary-validate.js b/test/dictionary-validate.js index 082f0aea..ccbd4ae4 100644 --- a/test/dictionary-validate.js +++ b/test/dictionary-validate.js @@ -28,11 +28,8 @@ async function validateDictionaryBanks(zip, fileNameFormat, schema) { } } -async function validateDictionary(fileName, schemas) { - const source = fs.readFileSync(fileName); - const zip = await JSZip.loadAsync(source); - - const indexFile = zip.files['index.json']; +async function validateDictionary(archive, schemas) { + const indexFile = archive.files['index.json']; if (!indexFile) { throw new Error('No dictionary index found in archive'); } @@ -42,11 +39,24 @@ async function validateDictionary(fileName, schemas) { JsonSchema.validate(index, schemas.index); - await validateDictionaryBanks(zip, 'term_bank_%s.json', version === 1 ? schemas.termBankV1 : schemas.termBankV3); - await validateDictionaryBanks(zip, 'term_meta_bank_%s.json', schemas.termMetaBankV3); - await validateDictionaryBanks(zip, 'kanji_bank_%s.json', version === 1 ? schemas.kanjiBankV1 : schemas.kanjiBankV3); - await validateDictionaryBanks(zip, 'kanji_meta_bank_%s.json', schemas.kanjiMetaBankV3); - await validateDictionaryBanks(zip, 'tag_bank_%s.json', schemas.tagBankV3); + await validateDictionaryBanks(archive, 'term_bank_%s.json', version === 1 ? schemas.termBankV1 : schemas.termBankV3); + await validateDictionaryBanks(archive, 'term_meta_bank_%s.json', schemas.termMetaBankV3); + await validateDictionaryBanks(archive, 'kanji_bank_%s.json', version === 1 ? schemas.kanjiBankV1 : schemas.kanjiBankV3); + await validateDictionaryBanks(archive, 'kanji_meta_bank_%s.json', schemas.kanjiMetaBankV3); + await validateDictionaryBanks(archive, 'tag_bank_%s.json', schemas.tagBankV3); +} + +function getSchemas() { + return { + index: readSchema('../ext/bg/data/dictionary-index-schema.json'), + kanjiBankV1: readSchema('../ext/bg/data/dictionary-kanji-bank-v1-schema.json'), + kanjiBankV3: readSchema('../ext/bg/data/dictionary-kanji-bank-v3-schema.json'), + kanjiMetaBankV3: readSchema('../ext/bg/data/dictionary-kanji-meta-bank-v3-schema.json'), + tagBankV3: readSchema('../ext/bg/data/dictionary-tag-bank-v3-schema.json'), + termBankV1: readSchema('../ext/bg/data/dictionary-term-bank-v1-schema.json'), + termBankV3: readSchema('../ext/bg/data/dictionary-term-bank-v3-schema.json'), + termMetaBankV3: readSchema('../ext/bg/data/dictionary-term-meta-bank-v3-schema.json') + }; } @@ -60,21 +70,14 @@ async function main() { return; } - const schemas = { - index: readSchema('../ext/bg/data/dictionary-index-schema.json'), - kanjiBankV1: readSchema('../ext/bg/data/dictionary-kanji-bank-v1-schema.json'), - kanjiBankV3: readSchema('../ext/bg/data/dictionary-kanji-bank-v3-schema.json'), - kanjiMetaBankV3: readSchema('../ext/bg/data/dictionary-kanji-meta-bank-v3-schema.json'), - tagBankV3: readSchema('../ext/bg/data/dictionary-tag-bank-v3-schema.json'), - termBankV1: readSchema('../ext/bg/data/dictionary-term-bank-v1-schema.json'), - termBankV3: readSchema('../ext/bg/data/dictionary-term-bank-v3-schema.json'), - termMetaBankV3: readSchema('../ext/bg/data/dictionary-term-meta-bank-v3-schema.json') - }; + const schemas = getSchemas(); for (const dictionaryFileName of dictionaryFileNames) { try { console.log(`Validating ${dictionaryFileName}...`); - await validateDictionary(dictionaryFileName, schemas); + const source = fs.readFileSync(dictionaryFileName); + const archive = await JSZip.loadAsync(source); + await validateDictionary(archive, schemas); console.log('No issues found'); } catch (e) { console.warn(e); @@ -84,3 +87,9 @@ async function main() { if (require.main === module) { main(); } + + +module.exports = { + getSchemas, + validateDictionary +}; -- cgit v1.2.3 From f2f4081d4572c48ebbe6685b8f89889823da0a69 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Mon, 17 Feb 2020 22:39:25 -0500 Subject: Use ? instead of %s --- test/dictionary-validate.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'test/dictionary-validate.js') diff --git a/test/dictionary-validate.js b/test/dictionary-validate.js index ccbd4ae4..25a5de88 100644 --- a/test/dictionary-validate.js +++ b/test/dictionary-validate.js @@ -16,7 +16,7 @@ function readSchema(relativeFileName) { async function validateDictionaryBanks(zip, fileNameFormat, schema) { let index = 1; while (true) { - const fileName = fileNameFormat.replace(/%s/, index); + const fileName = fileNameFormat.replace(/\?/, index); const file = zip.files[fileName]; if (!file) { break; } @@ -39,11 +39,11 @@ async function validateDictionary(archive, schemas) { JsonSchema.validate(index, schemas.index); - await validateDictionaryBanks(archive, 'term_bank_%s.json', version === 1 ? schemas.termBankV1 : schemas.termBankV3); - await validateDictionaryBanks(archive, 'term_meta_bank_%s.json', schemas.termMetaBankV3); - await validateDictionaryBanks(archive, 'kanji_bank_%s.json', version === 1 ? schemas.kanjiBankV1 : schemas.kanjiBankV3); - await validateDictionaryBanks(archive, 'kanji_meta_bank_%s.json', schemas.kanjiMetaBankV3); - await validateDictionaryBanks(archive, 'tag_bank_%s.json', schemas.tagBankV3); + await validateDictionaryBanks(archive, 'term_bank_?.json', version === 1 ? schemas.termBankV1 : schemas.termBankV3); + await validateDictionaryBanks(archive, 'term_meta_bank_?.json', schemas.termMetaBankV3); + await validateDictionaryBanks(archive, 'kanji_bank_?.json', version === 1 ? schemas.kanjiBankV1 : schemas.kanjiBankV3); + await validateDictionaryBanks(archive, 'kanji_meta_bank_?.json', schemas.kanjiMetaBankV3); + await validateDictionaryBanks(archive, 'tag_bank_?.json', schemas.tagBankV3); } function getSchemas() { -- cgit v1.2.3 From d87050bbbba6edb64f75e65b614526a3beef9105 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sun, 23 Feb 2020 13:05:48 -0500 Subject: Add missing copyright headers --- test/dictionary-validate.js | 18 ++++++++++++++++++ test/schema-validate.js | 18 ++++++++++++++++++ test/test-database.js | 18 ++++++++++++++++++ test/test-dictionary.js | 18 ++++++++++++++++++ test/test-schema.js | 18 ++++++++++++++++++ test/yomichan-test.js | 18 ++++++++++++++++++ 6 files changed, 108 insertions(+) (limited to 'test/dictionary-validate.js') diff --git a/test/dictionary-validate.js b/test/dictionary-validate.js index 25a5de88..14eee2ed 100644 --- a/test/dictionary-validate.js +++ b/test/dictionary-validate.js @@ -1,3 +1,21 @@ +/* + * Copyright (C) 2020 Alex Yatskov + * Author: Alex Yatskov + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + const fs = require('fs'); const path = require('path'); const yomichanTest = require('./yomichan-test'); diff --git a/test/schema-validate.js b/test/schema-validate.js index 1271a611..a4f2d94c 100644 --- a/test/schema-validate.js +++ b/test/schema-validate.js @@ -1,3 +1,21 @@ +/* + * Copyright (C) 2020 Alex Yatskov + * Author: Alex Yatskov + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + const fs = require('fs'); const yomichanTest = require('./yomichan-test'); diff --git a/test/test-database.js b/test/test-database.js index add04a03..c2317881 100644 --- a/test/test-database.js +++ b/test/test-database.js @@ -1,3 +1,21 @@ +/* + * Copyright (C) 2020 Alex Yatskov + * Author: Alex Yatskov + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + const fs = require('fs'); const url = require('url'); const path = require('path'); diff --git a/test/test-dictionary.js b/test/test-dictionary.js index b157dd5d..74f9e62b 100644 --- a/test/test-dictionary.js +++ b/test/test-dictionary.js @@ -1,3 +1,21 @@ +/* + * Copyright (C) 2020 Alex Yatskov + * Author: Alex Yatskov + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + const yomichanTest = require('./yomichan-test'); const dictionaryValidate = require('./dictionary-validate'); diff --git a/test/test-schema.js b/test/test-schema.js index 8ca63167..f4612f86 100644 --- a/test/test-schema.js +++ b/test/test-schema.js @@ -1,3 +1,21 @@ +/* + * Copyright (C) 2020 Alex Yatskov + * Author: Alex Yatskov + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + const assert = require('assert'); const yomichanTest = require('./yomichan-test'); diff --git a/test/yomichan-test.js b/test/yomichan-test.js index 939e0ad2..78bfb9c6 100644 --- a/test/yomichan-test.js +++ b/test/yomichan-test.js @@ -1,3 +1,21 @@ +/* + * Copyright (C) 2020 Alex Yatskov + * Author: Alex Yatskov + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + const fs = require('fs'); const path = require('path'); -- cgit v1.2.3