diff options
Diffstat (limited to 'test/test-database.js')
| -rw-r--r-- | test/test-database.js | 982 |
1 files changed, 982 insertions, 0 deletions
diff --git a/test/test-database.js b/test/test-database.js new file mode 100644 index 00000000..947e369b --- /dev/null +++ b/test/test-database.js @@ -0,0 +1,982 @@ +/* + * Copyright (C) 2023 Yomitan Authors + * Copyright (C) 2020-2022 Yomichan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +const path = require('path'); +const assert = require('assert'); +const {createDictionaryArchive, testMain} = require('../dev/util'); +const {DatabaseVM, DatabaseVMDictionaryImporterMediaLoader} = require('../dev/database-vm'); + + +const vm = new DatabaseVM(); +vm.execute([ + 'js/core.js', + 'js/core/extension-error.js', + 'js/general/cache-map.js', + 'js/data/json-schema.js', + 'js/media/media-util.js', + 'js/language/dictionary-importer.js', + 'js/data/database.js', + 'js/language/dictionary-database.js' +]); +/** @type {typeof DictionaryImporter} */ +const DictionaryImporter2 = vm.getSingle('DictionaryImporter'); +/** @type {typeof DictionaryDatabase} */ +const DictionaryDatabase2 = vm.getSingle('DictionaryDatabase'); + + +/** + * @param {string} dictionary + * @param {string} [dictionaryName] + * @returns {import('jszip')} + */ +function createTestDictionaryArchive(dictionary, dictionaryName) { + const dictionaryDirectory = path.join(__dirname, 'data', 'dictionaries', dictionary); + return createDictionaryArchive(dictionaryDirectory, dictionaryName); +} + + +/** + * @param {import('dictionary-importer').OnProgressCallback} [onProgress] + * @returns {DictionaryImporter} + */ +function createDictionaryImporter(onProgress) { + const dictionaryImporterMediaLoader = new DatabaseVMDictionaryImporterMediaLoader(); + return new DictionaryImporter2(dictionaryImporterMediaLoader, (...args) => { + const {stepIndex, stepCount, index, count} = args[0]; + assert.ok(stepIndex < stepCount); + assert.ok(index <= count); + if (typeof onProgress === 'function') { + onProgress(...args); + } + }); +} + + +/** + * @param {import('dictionary-database').TermEntry[]} dictionaryDatabaseEntries + * @param {string} term + * @returns {number} + */ +function countDictionaryDatabaseEntriesWithTerm(dictionaryDatabaseEntries, term) { + return dictionaryDatabaseEntries.reduce((i, v) => (i + (v.term === term ? 1 : 0)), 0); +} + +/** + * @param {import('dictionary-database').TermEntry[]} dictionaryDatabaseEntries + * @param {string} reading + * @returns {number} + */ +function countDictionaryDatabaseEntriesWithReading(dictionaryDatabaseEntries, reading) { + return dictionaryDatabaseEntries.reduce((i, v) => (i + (v.reading === reading ? 1 : 0)), 0); +} + +/** + * @param {import('dictionary-database').TermMeta[]|import('dictionary-database').KanjiMeta[]} metas + * @param {import('dictionary-database').TermMetaType|import('dictionary-database').KanjiMetaType} mode + * @returns {number} + */ +function countMetasWithMode(metas, mode) { + let i = 0; + for (const item of metas) { + if (item.mode === mode) { ++i; } + } + return i; +} + +/** + * @param {import('dictionary-database').KanjiEntry[]} kanji + * @param {string} character + * @returns {number} + */ +function countKanjiWithCharacter(kanji, character) { + let i = 0; + for (const item of kanji) { + if (item.character === character) { ++i; } + } + return i; +} + + +/** + * @param {number} timeout + * @returns {Promise<void>} + */ +function clearDatabase(timeout) { + return new Promise((resolve, reject) => { + /** @type {?number} */ + let timer = setTimeout(() => { + timer = null; + reject(new Error(`clearDatabase failed to resolve after ${timeout}ms`)); + }, timeout); + + (async () => { + const indexedDB = vm.indexedDB; + for (const {name} of await indexedDB.databases()) { + if (typeof name !== 'string') { continue; } + /** @type {Promise<void>} */ + const promise2 = new Promise((resolve2, reject2) => { + const request = indexedDB.deleteDatabase(name); + request.onerror = (e) => reject2(e); + request.onsuccess = () => resolve2(); + }); + await promise2; + } + if (timer !== null) { + clearTimeout(timer); + } + resolve(); + })(); + }); +} + + +/** */ +async function testDatabase1() { + // Load dictionary data + const testDictionary = createTestDictionaryArchive('valid-dictionary1'); + const testDictionarySource = await testDictionary.generateAsync({type: 'arraybuffer'}); + const testDictionaryIndex = JSON.parse(await testDictionary.files['index.json'].async('string')); + + const title = testDictionaryIndex.title; + const titles = new Map([ + [title, {priority: 0, allowSecondarySearches: false}] + ]); + + // Setup iteration data + const iterations = [ + { + cleanup: async () => { + // Test purge + await dictionaryDatabase.purge(); + await testDatabaseEmpty1(dictionaryDatabase); + } + }, + { + cleanup: async () => { + // Test deleteDictionary + let progressEvent = false; + await dictionaryDatabase.deleteDictionary( + title, + 1000, + () => { + progressEvent = true; + } + ); + assert.ok(progressEvent); + + await testDatabaseEmpty1(dictionaryDatabase); + } + }, + { + cleanup: async () => {} + } + ]; + + // Setup database + const dictionaryDatabase = new DictionaryDatabase2(); + await dictionaryDatabase.prepare(); + + for (const {cleanup} of iterations) { + const expectedSummary = { + title, + revision: 'test', + sequenced: true, + version: 3, + importDate: 0, + prefixWildcardsSupported: true, + counts: { + kanji: {total: 2}, + kanjiMeta: {total: 6, freq: 6}, + media: {total: 4}, + tagMeta: {total: 15}, + termMeta: {total: 38, freq: 31, pitch: 7}, + terms: {total: 21} + } + }; + + // Import data + let progressEvent = false; + const dictionaryImporter = createDictionaryImporter(() => { progressEvent = true; }); + const {result, errors} = await dictionaryImporter.importDictionary( + dictionaryDatabase, + testDictionarySource, + {prefixWildcardsSupported: true} + ); + expectedSummary.importDate = result.importDate; + vm.assert.deepStrictEqual(errors, []); + vm.assert.deepStrictEqual(result, expectedSummary); + assert.ok(progressEvent); + + // Get info summary + const info = await dictionaryDatabase.getDictionaryInfo(); + vm.assert.deepStrictEqual(info, [expectedSummary]); + + // Get counts + const counts = await dictionaryDatabase.getDictionaryCounts( + info.map((v) => v.title), + true + ); + vm.assert.deepStrictEqual(counts, { + counts: [{kanji: 2, kanjiMeta: 6, terms: 21, termMeta: 38, tagMeta: 15, media: 4}], + total: {kanji: 2, kanjiMeta: 6, terms: 21, termMeta: 38, tagMeta: 15, media: 4} + }); + + // Test find* functions + await testFindTermsBulkTest1(dictionaryDatabase, titles); + await testTindTermsExactBulk1(dictionaryDatabase, titles); + await testFindTermsBySequenceBulk1(dictionaryDatabase, title); + await testFindTermMetaBulk1(dictionaryDatabase, titles); + await testFindKanjiBulk1(dictionaryDatabase, titles); + await testFindKanjiMetaBulk1(dictionaryDatabase, titles); + await testFindTagForTitle1(dictionaryDatabase, title); + + // Cleanup + await cleanup(); + } + + await dictionaryDatabase.close(); +} + +/** + * @param {DictionaryDatabase} database + */ +async function testDatabaseEmpty1(database) { + const info = await database.getDictionaryInfo(); + vm.assert.deepStrictEqual(info, []); + + const counts = await database.getDictionaryCounts([], true); + vm.assert.deepStrictEqual(counts, { + counts: [], + total: {kanji: 0, kanjiMeta: 0, terms: 0, termMeta: 0, tagMeta: 0, media: 0} + }); +} + +/** + * @param {DictionaryDatabase} database + * @param {import('dictionary-database').DictionarySet} titles + */ +async function testFindTermsBulkTest1(database, titles) { + /** @type {{inputs: {matchType: import('dictionary-database').MatchType, termList: string[]}[], expectedResults: {total: number, terms: [key: string, count: number][], readings: [key: string, count: number][]}}[]} */ + const data = [ + { + inputs: [ + { + matchType: 'exact', + termList: ['打', '打つ', '打ち込む'] + }, + { + matchType: 'exact', + termList: ['だ', 'ダース', 'うつ', 'ぶつ', 'うちこむ', 'ぶちこむ'] + }, + { + matchType: 'prefix', + termList: ['打'] + } + ], + expectedResults: { + total: 10, + terms: [ + ['打', 2], + ['打つ', 4], + ['打ち込む', 4] + ], + readings: [ + ['だ', 1], + ['ダース', 1], + ['うつ', 2], + ['ぶつ', 2], + ['うちこむ', 2], + ['ぶちこむ', 2] + ] + } + }, + { + inputs: [ + { + matchType: 'exact', + termList: ['込む'] + } + ], + expectedResults: { + total: 0, + terms: [], + readings: [] + } + }, + { + inputs: [ + { + matchType: 'suffix', + termList: ['込む'] + } + ], + expectedResults: { + total: 4, + terms: [ + ['打ち込む', 4] + ], + readings: [ + ['うちこむ', 2], + ['ぶちこむ', 2] + ] + } + }, + { + inputs: [ + { + matchType: 'exact', + termList: [] + } + ], + expectedResults: { + total: 0, + terms: [], + readings: [] + } + } + ]; + + for (const {inputs, expectedResults} of data) { + for (const {termList, matchType} of inputs) { + const results = await database.findTermsBulk(termList, titles, matchType); + assert.strictEqual(results.length, expectedResults.total); + for (const [term, count] of expectedResults.terms) { + assert.strictEqual(countDictionaryDatabaseEntriesWithTerm(results, term), count); + } + for (const [reading, count] of expectedResults.readings) { + assert.strictEqual(countDictionaryDatabaseEntriesWithReading(results, reading), count); + } + } + } +} + +/** + * @param {DictionaryDatabase} database + * @param {import('dictionary-database').DictionarySet} titles + */ +async function testTindTermsExactBulk1(database, titles) { + /** @type {{inputs: {termList: {term: string, reading: string}[]}[], expectedResults: {total: number, terms: [key: string, count: number][], readings: [key: string, count: number][]}}[]} */ + const data = [ + { + inputs: [ + { + termList: [ + {term: '打', reading: 'だ'}, + {term: '打つ', reading: 'うつ'}, + {term: '打ち込む', reading: 'うちこむ'} + ] + } + ], + expectedResults: { + total: 5, + terms: [ + ['打', 1], + ['打つ', 2], + ['打ち込む', 2] + ], + readings: [ + ['だ', 1], + ['うつ', 2], + ['うちこむ', 2] + ] + } + }, + { + inputs: [ + { + termList: [ + {term: '打', reading: 'だ?'}, + {term: '打つ', reading: 'うつ?'}, + {term: '打ち込む', reading: 'うちこむ?'} + ] + } + ], + expectedResults: { + total: 0, + terms: [], + readings: [] + } + }, + { + inputs: [ + { + termList: [ + {term: '打つ', reading: 'うつ'}, + {term: '打つ', reading: 'ぶつ'} + ] + } + ], + expectedResults: { + total: 4, + terms: [ + ['打つ', 4] + ], + readings: [ + ['うつ', 2], + ['ぶつ', 2] + ] + } + }, + { + inputs: [ + { + termList: [ + {term: '打つ', reading: 'うちこむ'} + ] + } + ], + expectedResults: { + total: 0, + terms: [], + readings: [] + } + }, + { + inputs: [ + { + termList: [] + } + ], + expectedResults: { + total: 0, + terms: [], + readings: [] + } + } + ]; + + for (const {inputs, expectedResults} of data) { + for (const {termList} of inputs) { + const results = await database.findTermsExactBulk(termList, titles); + assert.strictEqual(results.length, expectedResults.total); + for (const [term, count] of expectedResults.terms) { + assert.strictEqual(countDictionaryDatabaseEntriesWithTerm(results, term), count); + } + for (const [reading, count] of expectedResults.readings) { + assert.strictEqual(countDictionaryDatabaseEntriesWithReading(results, reading), count); + } + } + } +} + +/** + * @param {DictionaryDatabase} database + * @param {string} mainDictionary + */ +async function testFindTermsBySequenceBulk1(database, mainDictionary) { + /** @type {{inputs: {sequenceList: number[]}[], expectedResults: {total: number, terms: [key: string, count: number][], readings: [key: string, count: number][]}}[]} */ + const data = [ + { + inputs: [ + { + sequenceList: [1, 2, 3, 4, 5] + } + ], + expectedResults: { + total: 11, + terms: [ + ['打', 2], + ['打つ', 4], + ['打ち込む', 4], + ['画像', 1] + ], + readings: [ + ['だ', 1], + ['ダース', 1], + ['うつ', 2], + ['ぶつ', 2], + ['うちこむ', 2], + ['ぶちこむ', 2], + ['がぞう', 1] + ] + } + }, + { + inputs: [ + { + sequenceList: [1] + } + ], + expectedResults: { + total: 1, + terms: [ + ['打', 1] + ], + readings: [ + ['だ', 1] + ] + } + }, + { + inputs: [ + { + sequenceList: [2] + } + ], + expectedResults: { + total: 1, + terms: [ + ['打', 1] + ], + readings: [ + ['ダース', 1] + ] + } + }, + { + inputs: [ + { + sequenceList: [3] + } + ], + expectedResults: { + total: 4, + terms: [ + ['打つ', 4] + ], + readings: [ + ['うつ', 2], + ['ぶつ', 2] + ] + } + }, + { + inputs: [ + { + sequenceList: [4] + } + ], + expectedResults: { + total: 4, + terms: [ + ['打ち込む', 4] + ], + readings: [ + ['うちこむ', 2], + ['ぶちこむ', 2] + ] + } + }, + { + inputs: [ + { + sequenceList: [5] + } + ], + expectedResults: { + total: 1, + terms: [ + ['画像', 1] + ], + readings: [ + ['がぞう', 1] + ] + } + }, + { + inputs: [ + { + sequenceList: [-1] + } + ], + expectedResults: { + total: 0, + terms: [], + readings: [] + } + }, + { + inputs: [ + { + sequenceList: [] + } + ], + expectedResults: { + total: 0, + terms: [], + readings: [] + } + } + ]; + + for (const {inputs, expectedResults} of data) { + for (const {sequenceList} of inputs) { + const results = await database.findTermsBySequenceBulk(sequenceList.map((query) => ({query, dictionary: mainDictionary}))); + assert.strictEqual(results.length, expectedResults.total); + for (const [term, count] of expectedResults.terms) { + assert.strictEqual(countDictionaryDatabaseEntriesWithTerm(results, term), count); + } + for (const [reading, count] of expectedResults.readings) { + assert.strictEqual(countDictionaryDatabaseEntriesWithReading(results, reading), count); + } + } + } +} + +/** + * @param {DictionaryDatabase} database + * @param {import('dictionary-database').DictionarySet} titles + */ +async function testFindTermMetaBulk1(database, titles) { + /** @type {{inputs: {termList: string[]}[], expectedResults: {total: number, modes: [key: import('dictionary-database').TermMetaType, count: number][]}}[]} */ + const data = [ + { + inputs: [ + { + termList: ['打'] + } + ], + expectedResults: { + total: 11, + modes: [ + ['freq', 11] + ] + } + }, + { + inputs: [ + { + termList: ['打つ'] + } + ], + expectedResults: { + total: 10, + modes: [ + ['freq', 10] + ] + } + }, + { + inputs: [ + { + termList: ['打ち込む'] + } + ], + expectedResults: { + total: 12, + modes: [ + ['freq', 10], + ['pitch', 2] + ] + } + }, + { + inputs: [ + { + termList: ['?'] + } + ], + expectedResults: { + total: 0, + modes: [] + } + } + ]; + + for (const {inputs, expectedResults} of data) { + for (const {termList} of inputs) { + const results = await database.findTermMetaBulk(termList, titles); + assert.strictEqual(results.length, expectedResults.total); + for (const [mode, count] of expectedResults.modes) { + assert.strictEqual(countMetasWithMode(results, mode), count); + } + } + } +} + +/** + * @param {DictionaryDatabase} database + * @param {import('dictionary-database').DictionarySet} titles + */ +async function testFindKanjiBulk1(database, titles) { + /** @type {{inputs: {kanjiList: string[]}[], expectedResults: {total: number, kanji: [key: string, count: number][]}}[]} */ + const data = [ + { + inputs: [ + { + kanjiList: ['打'] + } + ], + expectedResults: { + total: 1, + kanji: [ + ['打', 1] + ] + } + }, + { + inputs: [ + { + kanjiList: ['込'] + } + ], + expectedResults: { + total: 1, + kanji: [ + ['込', 1] + ] + } + }, + { + inputs: [ + { + kanjiList: ['?'] + } + ], + expectedResults: { + total: 0, + kanji: [] + } + } + ]; + + for (const {inputs, expectedResults} of data) { + for (const {kanjiList} of inputs) { + const results = await database.findKanjiBulk(kanjiList, titles); + assert.strictEqual(results.length, expectedResults.total); + for (const [kanji, count] of expectedResults.kanji) { + assert.strictEqual(countKanjiWithCharacter(results, kanji), count); + } + } + } +} + +/** + * @param {DictionaryDatabase} database + * @param {import('dictionary-database').DictionarySet} titles + */ +async function testFindKanjiMetaBulk1(database, titles) { + /** @type {{inputs: {kanjiList: string[]}[], expectedResults: {total: number, modes: [key: import('dictionary-database').KanjiMetaType, count: number][]}}[]} */ + const data = [ + { + inputs: [ + { + kanjiList: ['打'] + } + ], + expectedResults: { + total: 3, + modes: [ + ['freq', 3] + ] + } + }, + { + inputs: [ + { + kanjiList: ['込'] + } + ], + expectedResults: { + total: 3, + modes: [ + ['freq', 3] + ] + } + }, + { + inputs: [ + { + kanjiList: ['?'] + } + ], + expectedResults: { + total: 0, + modes: [] + } + } + ]; + + for (const {inputs, expectedResults} of data) { + for (const {kanjiList} of inputs) { + const results = await database.findKanjiMetaBulk(kanjiList, titles); + assert.strictEqual(results.length, expectedResults.total); + for (const [mode, count] of expectedResults.modes) { + assert.strictEqual(countMetasWithMode(results, mode), count); + } + } + } +} + +/** + * @param {DictionaryDatabase} database + * @param {string} title + */ +async function testFindTagForTitle1(database, title) { + const data = [ + { + inputs: [ + { + name: 'E1' + } + ], + expectedResults: { + value: {category: 'default', dictionary: title, name: 'E1', notes: 'example tag 1', order: 0, score: 0} + } + }, + { + inputs: [ + { + name: 'K1' + } + ], + expectedResults: { + value: {category: 'default', dictionary: title, name: 'K1', notes: 'example kanji tag 1', order: 0, score: 0} + } + }, + { + inputs: [ + { + name: 'kstat1' + } + ], + expectedResults: { + value: {category: 'class', dictionary: title, name: 'kstat1', notes: 'kanji stat 1', order: 0, score: 0} + } + }, + { + inputs: [ + { + name: 'invalid' + } + ], + expectedResults: { + value: null + } + } + ]; + + for (const {inputs, expectedResults} of data) { + for (const {name} of inputs) { + const result = await database.findTagForTitle(name, title); + vm.assert.deepStrictEqual(result, expectedResults.value); + } + } +} + + +/** */ +async function testDatabase2() { + // Load dictionary data + const testDictionary = createTestDictionaryArchive('valid-dictionary1'); + const testDictionarySource = await testDictionary.generateAsync({type: 'arraybuffer'}); + const testDictionaryIndex = JSON.parse(await testDictionary.files['index.json'].async('string')); + + const title = testDictionaryIndex.title; + const titles = new Map([ + [title, {priority: 0, allowSecondarySearches: false}] + ]); + + // Setup database + const dictionaryDatabase = new DictionaryDatabase2(); + /** @type {import('dictionary-importer').ImportDetails} */ + const detaultImportDetails = {prefixWildcardsSupported: false}; + + // Error: not prepared + await assert.rejects(async () => await dictionaryDatabase.deleteDictionary(title, 1000, () => {})); + await assert.rejects(async () => await dictionaryDatabase.findTermsBulk(['?'], titles, 'exact')); + await assert.rejects(async () => await dictionaryDatabase.findTermsExactBulk([{term: '?', reading: '?'}], titles)); + await assert.rejects(async () => await dictionaryDatabase.findTermsBySequenceBulk([{query: 1, dictionary: title}])); + await assert.rejects(async () => await dictionaryDatabase.findTermMetaBulk(['?'], titles)); + await assert.rejects(async () => await dictionaryDatabase.findTermMetaBulk(['?'], titles)); + await assert.rejects(async () => await dictionaryDatabase.findKanjiBulk(['?'], titles)); + await assert.rejects(async () => await dictionaryDatabase.findKanjiMetaBulk(['?'], titles)); + await assert.rejects(async () => await dictionaryDatabase.findTagForTitle('tag', title)); + await assert.rejects(async () => await dictionaryDatabase.getDictionaryInfo()); + await assert.rejects(async () => await dictionaryDatabase.getDictionaryCounts([...titles.keys()], true)); + await assert.rejects(async () => await createDictionaryImporter().importDictionary(dictionaryDatabase, testDictionarySource, detaultImportDetails)); + + await dictionaryDatabase.prepare(); + + // Error: already prepared + await assert.rejects(async () => await dictionaryDatabase.prepare()); + + await createDictionaryImporter().importDictionary(dictionaryDatabase, testDictionarySource, detaultImportDetails); + + // Error: dictionary already imported + await assert.rejects(async () => await createDictionaryImporter().importDictionary(dictionaryDatabase, testDictionarySource, detaultImportDetails)); + + await dictionaryDatabase.close(); +} + + +/** */ +async function testDatabase3() { + const invalidDictionaries = [ + 'invalid-dictionary1', + 'invalid-dictionary2', + 'invalid-dictionary3', + 'invalid-dictionary4', + 'invalid-dictionary5', + 'invalid-dictionary6' + ]; + + // Setup database + const dictionaryDatabase = new DictionaryDatabase2(); + /** @type {import('dictionary-importer').ImportDetails} */ + const detaultImportDetails = {prefixWildcardsSupported: false}; + await dictionaryDatabase.prepare(); + + for (const invalidDictionary of invalidDictionaries) { + const testDictionary = createTestDictionaryArchive(invalidDictionary); + const testDictionarySource = await testDictionary.generateAsync({type: 'arraybuffer'}); + + let error = null; + try { + await createDictionaryImporter().importDictionary(dictionaryDatabase, testDictionarySource, detaultImportDetails); + } catch (e) { + error = e; + } + + if (error === null) { + assert.ok(false, `Expected an error while importing ${invalidDictionary}`); + } else { + const prefix = 'Dictionary has invalid data'; + const message = /** @type {import('core').UnknownObject} */ (error).message; + assert.ok(typeof message, 'string'); + if (typeof message === 'string') { + assert.ok(message.startsWith(prefix), `Expected error message to start with '${prefix}': ${message}`); + } + } + } + + await dictionaryDatabase.close(); +} + + +/** */ +async function main() { + const clearTimeout = 5000; + try { + await testDatabase1(); + await clearDatabase(clearTimeout); + + await testDatabase2(); + await clearDatabase(clearTimeout); + + await testDatabase3(); + await clearDatabase(clearTimeout); + } catch (e) { + console.log(e); + process.exit(-1); + throw e; + } +} + + +if (require.main === module) { testMain(main); } |