/* * Copyright (C) 2023 Yomitan Authors * Copyright (C) 2020-2022 Yomichan Authors * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see <https://www.gnu.org/licenses/>. */ import {IDBFactory, IDBKeyRange} from 'fake-indexeddb'; import path from 'path'; import {beforeEach, describe, expect, test, vi} from 'vitest'; import {createDictionaryArchive} from '../dev/util.js'; import {DictionaryDatabase} from '../ext/js/language/dictionary-database.js'; import {DictionaryImporterMediaLoader} from '../ext/js/language/dictionary-importer-media-loader.js'; import {DictionaryImporter} from '../ext/js/language/dictionary-importer.js'; vi.stubGlobal('IDBKeyRange', IDBKeyRange); vi.mock('../ext/js/language/dictionary-importer-media-loader.js'); function createTestDictionaryArchive(dictionary, dictionaryName) { const dictionaryDirectory = path.join(__dirname, 'data', 'dictionaries', dictionary); return createDictionaryArchive(dictionaryDirectory, dictionaryName); } function createDictionaryImporter(onProgress) { const dictionaryImporterMediaLoader = new DictionaryImporterMediaLoader(); return new DictionaryImporter(dictionaryImporterMediaLoader, (...args) => { const {stepIndex, stepCount, index, count} = args[0]; expect(stepIndex < stepCount).toBe(true); expect(index <= count).toBe(true); if (typeof onProgress === 'function') { onProgress(...args); } }); } function countDictionaryDatabaseEntriesWithTerm(dictionaryDatabaseEntries, term) { return dictionaryDatabaseEntries.reduce((i, v) => (i + (v.term === term ? 1 : 0)), 0); } function countDictionaryDatabaseEntriesWithReading(dictionaryDatabaseEntries, reading) { return dictionaryDatabaseEntries.reduce((i, v) => (i + (v.reading === reading ? 1 : 0)), 0); } function countMetasWithMode(metas, mode) { return metas.reduce((i, v) => (i + (v.mode === mode ? 1 : 0)), 0); } function countKanjiWithCharacter(kanji, character) { return kanji.reduce((i, v) => (i + (v.character === character ? 1 : 0)), 0); } async function testDatabase1() { test('Database1', async () => { // Load dictionary data const testDictionary = createTestDictionaryArchive('valid-dictionary1'); const testDictionarySource = await testDictionary.generateAsync({type: 'arraybuffer'}); const testDictionaryIndex = JSON.parse(await testDictionary.files['index.json'].async('string')); const title = testDictionaryIndex.title; const titles = new Map([ [title, {priority: 0, allowSecondarySearches: false}] ]); // Setup iteration data const iterations = [ { cleanup: async () => { // Test purge await dictionaryDatabase.purge(); await testDatabaseEmpty1(dictionaryDatabase); } }, { cleanup: async () => { // Test deleteDictionary let progressEvent = false; await dictionaryDatabase.deleteDictionary( title, 1000, () => { progressEvent = true; } ); expect(progressEvent).toBe(true); await testDatabaseEmpty1(dictionaryDatabase); } }, { cleanup: async () => {} } ]; // Setup database const dictionaryDatabase = new DictionaryDatabase(); await dictionaryDatabase.prepare(); for (const {cleanup} of iterations) { const expectedSummary = { title, revision: 'test', sequenced: true, version: 3, importDate: 0, prefixWildcardsSupported: true, counts: { kanji: {total: 2}, kanjiMeta: {total: 6, freq: 6}, media: {total: 4}, tagMeta: {total: 15}, termMeta: {total: 38, freq: 31, pitch: 7}, terms: {total: 21} } }; // Import data let progressEvent = false; const dictionaryImporter = createDictionaryImporter(() => { progressEvent = true; }); const {result, errors} = await dictionaryImporter.importDictionary( dictionaryDatabase, testDictionarySource, {prefixWildcardsSupported: true} ); expectedSummary.importDate = result.importDate; expect(errors).toStrictEqual([]); expect(result).toStrictEqual(expectedSummary); expect(progressEvent).toBe(true); // Get info summary const info = await dictionaryDatabase.getDictionaryInfo(); expect(info).toStrictEqual([expectedSummary]); // Get counts const counts = await dictionaryDatabase.getDictionaryCounts( info.map((v) => v.title), true ); expect(counts).toStrictEqual({ counts: [{kanji: 2, kanjiMeta: 6, terms: 21, termMeta: 38, tagMeta: 15, media: 4}], total: {kanji: 2, kanjiMeta: 6, terms: 21, termMeta: 38, tagMeta: 15, media: 4} }); // Test find* functions await testFindTermsBulkTest1(dictionaryDatabase, titles); await testTindTermsExactBulk1(dictionaryDatabase, titles); await testFindTermsBySequenceBulk1(dictionaryDatabase, title); await testFindTermMetaBulk1(dictionaryDatabase, titles); await testFindKanjiBulk1(dictionaryDatabase, titles); await testFindKanjiMetaBulk1(dictionaryDatabase, titles); await testFindTagForTitle1(dictionaryDatabase, title); // Cleanup await cleanup(); } await dictionaryDatabase.close(); }); } async function testDatabaseEmpty1(database) { test('DatabaseEmpty1', async () => { const info = await database.getDictionaryInfo(); expect(info).toStrictEqual([]); const counts = await database.getDictionaryCounts([], true); expect(counts).toStrictEqual({ counts: [], total: {kanji: 0, kanjiMeta: 0, terms: 0, termMeta: 0, tagMeta: 0, media: 0} }); }); } async function testFindTermsBulkTest1(database, titles) { test('FindTermsBulkTest1', async () => { const data = [ { inputs: [ { matchType: null, termList: ['打', '打つ', '打ち込む'] }, { matchType: null, termList: ['だ', 'ダース', 'うつ', 'ぶつ', 'うちこむ', 'ぶちこむ'] }, { matchType: 'prefix', termList: ['打'] } ], expectedResults: { total: 10, terms: [ ['打', 2], ['打つ', 4], ['打ち込む', 4] ], readings: [ ['だ', 1], ['ダース', 1], ['うつ', 2], ['ぶつ', 2], ['うちこむ', 2], ['ぶちこむ', 2] ] } }, { inputs: [ { matchType: null, termList: ['込む'] } ], expectedResults: { total: 0, terms: [], readings: [] } }, { inputs: [ { matchType: 'suffix', termList: ['込む'] } ], expectedResults: { total: 4, terms: [ ['打ち込む', 4] ], readings: [ ['うちこむ', 2], ['ぶちこむ', 2] ] } }, { inputs: [ { matchType: null, termList: [] } ], expectedResults: { total: 0, terms: [], readings: [] } } ]; for (const {inputs, expectedResults} of data) { for (const {termList, matchType} of inputs) { const results = await database.findTermsBulk(termList, titles, matchType); expect(results.length).toStrictEqual(expectedResults.total); for (const [term, count] of expectedResults.terms) { expect(countDictionaryDatabaseEntriesWithTerm(results, term)).toStrictEqual(count); } for (const [reading, count] of expectedResults.readings) { expect(countDictionaryDatabaseEntriesWithReading(results, reading)).toStrictEqual(count); } } } }); } async function testTindTermsExactBulk1(database, titles) { test('TindTermsExactBulk1', async () => { const data = [ { inputs: [ { termList: [ {term: '打', reading: 'だ'}, {term: '打つ', reading: 'うつ'}, {term: '打ち込む', reading: 'うちこむ'} ] } ], expectedResults: { total: 5, terms: [ ['打', 1], ['打つ', 2], ['打ち込む', 2] ], readings: [ ['だ', 1], ['うつ', 2], ['うちこむ', 2] ] } }, { inputs: [ { termList: [ {term: '打', reading: 'だ?'}, {term: '打つ', reading: 'うつ?'}, {term: '打ち込む', reading: 'うちこむ?'} ] } ], expectedResults: { total: 0, terms: [], readings: [] } }, { inputs: [ { termList: [ {term: '打つ', reading: 'うつ'}, {term: '打つ', reading: 'ぶつ'} ] } ], expectedResults: { total: 4, terms: [ ['打つ', 4] ], readings: [ ['うつ', 2], ['ぶつ', 2] ] } }, { inputs: [ { termList: [ {term: '打つ', reading: 'うちこむ'} ] } ], expectedResults: { total: 0, terms: [], readings: [] } }, { inputs: [ { termList: [] } ], expectedResults: { total: 0, terms: [], readings: [] } } ]; for (const {inputs, expectedResults} of data) { for (const {termList} of inputs) { const results = await database.findTermsExactBulk(termList, titles); expect(results.length).toStrictEqual(expectedResults.total); for (const [term, count] of expectedResults.terms) { expect(countDictionaryDatabaseEntriesWithTerm(results, term)).toStrictEqual(count); } for (const [reading, count] of expectedResults.readings) { expect(countDictionaryDatabaseEntriesWithReading(results, reading)).toStrictEqual(count); } } } }); } async function testFindTermsBySequenceBulk1(database, mainDictionary) { test('FindTermsBySequenceBulk1', async () => { const data = [ { inputs: [ { sequenceList: [1, 2, 3, 4, 5] } ], expectedResults: { total: 11, terms: [ ['打', 2], ['打つ', 4], ['打ち込む', 4], ['画像', 1] ], readings: [ ['だ', 1], ['ダース', 1], ['うつ', 2], ['ぶつ', 2], ['うちこむ', 2], ['ぶちこむ', 2], ['がぞう', 1] ] } }, { inputs: [ { sequenceList: [1] } ], expectedResults: { total: 1, terms: [ ['打', 1] ], readings: [ ['だ', 1] ] } }, { inputs: [ { sequenceList: [2] } ], expectedResults: { total: 1, terms: [ ['打', 1] ], readings: [ ['ダース', 1] ] } }, { inputs: [ { sequenceList: [3] } ], expectedResults: { total: 4, terms: [ ['打つ', 4] ], readings: [ ['うつ', 2], ['ぶつ', 2] ] } }, { inputs: [ { sequenceList: [4] } ], expectedResults: { total: 4, terms: [ ['打ち込む', 4] ], readings: [ ['うちこむ', 2], ['ぶちこむ', 2] ] } }, { inputs: [ { sequenceList: [5] } ], expectedResults: { total: 1, terms: [ ['画像', 1] ], readings: [ ['がぞう', 1] ] } }, { inputs: [ { sequenceList: [-1] } ], expectedResults: { total: 0, terms: [], readings: [] } }, { inputs: [ { sequenceList: [] } ], expectedResults: { total: 0, terms: [], readings: [] } } ]; for (const {inputs, expectedResults} of data) { for (const {sequenceList} of inputs) { const results = await database.findTermsBySequenceBulk(sequenceList.map((query) => ({query, dictionary: mainDictionary}))); expect(results.length).toStrictEqual(expectedResults.total); for (const [term, count] of expectedResults.terms) { expect(countDictionaryDatabaseEntriesWithTerm(results, term)).toStrictEqual(count); } for (const [reading, count] of expectedResults.readings) { expect(countDictionaryDatabaseEntriesWithReading(results, reading)).toStrictEqual(count); } } } }); } async function testFindTermMetaBulk1(database, titles) { test('FindTermMetaBulk1', async () => { const data = [ { inputs: [ { termList: ['打'] } ], expectedResults: { total: 11, modes: [ ['freq', 11] ] } }, { inputs: [ { termList: ['打つ'] } ], expectedResults: { total: 10, modes: [ ['freq', 10] ] } }, { inputs: [ { termList: ['打ち込む'] } ], expectedResults: { total: 12, modes: [ ['freq', 10], ['pitch', 2] ] } }, { inputs: [ { termList: ['?'] } ], expectedResults: { total: 0, modes: [] } } ]; for (const {inputs, expectedResults} of data) { for (const {termList} of inputs) { const results = await database.findTermMetaBulk(termList, titles); expect(results.length).toStrictEqual(expectedResults.total); for (const [mode, count] of expectedResults.modes) { expect(countMetasWithMode(results, mode)).toStrictEqual(count); } } } }); } async function testFindKanjiBulk1(database, titles) { test('FindKanjiBulk1', async () => { const data = [ { inputs: [ { kanjiList: ['打'] } ], expectedResults: { total: 1, kanji: [ ['打', 1] ] } }, { inputs: [ { kanjiList: ['込'] } ], expectedResults: { total: 1, kanji: [ ['込', 1] ] } }, { inputs: [ { kanjiList: ['?'] } ], expectedResults: { total: 0, kanji: [] } } ]; for (const {inputs, expectedResults} of data) { for (const {kanjiList} of inputs) { const results = await database.findKanjiBulk(kanjiList, titles); expect(results.length).toStrictEqual(expectedResults.total); for (const [kanji, count] of expectedResults.kanji) { expect(countKanjiWithCharacter(results, kanji)).toStrictEqual(count); } } } }); } async function testFindKanjiMetaBulk1(database, titles) { test('FindKanjiMetaBulk1', async () => { const data = [ { inputs: [ { kanjiList: ['打'] } ], expectedResults: { total: 3, modes: [ ['freq', 3] ] } }, { inputs: [ { kanjiList: ['込'] } ], expectedResults: { total: 3, modes: [ ['freq', 3] ] } }, { inputs: [ { kanjiList: ['?'] } ], expectedResults: { total: 0, modes: [] } } ]; for (const {inputs, expectedResults} of data) { for (const {kanjiList} of inputs) { const results = await database.findKanjiMetaBulk(kanjiList, titles); expect(results.length).toStrictEqual(expectedResults.total); for (const [mode, count] of expectedResults.modes) { expect(countMetasWithMode(results, mode)).toStrictEqual(count); } } } }); } async function testFindTagForTitle1(database, title) { test('FindTagForTitle1', async () => { const data = [ { inputs: [ { name: 'E1' } ], expectedResults: { value: {category: 'default', dictionary: title, name: 'E1', notes: 'example tag 1', order: 0, score: 0} } }, { inputs: [ { name: 'K1' } ], expectedResults: { value: {category: 'default', dictionary: title, name: 'K1', notes: 'example kanji tag 1', order: 0, score: 0} } }, { inputs: [ { name: 'kstat1' } ], expectedResults: { value: {category: 'class', dictionary: title, name: 'kstat1', notes: 'kanji stat 1', order: 0, score: 0} } }, { inputs: [ { name: 'invalid' } ], expectedResults: { value: null } } ]; for (const {inputs, expectedResults} of data) { for (const {name} of inputs) { const result = await database.findTagForTitle(name, title); expect(result).toStrictEqual(expectedResults.value); } } }); } async function testDatabase2() { test('Database2', async () => { // Load dictionary data const testDictionary = createTestDictionaryArchive('valid-dictionary1'); const testDictionarySource = await testDictionary.generateAsync({type: 'arraybuffer'}); const testDictionaryIndex = JSON.parse(await testDictionary.files['index.json'].async('string')); const title = testDictionaryIndex.title; const titles = new Map([ [title, {priority: 0, allowSecondarySearches: false}] ]); // Setup database const dictionaryDatabase = new DictionaryDatabase(); // Database not open await expect(dictionaryDatabase.deleteDictionary(title, 1000)).rejects.toThrow('Database not open'); await expect(dictionaryDatabase.findTermsBulk(['?'], titles, null)).rejects.toThrow('Database not open'); await expect(dictionaryDatabase.findTermsExactBulk([{term: '?', reading: '?'}], titles)).rejects.toThrow('Database not open'); await expect(dictionaryDatabase.findTermsBySequenceBulk([{query: 1, dictionary: title}])).rejects.toThrow('Database not open'); await expect(dictionaryDatabase.findTermMetaBulk(['?'], titles)).rejects.toThrow('Database not open'); await expect(dictionaryDatabase.findTermMetaBulk(['?'], titles)).rejects.toThrow('Database not open'); await expect(dictionaryDatabase.findKanjiBulk(['?'], titles)).rejects.toThrow('Database not open'); await expect(dictionaryDatabase.findKanjiMetaBulk(['?'], titles)).rejects.toThrow('Database not open'); await expect(dictionaryDatabase.findTagForTitle('tag', title)).rejects.toThrow('Database not open'); await expect(dictionaryDatabase.getDictionaryInfo()).rejects.toThrow('Database not open'); await expect(dictionaryDatabase.getDictionaryCounts(titles, true)).rejects.toThrow('Database not open'); await expect(createDictionaryImporter().importDictionary(dictionaryDatabase, testDictionarySource, {})).rejects.toThrow('Database is not ready'); await dictionaryDatabase.prepare(); // already prepared await expect(dictionaryDatabase.prepare()).rejects.toThrow('Database already open'); await createDictionaryImporter().importDictionary(dictionaryDatabase, testDictionarySource, {}); // dictionary already imported await expect(createDictionaryImporter().importDictionary(dictionaryDatabase, testDictionarySource, {})).rejects.toThrow('Dictionary is already imported'); await dictionaryDatabase.close(); }); } async function testDatabase3() { const invalidDictionaries = [ 'invalid-dictionary1', 'invalid-dictionary2', 'invalid-dictionary3', 'invalid-dictionary4', 'invalid-dictionary5', 'invalid-dictionary6' ]; describe('Database3', () => { for (const invalidDictionary of invalidDictionaries) { test(`${invalidDictionary}`, async () => { // Setup database const dictionaryDatabase = new DictionaryDatabase(); await dictionaryDatabase.prepare(); const testDictionary = createTestDictionaryArchive(invalidDictionary); const testDictionarySource = await testDictionary.generateAsync({type: 'arraybuffer'}); await expect(createDictionaryImporter().importDictionary(dictionaryDatabase, testDictionarySource, {})).rejects.toThrow('Dictionary has invalid data'); await dictionaryDatabase.close(); }); } }); } async function main() { beforeEach(async () => { globalThis.indexedDB = new IDBFactory(); }); await testDatabase1(); await testDatabase2(); await testDatabase3(); } await main();