Merge pull request #307 from themoeway/modernize

Modernize codebase

13 files changed, 122 insertions, 143 deletions

diff --git a/ext/js/language/__mocks__/dictionary-importer-media-loader.js b/ext/js/language/__mocks__/dictionary-importer-media-loader.js
new file mode 100644
index 00000000..96f0f6dd
--- /dev/null
+++ b/ext/js/language/__mocks__/dictionary-importer-media-loader.js
@@ -0,0 +1,24 @@
+/*
+ * Copyright (C) 2023  Yomitan Authors
+ * Copyright (C) 2021-2022  Yomichan Authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+export class DictionaryImporterMediaLoader {
+    async getImageDetails(content) {
+        // Placeholder values
+        return {content, width: 100, height: 100};
+    }
+}
diff --git a/ext/js/language/deinflector.js b/ext/js/language/deinflector.js
index b11aa918..3012c29a 100644
--- a/ext/js/language/deinflector.js
+++ b/ext/js/language/deinflector.js
@@ -16,7 +16,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-class Deinflector {
+export class Deinflector {
     constructor(reasons) {
         this.reasons = Deinflector.normalizeReasons(reasons);
     }
diff --git a/ext/js/language/dictionary-database.js b/ext/js/language/dictionary-database.js
index 9f2b9e45..da365da7 100644
--- a/ext/js/language/dictionary-database.js
+++ b/ext/js/language/dictionary-database.js
@@ -16,11 +16,10 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-/* global
- * Database
- */
+import {log, stringReverse} from '../core.js';
+import {Database} from '../data/database.js';
 
-class DictionaryDatabase {
+export class DictionaryDatabase {
     constructor() {
         this._db = new Database();
         this._dbName = 'dict';
diff --git a/ext/js/language/dictionary-importer-media-loader.js b/ext/js/language/dictionary-importer-media-loader.js
index 4024d0b3..7d4f798c 100644
--- a/ext/js/language/dictionary-importer-media-loader.js
+++ b/ext/js/language/dictionary-importer-media-loader.js
@@ -16,10 +16,12 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
+import {EventListenerCollection} from '../core.js';
+
 /**
  * Class used for loading and validating media during the dictionary import process.
  */
-class DictionaryImporterMediaLoader {
+export class DictionaryImporterMediaLoader {
     /**
      * Attempts to load an image using an ArrayBuffer and a media type to return details about it.
      * @param {ArrayBuffer} content The binary content for the image, encoded as an ArrayBuffer.
diff --git a/ext/js/language/dictionary-importer.js b/ext/js/language/dictionary-importer.js
index 718d9f1c..791d1a77 100644
--- a/ext/js/language/dictionary-importer.js
+++ b/ext/js/language/dictionary-importer.js
@@ -16,13 +16,11 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-/* global
- * JSZip
- * JsonSchema
- * MediaUtil
- */
-
-class DictionaryImporter {
+import * as ajvSchemas from '../../lib/validate-schemas.js';
+import {BlobWriter, TextWriter, Uint8ArrayReader, ZipReader, configure} from '../../lib/zip.js';
+import {stringReverse} from '../core.js';
+import {MediaUtil} from '../media/media-util.js';
+export class DictionaryImporter {
     constructor(mediaLoader, onProgress) {
         this._mediaLoader = mediaLoader;
         this._onProgress = typeof onProgress === 'function' ? onProgress : () => {};
@@ -39,20 +37,36 @@ class DictionaryImporter {
 
         this._progressReset();
 
-        // Read archive
-        const archive = await JSZip.loadAsync(archiveContent);
+        configure({
+            workerScripts: {
+                deflate: ['../../lib/z-worker.js'],
+                inflate: ['../../lib/z-worker.js']
+            }
+        });
 
+        // Read archive
+        const zipFileReader = new Uint8ArrayReader(new Uint8Array(archiveContent));
+        const zipReader = new ZipReader(zipFileReader);
+        const zipEntries = await zipReader.getEntries();
+        const zipEntriesObject = {};
+        for (const entry of zipEntries) {
+            zipEntriesObject[entry.filename] = entry;
+        }
         // Read and validate index
         const indexFileName = 'index.json';
-        const indexFile = archive.file(indexFileName);
+        const indexFile = zipEntriesObject[indexFileName];
         if (!indexFile) {
             throw new Error('No dictionary index found in archive');
         }
 
-        const index = JSON.parse(await indexFile.async('string'));
+        const indexContent = await indexFile.getData(
+            new TextWriter()
+        );
+        const index = JSON.parse(indexContent);
 
-        const indexSchema = await this._getSchema('/data/schemas/dictionary-index-schema.json');
-        this._validateJsonSchema(index, indexSchema, indexFileName);
+        if (!ajvSchemas.dictionaryIndex(index)) {
+            throw this._formatAjvSchemaError(ajvSchemas.dictionaryIndex, indexFileName);
+        }
 
         const dictionaryTitle = index.title;
         const version = index.format || index.version;
@@ -75,15 +89,14 @@ class DictionaryImporter {
 
         // Load schemas
         this._progressNextStep(0);
-        const dataBankSchemaPaths = this._getDataBankSchemaPaths(version);
-        const dataBankSchemas = await Promise.all(dataBankSchemaPaths.map((path) => this._getSchema(path)));
+        const dataBankSchemas = this._getDataBankSchemas(version);
 
         // Files
-        const termFiles      = this._getArchiveFiles(archive, 'term_bank_?.json');
-        const termMetaFiles  = this._getArchiveFiles(archive, 'term_meta_bank_?.json');
-        const kanjiFiles     = this._getArchiveFiles(archive, 'kanji_bank_?.json');
-        const kanjiMetaFiles = this._getArchiveFiles(archive, 'kanji_meta_bank_?.json');
-        const tagFiles       = this._getArchiveFiles(archive, 'tag_bank_?.json');
+        const termFiles      = this._getArchiveFiles(zipEntriesObject, 'term_bank_?.json');
+        const termMetaFiles  = this._getArchiveFiles(zipEntriesObject, 'term_meta_bank_?.json');
+        const kanjiFiles     = this._getArchiveFiles(zipEntriesObject, 'kanji_bank_?.json');
+        const kanjiMetaFiles = this._getArchiveFiles(zipEntriesObject, 'kanji_meta_bank_?.json');
+        const tagFiles       = this._getArchiveFiles(zipEntriesObject, 'tag_bank_?.json');
 
         // Load data
         this._progressNextStep(termFiles.length + termMetaFiles.length + kanjiFiles.length + kanjiMetaFiles.length + tagFiles.length);
@@ -124,7 +137,7 @@ class DictionaryImporter {
 
         // Async requirements
         this._progressNextStep(requirements.length);
-        const {media} = await this._resolveAsyncRequirements(requirements, archive);
+        const {media} = await this._resolveAsyncRequirements(requirements, zipEntriesObject);
 
         // Add dictionary descriptor
         this._progressNextStep(termList.length + termMetaList.length + kanjiList.length + kanjiMetaList.length + tagList.length + media.length);
@@ -214,68 +227,27 @@ class DictionaryImporter {
         return summary;
     }
 
-    async _getSchema(fileName) {
-        const schema = await this._fetchJsonAsset(fileName);
-        return new JsonSchema(schema);
-    }
-
-    _validateJsonSchema(value, schema, fileName) {
-        try {
-            schema.validate(value);
-        } catch (e) {
-            throw this._formatSchemaError(e, fileName);
-        }
-    }
-
-    _formatSchemaError(e, fileName) {
-        const valuePathString = this._getSchemaErrorPathString(e.valueStack, 'dictionary');
-        const schemaPathString = this._getSchemaErrorPathString(e.schemaStack, 'schema');
-
-        const e2 = new Error(`Dictionary has invalid data in '${fileName}' for value '${valuePathString}', validated against '${schemaPathString}': ${e.message}`);
-        e2.data = e;
+    _formatAjvSchemaError(schema, fileName) {
+        const e2 = new Error(`Dictionary has invalid data in '${fileName}'`);
+        e2.data = schema.errors;
 
         return e2;
     }
 
-    _getSchemaErrorPathString(infoList, base='') {
-        let result = base;
-        for (const {path} of infoList) {
-            const pathArray = Array.isArray(path) ? path : [path];
-            for (const pathPart of pathArray) {
-                if (pathPart === null) {
-                    result = base;
-                } else {
-                    switch (typeof pathPart) {
-                        case 'string':
-                            if (result.length > 0) {
-                                result += '.';
-                            }
-                            result += pathPart;
-                            break;
-                        case 'number':
-                            result += `[${pathPart}]`;
-                            break;
-                    }
-                }
-            }
-        }
-        return result;
-    }
-
-    _getDataBankSchemaPaths(version) {
+    _getDataBankSchemas(version) {
         const termBank = (
             version === 1 ?
-            '/data/schemas/dictionary-term-bank-v1-schema.json' :
-            '/data/schemas/dictionary-term-bank-v3-schema.json'
+            'dictionaryTermBankV1' :
+            'dictionaryTermBankV3'
         );
-        const termMetaBank = '/data/schemas/dictionary-term-meta-bank-v3-schema.json';
+        const termMetaBank = 'dictionaryTermMetaBankV3';
         const kanjiBank = (
             version === 1 ?
-            '/data/schemas/dictionary-kanji-bank-v1-schema.json' :
-            '/data/schemas/dictionary-kanji-bank-v3-schema.json'
+            'dictionaryKanjiBankV1' :
+            'dictionaryKanjiBankV3'
         );
-        const kanjiMetaBank = '/data/schemas/dictionary-kanji-meta-bank-v3-schema.json';
-        const tagBank = '/data/schemas/dictionary-tag-bank-v3-schema.json';
+        const kanjiMetaBank = 'dictionaryKanjiMetaBankV3';
+        const tagBank = 'dictionaryTagBankV3';
 
         return [termBank, termMetaBank, kanjiBank, kanjiMetaBank, tagBank];
     }
@@ -335,9 +307,9 @@ class DictionaryImporter {
         return target;
     }
 
-    async _resolveAsyncRequirements(requirements, archive) {
+    async _resolveAsyncRequirements(requirements, zipEntriesObject) {
         const media = new Map();
-        const context = {archive, media};
+        const context = {zipEntriesObject, media};
 
         for (const requirement of requirements) {
             await this._resolveAsyncRequirement(context, requirement);
@@ -427,13 +399,16 @@ class DictionaryImporter {
         }
 
         // Find file in archive
-        const file = context.archive.file(path);
+        const file = context.zipEntriesObject[path];
         if (file === null) {
             throw createError('Could not find image');
         }
 
         // Load file content
-        let content = await file.async('arraybuffer');
+        let content = await (await file.getData(
+            new BlobWriter()
+        )).arrayBuffer();
+
         const mediaType = MediaUtil.getImageMediaTypeFromFileName(path);
         if (mediaType === null) {
             throw createError('Could not determine media type for image');
@@ -525,42 +500,36 @@ class DictionaryImporter {
         }
     }
 
-    _getArchiveFiles(archive, fileNameFormat) {
+    _getArchiveFiles(zipEntriesObject, fileNameFormat) {
         const indexPosition = fileNameFormat.indexOf('?');
         const prefix = fileNameFormat.substring(0, indexPosition);
         const suffix = fileNameFormat.substring(indexPosition + 1);
         const results = [];
-        for (let i = 1; true; ++i) {
-            const fileName = `${prefix}${i}${suffix}`;
-            const file = archive.file(fileName);
-            if (!file) { break; }
-            results.push(file);
+        for (const f of Object.keys(zipEntriesObject)) {
+            if (f.startsWith(prefix) && f.endsWith(suffix)) {
+                results.push(zipEntriesObject[f]);
+            }
         }
         return results;
     }
 
-    async _readFileSequence(files, convertEntry, schema, dictionaryTitle) {
+    async _readFileSequence(files, convertEntry, schemaName, dictionaryTitle) {
         const progressData = this._progressData;
-        let count = 0;
         let startIndex = 0;
-        if (typeof this._onProgress === 'function') {
-            schema.progressInterval = 1000;
-            schema.progress = (s) => {
-                const index = s.getValueStackLength() > 1 ? s.getValueStackItem(1).path : 0;
-                progressData.index = startIndex + (index / count);
-                this._progress();
-            };
-        }
 
         const results = [];
-        for (const file of files) {
-            const entries = JSON.parse(await file.async('string'));
+        for (const fileName of Object.keys(files)) {
+            const content = await files[fileName].getData(
+                new TextWriter()
+            );
+            const entries = JSON.parse(content);
 
-            count = Array.isArray(entries) ? Math.max(entries.length, 1) : 1;
             startIndex = progressData.index;
             this._progress();
 
-            this._validateJsonSchema(entries, schema, file.name);
+            if (!ajvSchemas[schemaName](entries)) {
+                throw this._formatAjvSchemaError(ajvSchemas[schemaName], fileName);
+            }
 
             progressData.index = startIndex + 1;
             this._progress();
diff --git a/ext/js/language/dictionary-worker-handler.js b/ext/js/language/dictionary-worker-handler.js
index 0e3e8495..b8c41b26 100644
--- a/ext/js/language/dictionary-worker-handler.js
+++ b/ext/js/language/dictionary-worker-handler.js
@@ -16,13 +16,12 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-/* global
- * DictionaryDatabase
- * DictionaryImporter
- * DictionaryWorkerMediaLoader
- */
+import {serializeError} from '../core.js';
+import {DictionaryDatabase} from './dictionary-database.js';
+import {DictionaryImporter} from './dictionary-importer.js';
+import {DictionaryWorkerMediaLoader} from './dictionary-worker-media-loader.js';
 
-class DictionaryWorkerHandler {
+export class DictionaryWorkerHandler {
     constructor() {
         this._mediaLoader = new DictionaryWorkerMediaLoader();
     }
diff --git a/ext/js/language/dictionary-worker-main.js b/ext/js/language/dictionary-worker-main.js
index cbb3247c..6d2386aa 100644
--- a/ext/js/language/dictionary-worker-main.js
+++ b/ext/js/language/dictionary-worker-main.js
@@ -16,22 +16,8 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-/* global
- * DictionaryWorkerHandler
- */
-
-self.importScripts(
-    '/lib/jszip.min.js',
-    '/js/core.js',
-    '/js/data/database.js',
-    '/js/data/json-schema.js',
-    '/js/general/cache-map.js',
-    '/js/language/dictionary-database.js',
-    '/js/language/dictionary-importer.js',
-    '/js/language/dictionary-worker-handler.js',
-    '/js/language/dictionary-worker-media-loader.js',
-    '/js/media/media-util.js'
-);
+import {log} from '../core.js';
+import {DictionaryWorkerHandler} from './dictionary-worker-handler.js';
 
 (() => {
     try {
diff --git a/ext/js/language/dictionary-worker-media-loader.js b/ext/js/language/dictionary-worker-media-loader.js
index d2ef4fae..d58e46c5 100644
--- a/ext/js/language/dictionary-worker-media-loader.js
+++ b/ext/js/language/dictionary-worker-media-loader.js
@@ -16,11 +16,13 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
+import {deserializeError, generateId} from '../core.js';
+
 /**
  * Class used for loading and validating media from a worker thread
  * during the dictionary import process.
  */
-class DictionaryWorkerMediaLoader {
+export class DictionaryWorkerMediaLoader {
     /**
      * Creates a new instance of the media loader.
      */
diff --git a/ext/js/language/dictionary-worker.js b/ext/js/language/dictionary-worker.js
index b415a346..18c300af 100644
--- a/ext/js/language/dictionary-worker.js
+++ b/ext/js/language/dictionary-worker.js
@@ -16,11 +16,10 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-/* global
- * DictionaryImporterMediaLoader
- */
+import {deserializeError, serializeError} from '../core.js';
+import {DictionaryImporterMediaLoader} from './dictionary-importer-media-loader.js';
 
-class DictionaryWorker {
+export class DictionaryWorker {
     constructor() {
         this._dictionaryImporterMediaLoader = new DictionaryImporterMediaLoader();
     }
@@ -47,7 +46,7 @@ class DictionaryWorker {
 
     _invoke(action, params, transfer, onProgress, formatResult) {
         return new Promise((resolve, reject) => {
-            const worker = new Worker('/js/language/dictionary-worker-main.js', {});
+            const worker = new Worker('/js/language/dictionary-worker-main.js', {type: 'module'});
             const details = {
                 complete: false,
                 worker,
diff --git a/ext/js/language/sandbox/dictionary-data-util.js b/ext/js/language/sandbox/dictionary-data-util.js
index 323076e8..1b71346a 100644
--- a/ext/js/language/sandbox/dictionary-data-util.js
+++ b/ext/js/language/sandbox/dictionary-data-util.js
@@ -16,7 +16,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-class DictionaryDataUtil {
+export class DictionaryDataUtil {
     static groupTermTags(dictionaryEntry) {
         const {headwords} = dictionaryEntry;
         const headwordCount = headwords.length;
diff --git a/ext/js/language/sandbox/japanese-util.js b/ext/js/language/sandbox/japanese-util.js
index eee70f9c..316b1c2e 100644
--- a/ext/js/language/sandbox/japanese-util.js
+++ b/ext/js/language/sandbox/japanese-util.js
@@ -16,7 +16,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-const JapaneseUtil = (() => {
+export const JapaneseUtil = (() => {
     const HIRAGANA_SMALL_TSU_CODE_POINT = 0x3063;
     const KATAKANA_SMALL_TSU_CODE_POINT = 0x30c3;
     const KATAKANA_SMALL_KA_CODE_POINT = 0x30f5;
diff --git a/ext/js/language/text-scanner.js b/ext/js/language/text-scanner.js
index af5cc8fe..bd5b0fbe 100644
--- a/ext/js/language/text-scanner.js
+++ b/ext/js/language/text-scanner.js
@@ -16,11 +16,11 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-/* global
- * DocumentUtil
- */
+import {EventDispatcher, EventListenerCollection, clone, isObject, log, promiseTimeout} from '../core.js';
+import {DocumentUtil} from '../dom/document-util.js';
+import {yomichan} from '../yomichan.js';
 
-class TextScanner extends EventDispatcher {
+export class TextScanner extends EventDispatcher {
     constructor({
         node,
         getSearchContext,
diff --git a/ext/js/language/translator.js b/ext/js/language/translator.js
index 3b47cc51..4044f379 100644
--- a/ext/js/language/translator.js
+++ b/ext/js/language/translator.js
@@ -16,16 +16,15 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-/* global
- * Deinflector
- * RegexUtil
- * TextSourceMap
- */
+import {RegexUtil} from '../general/regex-util.js';
+import {TextSourceMap} from '../general/text-source-map.js';
+import {Deinflector} from './deinflector.js';
+import {DictionaryDatabase} from './dictionary-database.js';
 
 /**
  * Class which finds term and kanji dictionary entries for text.
  */
-class Translator {
+export class Translator {
     /**
      * Information about how popup content should be shown, specifically related to the outer popup frame.
      * @typedef {object} TermFrequency