add lingua libre audio source (#1129)

* add lingua libre audio source

* mvp

* run file requests in parallel

* remove redundant language var

* redundant api function

---------

Co-authored-by: Cashew <52880648+cashewnuttynuts@users.noreply.github.com>

1 files changed, 60 insertions, 36 deletions

diff --git a/ext/js/media/audio-downloader.js b/ext/js/media/audio-downloader.js
index 2d1bc4ec..d378d043 100644
--- a/ext/js/media/audio-downloader.js
+++ b/ext/js/media/audio-downloader.js
@@ -25,6 +25,16 @@ import {NativeSimpleDOMParser} from '../dom/native-simple-dom-parser.js';
 import {SimpleDOMParser} from '../dom/simple-dom-parser.js';
 import {isStringEntirelyKana} from '../language/ja/japanese.js';
 
+/** @type {RequestInit} */
+const DEFAULT_REQUEST_INIT_PARAMS = {
+    method: 'GET',
+    mode: 'cors',
+    cache: 'default',
+    credentials: 'omit',
+    redirect: 'follow',
+    referrerPolicy: 'no-referrer',
+};
+
 export class AudioDownloader {
     /**
      * @param {RequestBuilder} requestBuilder
@@ -39,6 +49,7 @@ export class AudioDownloader {
             ['jpod101', this._getInfoJpod101.bind(this)],
             ['jpod101-alternate', this._getInfoJpod101Alternate.bind(this)],
             ['jisho', this._getInfoJisho.bind(this)],
+            ['lingua-libre', this._getInfoLinguaLibre.bind(this)],
             ['text-to-speech', this._getInfoTextToSpeech.bind(this)],
             ['text-to-speech-reading', this._getInfoTextToSpeechReading.bind(this)],
             ['custom', this._getInfoCustom.bind(this)],
@@ -50,13 +61,14 @@ export class AudioDownloader {
      * @param {import('audio').AudioSourceInfo} source
      * @param {string} term
      * @param {string} reading
+     * @param {import('language').LanguageSummary} languageSummary
      * @returns {Promise<import('audio-downloader').Info[]>}
      */
-    async getTermAudioInfoList(source, term, reading) {
+    async getTermAudioInfoList(source, term, reading, languageSummary) {
         const handler = this._getInfoHandlers.get(source.type);
         if (typeof handler === 'function') {
             try {
-                return await handler(term, reading, source);
+                return await handler(term, reading, source, languageSummary);
             } catch (e) {
                 // NOP
             }
@@ -70,12 +82,13 @@ export class AudioDownloader {
      * @param {string} term
      * @param {string} reading
      * @param {?number} idleTimeout
+     * @param {import('language').LanguageSummary} languageSummary
      * @returns {Promise<import('audio-downloader').AudioBinaryBase64>}
      */
-    async downloadTermAudio(sources, preferredAudioIndex, term, reading, idleTimeout) {
+    async downloadTermAudio(sources, preferredAudioIndex, term, reading, idleTimeout, languageSummary) {
         const errors = [];
         for (const source of sources) {
-            let infoList = await this.getTermAudioInfoList(source, term, reading);
+            let infoList = await this.getTermAudioInfoList(source, term, reading, languageSummary);
             if (typeof preferredAudioIndex === 'number') {
                 infoList = (preferredAudioIndex >= 0 && preferredAudioIndex < infoList.length ? [infoList[preferredAudioIndex]] : []);
             }
@@ -137,12 +150,8 @@ export class AudioDownloader {
             vulgar: 'true',
         });
         const response = await this._requestBuilder.fetchAnonymous(fetchUrl, {
+            ...DEFAULT_REQUEST_INIT_PARAMS,
             method: 'POST',
-            mode: 'cors',
-            cache: 'default',
-            credentials: 'omit',
-            redirect: 'follow',
-            referrerPolicy: 'no-referrer',
             headers: {
                 'Content-Type': 'application/x-www-form-urlencoded',
             },
@@ -181,14 +190,7 @@ export class AudioDownloader {
     /** @type {import('audio-downloader').GetInfoHandler} */
     async _getInfoJisho(term, reading) {
         const fetchUrl = `https://jisho.org/search/${term}`;
-        const response = await this._requestBuilder.fetchAnonymous(fetchUrl, {
-            method: 'GET',
-            mode: 'cors',
-            cache: 'default',
-            credentials: 'omit',
-            redirect: 'follow',
-            referrerPolicy: 'no-referrer',
-        });
+        const response = await this._requestBuilder.fetchAnonymous(fetchUrl, DEFAULT_REQUEST_INIT_PARAMS);
         const responseText = await response.text();
 
         const dom = this._createSimpleDOMParser(responseText);
@@ -212,6 +214,44 @@ export class AudioDownloader {
     }
 
     /** @type {import('audio-downloader').GetInfoHandler} */
+    async _getInfoLinguaLibre(term, _reading, _details, languageSummary) {
+        if (typeof languageSummary !== 'object' || languageSummary === null) {
+            throw new Error('Invalid arguments');
+        }
+        const {iso639_3} = languageSummary;
+        const searchCategory = `incategory:"Lingua_Libre_pronunciation-${iso639_3}"`;
+        const searchString = `-${term}.wav`;
+        const fetchUrl = `https://commons.wikimedia.org/w/api.php?action=query&format=json&list=search&srsearch=intitle:/${searchString}/i+${searchCategory}&srnamespace=6&origin=*`;
+
+        const response = await this._requestBuilder.fetchAnonymous(fetchUrl, DEFAULT_REQUEST_INIT_PARAMS);
+
+        /** @type {import('audio-downloader').LinguaLibreLookupResponse} */
+        const lookupResponse = await readResponseJson(response);
+
+        const lookupResults = lookupResponse.query.search;
+
+        const fetchFileInfos = lookupResults.map(async ({title}) => {
+            const fileInfoURL = `https://commons.wikimedia.org/w/api.php?action=query&format=json&titles=${title}&prop=imageinfo&iiprop=user|url&origin=*`;
+            const response2 = await this._requestBuilder.fetchAnonymous(fileInfoURL, DEFAULT_REQUEST_INIT_PARAMS);
+            /** @type {import('audio-downloader').LinguaLibreFileResponse} */
+            const fileResponse = await readResponseJson(response2);
+            const fileResults = fileResponse.query.pages;
+            const results = [];
+            for (const page of Object.values(fileResults)) {
+                const fileUrl = page.imageinfo[0].url;
+                const fileUser = page.imageinfo[0].user;
+                const validFilenameTest = new RegExp(`^File:LL-Q\\d+\\s+\\(${iso639_3}\\)-(\\w+ \\()?${fileUser}\\)?-${term}\\.wav$`, 'i');
+                if (validFilenameTest.test(title)) {
+                    results.push({type: 'url', url: fileUrl, name: fileUser});
+                }
+            }
+            return /** @type {import('audio-downloader').Info1[]} */ (results);
+        });
+
+        return (await Promise.all(fetchFileInfos)).flat();
+    }
+
+    /** @type {import('audio-downloader').GetInfoHandler} */
     async _getInfoTextToSpeech(term, reading, details) {
         if (typeof details !== 'object' || details === null) {
             throw new Error('Invalid arguments');
@@ -259,14 +299,7 @@ export class AudioDownloader {
         }
         url = this._getCustomUrl(term, reading, url);
 
-        const response = await this._requestBuilder.fetchAnonymous(url, {
-            method: 'GET',
-            mode: 'cors',
-            cache: 'default',
-            credentials: 'omit',
-            redirect: 'follow',
-            referrerPolicy: 'no-referrer',
-        });
+        const response = await this._requestBuilder.fetchAnonymous(url, DEFAULT_REQUEST_INIT_PARAMS);
 
         if (!response.ok) {
             throw new Error(`Invalid response: ${response.status}`);
@@ -345,12 +378,7 @@ export class AudioDownloader {
         }
 
         const response = await this._requestBuilder.fetchAnonymous(url, {
-            method: 'GET',
-            mode: 'cors',
-            cache: 'default',
-            credentials: 'omit',
-            redirect: 'follow',
-            referrerPolicy: 'no-referrer',
+            ...DEFAULT_REQUEST_INIT_PARAMS,
             signal,
         });
 
@@ -429,12 +457,8 @@ export class AudioDownloader {
     async _getCustomAudioListSchema() {
         const url = chrome.runtime.getURL('/data/schemas/custom-audio-list-schema.json');
         const response = await fetch(url, {
-            method: 'GET',
+            ...DEFAULT_REQUEST_INIT_PARAMS,
             mode: 'no-cors',
-            cache: 'default',
-            credentials: 'omit',
-            redirect: 'follow',
-            referrerPolicy: 'no-referrer',
         });
         return await readResponseJson(response);
     }