Add wiktionary commons audio source (#1143)

* add audio source for wiktionary format filenames on commons

* allow region codes

* fix files being saved in anki as mp3

6 files changed, 71 insertions, 7 deletions

diff --git a/ext/data/schemas/options-schema.json b/ext/data/schemas/options-schema.json
index 84619ab4..4bd7625a 100644
--- a/ext/data/schemas/options-schema.json
+++ b/ext/data/schemas/options-schema.json
@@ -401,6 +401,7 @@
                                                         "jpod101-alternate",
                                                         "jisho",
                                                         "lingua-libre",
+                                                        "wiktionary",
                                                         "text-to-speech",
                                                         "text-to-speech-reading",
                                                         "custom",
diff --git a/ext/js/display/display-audio.js b/ext/js/display/display-audio.js
index 0d1ca029..bb30c944 100644
--- a/ext/js/display/display-audio.js
+++ b/ext/js/display/display-audio.js
@@ -58,6 +58,7 @@ export class DisplayAudio {
             ['jpod101-alternate', 'JapanesePod101 (Alternate)'],
             ['jisho', 'Jisho.org'],
             ['lingua-libre', 'Lingua Libre'],
+            ['wiktionary', 'Wiktionary'],
             ['text-to-speech', 'Text-to-speech'],
             ['text-to-speech-reading', 'Text-to-speech (Kana reading)'],
             ['custom', 'Custom URL'],
diff --git a/ext/js/media/audio-downloader.js b/ext/js/media/audio-downloader.js
index d378d043..99ca1dfd 100644
--- a/ext/js/media/audio-downloader.js
+++ b/ext/js/media/audio-downloader.js
@@ -50,11 +50,14 @@ export class AudioDownloader {
             ['jpod101-alternate', this._getInfoJpod101Alternate.bind(this)],
             ['jisho', this._getInfoJisho.bind(this)],
             ['lingua-libre', this._getInfoLinguaLibre.bind(this)],
+            ['wiktionary', this._getInfoWiktionary.bind(this)],
             ['text-to-speech', this._getInfoTextToSpeech.bind(this)],
             ['text-to-speech-reading', this._getInfoTextToSpeechReading.bind(this)],
             ['custom', this._getInfoCustom.bind(this)],
             ['custom-json', this._getInfoCustomJson.bind(this)],
         ]));
+        /** @type {Intl.DisplayNames} */
+        this._regionNames = new Intl.DisplayNames(['en'], {type: 'region'});
     }
 
     /**
@@ -223,26 +226,80 @@ export class AudioDownloader {
         const searchString = `-${term}.wav`;
         const fetchUrl = `https://commons.wikimedia.org/w/api.php?action=query&format=json&list=search&srsearch=intitle:/${searchString}/i+${searchCategory}&srnamespace=6&origin=*`;
 
+        /**
+         * @param {string} filename
+         * @param {string} fileUser
+         * @returns {boolean}
+         */
+        const validateFilename = (filename, fileUser) => {
+            const validFilenameTest = new RegExp(`^File:LL-Q\\d+\\s+\\(${iso639_3}\\)-${fileUser}-${term}\\.wav$`, 'i');
+            return validFilenameTest.test(filename);
+        };
+
+        return await this.getInfoWikimediaCommons(fetchUrl, validateFilename);
+    }
+
+    /** @type {import('audio-downloader').GetInfoHandler} */
+    async _getInfoWiktionary(term, _reading, _details, languageSummary) {
+        if (typeof languageSummary !== 'object' || languageSummary === null) {
+            throw new Error('Invalid arguments');
+        }
+        const {iso} = languageSummary;
+        const searchString = `${iso}(-[a-zA-Z]{2})?-${term}[0123456789]*.ogg`;
+        const fetchUrl = `https://commons.wikimedia.org/w/api.php?action=query&format=json&list=search&srsearch=intitle:/${searchString}/i&srnamespace=6&origin=*`;
+
+        /**
+         * @param {string} filename
+         * @returns {boolean}
+         */
+        const validateFilename = (filename) => {
+            const validFilenameTest = new RegExp(`^File:${iso}(-\\w\\w)?-${term}\\d*\\.ogg$`, 'i');
+            return validFilenameTest.test(filename);
+        };
+
+        /**
+         * @param {string} filename
+         * @param {string} fileUser
+         * @returns {string}
+         */
+        const displayName = (filename, fileUser) => {
+            const match = filename.match(new RegExp(`^File:${iso}(-\\w\\w)-${term}`, 'i'));
+            if (match === null) {
+                return fileUser;
+            }
+            const region = match[1].substring(1).toUpperCase();
+            const regionName = this._regionNames.of(region);
+            return `(${regionName}) ${fileUser}`;
+        };
+
+        return await this.getInfoWikimediaCommons(fetchUrl, validateFilename, displayName);
+    }
+
+    /**
+     * @param {string} fetchUrl
+     * @param {(filename: string, fileUser: string) => boolean} validateFilename
+     * @param {(filename: string, fileUser: string) => string} [displayName]
+     * @returns {Promise<import('audio-downloader').Info1[]>}
+     */
+    async getInfoWikimediaCommons(fetchUrl, validateFilename, displayName = (_filename, fileUser) => fileUser) {
         const response = await this._requestBuilder.fetchAnonymous(fetchUrl, DEFAULT_REQUEST_INIT_PARAMS);
 
-        /** @type {import('audio-downloader').LinguaLibreLookupResponse} */
+        /** @type {import('audio-downloader').WikimediaCommonsLookupResponse} */
         const lookupResponse = await readResponseJson(response);
-
         const lookupResults = lookupResponse.query.search;
 
         const fetchFileInfos = lookupResults.map(async ({title}) => {
             const fileInfoURL = `https://commons.wikimedia.org/w/api.php?action=query&format=json&titles=${title}&prop=imageinfo&iiprop=user|url&origin=*`;
             const response2 = await this._requestBuilder.fetchAnonymous(fileInfoURL, DEFAULT_REQUEST_INIT_PARAMS);
-            /** @type {import('audio-downloader').LinguaLibreFileResponse} */
+            /** @type {import('audio-downloader').WikimediaCommonsFileResponse} */
             const fileResponse = await readResponseJson(response2);
             const fileResults = fileResponse.query.pages;
             const results = [];
             for (const page of Object.values(fileResults)) {
                 const fileUrl = page.imageinfo[0].url;
                 const fileUser = page.imageinfo[0].user;
-                const validFilenameTest = new RegExp(`^File:LL-Q\\d+\\s+\\(${iso639_3}\\)-(\\w+ \\()?${fileUser}\\)?-${term}\\.wav$`, 'i');
-                if (validFilenameTest.test(title)) {
-                    results.push({type: 'url', url: fileUrl, name: fileUser});
+                if (validateFilename(title, fileUser)) {
+                    results.push({type: 'url', url: fileUrl, name: displayName(title, fileUser)});
                 }
             }
             return /** @type {import('audio-downloader').Info1[]} */ (results);
diff --git a/ext/js/media/media-util.js b/ext/js/media/media-util.js
index 3e492e42..821748dd 100644
--- a/ext/js/media/media-util.js
+++ b/ext/js/media/media-util.js
@@ -113,6 +113,7 @@ export function getFileExtensionFromAudioMediaType(mediaType) {
             return '.mp4';
         case 'audio/ogg':
         case 'audio/vorbis':
+        case 'application/ogg':
             return '.ogg';
         case 'audio/vnd.wav':
         case 'audio/wave':
diff --git a/ext/js/pages/settings/audio-controller.js b/ext/js/pages/settings/audio-controller.js
index ac79b51a..b89ae2ba 100644
--- a/ext/js/pages/settings/audio-controller.js
+++ b/ext/js/pages/settings/audio-controller.js
@@ -222,6 +222,7 @@ export class AudioController extends EventDispatcher {
             'jpod101-alternate',
             'jisho',
             'lingua-libre',
+            'wiktionary',
             'custom',
         ];
         for (const type of typesAvailable) {
@@ -490,6 +491,7 @@ class AudioSourceEntry {
             case 'jpod101-alternate':
             case 'jisho':
             case 'lingua-libre':
+            case 'wiktionary':
             case 'text-to-speech':
             case 'text-to-speech-reading':
             case 'custom':
diff --git a/ext/templates-settings.html b/ext/templates-settings.html
index 2443843b..bea0aa3e 100644
--- a/ext/templates-settings.html
+++ b/ext/templates-settings.html
@@ -116,7 +116,8 @@
             <option value="jpod101">JapanesePod101</option>
             <option value="jpod101-alternate">JapanesePod101 (Alternate)</option>
             <option value="jisho">Jisho.org</option>
-            <option value="lingua-libre">Lingua Libre</option>
+            <option value="lingua-libre">(Commons) Lingua Libre</option>
+            <option value="wiktionary">(Commons) Wiktionary</option>
             <option value="text-to-speech">Text-to-speech</option>
             <option value="text-to-speech-reading">Text-to-speech (Kana reading)</option>
             <option value="custom">Custom URL</option>
@@ -431,6 +432,7 @@
         <option value="jpod101-alternate">JapanesePod101 (Alternate)</option>
         <option value="jisho">Jisho.org</option>
         <option value="lingua-libre">Lingua Libre</option>
+        <option value="wiktionary">Wiktionary</option>
         <option value="text-to-speech">Text-to-speech</option>
         <option value="text-to-speech-reading">Text-to-speech (Kana reading)</option>
         <option value="custom">Custom</option>