From b1b33f8beb26f97d91cb282682472c65f6eccae8 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sun, 2 Aug 2020 13:30:55 -0400 Subject: Fix fetch requests (#708) * Revert audio fetching functionality to use XMLHttpRequest * Replace requestJson * Replace requestJson * Replace requestJson * Replace requestJson and requestText * Fix tests * Include support for vulgar word searches * Remove request.js --- ext/bg/background.html | 1 - ext/bg/js/anki.js | 15 +++++++++----- ext/bg/js/audio-uri-builder.js | 36 ++++++++++++--------------------- ext/bg/js/backend.js | 29 ++++++++++++++++++--------- ext/bg/js/dictionary-importer.js | 18 +++++++++++++++-- ext/bg/js/request.js | 43 ---------------------------------------- ext/bg/js/translator.js | 19 +++++++++++++++--- ext/mixed/js/audio-system.js | 30 ++++++++++++++-------------- test/test-database.js | 6 ++++-- 9 files changed, 94 insertions(+), 103 deletions(-) delete mode 100644 ext/bg/js/request.js diff --git a/ext/bg/background.html b/ext/bg/background.html index 73dbc251..ab84f69a 100644 --- a/ext/bg/background.html +++ b/ext/bg/background.html @@ -40,7 +40,6 @@ - diff --git a/ext/bg/js/anki.js b/ext/bg/js/anki.js index a8872a52..a72dff2a 100644 --- a/ext/bg/js/anki.js +++ b/ext/bg/js/anki.js @@ -15,10 +15,6 @@ * along with this program. If not, see . */ -/* global - * requestJson - */ - class AnkiConnect { constructor(server) { this._enabled = false; @@ -110,7 +106,16 @@ class AnkiConnect { } async _invoke(action, params) { - const result = await requestJson(this._server, 'POST', {action, params, version: this._localVersion}, true); + const response = await fetch(this._server, { + method: 'POST', + mode: 'cors', + cache: 'default', + credentials: 'omit', + redirect: 'follow', + referrerPolicy: 'no-referrer', + body: JSON.stringify({action, params, version: this._localVersion}) + }); + const result = await response.json(); if (isObject(result)) { const error = result.error; if (typeof error !== 'undefined') { diff --git a/ext/bg/js/audio-uri-builder.js b/ext/bg/js/audio-uri-builder.js index 390e1e4d..11738ef3 100644 --- a/ext/bg/js/audio-uri-builder.js +++ b/ext/bg/js/audio-uri-builder.js @@ -82,21 +82,14 @@ class AudioUriBuilder { } async _getUriJpod101Alternate(definition) { - const fetchUrl = 'https://www.japanesepod101.com/learningcenter/reference/dictionary_post'; - const data = `post=dictionary_reference&match_type=exact&search_query=${encodeURIComponent(definition.expression)}`; - const response = await fetch(fetchUrl, { - method: 'POST', - mode: 'no-cors', - cache: 'default', - credentials: 'omit', - redirect: 'follow', - referrerPolicy: 'no-referrer', - headers: { - 'Content-Type': 'application/x-www-form-urlencoded' - }, - body: data + const responseText = await new Promise((resolve, reject) => { + const xhr = new XMLHttpRequest(); + xhr.open('POST', 'https://www.japanesepod101.com/learningcenter/reference/dictionary_post'); + xhr.setRequestHeader('Content-Type', 'application/x-www-form-urlencoded'); + xhr.addEventListener('error', () => reject(new Error('Failed to scrape audio data'))); + xhr.addEventListener('load', () => resolve(xhr.responseText)); + xhr.send(`post=dictionary_reference&match_type=exact&search_query=${encodeURIComponent(definition.expression)}&vulgar=true`); }); - const responseText = await response.text(); const dom = new DOMParser().parseFromString(responseText, 'text/html'); for (const row of dom.getElementsByClassName('dc-result-row')) { @@ -115,16 +108,13 @@ class AudioUriBuilder { } async _getUriJisho(definition) { - const fetchUrl = `https://jisho.org/search/${definition.expression}`; - const response = await fetch(fetchUrl, { - method: 'GET', - mode: 'no-cors', - cache: 'default', - credentials: 'omit', - redirect: 'follow', - referrerPolicy: 'no-referrer' + const responseText = await new Promise((resolve, reject) => { + const xhr = new XMLHttpRequest(); + xhr.open('GET', `https://jisho.org/search/${definition.expression}`); + xhr.addEventListener('error', () => reject(new Error('Failed to scrape audio data'))); + xhr.addEventListener('load', () => resolve(xhr.responseText)); + xhr.send(); }); - const responseText = await response.text(); const dom = new DOMParser().parseFromString(responseText, 'text/html'); try { diff --git a/ext/bg/js/backend.js b/ext/bg/js/backend.js index 2e772aa1..9e7ac76a 100644 --- a/ext/bg/js/backend.js +++ b/ext/bg/js/backend.js @@ -35,8 +35,6 @@ * jp * profileConditionsDescriptor * profileConditionsDescriptorPromise - * requestJson - * requestText */ class Backend { @@ -199,8 +197,8 @@ class Backend { await profileConditionsDescriptorPromise; - this._optionsSchema = await requestJson(chrome.runtime.getURL('/bg/data/options-schema.json'), 'GET'); - this._defaultAnkiFieldTemplates = (await requestText(chrome.runtime.getURL('/bg/data/default-anki-field-templates.handlebars'), 'GET')).trim(); + this._optionsSchema = await this._fetchAsset('/bg/data/options-schema.json', true); + this._defaultAnkiFieldTemplates = (await this._fetchAsset('/bg/data/default-anki-field-templates.handlebars')).trim(); this._options = await OptionsUtil.load(); this._options = JsonSchema.getValidValueOrDefault(this._optionsSchema, this._options); @@ -615,7 +613,7 @@ class Backend { if (!url.startsWith('/') || url.startsWith('//') || !url.endsWith('.css')) { throw new Error('Invalid URL'); } - return await requestText(url, 'GET'); + return await this._fetchAsset(url); } _onApiGetEnvironmentInfo() { @@ -653,13 +651,11 @@ class Backend { } async _onApiGetDisplayTemplatesHtml() { - const url = chrome.runtime.getURL('/mixed/display-templates.html'); - return await requestText(url, 'GET'); + return await this._fetchAsset('/mixed/display-templates.html'); } async _onApiGetQueryParserTemplatesHtml() { - const url = chrome.runtime.getURL('/bg/query-parser-templates.html'); - return await requestText(url, 'GET'); + return await this._fetchAsset('/bg/query-parser-templates.html'); } _onApiGetZoom(params, sender) { @@ -1522,4 +1518,19 @@ class Backend { } }); } + + async _fetchAsset(url, json=false) { + const response = await fetch(chrome.runtime.getURL(url), { + method: 'GET', + mode: 'no-cors', + cache: 'default', + credentials: 'omit', + redirect: 'follow', + referrerPolicy: 'no-referrer' + }); + if (!response.ok) { + throw new Error(`Failed to fetch ${url}: ${response.status}`); + } + return await (json ? response.json() : response.text()); + } } diff --git a/ext/bg/js/dictionary-importer.js b/ext/bg/js/dictionary-importer.js index 69d5c386..4374ff40 100644 --- a/ext/bg/js/dictionary-importer.js +++ b/ext/bg/js/dictionary-importer.js @@ -19,7 +19,6 @@ * JSZip * JsonSchema * mediaUtility - * requestJson */ class DictionaryImporter { @@ -235,7 +234,7 @@ class DictionaryImporter { return schemaPromise; } - schemaPromise = requestJson(chrome.runtime.getURL(fileName), 'GET'); + schemaPromise = this._fetchJsonAsset(fileName); this._schemas.set(fileName, schemaPromise); return schemaPromise; } @@ -365,4 +364,19 @@ class DictionaryImporter { return newData; } + + async _fetchJsonAsset(url) { + const response = await fetch(chrome.runtime.getURL(url), { + method: 'GET', + mode: 'no-cors', + cache: 'default', + credentials: 'omit', + redirect: 'follow', + referrerPolicy: 'no-referrer' + }); + if (!response.ok) { + throw new Error(`Failed to fetch ${url}: ${response.status}`); + } + return await response.json(); + } } diff --git a/ext/bg/js/request.js b/ext/bg/js/request.js deleted file mode 100644 index 4a455850..00000000 --- a/ext/bg/js/request.js +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (C) 2017-2020 Yomichan Authors - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - - -async function requestText(url, method, data, cors=false) { - const response = await fetch(url, { - method, - mode: (cors ? 'cors' : 'no-cors'), - cache: 'default', - credentials: 'omit', - redirect: 'follow', - referrerPolicy: 'no-referrer', - body: (data ? JSON.stringify(data) : void 0) - }); - return await response.text(); -} - -async function requestJson(url, method, data, cors=false) { - const response = await fetch(url, { - method, - mode: (cors ? 'cors' : 'no-cors'), - cache: 'default', - credentials: 'omit', - redirect: 'follow', - referrerPolicy: 'no-referrer', - body: (data ? JSON.stringify(data) : void 0) - }); - return await response.json(); -} diff --git a/ext/bg/js/translator.js b/ext/bg/js/translator.js index 3fd329d1..a1f30bd2 100644 --- a/ext/bg/js/translator.js +++ b/ext/bg/js/translator.js @@ -29,7 +29,6 @@ * dictTermsSort * dictTermsUndupe * jp - * requestJson */ class Translator { @@ -40,8 +39,7 @@ class Translator { } async prepare() { - const url = chrome.runtime.getURL('/bg/lang/deinflect.json'); - const reasons = await requestJson(url, 'GET'); + const reasons = await this._fetchJsonAsset('/bg/lang/deinflect.json'); this.deinflector = new Deinflector(reasons); } @@ -657,4 +655,19 @@ class Translator { return text; } + + async _fetchJsonAsset(url) { + const response = await fetch(chrome.runtime.getURL(url), { + method: 'GET', + mode: 'no-cors', + cache: 'default', + credentials: 'omit', + redirect: 'follow', + referrerPolicy: 'no-referrer' + }); + if (!response.ok) { + throw new Error(`Failed to fetch ${url}: ${response.status}`); + } + return await response.json(); + } } diff --git a/ext/mixed/js/audio-system.js b/ext/mixed/js/audio-system.js index c590b909..fdfb0b10 100644 --- a/ext/mixed/js/audio-system.js +++ b/ext/mixed/js/audio-system.js @@ -169,22 +169,22 @@ class AudioSystem { }); } - async _createAudioBinaryFromUrl(url) { - const response = await fetch(url, { - method: 'GET', - mode: 'no-cors', - cache: 'default', - credentials: 'omit', - redirect: 'follow', - referrerPolicy: 'no-referrer' + _createAudioBinaryFromUrl(url) { + return new Promise((resolve, reject) => { + const xhr = new XMLHttpRequest(); + xhr.responseType = 'arraybuffer'; + xhr.addEventListener('load', async () => { + const arrayBuffer = xhr.response; + if (!await this._isAudioBinaryValid(arrayBuffer)) { + reject(new Error('Could not retrieve audio')); + } else { + resolve(arrayBuffer); + } + }); + xhr.addEventListener('error', () => reject(new Error('Failed to connect'))); + xhr.open('GET', url); + xhr.send(); }); - const arrayBuffer = await response.arrayBuffer(); - - if (!await this._isAudioBinaryValid(arrayBuffer)) { - throw new Error('Could not retrieve audio'); - } - - return arrayBuffer; } _isAudioValid(audio) { diff --git a/test/test-database.js b/test/test-database.js index 3a090c1d..6d0aae07 100644 --- a/test/test-database.js +++ b/test/test-database.js @@ -30,7 +30,7 @@ const chrome = { removeListener() { /* NOP */ } }, getURL(path2) { - return url.pathToFileURL(path.join(__dirname, '..', 'ext', path2.replace(/^\//, ''))); + return url.pathToFileURL(path.join(__dirname, '..', 'ext', path2.replace(/^\//, ''))).href; }, sendMessage() { // NOP @@ -89,6 +89,9 @@ async function fetch(url2) { await Promise.resolve(); const content = fs.readFileSync(filePath, {encoding: null}); return { + ok: true, + status: 200, + statusText: 'OK', text: async () => Promise.resolve(content.toString('utf8')), json: async () => Promise.resolve(JSON.parse(content.toString('utf8'))) }; @@ -113,7 +116,6 @@ vm.execute([ 'bg/js/json-schema.js', 'bg/js/dictionary.js', 'bg/js/media-utility.js', - 'bg/js/request.js', 'bg/js/dictionary-importer.js', 'bg/js/database.js', 'bg/js/dictionary-database.js' -- cgit v1.2.3