aboutsummaryrefslogtreecommitdiff
path: root/ext/mixed
diff options
context:
space:
mode:
authortoasted-nutbread <toasted-nutbread@users.noreply.github.com>2020-09-26 13:41:26 -0400
committerGitHub <noreply@github.com>2020-09-26 13:41:26 -0400
commit0b51488f1f639885b518fabf683b70db577afa67 (patch)
tree5fca858efdecb345e8bcabec5ff4f727c7f1dc20 /ext/mixed
parent22932e02cbd1fb16d0180c79a3c9b4d1790fbfda (diff)
Audio system refactor (#858)
* Refactor AudioUriBuilder * Add downloadAudio function * Refactor AudioSystem * Update API usage * Rename file * Update scripts * Add prepare calls
Diffstat (limited to 'ext/mixed')
-rw-r--r--ext/mixed/js/api.js12
-rw-r--r--ext/mixed/js/audio-system.js183
-rw-r--r--ext/mixed/js/display.js14
3 files changed, 62 insertions, 147 deletions
diff --git a/ext/mixed/js/api.js b/ext/mixed/js/api.js
index 1973ca22..f7711cbd 100644
--- a/ext/mixed/js/api.js
+++ b/ext/mixed/js/api.js
@@ -85,8 +85,12 @@ const api = (() => {
return this._invoke('noteView', {noteId});
}
- audioGetUri(source, expression, reading, details) {
- return this._invoke('audioGetUri', {source, expression, reading, details});
+ getDefinitionAudioInfo(source, expression, reading, details) {
+ return this._invoke('getDefinitionAudioInfo', {source, expression, reading, details});
+ }
+
+ downloadDefinitionAudio(sources, expression, reading, details) {
+ return this._invoke('downloadDefinitionAudio', {sources, expression, reading, details});
}
commandExec(command, params) {
@@ -189,10 +193,6 @@ const api = (() => {
return this._invoke('isTabSearchPopup', {tabId});
}
- getDefinitionAudio(sources, expression, reading, details) {
- return this._invoke('getDefinitionAudio', {sources, expression, reading, details});
- }
-
triggerDatabaseUpdated(type, cause) {
return this._invoke('triggerDatabaseUpdated', {type, cause});
}
diff --git a/ext/mixed/js/audio-system.js b/ext/mixed/js/audio-system.js
index a8226820..89302ada 100644
--- a/ext/mixed/js/audio-system.js
+++ b/ext/mixed/js/audio-system.js
@@ -16,101 +16,72 @@
*/
/* global
+ * CacheMap
* TextToSpeechAudio
*/
class AudioSystem {
- constructor({audioUriBuilder, requestBuilder=null, useCache}) {
- this._cache = useCache ? new Map() : null;
- this._cacheSizeMaximum = 32;
- this._audioUriBuilder = audioUriBuilder;
- this._requestBuilder = requestBuilder;
+ constructor({getAudioInfo, cacheSize=32}) {
+ this._cache = new CacheMap(cacheSize);
+ this._getAudioInfo = getAudioInfo;
+ }
- if (typeof speechSynthesis !== 'undefined') {
- // speechSynthesis.getVoices() will not be populated unless some API call is made.
- speechSynthesis.addEventListener('voiceschanged', this._onVoicesChanged.bind(this));
- }
+ prepare() {
+ // speechSynthesis.getVoices() will not be populated unless some API call is made.
+ if (typeof speechSynthesis === 'undefined') { return; }
+
+ const eventListeners = new EventListenerCollection();
+ const onVoicesChanged = () => { eventListeners.removeAllEventListeners(); };
+ eventListeners.addEventListener(speechSynthesis, 'voiceschanged', onVoicesChanged, false);
}
- async getDefinitionAudio(sources, expression, reading, details) {
- const key = `${expression}:${reading}`;
- const hasCache = (this._cache !== null && !details.disableCache);
+ async createDefinitionAudio(sources, expression, reading, details) {
+ const key = [expression, reading];
- if (hasCache) {
- const cacheValue = this._cache.get(key);
- if (typeof cacheValue !== 'undefined') {
- const {audio, uri, source} = cacheValue;
- const index = sources.indexOf(source);
- if (index >= 0) {
- return {audio, uri, index};
- }
+ const cacheValue = this._cache.get(key);
+ if (typeof cacheValue !== 'undefined') {
+ const {audio, source} = cacheValue;
+ const index = sources.indexOf(source);
+ if (index >= 0) {
+ return {audio, index};
}
}
for (let i = 0, ii = sources.length; i < ii; ++i) {
const source = sources[i];
- const uri = await this._getAudioUri(source, expression, reading, details);
- if (uri === null) { continue; }
+ const info = await this._getAudioInfo(source, expression, reading, details);
+ if (info === null) { continue; }
+ let audio;
try {
- const audio = (
- details.binary ?
- await this._createAudioBinary(uri) :
- await this._createAudio(uri)
- );
- if (hasCache) {
- this._cacheCheck();
- this._cache.set(key, {audio, uri, source});
+ switch (info.type) {
+ case 'url':
+ {
+ const {details: {url}} = info;
+ audio = await this.createAudio(url);
+ }
+ break;
+ case 'tts':
+ {
+ const {details: {text, voice}} = info;
+ audio = this.createTextToSpeechAudio(text, voice);
+ }
+ break;
+ default:
+ throw new Error(`Unsupported type: ${info.type}`);
}
- return {audio, uri, index: i};
} catch (e) {
- // NOP
+ continue;
}
- }
-
- throw new Error('Could not create audio');
- }
-
- createTextToSpeechAudio(text, voiceUri) {
- const voice = this._getTextToSpeechVoiceFromVoiceUri(voiceUri);
- if (voice === null) {
- throw new Error('Invalid text-to-speech voice');
- }
- return new TextToSpeechAudio(text, voice);
- }
-
- _onVoicesChanged() {
- // NOP
- }
-
- _getAudioUri(source, expression, reading, details) {
- return (
- this._audioUriBuilder !== null ?
- this._audioUriBuilder.getUri(source, expression, reading, details) :
- null
- );
- }
-
- async _createAudio(uri) {
- const ttsParameters = this._getTextToSpeechParameters(uri);
- if (ttsParameters !== null) {
- const {text, voiceUri} = ttsParameters;
- return this.createTextToSpeechAudio(text, voiceUri);
- }
-
- return await this._createAudioFromUrl(uri);
- }
- async _createAudioBinary(uri) {
- const ttsParameters = this._getTextToSpeechParameters(uri);
- if (ttsParameters !== null) {
- throw new Error('Cannot create audio from text-to-speech');
+ this._cache.set(key, {audio, source});
+ return {audio, index: i};
}
- return await this._createAudioBinaryFromUrl(uri);
+ throw new Error('Could not create audio');
}
- _createAudioFromUrl(url) {
+ createAudio(url) {
return new Promise((resolve, reject) => {
const audio = new Audio(url);
audio.addEventListener('loadeddata', () => {
@@ -124,27 +95,15 @@ class AudioSystem {
});
}
- async _createAudioBinaryFromUrl(url) {
- const response = await this._requestBuilder.fetchAnonymous(url, {
- method: 'GET',
- mode: 'cors',
- cache: 'default',
- credentials: 'omit',
- redirect: 'follow',
- referrerPolicy: 'no-referrer'
- });
- const arrayBuffer = await response.arrayBuffer();
-
- if (!await this._isAudioBinaryValid(arrayBuffer)) {
- throw new Error('Could not retrieve audio');
+ createTextToSpeechAudio(text, voiceUri) {
+ const voice = this._getTextToSpeechVoiceFromVoiceUri(voiceUri);
+ if (voice === null) {
+ throw new Error('Invalid text-to-speech voice');
}
-
- return this._arrayBufferToBase64(arrayBuffer);
+ return new TextToSpeechAudio(text, voice);
}
- _arrayBufferToBase64(arrayBuffer) {
- return btoa(String.fromCharCode(...new Uint8Array(arrayBuffer)));
- }
+ // Private
_isAudioValid(audio) {
const duration = audio.duration;
@@ -154,16 +113,6 @@ class AudioSystem {
);
}
- async _isAudioBinaryValid(arrayBuffer) {
- const digest = await AudioSystem.arrayBufferDigest(arrayBuffer);
- switch (digest) {
- case 'ae6398b5a27bc8c0a771df6c907ade794be15518174773c58c7c7ddd17098906': // jpod101 invalid audio
- return false;
- default:
- return true;
- }
- }
-
_getTextToSpeechVoiceFromVoiceUri(voiceUri) {
try {
for (const voice of speechSynthesis.getVoices()) {
@@ -176,38 +125,4 @@ class AudioSystem {
}
return null;
}
-
- _getTextToSpeechParameters(uri) {
- const m = /^tts:[^#?]*\?([^#]*)/.exec(uri);
- if (m === null) { return null; }
-
- const searchParameters = new URLSearchParams(m[1]);
- const text = searchParameters.get('text');
- const voiceUri = searchParameters.get('voice');
- return (text !== null && voiceUri !== null ? {text, voiceUri} : null);
- }
-
- _cacheCheck() {
- const removeCount = this._cache.size - this._cacheSizeMaximum;
- if (removeCount <= 0) { return; }
-
- const removeKeys = [];
- for (const key of this._cache.keys()) {
- removeKeys.push(key);
- if (removeKeys.length >= removeCount) { break; }
- }
-
- for (const key of removeKeys) {
- this._cache.delete(key);
- }
- }
-
- static async arrayBufferDigest(arrayBuffer) {
- const hash = new Uint8Array(await crypto.subtle.digest('SHA-256', new Uint8Array(arrayBuffer)));
- let digest = '';
- for (const byte of hash) {
- digest += byte.toString(16).padStart(2, '0');
- }
- return digest;
- }
}
diff --git a/ext/mixed/js/display.js b/ext/mixed/js/display.js
index a62e0212..689fa7e4 100644
--- a/ext/mixed/js/display.js
+++ b/ext/mixed/js/display.js
@@ -43,12 +43,7 @@ class Display extends EventDispatcher {
this._audioPlaying = null;
this._audioFallback = null;
this._audioSystem = new AudioSystem({
- audioUriBuilder: {
- getUri: async (source, expression, reading, details) => {
- return await api.audioGetUri(source, expression, reading, details);
- }
- },
- useCache: true
+ getAudioInfo: this._getAudioInfo.bind(this)
});
this._styleNode = null;
this._eventListeners = new EventListenerCollection();
@@ -165,6 +160,7 @@ class Display extends EventDispatcher {
}
async prepare() {
+ this._audioSystem.prepare();
this._updateMode();
this._setInteractive(true);
await this._displayGenerator.prepare();
@@ -1096,7 +1092,7 @@ class Display extends EventDispatcher {
try {
const {sources, textToSpeechVoice, customSourceUrl} = this._options.audio;
let index;
- ({audio, index} = await this._audioSystem.getDefinitionAudio(sources, expression, reading, {textToSpeechVoice, customSourceUrl}));
+ ({audio, index} = await this._audioSystem.createDefinitionAudio(sources, expression, reading, {textToSpeechVoice, customSourceUrl}));
info = `From source ${1 + index}: ${sources[index]}`;
} catch (e) {
if (this._audioFallback === null) {
@@ -1419,4 +1415,8 @@ class Display extends EventDispatcher {
modeOptions
});
}
+
+ async _getAudioInfo(source, expression, reading, details) {
+ return await api.getDefinitionAudioInfo(source, expression, reading, details);
+ }
}