diff options
29 files changed, 620 insertions, 321 deletions
@@ -82,7 +82,7 @@ primary language is not English, you may consider also importing the English ver * **[Innocent Corpus](https://forum.koohii.com/post-168613.html#pid168613)** (Term and Kanji frequencies across 5000+ novels) * [innocent\_corpus.zip](https://foosoft.net/projects/yomichan/dl/dict/innocent_corpus.zip) * **[Kanjium](https://github.com/mifunetoshiro/kanjium)** (Pitch dictionary, see [related project page](https://github.com/toasted-nutbread/yomichan-pitch-accent-dictionary) for details) - * [kanjium_pitch_accents.zip](https://foosoft.net/projects/yomichan/dl/dict/kanjium_pitch_accents.zip) + * [kanjium_pitch_accents.zip](https://foosoft.net/projects/yomichan/dl/dict/kanjium_pitch_accents.zip) (currently available for **testing version only**) ## Basic Usage ## diff --git a/ext/bg/data/options-schema.json b/ext/bg/data/options-schema.json index da1f1ce0..4f9e694d 100644 --- a/ext/bg/data/options-schema.json +++ b/ext/bg/data/options-schema.json @@ -388,7 +388,8 @@ "convertNumericCharacters", "convertAlphabeticCharacters", "convertHiraganaToKatakana", - "convertKatakanaToHiragana" + "convertKatakanaToHiragana", + "collapseEmphaticSequences" ], "properties": { "convertHalfWidthCharacters": { @@ -415,6 +416,11 @@ "type": "string", "enum": ["false", "true", "variant"], "default": "variant" + }, + "collapseEmphaticSequences": { + "type": "string", + "enum": ["false", "true", "full"], + "default": "false" } } }, diff --git a/ext/bg/js/backend.js b/ext/bg/js/backend.js index be8ea322..04a3b932 100644 --- a/ext/bg/js/backend.js +++ b/ext/bg/js/backend.js @@ -30,7 +30,6 @@ * Translator * conditionsTestValue * dictConfigured - * dictEnabledSet * dictTermsSort * handlebarsRenderDynamic * jp @@ -85,7 +84,6 @@ class Backend { ['kanjiFind', {handler: this._onApiKanjiFind.bind(this), async: true}], ['termsFind', {handler: this._onApiTermsFind.bind(this), async: true}], ['textParse', {handler: this._onApiTextParse.bind(this), async: true}], - ['textParseMecab', {handler: this._onApiTextParseMecab.bind(this), async: true}], ['definitionAdd', {handler: this._onApiDefinitionAdd.bind(this), async: true}], ['definitionsAddable', {handler: this._onApiDefinitionsAddable.bind(this), async: true}], ['noteView', {handler: this._onApiNoteView.bind(this), async: true}], @@ -102,7 +100,13 @@ class Backend { ['getQueryParserTemplatesHtml', {handler: this._onApiGetQueryParserTemplatesHtml.bind(this), async: true}], ['getZoom', {handler: this._onApiGetZoom.bind(this), async: true}], ['getMessageToken', {handler: this._onApiGetMessageToken.bind(this), async: false}], - ['getDefaultAnkiFieldTemplates', {handler: this._onApiGetDefaultAnkiFieldTemplates.bind(this), async: false}] + ['getDefaultAnkiFieldTemplates', {handler: this._onApiGetDefaultAnkiFieldTemplates.bind(this), async: false}], + ['getAnkiDeckNames', {handler: this._onApiGetAnkiDeckNames.bind(this), async: true}], + ['getAnkiModelNames', {handler: this._onApiGetAnkiModelNames.bind(this), async: true}], + ['getAnkiModelFieldNames', {handler: this._onApiGetAnkiModelFieldNames.bind(this), async: true}], + ['getDictionaryInfo', {handler: this._onApiGetDictionaryInfo.bind(this), async: true}], + ['getDictionaryCounts', {handler: this._onApiGetDictionaryCounts.bind(this), async: true}], + ['purgeDatabase', {handler: this._onApiPurgeDatabase.bind(this), async: true}] ]); this._commandHandlers = new Map([ @@ -315,6 +319,60 @@ class Backend { return await this.dictionaryImporter.import(this.database, archiveSource, onProgress, details); } + async _textParseScanning(text, options) { + const results = []; + while (text.length > 0) { + const term = []; + const [definitions, sourceLength] = await this.translator.findTerms( + 'simple', + text.substring(0, options.scanning.length), + {}, + options + ); + if (definitions.length > 0 && sourceLength > 0) { + dictTermsSort(definitions); + const {expression, reading} = definitions[0]; + const source = text.substring(0, sourceLength); + for (const {text: text2, furigana} of jp.distributeFuriganaInflected(expression, reading, source)) { + const reading2 = jp.convertReading(text2, furigana, options.parsing.readingMode); + term.push({text: text2, reading: reading2}); + } + text = text.substring(source.length); + } else { + const reading = jp.convertReading(text[0], '', options.parsing.readingMode); + term.push({text: text[0], reading}); + text = text.substring(1); + } + results.push(term); + } + return results; + } + + async _textParseMecab(text, options) { + const results = []; + const rawResults = await this.mecab.parseText(text); + for (const [mecabName, parsedLines] of Object.entries(rawResults)) { + const result = []; + for (const parsedLine of parsedLines) { + for (const {expression, reading, source} of parsedLine) { + const term = []; + for (const {text: text2, furigana} of jp.distributeFuriganaInflected( + expression.length > 0 ? expression : source, + jp.convertKatakanaToHiragana(reading), + source + )) { + const reading2 = jp.convertReading(text2, furigana, options.parsing.readingMode); + term.push({text: text2, reading: reading2}); + } + result.push(term); + } + result.push([{text: '\n', reading: ''}]); + } + results.push([mecabName, result]); + } + return results; + } + // Message handlers _onApiYomichanCoreReady(_params, sender) { @@ -406,61 +464,27 @@ class Backend { async _onApiTextParse({text, optionsContext}) { const options = this.getOptions(optionsContext); const results = []; - while (text.length > 0) { - const term = []; - const [definitions, sourceLength] = await this.translator.findTerms( - 'simple', - text.substring(0, options.scanning.length), - {}, - options - ); - if (definitions.length > 0) { - dictTermsSort(definitions); - const {expression, reading} = definitions[0]; - const source = text.substring(0, sourceLength); - for (const {text: text2, furigana} of jp.distributeFuriganaInflected(expression, reading, source)) { - const reading2 = jp.convertReading(text2, furigana, options.parsing.readingMode); - term.push({text: text2, reading: reading2}); - } - text = text.substring(source.length); - } else { - const reading = jp.convertReading(text[0], null, options.parsing.readingMode); - term.push({text: text[0], reading}); - text = text.substring(1); - } - results.push(term); + + if (options.parsing.enableScanningParser) { + results.push({ + source: 'scanning-parser', + id: 'scan', + content: await this._textParseScanning(text, options) + }); } - return results; - } - async _onApiTextParseMecab({text, optionsContext}) { - const options = this.getOptions(optionsContext); - const results = []; - const rawResults = await this.mecab.parseText(text); - for (const [mecabName, parsedLines] of Object.entries(rawResults)) { - const result = []; - for (const parsedLine of parsedLines) { - for (const {expression, reading, source} of parsedLine) { - const term = []; - if (expression !== null && reading !== null) { - for (const {text: text2, furigana} of jp.distributeFuriganaInflected( - expression, - jp.convertKatakanaToHiragana(reading), - source - )) { - const reading2 = jp.convertReading(text2, furigana, options.parsing.readingMode); - term.push({text: text2, reading: reading2}); - } - } else { - const reading2 = jp.convertReading(source, null, options.parsing.readingMode); - term.push({text: source, reading: reading2}); - } - result.push(term); - } - result.push([{text: '\n'}]); + if (options.parsing.enableMecabParser) { + const mecabResults = await this._textParseMecab(text, options); + for (const [mecabDictName, mecabDictResults] of mecabResults) { + results.push({ + source: 'mecab', + dictionary: mecabDictName, + id: `mecab-${mecabDictName}`, + content: mecabDictResults + }); } - results.push([mecabName, result]); } + return results; } @@ -704,6 +728,36 @@ class Backend { return this.defaultAnkiFieldTemplates; } + async _onApiGetAnkiDeckNames(params, sender) { + this._validatePrivilegedMessageSender(sender); + return await this.anki.getDeckNames(); + } + + async _onApiGetAnkiModelNames(params, sender) { + this._validatePrivilegedMessageSender(sender); + return await this.anki.getModelNames(); + } + + async _onApiGetAnkiModelFieldNames({modelName}, sender) { + this._validatePrivilegedMessageSender(sender); + return await this.anki.getModelFieldNames(modelName); + } + + async _onApiGetDictionaryInfo(params, sender) { + this._validatePrivilegedMessageSender(sender); + return await this.translator.database.getDictionaryInfo(); + } + + async _onApiGetDictionaryCounts({dictionaryNames, getTotal}, sender) { + this._validatePrivilegedMessageSender(sender); + return await this.translator.database.getDictionaryCounts(dictionaryNames, getTotal); + } + + async _onApiPurgeDatabase(params, sender) { + this._validatePrivilegedMessageSender(sender); + return await this.translator.purgeDatabase(); + } + // Command handlers async _onCommandSearch(params) { @@ -800,6 +854,13 @@ class Backend { // Utilities + _validatePrivilegedMessageSender(sender) { + const url = sender.url; + if (!(typeof url === 'string' && yomichan.isExtensionUrl(url))) { + throw new Error('Invalid message sender'); + } + } + async _getAudioUri(definition, source, details) { let optionsContext = (typeof details === 'object' && details !== null ? details.optionsContext : null); if (!(typeof optionsContext === 'object' && optionsContext !== null)) { diff --git a/ext/bg/js/database.js b/ext/bg/js/database.js index ad4e3bad..260c815a 100644 --- a/ext/bg/js/database.js +++ b/ext/bg/js/database.js @@ -16,10 +16,7 @@ */ /* global - * JSZip - * JsonSchema * dictFieldSplit - * requestJson */ class Database { diff --git a/ext/bg/js/japanese.js b/ext/bg/js/japanese.js index 5c49cca7..ac81acb5 100644 --- a/ext/bg/js/japanese.js +++ b/ext/bg/js/japanese.js @@ -82,6 +82,9 @@ const ITERATION_MARK_CODE_POINT = 0x3005; + const HIRAGANA_SMALL_TSU_CODE_POINT = 0x3063; + const KATAKANA_SMALL_TSU_CODE_POINT = 0x30c3; + const KANA_PROLONGED_SOUND_MARK_CODE_POINT = 0x30fc; // Existing functions @@ -121,25 +124,25 @@ return wanakana.toRomaji(text); } - function convertReading(expressionFragment, readingFragment, readingMode) { + function convertReading(expression, reading, readingMode) { switch (readingMode) { case 'hiragana': - return convertKatakanaToHiragana(readingFragment || ''); + return convertKatakanaToHiragana(reading); case 'katakana': - return convertHiraganaToKatakana(readingFragment || ''); + return convertHiraganaToKatakana(reading); case 'romaji': - if (readingFragment) { - return convertToRomaji(readingFragment); + if (reading) { + return convertToRomaji(reading); } else { - if (isStringEntirelyKana(expressionFragment)) { - return convertToRomaji(expressionFragment); + if (isStringEntirelyKana(expression)) { + return convertToRomaji(expression); } } - return readingFragment; + return reading; case 'none': - return null; + return ''; default: - return readingFragment; + return reading; } } @@ -297,7 +300,7 @@ const readingLeft = reading2.substring(group.text.length); const segs = segmentize(readingLeft, groups.splice(1)); if (segs) { - return [{text: group.text}].concat(segs); + return [{text: group.text, furigana: ''}].concat(segs); } } } else { @@ -365,13 +368,47 @@ } if (stemLength !== source.length) { - output.push({text: source.substring(stemLength)}); + output.push({text: source.substring(stemLength), furigana: ''}); } return output; } + // Miscellaneous + + function collapseEmphaticSequences(text, fullCollapse, sourceMap=null) { + let result = ''; + let collapseCodePoint = -1; + const hasSourceMap = (sourceMap !== null); + for (const char of text) { + const c = char.codePointAt(0); + if ( + c === HIRAGANA_SMALL_TSU_CODE_POINT || + c === KATAKANA_SMALL_TSU_CODE_POINT || + c === KANA_PROLONGED_SOUND_MARK_CODE_POINT + ) { + if (collapseCodePoint !== c) { + collapseCodePoint = c; + if (!fullCollapse) { + result += char; + continue; + } + } + } else { + collapseCodePoint = -1; + result += char; + continue; + } + + if (hasSourceMap) { + sourceMap.combine(Math.max(0, result.length - 1), 1); + } + } + return result; + } + + // Exports Object.assign(jp, { @@ -383,6 +420,7 @@ convertHalfWidthKanaToFullWidth, convertAlphabeticToKana, distributeFurigana, - distributeFuriganaInflected + distributeFuriganaInflected, + collapseEmphaticSequences }); })(); diff --git a/ext/bg/js/mecab.js b/ext/bg/js/mecab.js index cd6e6c57..597dceae 100644 --- a/ext/bg/js/mecab.js +++ b/ext/bg/js/mecab.js @@ -40,7 +40,36 @@ class Mecab { } async parseText(text) { - return await this.invoke('parse_text', {text}); + const rawResults = await this.invoke('parse_text', {text}); + // { + // 'mecab-name': [ + // // line1 + // [ + // {str expression: 'expression', str reading: 'reading', str source: 'source'}, + // {str expression: 'expression2', str reading: 'reading2', str source: 'source2'} + // ], + // line2, + // ... + // ], + // 'mecab-name2': [...] + // } + const results = {}; + for (const [mecabName, parsedLines] of Object.entries(rawResults)) { + const result = []; + for (const parsedLine of parsedLines) { + const line = []; + for (const {expression, reading, source} of parsedLine) { + line.push({ + expression: expression || '', + reading: reading || '', + source: source || '' + }); + } + result.push(line); + } + results[mecabName] = result; + } + return results; } startListener() { diff --git a/ext/bg/js/options.js b/ext/bg/js/options.js index 20df2a68..da26b628 100644 --- a/ext/bg/js/options.js +++ b/ext/bg/js/options.js @@ -15,14 +15,23 @@ * along with this program. If not, see <https://www.gnu.org/licenses/>. */ -/* global - * utilStringHashCode - */ - /* * Generic options functions */ +function optionsGetStringHashCode(string) { + let hashCode = 0; + + if (typeof string !== 'string') { return hashCode; } + + for (let i = 0, charCode = string.charCodeAt(i); i < string.length; charCode = string.charCodeAt(++i)) { + hashCode = ((hashCode << 5) - hashCode) + charCode; + hashCode |= 0; + } + + return hashCode; +} + function optionsGenericApplyUpdates(options, updates) { const targetVersion = updates.length; const currentVersion = options.version; @@ -63,12 +72,12 @@ const profileOptionsVersionUpdates = [ options.anki.fieldTemplates = null; }, (options) => { - if (utilStringHashCode(options.anki.fieldTemplates) === 1285806040) { + if (optionsGetStringHashCode(options.anki.fieldTemplates) === 1285806040) { options.anki.fieldTemplates = null; } }, (options) => { - if (utilStringHashCode(options.anki.fieldTemplates) === -250091611) { + if (optionsGetStringHashCode(options.anki.fieldTemplates) === -250091611) { options.anki.fieldTemplates = null; } }, @@ -87,7 +96,7 @@ const profileOptionsVersionUpdates = [ (options) => { // Version 12 changes: // The preferred default value of options.anki.fieldTemplates has been changed to null. - if (utilStringHashCode(options.anki.fieldTemplates) === 1444379824) { + if (optionsGetStringHashCode(options.anki.fieldTemplates) === 1444379824) { options.anki.fieldTemplates = null; } }, @@ -170,7 +179,8 @@ function profileOptionsCreateDefaults() { convertNumericCharacters: 'false', convertAlphabeticCharacters: 'false', convertHiraganaToKatakana: 'false', - convertKatakanaToHiragana: 'variant' + convertKatakanaToHiragana: 'variant', + collapseEmphaticSequences: 'false' }, dictionaries: {}, diff --git a/ext/bg/js/search-frontend.js b/ext/bg/js/search-frontend.js index 9cc1436f..e534e771 100644 --- a/ext/bg/js/search-frontend.js +++ b/ext/bg/js/search-frontend.js @@ -19,18 +19,7 @@ * apiOptionsGet */ -async function searchFrontendSetup() { - await yomichan.prepare(); - - const optionsContext = { - depth: 0, - url: window.location.href - }; - const options = await apiOptionsGet(optionsContext); - if (!options.scanning.enableOnSearchPage) { return; } - - window.frontendInitializationData = {depth: 1, proxy: false}; - +function injectSearchFrontend() { const scriptSrcs = [ '/mixed/js/text-scanner.js', '/fg/js/frontend-api-receiver.js', @@ -62,4 +51,29 @@ async function searchFrontendSetup() { } } -searchFrontendSetup(); +async function main() { + await yomichan.prepare(); + + let optionsApplied = false; + + const applyOptions = async () => { + const optionsContext = { + depth: 0, + url: window.location.href + }; + const options = await apiOptionsGet(optionsContext); + if (!options.scanning.enableOnSearchPage || optionsApplied) { return; } + optionsApplied = true; + + window.frontendInitializationData = {depth: 1, proxy: false, isSearchPage: true}; + injectSearchFrontend(); + + yomichan.off('optionsUpdated', applyOptions); + }; + + yomichan.on('optionsUpdated', applyOptions); + + await applyOptions(); +} + +main(); diff --git a/ext/bg/js/search-query-parser-generator.js b/ext/bg/js/search-query-parser-generator.js index 390841c1..9e7ff8aa 100644 --- a/ext/bg/js/search-query-parser-generator.js +++ b/ext/bg/js/search-query-parser-generator.js @@ -36,7 +36,7 @@ class QueryParserGenerator { const termContainer = this._templateHandler.instantiate(preview ? 'term-preview' : 'term'); for (const segment of term) { if (!segment.text.trim()) { continue; } - if (!segment.reading || !segment.reading.trim()) { + if (!segment.reading.trim()) { termContainer.appendChild(this.createSegmentText(segment.text)); } else { termContainer.appendChild(this.createSegment(segment)); @@ -71,7 +71,17 @@ class QueryParserGenerator { for (const parseResult of parseResults) { const optionContainer = this._templateHandler.instantiate('select-option'); optionContainer.value = parseResult.id; - optionContainer.textContent = parseResult.name; + switch (parseResult.source) { + case 'scanning-parser': + optionContainer.textContent = 'Scanning parser'; + break; + case 'mecab': + optionContainer.textContent = `MeCab: ${parseResult.dictionary}`; + break; + default: + optionContainer.textContent = 'Unrecognized dictionary'; + break; + } optionContainer.defaultSelected = selectedParser === parseResult.id; selectContainer.appendChild(optionContainer); } diff --git a/ext/bg/js/search-query-parser.js b/ext/bg/js/search-query-parser.js index 01a0ace5..eb3b681c 100644 --- a/ext/bg/js/search-query-parser.js +++ b/ext/bg/js/search-query-parser.js @@ -21,13 +21,12 @@ * apiOptionsSet * apiTermsFind * apiTextParse - * apiTextParseMecab * docSentenceExtract */ class QueryParser extends TextScanner { constructor({getOptionsContext, setContent, setSpinnerVisible}) { - super(document.querySelector('#query-parser-content'), [], []); + super(document.querySelector('#query-parser-content'), () => [], []); this.getOptionsContext = getOptionsContext; this.setContent = setContent; @@ -128,7 +127,7 @@ class QueryParser extends TextScanner { this.setPreview(text); - this.parseResults = await this.parseText(text); + this.parseResults = await apiTextParse(text, this.getOptionsContext()); this.refreshSelectedParser(); this.renderParserSelect(); @@ -137,33 +136,11 @@ class QueryParser extends TextScanner { this.setSpinnerVisible(false); } - async parseText(text) { - const results = []; - if (this.options.parsing.enableScanningParser) { - results.push({ - name: 'Scanning parser', - id: 'scan', - parsedText: await apiTextParse(text, this.getOptionsContext()) - }); - } - if (this.options.parsing.enableMecabParser) { - const mecabResults = await apiTextParseMecab(text, this.getOptionsContext()); - for (const [mecabDictName, mecabDictResults] of mecabResults) { - results.push({ - name: `MeCab: ${mecabDictName}`, - id: `mecab-${mecabDictName}`, - parsedText: mecabDictResults - }); - } - } - return results; - } - setPreview(text) { const previewTerms = []; for (let i = 0, ii = text.length; i < ii; i += 2) { const tempText = text.substring(i, i + 2); - previewTerms.push([{text: tempText}]); + previewTerms.push([{text: tempText, reading: ''}]); } this.queryParser.textContent = ''; this.queryParser.appendChild(this.queryParserGenerator.createParseResult(previewTerms, true)); @@ -183,6 +160,6 @@ class QueryParser extends TextScanner { const parseResult = this.getParseResult(); this.queryParser.textContent = ''; if (!parseResult) { return; } - this.queryParser.appendChild(this.queryParserGenerator.createParseResult(parseResult.parsedText)); + this.queryParser.appendChild(this.queryParserGenerator.createParseResult(parseResult.content)); } } diff --git a/ext/bg/js/search.js b/ext/bg/js/search.js index 2ba3e468..871c576b 100644 --- a/ext/bg/js/search.js +++ b/ext/bg/js/search.js @@ -208,7 +208,7 @@ class DisplaySearch extends Display { onCopy() { // ignore copy from search page - this.clipboardMonitor.setPreviousText(document.getSelection().toString().trim()); + this.clipboardMonitor.setPreviousText(window.getSelection().toString().trim()); } onExternalSearchUpdate({text}) { diff --git a/ext/bg/js/settings/anki.js b/ext/bg/js/settings/anki.js index b32a9517..ff1277ed 100644 --- a/ext/bg/js/settings/anki.js +++ b/ext/bg/js/settings/anki.js @@ -16,13 +16,13 @@ */ /* global + * apiGetAnkiDeckNames + * apiGetAnkiModelFieldNames + * apiGetAnkiModelNames * getOptionsContext * getOptionsMutable * onFormOptionsChanged * settingsSaveOptions - * utilAnkiGetDeckNames - * utilAnkiGetModelFieldNames - * utilAnkiGetModelNames * utilBackgroundIsolate */ @@ -107,7 +107,7 @@ async function _ankiDeckAndModelPopulate(options) { const kanjiModel = {value: options.anki.kanji.model, selector: '#anki-kanji-model'}; try { _ankiSpinnerShow(true); - const [deckNames, modelNames] = await Promise.all([utilAnkiGetDeckNames(), utilAnkiGetModelNames()]); + const [deckNames, modelNames] = await Promise.all([apiGetAnkiDeckNames(), apiGetAnkiModelNames()]); deckNames.sort(); modelNames.sort(); termsDeck.values = deckNames; @@ -180,7 +180,7 @@ async function _onAnkiModelChanged(e) { let fieldNames; try { const modelName = node.value; - fieldNames = await utilAnkiGetModelFieldNames(modelName); + fieldNames = await apiGetAnkiModelFieldNames(modelName); _ankiSetError(null); } catch (error) { _ankiSetError(error); diff --git a/ext/bg/js/settings/dictionaries.js b/ext/bg/js/settings/dictionaries.js index 1a6d452b..7eed4273 100644 --- a/ext/bg/js/settings/dictionaries.js +++ b/ext/bg/js/settings/dictionaries.js @@ -17,8 +17,11 @@ /* global * PageExitPrevention + * apiGetDictionaryCounts + * apiGetDictionaryInfo * apiOptionsGet * apiOptionsGetFull + * apiPurgeDatabase * getOptionsContext * getOptionsFullMutable * getOptionsMutable @@ -27,10 +30,7 @@ * storageUpdateStats * utilBackgroundIsolate * utilDatabaseDeleteDictionary - * utilDatabaseGetDictionaryCounts - * utilDatabaseGetDictionaryInfo * utilDatabaseImport - * utilDatabasePurge */ let dictionaryUI = null; @@ -431,7 +431,7 @@ async function onDictionaryOptionsChanged() { async function onDatabaseUpdated() { try { - const dictionaries = await utilDatabaseGetDictionaryInfo(); + const dictionaries = await apiGetDictionaryInfo(); dictionaryUI.setDictionaries(dictionaries); document.querySelector('#dict-warning').hidden = (dictionaries.length > 0); @@ -439,7 +439,7 @@ async function onDatabaseUpdated() { updateMainDictionarySelectOptions(dictionaries); await updateMainDictionarySelectValue(); - const {counts, total} = await utilDatabaseGetDictionaryCounts(dictionaries.map((v) => v.title), true); + const {counts, total} = await apiGetDictionaryCounts(dictionaries.map((v) => v.title), true); dictionaryUI.setCounts(counts, total); } catch (e) { dictionaryErrorsShow([e]); @@ -618,7 +618,7 @@ async function onDictionaryPurge(e) { dictionaryErrorsShow(null); dictionarySpinnerShow(true); - await utilDatabasePurge(); + await apiPurgeDatabase(); for (const {options} of toIterable((await getOptionsFullMutable()).profiles)) { options.dictionaries = utilBackgroundIsolate({}); options.general.mainDictionary = ''; diff --git a/ext/bg/js/settings/main.js b/ext/bg/js/settings/main.js index 8fd94562..308e92eb 100644 --- a/ext/bg/js/settings/main.js +++ b/ext/bg/js/settings/main.js @@ -118,6 +118,7 @@ async function formRead(options) { options.translation.convertAlphabeticCharacters = $('#translation-convert-alphabetic-characters').val(); options.translation.convertHiraganaToKatakana = $('#translation-convert-hiragana-to-katakana').val(); options.translation.convertKatakanaToHiragana = $('#translation-convert-katakana-to-hiragana').val(); + options.translation.collapseEmphaticSequences = $('#translation-collapse-emphatic-sequences').val(); options.parsing.enableScanningParser = $('#parsing-scan-enable').prop('checked'); options.parsing.enableMecabParser = $('#parsing-mecab-enable').prop('checked'); @@ -199,6 +200,7 @@ async function formWrite(options) { $('#translation-convert-alphabetic-characters').val(options.translation.convertAlphabeticCharacters); $('#translation-convert-hiragana-to-katakana').val(options.translation.convertHiraganaToKatakana); $('#translation-convert-katakana-to-hiragana').val(options.translation.convertKatakanaToHiragana); + $('#translation-collapse-emphatic-sequences').val(options.translation.collapseEmphaticSequences); $('#parsing-scan-enable').prop('checked', options.parsing.enableScanningParser); $('#parsing-mecab-enable').prop('checked', options.parsing.enableMecabParser); diff --git a/ext/bg/js/translator.js b/ext/bg/js/translator.js index b6f8b8e5..8708e4d8 100644 --- a/ext/bg/js/translator.js +++ b/ext/bg/js/translator.js @@ -347,17 +347,27 @@ class Translator { getAllDeinflections(text, options) { const translationOptions = options.translation; + const collapseEmphaticOptions = [[false, false]]; + switch (translationOptions.collapseEmphaticSequences) { + case 'true': + collapseEmphaticOptions.push([true, false]); + break; + case 'full': + collapseEmphaticOptions.push([true, false], [true, true]); + break; + } const textOptionVariantArray = [ Translator.getTextOptionEntryVariants(translationOptions.convertHalfWidthCharacters), Translator.getTextOptionEntryVariants(translationOptions.convertNumericCharacters), Translator.getTextOptionEntryVariants(translationOptions.convertAlphabeticCharacters), Translator.getTextOptionEntryVariants(translationOptions.convertHiraganaToKatakana), - Translator.getTextOptionEntryVariants(translationOptions.convertKatakanaToHiragana) + Translator.getTextOptionEntryVariants(translationOptions.convertKatakanaToHiragana), + collapseEmphaticOptions ]; const deinflections = []; const used = new Set(); - for (const [halfWidth, numeric, alphabetic, katakana, hiragana] of Translator.getArrayVariants(textOptionVariantArray)) { + for (const [halfWidth, numeric, alphabetic, katakana, hiragana, [collapseEmphatic, collapseEmphaticFull]] of Translator.getArrayVariants(textOptionVariantArray)) { let text2 = text; const sourceMap = new TextSourceMap(text2); if (halfWidth) { @@ -375,6 +385,9 @@ class Translator { if (hiragana) { text2 = jp.convertKatakanaToHiragana(text2); } + if (collapseEmphatic) { + text2 = jp.collapseEmphaticSequences(text2, collapseEmphaticFull, sourceMap); + } for (let i = text2.length; i > 0; --i) { const text2Substring = text2.substring(0, i); diff --git a/ext/bg/js/util.js b/ext/bg/js/util.js index 69536f02..5edcc193 100644 --- a/ext/bg/js/util.js +++ b/ext/bg/js/util.js @@ -58,19 +58,6 @@ function utilBackgroundFunctionIsolate(func) { return backgroundPage.utilFunctionIsolate(func); } -function utilStringHashCode(string) { - let hashCode = 0; - - if (typeof string !== 'string') { return hashCode; } - - for (let i = 0, charCode = string.charCodeAt(i); i < string.length; charCode = string.charCodeAt(++i)) { - hashCode = ((hashCode << 5) - hashCode) + charCode; - hashCode |= 0; - } - - return hashCode; -} - function utilBackend() { const backend = chrome.extension.getBackgroundPage().yomichanBackend; if (!backend.isPrepared) { @@ -79,35 +66,6 @@ function utilBackend() { return backend; } -async function utilAnkiGetModelNames() { - return utilIsolate(await utilBackend().anki.getModelNames()); -} - -async function utilAnkiGetDeckNames() { - return utilIsolate(await utilBackend().anki.getDeckNames()); -} - -async function utilDatabaseGetDictionaryInfo() { - return utilIsolate(await utilBackend().translator.database.getDictionaryInfo()); -} - -async function utilDatabaseGetDictionaryCounts(dictionaryNames, getTotal) { - return utilIsolate(await utilBackend().translator.database.getDictionaryCounts( - utilBackgroundIsolate(dictionaryNames), - utilBackgroundIsolate(getTotal) - )); -} - -async function utilAnkiGetModelFieldNames(modelName) { - return utilIsolate(await utilBackend().anki.getModelFieldNames( - utilBackgroundIsolate(modelName) - )); -} - -async function utilDatabasePurge() { - return utilIsolate(await utilBackend().translator.purgeDatabase()); -} - async function utilDatabaseDeleteDictionary(dictionaryName, onProgress) { return utilIsolate(await utilBackend().translator.database.deleteDictionary( utilBackgroundIsolate(dictionaryName), diff --git a/ext/bg/settings.html b/ext/bg/settings.html index 1297a9cc..96c1db82 100644 --- a/ext/bg/settings.html +++ b/ext/bg/settings.html @@ -427,7 +427,7 @@ <p class="help-block"> The conversion options below are listed in the order that the conversions are applied to the input text. - Each conversion has three possible values: + Conversions commonly have three possible values: </p> <ul class="help-block"> @@ -490,6 +490,15 @@ <option value="variant">Use both variants</option> </select> </div> + + <div class="form-group"> + <label for="translation-collapse-emphatic-sequences">Collapse emphatic character sequences <span class="label-light">(すっっごーーい → すっごーい / すごい)</span></label> + <select class="form-control" id="translation-collapse-emphatic-sequences"> + <option value="false">Disabled</option> + <option value="true">Collapse into single character</option> + <option value="full">Remove all characters</option> + </select> + </div> </div> <div id="popup-content-scanning"> diff --git a/ext/fg/js/frontend-initialize.js b/ext/fg/js/frontend-initialize.js index 5af7fdf0..2b942258 100644 --- a/ext/fg/js/frontend-initialize.js +++ b/ext/fg/js/frontend-initialize.js @@ -24,49 +24,101 @@ * apiOptionsGet */ +async function createIframePopupProxy(url, frameOffsetForwarder) { + const rootPopupInformationPromise = yomichan.getTemporaryListenerResult( + chrome.runtime.onMessage, + ({action, params}, {resolve}) => { + if (action === 'rootPopupInformation') { + resolve(params); + } + } + ); + apiBroadcastTab('rootPopupRequestInformationBroadcast'); + const {popupId, frameId} = await rootPopupInformationPromise; + + const getFrameOffset = frameOffsetForwarder.getOffset.bind(frameOffsetForwarder); + + const popup = new PopupProxy(popupId, 0, null, frameId, url, getFrameOffset); + await popup.prepare(); + + return popup; +} + +async function getOrCreatePopup(depth) { + const popupHost = new PopupProxyHost(); + await popupHost.prepare(); + + const popup = popupHost.getOrCreatePopup(null, null, depth); + + return popup; +} + +async function createPopupProxy(depth, id, parentFrameId, url) { + const popup = new PopupProxy(null, depth + 1, id, parentFrameId, url); + await popup.prepare(); + + return popup; +} + async function main() { await yomichan.prepare(); const data = window.frontendInitializationData || {}; - const {id, depth=0, parentFrameId, url, proxy=false} = data; - - const optionsContext = {depth, url}; - const options = await apiOptionsGet(optionsContext); - - let popup; - if (!proxy && (window !== window.parent) && options.general.showIframePopupsInRootFrame) { - const rootPopupInformationPromise = yomichan.getTemporaryListenerResult( - chrome.runtime.onMessage, - ({action, params}, {resolve}) => { - if (action === 'rootPopupInformation') { - resolve(params); - } + const {id, depth=0, parentFrameId, url=window.location.href, proxy=false, isSearchPage=false} = data; + + const isIframe = !proxy && (window !== window.parent); + + const popups = { + iframe: null, + proxy: null, + normal: null + }; + + let frontend = null; + let frontendPreparePromise = null; + let frameOffsetForwarder = null; + + const applyOptions = async () => { + const optionsContext = {depth: isSearchPage ? 0 : depth, url}; + const options = await apiOptionsGet(optionsContext); + + if (!proxy && frameOffsetForwarder === null) { + frameOffsetForwarder = new FrameOffsetForwarder(); + frameOffsetForwarder.start(); + } + + let popup; + if (isIframe && options.general.showIframePopupsInRootFrame) { + popup = popups.iframe || await createIframePopupProxy(url, frameOffsetForwarder); + popups.iframe = popup; + } else if (proxy) { + popup = popups.proxy || await createPopupProxy(depth, id, parentFrameId, url); + popups.proxy = popup; + } else { + popup = popups.normal || await getOrCreatePopup(depth); + popups.normal = popup; + } + + if (frontend === null) { + frontend = new Frontend(popup); + frontendPreparePromise = frontend.prepare(); + await frontendPreparePromise; + } else { + await frontendPreparePromise; + if (isSearchPage) { + const disabled = !options.scanning.enableOnSearchPage; + frontend.setDisabledOverride(disabled); + } + + if (isIframe) { + await frontend.setPopup(popup); } - ); - apiBroadcastTab('rootPopupRequestInformationBroadcast'); - const {popupId, frameId} = await rootPopupInformationPromise; - - const frameOffsetForwarder = new FrameOffsetForwarder(); - frameOffsetForwarder.start(); - const getFrameOffset = frameOffsetForwarder.getOffset.bind(frameOffsetForwarder); - - popup = new PopupProxy(popupId, 0, null, frameId, url, getFrameOffset); - await popup.prepare(); - } else if (proxy) { - popup = new PopupProxy(null, depth + 1, id, parentFrameId, url); - await popup.prepare(); - } else { - const frameOffsetForwarder = new FrameOffsetForwarder(); - frameOffsetForwarder.start(); - - const popupHost = new PopupProxyHost(); - await popupHost.prepare(); - - popup = popupHost.getOrCreatePopup(null, null, depth); - } - - const frontend = new Frontend(popup); - await frontend.prepare(); + } + }; + + yomichan.on('optionsUpdated', applyOptions); + + await applyOptions(); } main(); diff --git a/ext/fg/js/frontend.js b/ext/fg/js/frontend.js index 55d699e5..eecfe2e1 100644 --- a/ext/fg/js/frontend.js +++ b/ext/fg/js/frontend.js @@ -29,11 +29,14 @@ class Frontend extends TextScanner { constructor(popup) { super( window, - popup.isProxy() ? [] : [popup.getContainer()], + () => this.popup.isProxy() ? [] : [this.popup.getContainer()], [(x, y) => this.popup.containsPoint(x, y)] ); this.popup = popup; + + this._disabledOverride = false; + this.options = null; this.optionsContext = { @@ -43,7 +46,7 @@ class Frontend extends TextScanner { this._pageZoomFactor = 1.0; this._contentScale = 1.0; - this._orphaned = true; + this._orphaned = false; this._lastShowPromise = Promise.resolve(); this._windowMessageHandlers = new Map([ @@ -132,8 +135,20 @@ class Frontend extends TextScanner { ]; } + setDisabledOverride(disabled) { + this._disabledOverride = disabled; + this.setEnabled(this.options.general.enable, this._canEnable()); + } + + async setPopup(popup) { + this.onSearchClear(false); + this.popup = popup; + await popup.setOptions(this.options); + } + async updateOptions() { - this.setOptions(await apiOptionsGet(this.getOptionsContext())); + this.options = await apiOptionsGet(this.getOptionsContext()); + this.setOptions(this.options, this._canEnable()); const ignoreNodes = ['.scan-disable', '.scan-disable *']; if (!this.options.scanning.enableOnPopupExpressions) { @@ -259,7 +274,7 @@ class Frontend extends TextScanner { } _broadcastRootPopupInformation() { - if (!this.popup.isProxy() && this.popup.depth === 0) { + if (!this.popup.isProxy() && this.popup.depth === 0 && this.popup.frameId === 0) { apiBroadcastTab('rootPopupInformation', {popupId: this.popup.id, frameId: this.popup.frameId}); } } @@ -272,6 +287,10 @@ class Frontend extends TextScanner { }); } + _canEnable() { + return this.popup.depth <= this.options.scanning.popupNestingMaxDepth && !this._disabledOverride; + } + async _updatePopupPosition() { const textSource = this.getCurrentTextSource(); if (textSource !== null && await this.popup.isVisible()) { diff --git a/ext/fg/js/popup-nested.js b/ext/fg/js/popup-nested.js index 1b24614b..c140f9c8 100644 --- a/ext/fg/js/popup-nested.js +++ b/ext/fg/js/popup-nested.js @@ -19,24 +19,7 @@ * apiOptionsGet */ -let popupNestedInitialized = false; - -async function popupNestedInitialize(id, depth, parentFrameId, url) { - if (popupNestedInitialized) { - return; - } - popupNestedInitialized = true; - - const optionsContext = {depth, url}; - const options = await apiOptionsGet(optionsContext); - const popupNestingMaxDepth = options.scanning.popupNestingMaxDepth; - - if (!(typeof popupNestingMaxDepth === 'number' && typeof depth === 'number' && depth < popupNestingMaxDepth)) { - return; - } - - window.frontendInitializationData = {id, depth, parentFrameId, url, proxy: true}; - +function injectPopupNested() { const scriptSrcs = [ '/mixed/js/text-scanner.js', '/fg/js/frontend-api-sender.js', @@ -52,3 +35,33 @@ async function popupNestedInitialize(id, depth, parentFrameId, url) { document.body.appendChild(script); } } + +async function popupNestedInitialize(id, depth, parentFrameId, url) { + let optionsApplied = false; + + const applyOptions = async () => { + const optionsContext = {depth, url}; + const options = await apiOptionsGet(optionsContext); + const popupNestingMaxDepth = options.scanning.popupNestingMaxDepth; + + const maxPopupDepthExceeded = !( + typeof popupNestingMaxDepth === 'number' && + typeof depth === 'number' && + depth < popupNestingMaxDepth + ); + if (maxPopupDepthExceeded || optionsApplied) { + return; + } + + optionsApplied = true; + + window.frontendInitializationData = {id, depth, parentFrameId, url, proxy: true}; + injectPopupNested(); + + yomichan.off('optionsUpdated', applyOptions); + }; + + yomichan.on('optionsUpdated', applyOptions); + + await applyOptions(); +} diff --git a/ext/fg/js/popup.js b/ext/fg/js/popup.js index 42f08afa..99610e17 100644 --- a/ext/fg/js/popup.js +++ b/ext/fg/js/popup.js @@ -539,19 +539,10 @@ class Popup { }; } - static _isOnExtensionPage() { - try { - const url = chrome.runtime.getURL('/'); - return window.location.href.substring(0, url.length) === url; - } catch (e) { - // NOP - } - } - static async _injectStylesheet(id, type, value, useWebExtensionApi) { const injectedStylesheets = Popup._injectedStylesheets; - if (Popup._isOnExtensionPage()) { + if (yomichan.isExtensionUrl(window.location.href)) { // Permissions error will occur if trying to use the WebExtension API to inject // into an extension page. useWebExtensionApi = false; diff --git a/ext/manifest.json b/ext/manifest.json index 788e1469..452b642c 100644 --- a/ext/manifest.json +++ b/ext/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 2, "name": "Yomichan", - "version": "20.4.10.0", + "version": "20.4.18.0", "description": "Japanese dictionary with Anki integration", "icons": {"16": "mixed/img/icon16.png", "48": "mixed/img/icon48.png", "128": "mixed/img/icon128.png"}, diff --git a/ext/mixed/js/api.js b/ext/mixed/js/api.js index 50b285a5..7080d93a 100644 --- a/ext/mixed/js/api.js +++ b/ext/mixed/js/api.js @@ -44,10 +44,6 @@ function apiTextParse(text, optionsContext) { return _apiInvoke('textParse', {text, optionsContext}); } -function apiTextParseMecab(text, optionsContext) { - return _apiInvoke('textParseMecab', {text, optionsContext}); -} - function apiKanjiFind(text, optionsContext) { return _apiInvoke('kanjiFind', {text, optionsContext}); } @@ -120,6 +116,30 @@ function apiGetDefaultAnkiFieldTemplates() { return _apiInvoke('getDefaultAnkiFieldTemplates'); } +function apiGetAnkiDeckNames() { + return _apiInvoke('getAnkiDeckNames'); +} + +function apiGetAnkiModelNames() { + return _apiInvoke('getAnkiModelNames'); +} + +function apiGetAnkiModelFieldNames(modelName) { + return _apiInvoke('getAnkiModelFieldNames', {modelName}); +} + +function apiGetDictionaryInfo() { + return _apiInvoke('getDictionaryInfo'); +} + +function apiGetDictionaryCounts(dictionaryNames, getTotal) { + return _apiInvoke('getDictionaryCounts', {dictionaryNames, getTotal}); +} + +function apiPurgeDatabase() { + return _apiInvoke('purgeDatabase'); +} + function _apiInvoke(action, params={}) { const data = {action, params}; return new Promise((resolve, reject) => { diff --git a/ext/mixed/js/core.js b/ext/mixed/js/core.js index 2d11c11a..6a3298fc 100644 --- a/ext/mixed/js/core.js +++ b/ext/mixed/js/core.js @@ -316,6 +316,15 @@ const yomichan = (() => { this.trigger('orphaned', {error}); } + isExtensionUrl(url) { + try { + const urlBase = chrome.runtime.getURL('/'); + return url.substring(0, urlBase.length) === urlBase; + } catch (e) { + return false; + } + } + getTemporaryListenerResult(eventHandler, userCallback, timeout=null) { if (!( typeof eventHandler.addListener === 'function' && diff --git a/ext/mixed/js/text-scanner.js b/ext/mixed/js/text-scanner.js index a1d96320..0cd12cd7 100644 --- a/ext/mixed/js/text-scanner.js +++ b/ext/mixed/js/text-scanner.js @@ -46,7 +46,7 @@ class TextScanner { } onMouseOver(e) { - if (this.ignoreElements.includes(e.target)) { + if (this.ignoreElements().includes(e.target)) { this.scanTimerClear(); } } @@ -133,7 +133,7 @@ class TextScanner { this.preventNextClick = false; const primaryTouch = e.changedTouches[0]; - if (DOM.isPointInSelection(primaryTouch.clientX, primaryTouch.clientY, this.node.getSelection())) { + if (DOM.isPointInSelection(primaryTouch.clientX, primaryTouch.clientY, window.getSelection())) { return; } @@ -224,8 +224,8 @@ class TextScanner { } } - setEnabled(enabled) { - if (enabled) { + setEnabled(enabled, canEnable) { + if (enabled && canEnable) { if (!this.enabled) { this.hookEvents(); this.enabled = true; @@ -271,9 +271,9 @@ class TextScanner { ]; } - setOptions(options) { + setOptions(options, canEnable=true) { this.options = options; - this.setEnabled(this.options.general.enable); + this.setEnabled(this.options.general.enable, canEnable); } async searchAt(x, y, cause) { diff --git a/package-lock.json b/package-lock.json index 920263d2..8f421a68 100644 --- a/package-lock.json +++ b/package-lock.json @@ -906,9 +906,9 @@ "dev": true }, "jsdom": { - "version": "16.2.1", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.2.1.tgz", - "integrity": "sha512-3p0gHs5EfT7PxW9v8Phz3mrq//4Dy8MQenU/PoKxhdT+c45S7NjIjKbGT3Ph0nkICweE1r36+yaknXA5WfVNAg==", + "version": "16.2.2", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.2.2.tgz", + "integrity": "sha512-pDFQbcYtKBHxRaP55zGXCJWgFHkDAYbKcsXEK/3Icu9nKYZkutUXfLBwbD+09XDutkYSHcgfQLZ0qvpAAm9mvg==", "dev": true, "requires": { "abab": "^2.0.3", @@ -931,11 +931,11 @@ "tough-cookie": "^3.0.1", "w3c-hr-time": "^1.0.2", "w3c-xmlserializer": "^2.0.0", - "webidl-conversions": "^5.0.0", + "webidl-conversions": "^6.0.0", "whatwg-encoding": "^1.0.5", "whatwg-mimetype": "^2.3.0", "whatwg-url": "^8.0.0", - "ws": "^7.2.1", + "ws": "^7.2.3", "xml-name-validator": "^3.0.0" }, "dependencies": { @@ -946,6 +946,14 @@ "dev": true, "requires": { "webidl-conversions": "^5.0.0" + }, + "dependencies": { + "webidl-conversions": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-5.0.0.tgz", + "integrity": "sha512-VlZwKPCkYKxQgeSbH5EyngOmRp7Ww7I9rQLERETtf5ofd9pGeswWiOtogpEO850jziPRarreGxn5QIiTqpb2wA==", + "dev": true + } } }, "tr46": { @@ -958,9 +966,9 @@ } }, "webidl-conversions": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-5.0.0.tgz", - "integrity": "sha512-VlZwKPCkYKxQgeSbH5EyngOmRp7Ww7I9rQLERETtf5ofd9pGeswWiOtogpEO850jziPRarreGxn5QIiTqpb2wA==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-6.0.0.tgz", + "integrity": "sha512-jTZAeJnc6D+yAOjygbJOs33kVQIk5H6fj9SFDOhIKjsf9HiAzL/c+tAJsc8ASWafvhNkH+wJZms47pmajkhatA==", "dev": true }, "whatwg-url": { @@ -972,6 +980,14 @@ "lodash.sortby": "^4.7.0", "tr46": "^2.0.0", "webidl-conversions": "^5.0.0" + }, + "dependencies": { + "webidl-conversions": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-5.0.0.tgz", + "integrity": "sha512-VlZwKPCkYKxQgeSbH5EyngOmRp7Ww7I9rQLERETtf5ofd9pGeswWiOtogpEO850jziPRarreGxn5QIiTqpb2wA==", + "dev": true + } } } } @@ -1199,9 +1215,9 @@ "dev": true }, "psl": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/psl/-/psl-1.7.0.tgz", - "integrity": "sha512-5NsSEDv8zY70ScRnOTn7bK7eanl2MvFrOrS/R6x+dBt5g1ghnj9Zv90kO8GwT8gxcu2ANyFprnFYB85IogIJOQ==", + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz", + "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==", "dev": true }, "punycode": { @@ -1362,9 +1378,9 @@ "dev": true }, "saxes": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.0.tgz", - "integrity": "sha512-LXTZygxhf8lfwKaTP/8N9CsVdjTlea3teze4lL6u37ivbgGbV0GGMuNtS/I9rnD/HC2/txUM7Df4S2LVl1qhiA==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz", + "integrity": "sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw==", "dev": true, "requires": { "xmlchars": "^2.2.0" diff --git a/package.json b/package.json index b02ec179..0729cda1 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,6 @@ "eslint": "^6.8.0", "eslint-plugin-no-unsanitized": "^3.0.2", "fake-indexeddb": "^3.0.0", - "jsdom": "^16.2.1" + "jsdom": "^16.2.2" } } diff --git a/test/lint/global-declarations.js b/test/lint/global-declarations.js index 07ba5570..2fc9a5e2 100644 --- a/test/lint/global-declarations.js +++ b/test/lint/global-declarations.js @@ -37,6 +37,18 @@ function getNewline(string) { } } +function getSubstringCount(string, substring) { + let start = 0; + let count = 0; + while (true) { + const pos = string.indexOf(substring, start); + if (pos < 0) { break; } + ++count; + start = pos + substring.length; + } + return count; +} + function validateGlobals(fileName, fix) { const pattern = /\/\*\s*global\s+([\w\W]*?)\*\//g; @@ -47,6 +59,7 @@ function validateGlobals(fileName, fix) { let first = true; let endIndex = 0; let newSource = ''; + const allGlobals = []; const newline = getNewline(source); while ((match = pattern.exec(source)) !== null) { if (!first) { @@ -74,15 +87,27 @@ function validateGlobals(fileName, fix) { newSource += source.substring(0, match.index); newSource += expected; endIndex = match.index + match[0].length; + + allGlobals.push(...parts); } newSource += source.substring(endIndex); + // This is an approximate check to see if a global variable is unused. + // If the global appears in a comment, string, or similar, the check will pass. + let errorCount = 0; + for (const global of allGlobals) { + if (getSubstringCount(newSource, global) <= 1) { + console.error(`Global variable ${global} appears to be unused in ${fileName}`); + ++errorCount; + } + } + if (fix) { fs.writeFileSync(fileName, newSource, {encoding: 'utf8'}); } - return true; + return errorCount === 0; } diff --git a/test/test-japanese.js b/test/test-japanese.js index f4b084ac..87efdfad 100644 --- a/test/test-japanese.js +++ b/test/test-japanese.js @@ -176,19 +176,19 @@ function testConvertReading() { [['アリガトウ', 'アリガトウ', 'hiragana'], 'ありがとう'], [['アリガトウ', 'アリガトウ', 'katakana'], 'アリガトウ'], [['アリガトウ', 'アリガトウ', 'romaji'], 'arigatou'], - [['アリガトウ', 'アリガトウ', 'none'], null], + [['アリガトウ', 'アリガトウ', 'none'], ''], [['アリガトウ', 'アリガトウ', 'default'], 'アリガトウ'], [['ありがとう', 'ありがとう', 'hiragana'], 'ありがとう'], [['ありがとう', 'ありがとう', 'katakana'], 'アリガトウ'], [['ありがとう', 'ありがとう', 'romaji'], 'arigatou'], - [['ありがとう', 'ありがとう', 'none'], null], + [['ありがとう', 'ありがとう', 'none'], ''], [['ありがとう', 'ありがとう', 'default'], 'ありがとう'], [['有り難う', 'ありがとう', 'hiragana'], 'ありがとう'], [['有り難う', 'ありがとう', 'katakana'], 'アリガトウ'], [['有り難う', 'ありがとう', 'romaji'], 'arigatou'], - [['有り難う', 'ありがとう', 'none'], null], + [['有り難う', 'ありがとう', 'none'], ''], [['有り難う', 'ありがとう', 'default'], 'ありがとう'], // Cases with falsy readings @@ -196,44 +196,20 @@ function testConvertReading() { [['ありがとう', '', 'hiragana'], ''], [['ありがとう', '', 'katakana'], ''], [['ありがとう', '', 'romaji'], 'arigatou'], - [['ありがとう', '', 'none'], null], + [['ありがとう', '', 'none'], ''], [['ありがとう', '', 'default'], ''], - [['ありがとう', null, 'hiragana'], ''], - [['ありがとう', null, 'katakana'], ''], - [['ありがとう', null, 'romaji'], 'arigatou'], - [['ありがとう', null, 'none'], null], - [['ありがとう', null, 'default'], null], - - [['ありがとう', void 0, 'hiragana'], ''], - [['ありがとう', void 0, 'katakana'], ''], - [['ありがとう', void 0, 'romaji'], 'arigatou'], - [['ありがとう', void 0, 'none'], null], - [['ありがとう', void 0, 'default'], void 0], - // Cases with falsy readings and kanji expressions [['有り難う', '', 'hiragana'], ''], [['有り難う', '', 'katakana'], ''], [['有り難う', '', 'romaji'], ''], - [['有り難う', '', 'none'], null], - [['有り難う', '', 'default'], ''], - - [['有り難う', null, 'hiragana'], ''], - [['有り難う', null, 'katakana'], ''], - [['有り難う', null, 'romaji'], null], - [['有り難う', null, 'none'], null], - [['有り難う', null, 'default'], null], - - [['有り難う', void 0, 'hiragana'], ''], - [['有り難う', void 0, 'katakana'], ''], - [['有り難う', void 0, 'romaji'], void 0], - [['有り難う', void 0, 'none'], null], - [['有り難う', void 0, 'default'], void 0] + [['有り難う', '', 'none'], ''], + [['有り難う', '', 'default'], ''] ]; - for (const [[expressionFragment, readingFragment, readingMode], expected] of data) { - assert.strictEqual(jp.convertReading(expressionFragment, readingFragment, readingMode), expected); + for (const [[expression, reading, readingMode], expected] of data) { + assert.strictEqual(jp.convertReading(expression, reading, readingMode), expected); } } @@ -303,9 +279,9 @@ function testDistributeFurigana() { ['有り難う', 'ありがとう'], [ {text: '有', furigana: 'あ'}, - {text: 'り'}, + {text: 'り', furigana: ''}, {text: '難', furigana: 'がと'}, - {text: 'う'} + {text: 'う', furigana: ''} ] ], [ @@ -317,23 +293,23 @@ function testDistributeFurigana() { [ ['お祝い', 'おいわい'], [ - {text: 'お'}, + {text: 'お', furigana: ''}, {text: '祝', furigana: 'いわ'}, - {text: 'い'} + {text: 'い', furigana: ''} ] ], [ ['美味しい', 'おいしい'], [ {text: '美味', furigana: 'おい'}, - {text: 'しい'} + {text: 'しい', furigana: ''} ] ], [ ['食べ物', 'たべもの'], [ {text: '食', furigana: 'た'}, - {text: 'べ'}, + {text: 'べ', furigana: ''}, {text: '物', furigana: 'もの'} ] ], @@ -341,9 +317,9 @@ function testDistributeFurigana() { ['試し切り', 'ためしぎり'], [ {text: '試', furigana: 'ため'}, - {text: 'し'}, + {text: 'し', furigana: ''}, {text: '切', furigana: 'ぎ'}, - {text: 'り'} + {text: 'り', furigana: ''} ] ], // Ambiguous @@ -373,16 +349,16 @@ function testDistributeFuriganaInflected() { ['美味しい', 'おいしい', '美味しかた'], [ {text: '美味', furigana: 'おい'}, - {text: 'し'}, - {text: 'かた'} + {text: 'し', furigana: ''}, + {text: 'かた', furigana: ''} ] ], [ ['食べる', 'たべる', '食べた'], [ {text: '食', furigana: 'た'}, - {text: 'べ'}, - {text: 'た'} + {text: 'べ', furigana: ''}, + {text: 'た', furigana: ''} ] ] ]; @@ -393,6 +369,59 @@ function testDistributeFuriganaInflected() { } } +function testCollapseEmphaticSequences() { + const data = [ + [['かこい', false], ['かこい', [1, 1, 1]]], + [['かこい', true], ['かこい', [1, 1, 1]]], + [['かっこい', false], ['かっこい', [1, 1, 1, 1]]], + [['かっこい', true], ['かこい', [2, 1, 1]]], + [['かっっこい', false], ['かっこい', [1, 2, 1, 1]]], + [['かっっこい', true], ['かこい', [3, 1, 1]]], + [['かっっっこい', false], ['かっこい', [1, 3, 1, 1]]], + [['かっっっこい', true], ['かこい', [4, 1, 1]]], + + [['こい', false], ['こい', [1, 1]]], + [['こい', true], ['こい', [1, 1]]], + [['っこい', false], ['っこい', [1, 1, 1]]], + [['っこい', true], ['こい', [2, 1]]], + [['っっこい', false], ['っこい', [2, 1, 1]]], + [['っっこい', true], ['こい', [3, 1]]], + [['っっっこい', false], ['っこい', [3, 1, 1]]], + [['っっっこい', true], ['こい', [4, 1]]], + + [['すごい', false], ['すごい', [1, 1, 1]]], + [['すごい', true], ['すごい', [1, 1, 1]]], + [['すごーい', false], ['すごーい', [1, 1, 1, 1]]], + [['すごーい', true], ['すごい', [1, 2, 1]]], + [['すごーーい', false], ['すごーい', [1, 1, 2, 1]]], + [['すごーーい', true], ['すごい', [1, 3, 1]]], + [['すっごーい', false], ['すっごーい', [1, 1, 1, 1, 1]]], + [['すっごーい', true], ['すごい', [2, 2, 1]]], + [['すっっごーーい', false], ['すっごーい', [1, 2, 1, 2, 1]]], + [['すっっごーーい', true], ['すごい', [3, 3, 1]]], + + [['', false], ['', []]], + [['', true], ['', []]], + [['っ', false], ['っ', [1]]], + [['っ', true], ['', [1]]], + [['っっ', false], ['っ', [2]]], + [['っっ', true], ['', [2]]], + [['っっっ', false], ['っ', [3]]], + [['っっっ', true], ['', [3]]] + ]; + + for (const [[text, fullCollapse], [expected, expectedSourceMapping]] of data) { + const sourceMap = new TextSourceMap(text); + const actual1 = jp.collapseEmphaticSequences(text, fullCollapse, null); + const actual2 = jp.collapseEmphaticSequences(text, fullCollapse, sourceMap); + assert.strictEqual(actual1, expected); + assert.strictEqual(actual2, expected); + if (typeof expectedSourceMapping !== 'undefined') { + assert.ok(sourceMap.equals(new TextSourceMap(text, expectedSourceMapping))); + } + } +} + function testIsMoraPitchHigh() { const data = [ [[0, 0], false], @@ -462,6 +491,7 @@ function main() { testConvertAlphabeticToKana(); testDistributeFurigana(); testDistributeFuriganaInflected(); + testCollapseEmphaticSequences(); testIsMoraPitchHigh(); testGetKanaMorae(); } |