From fb87b1ad69b37e75d1e2e46c91235aa6b44c2980 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Fri, 10 Apr 2020 12:31:06 -0400 Subject: Add tests --- test/test-japanese.js | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) (limited to 'test') diff --git a/test/test-japanese.js b/test/test-japanese.js index ca65dde2..ac28a579 100644 --- a/test/test-japanese.js +++ b/test/test-japanese.js @@ -394,6 +394,50 @@ function testDistributeFuriganaInflected() { } } +function testCollapseEmphaticSequences() { + const data = [ + [['かこい', false], ['かこい', [1, 1, 1]]], + [['かこい', true], ['かこい', [1, 1, 1]]], + [['かっこい', false], ['かっこい', [1, 1, 1, 1]]], + [['かっこい', true], ['かこい', [2, 1, 1]]], + [['かっっこい', false], ['かっこい', [1, 2, 1, 1]]], + [['かっっこい', true], ['かこい', [3, 1, 1]]], + [['かっっっこい', false], ['かっこい', [1, 3, 1, 1]]], + [['かっっっこい', true], ['かこい', [4, 1, 1]]], + + [['こい', false], ['こい', [1, 1]]], + [['こい', true], ['こい', [1, 1]]], + [['っこい', false], ['っこい', [1, 1, 1]]], + [['っこい', true], ['こい', [2, 1]]], + [['っっこい', false], ['っこい', [2, 1, 1]]], + [['っっこい', true], ['こい', [3, 1]]], + [['っっっこい', false], ['っこい', [3, 1, 1]]], + [['っっっこい', true], ['こい', [4, 1]]], + + [['', false], ['', []]], + [['', true], ['', []]], + [['っ', false], ['っ', [1]]], + [['っ', true], ['', [1]]], + [['っっ', false], ['っ', [2]]], + [['っっ', true], ['', [2]]], + [['っっっ', false], ['っ', [3]]], + [['っっっ', true], ['', [3]]] + ]; + + for (const [[text, fullCollapse], [expected, expectedSourceMapping]] of data) { + const sourceMap = new TextSourceMap(text); + const actual1 = jp.collapseEmphaticSequences(text, fullCollapse, null); + const actual2 = jp.collapseEmphaticSequences(text, fullCollapse, sourceMap); + assert.strictEqual(actual1, expected); + assert.strictEqual(actual2, expected); + if (typeof expectedSourceMapping !== 'undefined') { + console.log('actual', JSON.stringify(actual1), sourceMap); + console.log('expected', JSON.stringify(expected), new TextSourceMap(text, expectedSourceMapping)); + assert.ok(sourceMap.equals(new TextSourceMap(text, expectedSourceMapping))); + } + } +} + function testIsMoraPitchHigh() { const data = [ [[0, 0], false], @@ -463,6 +507,7 @@ function main() { testConvertAlphabeticToKana(); testDistributeFurigana(); testDistributeFuriganaInflected(); + testCollapseEmphaticSequences(); testIsMoraPitchHigh(); testGetKanaMorae(); } -- cgit v1.2.3 From e7abeb02a64c599474ce47c5a8cde46097882634 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Fri, 10 Apr 2020 18:45:16 -0400 Subject: Add simple check to see if globals are unused --- test/lint/global-declarations.js | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) (limited to 'test') diff --git a/test/lint/global-declarations.js b/test/lint/global-declarations.js index 07ba5570..2fc9a5e2 100644 --- a/test/lint/global-declarations.js +++ b/test/lint/global-declarations.js @@ -37,6 +37,18 @@ function getNewline(string) { } } +function getSubstringCount(string, substring) { + let start = 0; + let count = 0; + while (true) { + const pos = string.indexOf(substring, start); + if (pos < 0) { break; } + ++count; + start = pos + substring.length; + } + return count; +} + function validateGlobals(fileName, fix) { const pattern = /\/\*\s*global\s+([\w\W]*?)\*\//g; @@ -47,6 +59,7 @@ function validateGlobals(fileName, fix) { let first = true; let endIndex = 0; let newSource = ''; + const allGlobals = []; const newline = getNewline(source); while ((match = pattern.exec(source)) !== null) { if (!first) { @@ -74,15 +87,27 @@ function validateGlobals(fileName, fix) { newSource += source.substring(0, match.index); newSource += expected; endIndex = match.index + match[0].length; + + allGlobals.push(...parts); } newSource += source.substring(endIndex); + // This is an approximate check to see if a global variable is unused. + // If the global appears in a comment, string, or similar, the check will pass. + let errorCount = 0; + for (const global of allGlobals) { + if (getSubstringCount(newSource, global) <= 1) { + console.error(`Global variable ${global} appears to be unused in ${fileName}`); + ++errorCount; + } + } + if (fix) { fs.writeFileSync(fileName, newSource, {encoding: 'utf8'}); } - return true; + return errorCount === 0; } -- cgit v1.2.3 From 92f2466cfff40d47fb5e6350dae5d7ff82770973 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sat, 11 Apr 2020 16:53:29 -0400 Subject: Add tests, remove logs --- test/test-japanese.js | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'test') diff --git a/test/test-japanese.js b/test/test-japanese.js index ac28a579..b1bba9a7 100644 --- a/test/test-japanese.js +++ b/test/test-japanese.js @@ -414,6 +414,17 @@ function testCollapseEmphaticSequences() { [['っっっこい', false], ['っこい', [3, 1, 1]]], [['っっっこい', true], ['こい', [4, 1]]], + [['すごい', false], ['すごい', [1, 1, 1]]], + [['すごい', true], ['すごい', [1, 1, 1]]], + [['すごーい', false], ['すごーい', [1, 1, 1, 1]]], + [['すごーい', true], ['すごい', [1, 2, 1]]], + [['すごーーい', false], ['すごーい', [1, 1, 2, 1]]], + [['すごーーい', true], ['すごい', [1, 3, 1]]], + [['すっごーい', false], ['すっごーい', [1, 1, 1, 1, 1]]], + [['すっごーい', true], ['すごい', [2, 2, 1]]], + [['すっっごーーい', false], ['すっごーい', [1, 2, 1, 2, 1]]], + [['すっっごーーい', true], ['すごい', [3, 3, 1]]], + [['', false], ['', []]], [['', true], ['', []]], [['っ', false], ['っ', [1]]], @@ -431,8 +442,6 @@ function testCollapseEmphaticSequences() { assert.strictEqual(actual1, expected); assert.strictEqual(actual2, expected); if (typeof expectedSourceMapping !== 'undefined') { - console.log('actual', JSON.stringify(actual1), sourceMap); - console.log('expected', JSON.stringify(expected), new TextSourceMap(text, expectedSourceMapping)); assert.ok(sourceMap.equals(new TextSourceMap(text, expectedSourceMapping))); } } -- cgit v1.2.3 From f93dc857107e9b23ec06f1b568aad2c6f870ba4c Mon Sep 17 00:00:00 2001 From: siikamiika Date: Mon, 13 Apr 2020 22:55:33 +0300 Subject: assume and propagate strings for text parsing --- ext/bg/js/backend.js | 26 ++++++------- ext/bg/js/japanese.js | 10 ++--- ext/bg/js/search-query-parser-generator.js | 2 +- ext/bg/js/search-query-parser.js | 2 +- test/test-japanese.js | 60 +++++++++--------------------- 5 files changed, 36 insertions(+), 64 deletions(-) (limited to 'test') diff --git a/ext/bg/js/backend.js b/ext/bg/js/backend.js index 65e00f28..d5086af2 100644 --- a/ext/bg/js/backend.js +++ b/ext/bg/js/backend.js @@ -334,7 +334,7 @@ class Backend { } text = text.substring(source.length); } else { - const reading = jp.convertReading(text[0], null, options.parsing.readingMode); + const reading = jp.convertReading(text[0], '', options.parsing.readingMode); term.push({text: text[0], reading}); text = text.substring(1); } @@ -349,24 +349,20 @@ class Backend { for (const [mecabName, parsedLines] of Object.entries(rawResults)) { const result = []; for (const parsedLine of parsedLines) { - for (const {expression, reading, source} of parsedLine) { + for (let {expression, reading, source} of parsedLine) { const term = []; - if (expression !== null && reading !== null) { - for (const {text: text2, furigana} of jp.distributeFuriganaInflected( - expression, - jp.convertKatakanaToHiragana(reading), - source - )) { - const reading2 = jp.convertReading(text2, furigana, options.parsing.readingMode); - term.push({text: text2, reading: reading2}); - } - } else { - const reading2 = jp.convertReading(source, null, options.parsing.readingMode); - term.push({text: source, reading: reading2}); + if (expression === '') { expression = source; } + for (const {text: text2, furigana} of jp.distributeFuriganaInflected( + expression, + jp.convertKatakanaToHiragana(reading), + source + )) { + const reading2 = jp.convertReading(text2, furigana, options.parsing.readingMode); + term.push({text: text2, reading: reading2}); } result.push(term); } - result.push([{text: '\n'}]); + result.push([{text: '\n', reading: ''}]); } results.push([mecabName, result]); } diff --git a/ext/bg/js/japanese.js b/ext/bg/js/japanese.js index 5fef27a7..c74e4553 100644 --- a/ext/bg/js/japanese.js +++ b/ext/bg/js/japanese.js @@ -127,9 +127,9 @@ function convertReading(expressionFragment, readingFragment, readingMode) { switch (readingMode) { case 'hiragana': - return convertKatakanaToHiragana(readingFragment || ''); + return convertKatakanaToHiragana(readingFragment); case 'katakana': - return convertHiraganaToKatakana(readingFragment || ''); + return convertHiraganaToKatakana(readingFragment); case 'romaji': if (readingFragment) { return convertToRomaji(readingFragment); @@ -140,7 +140,7 @@ } return readingFragment; case 'none': - return null; + return ''; default: return readingFragment; } @@ -300,7 +300,7 @@ const readingLeft = reading2.substring(group.text.length); const segs = segmentize(readingLeft, groups.splice(1)); if (segs) { - return [{text: group.text}].concat(segs); + return [{text: group.text, furigana: ''}].concat(segs); } } } else { @@ -368,7 +368,7 @@ } if (stemLength !== source.length) { - output.push({text: source.substring(stemLength)}); + output.push({text: source.substring(stemLength), furigana: ''}); } return output; diff --git a/ext/bg/js/search-query-parser-generator.js b/ext/bg/js/search-query-parser-generator.js index d44829f7..527302ed 100644 --- a/ext/bg/js/search-query-parser-generator.js +++ b/ext/bg/js/search-query-parser-generator.js @@ -36,7 +36,7 @@ class QueryParserGenerator { const termContainer = this._templateHandler.instantiate(preview ? 'term-preview' : 'term'); for (const segment of term) { if (!segment.text.trim()) { continue; } - if (!segment.reading || !segment.reading.trim()) { + if (!segment.reading.trim()) { termContainer.appendChild(this.createSegmentText(segment.text)); } else { termContainer.appendChild(this.createSegment(segment)); diff --git a/ext/bg/js/search-query-parser.js b/ext/bg/js/search-query-parser.js index da61b045..692fb1a8 100644 --- a/ext/bg/js/search-query-parser.js +++ b/ext/bg/js/search-query-parser.js @@ -140,7 +140,7 @@ class QueryParser extends TextScanner { const previewTerms = []; for (let i = 0, ii = text.length; i < ii; i += 2) { const tempText = text.substring(i, i + 2); - previewTerms.push([{text: tempText}]); + previewTerms.push([{text: tempText, reading: ''}]); } this.queryParser.textContent = ''; this.queryParser.appendChild(this.queryParserGenerator.createParseResult(previewTerms, true)); diff --git a/test/test-japanese.js b/test/test-japanese.js index 89e41c36..7fd71ba8 100644 --- a/test/test-japanese.js +++ b/test/test-japanese.js @@ -176,19 +176,19 @@ function testConvertReading() { [['アリガトウ', 'アリガトウ', 'hiragana'], 'ありがとう'], [['アリガトウ', 'アリガトウ', 'katakana'], 'アリガトウ'], [['アリガトウ', 'アリガトウ', 'romaji'], 'arigatou'], - [['アリガトウ', 'アリガトウ', 'none'], null], + [['アリガトウ', 'アリガトウ', 'none'], ''], [['アリガトウ', 'アリガトウ', 'default'], 'アリガトウ'], [['ありがとう', 'ありがとう', 'hiragana'], 'ありがとう'], [['ありがとう', 'ありがとう', 'katakana'], 'アリガトウ'], [['ありがとう', 'ありがとう', 'romaji'], 'arigatou'], - [['ありがとう', 'ありがとう', 'none'], null], + [['ありがとう', 'ありがとう', 'none'], ''], [['ありがとう', 'ありがとう', 'default'], 'ありがとう'], [['有り難う', 'ありがとう', 'hiragana'], 'ありがとう'], [['有り難う', 'ありがとう', 'katakana'], 'アリガトウ'], [['有り難う', 'ありがとう', 'romaji'], 'arigatou'], - [['有り難う', 'ありがとう', 'none'], null], + [['有り難う', 'ありがとう', 'none'], ''], [['有り難う', 'ありがとう', 'default'], 'ありがとう'], // Cases with falsy readings @@ -196,40 +196,16 @@ function testConvertReading() { [['ありがとう', '', 'hiragana'], ''], [['ありがとう', '', 'katakana'], ''], [['ありがとう', '', 'romaji'], 'arigatou'], - [['ありがとう', '', 'none'], null], + [['ありがとう', '', 'none'], ''], [['ありがとう', '', 'default'], ''], - [['ありがとう', null, 'hiragana'], ''], - [['ありがとう', null, 'katakana'], ''], - [['ありがとう', null, 'romaji'], 'arigatou'], - [['ありがとう', null, 'none'], null], - [['ありがとう', null, 'default'], null], - - [['ありがとう', void 0, 'hiragana'], ''], - [['ありがとう', void 0, 'katakana'], ''], - [['ありがとう', void 0, 'romaji'], 'arigatou'], - [['ありがとう', void 0, 'none'], null], - [['ありがとう', void 0, 'default'], void 0], - // Cases with falsy readings and kanji expressions [['有り難う', '', 'hiragana'], ''], [['有り難う', '', 'katakana'], ''], [['有り難う', '', 'romaji'], ''], - [['有り難う', '', 'none'], null], - [['有り難う', '', 'default'], ''], - - [['有り難う', null, 'hiragana'], ''], - [['有り難う', null, 'katakana'], ''], - [['有り難う', null, 'romaji'], null], - [['有り難う', null, 'none'], null], - [['有り難う', null, 'default'], null], - - [['有り難う', void 0, 'hiragana'], ''], - [['有り難う', void 0, 'katakana'], ''], - [['有り難う', void 0, 'romaji'], void 0], - [['有り難う', void 0, 'none'], null], - [['有り難う', void 0, 'default'], void 0] + [['有り難う', '', 'none'], ''], + [['有り難う', '', 'default'], ''] ]; for (const [[expressionFragment, readingFragment, readingMode], expected] of data) { @@ -303,9 +279,9 @@ function testDistributeFurigana() { ['有り難う', 'ありがとう'], [ {text: '有', furigana: 'あ'}, - {text: 'り'}, + {text: 'り', furigana: ''}, {text: '難', furigana: 'がと'}, - {text: 'う'} + {text: 'う', furigana: ''} ] ], [ @@ -317,23 +293,23 @@ function testDistributeFurigana() { [ ['お祝い', 'おいわい'], [ - {text: 'お'}, + {text: 'お', furigana: ''}, {text: '祝', furigana: 'いわ'}, - {text: 'い'} + {text: 'い', furigana: ''} ] ], [ ['美味しい', 'おいしい'], [ {text: '美味', furigana: 'おい'}, - {text: 'しい'} + {text: 'しい', furigana: ''} ] ], [ ['食べ物', 'たべもの'], [ {text: '食', furigana: 'た'}, - {text: 'べ'}, + {text: 'べ', furigana: ''}, {text: '物', furigana: 'もの'} ] ], @@ -341,9 +317,9 @@ function testDistributeFurigana() { ['試し切り', 'ためしぎり'], [ {text: '試', furigana: 'ため'}, - {text: 'し'}, + {text: 'し', furigana: ''}, {text: '切', furigana: 'ぎ'}, - {text: 'り'} + {text: 'り', furigana: ''} ] ], // Ambiguous @@ -373,16 +349,16 @@ function testDistributeFuriganaInflected() { ['美味しい', 'おいしい', '美味しかた'], [ {text: '美味', furigana: 'おい'}, - {text: 'し'}, - {text: 'かた'} + {text: 'し', furigana: ''}, + {text: 'かた', furigana: ''} ] ], [ ['食べる', 'たべる', '食べた'], [ {text: '食', furigana: 'た'}, - {text: 'べ'}, - {text: 'た'} + {text: 'べ', furigana: ''}, + {text: 'た', furigana: ''} ] ] ]; -- cgit v1.2.3 From 6cc87ec9753d7482af2ff75bed2d405f7c31c072 Mon Sep 17 00:00:00 2001 From: siikamiika Date: Mon, 13 Apr 2020 22:59:56 +0300 Subject: remove "fragment" from jp.convertReading variables --- ext/bg/js/japanese.js | 18 +++++++++--------- test/test-japanese.js | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) (limited to 'test') diff --git a/ext/bg/js/japanese.js b/ext/bg/js/japanese.js index c74e4553..ac81acb5 100644 --- a/ext/bg/js/japanese.js +++ b/ext/bg/js/japanese.js @@ -124,25 +124,25 @@ return wanakana.toRomaji(text); } - function convertReading(expressionFragment, readingFragment, readingMode) { + function convertReading(expression, reading, readingMode) { switch (readingMode) { case 'hiragana': - return convertKatakanaToHiragana(readingFragment); + return convertKatakanaToHiragana(reading); case 'katakana': - return convertHiraganaToKatakana(readingFragment); + return convertHiraganaToKatakana(reading); case 'romaji': - if (readingFragment) { - return convertToRomaji(readingFragment); + if (reading) { + return convertToRomaji(reading); } else { - if (isStringEntirelyKana(expressionFragment)) { - return convertToRomaji(expressionFragment); + if (isStringEntirelyKana(expression)) { + return convertToRomaji(expression); } } - return readingFragment; + return reading; case 'none': return ''; default: - return readingFragment; + return reading; } } diff --git a/test/test-japanese.js b/test/test-japanese.js index 7fd71ba8..87efdfad 100644 --- a/test/test-japanese.js +++ b/test/test-japanese.js @@ -208,8 +208,8 @@ function testConvertReading() { [['有り難う', '', 'default'], ''] ]; - for (const [[expressionFragment, readingFragment, readingMode], expected] of data) { - assert.strictEqual(jp.convertReading(expressionFragment, readingFragment, readingMode), expected); + for (const [[expression, reading, readingMode], expected] of data) { + assert.strictEqual(jp.convertReading(expression, reading, readingMode), expected); } } -- cgit v1.2.3