From 4aaa9f15d97668203741c1731f15e710ae8b8294 Mon Sep 17 00:00:00 2001 From: StefanVukovic99 Date: Sat, 17 Feb 2024 02:45:24 +0100 Subject: add language select, abstract text transformations (#584) * Copy functions from JapaneseUtil * Remove JapaneseUtil * Update usages of JapaneseUtil functions * part1 * frotend done? * fix tests * offscreen and type complications * add tests * start fixing tests * keep fixing tests * fix tests * Copy functions from JapaneseUtil * Remove JapaneseUtil * Update usages of JapaneseUtil functions * delete pt * renames * add tests * kebab-case filenames * lint * minor fixes * merge * fixes * fix part of comments * fix more comments * delete unused types * comment * comment * do backend * other files * move fetch utils to own file * remove extra line * add extra line * remove unnecessary export * simplify folder structure * remove redundant async * fix param type in api * fix language index * undo changes to cssStyleApplier * undo changes to utilities.js * undo changes to utilities.js * simplify language util * lint * undo phantom changes to anki integration * require textTransformations options * explicit locale in localeCompare * punctuate notes * prefer early exit * rename LanguageOptionsObjectMap * rename to textPreprocessor * tuple with names instead of boolean array * safe data setting * optional chaining * simplify LanguageOptions * encapsulate languages * delete language util * nullable language in text preprocessors controller * rename transform to process * remove settings * make translation advanced again * remove unused getTextTransformations api call * comments * change language types * RIP flags * comments * fix tests * lint * Text preprocessor type changes (#10) * Add types * Update types * Simplify type check * Refactor typing and structuring of language definitions * lint * update translator benchmark * undo markdown changes * undo markdown changes * undo markdown changes * more merge * simplify language controller --------- Co-authored-by: toasted-nutbread Co-authored-by: Darius Jahandarie --- types/ext/api.d.ts | 5 ++++ types/ext/language-english.d.ts | 25 ++++++++++++++++ types/ext/language-japanese.d.ts | 29 +++++++++++++++++++ types/ext/language.d.ts | 57 +++++++++++++++++++++++++++++++++++++ types/ext/settings.d.ts | 1 + types/ext/translation-internal.d.ts | 3 ++ types/ext/translation.d.ts | 38 +++---------------------- types/test/translator.d.ts | 11 ++----- 8 files changed, 127 insertions(+), 42 deletions(-) create mode 100644 types/ext/language-english.d.ts create mode 100644 types/ext/language-japanese.d.ts create mode 100644 types/ext/language.d.ts (limited to 'types') diff --git a/types/ext/api.d.ts b/types/ext/api.d.ts index 1f4fc0a9..85f4c146 100644 --- a/types/ext/api.d.ts +++ b/types/ext/api.d.ts @@ -26,6 +26,7 @@ import type * as DictionaryDatabase from './dictionary-database'; import type * as DictionaryImporter from './dictionary-importer'; import type * as Environment from './environment'; import type * as Extension from './extension'; +import type * as Language from './language'; import type * as Log from './log'; import type * as Settings from './settings'; import type * as SettingsModifications from './settings-modifications'; @@ -380,6 +381,10 @@ type ApiSurface = { params: void; return: boolean; }; + getLanguageSummaries: { + params: void; + return: Language.LanguageSummary[]; + }; }; type ApiExtraArgs = [sender: chrome.runtime.MessageSender]; diff --git a/types/ext/language-english.d.ts b/types/ext/language-english.d.ts new file mode 100644 index 00000000..ed501d57 --- /dev/null +++ b/types/ext/language-english.d.ts @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2024 Yomitan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +import type {LanguageDescriptor, TextPreprocessor} from './language'; + +export type EnglishTextPreprocessorDescriptor = { + capitalizeFirstLetter: TextPreprocessor; + decapitalize: TextPreprocessor; +}; + +export type EnglishLanguageDescriptor = LanguageDescriptor; diff --git a/types/ext/language-japanese.d.ts b/types/ext/language-japanese.d.ts new file mode 100644 index 00000000..1a627ed1 --- /dev/null +++ b/types/ext/language-japanese.d.ts @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2024 Yomitan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +import type {LanguageDescriptor, TextPreprocessor} from './language'; + +export type JapaneseTextPreprocessorDescriptor = { + convertHalfWidthCharacters: TextPreprocessor; + convertNumericCharacters: TextPreprocessor; + convertAlphabeticCharacters: TextPreprocessor; + convertHiraganaToKatakana: TextPreprocessor; + convertKatakanaToHiragana: TextPreprocessor; + collapseEmphaticSequences: TextPreprocessor<[collapseEmphatic: boolean, collapseEmphaticFull: boolean]>; +}; + +export type JapaneseLanguageDescriptor = LanguageDescriptor; diff --git a/types/ext/language.d.ts b/types/ext/language.d.ts new file mode 100644 index 00000000..247c7795 --- /dev/null +++ b/types/ext/language.d.ts @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2024 Yomitan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +import type {TextSourceMap} from '../../ext/js/general/text-source-map.js'; +import type {SafeAny} from './core'; + +export type TextPreprocessorOptions = T[]; + +export type TextPreprocessorFunction = (str: string, setting: T, sourceMap: TextSourceMap) => string; + +export type TextPreprocessor = { + name: string; + description: string; + options: TextPreprocessorOptions; + process: TextPreprocessorFunction; +}; + +export type LanguageAndPreprocessors = { + iso: string; + textPreprocessors: TextPreprocessorWithId[]; +}; + +export type TextPreprocessorWithId = { + id: string; + textPreprocessor: TextPreprocessor; +}; + +export type LanguageSummary = { + name: string; + iso: string; + exampleText: string; +}; + +export type LanguageDescriptor = { + name: string; + iso: string; + exampleText: string; + textPreprocessors: TTextPreprocessorDescriptor; +}; + +export type TextPreprocessorDescriptor = { + [key: string]: TextPreprocessor; +}; diff --git a/types/ext/settings.d.ts b/types/ext/settings.d.ts index a900dbe6..45466c3d 100644 --- a/types/ext/settings.d.ts +++ b/types/ext/settings.d.ts @@ -101,6 +101,7 @@ export type ProfileOptions = { export type GeneralOptions = { enable: boolean; + language: string; resultOutputMode: ResultOutputMode; debugInfo: boolean; maxResults: number; diff --git a/types/ext/translation-internal.d.ts b/types/ext/translation-internal.d.ts index 82704c54..7006221e 100644 --- a/types/ext/translation-internal.d.ts +++ b/types/ext/translation-internal.d.ts @@ -18,6 +18,7 @@ import type * as DictionaryDatabase from './dictionary-database'; import type * as Dictionary from './dictionary'; import type * as Translation from './translation'; +import type * as Language from './language'; export type TextDeinflectionOptions = [ textReplacements: Translation.FindTermsTextReplacement[] | null, @@ -47,3 +48,5 @@ export type DatabaseDeinflection = { inflectionRuleChainCandidates: Dictionary.InflectionRuleChainCandidate[]; databaseEntries: DictionaryDatabase.TermEntry[]; }; + +export type PreprocessorOptionsSpace = Map>; diff --git a/types/ext/translation.d.ts b/types/ext/translation.d.ts index c9a61be0..2e4d1a66 100644 --- a/types/ext/translation.d.ts +++ b/types/ext/translation.d.ts @@ -80,30 +80,6 @@ export type FindTermsOptions = { * Whether or not non-Japanese characters should be searched. */ removeNonJapaneseCharacters: boolean; - /** - * Whether or not half-width characters should be converted to full-width characters. - */ - convertHalfWidthCharacters: FindTermsVariantMode; - /** - * Whether or not ASCII numeric characters should be converted to full-width numeric characters. - */ - convertNumericCharacters: FindTermsVariantMode; - /** - * Whether or not alphabetic characters should be converted to kana. - */ - convertAlphabeticCharacters: FindTermsVariantMode; - /** - * Whether or not hiragana characters should be converted to katakana. - */ - convertHiraganaToKatakana: FindTermsVariantMode; - /** - * Whether or not katakana characters should be converted to hiragana. - */ - convertKatakanaToHiragana: FindTermsVariantMode; - /** - * How emphatic character sequences should be collapsed. - */ - collapseEmphaticSequences: FindTermsEmphaticSequencesMode; /** * An iterable sequence of text replacements to be applied during the term lookup process. */ @@ -121,6 +97,10 @@ export type FindTermsOptions = { * Whether every substring should be searched for, or only whole words. */ searchResolution: SearchResolution; + /** + * ISO-639 code of the language. + */ + language: string; }; /** @@ -133,16 +113,6 @@ export type FindTermsMatchType = Dictionary.TermSourceMatchType; */ export type FindTermsSortOrder = 'ascending' | 'descending'; -/** - * Mode describing how to handle variations. - */ -export type FindTermsVariantMode = 'false' | 'true' | 'variant'; - -/** - * Mode describing how to handle emphatic sequence variations. - */ -export type FindTermsEmphaticSequencesMode = 'false' | 'true' | 'full'; - /** * Information about how text should be replaced when looking up terms. */ diff --git a/types/test/translator.d.ts b/types/test/translator.d.ts index e3199225..efd5cc3f 100644 --- a/types/test/translator.d.ts +++ b/types/test/translator.d.ts @@ -15,8 +15,8 @@ * along with this program. If not, see . */ -import type {FindTermsMatchType, FindTermsSortOrder, FindTermsVariantMode, FindTermsEmphaticSequencesMode, FindKanjiDictionary, FindTermDictionary} from '../ext/translation'; -import type {SearchResolution} from 'settings'; +import type {FindTermsMatchType, FindTermsSortOrder, FindKanjiDictionary, FindTermDictionary} from '../ext/translation'; +import type {SearchResolution} from '../ext/settings'; import type {FindTermsMode} from 'translator'; import type {DictionaryEntry} from 'dictionary'; import type {NoteData} from 'anki-templates'; @@ -44,16 +44,11 @@ export type FindTermsOptionsPreset = { sortFrequencyDictionary?: string | null; sortFrequencyDictionaryOrder?: FindTermsSortOrder; removeNonJapaneseCharacters?: boolean; - convertHalfWidthCharacters?: FindTermsVariantMode; - convertNumericCharacters?: FindTermsVariantMode; - convertAlphabeticCharacters?: FindTermsVariantMode; - convertHiraganaToKatakana?: FindTermsVariantMode; - convertKatakanaToHiragana?: FindTermsVariantMode; - collapseEmphaticSequences?: FindTermsEmphaticSequencesMode; textReplacements?: (FindTermsTextReplacement[] | null)[]; enabledDictionaryMap?: [key: string, value: FindTermDictionary][]; excludeDictionaryDefinitions?: string[] | null; searchResolution?: SearchResolution; + language?: string; }; export type OptionsType = OptionsPreset['type']; -- cgit v1.2.3