// SPDX-License-Identifier: AGPL-3.0-or-later use std::cmp::Reverse; use anyhow::{Result, bail}; use regex::Regex; use crate::po::Entry; const EN_GB_PROTECTED_REGION_TERMS: &[&str] = &[ "United States of America", "United States", "U.S.", "USA", "US", ]; #[derive(Clone, Debug, Eq, PartialEq)] pub struct IcuControl { pub argument: String, pub kind: String, } #[derive(Clone, Debug, Eq, PartialEq)] pub struct TokenAlias { pub token: String, pub alias: String, } pub fn should_keep_unchanged(source: &str, locale: &str) -> bool { if locale == "en-US" { return false; } if source.trim().is_empty() { return true; } if !has_any_letter(source) { return true; } if is_preserved_token_composition(source) { return true; } is_preserved_artifact(source) } pub fn has_any_letter(source: &str) -> bool { source.chars().any(char::is_alphabetic) } pub fn is_preserved_artifact(source: &str) -> bool { let trimmed = source.trim(); if [ r"(?i)^https?://\S+$", r"(?i)^(?:mailto:|tel:|file:|app://|fluxer://)\S+$", r"^[@#][\w-]+$", r"^\{[^{}]+\}$", r"^<[^>]+>$", r"^%[@sd]$", r"^\$\d+$", r#"^[\s{}#,._:/+()\[\]'"-]+$"#, r"^[A-Z][A-Z0-9_./:+-]+$", ] .iter() .any(|pattern| { Regex::new(pattern) .expect("valid artifact regex") .is_match(trimmed) }) { return true; } Regex::new(r"^[\w./:+-]+$") .expect("valid word artifact regex") .is_match(trimmed) && Regex::new(r"[\d_./:+-]") .expect("valid word artifact marker regex") .is_match(trimmed) } pub fn is_preserved_token_composition(source: &str) -> bool { let entry = Entry { msgid: source.to_string(), ..Entry::default() }; let mut tokens = extract_preserved_tokens(&entry); if tokens.is_empty() { return false; } tokens.sort_by_key(|token| Reverse(token.len())); let mut remainder = source.to_string(); for token in tokens { remainder = Regex::new(®ex::escape(&token)) .expect("valid preserved token cleanup regex") .replace_all(&remainder, "") .into_owned(); } remainder .chars() .all(|character| character.is_whitespace() || is_preserved_token_separator(character)) } fn is_preserved_token_separator(character: char) -> bool { matches!( character, ':' | ',' | '.' | '/' | '\\' | '-' | '–' | '—' | '(' | ')' | '[' | ']' | '{' | '}' | '|' | '·' | '•' | '#' | '+' | '\'' | '"' ) } pub fn validate_localization(entry: &Entry, localized: &str) -> Result<()> { if !entry.msgid.trim().is_empty() && localized.trim().is_empty() { bail!("Localized string is empty"); } let source_tokens = extract_preserved_tokens(entry); let localized_entry = Entry { msgid: localized.to_string(), comments: entry.comments.clone(), ..Entry::default() }; let localized_tokens = extract_preserved_tokens(&localized_entry); for token in source_tokens { if !localized_tokens.contains(&token) { bail!("Localized string did not preserve token {token}"); } } for control in extract_icu_controls(&entry.msgid) { let pattern = Regex::new(&format!( r"\{{\s*{}\s*,\s*{}\s*,", regex::escape(&control.argument), regex::escape(&control.kind) )) .expect("valid ICU validation regex"); if !pattern.is_match(localized) { bail!( "Localized string did not preserve ICU {} argument {}", control.kind, control.argument ); } } Ok(()) } pub fn validate_locale_specific_localization( entry: &Entry, locale: &str, localized: &str, ) -> Result<()> { if locale != "en-GB" { return Ok(()); } for term in extract_en_gb_protected_terms(&entry.msgid) { if !contains_exact_term(localized, term) { bail!("en-GB localization changed protected source term {term}"); } } Ok(()) } pub fn extract_en_gb_protected_terms(source: &str) -> Vec<&'static str> { EN_GB_PROTECTED_REGION_TERMS .iter() .copied() .filter(|term| contains_exact_term(source, term)) .collect() } pub fn contains_exact_term(source: &str, term: &str) -> bool { source.match_indices(term).any(|(start, _match)| { let before_ok = source[..start] .chars() .next_back() .is_none_or(|before| !is_word_char(before)); let end = start + term.len(); let after_ok = source[end..] .chars() .next() .is_none_or(|after| !is_word_char(after)); before_ok && after_ok }) } fn is_word_char(value: char) -> bool { value == '_' || value.is_alphanumeric() } pub fn normalize_localized_capitalization(entry: &Entry, locale: &str, localized: &str) -> String { if locale.starts_with("en") || !entry.msgid.starts_with('(') || !localized.starts_with('(') { return localized.to_string(); } let mut chars = localized.chars(); let Some(first) = chars.next() else { return localized.to_string(); }; let Some(second) = chars.next() else { return localized.to_string(); }; let second_string = second.to_string(); if second_string != second.to_uppercase().to_string() { return localized.to_string(); } let rest = chars.as_str(); format!("{first}{}{rest}", second.to_lowercase()) } pub fn extract_preserved_tokens(entry: &Entry) -> Vec { let mut tokens = Vec::new(); for token in extract_top_level_brace_tokens(&entry.msgid) { push_unique(&mut tokens, token); } let token_re = Regex::new(r"%[@sd]|\$\d+|<[^>]+>|https?://\S+|:[a-zA-Z0-9_+-]+:") .expect("valid preserved token regex"); for matched in token_re.find_iter(&entry.msgid) { push_unique(&mut tokens, matched.as_str().to_string()); } tokens } pub fn extract_top_level_brace_tokens(source: &str) -> Vec { let mut tokens = Vec::new(); let mut depth = 0usize; let mut start: Option = None; let double_brace_re = Regex::new(r"^\{\{[^{}]+\}\}$").expect("valid double brace regex"); let brace_re = Regex::new(r"^\{[^{},]+\}$").expect("valid brace regex"); for (index, character) in source.char_indices() { if character == '{' { if depth == 0 { start = Some(index); } depth += 1; } else if character == '}' && depth > 0 { depth -= 1; if depth == 0 { if let Some(start_index) = start { let token = &source[start_index..index + character.len_utf8()]; if double_brace_re.is_match(token) || brace_re.is_match(token) { tokens.push(token.to_string()); } } start = None; } } } tokens } pub fn has_icu_control(source: &str) -> bool { Regex::new(r"\{[^{}]+,\s*(?:plural|select|selectordinal)\s*,") .expect("valid ICU detection regex") .is_match(source) } pub fn extract_icu_controls(source: &str) -> Vec { Regex::new(r"\{\s*([^{}\s,]+)\s*,\s*(plural|select|selectordinal)\s*,") .expect("valid ICU control regex") .captures_iter(source) .map(|captures| IcuControl { argument: captures[1].to_string(), kind: captures[2].to_string(), }) .collect() } pub fn build_masked_source(entry: &Entry) -> (String, Vec) { let token_aliases = extract_preserved_tokens(entry) .into_iter() .enumerate() .map(|(index, token)| TokenAlias { token, alias: format!("{{FLUXER_TOKEN_{index}}}"), }) .collect::>(); ( mask_text_with_aliases(&entry.msgid, &token_aliases), token_aliases, ) } pub fn mask_text_with_aliases(text: &str, token_aliases: &[TokenAlias]) -> String { let mut masked = text.to_string(); let mut aliases = token_aliases.iter().collect::>(); aliases.sort_by_key(|alias| std::cmp::Reverse(alias.token.len())); for item in aliases { masked = Regex::new(®ex::escape(&item.token)) .expect("valid token mask regex") .replace_all(&masked, item.alias.as_str()) .into_owned(); } masked } pub fn restore_masked_tokens(localized: &str, token_aliases: &[TokenAlias]) -> String { let mut restored = localized.to_string(); for item in token_aliases { restored = restored.replace(&item.alias, &item.token); } restored } pub fn build_token_alias_context(token_aliases: &[TokenAlias]) -> String { let mut lines = vec!["Literal protected aliases used in the source string:".to_string()]; lines.extend( token_aliases .iter() .map(|item| format!("{} = {}", item.alias, item.token)), ); lines.extend([ "Aliases are required substrings, not words or labels to localize.".to_string(), "Copy every alias exactly in the localized output, with the same spelling and braces.".to_string(), "Do not replace an alias with translated words, even if the original token name looks meaningful." .to_string(), "Translate only the human-readable words around aliases.".to_string(), ]); lines.join("\n") } fn push_unique(tokens: &mut Vec, token: String) { if !tokens.contains(&token) { tokens.push(token); } } #[cfg(test)] mod tests { use super::*; #[test] fn detects_sources_that_should_stay_unchanged() { assert!(should_keep_unchanged("", "de")); assert!(should_keep_unchanged("https://fluxer.app/docs", "de")); assert!(should_keep_unchanged("{productName}", "de")); assert!(should_keep_unchanged("{authorName} {description}", "de")); assert!(should_keep_unchanged( "{emojiName}: {reactionCountText}, {actionText}", "de" )); assert!(should_keep_unchanged("{start} – {end}", "de")); assert!(should_keep_unchanged("C++", "de")); assert!(!should_keep_unchanged("DSA {reportId}", "de")); assert!(!should_keep_unchanged("Hello", "de")); assert!(!should_keep_unchanged("{productName}", "en-US")); } #[test] fn extracts_masks_and_restores_preserved_tokens() { let entry = Entry::with_msgid("{productName} needs <0>{permission} %@ $1 :wave:"); assert_eq!( extract_preserved_tokens(&entry), vec![ "{productName}", "{permission}", "<0>", "", "%@", "$1", ":wave:" ] ); let (masked, aliases) = build_masked_source(&entry); assert!(masked.contains("{FLUXER_TOKEN_0}")); assert_eq!( restore_masked_tokens(&masked, &aliases), "{productName} needs <0>{permission} %@ $1 :wave:" ); } #[test] fn validates_tokens_and_icu_controls() { let entry = Entry { msgid: "{0} {1, plural, one {member} other {members}}".to_string(), comments: vec!["#. placeholder {0}: count".to_string()], ..Entry::default() }; validate_localization(&entry, "{0} {1, plural, one {miembro} other {miembros}}").unwrap(); let error = validate_localization(&entry, "{0} {2, plural, one {x} other {x}}").unwrap_err(); assert!( error .to_string() .contains("did not preserve ICU plural argument 1") ); let missing_token = validate_localization(&Entry::with_msgid("Open {productName}"), "Open").unwrap_err(); assert!(missing_token.to_string().contains("{productName}")); } #[test] fn validates_en_gb_protected_region_terms() { let entry = Entry::with_msgid("US color settings for the United States"); validate_locale_specific_localization( &entry, "en-GB", "US colour settings for the United States", ) .unwrap(); let error = validate_locale_specific_localization( &entry, "en-GB", "UK colour settings for the United Kingdom", ) .unwrap_err(); assert!( error .to_string() .contains("protected source term United States") ); } #[test] fn normalizes_non_english_parenthesized_capitalization() { assert_eq!( normalize_localized_capitalization( &Entry::with_msgid("(No content)"), "es-ES", "(Sin contenido)", ), "(sin contenido)" ); assert_eq!( normalize_localized_capitalization( &Entry::with_msgid("(No content)"), "en-GB", "(No content)", ), "(No content)" ); } }