// SPDX-License-Identifier: AGPL-3.0-or-later use std::collections::HashSet; use std::fs; use std::path::{Path, PathBuf}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use anyhow::{Context, Result, bail}; use clap::{Parser, ValueEnum}; use regex::Regex; use serde_json::{Map, Value, json}; use crate::config::{ AUTO_I18N_REVIEWED_UNCHANGED_FILE, DEFAULT_LOCALE_CONCURRENCY, DEFAULT_OPENROUTER_APP_TITLE, DEFAULT_OPENROUTER_BASE_URL, DEFAULT_OPENROUTER_FALLBACK_MODELS, DEFAULT_OPENROUTER_HTTP_REFERER, DEFAULT_OPENROUTER_MODEL, DEFAULT_OPENROUTER_PROVIDER_SORT, DEFAULT_PROGRESS_INTERVAL_SECONDS, DEFAULT_REQUEST_TIMEOUT_SECONDS, DEFAULT_STRING_CONCURRENCY, EnvOverlay, SOURCE_LOCALE, default_app_dir, env_value, is_auto_i18n_unchanged_comment, locales_dir, positive_float_env, positive_int_env, trim_trailing_slash, }; use crate::json_catalog::{ StaticJsonCatalogConfig, read_static_json_entries, rebuild_static_json_allow_replacing, reset_static_json_translations, }; use crate::llm::{ LocalizationClient, LocalizationResult, base_options, clean_translation_response, }; use crate::locales::{display_name, is_supported_locale}; use crate::openrouter::{ OpenRouterClient, OpenRouterClientConfig, parse_openrouter_fallback_models, }; use crate::po::{ Entry, Translation, extract_placeholder_hints, extract_translator_comments, parse_po, rebuild_po_allow_replacing, reset_po_translations, }; use crate::prompts::{build_system_prompt, build_target_locale_prompt, load_prompt_guidance}; use crate::reviewed_unchanged::ReviewedUnchangedStore; use crate::tokens::{ TokenAlias, build_masked_source, build_token_alias_context, extract_icu_controls, extract_preserved_tokens, has_icu_control, mask_text_with_aliases, normalize_localized_capitalization, restore_masked_tokens, should_keep_unchanged, validate_locale_specific_localization, validate_localization, }; use crate::ts_catalog::StaticTsCatalogKind; const FULL_TRANSLATION_ATTEMPTS: usize = 3; const SEGMENT_TRANSLATION_ATTEMPTS: usize = 2; const POLISH_TRANSLATION_ATTEMPTS: usize = 2; #[derive(Clone, Debug, Eq, PartialEq)] pub struct TranslateArgs { pub all: bool, pub batch_size: usize, pub catalog: CatalogName, pub dry_run: bool, pub limit: Option, pub locales: Vec, pub msgctxts: Vec, pub refresh_source_equal: bool, pub reset: bool, pub string_concurrency: usize, pub locale_concurrency: usize, pub progress_interval: Duration, } #[derive(Clone, Debug, Eq, PartialEq)] pub enum CatalogLayout { NestedMessages, FlatPo, StaticJson(StaticJsonCatalogConfig), } #[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] pub enum CatalogName { #[value(name = "app")] App, #[value(name = "errors")] Errors, #[value(name = "api-content", alias = "content")] ApiContent, #[value(name = "email", alias = "emails")] Email, } impl CatalogName { fn label(self) -> &'static str { match self { CatalogName::App => "app", CatalogName::Errors => "errors", CatalogName::ApiContent => "api-content", CatalogName::Email => "email", } } } #[derive(Clone, Debug)] pub struct RuntimeConfig { pub app_dir: PathBuf, pub catalog_layout: CatalogLayout, pub locales_dir: PathBuf, pub reviewed_unchanged_path: PathBuf, pub openrouter_api_key: String, pub openrouter_app_title: String, pub openrouter_base_url: String, pub openrouter_fallback_models: Vec, pub openrouter_http_referer: String, pub openrouter_model: String, pub openrouter_provider_sort: String, pub request_timeout_seconds: f64, } #[derive(Clone, Debug, Eq, PartialEq)] pub struct CatalogPaths { pub locales_dir: PathBuf, pub catalog_layout: CatalogLayout, pub reviewed_unchanged_path: PathBuf, } pub fn catalog_paths(app_dir: &Path, catalog: CatalogName) -> CatalogPaths { let repo_root = app_dir .parent() .map(Path::to_path_buf) .unwrap_or_else(|| PathBuf::from(".")); match catalog { CatalogName::App => { let locales_dir = locales_dir(app_dir); CatalogPaths { reviewed_unchanged_path: locales_dir.join(AUTO_I18N_REVIEWED_UNCHANGED_FILE), locales_dir, catalog_layout: CatalogLayout::NestedMessages, } } CatalogName::Errors => static_json_catalog_paths( repo_root .join("packages") .join("errors") .join("src") .join("i18n"), StaticTsCatalogKind::SimpleMessages, ), CatalogName::ApiContent => static_json_catalog_paths( repo_root .join("fluxer_api") .join("src") .join("api") .join("content_i18n"), StaticTsCatalogKind::SimpleMessages, ), CatalogName::Email => static_json_catalog_paths( repo_root .join("fluxer_api") .join("pkgs") .join("email") .join("src") .join("email_i18n"), StaticTsCatalogKind::EmailTemplates, ), } } fn static_json_catalog_paths(i18n_dir: PathBuf, kind: StaticTsCatalogKind) -> CatalogPaths { let weblate_dir = i18n_dir.join("weblate"); CatalogPaths { locales_dir: weblate_dir.join("locales"), catalog_layout: CatalogLayout::StaticJson(StaticJsonCatalogConfig { kind, source_path: weblate_dir.join("messages.json"), }), reviewed_unchanged_path: i18n_dir .join("locales") .join(AUTO_I18N_REVIEWED_UNCHANGED_FILE), } } impl RuntimeConfig { pub fn from_env(env_overrides: &EnvOverlay, catalog: CatalogName) -> Self { let app_dir = default_app_dir(); let CatalogPaths { locales_dir, catalog_layout, reviewed_unchanged_path, } = catalog_paths(&app_dir, catalog); let openrouter_base_url = trim_trailing_slash(&env_value( "OPENROUTER_BASE_URL", env_overrides, DEFAULT_OPENROUTER_BASE_URL, )); let openrouter_model = env_value("I18N_LLM_MODEL", env_overrides, "") .if_empty(|| env_value("OPENROUTER_MODEL", env_overrides, DEFAULT_OPENROUTER_MODEL)); let openrouter_api_key = env_value("OPENROUTER_API_KEY", env_overrides, ""); let openrouter_fallback_models = parse_openrouter_fallback_models(&env_value( "OPENROUTER_FALLBACK_MODELS", env_overrides, DEFAULT_OPENROUTER_FALLBACK_MODELS, )); let openrouter_provider_sort = env_value( "OPENROUTER_PROVIDER_SORT", env_overrides, DEFAULT_OPENROUTER_PROVIDER_SORT, ); let openrouter_http_referer = env_value( "OPENROUTER_HTTP_REFERER", env_overrides, DEFAULT_OPENROUTER_HTTP_REFERER, ); let openrouter_app_title = env_value( "OPENROUTER_APP_TITLE", env_overrides, DEFAULT_OPENROUTER_APP_TITLE, ); let request_timeout_seconds = positive_float_env( "FLUXER_AUTO_I18N_REQUEST_TIMEOUT", DEFAULT_REQUEST_TIMEOUT_SECONDS, env_overrides, ); Self { app_dir, catalog_layout, locales_dir, reviewed_unchanged_path, openrouter_api_key, openrouter_app_title, openrouter_base_url, openrouter_fallback_models, openrouter_http_referer, openrouter_model, openrouter_provider_sort, request_timeout_seconds, } } } trait EmptyFallback { fn if_empty(self, fallback: impl FnOnce() -> String) -> String; } impl EmptyFallback for String { fn if_empty(self, fallback: impl FnOnce() -> String) -> String { if self.is_empty() { fallback() } else { self } } } #[derive(Clone, Debug, Eq, PartialEq)] pub struct LocaleResult { pub locale: String, pub translated: usize, pub errors: usize, } #[derive(Clone, Debug, Eq, PartialEq)] pub struct LocalePlan { pub locale: String, pub pending: usize, pub selected: usize, } #[derive(Clone, Debug)] pub struct Prompt { pub system_prompt: String, pub user_prompt: String, pub token_aliases: Vec, } #[derive(Parser, Debug)] #[command(about = "Translate Lingui PO catalogs with an LLM provider.")] struct RawTranslateArgs { #[arg(long, help = "Process all non-source locales")] all: bool, #[arg(long = "locale", help = "Process one locale; repeatable")] locales: Vec, #[arg( long, value_enum, help = "Catalog target to translate: app, errors, api-content, or email" )] catalog: Option, #[arg( long = "msgctxt", help = "Only translate entries with this PO msgctxt or static catalog key; repeatable. Email fields use